From 236fe713ab09e4515e25abae5e956242f87638fc Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 19 Apr 2024 17:21:11 -0400 Subject: [PATCH 001/115] feat: WIP multi-stage ami build --- .gitignore | 4 + amazon-arm64.pkr.hcl | 11 ++- ansible/tasks/setup-pgbouncer.yml | 5 + ansible/tasks/setup-postgres.yml | 159 +++++++++++++++++------------- scripts/nix-provision.sh | 18 ++++ stage2-nix-psql.pkr.hcl | 63 ++++++++++++ 6 files changed, 184 insertions(+), 76 deletions(-) create mode 100644 scripts/nix-provision.sh create mode 100644 stage2-nix-psql.pkr.hcl diff --git a/.gitignore b/.gitignore index 9e232d380..9711dcd0b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ __pycache__/ result* .env-local .history + + +#IDE +.vscode diff --git a/amazon-arm64.pkr.hcl b/amazon-arm64.pkr.hcl index 993537297..7ed1856f3 100644 --- a/amazon-arm64.pkr.hcl +++ b/amazon-arm64.pkr.hcl @@ -270,9 +270,10 @@ build { direction = "download" } - provisioner "file" { - source = "/tmp/pg_binaries.tar.gz" - destination = "/tmp/pg_binaries.tar.gz" - direction = "download" - } + // provisioner "file" { + // source = "/tmp/pg_binaries.tar.gz" + // destination = "/tmp/pg_binaries.tar.gz" + // direction = "download" + // } + // TODO RESOLVE IN NEW BUILD } diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 4381ba24d..7beeaf903 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -1,4 +1,9 @@ # PgBouncer +- name: create postgres group + group: + name: postgres + state: present + - name: PgBouncer - download & install dependencies apt: pkg: diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index c84142928..22cdb6f97 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -1,40 +1,54 @@ -- name: Postgres - copy package - copy: - src: files/postgres/ - dest: /tmp/build/ - -- name: Postgres - add PPA - apt_repository: - repo: "deb [ trusted=yes ] file:///tmp/build ./" +# - name: Postgres - copy package +# copy: +# src: files/postgres/ +# dest: /tmp/build/ + +# - name: Postgres - add PPA +# apt_repository: +# repo: "deb [ trusted=yes ] file:///tmp/build ./" +# state: present + +# - name: Postgres - install commons +# apt: +# name: postgresql-common +# install_recommends: no + +# - name: Do not create main cluster +# shell: +# cmd: sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf + +# - name: Postgres - install server +# apt: +# name: postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1 +# install_recommends: no + +# - name: Postgres - remove PPA +# apt_repository: +# repo: "deb [ trusted=yes ] file:///tmp/build ./" +# state: absent + +# - name: Postgres - cleanup package +# file: +# path: /tmp/build +# state: absent + +# - name: Create symlink to /usr/lib/postgresql/bin +# shell: +# cmd: ln -s /usr/lib/postgresql/{{ postgresql_major }}/bin /usr/lib/postgresql/bin +#TODO resolve in new build + +- name: create postgres group + group: + name: postgres state: present -- name: Postgres - install commons - apt: - name: postgresql-common - install_recommends: no - -- name: Do not create main cluster - shell: - cmd: sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf - -- name: Postgres - install server - apt: - name: postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1 - install_recommends: no - -- name: Postgres - remove PPA - apt_repository: - repo: "deb [ trusted=yes ] file:///tmp/build ./" - state: absent - -- name: Postgres - cleanup package - file: - path: /tmp/build - state: absent - -- name: Create symlink to /usr/lib/postgresql/bin - shell: - cmd: ln -s /usr/lib/postgresql/{{ postgresql_major }}/bin /usr/lib/postgresql/bin +- name: create users + user: + name: postgres + groups: postgres + createhome: yes + shell: /bin/bash # Set shell if needed, default is /bin/bash + password: '' - name: Create relevant directories file: @@ -102,9 +116,11 @@ owner: postgres group: postgres -# Install extensions before init -- name: Install Postgres extensions - import_tasks: tasks/setup-docker.yml +# # Install extensions before init +# - name: Install Postgres extensions +# import_tasks: tasks/setup-docker.yml +# TODO resolve in new build + # init DB - name: Create directory on data volume @@ -118,36 +134,37 @@ with_items: - "/data/pgdata" -- name: Link database data_dir to data volume directory - file: - src: "/data/pgdata" - path: "/var/lib/postgresql/data" - state: link - force: yes - -- name: Initialize the database - become: yes - become_user: postgres - shell: - cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" - vars: - ansible_command_timeout: 60 - # Circumvents the following error: - # "Timeout (12s) waiting for privilege escalation prompt" - -- name: copy PG systemd unit - template: - src: files/postgresql_config/postgresql.service.j2 - dest: /etc/systemd/system/postgresql.service - -- name: copy optimizations systemd unit - template: - src: files/database-optimizations.service.j2 - dest: /etc/systemd/system/database-optimizations.service - -# Reload -- name: System - systemd reload - systemd: - enabled: yes - name: postgresql - daemon_reload: yes +# - name: Link database data_dir to data volume directory +# file: +# src: "/data/pgdata" +# path: "/var/lib/postgresql/data" +# state: link +# force: yes + +# - name: Initialize the database +# become: yes +# become_user: postgres +# shell: +# cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" +# vars: +# ansible_command_timeout: 60 +# # Circumvents the following error: +# # "Timeout (12s) waiting for privilege escalation prompt" + +# - name: copy PG systemd unit +# template: +# src: files/postgresql_config/postgresql.service.j2 +# dest: /etc/systemd/system/postgresql.service + +# - name: copy optimizations systemd unit +# template: +# src: files/database-optimizations.service.j2 +# dest: /etc/systemd/system/database-optimizations.service + +# # Reload +# - name: System - systemd reload +# systemd: +# enabled: yes +# name: postgresql +# daemon_reload: yes +# TODO resolve in new build diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh new file mode 100644 index 000000000..f5e139517 --- /dev/null +++ b/scripts/nix-provision.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +set -o errexit +set -o pipefail +set -o xtrace + + +function install_nix() { + sudo su -c "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \ + --extra-conf \"substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com\" \ + --extra-conf \"trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=\" " -s /bin/bash root + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh + sudo -u postgres bash -c '. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres#psql_15/bin' + +} + +install_nix diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl new file mode 100644 index 000000000..29457a233 --- /dev/null +++ b/stage2-nix-psql.pkr.hcl @@ -0,0 +1,63 @@ +packer { + required_plugins { + amazon = { + version = ">= 0.0.2" + source = "github.com/hashicorp/amazon" + } + } +} + +source "amazon-ebs" "ubuntu" { + ami_name = "nix-packer-ubuntu" + instance_type = "t4g.xlarge" + region = "us-east-1" + source_ami_filter { + filters = { + image-id = "ami-0f96dd7e4b73241d8" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["194568623217"] + } + ssh_username = "ubuntu" + ena_support = true + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 40 + volume_type = "gp3" + delete_on_termination = true + } + launch_block_device_mappings { + device_name = "/dev/xvdf" + delete_on_termination = true + volume_size = 30 + volume_type = "gp3" + } + + launch_block_device_mappings { + device_name = "/dev/xvdh" + delete_on_termination = true + volume_size = 30 + volume_type = "gp3" + } + + launch_block_device_mappings { + device_name = "/dev/xvda" + delete_on_termination = true + volume_size = 30 + volume_type = "gp3" + } + + +} + +build { + name = "nix-packer-ubuntu" + sources = [ + "source.amazon-ebs.ubuntu" + ] + provisioner "shell" { + script = "scripts/nix-provision.sh" + } +} From 08b6fb23c3395ddb24e73a06cbbbddb4b8ca3b7d Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 19 Apr 2024 20:42:03 -0400 Subject: [PATCH 002/115] feat: WIP switch off 2nd phase items --- ansible/playbook.yml | 122 +++++++++++++++--------------- ansible/tasks/setup-pgbouncer.yml | 4 +- 2 files changed, 65 insertions(+), 61 deletions(-) diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 5aef718f0..089d6bfd3 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -71,42 +71,45 @@ tags: - install-supabase-internal - - name: Start Postgres Database - systemd: - name: postgresql - state: started - when: not ebssurrogate_mode - - - name: Start Postgres Database without Systemd - become: yes - become_user: postgres - shell: - cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start - when: ebssurrogate_mode + # - name: Start Postgres Database + # systemd: + # name: postgresql + # state: started + # when: not ebssurrogate_mode + + # - name: Start Postgres Database without Systemd + # become: yes + # become_user: postgres + # shell: + # cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start + # when: ebssurrogate_mode + # TODO bring into 2nd phase - name: Adjust APT update intervals copy: src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic - - name: Transfer init SQL files - copy: - src: files/{{ item.source }} - dest: /tmp/{{ item.dest }} - loop: "{{ sql_files }}" + # - name: Transfer init SQL files + # copy: + # src: files/{{ item.source }} + # dest: /tmp/{{ item.dest }} + # loop: "{{ sql_files }}" - - name: Execute init SQL files - become: yes - become_user: postgres - shell: - cmd: /usr/lib/postgresql/bin/psql -f /tmp/{{ item.dest }} - loop: "{{ sql_files }}" + # - name: Execute init SQL files + # become: yes + # become_user: postgres + # shell: + # cmd: /usr/lib/postgresql/bin/psql -f /tmp/{{ item.dest }} + # loop: "{{ sql_files }}" + + # - name: Delete SQL scripts + # file: + # path: /tmp/{{ item.dest }} + # state: absent + # loop: "{{ sql_files }}" + # TODO bring into 2nd phase - - name: Delete SQL scripts - file: - path: /tmp/{{ item.dest }} - state: absent - loop: "{{ sql_files }}" - name: First boot optimizations import_tasks: tasks/internal/optimizations.yml @@ -140,34 +143,35 @@ update_cache: yes cache_valid_time: 3600 - - name: Clean out build dependencies - import_tasks: tasks/clean-build-dependencies.yml - - - name: Restart Postgres Database without Systemd - become: yes - become_user: postgres - shell: - cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data restart -o "-c shared_preload_libraries='pg_tle'" - when: ebssurrogate_mode - - - name: Run migrations - import_tasks: tasks/setup-migrations.yml - tags: - - migrations - - - name: Stop Postgres Database without Systemd - become: yes - become_user: postgres - shell: - cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop - when: ebssurrogate_mode - - - name: Run unit tests - import_tasks: tasks/test-image.yml - tags: - - unit-tests - - - name: Collect Postgres binaries - import_tasks: tasks/internal/collect-pg-binaries.yml - tags: - - collect-binaries + # - name: Clean out build dependencies + # import_tasks: tasks/clean-build-dependencies.yml + + # - name: Restart Postgres Database without Systemd + # become: yes + # become_user: postgres + # shell: + # cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data restart -o "-c shared_preload_libraries='pg_tle'" + # when: ebssurrogate_mode + + # - name: Run migrations + # import_tasks: tasks/setup-migrations.yml + # tags: + # - migrations + + # - name: Stop Postgres Database without Systemd + # become: yes + # become_user: postgres + # shell: + # cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop + # when: ebssurrogate_mode + + # - name: Run unit tests + # import_tasks: tasks/test-image.yml + # tags: + # - unit-tests + + # - name: Collect Postgres binaries + # import_tasks: tasks/internal/collect-pg-binaries.yml + # tags: + # - collect-binaries + #TODO bring into 2nd phase diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 7beeaf903..eee208961 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -1,7 +1,7 @@ # PgBouncer -- name: create postgres group +- name: create ssl-cert group group: - name: postgres + name: ssl-cert state: present - name: PgBouncer - download & install dependencies From 25e46d603c139238961dc291e41dfdc9b8bb74bb Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 22 Apr 2024 16:46:07 -0400 Subject: [PATCH 003/115] feat: migrating and adjusting build steps to work with sourcing files from nix --- amazon-arm64.pkr.hcl | 2 +- ansible/tasks/finalize-ami.yml | 17 +- ansible/tasks/setup-gotrue.yml | 8 +- ansible/tasks/setup-postgres.yml | 68 +------- ansible/tasks/stage2/playbook.yml | 15 ++ .../tasks/stage2/stage2-setup-postgres.yml | 154 ++++++++++++++++++ flake.nix | 10 +- scripts/nix-provision.sh | 27 ++- stage2-nix-psql.pkr.hcl | 72 ++++---- 9 files changed, 261 insertions(+), 112 deletions(-) create mode 100644 ansible/tasks/stage2/playbook.yml create mode 100644 ansible/tasks/stage2/stage2-setup-postgres.yml diff --git a/amazon-arm64.pkr.hcl b/amazon-arm64.pkr.hcl index 7ed1856f3..7486d0f09 100644 --- a/amazon-arm64.pkr.hcl +++ b/amazon-arm64.pkr.hcl @@ -106,7 +106,7 @@ source "amazon-ebssurrogate" "source" { profile = "${var.profile}" #access_key = "${var.aws_access_key}" #ami_name = "${var.ami_name}-arm64-${formatdate("YYYY-MM-DD-hhmm", timestamp())}" - ami_name = "${var.ami_name}-${var.postgres-version}" + ami_name = "${var.ami_name}-${var.postgres-version}-stage-1" ami_virtualization_type = "hvm" ami_architecture = "arm64" ami_regions = "${var.ami_regions}" diff --git a/ansible/tasks/finalize-ami.yml b/ansible/tasks/finalize-ami.yml index 135f4748e..32aeea864 100644 --- a/ansible/tasks/finalize-ami.yml +++ b/ansible/tasks/finalize-ami.yml @@ -33,7 +33,7 @@ rule: allow port: https tags: - - install-supabase-internal + - install-supabase-internal - name: UFW - Deny all other incoming traffic by default ufw: @@ -71,10 +71,11 @@ systemctl reenable logrotate.timer become: yes -- name: import pgsodium_getkey script - template: - src: files/pgsodium_getkey_readonly.sh.j2 - dest: "{{ pg_bindir }}/pgsodium_getkey.sh" - owner: postgres - group: postgres - mode: 0700 +# - name: import pgsodium_getkey script +# template: +# src: files/pgsodium_getkey_readonly.sh.j2 +# dest: "{{ pg_bindir }}/pgsodium_getkey.sh" +# owner: postgres +# group: postgres +# mode: 0700 +# TODO bring into 2nd phase diff --git a/ansible/tasks/setup-gotrue.yml b/ansible/tasks/setup-gotrue.yml index e87034bcb..2a70caa4d 100644 --- a/ansible/tasks/setup-gotrue.yml +++ b/ansible/tasks/setup-gotrue.yml @@ -39,10 +39,10 @@ # libpq is a C library that enables user programs to communicate with # the PostgreSQL database server. -# - name: gotrue - system dependencies -# apt: -# pkg: -# - libpq-dev +- name: gotrue - system dependencies + apt: + pkg: + - libpq-dev - name: gotrue - create service file template: diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 22cdb6f97..afe50ccad 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -1,13 +1,3 @@ -# - name: Postgres - copy package -# copy: -# src: files/postgres/ -# dest: /tmp/build/ - -# - name: Postgres - add PPA -# apt_repository: -# repo: "deb [ trusted=yes ] file:///tmp/build ./" -# state: present - # - name: Postgres - install commons # apt: # name: postgresql-common @@ -16,26 +6,9 @@ # - name: Do not create main cluster # shell: # cmd: sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf - -# - name: Postgres - install server -# apt: -# name: postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1 -# install_recommends: no - -# - name: Postgres - remove PPA -# apt_repository: -# repo: "deb [ trusted=yes ] file:///tmp/build ./" -# state: absent - -# - name: Postgres - cleanup package -# file: -# path: /tmp/build -# state: absent - -# - name: Create symlink to /usr/lib/postgresql/bin -# shell: -# cmd: ln -s /usr/lib/postgresql/{{ postgresql_major }}/bin /usr/lib/postgresql/bin -#TODO resolve in new build +# +# TODO These lines appear to be installing and configuring https://launchpad.net/ubuntu/+source/postgresql-common +# as far as I can see, we don't need this now. - name: create postgres group group: @@ -133,38 +106,3 @@ mode: 0750 with_items: - "/data/pgdata" - -# - name: Link database data_dir to data volume directory -# file: -# src: "/data/pgdata" -# path: "/var/lib/postgresql/data" -# state: link -# force: yes - -# - name: Initialize the database -# become: yes -# become_user: postgres -# shell: -# cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -# vars: -# ansible_command_timeout: 60 -# # Circumvents the following error: -# # "Timeout (12s) waiting for privilege escalation prompt" - -# - name: copy PG systemd unit -# template: -# src: files/postgresql_config/postgresql.service.j2 -# dest: /etc/systemd/system/postgresql.service - -# - name: copy optimizations systemd unit -# template: -# src: files/database-optimizations.service.j2 -# dest: /etc/systemd/system/database-optimizations.service - -# # Reload -# - name: System - systemd reload -# systemd: -# enabled: yes -# name: postgresql -# daemon_reload: yes -# TODO resolve in new build diff --git a/ansible/tasks/stage2/playbook.yml b/ansible/tasks/stage2/playbook.yml new file mode 100644 index 000000000..de056fa60 --- /dev/null +++ b/ansible/tasks/stage2/playbook.yml @@ -0,0 +1,15 @@ +- hosts: localhost + become: yes + + + tasks: + - set_fact: + supabase_internal: true + tags: + - install-supabase-internal + + - set_fact: + parallel_jobs: 16 + + - name: Install Postgres from nix binary cache + import_tasks: stage2-setup-postgres.yml diff --git a/ansible/tasks/stage2/stage2-setup-postgres.yml b/ansible/tasks/stage2/stage2-setup-postgres.yml new file mode 100644 index 000000000..876483eb7 --- /dev/null +++ b/ansible/tasks/stage2/stage2-setup-postgres.yml @@ -0,0 +1,154 @@ +- name: Install Postgres from nix binary cache + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres#psql_15/bin" + +- name: Ensure /usr/lib/postgresql/bin directory exists + file: + path: /usr/lib/postgresql/bin + state: directory + owner: postgres + group: postgres + + +- name: Ensure /usr/lib/postgresql/share directory exists + file: + path: /usr/lib/postgresql/share/postgresql + state: directory + owner: postgres + group: postgres + +- name: Ensure /usr/lib/postgresql/share/contrib directory exists + file: + path: /usr/lib/postgresql/share/postgresql/contrib + state: directory + owner: postgres + group: postgres + +- name: Ensure /usr/lib/postgresql/share/timezonesets directory exists + file: + path: /usr/lib/postgresql/share/postgresql/timezonesets + state: directory + owner: postgres + group: postgres + +- name: Ensure /usr/lib/postgresql/share/tsearch_data directory exists + file: + path: /usr/lib/postgresql/share/postgresql/tsearch_data + state: directory + owner: postgres + group: postgres + +- name: Ensure /usr/lib/postgresql/share/extension directory exists + file: + path: /usr/lib/postgresql/share/postgresql/extension + state: directory + owner: postgres + group: postgres + +- name: Create symbolic links from /home/postgres/.nix-profile/bin to /usr/lib/postgresql/bin + file: + src: "{{ item }}" + dest: "/usr/lib/postgresql/bin/{{ item | basename }}" + state: link + with_fileglob: + - "/home/postgres/.nix-profile/bin/*" + become: yes + + +- name: Create symbolic links from /home/postgres/.nix-profile/share/postgresql to /usr/lib/postgresql/share/postgresql + file: + src: "{{ item }}" + dest: "/usr/lib/postgresql/share/postgresql/{{ item | basename }}" + state: link + with_fileglob: + - "/home/postgres/.nix-profile/share/postgresql/*" + become: yes + +- name: Create symbolic links from /home/postgres/.nix-profile/share/postgresql/extension to /usr/lib/postgresql/share/postgresql/extension + file: + src: "{{ item }}" + dest: "/usr/lib/postgresql/share/postgresql/extension/{{ item | basename }}" + state: link + with_fileglob: + - "/home/postgres/.nix-profile/share/postgresql/extension/*" + become: yes + + +# - name: Create directories recursively +# file: +# path: "/usr/lib/postgresql/share/postgresql/contrib/{{ item | dirname }}" +# state: directory +# with_fileglob: +# - "/home/postgres/.nix-profile/share/postgresql/contrib/*" + +# - name: Create symbolic links for files +# file: +# src: "{{ item }}" +# dest: "/usr/lib/postgresql/share/postgresql/contrib/{{ item | regex_replace('^/home/postgres/.nix-profile/share/postgresql/contrib/', '') }}" +# state: link +# with_find: +# paths: "/home/postgres/.nix-profile/share/postgresql/contrib/" +# recurse: yes +# patterns: "*" +# file_type: any + +- name: Create symbolic links from /home/postgres/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets + file: + src: "{{ item }}" + dest: "/usr/lib/postgresql/share/postgresql/timezonesets/{{ item | basename }}" + state: link + with_fileglob: + - "/home/postgres/.nix-profile/share/postgresql/timezonesets/*" + become: yes + +- name: Create symbolic links from /home/postgres/.nix-profile/share/postgresql/tsearch_data to /usr/lib/postgresql/share/postgresql/tsearch_data + file: + src: "{{ item }}" + dest: "/usr/lib/postgresql/share/postgresql/tsearch_data/{{ item | basename }}" + state: link + with_fileglob: + - "/home/postgres/.nix-profile/share/postgresql/tsearch_data/*" + become: yes + + +# Install extensions before init +# - name: Install Postgres extensions +# import_tasks: tasks/setup-docker.yml +# TODO resolve in new build + +- name: Link database data_dir to data volume directory + file: + src: "/data/pgdata" + path: "/var/lib/postgresql/data" + state: link + force: yes + owner: postgres + group: postgres + +- name: Initialize the database + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" + vars: + ansible_command_timeout: 60 +# # Circumvents the following error: +# # "Timeout (12s) waiting for privilege escalation prompt" + +- name: copy PG systemd unit + template: + src: files/postgresql_config/postgresql.service.j2 + dest: /etc/systemd/system/postgresql.service + +- name: copy optimizations systemd unit + template: + src: files/database-optimizations.service.j2 + dest: /etc/systemd/system/database-optimizations.service + +# # # Reload +- name: System - systemd reload + systemd: + enabled: yes + name: postgresql + daemon_reload: yes diff --git a/flake.nix b/flake.nix index 2dfc5fde9..2b4ce3644 100644 --- a/flake.nix +++ b/flake.nix @@ -29,6 +29,7 @@ # it also serves as a base for importing the orioldb/postgres overlay to #build the orioledb postgres patched version of postgresql16 oriole_pkgs = import nixpkgs { + config = { allowUnfree = true; }; inherit system; overlays = [ # NOTE (aseipp): add any needed overlays here. in theory we could @@ -44,6 +45,7 @@ #This variable works the same as 'oriole_pkgs' but builds using the upstream #nixpkgs builds of postgresql 15 and 16 + the overlays listed below pkgs = import nixpkgs { + config = { allowUnfree = true; }; inherit system; overlays = [ # NOTE (aseipp): add any needed overlays here. in theory we could @@ -450,7 +452,7 @@ mkdir -p $out/bin substitute ${./nix/tools/run-replica.sh.in} $out/bin/start-postgres-replica \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'\ + --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' chmod +x $out/bin/start-postgres-replica ''; sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } '' @@ -545,7 +547,11 @@ nix-update pg_prove shellcheck - + ansible + ansible-lint + (packer.overrideAttrs (oldAttrs: { + version = "1.7.8"; + })) basePackages.start-server basePackages.start-client basePackages.start-replica diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index f5e139517..2bfd569a8 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -5,14 +5,39 @@ set -o errexit set -o pipefail set -o xtrace +function install_packages { + # Setup Ansible on host VM + sudo apt-get update && sudo apt-get install software-properties-common -y + sudo add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y + ansible-galaxy collection install community.general + +} + + function install_nix() { sudo su -c "curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm \ --extra-conf \"substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com\" \ --extra-conf \"trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=\" " -s /bin/bash root . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh - sudo -u postgres bash -c '. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres#psql_15/bin' } + +function execute_stage2_playbook { + +sudo tee /etc/ansible/ansible.cfg < Date: Mon, 22 Apr 2024 16:58:13 -0400 Subject: [PATCH 004/115] feat: use jinja2 files in 2nd stage --- ansible/tasks/stage2/stage2-setup-postgres.yml | 4 ++-- stage2-nix-psql.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/stage2/stage2-setup-postgres.yml b/ansible/tasks/stage2/stage2-setup-postgres.yml index 876483eb7..ea1a9bc74 100644 --- a/ansible/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2/stage2-setup-postgres.yml @@ -138,12 +138,12 @@ - name: copy PG systemd unit template: - src: files/postgresql_config/postgresql.service.j2 + src: /tmp/ansible-playbook/files/postgresql_config/postgresql.service.j2 dest: /etc/systemd/system/postgresql.service - name: copy optimizations systemd unit template: - src: files/database-optimizations.service.j2 + src: /tmp/ansible-playbook/files/database-optimizations.service.j2 dest: /etc/systemd/system/database-optimizations.service # # # Reload diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 0fa1753ae..191094129 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -63,7 +63,7 @@ build { provisioner "file" { source = "ansible/files" - destination = "/tmp/ansible-playbook" + destination = "/tmp/ansible-playbook/files" } From 0acebe03c2743e9d74dda3a4eecbb055ac067510 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 23 Apr 2024 07:23:51 -0400 Subject: [PATCH 005/115] feat: postgis built in our bundle again and matching version we need (#954) --- flake.nix | 7 ++++--- nix/ext/postgis.nix | 24 +++++++++++++++--------- nix/overlays/postgis.nix | 7 +++++++ 3 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 nix/overlays/postgis.nix diff --git a/flake.nix b/flake.nix index 2b4ce3644..c3eaa5c7f 100644 --- a/flake.nix +++ b/flake.nix @@ -53,6 +53,7 @@ # want to have an arbitrary order, since it might matter. being # explicit is better. (import ./nix/overlays/cargo-pgrx-0-11-3.nix) + # (import ./nix/overlays/postgis.nix) #(import ./nix/overlays/gdal-small.nix) ]; @@ -80,7 +81,7 @@ # may also bring in new versions of the extensions. psqlExtensions = [ /* pljava */ - "postgis" + /*"postgis"*/ ]; #FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue @@ -110,7 +111,7 @@ ./nix/ext/plpgsql-check.nix ./nix/ext/pgjwt.nix ./nix/ext/pgaudit.nix - #./nix/ext/postgis.nix + ./nix/ext/postgis.nix ./nix/ext/pgrouting.nix ./nix/ext/pgtap.nix ./nix/ext/pg_cron.nix @@ -137,7 +138,7 @@ #this var is a convenience setting to import the orioledb patched version of postgresql postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; - postgis_override = pkgs.postgis_override; + #postgis_override = pkgs.postgis_override; # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index c5b97f1e2..dc9b31e34 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -5,38 +5,44 @@ , postgresql , geos , proj -, gdal +, gdalMinimal , json_c , pkg-config , file , protobufc , libiconv +, pcre2 , nixosTests }: + +let + gdal = gdalMinimal; +in stdenv.mkDerivation rec { pname = "postgis"; version = "3.3.2"; outputs = [ "out" "doc" ]; - src = fetchurl { + src = fetchurl { url = "https://download.osgeo.org/postgis/source/postgis-${version}.tar.gz"; sha256 = "sha256-miohnaAFoXMKOdGVmhx87GGbHvsAm2W+gP/CW60pkGg="; }; - buildInputs = [ libxml2 postgresql geos proj gdal json_c protobufc ] + buildInputs = [ libxml2 postgresql geos proj gdal json_c protobufc pcre2.dev ] ++ lib.optional stdenv.isDarwin libiconv; - nativeBuildInputs = [ perl pkg-config ] ++ lib.optional postgresql.jitSupport postgresql.llvm; + nativeBuildInputs = [ perl pkg-config ]; dontDisableStatic = true; # postgis config directory assumes /include /lib from the same root for json-c library - NIX_LDFLAGS = "-L${lib.getLib json_c}/lib"; + env.NIX_LDFLAGS = "-L${lib.getLib json_c}/lib"; + preConfigure = '' sed -i 's@/usr/bin/file@${file}/bin/file@' configure - configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev}" + configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --disable-extension-upgrades-install" - makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin" + makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${pname}" ''; postConfigure = '' sed -i "s|@mkdir -p \$(DESTDIR)\$(PGSQL_BINDIR)||g ; @@ -50,7 +56,7 @@ stdenv.mkDerivation rec { # postgis' build system assumes it is being installed to the same place as postgresql, and looks # for the postgres binary relative to $PREFIX. We gently support this system using an illusion. - ln -s ${postgresql_15}/bin/postgres $out/bin/postgres + ln -s ${postgresql}/bin/postgres $out/bin/postgres ''; # create aliases for all commands adding version information @@ -73,7 +79,7 @@ stdenv.mkDerivation rec { homepage = "https://postgis.net/"; changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${version}/NEWS"; license = licenses.gpl2; - maintainers = [ samrose ]; + maintainers = with maintainers; teams.geospatial.members ++ [ marcweber wolfgangwalther ]; inherit (postgresql.meta) platforms; }; } diff --git a/nix/overlays/postgis.nix b/nix/overlays/postgis.nix new file mode 100644 index 000000000..8d022f564 --- /dev/null +++ b/nix/overlays/postgis.nix @@ -0,0 +1,7 @@ +final: prev: { + postgis = prev.postgresqlPackages.postgis.overrideAttrs (old: { + version = "3.3.2"; + sha256 = ""; + }); + postgresqlPackages.postgis = final.postgis; +} From 36a5c3e9e55118adaef78dee14fb92d5e681dede Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 23 Apr 2024 10:48:50 -0400 Subject: [PATCH 006/115] feat: cleanup on postgis extension --- flake.nix | 1 - nix/overlays/postgis.nix | 7 ------- 2 files changed, 8 deletions(-) delete mode 100644 nix/overlays/postgis.nix diff --git a/flake.nix b/flake.nix index c3eaa5c7f..63955be91 100644 --- a/flake.nix +++ b/flake.nix @@ -53,7 +53,6 @@ # want to have an arbitrary order, since it might matter. being # explicit is better. (import ./nix/overlays/cargo-pgrx-0-11-3.nix) - # (import ./nix/overlays/postgis.nix) #(import ./nix/overlays/gdal-small.nix) ]; diff --git a/nix/overlays/postgis.nix b/nix/overlays/postgis.nix deleted file mode 100644 index 8d022f564..000000000 --- a/nix/overlays/postgis.nix +++ /dev/null @@ -1,7 +0,0 @@ -final: prev: { - postgis = prev.postgresqlPackages.postgis.overrideAttrs (old: { - version = "3.3.2"; - sha256 = ""; - }); - postgresqlPackages.postgis = final.postgis; -} From fe2c90c18f30820bc96d3cf62e8df6652057c694 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 23 Apr 2024 12:09:43 -0400 Subject: [PATCH 007/115] feat: make pg_prove an nix installable package in our set --- ansible/playbook.yml | 63 +--------------- .../tasks/internal/collect-pg-binaries.yml | 49 ------------ ansible/tasks/setup-extensions.yml | 2 +- .../tasks/stage2/stage2-setup-postgres.yml | 75 ++++++++++++++----- flake.nix | 3 +- nix/ext/postgis.nix | 2 +- 6 files changed, 62 insertions(+), 132 deletions(-) delete mode 100644 ansible/tasks/internal/collect-pg-binaries.yml diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 089d6bfd3..8bd64c224 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -71,46 +71,11 @@ tags: - install-supabase-internal - # - name: Start Postgres Database - # systemd: - # name: postgresql - # state: started - # when: not ebssurrogate_mode - - # - name: Start Postgres Database without Systemd - # become: yes - # become_user: postgres - # shell: - # cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start - # when: ebssurrogate_mode - # TODO bring into 2nd phase - - name: Adjust APT update intervals copy: src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic - # - name: Transfer init SQL files - # copy: - # src: files/{{ item.source }} - # dest: /tmp/{{ item.dest }} - # loop: "{{ sql_files }}" - - # - name: Execute init SQL files - # become: yes - # become_user: postgres - # shell: - # cmd: /usr/lib/postgresql/bin/psql -f /tmp/{{ item.dest }} - # loop: "{{ sql_files }}" - - # - name: Delete SQL scripts - # file: - # path: /tmp/{{ item.dest }} - # state: absent - # loop: "{{ sql_files }}" - # TODO bring into 2nd phase - - - name: First boot optimizations import_tasks: tasks/internal/optimizations.yml tags: @@ -143,32 +108,8 @@ update_cache: yes cache_valid_time: 3600 - # - name: Clean out build dependencies - # import_tasks: tasks/clean-build-dependencies.yml - - # - name: Restart Postgres Database without Systemd - # become: yes - # become_user: postgres - # shell: - # cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data restart -o "-c shared_preload_libraries='pg_tle'" - # when: ebssurrogate_mode - - # - name: Run migrations - # import_tasks: tasks/setup-migrations.yml - # tags: - # - migrations - - # - name: Stop Postgres Database without Systemd - # become: yes - # become_user: postgres - # shell: - # cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop - # when: ebssurrogate_mode - - # - name: Run unit tests - # import_tasks: tasks/test-image.yml - # tags: - # - unit-tests + - name: Clean out build dependencies + import_tasks: tasks/clean-build-dependencies.yml # - name: Collect Postgres binaries # import_tasks: tasks/internal/collect-pg-binaries.yml diff --git a/ansible/tasks/internal/collect-pg-binaries.yml b/ansible/tasks/internal/collect-pg-binaries.yml deleted file mode 100644 index 7f652f740..000000000 --- a/ansible/tasks/internal/collect-pg-binaries.yml +++ /dev/null @@ -1,49 +0,0 @@ -- name: Collect Postgres binaries - create collection directory - file: - path: /tmp/pg_binaries/{{ postgresql_major }}/ - state: directory - -- name: Collect Postgres binaries - collect binaries and libraries - copy: - remote_src: yes - src: /usr/lib/postgresql/{{ postgresql_major }}/{{ item }}/ - dest: /tmp/pg_binaries/{{ postgresql_major }}/{{ item }}/ - with_items: - - bin - - lib - -- name: Collect Postgres libraries - collect libraries which are in /usr/lib/postgresql/lib/ - copy: - remote_src: yes - src: /usr/lib/postgresql/lib/ - dest: /tmp/pg_binaries/{{ postgresql_major }}/lib/ - -- name: Collect Postgres libraries - collect libraries which are in /var/lib/postgresql/extension/ - copy: - remote_src: yes - src: /var/lib/postgresql/extension/ - dest: /tmp/pg_binaries/{{ postgresql_major }}/lib/ - -- name: Collect Postgres libraries - collect latest libpq - copy: - remote_src: yes - src: /usr/lib/aarch64-linux-gnu/libpq.so.5 - dest: /tmp/pg_binaries/{{ postgresql_major }}/lib/libpq.so.5 - -- name: Collect Postgres binaries - collect shared files - copy: - remote_src: yes - src: /usr/share/postgresql/{{ postgresql_major }}/ - dest: /tmp/pg_binaries/{{ postgresql_major }}/share/ - -- name: Collect Postgres binaries - create tarfile - archive: - path: /tmp/pg_binaries/ - dest: /tmp/pg_binaries.tar.gz - remove: yes - -- name: Fetch tarfile to local - fetch: - src: /tmp/pg_binaries.tar.gz - dest: /tmp/ - flat: true diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index dc1b759ec..0fb92c151 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -80,7 +80,7 @@ - name: Install hypopg import_tasks: tasks/postgres-extensions/26-hypopg.yml - - name: Install pg_repack +- name: Install pg_repack import_tasks: tasks/postgres-extensions/27-pg_repack.yml - name: Install pgvector diff --git a/ansible/tasks/stage2/stage2-setup-postgres.yml b/ansible/tasks/stage2/stage2-setup-postgres.yml index ea1a9bc74..ff7743b30 100644 --- a/ansible/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2/stage2-setup-postgres.yml @@ -74,24 +74,26 @@ - "/home/postgres/.nix-profile/share/postgresql/extension/*" become: yes +- name: Recursively create destination directory + file: + path: /usr/lib/postgresql/share/postgresql/contrib/ + state: directory + recurse: yes -# - name: Create directories recursively -# file: -# path: "/usr/lib/postgresql/share/postgresql/contrib/{{ item | dirname }}" -# state: directory -# with_fileglob: -# - "/home/postgres/.nix-profile/share/postgresql/contrib/*" - -# - name: Create symbolic links for files -# file: -# src: "{{ item }}" -# dest: "/usr/lib/postgresql/share/postgresql/contrib/{{ item | regex_replace('^/home/postgres/.nix-profile/share/postgresql/contrib/', '') }}" -# state: link -# with_find: -# paths: "/home/postgres/.nix-profile/share/postgresql/contrib/" -# recurse: yes -# patterns: "*" -# file_type: any +- name: Create symbolic link for each directory + file: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + state: link + loop: + - { src: "/home/postgres/.nix-profile/share/postgresql/contrib/", dest: "/usr/lib/postgresql/share/postgresql/contrib/" } + +- name: Create symbolic link for each file + file: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + state: link + loop: "{{ query('fileglob', '/home/postgres/.nix-profile/share/postgresql/contrib/*') }}" - name: Create symbolic links from /home/postgres/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets file: @@ -115,7 +117,7 @@ # Install extensions before init # - name: Install Postgres extensions # import_tasks: tasks/setup-docker.yml -# TODO resolve in new build +# TODO resolve in this stage - name: Link database data_dir to data volume directory file: @@ -146,9 +148,46 @@ src: /tmp/ansible-playbook/files/database-optimizations.service.j2 dest: /etc/systemd/system/database-optimizations.service +# - name: Transfer init SQL files +# copy: +# src: files/{{ item.source }} +# dest: /tmp/{{ item.dest }} +# loop: "{{ sql_files }}" +# probably do not need this step + +- name: Execute init SQL files + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/psql -f /tmp/ansible-playbook/files/{{ item.dest }} + loop: "{{ sql_files }}" + +- name: Delete SQL scripts + file: + path: /tmp/{{ item.dest }} + state: absent + loop: "{{ sql_files }}" + # # # Reload - name: System - systemd reload systemd: enabled: yes name: postgresql daemon_reload: yes + + +- name: Run migrations + import_tasks: tasks/setup-migrations.yml + tags: + - migrations + +- name: Stop Postgres with systemd + systemd: + state: stopped + name: postgresql + + +- name: Run unit tests + import_tasks: tasks/test-image.yml + tags: + - unit-tests diff --git a/flake.nix b/flake.nix index 63955be91..47ee4e4ba 100644 --- a/flake.nix +++ b/flake.nix @@ -137,7 +137,6 @@ #this var is a convenience setting to import the orioledb patched version of postgresql postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; - #postgis_override = pkgs.postgis_override; # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other @@ -399,7 +398,7 @@ psql_15 = makePostgres "15"; #psql_16 = makePostgres "16"; #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; - + pg_prove = pg_prove; # Start a version of the server. start-server = let diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index dc9b31e34..6ac5c31df 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -79,7 +79,7 @@ stdenv.mkDerivation rec { homepage = "https://postgis.net/"; changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${version}/NEWS"; license = licenses.gpl2; - maintainers = with maintainers; teams.geospatial.members ++ [ marcweber wolfgangwalther ]; + maintainers = with maintainers; [ samrose ]; inherit (postgresql.meta) platforms; }; } From 39f232b3a056eda0dc4b5255bf341f2b58d47508 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 24 Apr 2024 21:51:50 -0400 Subject: [PATCH 008/115] feat: integration of nix with packer/ansible stage2 steps --- ansible/playbook.yml | 13 +-- ansible/tasks/finalize-ami.yml | 9 -- ansible/tasks/internal/optimizations.yml | 47 -------- ansible/tasks/stage2/optimizations.yml | 46 ++++++++ ansible/tasks/stage2/playbook.yml | 13 +++ .../setup-extensions.yml} | 40 +++---- ansible/tasks/stage2/setup-migrations.yml | 13 +++ .../tasks/stage2/stage2-setup-postgres.yml | 100 +++++++++++------- ansible/tasks/stage2/test-image.yml | 55 ++++++++++ ansible/tasks/test-image.yml | 7 -- nix/overlays/postgis.nix | 7 -- stage2-nix-psql.pkr.hcl | 8 +- 12 files changed, 213 insertions(+), 145 deletions(-) delete mode 100644 ansible/tasks/internal/optimizations.yml create mode 100644 ansible/tasks/stage2/optimizations.yml rename ansible/tasks/{setup-docker.yml => stage2/setup-extensions.yml} (72%) create mode 100644 ansible/tasks/stage2/setup-migrations.yml create mode 100644 ansible/tasks/stage2/test-image.yml delete mode 100644 nix/overlays/postgis.nix diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 8bd64c224..e23f9652f 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -27,7 +27,7 @@ - set_fact: parallel_jobs: 16 - - name: Install Postgres from source + - name: Prepare machine for Postgres installation import_tasks: tasks/setup-postgres.yml - name: Install PgBouncer @@ -76,11 +76,6 @@ src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic - - name: First boot optimizations - import_tasks: tasks/internal/optimizations.yml - tags: - - install-supabase-internal - - name: Finalize AMI import_tasks: tasks/finalize-ami.yml tags: @@ -110,9 +105,3 @@ - name: Clean out build dependencies import_tasks: tasks/clean-build-dependencies.yml - - # - name: Collect Postgres binaries - # import_tasks: tasks/internal/collect-pg-binaries.yml - # tags: - # - collect-binaries - #TODO bring into 2nd phase diff --git a/ansible/tasks/finalize-ami.yml b/ansible/tasks/finalize-ami.yml index 7e1461d49..81a89abeb 100644 --- a/ansible/tasks/finalize-ami.yml +++ b/ansible/tasks/finalize-ami.yml @@ -70,12 +70,3 @@ sed -i -e 's;daily;*:0/5;' /etc/systemd/system/logrotate.timer systemctl reenable logrotate.timer become: yes - -# - name: import pgsodium_getkey script -# template: -# src: files/pgsodium_getkey_readonly.sh.j2 -# dest: "{{ pg_bindir }}/pgsodium_getkey.sh" -# owner: postgres -# group: postgres -# mode: 0700 -# TODO bring into 2nd phase diff --git a/ansible/tasks/internal/optimizations.yml b/ansible/tasks/internal/optimizations.yml deleted file mode 100644 index 895acccd9..000000000 --- a/ansible/tasks/internal/optimizations.yml +++ /dev/null @@ -1,47 +0,0 @@ -- name: ensure services are stopped - community.general.snap: - name: amazon-ssm-agent - state: absent - failed_when: not ebssurrogate_mode - -- name: ensure services are stopped and disabled for first boot - systemd: - enabled: no - name: '{{ item }}' - state: stopped - with_items: - - snapd - - postgresql - - pgbouncer - - fail2ban - - motd-news - - vector - failed_when: not ebssurrogate_mode - -- name: Remove snapd - apt: - state: absent - pkg: - - snapd - failed_when: not ebssurrogate_mode - -- name: ensure services are stopped and disabled for first boot - systemd: - enabled: no - name: '{{ item }}' - state: stopped - masked: yes - with_items: - - lvm2-monitor - failed_when: not ebssurrogate_mode - -- name: disable man-db - become: yes - file: - state: absent - path: "/etc/cron.daily/{{ item }}" - with_items: - - man-db - - popularity-contest - - ubuntu-advantage-tools - failed_when: not ebssurrogate_mode diff --git a/ansible/tasks/stage2/optimizations.yml b/ansible/tasks/stage2/optimizations.yml new file mode 100644 index 000000000..6e7052dc2 --- /dev/null +++ b/ansible/tasks/stage2/optimizations.yml @@ -0,0 +1,46 @@ +# - name: ensure services are stopped +# community.general.snap: +# name: amazon-ssm-agent +# state: absent +# TODO taking this out as machine at this stage reports +# that aws-ssm-agent is not installed at all + +- name: ensure services are stopped and disabled for first boot + systemd: + enabled: no + name: '{{ item }}' + state: stopped + with_items: + #- snapd + - postgresql + - pgbouncer + - fail2ban + - motd-news + - vector + +- name: Remove snapd + apt: + state: absent + pkg: + - snapd + +# - name: ensure services are stopped and disabled for first boot +# systemd: +# enabled: no +# name: '{{ item }}' +# state: stopped +# masked: yes +# with_items: +# - lvm2-monitor +# machine at this stage reports this service is stopped and disabled + + +- name: disable man-db + become: yes + file: + state: absent + path: "/etc/cron.daily/{{ item }}" + with_items: + - man-db + - popularity-contest + - ubuntu-advantage-tools diff --git a/ansible/tasks/stage2/playbook.yml b/ansible/tasks/stage2/playbook.yml index de056fa60..24fdaa3df 100644 --- a/ansible/tasks/stage2/playbook.yml +++ b/ansible/tasks/stage2/playbook.yml @@ -1,6 +1,11 @@ - hosts: localhost become: yes + vars: + sql_files: + - "/tmp/ansible-playbook/files/pgbouncer_config/pgbouncer_auth_schema.sql" + - "/tmp/ansible-playbook/files/stat_extension.sql" + tasks: - set_fact: @@ -13,3 +18,11 @@ - name: Install Postgres from nix binary cache import_tasks: stage2-setup-postgres.yml + + - name: First boot optimizations + import_tasks: optimizations.yml + + - name: Run unit tests + import_tasks: test-image.yml + tags: + - unit-tests diff --git a/ansible/tasks/setup-docker.yml b/ansible/tasks/stage2/setup-extensions.yml similarity index 72% rename from ansible/tasks/setup-docker.yml rename to ansible/tasks/stage2/setup-extensions.yml index c2a56cc30..b699633a3 100644 --- a/ansible/tasks/setup-docker.yml +++ b/ansible/tasks/stage2/setup-extensions.yml @@ -1,14 +1,16 @@ -- name: Copy extension packages - copy: - src: files/extensions/ - dest: /tmp/extensions/ +# - name: Copy extension packages +# copy: +# src: files/extensions/ +# dest: /tmp/extensions/ + +# # Builtin apt module does not support wildcard for deb paths +# - name: Install extensions +# shell: | +# set -e +# apt-get update +# apt-get install -y --no-install-recommends /tmp/extensions/*.deb +# TODO remove -# Builtin apt module does not support wildcard for deb paths -- name: Install extensions - shell: | - set -e - apt-get update - apt-get install -y --no-install-recommends /tmp/extensions/*.deb - name: pg_cron - set cron.database_name become: yes @@ -17,11 +19,8 @@ state: present line: cron.database_name = 'postgres' -- name: pgsodium - determine postgres bin directory - shell: pg_config --bindir - register: pg_bindir_output - set_fact: - pg_bindir: "{{ pg_bindir_output.stdout }}" + pg_bindir: "/usr/lib/postgresql/bin" - name: pgsodium - set pgsodium.getkey_script become: yes @@ -48,7 +47,7 @@ - name: supautils - write custom supautils.conf template: - src: "files/postgresql_config/supautils.conf.j2" + src: "/tmp/ansible-playbook/files/postgresql_config/supautils.conf.j2" dest: /etc/postgresql-custom/supautils.conf mode: 0664 owner: postgres @@ -56,7 +55,7 @@ - name: supautils - copy extension custom scripts copy: - src: files/postgresql_extension_custom_scripts/ + src: /tmp/ansible-playbook/files/postgresql_extension_custom_scripts/ dest: /etc/postgresql-custom/extension-custom-scripts become: yes @@ -76,7 +75,8 @@ regexp: "#include = '/etc/postgresql-custom/supautils.conf'" replace: "include = '/etc/postgresql-custom/supautils.conf'" -- name: Cleanup - extension packages - file: - path: /tmp/extensions - state: absent +# - name: Cleanup - extension packages +# file: +# path: /tmp/extensions +# state: absent +# TODO remove diff --git a/ansible/tasks/stage2/setup-migrations.yml b/ansible/tasks/stage2/setup-migrations.yml new file mode 100644 index 000000000..d69fba024 --- /dev/null +++ b/ansible/tasks/stage2/setup-migrations.yml @@ -0,0 +1,13 @@ +- name: Run migrate.sh script + shell: . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && ./migrate.sh + register: retval + args: + chdir: /tmp/migrations/db + become: yes + become_user: postgres + failed_when: retval.rc != 0 + +- name: Create /root/MIGRATION-AMI file + file: + path: "/root/MIGRATION-AMI" + state: touch diff --git a/ansible/tasks/stage2/stage2-setup-postgres.yml b/ansible/tasks/stage2/stage2-setup-postgres.yml index ff7743b30..4a670e606 100644 --- a/ansible/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2/stage2-setup-postgres.yml @@ -3,6 +3,13 @@ shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres#psql_15/bin" +- name: Install glibcLocales from nix binary cache + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#glibcLocales" + +#TODO include ls $(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale-archive + - name: Ensure /usr/lib/postgresql/bin directory exists file: path: /usr/lib/postgresql/bin @@ -46,6 +53,14 @@ owner: postgres group: postgres +- name: import pgsodium_getkey script + template: + src: /tmp/ansible-playbook/files/pgsodium_getkey_readonly.sh.j2 + dest: "/usr/lib/postgresql/bin/pgsodium_getkey.sh" + owner: postgres + group: postgres + mode: 0700 + - name: Create symbolic links from /home/postgres/.nix-profile/bin to /usr/lib/postgresql/bin file: src: "{{ item }}" @@ -74,26 +89,19 @@ - "/home/postgres/.nix-profile/share/postgresql/extension/*" become: yes -- name: Recursively create destination directory +- name: create destination directory file: path: /usr/lib/postgresql/share/postgresql/contrib/ state: directory recurse: yes -- name: Create symbolic link for each directory - file: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - state: link - loop: - - { src: "/home/postgres/.nix-profile/share/postgresql/contrib/", dest: "/usr/lib/postgresql/share/postgresql/contrib/" } - -- name: Create symbolic link for each file - file: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - state: link - loop: "{{ query('fileglob', '/home/postgres/.nix-profile/share/postgresql/contrib/*') }}" +# TODO check that these dirs were created correctly in final AMI +- name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir + shell: > + sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \ + sudo find /home/postgres/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/" + become: yes - name: Create symbolic links from /home/postgres/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets file: @@ -114,10 +122,9 @@ become: yes -# Install extensions before init -# - name: Install Postgres extensions -# import_tasks: tasks/setup-docker.yml -# TODO resolve in this stage +#Install configure before init +- name: Install Postgres extensions + import_tasks: setup-extensions.yml - name: Link database data_dir to data volume directory file: @@ -128,11 +135,20 @@ owner: postgres group: postgres +- name: Ensure en_US.UTF-8 locale is enabled + shell: | + if ! grep -q "^en_US.UTF-8" /etc/locale.gen ; then + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen + locale-gen + fi + become: yes + - name: Initialize the database become: yes become_user: postgres - shell: - cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" + shell: | + export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive + /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access --locale=en_US.UTF-8" vars: ansible_command_timeout: 60 # # Circumvents the following error: @@ -148,26 +164,40 @@ src: /tmp/ansible-playbook/files/database-optimizations.service.j2 dest: /etc/systemd/system/database-optimizations.service -# - name: Transfer init SQL files -# copy: -# src: files/{{ item.source }} -# dest: /tmp/{{ item.dest }} -# loop: "{{ sql_files }}" -# probably do not need this step +- name: Restart Postgres Database without Systemd + become: yes + become_user: postgres + shell: | + export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive + /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start - name: Execute init SQL files become: yes become_user: postgres shell: - cmd: /usr/lib/postgresql/bin/psql -f /tmp/ansible-playbook/files/{{ item.dest }} + cmd: /usr/lib/postgresql/bin/psql -f {{ item }} loop: "{{ sql_files }}" - name: Delete SQL scripts file: - path: /tmp/{{ item.dest }} + path: "{{ item }}" state: absent loop: "{{ sql_files }}" +- name: Restart Postgres Database without Systemd + become: yes + become_user: postgres + shell: | + export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive + /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data restart -o "-c shared_preload_libraries='pg_tle'" + + +- name: Run migrations + import_tasks: setup-migrations.yml + tags: + - migrations + + # # # Reload - name: System - systemd reload systemd: @@ -175,19 +205,7 @@ name: postgresql daemon_reload: yes - -- name: Run migrations - import_tasks: tasks/setup-migrations.yml - tags: - - migrations - - name: Stop Postgres with systemd systemd: state: stopped name: postgresql - - -- name: Run unit tests - import_tasks: tasks/test-image.yml - tags: - - unit-tests diff --git a/ansible/tasks/stage2/test-image.yml b/ansible/tasks/stage2/test-image.yml new file mode 100644 index 000000000..aecef204b --- /dev/null +++ b/ansible/tasks/stage2/test-image.yml @@ -0,0 +1,55 @@ +- name: install pg_prove + apt: + pkg: + - libtap-parser-sourcehandler-pgtap-perl + +- name: Temporarily disable PG Sodium references in config + become: yes + become_user: postgres + shell: + cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf + +- name: Start Postgres Database to load all extensions. + become: yes + become_user: postgres + shell: | + export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start "-o -c config_file=/etc/postgresql/postgresql.conf" + +- name: Run Unit tests (with filename unit-test-*) on Postgres Database + shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql + register: retval + failed_when: retval.rc != 0 + +- name: Run migrations tests + shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql + register: retval + failed_when: retval.rc != 0 + args: + chdir: /tmp/migrations + +- name: Re-enable PG Sodium references in config + become: yes + become_user: postgres + shell: + cmd: mv /etc/postgresql/postgresql.conf.bak /etc/postgresql/postgresql.conf + +- name: Reset db stats + become: yes + become_user: postgres + shell: | + export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive + /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();' + +- name: remove pg_prove + apt: + pkg: + - libtap-parser-sourcehandler-pgtap-perl + state: absent + autoremove: yes + +- name: Stop Postgres Database + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index cd92a27d6..cad503a39 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -8,26 +8,22 @@ become_user: postgres shell: cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf - when: ebssurrogate_mode - name: Start Postgres Database to load all extensions. become: yes become_user: postgres shell: cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start "-o -c config_file=/etc/postgresql/postgresql.conf" - when: ebssurrogate_mode - name: Run Unit tests (with filename unit-test-*) on Postgres Database shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql register: retval failed_when: retval.rc != 0 - when: ebssurrogate_mode - name: Run migrations tests shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql register: retval failed_when: retval.rc != 0 - when: ebssurrogate_mode args: chdir: /tmp/migrations @@ -36,11 +32,9 @@ become_user: postgres shell: cmd: mv /etc/postgresql/postgresql.conf.bak /etc/postgresql/postgresql.conf - when: ebssurrogate_mode - name: Reset db stats shell: /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();' - when: ebssurrogate_mode - name: remove pg_prove apt: @@ -54,4 +48,3 @@ become_user: postgres shell: cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop - when: ebssurrogate_mode diff --git a/nix/overlays/postgis.nix b/nix/overlays/postgis.nix deleted file mode 100644 index 8d022f564..000000000 --- a/nix/overlays/postgis.nix +++ /dev/null @@ -1,7 +0,0 @@ -final: prev: { - postgis = prev.postgresqlPackages.postgis.overrideAttrs (old: { - version = "3.3.2"; - sha256 = ""; - }); - postgresqlPackages.postgis = final.postgis; -} diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 191094129..a490d9882 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -33,7 +33,7 @@ source "amazon-ebs" "ubuntu" { region = "${var.region}" source_ami_filter { filters = { - name = "supabase-postgres-15.1.1.41-stage-1" + name = "supabase-postgres-15.1.1.42-stage-1" root-device-type = "ebs" virtualization-type = "hvm" } @@ -62,10 +62,14 @@ build { } provisioner "file" { - source = "ansible/files" + source = "ansible/files" destination = "/tmp/ansible-playbook/files" } + provisioner "file" { + source = "migrations" + destination = "/tmp" + } provisioner "shell" { script = "scripts/nix-provision.sh" From e83886d37989884ab6e5a05dba6c20133f66bc5a Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 26 Apr 2024 22:12:13 -0400 Subject: [PATCH 009/115] WIP: this build is working up to running unit tests on image with pg_prove --- ansible/tasks/setup-postgres.yml | 25 ++++++ ansible/tasks/stage2/playbook.yml | 15 ++++ .../tasks/stage2/stage2-setup-postgres.yml | 80 ++++++++++++++++--- ansible/tasks/stage2/test-image.yml | 24 ++++-- ebssurrogate/scripts/surrogate-bootstrap.sh | 2 +- scripts/nix-provision.sh | 2 +- stage2-nix-psql.pkr.hcl | 9 ++- 7 files changed, 135 insertions(+), 22 deletions(-) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index afe50ccad..233e7c798 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -58,6 +58,31 @@ - 'generated-optimizations.conf' - 'custom-overrides.conf' +- name: locale-gen + command: sudo locale-gen en_US.UTF-8 + +- name: Add LOCALE_ARCHIVE to .bashrc + lineinfile: + dest: "/home/postgres/.bashrc" + line: 'export LOCALE_ARCHIVE=/usr/lib/locale/locale-archive' + create: yes + become: yes + + +- name: Add LANG items to .bashrc + lineinfile: + dest: "/home/postgres/.bashrc" + line: "{{ item }}" + + loop: + - 'export LANG="en_US.UTF-8"' + - 'export LANGUAGE="en_US.UTF-8"' + - 'export LC_ALL="en_US.UTF-8"' + - 'export LANG="en_US.UTF-8"' + - 'export LC_CTYPE="en_US.UTF-8"' + become: yes + + # Move Postgres configuration files into /etc/postgresql # Add postgresql.conf - name: import postgresql.conf diff --git a/ansible/tasks/stage2/playbook.yml b/ansible/tasks/stage2/playbook.yml index 24fdaa3df..cf20b71d2 100644 --- a/ansible/tasks/stage2/playbook.yml +++ b/ansible/tasks/stage2/playbook.yml @@ -15,6 +15,21 @@ - set_fact: parallel_jobs: 16 + + - name: locale-gen + command: sudo locale-gen en_US.UTF-8 + + # - name: Ensure en_US.UTF-8 locale is enabled + # shell: | + # echo "LC_ALL=en_US.UTF-8" >> /etc/environment + # echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen + # echo "LANG=en_US.UTF-8" > /etc/locale.conf + # locale-gen en_US.UTF-8 + # become: yes + + - name: check locales + command: locale -a + register: locales - name: Install Postgres from nix binary cache import_tasks: stage2-setup-postgres.yml diff --git a/ansible/tasks/stage2/stage2-setup-postgres.yml b/ansible/tasks/stage2/stage2-setup-postgres.yml index 4a670e606..20e2b8a57 100644 --- a/ansible/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2/stage2-setup-postgres.yml @@ -8,6 +8,34 @@ shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#glibcLocales" + +# - name: Generate en_US.UTF-8 locale +# command: sudo localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 + + +# - name: Add LOCALE_ARCHIVE to .bashrc +# ansible.builtin.lineinfile: +# dest: "/home/postgres/.bashrc" +# line: 'export LOCALE_ARCHIVE=/usr/lib/locale/locale-archive' +# create: yes +# become: yes +# become_user: postgres + +# - name: Add LANG items to .bashrc +# ansible.builtin.lineinfile: +# dest: "/home/postgres/.bashrc" +# line: "{{ item }}" + +# loop: +# - 'export LANG="en_US.UTF-8"' +# - 'export LANGUAGE="en_US.UTF-8"' +# - 'export LC_ALL="en_US.UTF-8"' +# - 'export LANG="en_US.UTF-8"' +# - 'export LC_CTYPE="en_US.UTF-8"' +# become: yes +# become_user: postgres + + #TODO include ls $(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale-archive - name: Ensure /usr/lib/postgresql/bin directory exists @@ -135,20 +163,26 @@ owner: postgres group: postgres -- name: Ensure en_US.UTF-8 locale is enabled - shell: | - if ! grep -q "^en_US.UTF-8" /etc/locale.gen ; then - echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen - locale-gen - fi - become: yes +# - name: Set LANG environment variable +# environment: +# LANG: en_US.UTF-8 +# #become: yes +#source /home/postgres/.bashrc && - name: Initialize the database become: yes become_user: postgres - shell: | - export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive - /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access --locale=en_US.UTF-8" + shell: source /home/postgres/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + # LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + # LOCALE_ARCHIVE: /usr/lib/locale/locale-archive vars: ansible_command_timeout: 60 # # Circumvents the following error: @@ -168,8 +202,14 @@ become: yes become_user: postgres shell: | - export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive + source /home/postgres/.bashrc /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive - name: Execute init SQL files become: yes @@ -188,9 +228,16 @@ become: yes become_user: postgres shell: | - export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive + source /home/postgres/.bashrc && \ /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data restart -o "-c shared_preload_libraries='pg_tle'" - + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive - name: Run migrations import_tasks: setup-migrations.yml @@ -209,3 +256,10 @@ systemd: state: stopped name: postgresql + + +- name: Stop Postgres Database without systemd + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop diff --git a/ansible/tasks/stage2/test-image.yml b/ansible/tasks/stage2/test-image.yml index aecef204b..d7f53c1d9 100644 --- a/ansible/tasks/stage2/test-image.yml +++ b/ansible/tasks/stage2/test-image.yml @@ -12,12 +12,18 @@ - name: Start Postgres Database to load all extensions. become: yes become_user: postgres - shell: | - export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive - cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start "-o -c config_file=/etc/postgresql/postgresql.conf" + shell: source /home/postgres/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start "-o -c config_file=/etc/postgresql/postgresql.conf" + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive - name: Run Unit tests (with filename unit-test-*) on Postgres Database - shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql + shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-01.sql register: retval failed_when: retval.rc != 0 @@ -38,8 +44,16 @@ become: yes become_user: postgres shell: | - export LOCALE_ARCHIVE=$(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale/locale-archive + source /home/postgres/.bashrc && \ /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();' + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive - name: remove pg_prove apt: diff --git a/ebssurrogate/scripts/surrogate-bootstrap.sh b/ebssurrogate/scripts/surrogate-bootstrap.sh index a24e1d35e..2bf68ae99 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap.sh @@ -214,7 +214,7 @@ EOF # Run Ansible playbook #export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp - ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml $ARGS + ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml -vvv $ARGS } function update_systemd_services { diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 2bfd569a8..cac4f1451 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -33,7 +33,7 @@ EOF # Run Ansible playbook #export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/tmp export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp - ansible-playbook /tmp/ansible-playbook/stage2/playbook.yml $ARGS + ansible-playbook /tmp/ansible-playbook/stage2/playbook.yml -vvv $ARGS } diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index a490d9882..43448406d 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -70,8 +70,13 @@ build { source = "migrations" destination = "/tmp" } - - provisioner "shell" { + + provisioner "shell" { script = "scripts/nix-provision.sh" } + + provisioner "file" { + source = "ebssurrogate/files/unit-tests" + destination = "/tmp" + } } From b8f2b221345846c5e8ed713677575399b684ab20 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 29 Apr 2024 16:37:30 -0400 Subject: [PATCH 010/115] feat: ansible run unit and migration tests passing! --- ansible/tasks/stage2/playbook.yml | 11 +++- .../tasks/stage2/stage2-setup-postgres.yml | 5 +- ansible/tasks/stage2/test-image.yml | 61 +++++++++++++++---- stage2-nix-psql.pkr.hcl | 11 ++-- 4 files changed, 67 insertions(+), 21 deletions(-) diff --git a/ansible/tasks/stage2/playbook.yml b/ansible/tasks/stage2/playbook.yml index cf20b71d2..fb53402dc 100644 --- a/ansible/tasks/stage2/playbook.yml +++ b/ansible/tasks/stage2/playbook.yml @@ -15,7 +15,16 @@ - set_fact: parallel_jobs: 16 - + + - name: stat unit test file copy + copy: + src: /tmp/unit-tests/unit-test-01.sql + dest: /home/postgres/unit-test-01.sql + # state: present + owner: postgres + group: postgres + mode: '0644' + - name: locale-gen command: sudo locale-gen en_US.UTF-8 diff --git a/ansible/tasks/stage2/stage2-setup-postgres.yml b/ansible/tasks/stage2/stage2-setup-postgres.yml index 20e2b8a57..021cbdf90 100644 --- a/ansible/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2/stage2-setup-postgres.yml @@ -3,11 +3,12 @@ shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres#psql_15/bin" -- name: Install glibcLocales from nix binary cache +- name: Install pg_prove from nix binary cache become: yes shell: | - sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#glibcLocales" + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/sam/2-stage-ami-nix#pg_prove" +#TODO switch pg_prove sourcing to develop branch once PR is merged # - name: Generate en_US.UTF-8 locale # command: sudo localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 diff --git a/ansible/tasks/stage2/test-image.yml b/ansible/tasks/stage2/test-image.yml index d7f53c1d9..bc46efd44 100644 --- a/ansible/tasks/stage2/test-image.yml +++ b/ansible/tasks/stage2/test-image.yml @@ -1,8 +1,3 @@ -- name: install pg_prove - apt: - pkg: - - libtap-parser-sourcehandler-pgtap-perl - - name: Temporarily disable PG Sodium references in config become: yes become_user: postgres @@ -22,17 +17,58 @@ LC_CTYPE: en_US.UTF-8 LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + +- name: check contents of unit test file + become: yes + become_user: postgres + shell: cat /home/postgres/unit-test-01.sql && ls -l /home/postgres/unit-test-01.sql + +- name: verify postgres server is running + become: yes + become_user: postgres + shell: psql -U postgres -h localhost -c "SELECT version();" + register: pg_isready + failed_when: pg_isready.rc != 0 + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + PATH: /home/postgres/.nix-profile/bin:$PATH + - name: Run Unit tests (with filename unit-test-*) on Postgres Database - shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-01.sql + become: yes + become_user: postgres + shell: source /home/postgres/.bashrc && /home/postgres/.nix-profile/bin/pg_prove -U postgres -h localhost -d postgres -f /home/postgres/unit-test-01.sql register: retval failed_when: retval.rc != 0 + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + PATH: /home/postgres/.nix-profile/bin:$PATH - name: Run migrations tests - shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql + shell: /home/postgres/.nix-profile/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql register: retval failed_when: retval.rc != 0 args: + executable: /bin/bash chdir: /tmp/migrations + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + PATH: /home/postgres/.nix-profile/bin:$PATH - name: Re-enable PG Sodium references in config become: yes @@ -55,12 +91,11 @@ LC_CTYPE: en_US.UTF-8 LOCALE_ARCHIVE: /usr/lib/locale/locale-archive -- name: remove pg_prove - apt: - pkg: - - libtap-parser-sourcehandler-pgtap-perl - state: absent - autoremove: yes +# - name: remove pg_prove +- name: Remove pg_prove + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile remove pg_prove" - name: Stop Postgres Database become: yes diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 43448406d..8717e8f54 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -70,13 +70,14 @@ build { source = "migrations" destination = "/tmp" } - + + provisioner "file" { + source = "ebssurrogate/files/unit-tests" + destination = "/tmp/unit-tests" + } + provisioner "shell" { script = "scripts/nix-provision.sh" } - provisioner "file" { - source = "ebssurrogate/files/unit-tests" - destination = "/tmp" - } } From 5fb6ead4c521cb20d2e3c4165a2aa3d9c88f8d99 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 30 Apr 2024 17:26:54 -0400 Subject: [PATCH 011/115] feat: create a nix package for sfcgal --- ansible/tasks/stage2/setup-extensions.yml | 6 ++++- flake.nix | 5 ++++ nix/ext/postgis.nix | 6 +++-- nix/ext/sfcgal/sfcgal.nix | 30 +++++++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 nix/ext/sfcgal/sfcgal.nix diff --git a/ansible/tasks/stage2/setup-extensions.yml b/ansible/tasks/stage2/setup-extensions.yml index b699633a3..df769dbb6 100644 --- a/ansible/tasks/stage2/setup-extensions.yml +++ b/ansible/tasks/stage2/setup-extensions.yml @@ -11,6 +11,10 @@ # apt-get install -y --no-install-recommends /tmp/extensions/*.deb # TODO remove +# The following tasks were moved to stage 2 from the original ansible/tasks/setup-docker.yml +# due to these configurations being required to be applied at this stage, after the database package +# has been installed and before the database is started. +# These tasks are required to be run before the database is started, as they modify the database configuration - name: pg_cron - set cron.database_name become: yes @@ -79,4 +83,4 @@ # file: # path: /tmp/extensions # state: absent -# TODO remove +# TODO remove extensions are already packaged in the nix build diff --git a/flake.nix b/flake.nix index 8b4d8d13d..8778c4a3f 100644 --- a/flake.nix +++ b/flake.nix @@ -57,8 +57,10 @@ #(import ./nix/overlays/gdal-small.nix) ]; + }; + sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { }; # FIXME (aseipp): pg_prove is yet another perl program that needs # LOCALE_ARCHIVE set in non-NixOS environments. upstream this. once that's done, we @@ -139,6 +141,7 @@ #this var is a convenience setting to import the orioledb patched version of postgresql postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; + # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other # tools to inspect what the contents of the install are: the PSQL @@ -400,6 +403,8 @@ #psql_16 = makePostgres "16"; #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; pg_prove = pg_prove; + sfcgal = sfcgal; + # Start a version of the server. start-server = let diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index 6ac5c31df..e0b6dfbeb 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -13,10 +13,12 @@ , libiconv , pcre2 , nixosTests +, callPackage }: let gdal = gdalMinimal; + sfcgal = callPackage ./sfcgal/sfcgal.nix { }; in stdenv.mkDerivation rec { pname = "postgis"; @@ -29,7 +31,7 @@ stdenv.mkDerivation rec { sha256 = "sha256-miohnaAFoXMKOdGVmhx87GGbHvsAm2W+gP/CW60pkGg="; }; - buildInputs = [ libxml2 postgresql geos proj gdal json_c protobufc pcre2.dev ] + buildInputs = [ libxml2 postgresql geos proj gdal json_c protobufc pcre2.dev sfcgal ] ++ lib.optional stdenv.isDarwin libiconv; nativeBuildInputs = [ perl pkg-config ]; dontDisableStatic = true; @@ -40,7 +42,7 @@ stdenv.mkDerivation rec { preConfigure = '' sed -i 's@/usr/bin/file@${file}/bin/file@' configure - configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --disable-extension-upgrades-install" + configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --disable-extension-upgrades-install --with-sfcgal" makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${pname}" ''; diff --git a/nix/ext/sfcgal/sfcgal.nix b/nix/ext/sfcgal/sfcgal.nix new file mode 100644 index 000000000..8ca22dc55 --- /dev/null +++ b/nix/ext/sfcgal/sfcgal.nix @@ -0,0 +1,30 @@ +{ lib, stdenv, fetchFromGitLab, cgal, cmake, pkg-config, gmp, mpfr, boost }: + +stdenv.mkDerivation rec { + pname = "sfcgal"; + version = "61f3b08ade49493b56c6bafa98c7c1f84addbc10"; + + src = fetchFromGitLab { + owner = "sfcgal"; + repo = "SFCGAL"; + rev = "${version}"; + hash = "sha256-nKSqiFyMkZAYptIeShb1zFg9lYSny3kcGJfxdeTFqxw="; + }; + + nativeBuildInputs = [ cmake pkg-config cgal gmp mpfr boost ]; + + cmakeFlags = [ "-DCGAL_DIR=${cgal}" "-DCMAKE_PREFIX_PATH=${cgal}" ]; + + + postPatch = '' + substituteInPlace sfcgal.pc.in \ + --replace '$'{prefix}/@CMAKE_INSTALL_LIBDIR@ @CMAKE_INSTALL_FULL_LIBDIR@ + ''; + + meta = with lib; { + description = "A wrapper around CGAL that intents to implement 2D and 3D operations on OGC standards models"; + homepage = "https://sfcgal.gitlab.io/SFCGAL/"; + license = licenses.gpl2Only; + maintainers = with maintainers; [ samrose ]; + }; +} From 9f63f35cf47b368b472ade10441811d6d3748fc5 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 30 Apr 2024 22:26:49 -0400 Subject: [PATCH 012/115] chore: correct license --- nix/ext/sfcgal/sfcgal.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nix/ext/sfcgal/sfcgal.nix b/nix/ext/sfcgal/sfcgal.nix index 8ca22dc55..54d7b52cb 100644 --- a/nix/ext/sfcgal/sfcgal.nix +++ b/nix/ext/sfcgal/sfcgal.nix @@ -24,7 +24,8 @@ stdenv.mkDerivation rec { meta = with lib; { description = "A wrapper around CGAL that intents to implement 2D and 3D operations on OGC standards models"; homepage = "https://sfcgal.gitlab.io/SFCGAL/"; - license = licenses.gpl2Only; + license = with licenses; [ gpl3Plus lgpl3Plus]; + platforms = platforms.all; maintainers = with maintainers; [ samrose ]; }; } From 3e9e76c64f5258c7df23afc1b1e55696c9378c42 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 1 May 2024 13:51:34 -0400 Subject: [PATCH 013/115] feat: tmp sourcing from the right flake url for successful build of sfcgal --- ansible/tasks/stage2/stage2-setup-postgres.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/stage2/stage2-setup-postgres.yml b/ansible/tasks/stage2/stage2-setup-postgres.yml index 021cbdf90..fa45f6ea3 100644 --- a/ansible/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2/stage2-setup-postgres.yml @@ -1,7 +1,7 @@ - name: Install Postgres from nix binary cache become: yes shell: | - sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres#psql_15/bin" + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/sam/2-stage-ami-nix#psql_15/bin" - name: Install pg_prove from nix binary cache become: yes From e82dd6de4155cc54d665ba3970e877596bb3a694 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 1 May 2024 15:01:00 -0400 Subject: [PATCH 014/115] feat: retain old and new ansible/packer files until change is complete --- amazon-arm64-nix.pkr.hcl | 272 ++++++ amazon-arm64.pkr.hcl | 13 +- .../files/admin_api_scripts/grow_fs.sh | 23 + .../admin_api_scripts/manage_readonly_mode.sh | 45 + .../admin_api_scripts/pg_egress_collect.pl | 132 +++ .../pg_upgrade_scripts/check.sh | 16 + .../pg_upgrade_scripts/common.sh | 63 ++ .../pg_upgrade_scripts/complete.sh | 153 ++++ .../pg_upgrade_scripts/initiate.sh | 333 ++++++++ .../pg_upgrade_scripts/pgsodium_getkey.sh | 12 + .../pg_upgrade_scripts/prepare.sh | 15 + ansible-nix/files/adminapi.service.j2 | 13 + ansible-nix/files/adminapi.sudoers.conf | 28 + ansible-nix/files/ansible-pull.service | 20 + ansible-nix/files/ansible-pull.timer | 11 + ansible-nix/files/apt_periodic | 4 + ansible-nix/files/cron.deny | 2 + .../files/database-optimizations.service.j2 | 12 + ansible-nix/files/default.sysstat | 9 + ansible-nix/files/envoy.service | 31 + ansible-nix/files/envoy_config/cds.yaml | 46 ++ ansible-nix/files/envoy_config/envoy.yaml | 23 + ansible-nix/files/envoy_config/lds.yaml | 315 +++++++ .../remove_apikey_query_parameter.lua | 8 + .../fail2ban_config/fail2ban.service.conf | 6 + .../fail2ban_config/filter-pgbouncer.conf.j2 | 3 + .../fail2ban_config/filter-postgresql.conf.j2 | 3 + .../fail2ban_config/jail-pgbouncer.conf.j2 | 7 + .../fail2ban_config/jail-postgresql.conf.j2 | 8 + .../files/fail2ban_config/jail-ssh.conf | 4 + ansible-nix/files/fail2ban_config/jail.local | 4 + ansible-nix/files/gotrue.service.j2 | 21 + ansible-nix/files/journald.conf | 6 + ansible-nix/files/kong_config/kong.conf.j2 | 7 + ansible-nix/files/kong_config/kong.env.j2 | 8 + ansible-nix/files/kong_config/kong.service.j2 | 28 + ansible-nix/files/logind.conf | 2 + .../logrotate-postgres-auth.conf | 8 + .../logrotate-postgres-csv.conf | 11 + .../logrotate_config/logrotate-postgres.conf | 9 + .../logrotate_config/logrotate-walg.conf | 9 + ansible-nix/files/manifest.json | 1 + ansible-nix/files/nginx.service.j2 | 22 + .../files/pg_egress_collect.service.j2 | 13 + .../files/pgbouncer_config/pgbouncer.ini.j2 | 364 ++++++++ .../pgbouncer_config/pgbouncer.service.j2 | 20 + .../pgbouncer_auth_schema.sql | 20 + .../tmpfiles.d-pgbouncer.conf.j2 | 2 + .../files/pgsodium_getkey_readonly.sh.j2 | 14 + .../files/pgsodium_getkey_urandom.sh.j2 | 10 + .../files/postgres_exporter.service.j2 | 14 + .../custom_read_replica.conf.j2 | 5 + .../postgresql_config/custom_walg.conf.j2 | 21 + .../files/postgresql_config/pg_hba.conf.j2 | 94 +++ .../files/postgresql_config/pg_ident.conf.j2 | 50 ++ .../postgresql_config/postgresql-csvlog.conf | 33 + .../postgresql-stdout-log.conf | 4 + .../postgresql_config/postgresql.conf.j2 | 776 ++++++++++++++++++ .../postgresql_config/postgresql.service.j2 | 23 + .../files/postgresql_config/supautils.conf.j2 | 12 + .../tmpfiles.postgresql.conf | 5 + .../before-create.sql | 84 ++ .../dblink/after-create.sql | 14 + .../pg_cron/after-create.sql | 13 + .../pg_tle/after-create.sql | 1 + .../pgsodium/after-create.sql | 3 + .../postgis_tiger_geocoder/after-create.sql | 10 + .../postgres_fdw/after-create.sql | 21 + .../files/postgrest-optimizations.service.j2 | 11 + ansible-nix/files/postgrest.service.j2 | 18 + ansible-nix/files/services.slice.j2 | 6 + ansible-nix/files/sodium_extension.sql | 6 + ansible-nix/files/start-envoy.sh | 12 + ansible-nix/files/stat_extension.sql | 2 + ansible-nix/files/supabase_facts.ini | 2 + ansible-nix/files/sysstat.sysstat | 36 + ansible-nix/files/systemd-resolved.conf | 8 + ansible-nix/files/ufw.service.conf | 4 + ansible-nix/files/vector.service.j2 | 20 + .../wal_change_ownership.sh | 42 + .../files/walg_helper_scripts/wal_fetch.sh | 12 + ansible-nix/manifest-playbook.yml | 91 ++ ansible-nix/playbook.yml | 107 +++ .../tasks/clean-build-dependencies.yml | 21 + ansible-nix/tasks/finalize-ami.yml | 72 ++ ansible-nix/tasks/internal/admin-api.yml | 92 +++ ansible-nix/tasks/internal/admin-mgr.yml | 22 + .../tasks/internal/pg_egress_collect.yml | 15 + .../tasks/internal/postgres-exporter.yml | 48 ++ .../tasks/internal/setup-ansible-pull.yml | 29 + ansible-nix/tasks/internal/setup-nftables.yml | 34 + ansible-nix/tasks/internal/supautils.yml | 77 ++ ansible-nix/tasks/setup-envoy.yml | 60 ++ ansible-nix/tasks/setup-extensions.yml | 94 +++ ansible-nix/tasks/setup-fail2ban.yml | 70 ++ ansible-nix/tasks/setup-gotrue.yml | 54 ++ ansible-nix/tasks/setup-kong.yml | 62 ++ ansible-nix/tasks/setup-migrations.yml | 13 + ansible-nix/tasks/setup-nginx.yml | 82 ++ ansible-nix/tasks/setup-pgbouncer.yml | 140 ++++ ansible-nix/tasks/setup-postgres.yml | 133 +++ ansible-nix/tasks/setup-postgrest.yml | 84 ++ ansible-nix/tasks/setup-supabase-internal.yml | 108 +++ ansible-nix/tasks/setup-system.yml | 154 ++++ ansible-nix/tasks/setup-wal-g.yml | 130 +++ .../tasks/stage2/optimizations.yml | 0 .../tasks/stage2/playbook.yml | 0 .../tasks/stage2/setup-extensions.yml | 0 .../tasks/stage2/setup-migrations.yml | 0 .../tasks/stage2/stage2-setup-postgres.yml | 0 .../tasks/stage2/test-image.yml | 0 ansible-nix/tasks/test-image.yml | 50 ++ ansible-nix/vars.yml | 145 ++++ ansible/playbook.yml | 68 +- ansible/tasks/finalize-ami.yml | 10 +- .../tasks/internal/collect-pg-binaries.yml | 49 ++ ansible/tasks/internal/optimizations.yml | 47 ++ ansible/tasks/setup-docker.yml | 82 ++ ansible/tasks/setup-extensions.yml | 2 +- ansible/tasks/setup-gotrue.yml | 8 +- ansible/tasks/setup-pgbouncer.yml | 5 - ansible/tasks/setup-postgres.yml | 124 +-- ansible/tasks/test-image.yml | 7 + ansible/vars.yml | 12 +- stage2-nix-psql.pkr.hcl | 2 +- 125 files changed, 5896 insertions(+), 78 deletions(-) create mode 100644 amazon-arm64-nix.pkr.hcl create mode 100644 ansible-nix/files/admin_api_scripts/grow_fs.sh create mode 100644 ansible-nix/files/admin_api_scripts/manage_readonly_mode.sh create mode 100644 ansible-nix/files/admin_api_scripts/pg_egress_collect.pl create mode 100755 ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/check.sh create mode 100755 ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/common.sh create mode 100755 ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/complete.sh create mode 100755 ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh create mode 100755 ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/pgsodium_getkey.sh create mode 100755 ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh create mode 100644 ansible-nix/files/adminapi.service.j2 create mode 100644 ansible-nix/files/adminapi.sudoers.conf create mode 100644 ansible-nix/files/ansible-pull.service create mode 100644 ansible-nix/files/ansible-pull.timer create mode 100644 ansible-nix/files/apt_periodic create mode 100644 ansible-nix/files/cron.deny create mode 100644 ansible-nix/files/database-optimizations.service.j2 create mode 100644 ansible-nix/files/default.sysstat create mode 100644 ansible-nix/files/envoy.service create mode 100644 ansible-nix/files/envoy_config/cds.yaml create mode 100644 ansible-nix/files/envoy_config/envoy.yaml create mode 100644 ansible-nix/files/envoy_config/lds.yaml create mode 100644 ansible-nix/files/envoy_config/remove_apikey_query_parameter.lua create mode 100644 ansible-nix/files/fail2ban_config/fail2ban.service.conf create mode 100644 ansible-nix/files/fail2ban_config/filter-pgbouncer.conf.j2 create mode 100644 ansible-nix/files/fail2ban_config/filter-postgresql.conf.j2 create mode 100644 ansible-nix/files/fail2ban_config/jail-pgbouncer.conf.j2 create mode 100644 ansible-nix/files/fail2ban_config/jail-postgresql.conf.j2 create mode 100644 ansible-nix/files/fail2ban_config/jail-ssh.conf create mode 100644 ansible-nix/files/fail2ban_config/jail.local create mode 100644 ansible-nix/files/gotrue.service.j2 create mode 100644 ansible-nix/files/journald.conf create mode 100644 ansible-nix/files/kong_config/kong.conf.j2 create mode 100644 ansible-nix/files/kong_config/kong.env.j2 create mode 100644 ansible-nix/files/kong_config/kong.service.j2 create mode 100644 ansible-nix/files/logind.conf create mode 100644 ansible-nix/files/logrotate_config/logrotate-postgres-auth.conf create mode 100644 ansible-nix/files/logrotate_config/logrotate-postgres-csv.conf create mode 100644 ansible-nix/files/logrotate_config/logrotate-postgres.conf create mode 100644 ansible-nix/files/logrotate_config/logrotate-walg.conf create mode 100644 ansible-nix/files/manifest.json create mode 100644 ansible-nix/files/nginx.service.j2 create mode 100644 ansible-nix/files/pg_egress_collect.service.j2 create mode 100644 ansible-nix/files/pgbouncer_config/pgbouncer.ini.j2 create mode 100644 ansible-nix/files/pgbouncer_config/pgbouncer.service.j2 create mode 100644 ansible-nix/files/pgbouncer_config/pgbouncer_auth_schema.sql create mode 100644 ansible-nix/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 create mode 100644 ansible-nix/files/pgsodium_getkey_readonly.sh.j2 create mode 100755 ansible-nix/files/pgsodium_getkey_urandom.sh.j2 create mode 100644 ansible-nix/files/postgres_exporter.service.j2 create mode 100644 ansible-nix/files/postgresql_config/custom_read_replica.conf.j2 create mode 100644 ansible-nix/files/postgresql_config/custom_walg.conf.j2 create mode 100755 ansible-nix/files/postgresql_config/pg_hba.conf.j2 create mode 100755 ansible-nix/files/postgresql_config/pg_ident.conf.j2 create mode 100644 ansible-nix/files/postgresql_config/postgresql-csvlog.conf create mode 100644 ansible-nix/files/postgresql_config/postgresql-stdout-log.conf create mode 100644 ansible-nix/files/postgresql_config/postgresql.conf.j2 create mode 100644 ansible-nix/files/postgresql_config/postgresql.service.j2 create mode 100644 ansible-nix/files/postgresql_config/supautils.conf.j2 create mode 100644 ansible-nix/files/postgresql_config/tmpfiles.postgresql.conf create mode 100644 ansible-nix/files/postgresql_extension_custom_scripts/before-create.sql create mode 100644 ansible-nix/files/postgresql_extension_custom_scripts/dblink/after-create.sql create mode 100644 ansible-nix/files/postgresql_extension_custom_scripts/pg_cron/after-create.sql create mode 100644 ansible-nix/files/postgresql_extension_custom_scripts/pg_tle/after-create.sql create mode 100644 ansible-nix/files/postgresql_extension_custom_scripts/pgsodium/after-create.sql create mode 100644 ansible-nix/files/postgresql_extension_custom_scripts/postgis_tiger_geocoder/after-create.sql create mode 100644 ansible-nix/files/postgresql_extension_custom_scripts/postgres_fdw/after-create.sql create mode 100644 ansible-nix/files/postgrest-optimizations.service.j2 create mode 100644 ansible-nix/files/postgrest.service.j2 create mode 100644 ansible-nix/files/services.slice.j2 create mode 100644 ansible-nix/files/sodium_extension.sql create mode 100644 ansible-nix/files/start-envoy.sh create mode 100644 ansible-nix/files/stat_extension.sql create mode 100644 ansible-nix/files/supabase_facts.ini create mode 100644 ansible-nix/files/sysstat.sysstat create mode 100644 ansible-nix/files/systemd-resolved.conf create mode 100644 ansible-nix/files/ufw.service.conf create mode 100644 ansible-nix/files/vector.service.j2 create mode 100644 ansible-nix/files/walg_helper_scripts/wal_change_ownership.sh create mode 100644 ansible-nix/files/walg_helper_scripts/wal_fetch.sh create mode 100644 ansible-nix/manifest-playbook.yml create mode 100644 ansible-nix/playbook.yml create mode 100644 ansible-nix/tasks/clean-build-dependencies.yml create mode 100644 ansible-nix/tasks/finalize-ami.yml create mode 100644 ansible-nix/tasks/internal/admin-api.yml create mode 100644 ansible-nix/tasks/internal/admin-mgr.yml create mode 100644 ansible-nix/tasks/internal/pg_egress_collect.yml create mode 100644 ansible-nix/tasks/internal/postgres-exporter.yml create mode 100644 ansible-nix/tasks/internal/setup-ansible-pull.yml create mode 100644 ansible-nix/tasks/internal/setup-nftables.yml create mode 100644 ansible-nix/tasks/internal/supautils.yml create mode 100644 ansible-nix/tasks/setup-envoy.yml create mode 100644 ansible-nix/tasks/setup-extensions.yml create mode 100644 ansible-nix/tasks/setup-fail2ban.yml create mode 100644 ansible-nix/tasks/setup-gotrue.yml create mode 100644 ansible-nix/tasks/setup-kong.yml create mode 100644 ansible-nix/tasks/setup-migrations.yml create mode 100644 ansible-nix/tasks/setup-nginx.yml create mode 100644 ansible-nix/tasks/setup-pgbouncer.yml create mode 100644 ansible-nix/tasks/setup-postgres.yml create mode 100644 ansible-nix/tasks/setup-postgrest.yml create mode 100644 ansible-nix/tasks/setup-supabase-internal.yml create mode 100644 ansible-nix/tasks/setup-system.yml create mode 100644 ansible-nix/tasks/setup-wal-g.yml rename {ansible => ansible-nix}/tasks/stage2/optimizations.yml (100%) rename {ansible => ansible-nix}/tasks/stage2/playbook.yml (100%) rename {ansible => ansible-nix}/tasks/stage2/setup-extensions.yml (100%) rename {ansible => ansible-nix}/tasks/stage2/setup-migrations.yml (100%) rename {ansible => ansible-nix}/tasks/stage2/stage2-setup-postgres.yml (100%) rename {ansible => ansible-nix}/tasks/stage2/test-image.yml (100%) create mode 100644 ansible-nix/tasks/test-image.yml create mode 100644 ansible-nix/vars.yml create mode 100644 ansible/tasks/internal/collect-pg-binaries.yml create mode 100644 ansible/tasks/internal/optimizations.yml create mode 100644 ansible/tasks/setup-docker.yml diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl new file mode 100644 index 000000000..559387b1e --- /dev/null +++ b/amazon-arm64-nix.pkr.hcl @@ -0,0 +1,272 @@ +variable "ami" { + type = string + default = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-*" +} + +variable "profile" { + type = string + default = "${env("AWS_PROFILE")}" +} + +variable "ami_name" { + type = string + default = "supabase-postgres" +} + +variable "ami_regions" { + type = list(string) + default = ["ap-southeast-2"] +} + +variable "ansible_arguments" { + type = string + default = "--skip-tags install-postgrest,install-pgbouncer,install-supabase-internal" +} + +variable "aws_access_key" { + type = string + default = "" +} + +variable "aws_secret_key" { + type = string + default = "" +} + +variable "environment" { + type = string + default = "prod" +} + +variable "region" { + type = string +} + +variable "build-vol" { + type = string + default = "xvdc" +} + +# ccache docker image details +variable "docker_user" { + type = string + default = "" +} + +variable "docker_passwd" { + type = string + default = "" +} + +variable "docker_image" { + type = string + default = "" +} + +variable "docker_image_tag" { + type = string + default = "latest" +} + +locals { + creator = "packer" +} + +variable "postgres-version" { + type = string + default = "" +} + +variable "git-head-version" { + type = string + default = "unknown" +} + +variable "packer-execution-id" { + type = string + default = "unknown" +} + +variable "force-deregister" { + type = bool + default = false +} + +packer { + required_plugins { + amazon = { + source = "github.com/hashicorp/amazon" + version = "~> 1" + } + } +} + +# source block +source "amazon-ebssurrogate" "source" { + profile = "${var.profile}" + #access_key = "${var.aws_access_key}" + #ami_name = "${var.ami_name}-arm64-${formatdate("YYYY-MM-DD-hhmm", timestamp())}" + ami_name = "${var.ami_name}-${var.postgres-version}-stage-1" + ami_virtualization_type = "hvm" + ami_architecture = "arm64" + ami_regions = "${var.ami_regions}" + instance_type = "c6g.4xlarge" + region = "${var.region}" + #secret_key = "${var.aws_secret_key}" + force_deregister = var.force-deregister + + # Use latest official ubuntu focal ami owned by Canonical. + source_ami_filter { + filters = { + virtualization-type = "hvm" + name = "${var.ami}" + root-device-type = "ebs" + } + owners = [ "099720109477" ] + most_recent = true + } + ena_support = true + launch_block_device_mappings { + device_name = "/dev/xvdf" + delete_on_termination = true + volume_size = 10 + volume_type = "gp3" + } + + launch_block_device_mappings { + device_name = "/dev/xvdh" + delete_on_termination = true + volume_size = 8 + volume_type = "gp3" + } + + launch_block_device_mappings { + device_name = "/dev/${var.build-vol}" + delete_on_termination = true + volume_size = 16 + volume_type = "gp2" + omit_from_artifact = true + } + + run_tags = { + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" + } + run_volume_tags = { + creator = "packer" + appType = "postgres" + } + snapshot_tags = { + creator = "packer" + appType = "postgres" + } + tags = { + creator = "packer" + appType = "postgres" + postgresVersion = "${var.postgres-version}" + sourceSha = "${var.git-head-version}" + } + + communicator = "ssh" + ssh_pty = true + ssh_username = "ubuntu" + ssh_timeout = "5m" + + ami_root_device { + source_device_name = "/dev/xvdf" + device_name = "/dev/xvda" + delete_on_termination = true + volume_size = 10 + volume_type = "gp2" + } + + associate_public_ip_address = true +} + +# a build block invokes sources and runs provisioning steps on them. +build { + sources = ["source.amazon-ebssurrogate.source"] + + provisioner "file" { + source = "ebssurrogate/files/sources-arm64.cfg" + destination = "/tmp/sources.list" + } + + provisioner "file" { + source = "ebssurrogate/files/ebsnvme-id" + destination = "/tmp/ebsnvme-id" + } + + provisioner "file" { + source = "ebssurrogate/files/70-ec2-nvme-devices.rules" + destination = "/tmp/70-ec2-nvme-devices.rules" + } + + provisioner "file" { + source = "ebssurrogate/scripts/chroot-bootstrap.sh" + destination = "/tmp/chroot-bootstrap.sh" + } + + provisioner "file" { + source = "ebssurrogate/files/cloud.cfg" + destination = "/tmp/cloud.cfg" + } + + provisioner "file" { + source = "ebssurrogate/files/vector.timer" + destination = "/tmp/vector.timer" + } + + provisioner "file" { + source = "ebssurrogate/files/apparmor_profiles" + destination = "/tmp" + } + + provisioner "file" { + source = "migrations" + destination = "/tmp" + } + + provisioner "file" { + source = "ebssurrogate/files/unit-tests" + destination = "/tmp" + } + + # Copy ansible playbook + provisioner "shell" { + inline = ["mkdir /tmp/ansible-playbook"] + } + + provisioner "file" { + source = "ansible-nix" + destination = "/tmp/ansible-playbook" + } + + provisioner "file" { + source = "scripts" + destination = "/tmp/ansible-playbook" + } + + provisioner "shell" { + environment_vars = [ + "ARGS=${var.ansible_arguments}", + "DOCKER_USER=${var.docker_user}", + "DOCKER_PASSWD=${var.docker_passwd}", + "DOCKER_IMAGE=${var.docker_image}", + "DOCKER_IMAGE_TAG=${var.docker_image_tag}", + "POSTGRES_SUPABASE_VERSION=${var.postgres-version}" + ] + use_env_var_file = true + script = "ebssurrogate/scripts/surrogate-bootstrap.sh" + execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && {{.Path}}'" + start_retry_timeout = "5m" + skip_clean = true + } + + provisioner "file" { + source = "/tmp/ansible.log" + destination = "/tmp/ansible.log" + direction = "download" + } +} diff --git a/amazon-arm64.pkr.hcl b/amazon-arm64.pkr.hcl index 7486d0f09..993537297 100644 --- a/amazon-arm64.pkr.hcl +++ b/amazon-arm64.pkr.hcl @@ -106,7 +106,7 @@ source "amazon-ebssurrogate" "source" { profile = "${var.profile}" #access_key = "${var.aws_access_key}" #ami_name = "${var.ami_name}-arm64-${formatdate("YYYY-MM-DD-hhmm", timestamp())}" - ami_name = "${var.ami_name}-${var.postgres-version}-stage-1" + ami_name = "${var.ami_name}-${var.postgres-version}" ami_virtualization_type = "hvm" ami_architecture = "arm64" ami_regions = "${var.ami_regions}" @@ -270,10 +270,9 @@ build { direction = "download" } - // provisioner "file" { - // source = "/tmp/pg_binaries.tar.gz" - // destination = "/tmp/pg_binaries.tar.gz" - // direction = "download" - // } - // TODO RESOLVE IN NEW BUILD + provisioner "file" { + source = "/tmp/pg_binaries.tar.gz" + destination = "/tmp/pg_binaries.tar.gz" + direction = "download" + } } diff --git a/ansible-nix/files/admin_api_scripts/grow_fs.sh b/ansible-nix/files/admin_api_scripts/grow_fs.sh new file mode 100644 index 000000000..6d2a4e5e4 --- /dev/null +++ b/ansible-nix/files/admin_api_scripts/grow_fs.sh @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +set -euo pipefail + +VOLUME_TYPE=${1:-data} + +if [ -b /dev/nvme1n1 ] ; then + if [[ "${VOLUME_TYPE}" == "data" ]]; then + resize2fs /dev/nvme1n1 + + elif [[ "${VOLUME_TYPE}" == "root" ]] ; then + growpart /dev/nvme0n1 2 + resize2fs /dev/nvme0n1p2 + + else + echo "Invalid disk specified: ${VOLUME_TYPE}" + exit 1 + fi +else + growpart /dev/nvme0n1 2 + resize2fs /dev/nvme0n1p2 +fi +echo "Done resizing disk" diff --git a/ansible-nix/files/admin_api_scripts/manage_readonly_mode.sh b/ansible-nix/files/admin_api_scripts/manage_readonly_mode.sh new file mode 100644 index 000000000..41c9f5a1e --- /dev/null +++ b/ansible-nix/files/admin_api_scripts/manage_readonly_mode.sh @@ -0,0 +1,45 @@ +#! /usr/bin/env bash + +set -euo pipefail + +SUBCOMMAND=$1 + +function set_mode { + MODE=$1 + psql -h localhost -U supabase_admin -d postgres -c "ALTER SYSTEM SET default_transaction_read_only to ${MODE};" + psql -h localhost -U supabase_admin -d postgres -c "SELECT pg_reload_conf();" +} + +function check_override { + COMMAND=$(cat < 220.235.16.223.62599: Flags [S.], cksum 0x5de3 (incorrect -> 0x63da), seq 2314200657, ack 2071735457, win 62643, options [mss 8961,sackOK,TS val 3358598837 ecr 1277499190,nop,wscale 7], length 0 +# 1674013833.989257 IP (tos 0x0, ttl 64, id 24975, offset 0, flags [DF], proto TCP (6), length 52) +# 10.112.101.122.5432 > 220.235.16.223.62599: Flags [.], cksum 0x5ddb (incorrect -> 0xa25b), seq 1, ack 9, win 490, options [nop,nop,TS val 3358598885 ecr 1277499232], length 0 +# +# Sample IPv6 input lines: +# +# 1706483718.836526 IP6 (flowlabel 0x0bf27, hlim 64, next-header TCP (6) payload length: 125) 2406:da18:4fd:9b00:959:c52:ce68:10c8.5432 > 2406:da12:d78:f501:1273:296c:2482:c7a7.50530: Flags [P.], seq 25:118, ack 125, win 488, options [nop,nop,TS val 1026340732 ecr 1935666426], length 93 +# 1706483718.912083 IP6 (flowlabel 0x0bf27, hlim 64, next-header TCP (6) payload length: 501) 2406:da18:4fd:9b00:959:c52:ce68:10c8.5432 > 2406:da12:d78:f501:1273:296c:2482:c7a7.50530: Flags [P.], seq 118:587, ack 234, win 488, options [nop,nop,TS val 1026340807 ecr 1935666497], length 469 +# 1706483718.984001 IP6 (flowlabel 0x0bf27, hlim 64, next-header TCP (6) payload length: 151) 2406:da18:4fd:9b00:959:c52:ce68:10c8.5432 > 2406:da12:d78:f501:1273:296c:2482:c7a7.50530: Flags [P.], seq 587:706, ack 448, win 487, options [nop,nop,TS val 1026340879 ecr 1935666569], length 119 +sub extract_packet_length { + my ($line) = @_; + + #print("debug: >> " . $line); + + if ($line =~ /^.*, length (\d+)$/) { + # extract tcp packet length and add it up + my $len = $1; + $captured_len += $len; + } +} + +# write total length to file +sub write_file { + my ($output) = @_; + + my $now = strftime "%F %T", localtime time; + print "[$now] write captured len $captured_len to $output\n"; + + open(my $fh, "+>", $output) or die "Could not open file '$output' $!"; + print $fh "$captured_len"; + close($fh) or die "Could not write file '$output' $!"; +} + +# main +sub main { + # get arguments + GetOptions( + "interval:i" => \(my $interval = 60), + "output:s" => \(my $output = "/tmp/pg_egress_collect.txt"), + "help" => sub { HelpMessage(0) }, + ) or HelpMessage(1); + + my $loop = IO::Async::Loop->new; + + # tcpdump extractor + my $extractor = IO::Async::Stream->new_for_stdin( + on_read => sub { + my ($self, $buffref, $eof) = @_; + + while($$buffref =~ s/^(.*\n)//) { + my $line = $1; + extract_packet_length($line); + } + + return 0; + }, + ); + + # schedule file writer per minute + my $writer = IO::Async::Timer::Periodic->new( + interval => $interval, + on_tick => sub { + write_file($output); + + # reset total captured length + $captured_len = 0; + }, + ); + $writer->start; + + print "pg_egress_collect started, egress data will be saved to $output at interval $interval seconds.\n"; + + $loop->add($extractor); + $loop->add($writer); + $loop->run; +} + +main(); + +__END__ + +=head1 NAME + +pg_egress_collect.pl - collect egress from tcpdump output, extract TCP packet length, aggregate in specified interval and write to output file. + +=head1 SYNOPSIS + +pg_egress_collect.pl [-i interval] [-o output] + +Options: + + -i, --interval interval + output file write interval, in seconds, default is 60 seconds + + -o, --output output + output file path, default is /tmp/pg_egress_collect.txt + + -h, --help + print this help message + +=cut diff --git a/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/check.sh b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/check.sh new file mode 100755 index 000000000..f85e9571b --- /dev/null +++ b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/check.sh @@ -0,0 +1,16 @@ +#! /usr/bin/env bash +## This script provides a method to check the status of the database upgrade +## process, which is updated in /tmp/pg-upgrade-status by initiate.sh +## This runs on the old (source) instance. + +set -euo pipefail + +STATUS_FILE="/tmp/pg-upgrade-status" + +if [ -f "${STATUS_FILE}" ]; then + STATUS=$(cat "${STATUS_FILE}") + echo -n "${STATUS}" +else + echo -n "unknown" +fi + diff --git a/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/common.sh new file mode 100755 index 000000000..bf549b212 --- /dev/null +++ b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -0,0 +1,63 @@ +#! /usr/bin/env bash + +# Common functions and variables used by initiate.sh and complete.sh + +REPORTING_PROJECT_REF="ihmaxnjpcccasmrbkpvo" +REPORTING_CREDENTIALS_FILE="/root/upgrade-reporting-credentials" + +REPORTING_ANON_KEY="" +if [ -f "$REPORTING_CREDENTIALS_FILE" ]; then + REPORTING_ANON_KEY=$(cat "$REPORTING_CREDENTIALS_FILE") +fi + +function run_sql { + psql -h localhost -U supabase_admin -d postgres "$@" +} + +function ship_logs { + LOG_FILE=$1 + + if [ -z "$REPORTING_ANON_KEY" ]; then + echo "No reporting key found. Skipping log upload." + return 0 + fi + + if [ ! -f "$LOG_FILE" ]; then + echo "No log file found. Skipping log upload." + return 0 + fi + + if [ ! -s "$LOG_FILE" ]; then + echo "Log file is empty. Skipping log upload." + return 0 + fi + + HOSTNAME=$(hostname) + DERIVED_REF="${HOSTNAME##*-}" + + printf -v BODY '{ "ref": "%s", "step": "%s", "content": %s }' "$DERIVED_REF" "completion" "$(cat "$LOG_FILE" | jq -Rs '.')" + curl -sf -X POST "https://$REPORTING_PROJECT_REF.supabase.co/rest/v1/error_logs" \ + -H "apikey: ${REPORTING_ANON_KEY}" \ + -H 'Content-type: application/json' \ + -d "$BODY" +} + +function retry { + local retries=$1 + shift + + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** (count + 1))) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + echo "Command $* exited with code $exit, retrying..." + sleep $wait + else + echo "Command $* exited with code $exit, no more retries left." + return $exit + fi + done + return 0 +} diff --git a/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/complete.sh new file mode 100755 index 000000000..b139091ed --- /dev/null +++ b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -0,0 +1,153 @@ +#! /usr/bin/env bash + +## This script is run on the newly launched instance which is to be promoted to +## become the primary database instance once the upgrade successfully completes. +## The following commands copy custom PG configs and enable previously disabled +## extensions, containing regtypes referencing system OIDs. + +set -eEuo pipefail + +SCRIPT_DIR=$(dirname -- "$0";) +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/common.sh" + +LOG_FILE="/var/log/pg-upgrade-complete.log" + +function cleanup { + UPGRADE_STATUS=${1:-"failed"} + EXIT_CODE=${?:-0} + + echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status + + ship_logs "$LOG_FILE" || true + + exit "$EXIT_CODE" +} + +function execute_patches { + # Patch pg_net grants + PG_NET_ENABLED=$(run_sql -A -t -c "select count(*) > 0 from pg_extension where extname = 'pg_net';") + + if [ "$PG_NET_ENABLED" = "t" ]; then + PG_NET_GRANT_QUERY=$(cat < 0 from pg_extension where extname = 'pg_cron' and extowner::regrole::text = 'postgres';") + + if [ "$HAS_PG_CRON_OWNED_BY_POSTGRES" = "t" ]; then + RECREATE_PG_CRON_QUERY=$(cat < /tmp/pg-upgrade-status + + echo "1. Mounting data disk" + retry 3 mount -a -v + + # copying custom configurations + echo "2. Copying custom configurations" + retry 3 copy_configs + + echo "3. Starting postgresql" + retry 3 service postgresql start + + echo "4. Running generated SQL files" + retry 3 run_generated_sql + + echo "4.1. Applying patches" + execute_patches || true + + run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" + + echo "4.2. Applying authentication scheme updates" + retry 3 apply_auth_scheme_updates + + sleep 5 + + echo "5. Restarting postgresql" + retry 3 service postgresql restart + + echo "5.1. Restarting gotrue and postgrest" + retry 3 service gotrue restart + retry 3 service postgrest restart + + echo "6. Starting vacuum analyze" + retry 3 start_vacuum_analyze +} + +function copy_configs { + cp -R /data/conf/* /etc/postgresql-custom/ + chown -R postgres:postgres /var/lib/postgresql/data + chown -R postgres:postgres /data/pgdata +} + +function run_generated_sql { + if [ -d /data/sql ]; then + for FILE in /data/sql/*.sql; do + if [ -f "$FILE" ]; then + run_sql -f "$FILE" + fi + done + fi +} + +# Projects which had their passwords hashed using md5 need to have their passwords reset +# Passwords for managed roles are already present in /etc/postgresql.schema.sql +function apply_auth_scheme_updates { + PASSWORD_ENCRYPTION_SETTING=$(run_sql -A -t -c "SHOW password_encryption;") + if [ "$PASSWORD_ENCRYPTION_SETTING" = "md5" ]; then + run_sql -c "ALTER SYSTEM SET password_encryption TO 'scram-sha-256';" + run_sql -c "SELECT pg_reload_conf();" + run_sql -f /etc/postgresql.schema.sql + fi +} + +function start_vacuum_analyze { + echo "complete" > /tmp/pg-upgrade-status + su -c 'vacuumdb --all --analyze-in-stages' -s "$SHELL" postgres + echo "Upgrade job completed" +} + +trap cleanup ERR + + +complete_pg_upgrade >> $LOG_FILE 2>&1 & diff --git a/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh new file mode 100755 index 000000000..492890a1e --- /dev/null +++ b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -0,0 +1,333 @@ +#! /usr/bin/env bash + +## This script is run on the old (source) instance, mounting the data disk +## of the newly launched instance, disabling extensions containing regtypes, +## and running pg_upgrade. +## It reports the current status of the upgrade process to /tmp/pg-upgrade-status, +## which can then be subsequently checked through check.sh. + +# Extensions to disable before running pg_upgrade. +# Running an upgrade with these extensions enabled will result in errors due to +# them depending on regtypes referencing system OIDs or outdated library files. +EXTENSIONS_TO_DISABLE=( + "pg_graphql" +) + +PG14_EXTENSIONS_TO_DISABLE=( + "wrappers" + "pgrouting" +) + +PG13_EXTENSIONS_TO_DISABLE=( + "pgrouting" +) + +set -eEuo pipefail + +SCRIPT_DIR=$(dirname -- "$0";) +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/common.sh" + +LOG_FILE="/var/log/pg-upgrade-initiate.log" + +PGVERSION=$1 +IS_DRY_RUN=${2:-false} +if [ "$IS_DRY_RUN" != false ]; then + IS_DRY_RUN=true +fi + +MOUNT_POINT="/data_migration" + +POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" +OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") + +POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" +PGBINOLD="/usr/lib/postgresql/bin" +PGLIBOLD="/usr/lib/postgresql/lib" + +# If upgrading from older major PG versions, disable specific extensions +if [[ "$OLD_PGVERSION" =~ ^14.* ]]; then + EXTENSIONS_TO_DISABLE+=("${PG14_EXTENSIONS_TO_DISABLE[@]}") +elif [[ "$OLD_PGVERSION" =~ ^13.* ]]; then + EXTENSIONS_TO_DISABLE+=("${PG13_EXTENSIONS_TO_DISABLE[@]}") +elif [[ "$OLD_PGVERSION" =~ ^12.* ]]; then + POSTGRES_CONFIG_PATH="/etc/postgresql/12/main/postgresql.conf" + PGBINOLD="/usr/lib/postgresql/12/bin" +fi + +echo "Detected PG version: $PGVERSION" + +cleanup() { + UPGRADE_STATUS=${1:-"failed"} + EXIT_CODE=${?:-0} + + if [ "$UPGRADE_STATUS" = "failed" ]; then + echo "Upgrade job failed. Cleaning up and exiting." + fi + + if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then + echo "Copying pg_upgrade output to /var/log" + cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ || true + ship_logs "$LOG_FILE" || true + tail -n +1 /var/log/pg_upgrade_output.d/*/* > /var/log/pg_upgrade_output.d/pg_upgrade.log || true + ship_logs "/var/log/pg_upgrade_output.d/pg_upgrade.log" || true + fi + + if [ -L "/usr/share/postgresql/${PGVERSION}" ]; then + rm "/usr/share/postgresql/${PGVERSION}" + + if [ -f "/usr/share/postgresql/${PGVERSION}.bak" ]; then + mv "/usr/share/postgresql/${PGVERSION}.bak" "/usr/share/postgresql/${PGVERSION}" + fi + + if [ -d "/usr/share/postgresql/${PGVERSION}.bak" ]; then + mv "/usr/share/postgresql/${PGVERSION}.bak" "/usr/share/postgresql/${PGVERSION}" + fi + fi + + if [ "$IS_DRY_RUN" = false ]; then + echo "Restarting postgresql" + systemctl enable postgresql + retry 5 systemctl restart postgresql + fi + + echo "Re-enabling extensions" + if [ -f $POST_UPGRADE_EXTENSION_SCRIPT ]; then + run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT + fi + + echo "Removing SUPERUSER grant from postgres" + run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" + + if [ "$IS_DRY_RUN" = false ]; then + echo "Unmounting data disk from ${MOUNT_POINT}" + umount $MOUNT_POINT + fi + echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status + + exit "$EXIT_CODE" +} + +function handle_extensions { + rm -f $POST_UPGRADE_EXTENSION_SCRIPT + touch $POST_UPGRADE_EXTENSION_SCRIPT + + PASSWORD_ENCRYPTION_SETTING=$(run_sql -A -t -c "SHOW password_encryption;") + if [ "$PASSWORD_ENCRYPTION_SETTING" = "md5" ]; then + echo "ALTER SYSTEM SET password_encryption = 'md5';" >> $POST_UPGRADE_EXTENSION_SCRIPT + fi + + cat << EOF >> $POST_UPGRADE_EXTENSION_SCRIPT +ALTER SYSTEM SET jit = off; +SELECT pg_reload_conf(); +EOF + + # Disable extensions if they're enabled + # Generate SQL script to re-enable them after upgrade + for EXTENSION in "${EXTENSIONS_TO_DISABLE[@]}"; do + EXTENSION_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = '${EXTENSION}');") + if [ "$EXTENSION_ENABLED" = "t" ]; then + echo "Disabling extension ${EXTENSION}" + run_sql -c "DROP EXTENSION IF EXISTS ${EXTENSION} CASCADE;" + cat << EOF >> $POST_UPGRADE_EXTENSION_SCRIPT +DO \$\$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = '${EXTENSION}') THEN + CREATE EXTENSION IF NOT EXISTS ${EXTENSION} CASCADE; + END IF; +END; +\$\$; +EOF + fi + done +} + +function initiate_upgrade { + mkdir -p "$MOUNT_POINT" + SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/") + + # Wrappers officially launched in PG15; PG14 version is incompatible + if [[ "$OLD_PGVERSION" =~ 14* ]]; then + SHARED_PRELOAD_LIBRARIES=$(echo "$SHARED_PRELOAD_LIBRARIES" | sed "s/wrappers,//" | xargs) + fi + SHARED_PRELOAD_LIBRARIES=$(echo "$SHARED_PRELOAD_LIBRARIES" | sed "s/pg_cron,//" | xargs) + SHARED_PRELOAD_LIBRARIES=$(echo "$SHARED_PRELOAD_LIBRARIES" | sed "s/check_role_membership,//" | xargs) + + PGDATAOLD=$(cat "$POSTGRES_CONFIG_PATH" | grep data_directory | sed "s/data_directory = '\(.*\)'.*/\1/") + + PGDATANEW="$MOUNT_POINT/pgdata" + PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" + PGBINNEW="$PG_UPGRADE_BIN_DIR/bin" + PGLIBNEW="$PG_UPGRADE_BIN_DIR/lib" + PGSHARENEW="$PG_UPGRADE_BIN_DIR/share" + + # running upgrade using at least 1 cpu core + WORKERS=$(nproc | awk '{ print ($1 == 1 ? 1 : $1 - 1) }') + + echo "1. Extracting pg_upgrade binaries" + mkdir -p "/tmp/pg_upgrade_bin" + tar zxf "/tmp/persistent/pg_upgrade_bin.tar.gz" -C "/tmp/pg_upgrade_bin" + + # copy upgrade-specific pgsodium_getkey script into the share dir + chmod +x "$SCRIPT_DIR/pgsodium_getkey.sh" + cp "$SCRIPT_DIR/pgsodium_getkey.sh" "$PGSHARENEW/extension/pgsodium_getkey" + if [ -d "/var/lib/postgresql/extension/" ]; then + cp "$SCRIPT_DIR/pgsodium_getkey.sh" "/var/lib/postgresql/extension/pgsodium_getkey" + chown postgres:postgres "/var/lib/postgresql/extension/pgsodium_getkey" + fi + + chown -R postgres:postgres "/tmp/pg_upgrade_bin/$PGVERSION" + + # upgrade job outputs a log in the cwd; needs write permissions + mkdir -p /tmp/pg_upgrade/ + chown -R postgres:postgres /tmp/pg_upgrade/ + cd /tmp/pg_upgrade/ + + # Fixing erros generated by previous dpkg executions (package upgrades et co) + echo "2. Fixing potential errors generated by dpkg" + DEBIAN_FRONTEND=noninteractive dpkg --configure -a --force-confold || true # handle errors generated by dpkg + + # Needed for PostGIS, since it's compiled with Protobuf-C support now + echo "3. Installing libprotobuf-c1 if missing" + if [[ ! "$(apt list --installed libprotobuf-c1 | grep "installed")" ]]; then + apt-get update && apt --fix-broken install -y libprotobuf-c1 + fi + + echo "4. Setup locale if required" + if ! grep -q "^en_US.UTF-8" /etc/locale.gen ; then + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen + locale-gen + fi + + if [ "$IS_DRY_RUN" = false ]; then + # awk NF==3 prints lines with exactly 3 fields, which are the block devices currently not mounted anywhere + # excluding nvme0 since it is the root disk + echo "5. Determining block device to mount" + BLOCK_DEVICE=$(lsblk -dprno name,size,mountpoint,type | grep "disk" | grep -v "nvme0" | awk 'NF==3 { print $1; }') + echo "Block device found: $BLOCK_DEVICE" + + mkdir -p "$MOUNT_POINT" + echo "6. Mounting block device" + + sleep 5 + e2fsck -pf "$BLOCK_DEVICE" + + sleep 1 + mount "$BLOCK_DEVICE" "$MOUNT_POINT" + + sleep 1 + resize2fs "$BLOCK_DEVICE" + fi + + if [ -f "$MOUNT_POINT/pgsodium_root.key" ]; then + cp "$MOUNT_POINT/pgsodium_root.key" /etc/postgresql-custom/pgsodium_root.key + chown postgres:postgres /etc/postgresql-custom/pgsodium_root.key + chmod 600 /etc/postgresql-custom/pgsodium_root.key + fi + + echo "7. Disabling extensions and generating post-upgrade script" + handle_extensions + + echo "8. Granting SUPERUSER to postgres user" + run_sql -c "ALTER USER postgres WITH SUPERUSER;" + + if [ -d "/usr/share/postgresql/${PGVERSION}" ]; then + mv "/usr/share/postgresql/${PGVERSION}" "/usr/share/postgresql/${PGVERSION}.bak" + fi + ln -s "$PGSHARENEW" "/usr/share/postgresql/${PGVERSION}" + + cp --remove-destination "$PGLIBNEW"/*.control "$PGSHARENEW/extension/" + cp --remove-destination "$PGLIBNEW"/*.sql "$PGSHARENEW/extension/" + + # This is a workaround for older versions of wrappers which don't have the expected + # naming scheme, containing the version in their library's file name + # e.g. wrappers-0.1.16.so, rather than wrappers.so + # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's + # library directory, so we're making sure the new version has the expected (old version's) + # file name. + # After the upgrade completes, the new version's library file is used. + # i.e. + # - old version: wrappers-0.1.16.so + # - new version: wrappers-0.1.18.so + # - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so + if [ -d "$PGLIBOLD" ]; then + WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit) + if [ -f "$WRAPPERS_LIB_PATH" ]; then + OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) + if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then + LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") + if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then + echo "Copying $OLD_WRAPPER_LIB_PATH to $WRAPPERS_LIB_PATH" + cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" + fi + fi + fi + fi + + export LD_LIBRARY_PATH="${PGLIBNEW}" + + echo "9. Creating new data directory, initializing database" + chown -R postgres:postgres "$MOUNT_POINT/" + rm -rf "${PGDATANEW:?}/" + su -c "$PGBINNEW/initdb -L $PGSHARENEW -D $PGDATANEW/" -s "$SHELL" postgres + + UPGRADE_COMMAND=$(cat < /tmp/pg-upgrade-status +if [ "$IS_DRY_RUN" = true ]; then + initiate_upgrade +else + initiate_upgrade >> "$LOG_FILE" 2>&1 & + echo "Upgrade initiate job completed" +fi diff --git a/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/pgsodium_getkey.sh b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/pgsodium_getkey.sh new file mode 100755 index 000000000..5a5a90e44 --- /dev/null +++ b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/pgsodium_getkey.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +KEY_FILE=/etc/postgresql-custom/pgsodium_root.key + +# if key file doesn't exist (project previously didn't use pgsodium), generate a new key +if [[ ! -f "${KEY_FILE}" ]]; then + head -c 32 /dev/urandom | od -A n -t x1 | tr -d ' \n' > $KEY_FILE +fi + +cat $KEY_FILE diff --git a/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh new file mode 100755 index 000000000..7d7eb9890 --- /dev/null +++ b/ansible-nix/files/admin_api_scripts/pg_upgrade_scripts/prepare.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env bash +## This script is runs in advance of the database version upgrade, on the newly +## launched instance which will eventually be promoted to become the primary +## database instance once the upgrade successfully completes, terminating the +## previous (source) instance. +## The following commands safely stop the Postgres service and unmount +## the data disk off the newly launched instance, to be re-attached to the +## source instance and run the upgrade there. + +set -euo pipefail + +systemctl stop postgresql + +cp /etc/postgresql-custom/pgsodium_root.key /data/pgsodium_root.key +umount /data diff --git a/ansible-nix/files/adminapi.service.j2 b/ansible-nix/files/adminapi.service.j2 new file mode 100644 index 000000000..6078f3d1a --- /dev/null +++ b/ansible-nix/files/adminapi.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=AdminAPI + +[Service] +Type=simple +ExecStart=/opt/supabase-admin-api +User=adminapi +Restart=always +RestartSec=3 +Environment="AWS_USE_DUALSTACK_ENDPOINT=true" + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/adminapi.sudoers.conf b/ansible-nix/files/adminapi.sudoers.conf new file mode 100644 index 000000000..eada0a94b --- /dev/null +++ b/ansible-nix/files/adminapi.sudoers.conf @@ -0,0 +1,28 @@ +Cmnd_Alias ENVOY = /bin/systemctl start envoy.service, /bin/systemctl stop envoy.service, /bin/systemctl restart envoy.service, /bin/systemctl disable envoy.service, /bin/systemctl enable envoy.service, /bin/systemctl reload envoy.service, /bin/systemctl try-restart envoy.service +Cmnd_Alias KONG = /bin/systemctl start kong.service, /bin/systemctl stop kong.service, /bin/systemctl restart kong.service, /bin/systemctl disable kong.service, /bin/systemctl enable kong.service, /bin/systemctl reload kong.service, /bin/systemctl try-restart kong.service +Cmnd_Alias POSTGREST = /bin/systemctl start postgrest.service, /bin/systemctl stop postgrest.service, /bin/systemctl restart postgrest.service, /bin/systemctl disable postgrest.service, /bin/systemctl enable postgrest.service, /bin/systemctl try-restart postgrest.service +Cmnd_Alias GOTRUE = /bin/systemctl start gotrue.service, /bin/systemctl stop gotrue.service, /bin/systemctl restart gotrue.service, /bin/systemctl disable gotrue.service, /bin/systemctl enable gotrue.service, /bin/systemctl try-restart gotrue.service +Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl stop pgbouncer.service, /bin/systemctl restart pgbouncer.service, /bin/systemctl disable pgbouncer.service, /bin/systemctl enable pgbouncer.service, /bin/systemctl reload pgbouncer.service, /bin/systemctl try-restart pgbouncer.service + +%adminapi ALL= NOPASSWD: /root/grow_fs.sh +%adminapi ALL= NOPASSWD: /root/manage_readonly_mode.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/prepare.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/initiate.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/complete.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/check.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/common.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/pgsodium_getkey.sh +%adminapi ALL= NOPASSWD: /usr/bin/systemctl daemon-reload +%adminapi ALL= NOPASSWD: /usr/bin/systemctl reload postgresql.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl restart postgresql.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl show -p NRestarts postgresql.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl restart adminapi.service +%adminapi ALL= NOPASSWD: /bin/systemctl daemon-reload +%adminapi ALL= NOPASSWD: /bin/systemctl restart services.slice +%adminapi ALL= NOPASSWD: /usr/sbin/nft -f /etc/nftables/supabase_managed.conf +%adminapi ALL= NOPASSWD: /usr/bin/admin-mgr +%adminapi ALL= NOPASSWD: ENVOY +%adminapi ALL= NOPASSWD: KONG +%adminapi ALL= NOPASSWD: POSTGREST +%adminapi ALL= NOPASSWD: GOTRUE +%adminapi ALL= NOPASSWD: PGBOUNCER diff --git a/ansible-nix/files/ansible-pull.service b/ansible-nix/files/ansible-pull.service new file mode 100644 index 000000000..3e061b31b --- /dev/null +++ b/ansible-nix/files/ansible-pull.service @@ -0,0 +1,20 @@ +[Unit] +Description=Ansible pull + +[Service] +Type=simple +User=ubuntu + +ExecStart=/usr/bin/ansible-pull --private-key "$SSH_READ_KEY_FILE" -U "$REPO" --accept-host-key -t "$REGION,db-all" -i localhost --clean --full "$PLAYBOOK" -v -o -C "$REPO_BRANCH" + +# --verify-commit +# temporarily disable commit verification, while we figure out how we want to balance commit signatures +# and PR reviews; an --ff-only merge options would have allowed us to use this pretty nicely + +MemoryAccounting=true +MemoryMax=30% + +StandardOutput=append:/var/log/ansible-pull.stdout +StandardError=append:/var/log/ansible-pull.error + +TimeoutStopSec=600 diff --git a/ansible-nix/files/ansible-pull.timer b/ansible-nix/files/ansible-pull.timer new file mode 100644 index 000000000..27ce24b99 --- /dev/null +++ b/ansible-nix/files/ansible-pull.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run ansible roughly every 3 hours + +[Timer] +OnBootSec=1h +OnUnitActiveSec=3h +RandomizedDelaySec=1h +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible-nix/files/apt_periodic b/ansible-nix/files/apt_periodic new file mode 100644 index 000000000..75870203d --- /dev/null +++ b/ansible-nix/files/apt_periodic @@ -0,0 +1,4 @@ +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Download-Upgradeable-Packages "1"; +APT::Periodic::AutocleanInterval "7"; +APT::Periodic::Unattended-Upgrade "1"; \ No newline at end of file diff --git a/ansible-nix/files/cron.deny b/ansible-nix/files/cron.deny new file mode 100644 index 000000000..3b5199b06 --- /dev/null +++ b/ansible-nix/files/cron.deny @@ -0,0 +1,2 @@ +ubuntu +postgres diff --git a/ansible-nix/files/database-optimizations.service.j2 b/ansible-nix/files/database-optimizations.service.j2 new file mode 100644 index 000000000..f25fc09c6 --- /dev/null +++ b/ansible-nix/files/database-optimizations.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Postgresql optimizations + +[Service] +Type=oneshot +# we do not want failures from these commands to cause downstream service startup to fail +ExecStart=-/opt/supabase-admin-api optimize db --destination-config-file-path /etc/postgresql-custom/generated-optimizations.conf +ExecStart=-/opt/supabase-admin-api optimize pgbouncer --destination-config-file-path /etc/pgbouncer-custom/generated-optimizations.ini +User=adminapi + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/default.sysstat b/ansible-nix/files/default.sysstat new file mode 100644 index 000000000..1b029ba6b --- /dev/null +++ b/ansible-nix/files/default.sysstat @@ -0,0 +1,9 @@ +# +# Default settings for /etc/init.d/sysstat, /etc/cron.d/sysstat +# and /etc/cron.daily/sysstat files +# + +# Should sadc collect system activity informations? Valid values +# are "true" and "false". Please do not put other values, they +# will be overwritten by debconf! +ENABLED="true" diff --git a/ansible-nix/files/envoy.service b/ansible-nix/files/envoy.service new file mode 100644 index 000000000..d739ffdd5 --- /dev/null +++ b/ansible-nix/files/envoy.service @@ -0,0 +1,31 @@ +[Unit] +Description=Envoy +After=postgrest.service gotrue.service adminapi.service +Wants=postgrest.service gotrue.service adminapi.service +Conflicts=kong.service + +[Service] +Type=simple + +ExecStartPre=sh -c 'if ss -lnt | grep -Eq ":(80|443) "; then echo "Port 80 or 443 already in use"; exit 1; fi' + +# Need to run via a restarter script to support hot restart when using a process +# manager, see: +# https://www.envoyproxy.io/docs/envoy/latest/operations/hot_restarter +ExecStart=/opt/envoy-hot-restarter.py /opt/start-envoy.sh + +ExecReload=/bin/kill -HUP $MAINPID +ExecStop=/bin/kill -TERM $MAINPID +User=envoy +Slice=services.slice +Restart=always +RestartSec=3 +LimitNOFILE=100000 + +# The envoy user is unprivileged and thus not permitted to bind on ports < 1024 +# Via systemd we grant the process a set of privileges to bind to 80/443 +# See http://archive.vn/36zJU +AmbientCapabilities=CAP_NET_BIND_SERVICE + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/envoy_config/cds.yaml b/ansible-nix/files/envoy_config/cds.yaml new file mode 100644 index 000000000..8f921396a --- /dev/null +++ b/ansible-nix/files/envoy_config/cds.yaml @@ -0,0 +1,46 @@ +resources: + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: admin_api + load_assignment: + cluster_name: admin_api + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 8085 + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: gotrue + load_assignment: + cluster_name: gotrue + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 9999 + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: postgrest + load_assignment: + cluster_name: postgrest + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 3000 + - '@type': type.googleapis.com/envoy.config.cluster.v3.Cluster + name: postgrest_admin + load_assignment: + cluster_name: postgrest_admin + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 3001 + diff --git a/ansible-nix/files/envoy_config/envoy.yaml b/ansible-nix/files/envoy_config/envoy.yaml new file mode 100644 index 000000000..3d25c13cd --- /dev/null +++ b/ansible-nix/files/envoy_config/envoy.yaml @@ -0,0 +1,23 @@ +dynamic_resources: + cds_config: + path_config_source: + path: /etc/envoy/cds.yaml + resource_api_version: V3 + lds_config: + path_config_source: + path: /etc/envoy/lds.yaml + resource_api_version: V3 +node: + cluster: cluster_0 + id: node_0 +overload_manager: + resource_monitors: + - name: envoy.resource_monitors.global_downstream_max_connections + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 30000 +stats_config: + stats_matcher: + reject_all: true + diff --git a/ansible-nix/files/envoy_config/lds.yaml b/ansible-nix/files/envoy_config/lds.yaml new file mode 100644 index 000000000..84acfc041 --- /dev/null +++ b/ansible-nix/files/envoy_config/lds.yaml @@ -0,0 +1,315 @@ +resources: + - '@type': type.googleapis.com/envoy.config.listener.v3.Listener + name: http_listener + address: + socket_address: + address: '::' + port_value: 80 + ipv4_compat: true + filter_chains: + - filters: &ref_1 + - name: envoy.filters.network.http_connection_manager + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + access_log: + - name: envoy.access_loggers.stdout + filter: + status_code_filter: + comparison: + op: GE + value: + default_value: 400 + runtime_key: unused + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog + generate_request_id: false + http_filters: + - name: envoy.filters.http.cors + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors + - name: envoy.filters.http.rbac + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC + rules: + action: DENY + policies: + api_key_missing: + permissions: + - any: true + principals: + - not_id: + or_ids: + ids: + - header: + name: apikey + present_match: true + - header: + name: ':path' + string_match: + contains: apikey= + api_key_not_valid: + permissions: + - any: true + principals: + - not_id: + or_ids: + ids: + - header: + name: apikey + string_match: + exact: anon_key + - header: + name: apikey + string_match: + exact: service_key + - header: + name: apikey + string_match: + exact: supabase_admin_key + - header: + name: ':path' + string_match: + contains: apikey=anon_key + - header: + name: ':path' + string_match: + contains: apikey=service_key + - header: + name: ':path' + string_match: + contains: apikey=supabase_admin_key + - name: envoy.filters.http.lua + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + source_codes: + remove_apikey_query_parameter: + filename: /etc/envoy/remove_apikey_query_parameter.lua + - name: envoy.filters.http.router + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + dynamic_stats: false + local_reply_config: + mappers: + - filter: + and_filter: + filters: + - status_code_filter: + comparison: + value: + default_value: 403 + runtime_key: unused + - header_filter: + header: + name: ':path' + string_match: + prefix: /customer/v1/privileged/ + status_code: 401 + body: + inline_string: Unauthorized + headers_to_add: + - header: + key: WWW-Authenticate + value: Basic realm="Unknown" + - filter: + and_filter: + filters: + - status_code_filter: + comparison: + value: + default_value: 403 + runtime_key: unused + - header_filter: + header: + name: ':path' + string_match: + prefix: /metrics/aggregated + invert_match: true + status_code: 401 + body_format_override: + json_format: + message: >- + `apikey` request header or query parameter is either + missing or invalid. Double check your Supabase `anon` + or `service_role` API key. + hint: '%RESPONSE_CODE_DETAILS%' + json_format_options: + sort_properties: false + route_config: + name: route_config_0 + virtual_hosts: + - name: virtual_host_0 + domains: + - '*' + typed_per_filter_config: + envoy.filters.http.cors: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.cors.v3.CorsPolicy + allow_origin_string_match: + - safe_regex: + regex: \* + allow_methods: GET,HEAD,PUT,PATCH,POST,DELETE,OPTIONS,TRACE,CONNECT + allow_headers: apikey,authorization,x-client-info + max_age: '3600' + routes: + - match: + path: /health + direct_response: + status: 200 + body: + inline_string: Healthy + typed_per_filter_config: &ref_0 + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + - match: + safe_regex: + regex: >- + /auth/v1/(verify|callback|authorize|sso/saml/(acs|metadata|slo)) + route: + cluster: gotrue + regex_rewrite: + pattern: + regex: ^/auth/v1 + substitution: '' + retry_policy: + num_retries: 3 + retry_on: 5xx + typed_per_filter_config: *ref_0 + - match: + prefix: /auth/v1/ + route: + cluster: gotrue + prefix_rewrite: / + - match: + prefix: /rest/v1/ + query_parameters: + - name: apikey + present_match: true + request_headers_to_remove: + - apikey + route: + cluster: postgrest + prefix_rewrite: / + timeout: 120s + typed_per_filter_config: + envoy.filters.http.lua: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute + name: remove_apikey_query_parameter + - match: + prefix: /rest/v1/ + request_headers_to_remove: + - apikey + route: + cluster: postgrest + prefix_rewrite: / + timeout: 120s + - match: + prefix: /rest-admin/v1/ + query_parameters: + - name: apikey + present_match: true + request_headers_to_remove: + - apikey + route: + cluster: postgrest_admin + prefix_rewrite: / + typed_per_filter_config: + envoy.filters.http.lua: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.lua.v3.LuaPerRoute + name: remove_apikey_query_parameter + - match: + prefix: /rest-admin/v1/ + request_headers_to_remove: + - apikey + route: + cluster: postgrest_admin + prefix_rewrite: / + - match: + path: /graphql/v1 + request_headers_to_add: + header: + key: Content-Profile + value: graphql_public + route: + cluster: postgrest + prefix_rewrite: /rpc/graphql + timeout: 120s + - match: + prefix: /admin/v1/ + route: + cluster: admin_api + prefix_rewrite: / + - match: + prefix: /customer/v1/privileged/ + route: + cluster: admin_api + prefix_rewrite: /privileged/ + typed_per_filter_config: + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: DENY + policies: + basic_auth: + permissions: + - any: true + principals: + - header: + name: authorization + invert_match: true + string_match: + exact: Basic c2VydmljZV9yb2xlOnNlcnZpY2Vfa2V5 + treat_missing_header_as_empty: true + - match: + prefix: /metrics/aggregated + route: + cluster: admin_api + prefix_rewrite: /supabase-internal/metrics + typed_per_filter_config: + envoy.filters.http.rbac: + '@type': >- + type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute + rbac: + rules: + action: DENY + policies: + not_private_ip: + permissions: + - any: true + principals: + - not_id: + direct_remote_ip: + address_prefix: 10.0.0.0 + prefix_len: 8 + stat_prefix: ingress_http + - '@type': type.googleapis.com/envoy.config.listener.v3.Listener + name: https_listener + address: + socket_address: + address: '::' + port_value: 443 + ipv4_compat: true + filter_chains: + - filters: *ref_1 + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + '@type': >- + type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + tls_certificates: + - certificate_chain: + filename: /etc/envoy/fullChain.pem + private_key: + filename: /etc/envoy/privKey.pem + diff --git a/ansible-nix/files/envoy_config/remove_apikey_query_parameter.lua b/ansible-nix/files/envoy_config/remove_apikey_query_parameter.lua new file mode 100644 index 000000000..0aeabaeb1 --- /dev/null +++ b/ansible-nix/files/envoy_config/remove_apikey_query_parameter.lua @@ -0,0 +1,8 @@ +function envoy_on_request(request_handle) + local path = request_handle:headers():get(":path") + + -- Remove `apikey` query parameter since PostgREST treats query parameters as conditions. + request_handle + :headers() + :replace(":path", path:gsub("([&?])apikey=[^&]+&?", "%1"):gsub("&$", "")) +end diff --git a/ansible-nix/files/fail2ban_config/fail2ban.service.conf b/ansible-nix/files/fail2ban_config/fail2ban.service.conf new file mode 100644 index 000000000..431d1db5b --- /dev/null +++ b/ansible-nix/files/fail2ban_config/fail2ban.service.conf @@ -0,0 +1,6 @@ +[Unit] +After=nftables.service +Wants=nftables.service + +[Service] +ExecStartPost=/bin/bash -c "sleep 5 && chmod g+w /var/run/fail2ban/fail2ban.sock" diff --git a/ansible-nix/files/fail2ban_config/filter-pgbouncer.conf.j2 b/ansible-nix/files/fail2ban_config/filter-pgbouncer.conf.j2 new file mode 100644 index 000000000..3a3a52ec6 --- /dev/null +++ b/ansible-nix/files/fail2ban_config/filter-pgbouncer.conf.j2 @@ -0,0 +1,3 @@ +[Definition] +failregex = ^.+@:.+password authentication failed$ +journalmatch = _SYSTEMD_UNIT=pgbouncer.service diff --git a/ansible-nix/files/fail2ban_config/filter-postgresql.conf.j2 b/ansible-nix/files/fail2ban_config/filter-postgresql.conf.j2 new file mode 100644 index 000000000..fd0895aee --- /dev/null +++ b/ansible-nix/files/fail2ban_config/filter-postgresql.conf.j2 @@ -0,0 +1,3 @@ +[Definition] +failregex = ^.*,.*,.*,.*,":.*password authentication failed for user.*$ +ignoreregex = ^.*,.*,.*,.*,"127\.0\.0\.1.*password authentication failed for user.*$ \ No newline at end of file diff --git a/ansible-nix/files/fail2ban_config/jail-pgbouncer.conf.j2 b/ansible-nix/files/fail2ban_config/jail-pgbouncer.conf.j2 new file mode 100644 index 000000000..60a9eb3d9 --- /dev/null +++ b/ansible-nix/files/fail2ban_config/jail-pgbouncer.conf.j2 @@ -0,0 +1,7 @@ +[pgbouncer] +enabled = true +port = 6543 +protocol = tcp +filter = pgbouncer +backend = systemd[journalflags=1] +maxretry = 3 diff --git a/ansible-nix/files/fail2ban_config/jail-postgresql.conf.j2 b/ansible-nix/files/fail2ban_config/jail-postgresql.conf.j2 new file mode 100644 index 000000000..a021035a9 --- /dev/null +++ b/ansible-nix/files/fail2ban_config/jail-postgresql.conf.j2 @@ -0,0 +1,8 @@ +[postgresql] +enabled = true +port = 5432 +protocol = tcp +filter = postgresql +logpath = /var/log/postgresql/auth-failures.csv +maxretry = 3 +ignoreip = 192.168.0.0/16 172.17.1.0/20 diff --git a/ansible-nix/files/fail2ban_config/jail-ssh.conf b/ansible-nix/files/fail2ban_config/jail-ssh.conf new file mode 100644 index 000000000..5476c3093 --- /dev/null +++ b/ansible-nix/files/fail2ban_config/jail-ssh.conf @@ -0,0 +1,4 @@ +[sshd] + +backend = systemd +mode = aggressive diff --git a/ansible-nix/files/fail2ban_config/jail.local b/ansible-nix/files/fail2ban_config/jail.local new file mode 100644 index 000000000..44e8210f1 --- /dev/null +++ b/ansible-nix/files/fail2ban_config/jail.local @@ -0,0 +1,4 @@ +[DEFAULT] + +banaction = nftables-multiport +banaction_allports = nftables-allports diff --git a/ansible-nix/files/gotrue.service.j2 b/ansible-nix/files/gotrue.service.j2 new file mode 100644 index 000000000..f0bd60ab0 --- /dev/null +++ b/ansible-nix/files/gotrue.service.j2 @@ -0,0 +1,21 @@ +[Unit] +Description=Gotrue + +[Service] +Type=simple +WorkingDirectory=/opt/gotrue +ExecStart=/opt/gotrue/gotrue +User=gotrue +Restart=always +RestartSec=3 + +MemoryAccounting=true +MemoryMax=50% + +EnvironmentFile=/etc/gotrue.env +EnvironmentFile=-/etc/gotrue.overrides.env + +Slice=services.slice + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/journald.conf b/ansible-nix/files/journald.conf new file mode 100644 index 000000000..2eb89f91f --- /dev/null +++ b/ansible-nix/files/journald.conf @@ -0,0 +1,6 @@ +[Journal] +Storage=persistent +SystemMaxUse=3G +SystemKeepFree=3G +SystemMaxFileSize=200M +ForwardToSyslog=no diff --git a/ansible-nix/files/kong_config/kong.conf.j2 b/ansible-nix/files/kong_config/kong.conf.j2 new file mode 100644 index 000000000..39067575e --- /dev/null +++ b/ansible-nix/files/kong_config/kong.conf.j2 @@ -0,0 +1,7 @@ +database = off +declarative_config = /etc/kong/kong.yml + +# plugins defined in the dockerfile +plugins = request-transformer,cors,key-auth,http-log + +proxy_listen = 0.0.0.0:80 reuseport backlog=16384, 0.0.0.0:443 http2 ssl reuseport backlog=16834, [::]:80 reuseport backlog=16384, [::]:443 http2 ssl reuseport backlog=16384 diff --git a/ansible-nix/files/kong_config/kong.env.j2 b/ansible-nix/files/kong_config/kong.env.j2 new file mode 100644 index 000000000..57613fdbb --- /dev/null +++ b/ansible-nix/files/kong_config/kong.env.j2 @@ -0,0 +1,8 @@ +KONG_NGINX_HTTP_GZIP=on +KONG_NGINX_HTTP_GZIP_COMP_LEVEL=6 +KONG_NGINX_HTTP_GZIP_MIN_LENGTH=256 +KONG_NGINX_HTTP_GZIP_PROXIED=any +KONG_NGINX_HTTP_GZIP_VARY=on +KONG_NGINX_HTTP_GZIP_TYPES=text/plain application/xml application/openapi+json application/json +KONG_PROXY_ERROR_LOG=syslog:server=unix:/dev/log +KONG_ADMIN_ERROR_LOG=syslog:server=unix:/dev/log diff --git a/ansible-nix/files/kong_config/kong.service.j2 b/ansible-nix/files/kong_config/kong.service.j2 new file mode 100644 index 000000000..6a36520bc --- /dev/null +++ b/ansible-nix/files/kong_config/kong.service.j2 @@ -0,0 +1,28 @@ +[Unit] +Description=Kong server +After=postgrest.service gotrue.service adminapi.service +Wants=postgrest.service gotrue.service adminapi.service +Conflicts=envoy.service + +# Ensures that Kong service is stopped before Envoy service is started +Before=envoy.service + +[Service] +Type=forking +ExecStart=/usr/local/bin/kong start -c /etc/kong/kong.conf +ExecReload=/usr/local/bin/kong reload -c /etc/kong/kong.conf +ExecStop=/usr/local/bin/kong quit +User=kong +EnvironmentFile=/etc/kong/kong.env +Slice=services.slice +Restart=always +RestartSec=3 +LimitNOFILE=100000 + +# The kong user is unprivileged and thus not permitted to bind on ports < 1024 +# Via systemd we grant the process a set of privileges to bind to 80/443 +# See http://archive.vn/36zJU +AmbientCapabilities=CAP_NET_BIND_SERVICE + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/logind.conf b/ansible-nix/files/logind.conf new file mode 100644 index 000000000..732900f3c --- /dev/null +++ b/ansible-nix/files/logind.conf @@ -0,0 +1,2 @@ +[Login] +RemoveIPC=no diff --git a/ansible-nix/files/logrotate_config/logrotate-postgres-auth.conf b/ansible-nix/files/logrotate_config/logrotate-postgres-auth.conf new file mode 100644 index 000000000..050210e60 --- /dev/null +++ b/ansible-nix/files/logrotate_config/logrotate-postgres-auth.conf @@ -0,0 +1,8 @@ +/var/log/postgresql/auth-failures.csv { + size 10M + rotate 5 + compress + delaycompress + notifempty + missingok +} diff --git a/ansible-nix/files/logrotate_config/logrotate-postgres-csv.conf b/ansible-nix/files/logrotate_config/logrotate-postgres-csv.conf new file mode 100644 index 000000000..e5418e8e0 --- /dev/null +++ b/ansible-nix/files/logrotate_config/logrotate-postgres-csv.conf @@ -0,0 +1,11 @@ +/var/log/postgresql/postgresql.csv { + size 50M + rotate 9 + compress + delaycompress + notifempty + missingok + postrotate + sudo -u postgres /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data logrotate + endscript +} diff --git a/ansible-nix/files/logrotate_config/logrotate-postgres.conf b/ansible-nix/files/logrotate_config/logrotate-postgres.conf new file mode 100644 index 000000000..c802320c4 --- /dev/null +++ b/ansible-nix/files/logrotate_config/logrotate-postgres.conf @@ -0,0 +1,9 @@ +/var/log/postgresql/postgresql.log { + size 50M + rotate 3 + copytruncate + delaycompress + compress + notifempty + missingok +} diff --git a/ansible-nix/files/logrotate_config/logrotate-walg.conf b/ansible-nix/files/logrotate_config/logrotate-walg.conf new file mode 100644 index 000000000..49eeb59eb --- /dev/null +++ b/ansible-nix/files/logrotate_config/logrotate-walg.conf @@ -0,0 +1,9 @@ +/var/log/wal-g/*.log { + size 50M + rotate 3 + copytruncate + delaycompress + compress + notifempty + missingok +} diff --git a/ansible-nix/files/manifest.json b/ansible-nix/files/manifest.json new file mode 100644 index 000000000..3a20e76a0 --- /dev/null +++ b/ansible-nix/files/manifest.json @@ -0,0 +1 @@ +{{ vars | to_json }} diff --git a/ansible-nix/files/nginx.service.j2 b/ansible-nix/files/nginx.service.j2 new file mode 100644 index 000000000..872e3346a --- /dev/null +++ b/ansible-nix/files/nginx.service.j2 @@ -0,0 +1,22 @@ +[Unit] +Description=nginx server +After=postgrest.service gotrue.service adminapi.service +Wants=postgrest.service gotrue.service adminapi.service + +[Service] +Type=forking +ExecStart=/usr/local/nginx/sbin/nginx -c /etc/nginx/nginx.conf +ExecReload=/usr/local/nginx/sbin/nginx -s reload -c /etc/nginx/nginx.conf +ExecStop=/usr/local/nginx/sbin/nginx -s quit +User=nginx +Slice=services.slice +Restart=always +RestartSec=3 +LimitNOFILE=100000 + +# Via systemd we grant the process a set of privileges to bind to 80/443 +# See http://archive.vn/36zJU +AmbientCapabilities=CAP_NET_BIND_SERVICE + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/pg_egress_collect.service.j2 b/ansible-nix/files/pg_egress_collect.service.j2 new file mode 100644 index 000000000..7ac04f47d --- /dev/null +++ b/ansible-nix/files/pg_egress_collect.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Postgres Egress Collector + +[Service] +Type=simple +ExecStart=/bin/bash -c "tcpdump -s 128 -Q out -nn -tt -vv -p -l 'tcp and (port 5432 or port 6543)' | perl /root/pg_egress_collect.pl" +User=root +Slice=services.slice +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/pgbouncer_config/pgbouncer.ini.j2 b/ansible-nix/files/pgbouncer_config/pgbouncer.ini.j2 new file mode 100644 index 000000000..e4518c007 --- /dev/null +++ b/ansible-nix/files/pgbouncer_config/pgbouncer.ini.j2 @@ -0,0 +1,364 @@ +;;; +;;; PgBouncer configuration file +;;; + +;; database name = connect string +;; +;; connect string params: +;; dbname= host= port= user= password= auth_user= +;; client_encoding= datestyle= timezone= +;; pool_size= reserve_pool= max_db_connections= +;; pool_mode= connect_query= application_name= +[databases] +* = host=localhost auth_user=pgbouncer + +;; foodb over Unix socket +;foodb = + +;; redirect bardb to bazdb on localhost +;bardb = host=localhost dbname=bazdb + +;; access to dest database will go with single user +;forcedb = host=localhost port=300 user=baz password=foo client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1' + +;; use custom pool sizes +;nondefaultdb = pool_size=50 reserve_pool=10 + +;; use auth_user with auth_query if user not present in auth_file +;; auth_user must exist in auth_file +; foodb = auth_user=bar + +;; fallback connect string +;* = host=testserver + +;; User-specific configuration +[users] + +;user1 = pool_mode=transaction max_user_connections=10 + +;; Configuration section +[pgbouncer] + +;;; +;;; Administrative settings +;;; + +;logfile = /var/log/pgbouncer.log +pidfile = /var/run/pgbouncer/pgbouncer.pid + +;;; +;;; Where to wait for clients +;;; + +;; IP address or * which means all IPs +listen_addr = * +listen_port = 6543 + +;; Unix socket is also used for -R. +;; On Debian it should be /var/run/postgresql +unix_socket_dir = /tmp +;unix_socket_mode = 0777 +;unix_socket_group = + +;;; +;;; TLS settings for accepting clients +;;; + +;; disable, allow, require, verify-ca, verify-full +;client_tls_sslmode = disable + +;; Path to file that contains trusted CA certs +;client_tls_ca_file = + +;; Private key and cert to present to clients. +;; Required for accepting TLS connections from clients. +;client_tls_key_file = +;client_tls_cert_file = + +;; fast, normal, secure, legacy, +;client_tls_ciphers = fast + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;client_tls_protocols = secure + +;; none, auto, legacy +;client_tls_dheparams = auto + +;; none, auto, +;client_tls_ecdhcurve = auto + +;;; +;;; TLS settings for connecting to backend databases +;;; + +;; disable, allow, require, verify-ca, verify-full +;server_tls_sslmode = disable + +;; Path to that contains trusted CA certs +;server_tls_ca_file = + +;; Private key and cert to present to backend. +;; Needed only if backend server require client cert. +;server_tls_key_file = +;server_tls_cert_file = + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;server_tls_protocols = secure + +;; fast, normal, secure, legacy, +;server_tls_ciphers = fast + +;;; +;;; Authentication settings +;;; + +;; any, trust, plain, md5, cert, hba, pam +auth_type = scram-sha-256 +auth_file = /etc/pgbouncer/userlist.txt + +;; Path to HBA-style auth config +;auth_hba_file = + +;; Query to use to fetch password from database. Result +;; must have 2 columns - username and password hash. +auth_query = SELECT * FROM pgbouncer.get_auth($1) + +;;; +;;; Users allowed into database 'pgbouncer' +;;; + +;; comma-separated list of users who are allowed to change settings +admin_users = pgbouncer + +;; comma-separated list of users who are just allowed to use SHOW command +stats_users = pgbouncer + +;;; +;;; Pooler personality questions +;;; + +;; When server connection is released back to pool: +;; session - after client disconnects (default) +;; transaction - after transaction finishes +;; statement - after statement finishes +pool_mode = transaction + +;; Query for cleaning connection immediately after releasing from +;; client. No need to put ROLLBACK here, pgbouncer does not reuse +;; connections where transaction is left open. +;server_reset_query = DISCARD ALL + +;; Whether server_reset_query should run in all pooling modes. If it +;; is off, server_reset_query is used only for session-pooling. +;server_reset_query_always = 0 + +;; Comma-separated list of parameters to ignore when given in startup +;; packet. Newer JDBC versions require the extra_float_digits here. +ignore_startup_parameters = extra_float_digits + +;; When taking idle server into use, this query is run first. +;server_check_query = select 1 + +;; If server was used more recently that this many seconds ago, +; skip the check query. Value 0 may or may not run in immediately. +;server_check_delay = 30 + +;; Close servers in session pooling mode after a RECONNECT, RELOAD, +;; etc. when they are idle instead of at the end of the session. +;server_fast_close = 0 + +;; Use as application_name on server. +;application_name_add_host = 0 + +;; Period for updating aggregated stats. +;stats_period = 60 + +;;; +;;; Connection limits +;;; + +;; Total number of clients that can connect +;max_client_conn = 100 + +;; Default pool size. 20 is good number when transaction pooling +;; is in use, in session pooling it needs to be the number of +;; max clients you want to handle at any moment +default_pool_size = 15 + +;; Minimum number of server connections to keep in pool. +;min_pool_size = 0 + +; how many additional connection to allow in case of trouble +;reserve_pool_size = 0 + +;; If a clients needs to wait more than this many seconds, use reserve +;; pool. +;reserve_pool_timeout = 5 + +;; Maximum number of server connections for a database +;max_db_connections = 0 + +;; Maximum number of server connections for a user +;max_user_connections = 0 + +;; If off, then server connections are reused in LIFO manner +;server_round_robin = 0 + +;;; +;;; Logging +;;; + +;; Syslog settings +;syslog = 0 +;syslog_facility = daemon +;syslog_ident = pgbouncer + +;; log if client connects or server connection is made +;log_connections = 1 + +;; log if and why connection was closed +;log_disconnections = 1 + +;; log error messages pooler sends to clients +;log_pooler_errors = 1 + +;; write aggregated stats into log +;log_stats = 1 + +;; Logging verbosity. Same as -v switch on command line. +;verbose = 0 + +;;; +;;; Timeouts +;;; + +;; Close server connection if its been connected longer. +;server_lifetime = 3600 + +;; Close server connection if its not been used in this time. Allows +;; to clean unnecessary connections from pool after peak. +;server_idle_timeout = 600 + +;; Cancel connection attempt if server does not answer takes longer. +;server_connect_timeout = 15 + +;; If server login failed (server_connect_timeout or auth failure) +;; then wait this many second. +;server_login_retry = 15 + +;; Dangerous. Server connection is closed if query does not return in +;; this time. Should be used to survive network problems, _not_ as +;; statement_timeout. (default: 0) +;query_timeout = 0 + +;; Dangerous. Client connection is closed if the query is not +;; assigned to a server in this time. Should be used to limit the +;; number of queued queries in case of a database or network +;; failure. (default: 120) +;query_wait_timeout = 120 + +;; Dangerous. Client connection is closed if no activity in this +;; time. Should be used to survive network problems. (default: 0) +;client_idle_timeout = 0 + +;; Disconnect clients who have not managed to log in after connecting +;; in this many seconds. +;client_login_timeout = 60 + +;; Clean automatically created database entries (via "*") if they stay +;; unused in this many seconds. +; autodb_idle_timeout = 3600 + +;; Close connections which are in "IDLE in transaction" state longer +;; than this many seconds. +;idle_transaction_timeout = 0 + +;; How long SUSPEND/-R waits for buffer flush before closing +;; connection. +;suspend_timeout = 10 + +;;; +;;; Low-level tuning options +;;; + +;; buffer for streaming packets +;pkt_buf = 4096 + +;; man 2 listen +;listen_backlog = 128 + +;; Max number pkt_buf to process in one event loop. +;sbuf_loopcnt = 5 + +;; Maximum PostgreSQL protocol packet size. +;max_packet_size = 2147483647 + +;; Set SO_REUSEPORT socket option +;so_reuseport = 0 + +;; networking options, for info: man 7 tcp + +;; Linux: Notify program about new connection only if there is also +;; data received. (Seconds to wait.) On Linux the default is 45, on +;; other OS'es 0. +;tcp_defer_accept = 0 + +;; In-kernel buffer size (Linux default: 4096) +;tcp_socket_buffer = 0 + +;; whether tcp keepalive should be turned on (0/1) +;tcp_keepalive = 1 + +;; The following options are Linux-specific. They also require +;; tcp_keepalive=1. + +;; Count of keepalive packets +;tcp_keepcnt = 0 + +;; How long the connection can be idle before sending keepalive +;; packets +;tcp_keepidle = 0 + +;; The time between individual keepalive probes +;tcp_keepintvl = 0 + +;; How long may transmitted data remain unacknowledged before TCP +;; connection is closed (in milliseconds) +;tcp_user_timeout = 0 + +;; DNS lookup caching time +;dns_max_ttl = 15 + +;; DNS zone SOA lookup period +;dns_zone_check_period = 0 + +;; DNS negative result caching time +;dns_nxdomain_ttl = 15 + +;; Custom resolv.conf file, to set custom DNS servers or other options +;; (default: empty = use OS settings) +;resolv_conf = /etc/pgbouncer/resolv.conf + +;;; +;;; Random stuff +;;; + +;; Hackish security feature. Helps against SQL injection: when PQexec +;; is disabled, multi-statement cannot be made. +;disable_pqexec = 0 + +;; Config file to use for next RELOAD/SIGHUP +;; By default contains config file from command line. +;conffile + +;; Windows service name to register as. job_name is alias for +;; service_name, used by some Skytools scripts. +;service_name = pgbouncer +;job_name = pgbouncer + +;; Read additional config from other file +;%include /etc/pgbouncer/pgbouncer-other.ini + +%include /etc/pgbouncer-custom/generated-optimizations.ini +%include /etc/pgbouncer-custom/custom-overrides.ini +%include /etc/pgbouncer-custom/ssl-config.ini diff --git a/ansible-nix/files/pgbouncer_config/pgbouncer.service.j2 b/ansible-nix/files/pgbouncer_config/pgbouncer.service.j2 new file mode 100644 index 000000000..1ec5ea378 --- /dev/null +++ b/ansible-nix/files/pgbouncer_config/pgbouncer.service.j2 @@ -0,0 +1,20 @@ +[Unit] +Description=connection pooler for PostgreSQL +Documentation=man:pgbouncer(1) +Documentation=https://www.pgbouncer.org/ +After=network.target +{% if supabase_internal is defined %} +Requires=database-optimizations.service +After=database-optimizations.service +{% endif %} + +[Service] +Type=notify +User=pgbouncer +ExecStart=/usr/local/bin/pgbouncer /etc/pgbouncer/pgbouncer.ini +ExecReload=/bin/kill -HUP $MAINPID +KillSignal=SIGINT +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/pgbouncer_config/pgbouncer_auth_schema.sql b/ansible-nix/files/pgbouncer_config/pgbouncer_auth_schema.sql new file mode 100644 index 000000000..c10ce44fd --- /dev/null +++ b/ansible-nix/files/pgbouncer_config/pgbouncer_auth_schema.sql @@ -0,0 +1,20 @@ +CREATE USER pgbouncer; + +REVOKE ALL PRIVILEGES ON SCHEMA public FROM pgbouncer; + +CREATE SCHEMA pgbouncer AUTHORIZATION pgbouncer; + +CREATE OR REPLACE FUNCTION pgbouncer.get_auth(p_usename TEXT) +RETURNS TABLE(username TEXT, password TEXT) AS +$$ +BEGIN + RAISE WARNING 'PgBouncer auth request: %', p_usename; + + RETURN QUERY + SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow + WHERE usename = p_usename; +END; +$$ LANGUAGE plpgsql SECURITY DEFINER; + +REVOKE ALL ON FUNCTION pgbouncer.get_auth(p_usename TEXT) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(p_usename TEXT) TO pgbouncer; diff --git a/ansible-nix/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 b/ansible-nix/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 new file mode 100644 index 000000000..d5d2cd49d --- /dev/null +++ b/ansible-nix/files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 @@ -0,0 +1,2 @@ +# Directory for PostgreSQL sockets, lockfiles and stats tempfiles +d /run/pgbouncer 2775 pgbouncer postgres - - \ No newline at end of file diff --git a/ansible-nix/files/pgsodium_getkey_readonly.sh.j2 b/ansible-nix/files/pgsodium_getkey_readonly.sh.j2 new file mode 100644 index 000000000..e0a72733f --- /dev/null +++ b/ansible-nix/files/pgsodium_getkey_readonly.sh.j2 @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +KEY_FILE=/etc/postgresql-custom/pgsodium_root.key + +# On the hosted platform, the root key is generated and managed for each project +# If for some reason the key is missing, we want to fail loudly, +# rather than generating a new one. +if [[ ! -f "${KEY_FILE}" ]]; then + echo "Key file ${KEY_FILE} does not exist." >&2 + exit 1 +fi +cat $KEY_FILE diff --git a/ansible-nix/files/pgsodium_getkey_urandom.sh.j2 b/ansible-nix/files/pgsodium_getkey_urandom.sh.j2 new file mode 100755 index 000000000..e8039d037 --- /dev/null +++ b/ansible-nix/files/pgsodium_getkey_urandom.sh.j2 @@ -0,0 +1,10 @@ +#!/bin/bash + +set -euo pipefail + +KEY_FILE=/etc/postgresql-custom/pgsodium_root.key + +if [[ ! -f "${KEY_FILE}" ]]; then + head -c 32 /dev/urandom | od -A n -t x1 | tr -d ' \n' > "${KEY_FILE}" +fi +cat $KEY_FILE diff --git a/ansible-nix/files/postgres_exporter.service.j2 b/ansible-nix/files/postgres_exporter.service.j2 new file mode 100644 index 000000000..0066a76b2 --- /dev/null +++ b/ansible-nix/files/postgres_exporter.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Postgres Exporter + +[Service] +Type=simple +ExecStart=/opt/postgres_exporter/postgres_exporter --disable-settings-metrics --extend.query-path="/opt/postgres_exporter/queries.yml" --disable-default-metrics --no-collector.locks --no-collector.replication --no-collector.replication_slot --no-collector.stat_bgwriter --no-collector.stat_database --no-collector.stat_user_tables --no-collector.statio_user_tables --no-collector.wal +User=postgres +Group=postgres +Restart=always +RestartSec=3 +Environment="DATA_SOURCE_NAME=host=localhost dbname=postgres sslmode=disable user=supabase_admin pg_stat_statements.track=none application_name=postgres_exporter" + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/postgresql_config/custom_read_replica.conf.j2 b/ansible-nix/files/postgresql_config/custom_read_replica.conf.j2 new file mode 100644 index 000000000..7d52f92a7 --- /dev/null +++ b/ansible-nix/files/postgresql_config/custom_read_replica.conf.j2 @@ -0,0 +1,5 @@ +# hot_standby = on +# restore_command = '/usr/bin/admin-mgr wal-fetch %f %p >> /var/log/wal-g/wal-fetch.log 2>&1' +# recovery_target_timeline = 'latest' + +# primary_conninfo = 'host=localhost port=6543 user=replication' diff --git a/ansible-nix/files/postgresql_config/custom_walg.conf.j2 b/ansible-nix/files/postgresql_config/custom_walg.conf.j2 new file mode 100644 index 000000000..7ef7256d8 --- /dev/null +++ b/ansible-nix/files/postgresql_config/custom_walg.conf.j2 @@ -0,0 +1,21 @@ +# - Archiving - + +#archive_mode = on +#archive_command = '/usr/bin/admin-mgr wal-push %p >> /var/log/wal-g/wal-push.log 2>&1' +#archive_timeout = 120 + + +# - Archive Recovery - + +#restore_command = '/usr/bin/admin-mgr wal-fetch %f %p >> /var/log/wal-g/wal-fetch.log 2>&1' + +# - Recovery Target - + +#recovery_target_lsn = '' +#recovery_target_time = '' +#recovery_target_action = 'promote' +#recovery_target_timeline = 'current' +#recovery_target_inclusive = off + +# - Hot Standby - +hot_standby = off diff --git a/ansible-nix/files/postgresql_config/pg_hba.conf.j2 b/ansible-nix/files/postgresql_config/pg_hba.conf.j2 new file mode 100755 index 000000000..9cafd4146 --- /dev/null +++ b/ansible-nix/files/postgresql_config/pg_hba.conf.j2 @@ -0,0 +1,94 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# non-SSL TCP/IP socket. Similarly, "hostgssenc" uses a +# GSSAPI-encrypted TCP/IP socket, while "hostnogssenc" uses a +# non-GSSAPI socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", +# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". +# Note that "password" sends passwords in clear text; "md5" or +# "scram-sha-256" are preferred since they send encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the server receives a +# SIGHUP signal. If you edit the file on a running system, you have to +# SIGHUP the server for the changes to take effect, run "pg_ctl reload", +# or execute "SELECT pg_reload_conf()". +# +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + +# TYPE DATABASE USER ADDRESS METHOD + +# trust local connections +local all supabase_admin scram-sha-256 +local all all peer map=supabase_map +host all all 127.0.0.1/32 trust +host all all ::1/128 trust + +# IPv4 external connections +host all all 10.0.0.0/8 scram-sha-256 +host all all 172.16.0.0/12 scram-sha-256 +host all all 192.168.0.0/16 scram-sha-256 +host all all 0.0.0.0/0 scram-sha-256 + +# IPv6 external connections +host all all ::0/0 scram-sha-256 diff --git a/ansible-nix/files/postgresql_config/pg_ident.conf.j2 b/ansible-nix/files/postgresql_config/pg_ident.conf.j2 new file mode 100755 index 000000000..d8891f416 --- /dev/null +++ b/ansible-nix/files/postgresql_config/pg_ident.conf.j2 @@ -0,0 +1,50 @@ +# PostgreSQL User Name Maps +# ========================= +# +# Refer to the PostgreSQL documentation, chapter "Client +# Authentication" for a complete description. A short synopsis +# follows. +# +# This file controls PostgreSQL user name mapping. It maps external +# user names to their corresponding PostgreSQL user names. Records +# are of the form: +# +# MAPNAME SYSTEM-USERNAME PG-USERNAME +# +# (The uppercase quantities must be replaced by actual values.) +# +# MAPNAME is the (otherwise freely chosen) map name that was used in +# pg_hba.conf. SYSTEM-USERNAME is the detected user name of the +# client. PG-USERNAME is the requested PostgreSQL user name. The +# existence of a record specifies that SYSTEM-USERNAME may connect as +# PG-USERNAME. +# +# If SYSTEM-USERNAME starts with a slash (/), it will be treated as a +# regular expression. Optionally this can contain a capture (a +# parenthesized subexpression). The substring matching the capture +# will be substituted for \1 (backslash-one) if present in +# PG-USERNAME. +# +# Multiple maps may be specified in this file and used by pg_hba.conf. +# +# No map names are defined in the default configuration. If all +# system user names and PostgreSQL user names are the same, you don't +# need anything in this file. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can +# use "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- + +# MAPNAME SYSTEM-USERNAME PG-USERNAME +supabase_map postgres postgres +supabase_map root postgres +supabase_map ubuntu postgres + +# supabase-specific users +supabase_map gotrue supabase_auth_admin +supabase_map postgrest authenticator +supabase_map adminapi postgres diff --git a/ansible-nix/files/postgresql_config/postgresql-csvlog.conf b/ansible-nix/files/postgresql_config/postgresql-csvlog.conf new file mode 100644 index 000000000..b8d64da51 --- /dev/null +++ b/ansible-nix/files/postgresql_config/postgresql-csvlog.conf @@ -0,0 +1,33 @@ +# - Where to Log - + +log_destination = 'csvlog' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. + +# This is used when logging to stderr: +logging_collector = on # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +log_directory = '/var/log/postgresql' # directory where log files are written, + # can be absolute or relative to PGDATA +log_filename = 'postgresql.log' # log file name pattern, + # can include strftime() escapes +log_file_mode = 0640 # creation mode for log files, + # begin with 0 to use octal notation +log_rotation_age = 0 # Automatic rotation of logfiles will + # happen after that time. 0 disables. +log_rotation_size = 0 # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. diff --git a/ansible-nix/files/postgresql_config/postgresql-stdout-log.conf b/ansible-nix/files/postgresql_config/postgresql-stdout-log.conf new file mode 100644 index 000000000..6ae4ff456 --- /dev/null +++ b/ansible-nix/files/postgresql_config/postgresql-stdout-log.conf @@ -0,0 +1,4 @@ +logging_collector = off # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) diff --git a/ansible-nix/files/postgresql_config/postgresql.conf.j2 b/ansible-nix/files/postgresql_config/postgresql.conf.j2 new file mode 100644 index 000000000..fdca6643d --- /dev/null +++ b/ansible-nix/files/postgresql_config/postgresql.conf.j2 @@ -0,0 +1,776 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, run "pg_ctl reload", or execute +# "SELECT pg_reload_conf()". Some parameters, which are marked below, +# require a server shutdown and restart to take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: B = bytes Time units: us = microseconds +# kB = kilobytes ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +data_directory = '/var/lib/postgresql/data' # use data in another directory + # (change requires restart) +hba_file = '/etc/postgresql/pg_hba.conf' # host-based authentication file + # (change requires restart) +ident_file = '/etc/postgresql/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +#external_pid_file = '' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +listen_addresses = '*' # what IP address(es) to listen on; + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +#port = 5432 # (change requires restart) +#max_connections = 100 # (change requires restart) +#superuser_reserved_connections = 3 # (change requires restart) +#unix_socket_directories = '/tmp' # comma-separated list of directories + # (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation + # (change requires restart) +#bonjour = off # advertise server via Bonjour + # (change requires restart) +#bonjour_name = '' # defaults to the computer name + # (change requires restart) + +# - TCP settings - +# see "man tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default +#tcp_user_timeout = 0 # TCP_USER_TIMEOUT, in milliseconds; + # 0 selects the system default + +#client_connection_check_interval = 0 # time between checks for client + # disconnection while running queries; + # 0 for never + +# - Authentication - + +authentication_timeout = 1min # 1s-600s +password_encryption = scram-sha-256 # scram-sha-256 or md5 +db_user_namespace = off + +# GSSAPI using Kerberos +#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab' +#krb_caseins_users = off + +# - SSL - + +ssl = off +ssl_ca_file = '' +ssl_cert_file = '' +ssl_crl_file = '' +ssl_crl_dir = '' +ssl_key_file = '' +ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +ssl_prefer_server_ciphers = on +ssl_ecdh_curve = 'prime256v1' +ssl_min_protocol_version = 'TLSv1.2' +ssl_max_protocol_version = '' +ssl_dh_params_file = '' +ssl_passphrase_command = '' +ssl_passphrase_command_supports_reload = off + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +shared_buffers = 128MB # min 128kB + # (change requires restart) +#huge_pages = try # on, off, or try + # (change requires restart) +#huge_page_size = 0 # zero for system default + # (change requires restart) +#temp_buffers = 8MB # min 800kB +#max_prepared_transactions = 0 # zero disables the feature + # (change requires restart) +# Caution: it is not advisable to set max_prepared_transactions nonzero unless +# you actively intend to use prepared transactions. +#work_mem = 4MB # min 64kB +#hash_mem_multiplier = 1.0 # 1-1000.0 multiplier on hash table work_mem +#maintenance_work_mem = 64MB # min 1MB +#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +#logical_decoding_work_mem = 64MB # min 64kB +#max_stack_depth = 2MB # min 100kB +#shared_memory_type = mmap # the default is the first option + # supported by the operating system: + # mmap + # sysv + # windows + # (change requires restart) +#dynamic_shared_memory_type = posix # the default is the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # (change requires restart) +#min_dynamic_shared_memory = 0MB # (change requires restart) + +# - Disk - + +#temp_file_limit = -1 # limits per-process temp file space + # in kilobytes, or -1 for no limit + +# - Kernel Resources - + +#max_files_per_process = 1000 # min 64 + # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds (0 disables) +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 2 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round +#bgwriter_flush_after = 0 # measured in pages, 0 disables + +# - Asynchronous Behavior - + +#backend_flush_after = 0 # measured in pages, 0 disables +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#maintenance_io_concurrency = 10 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 # (change requires restart) +#max_parallel_workers_per_gather = 2 # taken from max_parallel_workers +#max_parallel_maintenance_workers = 2 # taken from max_parallel_workers +#max_parallel_workers = 8 # maximum number of max_worker_processes that + # can be used in parallel operations +#parallel_leader_participation = on +#old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate + # (change requires restart) + + +#------------------------------------------------------------------------------ +# WRITE-AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +wal_level = logical # minimal, replica, or logical + # (change requires restart) +#fsync = on # flush data to disk for crash safety + # (turning this off can cause + # unrecoverable data corruption) +#synchronous_commit = on # synchronization level; + # off, local, remote_write, remote_apply, or on +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux and FreeBSD) + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +#wal_log_hints = off # also do full page writes of non-critical updates + # (change requires restart) +#wal_compression = off # enable compression of full-page writes +#wal_init_zero = on # zero-fill new WAL files +#wal_recycle = on # recycle WAL files +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers + # (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds +#wal_writer_flush_after = 1MB # measured in pages, 0 disables +#wal_skip_threshold = 2MB + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_timeout = 5min # range 30s-1d +checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 +checkpoint_flush_after = 256kB # measured in pages, 0 disables +#checkpoint_warning = 30s # 0 disables +#max_wal_size = 1GB +#min_wal_size = 80MB + +# - Archiving - + +#archive_mode = off # enables archiving; off, on, or always + # (change requires restart) +#archive_command = '' # command to use to archive a logfile segment + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this + # number of seconds; 0 disables + +# - Archive Recovery - + +# These are only used in recovery mode. + +#restore_command = '' # command to use to restore an archived logfile segment + # placeholders: %p = path of file to restore + # %f = file name only + # e.g. 'cp /mnt/server/archivedir/%f %p' +#archive_cleanup_command = '' # command to execute at every restartpoint +#recovery_end_command = '' # command to execute at completion of recovery + +# - Recovery Target - + +# Set these only when performing a targeted recovery. + +#recovery_target = '' # 'immediate' to end recovery as soon as a + # consistent state is reached + # (change requires restart) +#recovery_target_name = '' # the named restore point to which recovery will proceed + # (change requires restart) +#recovery_target_time = '' # the time stamp up to which recovery will proceed + # (change requires restart) +#recovery_target_xid = '' # the transaction ID up to which recovery will proceed + # (change requires restart) +#recovery_target_lsn = '' # the WAL LSN up to which recovery will proceed + # (change requires restart) +#recovery_target_inclusive = on # Specifies whether to stop: + # just after the specified recovery target (on) + # just before the recovery target (off) + # (change requires restart) +#recovery_target_timeline = 'latest' # 'current', 'latest', or timeline ID + # (change requires restart) +#recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' + # (change requires restart) + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Servers - + +# Set these on the primary and on any standby that will send replication data. + +max_wal_senders = 10 # max number of walsender processes + # (change requires restart) +max_replication_slots = 5 # max number of replication slots + # (change requires restart) +#wal_keep_size = 0 # in megabytes; 0 disables +max_slot_wal_keep_size = 1024 # in megabytes; -1 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables +#track_commit_timestamp = off # collect timestamp of transaction commit + # (change requires restart) + +# - Primary Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep + # method to choose sync standbys, number of sync standbys, + # and comma-separated list of application_name + # from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a primary server. + +#primary_conninfo = '' # connection string to sending server +#primary_slot_name = '' # replication slot on sending server +#promote_trigger_file = '' # file name whose presence ends recovery +#hot_standby = on # "off" disallows queries during recovery + # (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries + # when reading WAL from archive; + # -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries + # when reading streaming WAL; + # -1 allows indefinite delay +#wal_receiver_create_temp_slot = off # create temp slot if primary_slot_name + # is not set +#wal_receiver_status_interval = 10s # send replies at least this often + # 0 disables +#hot_standby_feedback = off # send info from standby to prevent + # query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for + # communication from primary + # in milliseconds; 0 disables +#wal_retrieve_retry_interval = 5s # time to wait before retrying to + # retrieve WAL after a failed attempt +#recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery + +# - Subscribers - + +# These settings are ignored on a publisher. + +#max_logical_replication_workers = 4 # taken from max_worker_processes + # (change requires restart) +#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_async_append = on +#enable_bitmapscan = on +#enable_gathermerge = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_incremental_sort = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_resultcache = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_parallel_append = on +#enable_parallel_hash = on +#enable_partition_pruning = on +#enable_partitionwise_join = off +#enable_partitionwise_aggregate = off +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#parallel_setup_cost = 1000.0 # same scale as above +#parallel_tuple_cost = 0.1 # same scale as above +#min_parallel_table_scan_size = 8MB +#min_parallel_index_scan_size = 512kB +effective_cache_size = 128MB + +#jit_above_cost = 100000 # perform JIT compilation if available + # and query more expensive than this; + # -1 disables +#jit_inline_above_cost = 500000 # inline small functions if query is + # more expensive than this; -1 disables +#jit_optimize_above_cost = 500000 # use expensive JIT optimizations if + # query is more expensive than this; + # -1 disables + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#jit = on # allow JIT compilation +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOIN clauses +#plan_cache_mode = auto # auto, force_generic_plan or + # force_custom_plan + + +#------------------------------------------------------------------------------ +# REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +include = '/etc/postgresql/logging.conf' + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' +#syslog_sequence_numbers = on +#syslog_split_messages = on + +# This is only relevant when logging to eventlog (Windows): +# (change requires restart) +#event_source = 'PostgreSQL' + +# - When to Log - + +#log_min_messages = warning # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_min_error_statement = error # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds + +#log_min_duration_sample = -1 # -1 is disabled, 0 logs a sample of statements + # and their durations, > 0 logs only a sample of + # statements running at least this number + # of milliseconds; + # sample fraction is determined by log_statement_sample_rate + +#log_statement_sample_rate = 1.0 # fraction of logged statements exceeding + # log_min_duration_sample to be logged; + # 1.0 logs all such statements, 0.0 never logs + + +#log_transaction_sample_rate = 0.0 # fraction of transactions whose statements + # are logged regardless of their duration; 1.0 logs all + # statements from all transactions, 0.0 never logs + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_autovacuum_min_duration = -1 # log autovacuum activity; + # -1 disables, 0 logs all actions and + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. +#log_checkpoints = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +log_line_prefix = '%h %m [%p] %q%u@%d ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %b = backend type + # %p = process ID + # %P = process ID of parallel group leader + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %n = timestamp with milliseconds (as a Unix epoch) + # %Q = query ID (0 if none or not computed) + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_recovery_conflict_waits = off # log standby recovery conflict waits + # >= deadlock_timeout +#log_parameter_max_length = -1 # when logging statements, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_parameter_max_length_on_error = 0 # when logging an error, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +log_statement = 'none' # none, ddl, mod, all +#log_replication_commands = off +#log_temp_files = -1 # log temporary files equal or larger + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +log_timezone = 'UTC' + +#------------------------------------------------------------------------------ +# PROCESS TITLE +#------------------------------------------------------------------------------ + +cluster_name = 'main' # added to process titles if nonempty + # (change requires restart) +#update_process_title = on + + +#------------------------------------------------------------------------------ +# STATISTICS +#------------------------------------------------------------------------------ + +# - Query and Index Statistics Collector - + +#track_activities = on +#track_activity_query_size = 1024 # (change requires restart) +#track_counts = on +#track_io_timing = off +#track_wal_io_timing = off +#track_functions = none # none, pl, all +#stats_temp_directory = 'pg_stat_tmp' + + +# - Monitoring - + +#compute_query_id = auto +#log_statement_stats = off +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' + # requires track_counts to also be on. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses + # (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before + # vacuum +#autovacuum_vacuum_insert_threshold = 1000 # min number of row inserts + # before vacuum; -1 disables insert + # vacuums +#autovacuum_analyze_threshold = 50 # min number of row updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_vacuum_insert_scale_factor = 0.2 # fraction of inserts over table + # size before insert vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum + # (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age + # before forced vacuum + # (change requires restart) +#autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovacuum, -1 means use + # vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#client_min_messages = notice # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error +#search_path = '"$user", public' # schema names +row_security = on +#default_table_access_method = 'heap' +#default_tablespace = '' # a tablespace name, '' uses the default +#default_toast_compression = 'pglz' # 'pglz' or 'lz4' +#temp_tablespaces = '' # a list of tablespace names, '' uses + # only default tablespace +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled +#idle_session_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_table_age = 150000000 +#vacuum_freeze_min_age = 50000000 +#vacuum_failsafe_age = 1600000000 +#vacuum_multixact_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_failsafe_age = 1600000000 +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' +#gin_pending_list_limit = 4MB + +# - Locale and Formatting - + +#datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +timezone = 'UTC' +#timezone_abbreviations = 'Default' # Select the set of available time zone + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. +extra_float_digits = 0 # min -15, max 3; any value >0 actually + # selects precise output mode +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb, but they can be changed. +lc_messages = 'en_US.UTF-8' # locale for system error message + # strings +lc_monetary = 'en_US.UTF-8' # locale for monetary formatting +lc_numeric = 'en_US.UTF-8' # locale for number formatting +lc_time = 'en_US.UTF-8' # locale for time formatting + +# default configuration for text search +default_text_search_config = 'pg_catalog.english' + +# - Shared Library Preloading - + +#local_preload_libraries = '' +#session_preload_libraries = '' + +shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle' # (change requires restart) +jit_provider = 'llvmjit' # JIT library to use + +# - Other Defaults - + +#dynamic_library_path = '$libdir' +#gin_fuzzy_search_limit = 0 + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_relation = -2 # negative values mean + # (max_pred_locks_per_transaction + # / -max_pred_locks_per_relation) - 1 +#max_pred_locks_per_page = 2 # min 0 + + +#------------------------------------------------------------------------------ +# VERSION AND PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#escape_string_warning = on +#lo_compat_privileges = off +#quote_all_identifiers = off +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? +#data_sync_retry = off # retry or panic on failure to fsync + # data? + # (change requires restart) +#recovery_init_sync_method = fsync # fsync, syncfs (Linux 5.8+) + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. Note that these are directives, not variable +# assignments, so they can usefully be given more than once. + +#include_dir = '...' # include files ending in '.conf' from + # a directory, e.g., 'conf.d' +#include_if_exists = '...' # include file only if it exists +#include = '...' # include file + +# Automatically generated optimizations +#include = '/etc/postgresql-custom/generated-optimizations.conf' +# User-supplied custom parameters, override any automatically generated ones +#include = '/etc/postgresql-custom/custom-overrides.conf' + +# WAL-G specific configurations +#include = '/etc/postgresql-custom/wal-g.conf' + +# read replica specific configurations +include = '/etc/postgresql-custom/read-replica.conf' + +# supautils specific configurations +#include = '/etc/postgresql-custom/supautils.conf' + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here diff --git a/ansible-nix/files/postgresql_config/postgresql.service.j2 b/ansible-nix/files/postgresql_config/postgresql.service.j2 new file mode 100644 index 000000000..ac44e0a45 --- /dev/null +++ b/ansible-nix/files/postgresql_config/postgresql.service.j2 @@ -0,0 +1,23 @@ +[Unit] +Description=PostgreSQL database server +Documentation=man:postgres(1) +{% if supabase_internal is defined %} +Requires=database-optimizations.service +After=database-optimizations.service +{% endif %} + +[Service] +Type=notify +User=postgres +ExecStart=/usr/lib/postgresql/bin/postgres -D /etc/postgresql +ExecReload=/bin/kill -HUP $MAINPID +KillMode=mixed +KillSignal=SIGINT +TimeoutStopSec=90 +TimeoutStartSec=86400 +Restart=always +RestartSec=5 +OOMScoreAdjust=-1000 + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/postgresql_config/supautils.conf.j2 b/ansible-nix/files/postgresql_config/supautils.conf.j2 new file mode 100644 index 000000000..bfbd590f9 --- /dev/null +++ b/ansible-nix/files/postgresql_config/supautils.conf.j2 @@ -0,0 +1,12 @@ +supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}' +supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.broadcasts","realtime.channels","realtime.presences","storage.buckets","storage.migrations","storage.objects"]}' +# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 +# omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility +# omitted because deprecated: intagg, xml2 +supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' +supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' +supautils.privileged_extensions_superuser = 'supabase_admin' +supautils.privileged_role = 'postgres' +supautils.privileged_role_allowed_configs = 'log_min_messages, pgaudit.log, pgaudit.log_catalog, pgaudit.log_client, pgaudit.log_level, pgaudit.log_relation, pgaudit.log_rows, pgaudit.log_statement, pgaudit.log_statement_once, pgaudit.role, pgrst.*, session_replication_role, track_io_timing' +supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, authenticator' +supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*' diff --git a/ansible-nix/files/postgresql_config/tmpfiles.postgresql.conf b/ansible-nix/files/postgresql_config/tmpfiles.postgresql.conf new file mode 100644 index 000000000..b5ea54948 --- /dev/null +++ b/ansible-nix/files/postgresql_config/tmpfiles.postgresql.conf @@ -0,0 +1,5 @@ +# unchanged from upstream package +d /run/postgresql 2775 postgres postgres - - +# Log directory - ensure that our logging setup gets preserved +# and that vector can keep writing to a file here as well +d /var/log/postgresql 1775 postgres postgres - - diff --git a/ansible-nix/files/postgresql_extension_custom_scripts/before-create.sql b/ansible-nix/files/postgresql_extension_custom_scripts/before-create.sql new file mode 100644 index 000000000..f2f238668 --- /dev/null +++ b/ansible-nix/files/postgresql_extension_custom_scripts/before-create.sql @@ -0,0 +1,84 @@ +-- If the following are true: +-- * the extension to be created is a TLE +-- * the extension is created with `cascade` +-- +-- then we pre-`create` all nested extension dependencies which are part of +-- `supautils.privileged_extensions`. This is because supautils can't intercept +-- the extension creation for dependencies - it can only intercept the `create +-- extension` statement. +do $$ +declare + _extname text := @extname@; + _extschema text := @extschema@; + _extversion text := @extversion@; + _extcascade bool := @extcascade@; + _r record; +begin + if not _extcascade then + return; + end if; + + if not exists (select from pg_extension where extname = 'pg_tle') then + return; + end if; + + if not exists (select from pgtle.available_extensions() where name = _extname) then + return; + end if; + + if _extversion is null then + select default_version + from pgtle.available_extensions() + where name = _extname + into _extversion; + end if; + + if _extschema is null then + select schema + from pgtle.available_extension_versions() + where name = _extname and version = _extversion + into _extschema; + end if; + + for _r in ( + with recursive available_extensions(name, default_version) as ( + select name, default_version + from pg_available_extensions + union + select name, default_version + from pgtle.available_extensions() + ) + , available_extension_versions(name, version, requires) as ( + select name, version, requires + from pg_available_extension_versions + union + select name, version, requires + from pgtle.available_extension_versions() + ) + , all_dependencies(name, dependency) as ( + select e.name, unnest(ev.requires) as dependency + from available_extensions e + join available_extension_versions ev on ev.name = e.name and ev.version = e.default_version + ) + , dependencies(name) AS ( + select unnest(requires) + from available_extension_versions + where name = _extname and version = _extversion + union + select all_dependencies.dependency + from all_dependencies + join dependencies d on d.name = all_dependencies.name + ) + select name + from dependencies + intersect + select name + from regexp_split_to_table(current_setting('supautils.privileged_extensions', true), '\s*,\s*') as t(name) + ) loop + if _extschema is null then + execute(format('create extension if not exists %I cascade', _r.name)); + else + execute(format('create extension if not exists %I schema %I cascade', _r.name, _extschema)); + end if; + end loop; +end $$; diff --git a/ansible-nix/files/postgresql_extension_custom_scripts/dblink/after-create.sql b/ansible-nix/files/postgresql_extension_custom_scripts/dblink/after-create.sql new file mode 100644 index 000000000..22261bc77 --- /dev/null +++ b/ansible-nix/files/postgresql_extension_custom_scripts/dblink/after-create.sql @@ -0,0 +1,14 @@ +do $$ +declare + r record; +begin + for r in (select oid, (aclexplode(proacl)).grantee from pg_proc where proname = 'dblink_connect_u') loop + continue when r.grantee = 'supabase_admin'::regrole; + execute( + format( + 'revoke all on function %s(%s) from %s;', r.oid::regproc, pg_get_function_identity_arguments(r.oid), r.grantee::regrole + ) + ); + end loop; +end +$$; diff --git a/ansible-nix/files/postgresql_extension_custom_scripts/pg_cron/after-create.sql b/ansible-nix/files/postgresql_extension_custom_scripts/pg_cron/after-create.sql new file mode 100644 index 000000000..6ac9d6b6e --- /dev/null +++ b/ansible-nix/files/postgresql_extension_custom_scripts/pg_cron/after-create.sql @@ -0,0 +1,13 @@ +grant usage on schema cron to postgres with grant option; +grant all on all functions in schema cron to postgres with grant option; + +alter default privileges for user supabase_admin in schema cron grant all + on sequences to postgres with grant option; +alter default privileges for user supabase_admin in schema cron grant all + on tables to postgres with grant option; +alter default privileges for user supabase_admin in schema cron grant all + on functions to postgres with grant option; + +grant all privileges on all tables in schema cron to postgres with grant option; +revoke all on table cron.job from postgres; +grant select on table cron.job to postgres with grant option; diff --git a/ansible-nix/files/postgresql_extension_custom_scripts/pg_tle/after-create.sql b/ansible-nix/files/postgresql_extension_custom_scripts/pg_tle/after-create.sql new file mode 100644 index 000000000..eb8aeff7e --- /dev/null +++ b/ansible-nix/files/postgresql_extension_custom_scripts/pg_tle/after-create.sql @@ -0,0 +1 @@ +grant pgtle_admin to postgres; diff --git a/ansible-nix/files/postgresql_extension_custom_scripts/pgsodium/after-create.sql b/ansible-nix/files/postgresql_extension_custom_scripts/pgsodium/after-create.sql new file mode 100644 index 000000000..907c67ebf --- /dev/null +++ b/ansible-nix/files/postgresql_extension_custom_scripts/pgsodium/after-create.sql @@ -0,0 +1,3 @@ +grant execute on function pgsodium.crypto_aead_det_decrypt(bytea, bytea, uuid, bytea) to service_role; +grant execute on function pgsodium.crypto_aead_det_encrypt(bytea, bytea, uuid, bytea) to service_role; +grant execute on function pgsodium.crypto_aead_det_keygen to service_role; diff --git a/ansible-nix/files/postgresql_extension_custom_scripts/postgis_tiger_geocoder/after-create.sql b/ansible-nix/files/postgresql_extension_custom_scripts/postgis_tiger_geocoder/after-create.sql new file mode 100644 index 000000000..f8ec163bb --- /dev/null +++ b/ansible-nix/files/postgresql_extension_custom_scripts/postgis_tiger_geocoder/after-create.sql @@ -0,0 +1,10 @@ +-- These schemas are created by extension to house all tiger related functions, owned by supabase_admin +grant usage on schema tiger, tiger_data to postgres with grant option; +-- Give postgres permission to all existing entities, also allows postgres to grant other roles +grant all on all tables in schema tiger, tiger_data to postgres with grant option; +grant all on all routines in schema tiger, tiger_data to postgres with grant option; +grant all on all sequences in schema tiger, tiger_data to postgres with grant option; +-- Update default privileges so that new entities are also accessible by postgres +alter default privileges in schema tiger, tiger_data grant all on tables to postgres with grant option; +alter default privileges in schema tiger, tiger_data grant all on routines to postgres with grant option; +alter default privileges in schema tiger, tiger_data grant all on sequences to postgres with grant option; diff --git a/ansible-nix/files/postgresql_extension_custom_scripts/postgres_fdw/after-create.sql b/ansible-nix/files/postgresql_extension_custom_scripts/postgres_fdw/after-create.sql new file mode 100644 index 000000000..1e83ee90e --- /dev/null +++ b/ansible-nix/files/postgresql_extension_custom_scripts/postgres_fdw/after-create.sql @@ -0,0 +1,21 @@ +do $$ +declare + is_super boolean; +begin + is_super = ( + select usesuper + from pg_user + where usename = 'postgres' + ); + + -- Need to be superuser to own FDWs, so we temporarily make postgres superuser. + if not is_super then + alter role postgres superuser; + end if; + + alter foreign data wrapper postgres_fdw owner to postgres; + + if not is_super then + alter role postgres nosuperuser; + end if; +end $$; diff --git a/ansible-nix/files/postgrest-optimizations.service.j2 b/ansible-nix/files/postgrest-optimizations.service.j2 new file mode 100644 index 000000000..38604b67a --- /dev/null +++ b/ansible-nix/files/postgrest-optimizations.service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Postgrest optimizations + +[Service] +Type=oneshot +# we don't want failures from this command to cause PG startup to fail +ExecStart=/bin/bash -c "/opt/supabase-admin-api optimize postgrest --destination-config-file-path /etc/postgrest/generated.conf ; exit 0" +User=postgrest + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/postgrest.service.j2 b/ansible-nix/files/postgrest.service.j2 new file mode 100644 index 000000000..290f07720 --- /dev/null +++ b/ansible-nix/files/postgrest.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=PostgREST +Requires=postgrest-optimizations.service +After=postgrest-optimizations.service + +[Service] +Type=simple +# We allow the base config (sent from the worker) to override the generated config +ExecStartPre=/etc/postgrest/merge.sh /etc/postgrest/generated.conf /etc/postgrest/base.conf +ExecStart=/opt/postgrest /etc/postgrest/merged.conf +User=postgrest +Slice=services.slice +Restart=always +RestartSec=3 +LimitNOFILE=100000 + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/services.slice.j2 b/ansible-nix/files/services.slice.j2 new file mode 100644 index 000000000..d45187fb8 --- /dev/null +++ b/ansible-nix/files/services.slice.j2 @@ -0,0 +1,6 @@ +# Used for general services grouping for easy visibility when running +# systemctl status +# See http://archive.vn/94IGa for an in depth article on systemd slices +[Unit] +Description=Slice used for PostgreSQL +Before=slices.target diff --git a/ansible-nix/files/sodium_extension.sql b/ansible-nix/files/sodium_extension.sql new file mode 100644 index 000000000..a19cabf72 --- /dev/null +++ b/ansible-nix/files/sodium_extension.sql @@ -0,0 +1,6 @@ +create schema if not exists pgsodium; +create extension if not exists pgsodium with schema pgsodium cascade; + +grant pgsodium_keyiduser to postgres with admin option; +grant pgsodium_keyholder to postgres with admin option; +grant pgsodium_keymaker to postgres with admin option; diff --git a/ansible-nix/files/start-envoy.sh b/ansible-nix/files/start-envoy.sh new file mode 100644 index 000000000..edd6fe09e --- /dev/null +++ b/ansible-nix/files/start-envoy.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -eou pipefail + +if [[ $(cat /sys/module/ipv6/parameters/disable) = 1 ]]; then + sed -i -e "s/address: '::'/address: '0.0.0.0'/" -e 's/ipv4_compat: true/ipv4_compat: false/' /etc/envoy/lds.yaml +else + sed -i -e "s/address: '0.0.0.0'/address: '::'/" -e 's/ipv4_compat: false/ipv4_compat: true/' /etc/envoy/lds.yaml +fi + +# Workaround using `tee` to get `/dev/stdout` access logging to work, see: +# https://github.com/envoyproxy/envoy/issues/8297#issuecomment-620659781 +exec /opt/envoy --config-path /etc/envoy/envoy.yaml --restart-epoch "${RESTART_EPOCH}" 2>&1 | tee diff --git a/ansible-nix/files/stat_extension.sql b/ansible-nix/files/stat_extension.sql new file mode 100644 index 000000000..937834004 --- /dev/null +++ b/ansible-nix/files/stat_extension.sql @@ -0,0 +1,2 @@ +CREATE SCHEMA IF NOT exists extensions; +CREATE EXTENSION IF NOT EXISTS pg_stat_statements with schema extensions; diff --git a/ansible-nix/files/supabase_facts.ini b/ansible-nix/files/supabase_facts.ini new file mode 100644 index 000000000..44e01b4c3 --- /dev/null +++ b/ansible-nix/files/supabase_facts.ini @@ -0,0 +1,2 @@ +[general] +postgres_version=15 diff --git a/ansible-nix/files/sysstat.sysstat b/ansible-nix/files/sysstat.sysstat new file mode 100644 index 000000000..52b7d07d1 --- /dev/null +++ b/ansible-nix/files/sysstat.sysstat @@ -0,0 +1,36 @@ +# How long to keep log files (in days). +# Used by sa2(8) script +# If value is greater than 28, then use sadc's option -D to prevent older +# data files from being overwritten. See sadc(8) and sysstat(5) manual pages. +HISTORY=7 + +# Compress (using xz, gzip or bzip2) sa and sar files older than (in days): +COMPRESSAFTER=10 + +# Parameters for the system activity data collector (see sadc(8) manual page) +# which are used for the generation of log files. +# By default contains the `-S DISK' option responsible for generating disk +# statisitcs. Use `-S XALL' to collect all available statistics. +SADC_OPTIONS="-S DISK" + +# Directory where sa and sar files are saved. The directory must exist. +SA_DIR=/var/log/sysstat + +# Compression program to use. +ZIP="xz" + +# By default sa2 script generates yesterday's summary, since the cron job +# usually runs right after midnight. If you want sa2 to generate the summary +# of the same day (for example when cron job runs at 23:53) set this variable. +#YESTERDAY=no + +# By default sa2 script generates reports files (the so called sarDD files). +# Set this variable to false to disable reports generation. +#REPORTS=false + +# The sa1 and sa2 scripts generate system activity data and report files in +# the /var/log/sysstat directory. By default the files are created with umask 0022 +# and are therefore readable for all users. Change this variable to restrict +# the permissions on the files (e.g. use 0027 to adhere to more strict +# security standards). +UMASK=0022 diff --git a/ansible-nix/files/systemd-resolved.conf b/ansible-nix/files/systemd-resolved.conf new file mode 100644 index 000000000..9280d88a1 --- /dev/null +++ b/ansible-nix/files/systemd-resolved.conf @@ -0,0 +1,8 @@ +# the default is RestartSec=0. If the service fails to start because +# of a systemic issue (e.g. rare case when disk is full) it will +# quickly hit the burst limit (default of 5 failures within 10secs) +# and thereafter be placed in a failed state. By increasing the +# restart interval, we avoid that, and ensure that the service will be +# started back up once any underlying issues are resolved. +[Service] +RestartSec=3 diff --git a/ansible-nix/files/ufw.service.conf b/ansible-nix/files/ufw.service.conf new file mode 100644 index 000000000..83b82efc7 --- /dev/null +++ b/ansible-nix/files/ufw.service.conf @@ -0,0 +1,4 @@ +[Unit] +After=nftables.service +Requires=nftables.service +PartOf=nftables.service diff --git a/ansible-nix/files/vector.service.j2 b/ansible-nix/files/vector.service.j2 new file mode 100644 index 000000000..1c88baa20 --- /dev/null +++ b/ansible-nix/files/vector.service.j2 @@ -0,0 +1,20 @@ +[Unit] +Description=Vector +Documentation=https://vector.dev +After=network-online.target +Requires=network-online.target + +[Service] +User=vector +Group=vector +ExecStartPre=/usr/bin/vector validate --config-yaml /etc/vector/vector.yaml +ExecStart=/usr/bin/vector --config-yaml /etc/vector/vector.yaml +ExecReload=/usr/bin/vector validate --config-yaml /etc/vector/vector.yaml +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +RestartSec=3 +AmbientCapabilities=CAP_NET_BIND_SERVICE +EnvironmentFile=-/etc/default/vector + +[Install] +WantedBy=multi-user.target diff --git a/ansible-nix/files/walg_helper_scripts/wal_change_ownership.sh b/ansible-nix/files/walg_helper_scripts/wal_change_ownership.sh new file mode 100644 index 000000000..3f0112d2f --- /dev/null +++ b/ansible-nix/files/walg_helper_scripts/wal_change_ownership.sh @@ -0,0 +1,42 @@ +#! /usr/bin/env bash + +set -euo pipefail + +filename=$1 + +if [[ -z "$filename" ]]; then + echo "Nothing supplied. Exiting." + exit 1 +fi + +full_path=/tmp/wal_fetch_dir/$filename + +num_paths=$(readlink -f "$full_path" | wc -l) + +# Checks if supplied filename string contains multiple paths +# For example, "correct/path /var/lib/injected/path /var/lib/etc" +if [[ "$num_paths" -gt 1 ]]; then + echo "Multiple paths supplied. Exiting." + exit 1 +fi + +base_dir=$(readlink -f "$full_path" | cut -d'/' -f2) + +# Checks if directory/ file to be manipulated +# is indeed within the /tmp directory +# For example, "/tmp/../var/lib/postgresql/..." +# will return "var" as the value for $base_dir +if [[ "$base_dir" != "tmp" ]]; then + echo "Attempt to manipulate a file not in /tmp. Exiting." + exit 1 +fi + +# Checks if change of ownership will be applied to a file +# If not, exit +if [[ ! -f $full_path ]]; then + echo "Either file does not exist or is a directory. Exiting." + exit 1 +fi + +# once valid, proceed to change ownership +chown postgres:postgres "$full_path" diff --git a/ansible-nix/files/walg_helper_scripts/wal_fetch.sh b/ansible-nix/files/walg_helper_scripts/wal_fetch.sh new file mode 100644 index 000000000..33448ac95 --- /dev/null +++ b/ansible-nix/files/walg_helper_scripts/wal_fetch.sh @@ -0,0 +1,12 @@ +#! /usr/bin/env bash + +set -euo pipefail + +# Fetch the WAL file and temporarily store them in /tmp +sudo -u wal-g wal-g wal-fetch "$1" /tmp/wal_fetch_dir/"$1" --config /etc/wal-g/config.json + +# Ensure WAL file is owned by the postgres Linux user +sudo -u root /root/wal_change_ownership.sh "$1" + +# Move file to its final destination +mv /tmp/wal_fetch_dir/"$1" /var/lib/postgresql/data/"$2" diff --git a/ansible-nix/manifest-playbook.yml b/ansible-nix/manifest-playbook.yml new file mode 100644 index 000000000..93f0e15c5 --- /dev/null +++ b/ansible-nix/manifest-playbook.yml @@ -0,0 +1,91 @@ +- hosts: localhost + gather_facts: no + + vars_files: + - ./vars.yml + + tasks: + - name: Write out image manifest + action: template src=files/manifest.json dest=./image-manifest-{{ ami_release_version }}.json + + - name: Upload image manifest + shell: | + aws s3 cp ./image-manifest-{{ ami_release_version }}.json s3://{{ internal_artifacts_bucket }}/manifests/postgres-{{ ami_release_version }}/software-manifest.json + + # upload software artifacts of interest + # Generally - download, extract, repack as xz archive, upload + # currently, we upload gotrue, adminapi, postgrest + - name: gotrue - download commit archive + get_url: + url: "https://github.com/supabase/gotrue/releases/download/v{{ gotrue_release }}/auth-v{{ gotrue_release }}-arm64.tar.gz" + dest: /tmp/gotrue.tar.gz + checksum: "{{ gotrue_release_checksum }}" + timeout: 60 + + - name: gotrue - create /tmp/gotrue + file: + path: /tmp/gotrue + state: directory + mode: 0775 + + - name: gotrue - unpack archive in /tmp/gotrue + unarchive: + remote_src: yes + src: /tmp/gotrue.tar.gz + dest: /tmp/gotrue + + - name: gotrue - pack archive + shell: | + cd /tmp && tar -cJf gotrue-v{{ gotrue_release }}-arm64.tar.xz gotrue + + - name: PostgREST - download ubuntu binary archive (arm) + get_url: + url: "https://github.com/PostgREST/postgrest/releases/download/v{{ postgrest_release }}/postgrest-v{{ postgrest_release }}-ubuntu-aarch64.tar.xz" + dest: /tmp/postgrest-{{ postgrest_release }}-arm64.tar.xz + checksum: "{{ postgrest_arm_release_checksum }}" + timeout: 60 + + - name: Download adminapi archive + get_url: + url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-api/v{{ adminapi_release }}/supabase-admin-api_{{ adminapi_release }}_linux_arm64.tar.gz" + dest: "/tmp/adminapi.tar.gz" + timeout: 90 + + - name: adminapi - unpack archive in /tmp + unarchive: + remote_src: yes + src: /tmp/adminapi.tar.gz + dest: /tmp + + - name: adminapi - pack archive + shell: | + cd /tmp && tar -cJf supabase-admin-api-{{ adminapi_release }}-arm64.tar.xz supabase-admin-api + + - name: Download admin-mgr archive + get_url: + url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/admin-mgr/v{{ adminmgr_release }}/admin-mgr_{{ adminmgr_release }}_linux_arm64.tar.gz" + dest: "/tmp/admin-mgr.tar.gz" + timeout: 90 + + - name: admin-mgr - unpack archive in /tmp + unarchive: + remote_src: yes + src: /tmp/admin-mgr.tar.gz + dest: /tmp + + - name: admin-mgr - pack archive + shell: | + cd /tmp && tar -cJf admin-mgr-{{ adminmgr_release }}-arm64.tar.xz admin-mgr + + - name: upload archives + shell: | + aws s3 cp /tmp/{{ item.file }} s3://{{ internal_artifacts_bucket }}/upgrades/{{ item.service }}/{{ item.file }} + with_items: + - service: gotrue + file: gotrue-v{{ gotrue_release }}-arm64.tar.xz + - service: postgrest + file: postgrest-{{ postgrest_release }}-arm64.tar.xz + - service: supabase-admin-api + file: supabase-admin-api-{{ adminapi_release }}-arm64.tar.xz + - service: admin-mgr + file: admin-mgr-{{ adminmgr_release }}-arm64.tar.xz diff --git a/ansible-nix/playbook.yml b/ansible-nix/playbook.yml new file mode 100644 index 000000000..e23f9652f --- /dev/null +++ b/ansible-nix/playbook.yml @@ -0,0 +1,107 @@ +- hosts: all + become: yes + + pre_tasks: + - import_tasks: tasks/setup-system.yml + + vars_files: + - ./vars.yml + + vars: + sql_files: + - { + source: "pgbouncer_config/pgbouncer_auth_schema.sql", + dest: "00-schema.sql", + } + - { source: "stat_extension.sql", dest: "01-extension.sql" } + + environment: + PATH: /usr/lib/postgresql/bin:{{ ansible_env.PATH }} + + tasks: + - set_fact: + supabase_internal: true + tags: + - install-supabase-internal + + - set_fact: + parallel_jobs: 16 + + - name: Prepare machine for Postgres installation + import_tasks: tasks/setup-postgres.yml + + - name: Install PgBouncer + import_tasks: tasks/setup-pgbouncer.yml + tags: + - install-pgbouncer + - install-supabase-internal + + - name: Install WAL-G + import_tasks: tasks/setup-wal-g.yml + + - name: Install Gotrue + import_tasks: tasks/setup-gotrue.yml + tags: + - install-gotrue + - install-supabase-internal + + - name: Install PostgREST + import_tasks: tasks/setup-postgrest.yml + tags: + - install-postgrest + - install-supabase-internal + + - name: Install Envoy + import_tasks: tasks/setup-envoy.yml + tags: + - install-supabase-internal + + - name: Install Kong + import_tasks: tasks/setup-kong.yml + tags: + - install-supabase-internal + + - name: Install nginx + import_tasks: tasks/setup-nginx.yml + tags: + - install-supabase-internal + + - name: Install Supabase specific content + import_tasks: tasks/setup-supabase-internal.yml + tags: + - install-supabase-internal + + - name: Adjust APT update intervals + copy: + src: files/apt_periodic + dest: /etc/apt/apt.conf.d/10periodic + + - name: Finalize AMI + import_tasks: tasks/finalize-ami.yml + tags: + - install-supabase-internal + + - name: Enhance fail2ban + import_tasks: tasks/setup-fail2ban.yml + + # Install EC2 instance connect + # Only for AWS images + - name: install EC2 instance connect + become: yes + apt: + pkg: + - ec2-instance-connect + tags: + - aws-only + + # Install this at the end to prevent it from kicking in during the apt process, causing conflicts + - name: Install security tools + become: yes + apt: + pkg: + - unattended-upgrades + update_cache: yes + cache_valid_time: 3600 + + - name: Clean out build dependencies + import_tasks: tasks/clean-build-dependencies.yml diff --git a/ansible-nix/tasks/clean-build-dependencies.yml b/ansible-nix/tasks/clean-build-dependencies.yml new file mode 100644 index 000000000..43ec05179 --- /dev/null +++ b/ansible-nix/tasks/clean-build-dependencies.yml @@ -0,0 +1,21 @@ +- name: Remove build dependencies + apt: + pkg: + - bison + - build-essential + - clang-11 + - cmake + - cpp + - flex + - g++ + - g++-10 + - g++-9 + - gcc-10 + - make + - manpages + - manpages-dev + - ninja-build + - patch + - python2 + state: absent + autoremove: yes diff --git a/ansible-nix/tasks/finalize-ami.yml b/ansible-nix/tasks/finalize-ami.yml new file mode 100644 index 000000000..81a89abeb --- /dev/null +++ b/ansible-nix/tasks/finalize-ami.yml @@ -0,0 +1,72 @@ +- name: PG logging conf + template: + src: files/postgresql_config/postgresql-csvlog.conf + dest: /etc/postgresql/logging.conf + group: postgres + +- name: UFW - Allow SSH connections + ufw: + rule: allow + name: OpenSSH + +- name: UFW - Allow connections to postgreSQL (5432) + ufw: + rule: allow + port: "5432" + +- name: UFW - Allow connections to postgreSQL (6543) + ufw: + rule: allow + port: "6543" + tags: + - install-pgbouncer + +- name: UFW - Allow connections to http (80) + ufw: + rule: allow + port: http + tags: + - install-supabase-internal + +- name: UFW - Allow connections to https (443) + ufw: + rule: allow + port: https + tags: + - install-supabase-internal + +- name: UFW - Deny all other incoming traffic by default + ufw: + state: enabled + policy: deny + direction: incoming + +- name: Move logrotate files to /etc/logrotate.d/ + copy: + src: "files/logrotate_config/{{ item.file }}" + dest: "/etc/logrotate.d/{{ item.file }}" + mode: "0700" + owner: root + loop: + - { file: "logrotate-postgres-csv.conf" } + - { file: "logrotate-postgres.conf" } + - { file: "logrotate-walg.conf" } + - { file: "logrotate-postgres-auth.conf" } + +- name: Ensure default Postgres logrotate config is removed + file: + path: /etc/logrotate.d/postgresql-common + state: absent + +- name: Disable cron access + copy: + src: files/cron.deny + dest: /etc/cron.deny + +- name: Configure logrotation to run every hour + shell: + cmd: | + cp /usr/lib/systemd/system/logrotate.timer /etc/systemd/system/logrotate.timer + sed -i -e 's;daily;*:0/5;' /etc/systemd/system/logrotate.timer + systemctl reenable logrotate.timer + become: yes diff --git a/ansible-nix/tasks/internal/admin-api.yml b/ansible-nix/tasks/internal/admin-api.yml new file mode 100644 index 000000000..cea0109fd --- /dev/null +++ b/ansible-nix/tasks/internal/admin-api.yml @@ -0,0 +1,92 @@ +- name: adminapi - system user + user: + name: adminapi + groups: root,admin,envoy,kong,pgbouncer,postgres,postgrest,systemd-journal,vector,wal-g + append: yes + +- name: Move shell scripts to /root dir + copy: + src: "files/admin_api_scripts/{{ item.file }}" + dest: "/root/{{ item.file }}" + mode: "0700" + owner: root + loop: + - { file: "grow_fs.sh" } + - { file: "manage_readonly_mode.sh" } + - { file: "pg_egress_collect.pl" } + +- name: give adminapi user permissions + copy: + src: files/adminapi.sudoers.conf + dest: /etc/sudoers.d/adminapi + mode: "0644" + +- name: perms for adminapi + shell: | + chmod g+w /etc + +- name: Setting arch (x86) + set_fact: + arch: "x86" + when: platform == "amd64" + +- name: Setting arch (arm) + set_fact: + arch: "arm64" + when: platform == "arm64" + +- name: Download adminapi archive + get_url: + url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-api/v{{ adminapi_release }}/supabase-admin-api_{{ adminapi_release }}_linux_{{ arch }}.tar.gz" + dest: "/tmp/adminapi.tar.gz" + timeout: 90 + +- name: adminapi - unpack archive in /opt + unarchive: + remote_src: yes + src: /tmp/adminapi.tar.gz + dest: /opt + owner: adminapi + +- name: adminapi - config dir + file: + path: /etc/adminapi + owner: adminapi + state: directory + +- name: adminapi - pg_upgrade scripts dir + file: + path: /etc/adminapi/pg_upgrade_scripts + owner: adminapi + state: directory + +- name: Move shell scripts to /etc/adminapi/pg_upgrade_scripts/ + copy: + src: "files/admin_api_scripts/pg_upgrade_scripts/{{ item.file }}" + dest: "/etc/adminapi/pg_upgrade_scripts/{{ item.file }}" + mode: "0755" + owner: adminapi + loop: + - { file: "check.sh" } + - { file: "complete.sh" } + - { file: "initiate.sh" } + - { file: "prepare.sh" } + - { file: "pgsodium_getkey.sh" } + - { file: "common.sh" } + +- name: adminapi - create service file + template: + src: files/adminapi.service.j2 + dest: /etc/systemd/system/adminapi.service + +- name: UFW - Allow connections to adminapi ports + ufw: + rule: allow + port: "8085" + +- name: adminapi - reload systemd + systemd: + daemon_reload: yes + +- name: adminapi - grant extra priviliges to user + shell: chmod 775 /etc && chmod 775 /etc/kong diff --git a/ansible-nix/tasks/internal/admin-mgr.yml b/ansible-nix/tasks/internal/admin-mgr.yml new file mode 100644 index 000000000..073b8661f --- /dev/null +++ b/ansible-nix/tasks/internal/admin-mgr.yml @@ -0,0 +1,22 @@ +- name: Setting arch (x86) + set_fact: + arch: "amd64" + when: platform == "amd64" + +- name: Setting arch (arm) + set_fact: + arch: "arm64" + when: platform == "arm64" + +- name: Download admin-mgr archive + get_url: + url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/admin-mgr/v{{ adminmgr_release }}/admin-mgr_{{ adminmgr_release }}_linux_{{ arch }}.tar.gz" + dest: "/tmp/admin-mgr.tar.gz" + timeout: 90 + +- name: admin-mgr - unpack archive in /usr/bin/ + unarchive: + remote_src: yes + src: /tmp/admin-mgr.tar.gz + dest: /usr/bin/ + owner: root diff --git a/ansible-nix/tasks/internal/pg_egress_collect.yml b/ansible-nix/tasks/internal/pg_egress_collect.yml new file mode 100644 index 000000000..be9fefe02 --- /dev/null +++ b/ansible-nix/tasks/internal/pg_egress_collect.yml @@ -0,0 +1,15 @@ +- name: pg_egress_collect - install tcpdump and perl async lib + apt: + pkg: + - tcpdump + - libio-async-perl + +- name: pg_egress_collect - create service file + template: + src: files/pg_egress_collect.service.j2 + dest: /etc/systemd/system/pg_egress_collect.service + +- name: pg_egress_collect - reload systemd + systemd: + daemon_reload: yes + diff --git a/ansible-nix/tasks/internal/postgres-exporter.yml b/ansible-nix/tasks/internal/postgres-exporter.yml new file mode 100644 index 000000000..0292157b2 --- /dev/null +++ b/ansible-nix/tasks/internal/postgres-exporter.yml @@ -0,0 +1,48 @@ +- name: UFW - Allow connections to exporter for prometheus + ufw: + rule: allow + port: "9187" + +- name: create directories - systemd unit + file: + state: directory + path: /etc/systemd/system/postgres_exporter.service.d + owner: root + mode: '0700' + become: yes + +- name: create directories - service files + file: + state: directory + path: /opt/postgres_exporter + owner: postgres + group: postgres + mode: '0775' + become: yes + +- name: download postgres exporter + get_url: + url: "https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ postgres_exporter_release }}/postgres_exporter-{{ postgres_exporter_release }}.linux-{{ platform }}.tar.gz" + dest: /tmp/postgres_exporter.tar.gz + checksum: "{{ postgres_exporter_release_checksum[platform] }}" + timeout: 60 + +- name: expand postgres exporter + unarchive: + remote_src: yes + src: /tmp/postgres_exporter.tar.gz + dest: /opt/postgres_exporter + extra_opts: [--strip-components=1] + become: yes + +- name: exporter create a service + template: + src: files/postgres_exporter.service.j2 + dest: /etc/systemd/system/postgres_exporter.service + +- name: exporter ensure service is present + systemd: + enabled: no + name: postgres_exporter + daemon_reload: yes + state: stopped diff --git a/ansible-nix/tasks/internal/setup-ansible-pull.yml b/ansible-nix/tasks/internal/setup-ansible-pull.yml new file mode 100644 index 000000000..7cce74a8c --- /dev/null +++ b/ansible-nix/tasks/internal/setup-ansible-pull.yml @@ -0,0 +1,29 @@ +- name: install ansible + shell: + cmd: | + apt install -y software-properties-common + add-apt-repository --yes --update ppa:ansible/ansible + apt install -y ansible + sed -i -e 's/#callback_whitelist.*/callback_whitelist = profile_tasks/' /etc/ansible/ansible.cfg + +- name: ansible pull systemd units + copy: + src: files/{{ item }} + dest: /etc/systemd/system/{{ item }} + with_items: + - ansible-pull.service + - ansible-pull.timer + +- name: create facts dir + file: + path: /etc/ansible/facts.d + state: directory + +- name: ansible facts + copy: + src: files/supabase_facts.ini + dest: /etc/ansible/facts.d/supabase.fact + +- name: reload systemd + systemd: + daemon_reload: yes diff --git a/ansible-nix/tasks/internal/setup-nftables.yml b/ansible-nix/tasks/internal/setup-nftables.yml new file mode 100644 index 000000000..fc8d0235a --- /dev/null +++ b/ansible-nix/tasks/internal/setup-nftables.yml @@ -0,0 +1,34 @@ +- name: nftables overrides + file: + state: directory + path: /etc/nftables + owner: adminapi + +- name: nftables empty config + file: + state: touch + path: /etc/nftables/supabase_managed.conf + owner: adminapi + +- name: include managed config + shell: | + cat >> "/etc/nftables.conf" << EOF + table inet supabase_managed { } + include "/etc/nftables/supabase_managed.conf"; + + EOF + +- name: ufw overrides dir + file: + state: directory + path: /etc/systemd/system/ufw.service.d + owner: root + +- name: Custom systemd overrides + copy: + src: files/ufw.service.conf + dest: /etc/systemd/system/ufw.service.d/overrides.conf + +- name: reload systemd + systemd: + daemon_reload: yes diff --git a/ansible-nix/tasks/internal/supautils.yml b/ansible-nix/tasks/internal/supautils.yml new file mode 100644 index 000000000..33811b5ac --- /dev/null +++ b/ansible-nix/tasks/internal/supautils.yml @@ -0,0 +1,77 @@ +# supautils +- name: supautils - download & install dependencies + apt: + pkg: + - build-essential + - clang-11 + update_cache: yes + cache_valid_time: 3600 + +- name: supautils - download latest release + get_url: + url: "https://github.com/supabase/supautils/archive/refs/tags/v{{ supautils_release }}.tar.gz" + dest: /tmp/supautils-{{ supautils_release }}.tar.gz + checksum: "{{ supautils_release_checksum }}" + timeout: 60 + +- name: supautils - unpack archive + unarchive: + remote_src: yes + src: /tmp/supautils-{{ supautils_release }}.tar.gz + dest: /tmp + become: yes + +- name: supautils - build + make: + chdir: /tmp/supautils-{{ supautils_release }} + become: yes + +- name: supautils - install + make: + chdir: /tmp/supautils-{{ supautils_release }} + target: install + become: yes + +- name: supautils - add supautils to session_preload_libraries + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: "#session_preload_libraries = ''" + replace: session_preload_libraries = 'supautils' + +- name: supautils - write custom supautils.conf + template: + src: "files/postgresql_config/supautils.conf.j2" + dest: /etc/postgresql-custom/supautils.conf + mode: 0664 + owner: postgres + group: postgres + +- name: supautils - copy extension custom scripts + copy: + src: files/postgresql_extension_custom_scripts/ + dest: /etc/postgresql-custom/extension-custom-scripts + become: yes + +- name: supautils - chown extension custom scripts + file: + mode: 0775 + owner: postgres + group: postgres + path: /etc/postgresql-custom/extension-custom-scripts + recurse: yes + become: yes + +- name: supautils - include /etc/postgresql-custom/supautils.conf in postgresql.conf + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: "#include = '/etc/postgresql-custom/supautils.conf'" + replace: "include = '/etc/postgresql-custom/supautils.conf'" + +- name: supautils - remove build dependencies + apt: + pkg: + - build-essential + - clang-11 + state: absent diff --git a/ansible-nix/tasks/setup-envoy.yml b/ansible-nix/tasks/setup-envoy.yml new file mode 100644 index 000000000..9843b5546 --- /dev/null +++ b/ansible-nix/tasks/setup-envoy.yml @@ -0,0 +1,60 @@ +- name: Envoy - system user + ansible.builtin.user: + name: envoy + +- name: Envoy - download binary + ansible.builtin.get_url: + checksum: "{{ envoy_release_checksum }}" + dest: /opt/envoy + group: envoy + mode: u+x + owner: envoy + # yamllint disable-line rule:line-length + url: "https://github.com/envoyproxy/envoy/releases/download/v{{ envoy_release }}/envoy-{{ envoy_release }}-linux-aarch_64" + +- name: Envoy - download hot restarter script + ansible.builtin.get_url: + checksum: "{{ envoy_hot_restarter_release_checksum }}" + dest: /opt/envoy-hot-restarter.py + group: envoy + mode: u+x + owner: envoy + # yamllint disable-line rule:line-length + url: https://raw.githubusercontent.com/envoyproxy/envoy/v{{ envoy_release }}/restarter/hot-restarter.py + +- name: Envoy - bump up ulimit + community.general.pam_limits: + domain: envoy + limit_item: nofile + limit_type: soft + value: 4096 + +- name: Envoy - create script to start envoy + ansible.builtin.copy: + dest: /opt/start-envoy.sh + group: envoy + mode: u+x + owner: envoy + src: files/start-envoy.sh + +- name: Envoy - create configuration files + ansible.builtin.copy: + dest: /etc/envoy/ + directory_mode: u=rwx,g=rwx,o=rx + group: envoy + mode: u=rw,g=rw,o=r + owner: envoy + src: files/envoy_config/ + +- name: Envoy - create service file + ansible.builtin.copy: + dest: /etc/systemd/system/envoy.service + mode: u=rw,g=r,o=r + src: files/envoy.service + +- name: Envoy - disable service + ansible.builtin.systemd: + daemon_reload: true + enabled: false + name: envoy + state: stopped diff --git a/ansible-nix/tasks/setup-extensions.yml b/ansible-nix/tasks/setup-extensions.yml new file mode 100644 index 000000000..0fb92c151 --- /dev/null +++ b/ansible-nix/tasks/setup-extensions.yml @@ -0,0 +1,94 @@ +- name: Install plv8 + import_tasks: tasks/postgres-extensions/13-plv8.yml + +- name: Install pg_jsonschema + import_tasks: tasks/postgres-extensions/22-pg_jsonschema.yml + +- name: Install postgis + import_tasks: tasks/postgres-extensions/01-postgis.yml + +- name: Install pgrouting + import_tasks: tasks/postgres-extensions/02-pgrouting.yml + +- name: Install pgtap + import_tasks: tasks/postgres-extensions/03-pgtap.yml + +- name: Install pg_cron + import_tasks: tasks/postgres-extensions/04-pg_cron.yml + +- name: Install pgaudit + import_tasks: tasks/postgres-extensions/05-pgaudit.yml + +- name: Install pgjwt + import_tasks: tasks/postgres-extensions/06-pgjwt.yml + +- name: Install pgsql-http + import_tasks: tasks/postgres-extensions/07-pgsql-http.yml + +- name: Install plpgsql_check + import_tasks: tasks/postgres-extensions/08-plpgsql_check.yml + +- name: Install pg-safeupdate + import_tasks: tasks/postgres-extensions/09-pg-safeupdate.yml + +- name: Install timescaledb + import_tasks: tasks/postgres-extensions/10-timescaledb.yml + +- name: Install wal2json + import_tasks: tasks/postgres-extensions/11-wal2json.yml + +- name: Install pljava + import_tasks: tasks/postgres-extensions/12-pljava.yml + tags: + - legacy-incompatible + +- name: Install pg_plan_filter + import_tasks: tasks/postgres-extensions/14-pg_plan_filter.yml + +- name: Install pg_net + import_tasks: tasks/postgres-extensions/15-pg_net.yml + +- name: Install rum + import_tasks: tasks/postgres-extensions/16-rum.yml + +- name: Install pg_hashids + import_tasks: tasks/postgres-extensions/17-pg_hashids.yml + +- name: Install pgsodium + import_tasks: tasks/postgres-extensions/18-pgsodium.yml + +- name: Install pg_graphql + import_tasks: tasks/postgres-extensions/19-pg_graphql.yml + tags: + - legacy-incompatible + +- name: Install pg_stat_monitor + import_tasks: tasks/postgres-extensions/20-pg_stat_monitor.yml + +- name: Install auto_explain + import_tasks: tasks/postgres-extensions/21-auto_explain.yml + +- name: Install vault + import_tasks: tasks/postgres-extensions/23-vault.yml + +- name: Install PGroonga + import_tasks: tasks/postgres-extensions/24-pgroonga.yml + +- name: Install wrappers + import_tasks: tasks/postgres-extensions/25-wrappers.yml + +- name: Install hypopg + import_tasks: tasks/postgres-extensions/26-hypopg.yml + +- name: Install pg_repack + import_tasks: tasks/postgres-extensions/27-pg_repack.yml + +- name: Install pgvector + import_tasks: tasks/postgres-extensions/28-pgvector.yml + +- name: Install Trusted Language Extensions + import_tasks: tasks/postgres-extensions/29-pg_tle.yml + +- name: Verify async task status + import_tasks: tasks/postgres-extensions/99-finish_async_tasks.yml + when: async_mode diff --git a/ansible-nix/tasks/setup-fail2ban.yml b/ansible-nix/tasks/setup-fail2ban.yml new file mode 100644 index 000000000..e1cec2d92 --- /dev/null +++ b/ansible-nix/tasks/setup-fail2ban.yml @@ -0,0 +1,70 @@ +# set default bantime to 1 hour +- name: extend bantime + become: yes + replace: + path: /etc/fail2ban/jail.conf + regexp: bantime = 10m + replace: bantime = 3600 + +- name: Configure journald + copy: + src: files/fail2ban_config/jail-ssh.conf + dest: /etc/fail2ban/jail.d/sshd.local + +- name: configure fail2ban to use nftables + copy: + src: files/fail2ban_config/jail.local + dest: /etc/fail2ban/jail.local + +# postgresql +- name: import jail.d/postgresql.conf + template: + src: files/fail2ban_config/jail-postgresql.conf.j2 + dest: /etc/fail2ban/jail.d/postgresql.conf + become: yes + +- name: import filter.d/postgresql.conf + template: + src: files/fail2ban_config/filter-postgresql.conf.j2 + dest: /etc/fail2ban/filter.d/postgresql.conf + become: yes + +- name: create overrides dir + file: + state: directory + owner: root + group: root + path: /etc/systemd/system/fail2ban.service.d + mode: '0700' + +- name: Custom systemd overrides + copy: + src: files/fail2ban_config/fail2ban.service.conf + dest: /etc/systemd/system/fail2ban.service.d/overrides.conf + +- name: add in supabase specific ignore filters + lineinfile: + path: /etc/fail2ban/filter.d/postgresql.conf + state: present + line: "{{ item.line }}" + loop: + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_admin".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_auth_admin".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_storage_admin".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""authenticator".*$' } + - { line: ' ^.*,.*,.*,.*,":.*password authentication failed for user ""pgbouncer".*$' } + become: yes + tags: + - install-supabase-internal + +# Restart +- name: fail2ban - restart + systemd: + name: fail2ban + state: restarted + +- name: fail2ban - disable service + systemd: + name: fail2ban + enabled: no + daemon_reload: yes diff --git a/ansible-nix/tasks/setup-gotrue.yml b/ansible-nix/tasks/setup-gotrue.yml new file mode 100644 index 000000000..2a70caa4d --- /dev/null +++ b/ansible-nix/tasks/setup-gotrue.yml @@ -0,0 +1,54 @@ +- name: UFW - Allow connections to GoTrue metrics exporter + ufw: + rule: allow + port: "9122" + +# use this user for the Gotrue build and for running the service +- name: Gotrue - system user + user: name=gotrue + +- name: Setting arch (x86) + set_fact: + arch: "x86" + when: platform == "amd64" + +- name: Setting arch (arm) + set_fact: + arch: "arm64" + when: platform == "arm64" + +- name: gotrue - download commit archive + get_url: + url: "https://github.com/supabase/gotrue/releases/download/v{{ gotrue_release }}/auth-v{{ gotrue_release }}-{{ arch }}.tar.gz" + dest: /tmp/gotrue.tar.gz + checksum: "{{ gotrue_release_checksum }}" + +- name: gotrue - create /opt/gotrue + file: + path: /opt/gotrue + state: directory + owner: gotrue + mode: 0775 + +- name: gotrue - unpack archive in /opt/gotrue + unarchive: + remote_src: yes + src: /tmp/gotrue.tar.gz + dest: /opt/gotrue + owner: gotrue + +# libpq is a C library that enables user programs to communicate with +# the PostgreSQL database server. +- name: gotrue - system dependencies + apt: + pkg: + - libpq-dev + +- name: gotrue - create service file + template: + src: files/gotrue.service.j2 + dest: /etc/systemd/system/gotrue.service + +- name: gotrue - reload systemd + systemd: + daemon_reload: yes diff --git a/ansible-nix/tasks/setup-kong.yml b/ansible-nix/tasks/setup-kong.yml new file mode 100644 index 000000000..09d6d2b44 --- /dev/null +++ b/ansible-nix/tasks/setup-kong.yml @@ -0,0 +1,62 @@ +- name: Kong - system user + user: name=kong + +# Kong installation steps from http://archive.vn/3HRQx +- name: Kong - system dependencies + apt: + pkg: + - openssl + - libpcre3 + - procps + - perl + +- name: Kong - download deb package + get_url: + url: "https://download.konghq.com/gateway-2.x-ubuntu-{{ kong_release_target }}/pool/all/k/kong/{{ kong_deb }}" + dest: /tmp/kong.deb + checksum: "{{ kong_deb_checksum }}" + +- name: Kong - deb installation + apt: deb=file:///tmp/kong.deb + +- name: Kong - ensure it is NOT autoremoved + shell: | + set -e + apt-mark manual kong zlib1g* + +- name: Kong - configuration + template: + src: files/kong_config/kong.conf.j2 + dest: /etc/kong/kong.conf + +- name: Kong - hand over ownership of /usr/local/kong to user kong + file: + path: /usr/local/kong + recurse: yes + owner: kong + +# [warn] ulimit is currently set to "1024". For better performance set it to at least +# "4096" using "ulimit -n" +- name: Kong - bump up ulimit + pam_limits: + limit_item: nofile + limit_type: soft + domain: kong + value: "4096" + +- name: Kong - create env file + template: + src: files/kong_config/kong.env.j2 + dest: /etc/kong/kong.env + +- name: Kong - create service file + template: + src: files/kong_config/kong.service.j2 + dest: /etc/systemd/system/kong.service + +- name: Kong - disable service + systemd: + enabled: no + name: kong + state: stopped + daemon_reload: yes diff --git a/ansible-nix/tasks/setup-migrations.yml b/ansible-nix/tasks/setup-migrations.yml new file mode 100644 index 000000000..570f7763c --- /dev/null +++ b/ansible-nix/tasks/setup-migrations.yml @@ -0,0 +1,13 @@ +- name: Run migrate.sh script + shell: ./migrate.sh + register: retval + when: ebssurrogate_mode + args: + chdir: /tmp/migrations/db + failed_when: retval.rc != 0 + +- name: Create /root/MIGRATION-AMI file + file: + path: "/root/MIGRATION-AMI" + state: touch + when: ebssurrogate_mode diff --git a/ansible-nix/tasks/setup-nginx.yml b/ansible-nix/tasks/setup-nginx.yml new file mode 100644 index 000000000..77fb7707a --- /dev/null +++ b/ansible-nix/tasks/setup-nginx.yml @@ -0,0 +1,82 @@ +- name: nginx - system user + user: name=nginx + +# Kong installation steps from http://archive.vn/3HRQx +- name: nginx - system dependencies + apt: + pkg: + - openssl + - libpcre3-dev + - libssl-dev + - zlib1g-dev + +- name: nginx - download source + get_url: + url: "https://nginx.org/download/nginx-{{ nginx_release }}.tar.gz" + dest: /tmp/nginx-{{ nginx_release }}.tar.gz + checksum: "{{ nginx_release_checksum }}" + +- name: nginx - unpack archive + unarchive: + remote_src: yes + src: /tmp/nginx-{{ nginx_release }}.tar.gz + dest: /tmp + +- name: nginx - configure + shell: + chdir: /tmp/nginx-{{ nginx_release }} + cmd: | + set -e + + ./configure \ + --prefix=/usr/local/nginx \ + --conf-path=/etc/nginx/nginx.conf \ + --with-http_ssl_module \ + --with-http_realip_module \ + --with-threads + become: yes + +- name: nginx - build + community.general.make: + target: build + chdir: /tmp/nginx-{{ nginx_release }} + jobs: "{{ parallel_jobs | default(omit) }}" + become: yes + +- name: nginx - install + make: + chdir: /tmp/nginx-{{ nginx_release }} + target: install + become: yes + +- name: nginx - hand over ownership of /usr/local/nginx to user nginx + file: + path: /usr/local/nginx + recurse: yes + owner: nginx + +- name: nginx - hand over ownership of /etc/nginx to user nginx + file: + path: /etc/nginx + recurse: yes + owner: nginx + +# [warn] ulimit is currently set to "1024". For better performance set it to at least +# "4096" using "ulimit -n" +- name: nginx - bump up ulimit + pam_limits: + limit_item: nofile + limit_type: soft + domain: nginx + value: "4096" + +- name: nginx - create service file + template: + src: files/nginx.service.j2 + dest: /etc/systemd/system/nginx.service + +# Keep it dormant for the timebeing + +# - name: nginx - reload systemd +# systemd: +# daemon_reload: yes diff --git a/ansible-nix/tasks/setup-pgbouncer.yml b/ansible-nix/tasks/setup-pgbouncer.yml new file mode 100644 index 000000000..eee208961 --- /dev/null +++ b/ansible-nix/tasks/setup-pgbouncer.yml @@ -0,0 +1,140 @@ +# PgBouncer +- name: create ssl-cert group + group: + name: ssl-cert + state: present + +- name: PgBouncer - download & install dependencies + apt: + pkg: + - build-essential + - libssl-dev + - pkg-config + - libevent-dev + - libsystemd-dev + update_cache: yes + cache_valid_time: 3600 + +- name: PgBouncer - download latest release + get_url: + url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" + dest: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz + checksum: "{{ pgbouncer_release_checksum }}" + timeout: 60 + +- name: PgBouncer - unpack archive + unarchive: + remote_src: yes + src: /tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz + dest: /tmp + become: yes + +- name: PgBouncer - configure + shell: + cmd: "./configure --prefix=/usr/local --with-systemd" + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + become: yes + +- name: PgBouncer - build + make: + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + become: yes + +- name: PgBouncer - install + make: + chdir: /tmp/pgbouncer-{{ pgbouncer_release }} + target: install + become: yes + +- name: Create pgbouncer user + user: + name: pgbouncer + shell: /bin/false + comment: PgBouncer user + groups: postgres,ssl-cert + +- name: PgBouncer - create a directory if it does not exist + file: + path: /etc/pgbouncer + state: directory + owner: pgbouncer + group: pgbouncer + mode: '0700' + +- name: PgBouncer - create a directory if it does not exist + file: + state: directory + owner: pgbouncer + group: pgbouncer + path: '{{ item }}' + mode: '0775' + with_items: + - '/etc/pgbouncer-custom' + +- name: create placeholder config files + file: + path: '/etc/pgbouncer-custom/{{ item }}' + state: touch + owner: pgbouncer + group: pgbouncer + mode: 0664 + with_items: + - 'generated-optimizations.ini' + - 'custom-overrides.ini' + - 'ssl-config.ini' + +- name: PgBouncer - adjust pgbouncer.ini + copy: + src: files/pgbouncer_config/pgbouncer.ini.j2 + dest: /etc/pgbouncer/pgbouncer.ini + owner: pgbouncer + mode: '0700' + +- name: PgBouncer - create a directory if it does not exist + file: + path: /etc/pgbouncer/userlist.txt + state: touch + owner: pgbouncer + mode: '0700' + +- name: import /etc/tmpfiles.d/pgbouncer.conf + template: + src: files/pgbouncer_config/tmpfiles.d-pgbouncer.conf.j2 + dest: /etc/tmpfiles.d/pgbouncer.conf + become: yes + +- name: PgBouncer - By default allow ssl connections. + become: yes + copy: + dest: /etc/pgbouncer-custom/ssl-config.ini + content: | + client_tls_sslmode = allow + +- name: Grant pg_hba and pgbouncer grp perm for adminapi updates + shell: | + chmod g+w /etc/postgresql/pg_hba.conf + chmod g+w /etc/pgbouncer-custom/ssl-config.ini + +# Add fail2ban filter +- name: import jail.d/pgbouncer.conf + template: + src: files/fail2ban_config/jail-pgbouncer.conf.j2 + dest: /etc/fail2ban/jail.d/pgbouncer.conf + become: yes + +- name: import filter.d/pgbouncer.conf + template: + src: files/fail2ban_config/filter-pgbouncer.conf.j2 + dest: /etc/fail2ban/filter.d/pgbouncer.conf + become: yes + +# Add systemd file for PgBouncer +- name: PgBouncer - import postgresql.service + template: + src: files/pgbouncer_config/pgbouncer.service.j2 + dest: /etc/systemd/system/pgbouncer.service + become: yes + +- name: PgBouncer - reload systemd + systemd: + daemon_reload: yes diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml new file mode 100644 index 000000000..233e7c798 --- /dev/null +++ b/ansible-nix/tasks/setup-postgres.yml @@ -0,0 +1,133 @@ +# - name: Postgres - install commons +# apt: +# name: postgresql-common +# install_recommends: no + +# - name: Do not create main cluster +# shell: +# cmd: sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf +# +# TODO These lines appear to be installing and configuring https://launchpad.net/ubuntu/+source/postgresql-common +# as far as I can see, we don't need this now. + +- name: create postgres group + group: + name: postgres + state: present + +- name: create users + user: + name: postgres + groups: postgres + createhome: yes + shell: /bin/bash # Set shell if needed, default is /bin/bash + password: '' + +- name: Create relevant directories + file: + path: '{{ item }}' + recurse: yes + state: directory + owner: postgres + group: postgres + with_items: + - '/home/postgres' + - '/var/log/postgresql' + - '/var/lib/postgresql' + +- name: Allow adminapi to write custom config + file: + path: '{{ item }}' + recurse: yes + state: directory + owner: postgres + group: postgres + mode: 0775 + with_items: + - '/etc/postgresql' + - '/etc/postgresql-custom' + +- name: create placeholder config files + file: + path: '/etc/postgresql-custom/{{ item }}' + state: touch + owner: postgres + group: postgres + mode: 0664 + with_items: + - 'generated-optimizations.conf' + - 'custom-overrides.conf' + +- name: locale-gen + command: sudo locale-gen en_US.UTF-8 + +- name: Add LOCALE_ARCHIVE to .bashrc + lineinfile: + dest: "/home/postgres/.bashrc" + line: 'export LOCALE_ARCHIVE=/usr/lib/locale/locale-archive' + create: yes + become: yes + + +- name: Add LANG items to .bashrc + lineinfile: + dest: "/home/postgres/.bashrc" + line: "{{ item }}" + + loop: + - 'export LANG="en_US.UTF-8"' + - 'export LANGUAGE="en_US.UTF-8"' + - 'export LC_ALL="en_US.UTF-8"' + - 'export LANG="en_US.UTF-8"' + - 'export LC_CTYPE="en_US.UTF-8"' + become: yes + + +# Move Postgres configuration files into /etc/postgresql +# Add postgresql.conf +- name: import postgresql.conf + template: + src: files/postgresql_config/postgresql.conf.j2 + dest: /etc/postgresql/postgresql.conf + group: postgres + +# Add pg_hba.conf +- name: import pg_hba.conf + template: + src: files/postgresql_config/pg_hba.conf.j2 + dest: /etc/postgresql/pg_hba.conf + group: postgres + +# Add pg_ident.conf +- name: import pg_ident.conf + template: + src: files/postgresql_config/pg_ident.conf.j2 + dest: /etc/postgresql/pg_ident.conf + group: postgres + +# Add custom config for read replicas set up +- name: Move custom read-replica.conf file to /etc/postgresql-custom/read-replica.conf + template: + src: "files/postgresql_config/custom_read_replica.conf.j2" + dest: /etc/postgresql-custom/read-replica.conf + mode: 0664 + owner: postgres + group: postgres + +# # Install extensions before init +# - name: Install Postgres extensions +# import_tasks: tasks/setup-docker.yml +# TODO resolve in new build + + +# init DB +- name: Create directory on data volume + file: + path: '{{ item }}' + recurse: yes + state: directory + owner: postgres + group: postgres + mode: 0750 + with_items: + - "/data/pgdata" diff --git a/ansible-nix/tasks/setup-postgrest.yml b/ansible-nix/tasks/setup-postgrest.yml new file mode 100644 index 000000000..57b76e1ee --- /dev/null +++ b/ansible-nix/tasks/setup-postgrest.yml @@ -0,0 +1,84 @@ +- name: PostgREST - system user + user: name=postgrest + +# libpq is a C library that enables user programs to communicate with +# the PostgreSQL database server. +- name: PostgREST - system dependencies + apt: + pkg: + - libpq5 + - libnuma-dev + +- name: postgis - ensure dependencies do not get autoremoved + shell: | + set -e + apt-mark manual libnuma* + apt-mark auto libnuma*-dev + +- name: PostgREST - download ubuntu binary archive (arm) + get_url: + url: "https://github.com/PostgREST/postgrest/releases/download/v{{ postgrest_release }}/postgrest-v{{ postgrest_release }}-ubuntu-aarch64.tar.xz" + dest: /tmp/postgrest.tar.xz + checksum: "{{ postgrest_arm_release_checksum }}" + timeout: 60 + when: platform == "arm64" + +- name: PostgREST - download ubuntu binary archive (x86) + get_url: + url: "https://github.com/PostgREST/postgrest/releases/download/v{{ postgrest_release }}/postgrest-v{{ postgrest_release }}-linux-static-x64.tar.xz" + dest: /tmp/postgrest.tar.xz + checksum: "{{ postgrest_x86_release_checksum }}" + timeout: 60 + when: platform == "amd64" + +- name: PostgREST - unpack archive in /opt + unarchive: + remote_src: yes + src: /tmp/postgrest.tar.xz + dest: /opt + owner: postgrest + mode: '0755' + +- name: create directories + file: + state: directory + owner: postgrest + group: postgrest + mode: '0775' + path: /etc/postgrest + +- name: empty files + file: + state: touch + owner: postgrest + group: postgrest + path: /etc/postgrest/{{ item }} + with_items: + - base.conf + - generated.conf + +- name: create conf merging script + copy: + content: | + #! /usr/bin/env bash + set -euo pipefail + set -x + + cd "$(dirname "$0")" + cat $@ > merged.conf + dest: /etc/postgrest/merge.sh + mode: 0750 + owner: postgrest + group: postgrest + +- name: PostgREST - create service files + template: + src: files/{{ item }}.j2 + dest: /etc/systemd/system/{{ item }} + with_items: + - postgrest.service + - postgrest-optimizations.service + +- name: PostgREST - reload systemd + systemd: + daemon_reload: yes diff --git a/ansible-nix/tasks/setup-supabase-internal.yml b/ansible-nix/tasks/setup-supabase-internal.yml new file mode 100644 index 000000000..849c76954 --- /dev/null +++ b/ansible-nix/tasks/setup-supabase-internal.yml @@ -0,0 +1,108 @@ +- name: AWS CLI dep + apt: + pkg: + - unzip + - jq + install_recommends: no + +- name: AWS CLI (arm) + get_url: + url: "https://awscli.amazonaws.com/awscli-exe-linux-aarch64-{{ aws_cli_release }}.zip" + dest: "/tmp/awscliv2.zip" + timeout: 60 + when: platform == "arm64" + +- name: AWS CLI (x86) + get_url: + url: "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-{{ aws_cli_release }}.zip" + dest: "/tmp/awscliv2.zip" + timeout: 60 + when: platform == "amd64" + +- name: AWS CLI - expand + unarchive: + remote_src: yes + src: "/tmp/awscliv2.zip" + dest: "/tmp" + +- name: AWS CLI - install + shell: "/tmp/aws/install --update" + become: true + +- name: AWS CLI - configure ipv6 support for s3 + shell: | + aws configure set default.s3.use_dualstack_endpoint true + +- name: install Vector for logging + become: yes + apt: + deb: "{{ vector_x86_deb }}" + when: platform == "amd64" + +- name: install Vector for logging + become: yes + apt: + deb: "{{ vector_arm_deb }}" + when: platform == "arm64" + +- name: add Vector to postgres group + become: yes + shell: + cmd: | + usermod -a -G postgres vector + +- name: create service files for Vector + template: + src: files/vector.service.j2 + dest: /etc/systemd/system/vector.service + +- name: configure tmpfiles for postgres - overwrites upstream package + template: + src: files/postgresql_config/tmpfiles.postgresql.conf + dest: /etc/tmpfiles.d/postgresql-common.conf + +- name: fix permissions for vector config to be managed + shell: + cmd: | + chown -R vector:vector /etc/vector + chmod 0775 /etc/vector + +- name: vector - reload systemd + systemd: + daemon_reload: yes + +- name: Create checkpoints dir + become: yes + file: + path: /var/lib/vector + state: directory + owner: vector + +- name: Include file for generated optimizations in postgresql.conf + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: "#include = '/etc/postgresql-custom/generated-optimizations.conf'" + replace: "include = '/etc/postgresql-custom/generated-optimizations.conf'" + +- name: Include file for custom overrides in postgresql.conf + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: "#include = '/etc/postgresql-custom/custom-overrides.conf'" + replace: "include = '/etc/postgresql-custom/custom-overrides.conf'" + +- name: Install Postgres exporter + import_tasks: internal/postgres-exporter.yml + +- name: Install admin-mgr + import_tasks: internal/admin-mgr.yml + +- name: Install adminapi + import_tasks: internal/admin-api.yml + +- name: Init nftabless + import_tasks: internal/setup-nftables.yml + +- name: Install pg_egress_collect + import_tasks: internal/pg_egress_collect.yml diff --git a/ansible-nix/tasks/setup-system.yml b/ansible-nix/tasks/setup-system.yml new file mode 100644 index 000000000..860d75cc2 --- /dev/null +++ b/ansible-nix/tasks/setup-system.yml @@ -0,0 +1,154 @@ +- name: System - apt update and apt upgrade + apt: update_cache=yes upgrade=yes + when: not ebssurrogate_mode + # SEE http://archive.vn/DKJjs#parameter-upgrade + +- name: Install required security updates + apt: + pkg: + - tzdata + - linux-libc-dev + +# SEE https://github.com/georchestra/ansible/issues/55#issuecomment-588313638 +# Without this, a similar error is faced +- name: Install Ansible dependencies + apt: + pkg: + - acl + +- name: Install security tools + apt: + pkg: + - nftables + - fail2ban + update_cache: yes + cache_valid_time: 3600 + +- name: Use nftables backend + shell: | + update-alternatives --set iptables /usr/sbin/iptables-nft + update-alternatives --set ip6tables /usr/sbin/ip6tables-nft + update-alternatives --set arptables /usr/sbin/arptables-nft + update-alternatives --set ebtables /usr/sbin/ebtables-nft + systemctl restart ufw + +- name: Create Sysstat log directory + file: + path: /var/log/sysstat + state: directory + +- name: Install other useful tools + apt: + pkg: + - bwm-ng + - htop + - net-tools + - ngrep + - sysstat + - vim-tiny + update_cache: yes + +- name: Configure sysstat + copy: + src: files/sysstat.sysstat + dest: /etc/sysstat/sysstat + +- name: Configure default sysstat + copy: + src: files/default.sysstat + dest: /etc/default/sysstat + + +- name: Adjust APT update intervals + copy: + src: files/apt_periodic + dest: /etc/apt/apt.conf.d/10periodic + +# Find platform architecture and set as a variable +- name: finding platform architecture + shell: if [ $(uname -m) = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi + register: platform_output + tags: + - update + - update-only +- set_fact: + platform: "{{ platform_output.stdout }}" + tags: + - update + - update-only + +- name: create overrides dir + file: + state: directory + owner: root + group: root + path: /etc/systemd/system/systemd-resolved.service.d + mode: '0700' + +- name: Custom systemd overrides for resolved + copy: + src: files/systemd-resolved.conf + dest: /etc/systemd/system/systemd-resolved.service.d/override.conf + +- name: System - Create services.slice + template: + src: files/services.slice.j2 + dest: /etc/systemd/system/services.slice + when: not ebssurrogate_mode + +- name: System - systemd reload + systemd: daemon_reload=yes + +- name: Configure journald + copy: + src: files/journald.conf + dest: /etc/systemd/journald.conf + +- name: reload systemd-journald + systemd: + name: systemd-journald + state: restarted + +- name: Configure logind + copy: + src: files/logind.conf + dest: /etc/systemd/logind.conf + +- name: reload systemd-logind + systemd: + name: systemd-logind + state: restarted + +- name: enable timestamps for shell history + copy: + content: | + export HISTTIMEFORMAT='%d/%m/%y %T ' + dest: /etc/profile.d/09-history-timestamps.sh + mode: 0644 + owner: root + group: root + +- name: set hosts file + copy: + content: | + 127.0.0.1 localhost + ::1 localhost + dest: /etc/hosts + mode: 0644 + owner: root + group: root + +#Set Sysctl params for restarting the OS on oom after 10 +- name: Set vm.panic_on_oom=1 + ansible.builtin.sysctl: + name: vm.panic_on_oom + value: '1' + state: present + reload: yes + +- name: Set kernel.panic=10 + ansible.builtin.sysctl: + name: kernel.panic + value: '10' + state: present + reload: yes diff --git a/ansible-nix/tasks/setup-wal-g.yml b/ansible-nix/tasks/setup-wal-g.yml new file mode 100644 index 000000000..bbc64cdde --- /dev/null +++ b/ansible-nix/tasks/setup-wal-g.yml @@ -0,0 +1,130 @@ +# Downloading dependencies +- name: wal-g dependencies + become: yes + apt: + pkg: + - libbrotli-dev + - liblzo2-dev + - libsodium-dev + - cmake + +# install go dependency for WAL-G +- name: wal-g go dependency + get_url: + url: "https://golang.org/dl/go{{ golang_version }}.linux-{{ platform }}.tar.gz" + dest: /tmp + checksum: "{{ golang_version_checksum[platform] }}" + timeout: 60 + +- name: unpack go archive + unarchive: + remote_src: yes + src: "/tmp/go{{ golang_version }}.linux-{{ platform }}.tar.gz" + dest: /usr/local + +# Download WAL-G +- name: wal-g - download latest version + git: + repo: https://github.com/wal-g/wal-g.git + dest: /tmp/wal-g + version: "v{{ wal_g_release }}" + become: yes + +- name: wal-g - pg_clean + make: + chdir: /tmp/wal-g + target: pg_clean + params: + GOBIN: "/usr/local/bin" + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + USE_LIBSODIUM: true + become: yes + ignore_errors: yes + +- name: wal-g - deps + make: + chdir: /tmp/wal-g + target: deps + params: + GOBIN: "/usr/local/bin" + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + USE_LIBSODIUM: true + become: yes + ignore_errors: yes + +- name: wal-g - build and install + community.general.make: + chdir: /tmp/wal-g + target: pg_install + jobs: "{{ parallel_jobs | default(omit) }}" + params: + GOBIN: "/usr/local/bin" + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + USE_LIBSODIUM: true + become: yes + +- name: Create wal-g group + group: + name: wal-g + state: present + +- name: Create wal-g user + user: + name: wal-g + shell: /bin/false + comment: WAL-G user + group: wal-g + groups: wal-g, postgres + +- name: Create a config directory owned by wal-g + file: + path: /etc/wal-g + state: directory + owner: wal-g + group: wal-g + mode: '0770' + +- name: Create /etc/wal-g/config.json + file: + path: /etc/wal-g/config.json + state: touch + owner: wal-g + group: wal-g + mode: '0664' + +- name: Move custom wal-g.conf file to /etc/postgresql-custom/wal-g.conf + template: + src: "files/postgresql_config/custom_walg.conf.j2" + dest: /etc/postgresql-custom/wal-g.conf + mode: 0664 + owner: postgres + group: postgres + +- name: Add script to be run for restore_command + template: + src: "files/walg_helper_scripts/wal_fetch.sh" + dest: /home/postgres/wal_fetch.sh + mode: 0500 + owner: postgres + group: postgres + +- name: Add helper script for wal_fetch.sh + template: + src: "files/walg_helper_scripts/wal_change_ownership.sh" + dest: /root/wal_change_ownership.sh + mode: 0700 + owner: root + +- name: Include /etc/postgresql-custom/wal-g.conf in postgresql.conf + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: "#include = '/etc/postgresql-custom/wal-g.conf'" + replace: "include = '/etc/postgresql-custom/wal-g.conf'" + +# Clean up Go +- name: Uninstall Go + become: yes + file: + path: /usr/local/go + state: absent diff --git a/ansible/tasks/stage2/optimizations.yml b/ansible-nix/tasks/stage2/optimizations.yml similarity index 100% rename from ansible/tasks/stage2/optimizations.yml rename to ansible-nix/tasks/stage2/optimizations.yml diff --git a/ansible/tasks/stage2/playbook.yml b/ansible-nix/tasks/stage2/playbook.yml similarity index 100% rename from ansible/tasks/stage2/playbook.yml rename to ansible-nix/tasks/stage2/playbook.yml diff --git a/ansible/tasks/stage2/setup-extensions.yml b/ansible-nix/tasks/stage2/setup-extensions.yml similarity index 100% rename from ansible/tasks/stage2/setup-extensions.yml rename to ansible-nix/tasks/stage2/setup-extensions.yml diff --git a/ansible/tasks/stage2/setup-migrations.yml b/ansible-nix/tasks/stage2/setup-migrations.yml similarity index 100% rename from ansible/tasks/stage2/setup-migrations.yml rename to ansible-nix/tasks/stage2/setup-migrations.yml diff --git a/ansible/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml similarity index 100% rename from ansible/tasks/stage2/stage2-setup-postgres.yml rename to ansible-nix/tasks/stage2/stage2-setup-postgres.yml diff --git a/ansible/tasks/stage2/test-image.yml b/ansible-nix/tasks/stage2/test-image.yml similarity index 100% rename from ansible/tasks/stage2/test-image.yml rename to ansible-nix/tasks/stage2/test-image.yml diff --git a/ansible-nix/tasks/test-image.yml b/ansible-nix/tasks/test-image.yml new file mode 100644 index 000000000..cad503a39 --- /dev/null +++ b/ansible-nix/tasks/test-image.yml @@ -0,0 +1,50 @@ +- name: install pg_prove + apt: + pkg: + - libtap-parser-sourcehandler-pgtap-perl + +- name: Temporarily disable PG Sodium references in config + become: yes + become_user: postgres + shell: + cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf + +- name: Start Postgres Database to load all extensions. + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start "-o -c config_file=/etc/postgresql/postgresql.conf" + +- name: Run Unit tests (with filename unit-test-*) on Postgres Database + shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql + register: retval + failed_when: retval.rc != 0 + +- name: Run migrations tests + shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql + register: retval + failed_when: retval.rc != 0 + args: + chdir: /tmp/migrations + +- name: Re-enable PG Sodium references in config + become: yes + become_user: postgres + shell: + cmd: mv /etc/postgresql/postgresql.conf.bak /etc/postgresql/postgresql.conf + +- name: Reset db stats + shell: /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();' + +- name: remove pg_prove + apt: + pkg: + - libtap-parser-sourcehandler-pgtap-perl + state: absent + autoremove: yes + +- name: Stop Postgres Database + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop diff --git a/ansible-nix/vars.yml b/ansible-nix/vars.yml new file mode 100644 index 000000000..dcb11dbab --- /dev/null +++ b/ansible-nix/vars.yml @@ -0,0 +1,145 @@ +supabase_internal: true +ebssurrogate_mode: true +async_mode: true + +postgresql_major: "15" +postgresql_release: "15.1" +postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c + +# Non Postgres Extensions +pgbouncer_release: "1.19.0" +pgbouncer_release_checksum: sha256:af0b05e97d0e1fd9ad45fe00ea6d2a934c63075f67f7e2ccef2ca59e3d8ce682 + +# to get these use `wget https://github.com/PostgREST/postgrest/releases/download/v12.0.2/postgrest-v12.0.2-ubuntu-aarch64.tar.xz -q -O- | sha1sum` +postgrest_release: "12.0.2" +postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 +postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 + +gotrue_release: 2.148.0 +gotrue_release_checksum: sha1:e3ab39c10605b6a00a953528022b563c8fa1ca33 + +aws_cli_release: "2.2.7" + +golang_version: "1.19.3" +golang_version_checksum: + arm64: sha256:99de2fe112a52ab748fb175edea64b313a0c8d51d6157dba683a6be163fd5eab + amd64: sha256:74b9640724fd4e6bb0ed2a1bc44ae813a03f1e72a4c76253e2d5c015494430ba + +envoy_release: 1.28.0 +envoy_release_checksum: sha1:b0a06e9cfb170f1993f369beaa5aa9d7ec679ce5 +envoy_hot_restarter_release_checksum: sha1:6d43b89d266fb2427a4b51756b649883b0617eda + +kong_release_target: focal # if it works, it works +kong_deb: kong_2.8.1_arm64.deb +kong_deb_checksum: sha1:2086f6ccf8454fe64435252fea4d29d736d7ec61 + +nginx_release: 1.22.0 +nginx_release_checksum: sha1:419efb77b80f165666e2ee406ad8ae9b845aba93 + +wal_g_release: "2.0.1" + +sfcgal_release: "1.3.10" +sfcgal_release_checksum: sha256:4e39b3b2adada6254a7bdba6d297bb28e1a9835a9f879b74f37e2dab70203232 + +postgres_exporter_release: "0.15.0" +postgres_exporter_release_checksum: + arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0 + amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84 + +adminapi_release: 0.63.5 +adminmgr_release: 0.19.0 + +# Postgres Extensions +postgis_release: "3.3.2" +postgis_release_checksum: sha256:9a2a219da005a1730a39d1959a1c7cec619b1efb009b65be80ffc25bad299068 + +pgrouting_release: "3.4.1" +pgrouting_release_checksum: sha256:a4e034efee8cf67582b67033d9c3ff714a09d8f5425339624879df50aff3f642 + +pgtap_release: "1.2.0" +pgtap_release_checksum: sha256:9c7c3de67ea41638e14f06da5da57bac6f5bd03fea05c165a0ec862205a5c052 + +pg_cron_release: "1.6.2" +pg_cron_release_checksum: sha256:9f4eb3193733c6fa93a6591406659aac54b82c24a5d91ffaf4ec243f717d94a0 + +pgaudit_release: "1.7.0" +pgaudit_release_checksum: sha256:8f4a73e451c88c567e516e6cba7dc1e23bc91686bb6f1f77f8f3126d428a8bd8 + +pgjwt_release: 9742dab1b2f297ad3811120db7b21451bca2d3c9 + +pgsql_http_release: "1.5.0" +pgsql_http_release_checksum: sha256:43efc9e82afcd110f205b86b8d28d1355d39b6b134161e9661a33a1346818f5d + +plpgsql_check_release: "2.2.5" +plpgsql_check_release_checksum: sha256:6c3a3c5faf3f9689425c6db8a6b20bf4cd5e7144a055e29538eae980c7232573 + +pg_safeupdate_release: "1.4" +pg_safeupdate_release_checksum: sha256:ff01d3d444d35924bd3d745c5695696292e2855042da4c30fe728fb3b6648122 + +timescaledb_release: "2.9.1" +timescaledb_release_checksum: sha256:883638f2e79d25ec88ee58f603f3c81c999b6364cb4c799919d363f04089b47b + +wal2json_release: "2_5" +wal2json_release_checksum: sha256:b516653575541cf221b99cf3f8be9b6821f6dbcfc125675c85f35090f824f00e + +supautils_release: "2.2.0" +supautils_release_checksum: sha256:0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5 + +pljava_release: master +pljava_release_checksum: sha256:e99b1c52f7b57f64c8986fe6ea4a6cc09d78e779c1643db060d0ac66c93be8b6 + +plv8_release: "3.1.5" +plv8_release_checksum: sha256:2edf9a219844b2b6abae09c0bdb840c5b0d6e3dd418631744c7326c0b107cc10 + +pg_plan_filter_release: 5081a7b5cb890876e67d8e7486b6a64c38c9a492 + +pg_net_release: "0.8.0" +# To obtain the checksum use `wget https://github.com/supabase/pg_net/archive/refs/tags/v0.8.0.tar.gz -q -O- | sha256sum` +pg_net_release_checksum: sha256:3a50845fdb2d41b0a1053e9e006d068638515b53efd1ad38a2c26fdf4935ce4e + +rum_release: "1.3.13" +rum_release_checksum: sha256:6ab370532c965568df6210bd844ac6ba649f53055e48243525b0b7e5c4d69a7d + +pg_hashids_release: cd0e1b31d52b394a0df64079406a14a4f7387cd6 + +vector_x86_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_amd64.deb" +vector_arm_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_arm64.deb" + +libsodium_release: "1.0.18" +libsodium_release_checksum: sha256:6f504490b342a4f8a4c4a02fc9b866cbef8622d5df4e5452b46be121e46636c1 + +pgsodium_release: "3.1.8" +pgsodium_release_checksum: sha256:4d027aeee5163f3f33740d269938a120d1593a41c3701c920d2a1de80aa97486 + +pg_graphql_release: "1.5.1" + +pg_jsonschema_release: "0.2.0" + +pg_stat_monitor_release: "1.1.1" +pg_stat_monitor_release_checksum: sha256:1756a02d5a6dd66b892d15920257c69a17a67d48d3d4e2f189b681b83001ec2a + +vault_release: "0.2.9" +vault_release_checksum: sha256:1e813216395c59bb94c92be47ce8b70ba19ccc0efbcdb1fb14ed6d34a42c6cdb + +groonga_release: "13.0.1" +groonga_release_checksum: sha256:1c2d1a6981c1ad3f02a11aff202b15ba30cb1c6147f1fa9195b519a2b728f8ba + +pgroonga_release: "3.0.7" +pgroonga_release_checksum: sha256:885ff3878cc30e9030e5fc56d561bc8b66df3ede1562c9d802bc0ea04fe5c203 + +wrappers_release: "0.3.0" + +hypopg_release: "1.3.1" +hypopg_release_checksum: sha256:e7f01ee0259dc1713f318a108f987663d60f3041948c2ada57a94b469565ca8e + +pg_repack_release: "1.5.0" +pg_repack_release_checksum: sha256:9a14d6a95bfa29f856aa10538238622c1f351d38eb350b196c06720a878ccc52 + +pgvector_release: "0.6.2" +pgvector_release_checksum: sha256:a11cc249a9f3f3d7b13069a1696f2915ac28991a72d7ba4e2bcfdceddbaeae49 + +pg_tle_release: "1.3.2" +pg_tle_release_checksum: sha256:d04f72d88b21b954656609743560684ac42645b64a36c800d4d2f84d1f180de1 + +index_advisor_release: "0.2.0" +index_advisor_checksum: sha256:2d3642012a9185cda51f1e82ba43d64a81b24a2655a3ac3afdcbbd95d46a1a27 diff --git a/ansible/playbook.yml b/ansible/playbook.yml index e23f9652f..5aef718f0 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -27,7 +27,7 @@ - set_fact: parallel_jobs: 16 - - name: Prepare machine for Postgres installation + - name: Install Postgres from source import_tasks: tasks/setup-postgres.yml - name: Install PgBouncer @@ -71,11 +71,48 @@ tags: - install-supabase-internal + - name: Start Postgres Database + systemd: + name: postgresql + state: started + when: not ebssurrogate_mode + + - name: Start Postgres Database without Systemd + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start + when: ebssurrogate_mode + - name: Adjust APT update intervals copy: src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic + - name: Transfer init SQL files + copy: + src: files/{{ item.source }} + dest: /tmp/{{ item.dest }} + loop: "{{ sql_files }}" + + - name: Execute init SQL files + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/psql -f /tmp/{{ item.dest }} + loop: "{{ sql_files }}" + + - name: Delete SQL scripts + file: + path: /tmp/{{ item.dest }} + state: absent + loop: "{{ sql_files }}" + + - name: First boot optimizations + import_tasks: tasks/internal/optimizations.yml + tags: + - install-supabase-internal + - name: Finalize AMI import_tasks: tasks/finalize-ami.yml tags: @@ -105,3 +142,32 @@ - name: Clean out build dependencies import_tasks: tasks/clean-build-dependencies.yml + + - name: Restart Postgres Database without Systemd + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data restart -o "-c shared_preload_libraries='pg_tle'" + when: ebssurrogate_mode + + - name: Run migrations + import_tasks: tasks/setup-migrations.yml + tags: + - migrations + + - name: Stop Postgres Database without Systemd + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop + when: ebssurrogate_mode + + - name: Run unit tests + import_tasks: tasks/test-image.yml + tags: + - unit-tests + + - name: Collect Postgres binaries + import_tasks: tasks/internal/collect-pg-binaries.yml + tags: + - collect-binaries diff --git a/ansible/tasks/finalize-ami.yml b/ansible/tasks/finalize-ami.yml index 81a89abeb..9818f2add 100644 --- a/ansible/tasks/finalize-ami.yml +++ b/ansible/tasks/finalize-ami.yml @@ -33,7 +33,7 @@ rule: allow port: https tags: - - install-supabase-internal + - install-supabase-internal - name: UFW - Deny all other incoming traffic by default ufw: @@ -70,3 +70,11 @@ sed -i -e 's;daily;*:0/5;' /etc/systemd/system/logrotate.timer systemctl reenable logrotate.timer become: yes + +- name: import pgsodium_getkey script + template: + src: files/pgsodium_getkey_readonly.sh.j2 + dest: "{{ pg_bindir }}/pgsodium_getkey.sh" + owner: postgres + group: postgres + mode: 0700 diff --git a/ansible/tasks/internal/collect-pg-binaries.yml b/ansible/tasks/internal/collect-pg-binaries.yml new file mode 100644 index 000000000..7f652f740 --- /dev/null +++ b/ansible/tasks/internal/collect-pg-binaries.yml @@ -0,0 +1,49 @@ +- name: Collect Postgres binaries - create collection directory + file: + path: /tmp/pg_binaries/{{ postgresql_major }}/ + state: directory + +- name: Collect Postgres binaries - collect binaries and libraries + copy: + remote_src: yes + src: /usr/lib/postgresql/{{ postgresql_major }}/{{ item }}/ + dest: /tmp/pg_binaries/{{ postgresql_major }}/{{ item }}/ + with_items: + - bin + - lib + +- name: Collect Postgres libraries - collect libraries which are in /usr/lib/postgresql/lib/ + copy: + remote_src: yes + src: /usr/lib/postgresql/lib/ + dest: /tmp/pg_binaries/{{ postgresql_major }}/lib/ + +- name: Collect Postgres libraries - collect libraries which are in /var/lib/postgresql/extension/ + copy: + remote_src: yes + src: /var/lib/postgresql/extension/ + dest: /tmp/pg_binaries/{{ postgresql_major }}/lib/ + +- name: Collect Postgres libraries - collect latest libpq + copy: + remote_src: yes + src: /usr/lib/aarch64-linux-gnu/libpq.so.5 + dest: /tmp/pg_binaries/{{ postgresql_major }}/lib/libpq.so.5 + +- name: Collect Postgres binaries - collect shared files + copy: + remote_src: yes + src: /usr/share/postgresql/{{ postgresql_major }}/ + dest: /tmp/pg_binaries/{{ postgresql_major }}/share/ + +- name: Collect Postgres binaries - create tarfile + archive: + path: /tmp/pg_binaries/ + dest: /tmp/pg_binaries.tar.gz + remove: yes + +- name: Fetch tarfile to local + fetch: + src: /tmp/pg_binaries.tar.gz + dest: /tmp/ + flat: true diff --git a/ansible/tasks/internal/optimizations.yml b/ansible/tasks/internal/optimizations.yml new file mode 100644 index 000000000..895acccd9 --- /dev/null +++ b/ansible/tasks/internal/optimizations.yml @@ -0,0 +1,47 @@ +- name: ensure services are stopped + community.general.snap: + name: amazon-ssm-agent + state: absent + failed_when: not ebssurrogate_mode + +- name: ensure services are stopped and disabled for first boot + systemd: + enabled: no + name: '{{ item }}' + state: stopped + with_items: + - snapd + - postgresql + - pgbouncer + - fail2ban + - motd-news + - vector + failed_when: not ebssurrogate_mode + +- name: Remove snapd + apt: + state: absent + pkg: + - snapd + failed_when: not ebssurrogate_mode + +- name: ensure services are stopped and disabled for first boot + systemd: + enabled: no + name: '{{ item }}' + state: stopped + masked: yes + with_items: + - lvm2-monitor + failed_when: not ebssurrogate_mode + +- name: disable man-db + become: yes + file: + state: absent + path: "/etc/cron.daily/{{ item }}" + with_items: + - man-db + - popularity-contest + - ubuntu-advantage-tools + failed_when: not ebssurrogate_mode diff --git a/ansible/tasks/setup-docker.yml b/ansible/tasks/setup-docker.yml new file mode 100644 index 000000000..c2a56cc30 --- /dev/null +++ b/ansible/tasks/setup-docker.yml @@ -0,0 +1,82 @@ +- name: Copy extension packages + copy: + src: files/extensions/ + dest: /tmp/extensions/ + +# Builtin apt module does not support wildcard for deb paths +- name: Install extensions + shell: | + set -e + apt-get update + apt-get install -y --no-install-recommends /tmp/extensions/*.deb + +- name: pg_cron - set cron.database_name + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: cron.database_name = 'postgres' + +- name: pgsodium - determine postgres bin directory + shell: pg_config --bindir + register: pg_bindir_output +- set_fact: + pg_bindir: "{{ pg_bindir_output.stdout }}" + +- name: pgsodium - set pgsodium.getkey_script + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + # script is expected to be placed by finalization tasks for different target platforms + line: pgsodium.getkey_script= '{{ pg_bindir }}/pgsodium_getkey.sh' + +- name: auto_explain - set auto_explain.log_min_duration + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: auto_explain.log_min_duration = 10s + +# supautils +- name: supautils - add supautils to session_preload_libraries + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: "#session_preload_libraries = ''" + replace: session_preload_libraries = 'supautils' + +- name: supautils - write custom supautils.conf + template: + src: "files/postgresql_config/supautils.conf.j2" + dest: /etc/postgresql-custom/supautils.conf + mode: 0664 + owner: postgres + group: postgres + +- name: supautils - copy extension custom scripts + copy: + src: files/postgresql_extension_custom_scripts/ + dest: /etc/postgresql-custom/extension-custom-scripts + become: yes + +- name: supautils - chown extension custom scripts + file: + mode: 0775 + owner: postgres + group: postgres + path: /etc/postgresql-custom/extension-custom-scripts + recurse: yes + become: yes + +- name: supautils - include /etc/postgresql-custom/supautils.conf in postgresql.conf + become: yes + replace: + path: /etc/postgresql/postgresql.conf + regexp: "#include = '/etc/postgresql-custom/supautils.conf'" + replace: "include = '/etc/postgresql-custom/supautils.conf'" + +- name: Cleanup - extension packages + file: + path: /tmp/extensions + state: absent diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml index 0fb92c151..dc1b759ec 100644 --- a/ansible/tasks/setup-extensions.yml +++ b/ansible/tasks/setup-extensions.yml @@ -80,7 +80,7 @@ - name: Install hypopg import_tasks: tasks/postgres-extensions/26-hypopg.yml -- name: Install pg_repack + - name: Install pg_repack import_tasks: tasks/postgres-extensions/27-pg_repack.yml - name: Install pgvector diff --git a/ansible/tasks/setup-gotrue.yml b/ansible/tasks/setup-gotrue.yml index 2a70caa4d..e87034bcb 100644 --- a/ansible/tasks/setup-gotrue.yml +++ b/ansible/tasks/setup-gotrue.yml @@ -39,10 +39,10 @@ # libpq is a C library that enables user programs to communicate with # the PostgreSQL database server. -- name: gotrue - system dependencies - apt: - pkg: - - libpq-dev +# - name: gotrue - system dependencies +# apt: +# pkg: +# - libpq-dev - name: gotrue - create service file template: diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index eee208961..4381ba24d 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -1,9 +1,4 @@ # PgBouncer -- name: create ssl-cert group - group: - name: ssl-cert - state: present - - name: PgBouncer - download & install dependencies apt: pkg: diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 233e7c798..c84142928 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -1,27 +1,40 @@ -# - name: Postgres - install commons -# apt: -# name: postgresql-common -# install_recommends: no - -# - name: Do not create main cluster -# shell: -# cmd: sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf -# -# TODO These lines appear to be installing and configuring https://launchpad.net/ubuntu/+source/postgresql-common -# as far as I can see, we don't need this now. - -- name: create postgres group - group: - name: postgres +- name: Postgres - copy package + copy: + src: files/postgres/ + dest: /tmp/build/ + +- name: Postgres - add PPA + apt_repository: + repo: "deb [ trusted=yes ] file:///tmp/build ./" state: present -- name: create users - user: - name: postgres - groups: postgres - createhome: yes - shell: /bin/bash # Set shell if needed, default is /bin/bash - password: '' +- name: Postgres - install commons + apt: + name: postgresql-common + install_recommends: no + +- name: Do not create main cluster + shell: + cmd: sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf + +- name: Postgres - install server + apt: + name: postgresql-{{ postgresql_major }}={{ postgresql_release }}-1.pgdg20.04+1 + install_recommends: no + +- name: Postgres - remove PPA + apt_repository: + repo: "deb [ trusted=yes ] file:///tmp/build ./" + state: absent + +- name: Postgres - cleanup package + file: + path: /tmp/build + state: absent + +- name: Create symlink to /usr/lib/postgresql/bin + shell: + cmd: ln -s /usr/lib/postgresql/{{ postgresql_major }}/bin /usr/lib/postgresql/bin - name: Create relevant directories file: @@ -58,31 +71,6 @@ - 'generated-optimizations.conf' - 'custom-overrides.conf' -- name: locale-gen - command: sudo locale-gen en_US.UTF-8 - -- name: Add LOCALE_ARCHIVE to .bashrc - lineinfile: - dest: "/home/postgres/.bashrc" - line: 'export LOCALE_ARCHIVE=/usr/lib/locale/locale-archive' - create: yes - become: yes - - -- name: Add LANG items to .bashrc - lineinfile: - dest: "/home/postgres/.bashrc" - line: "{{ item }}" - - loop: - - 'export LANG="en_US.UTF-8"' - - 'export LANGUAGE="en_US.UTF-8"' - - 'export LC_ALL="en_US.UTF-8"' - - 'export LANG="en_US.UTF-8"' - - 'export LC_CTYPE="en_US.UTF-8"' - become: yes - - # Move Postgres configuration files into /etc/postgresql # Add postgresql.conf - name: import postgresql.conf @@ -114,11 +102,9 @@ owner: postgres group: postgres -# # Install extensions before init -# - name: Install Postgres extensions -# import_tasks: tasks/setup-docker.yml -# TODO resolve in new build - +# Install extensions before init +- name: Install Postgres extensions + import_tasks: tasks/setup-docker.yml # init DB - name: Create directory on data volume @@ -131,3 +117,37 @@ mode: 0750 with_items: - "/data/pgdata" + +- name: Link database data_dir to data volume directory + file: + src: "/data/pgdata" + path: "/var/lib/postgresql/data" + state: link + force: yes + +- name: Initialize the database + become: yes + become_user: postgres + shell: + cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" + vars: + ansible_command_timeout: 60 + # Circumvents the following error: + # "Timeout (12s) waiting for privilege escalation prompt" + +- name: copy PG systemd unit + template: + src: files/postgresql_config/postgresql.service.j2 + dest: /etc/systemd/system/postgresql.service + +- name: copy optimizations systemd unit + template: + src: files/database-optimizations.service.j2 + dest: /etc/systemd/system/database-optimizations.service + +# Reload +- name: System - systemd reload + systemd: + enabled: yes + name: postgresql + daemon_reload: yes diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index cad503a39..cd92a27d6 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -8,22 +8,26 @@ become_user: postgres shell: cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf + when: ebssurrogate_mode - name: Start Postgres Database to load all extensions. become: yes become_user: postgres shell: cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start "-o -c config_file=/etc/postgresql/postgresql.conf" + when: ebssurrogate_mode - name: Run Unit tests (with filename unit-test-*) on Postgres Database shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql register: retval failed_when: retval.rc != 0 + when: ebssurrogate_mode - name: Run migrations tests shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql register: retval failed_when: retval.rc != 0 + when: ebssurrogate_mode args: chdir: /tmp/migrations @@ -32,9 +36,11 @@ become_user: postgres shell: cmd: mv /etc/postgresql/postgresql.conf.bak /etc/postgresql/postgresql.conf + when: ebssurrogate_mode - name: Reset db stats shell: /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();' + when: ebssurrogate_mode - name: remove pg_prove apt: @@ -48,3 +54,4 @@ become_user: postgres shell: cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop + when: ebssurrogate_mode diff --git a/ansible/vars.yml b/ansible/vars.yml index dcb11dbab..d499b4fd2 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -15,8 +15,8 @@ postgrest_release: "12.0.2" postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 -gotrue_release: 2.148.0 -gotrue_release_checksum: sha1:e3ab39c10605b6a00a953528022b563c8fa1ca33 +gotrue_release: 2.150.0 +gotrue_release_checksum: sha1:d785c1a4aadcee88738c3e016ac12f9a7c3db858 aws_cli_release: "2.2.7" @@ -129,14 +129,14 @@ pgroonga_release_checksum: sha256:885ff3878cc30e9030e5fc56d561bc8b66df3ede1562c9 wrappers_release: "0.3.0" -hypopg_release: "1.3.1" -hypopg_release_checksum: sha256:e7f01ee0259dc1713f318a108f987663d60f3041948c2ada57a94b469565ca8e +hypopg_release: "1.4.1" +hypopg_release_checksum: sha256:9afe6357fd389d8d33fad81703038ce520b09275ec00153c6c89282bcdedd6bc pg_repack_release: "1.5.0" pg_repack_release_checksum: sha256:9a14d6a95bfa29f856aa10538238622c1f351d38eb350b196c06720a878ccc52 -pgvector_release: "0.6.2" -pgvector_release_checksum: sha256:a11cc249a9f3f3d7b13069a1696f2915ac28991a72d7ba4e2bcfdceddbaeae49 +pgvector_release: "0.7.0" +pgvector_release_checksum: sha256:1b5503a35c265408b6eb282621c5e1e75f7801afc04eecb950796cfee2e3d1d8 pg_tle_release: "1.3.2" pg_tle_release_checksum: sha256:d04f72d88b21b954656609743560684ac42645b64a36c800d4d2f84d1f180de1 diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 8717e8f54..2dcfd19a4 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -57,7 +57,7 @@ build { } provisioner "file" { - source = "ansible/tasks/stage2" + source = "ansible-nix/tasks/stage2" destination = "/tmp/ansible-playbook" } From 971fb6f038491552bc2d96870f6e297e1ef2dd29 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 1 May 2024 15:03:43 -0400 Subject: [PATCH 015/115] chore: remove very old file no longer in use --- digitalOcean.json | 45 --------------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 digitalOcean.json diff --git a/digitalOcean.json b/digitalOcean.json deleted file mode 100644 index 36396fc29..000000000 --- a/digitalOcean.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "variables": { - "do_token": "", - "image_name": "ubuntu-20-04-x64", - "region": "sgp1", - "snapshot_regions": "sgp1", - "snapshot_name": "supabase-postgres-13.3.0", - "ansible_arguments": "--skip-tags,update-only,--skip-tags,aws-only,-e,supabase_internal='false'" - }, - "builders": [ - { - "type": "digitalocean", - "api_token": "{{user `do_token`}}", - "image": "{{user `image_name`}}", - "region": "{{user `region`}}", - "snapshot_regions": "{{user `snapshot_regions`}}", - "size": "s-1vcpu-1gb", - "ssh_username": "root", - "snapshot_name": "{{user `snapshot_name`}}" - } - ], - "provisioners": [ - { - "type": "shell", - "inline": [ - "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done" - ] - }, - { - "type": "ansible", - "user": "root", - "playbook_file": "ansible/playbook.yml", - "extra_arguments": "{{user `ansible_arguments`}}" - }, - { - "type": "shell", - "scripts": [ - "scripts/01-postgres_check.sh", - "scripts/90-cleanup.sh", - "scripts/91-log_cleanup.sh", - "scripts/99-img_check.sh" - ] - } - ] -} From 5c46f972bb6f1b6781d8ae73eba7fe11522187cc Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 1 May 2024 15:08:35 -0400 Subject: [PATCH 016/115] chore: remove deprecated workflow --- .github/workflows/nix-cache-upload.yml | 52 -------------------------- 1 file changed, 52 deletions(-) delete mode 100644 .github/workflows/nix-cache-upload.yml diff --git a/.github/workflows/nix-cache-upload.yml b/.github/workflows/nix-cache-upload.yml deleted file mode 100644 index ea7f628e9..000000000 --- a/.github/workflows/nix-cache-upload.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Nix Cache upload - -on: - push: - branches: - - main - -permissions: - contents: write - packages: write - id-token: write - -jobs: - build: - strategy: - fail-fast: false - matrix: - include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 - runs-on: ${{ matrix.runner }} - name: nix-build - steps: - - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 - with: - fetch-depth: 0 - - uses: DeterminateSystems/nix-installer-action@65d7c888b2778e8cf30a07a88422ccb23499bfb8 - - uses: DeterminateSystems/magic-nix-cache-action@749fc5bbc9fa49d60c2b93f6c4bc867b82e1d295 - - name: configure aws credentials for s3 - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" - kvm: true - extra-conf: | - system-features = kvm - - - name: write secret key - # use python so we don't interpolate the secret into the workflow logs, in case of bugs - run: | - python -c "import os; file = open('nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()" - env: - NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - - name: build and copy to S3 - run: | - for x in 15 16 orioledb_16; do - nix build .#psql_$x/bin -o result-$x - done - nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./result* From c15b960f9ce1977af23dd456ad42ce6ed151a2ea Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 2 May 2024 19:47:23 -0400 Subject: [PATCH 017/115] feat: clean up and attach stages in packer --- .github/workflows/ami-release-nix.yml | 169 ++++++++ .github/workflows/text-nix.yml | 117 ++++++ .github/workflows/textinfra-nix.yml | 145 +++++++ amazon-arm64-nix.pkr.hcl | 6 +- common-nix.vars.pkr.hcl | 1 + docker/all-in-one-nix/Dockerfile | 293 ++++++++++++++ docker/all-in-one-nix/README.md | 59 +++ docker/all-in-one-nix/configure-shim.sh | 16 + docker/all-in-one-nix/entrypoint.sh | 304 +++++++++++++++ .../all-in-one-nix/etc/adminapi/adminapi.yaml | 76 ++++ .../etc/fail2ban/filter.d/pgbouncer.conf | 2 + .../etc/fail2ban/filter.d/postgresql.conf | 8 + .../etc/fail2ban/jail.d/jail.local | 4 + .../etc/fail2ban/jail.d/pgbouncer.conf | 7 + .../etc/fail2ban/jail.d/postgresql.conf | 8 + .../etc/fail2ban/jail.d/sshd.local | 3 + docker/all-in-one-nix/etc/gotrue.env | 9 + docker/all-in-one-nix/etc/kong/kong.conf | 35 ++ docker/all-in-one-nix/etc/kong/kong.yml | 88 +++++ .../etc/logrotate.d/postgresql.conf | 11 + .../all-in-one-nix/etc/logrotate.d/walg.conf | 9 + .../etc/pgbouncer-custom/custom-overrides.ini | 0 .../generated-optimizations.ini | 0 .../etc/pgbouncer-custom/ssl-config.ini | 4 + .../etc/pgbouncer/pgbouncer.ini | 363 ++++++++++++++++++ .../all-in-one-nix/etc/pgbouncer/userlist.txt | 0 .../postgresql-custom/custom-overrides.conf | 0 .../generated-optimizations.conf | 0 .../postgresql-platform-defaults.conf | 9 + .../all-in-one-nix/etc/postgresql.schema.sql | 16 + .../etc/postgresql/logging.conf | 33 ++ .../all-in-one-nix/etc/postgresql/pg_hba.conf | 94 +++++ docker/all-in-one-nix/etc/postgrest/base.conf | 7 + .../all-in-one-nix/etc/postgrest/bootstrap.sh | 8 + .../etc/postgrest/generated.conf | 0 docker/all-in-one-nix/etc/sudoers.d/adminapi | 27 ++ .../etc/supa-shutdown/shutdown.conf | 1 + .../supervisor/base-services/adminapi.conf | 9 + .../supervisor/base-services/logrotate.conf | 10 + .../base-services/lsn-checkpoint-push.conf | 9 + .../base-services/pg_egress_collect.conf | 9 + .../supervisor/base-services/postgresql.conf | 12 + .../base-services/supa-shutdown.conf | 10 + .../etc/supervisor/services/envoy.conf | 9 + .../etc/supervisor/services/exporter.conf | 10 + .../etc/supervisor/services/fail2ban.conf | 9 + .../etc/supervisor/services/gotrue.conf | 10 + .../etc/supervisor/services/group.conf | 3 + .../etc/supervisor/services/kong.conf | 10 + .../etc/supervisor/services/pgbouncer.conf | 10 + .../etc/supervisor/services/postgrest.conf | 9 + .../etc/supervisor/supervisord.conf | 170 ++++++++ .../etc/tmpfiles.d/pgbouncer.conf | 2 + docker/all-in-one-nix/etc/vector/vector.yaml | 306 +++++++++++++++ docker/all-in-one-nix/healthcheck.sh | 49 +++ .../init/configure-admin-mgr.sh | 8 + .../all-in-one-nix/init/configure-adminapi.sh | 55 +++ .../init/configure-autoshutdown.sh | 21 + docker/all-in-one-nix/init/configure-envoy.sh | 53 +++ .../all-in-one-nix/init/configure-exporter.sh | 5 + .../all-in-one-nix/init/configure-fail2ban.sh | 6 + .../all-in-one-nix/init/configure-gotrue.sh | 38 ++ docker/all-in-one-nix/init/configure-kong.sh | 48 +++ .../init/configure-pg_egress_collect.sh | 9 + .../init/configure-pgbouncer.sh | 46 +++ .../init/configure-postgrest.sh | 41 ++ .../all-in-one-nix/init/configure-vector.sh | 56 +++ docker/all-in-one-nix/init/start-kong.sh | 7 + .../pg_egress_collect/pg_egress_collect.pl | 126 ++++++ .../opt/postgres_exporter/queries.yml | 287 ++++++++++++++ docker/all-in-one-nix/postgres-entrypoint.sh | 358 +++++++++++++++++ docker/all-in-one-nix/run-logrotate.sh | 8 + docker/all-in-one-nix/shutdown.sh | 96 +++++ ebssurrogate/scripts/chroot-bootstrap-nix.sh | 204 ++++++++++ .../scripts/surrogate-bootstrap-nix.sh | 324 ++++++++++++++++ flake.nix | 2 +- stage2-nix-psql.pkr.hcl | 19 +- 77 files changed, 4396 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/ami-release-nix.yml create mode 100644 .github/workflows/text-nix.yml create mode 100644 .github/workflows/textinfra-nix.yml create mode 100644 common-nix.vars.pkr.hcl create mode 100644 docker/all-in-one-nix/Dockerfile create mode 100644 docker/all-in-one-nix/README.md create mode 100755 docker/all-in-one-nix/configure-shim.sh create mode 100755 docker/all-in-one-nix/entrypoint.sh create mode 100644 docker/all-in-one-nix/etc/adminapi/adminapi.yaml create mode 100644 docker/all-in-one-nix/etc/fail2ban/filter.d/pgbouncer.conf create mode 100644 docker/all-in-one-nix/etc/fail2ban/filter.d/postgresql.conf create mode 100644 docker/all-in-one-nix/etc/fail2ban/jail.d/jail.local create mode 100644 docker/all-in-one-nix/etc/fail2ban/jail.d/pgbouncer.conf create mode 100644 docker/all-in-one-nix/etc/fail2ban/jail.d/postgresql.conf create mode 100644 docker/all-in-one-nix/etc/fail2ban/jail.d/sshd.local create mode 100644 docker/all-in-one-nix/etc/gotrue.env create mode 100644 docker/all-in-one-nix/etc/kong/kong.conf create mode 100644 docker/all-in-one-nix/etc/kong/kong.yml create mode 100644 docker/all-in-one-nix/etc/logrotate.d/postgresql.conf create mode 100644 docker/all-in-one-nix/etc/logrotate.d/walg.conf create mode 100644 docker/all-in-one-nix/etc/pgbouncer-custom/custom-overrides.ini create mode 100644 docker/all-in-one-nix/etc/pgbouncer-custom/generated-optimizations.ini create mode 100644 docker/all-in-one-nix/etc/pgbouncer-custom/ssl-config.ini create mode 100644 docker/all-in-one-nix/etc/pgbouncer/pgbouncer.ini create mode 100644 docker/all-in-one-nix/etc/pgbouncer/userlist.txt create mode 100644 docker/all-in-one-nix/etc/postgresql-custom/custom-overrides.conf create mode 100644 docker/all-in-one-nix/etc/postgresql-custom/generated-optimizations.conf create mode 100644 docker/all-in-one-nix/etc/postgresql-custom/postgresql-platform-defaults.conf create mode 100644 docker/all-in-one-nix/etc/postgresql.schema.sql create mode 100644 docker/all-in-one-nix/etc/postgresql/logging.conf create mode 100755 docker/all-in-one-nix/etc/postgresql/pg_hba.conf create mode 100644 docker/all-in-one-nix/etc/postgrest/base.conf create mode 100755 docker/all-in-one-nix/etc/postgrest/bootstrap.sh create mode 100644 docker/all-in-one-nix/etc/postgrest/generated.conf create mode 100644 docker/all-in-one-nix/etc/sudoers.d/adminapi create mode 100644 docker/all-in-one-nix/etc/supa-shutdown/shutdown.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/base-services/adminapi.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/base-services/logrotate.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/base-services/lsn-checkpoint-push.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/base-services/pg_egress_collect.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/base-services/postgresql.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/base-services/supa-shutdown.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/services/envoy.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/services/exporter.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/services/fail2ban.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/services/gotrue.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/services/group.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/services/kong.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/services/pgbouncer.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/services/postgrest.conf create mode 100644 docker/all-in-one-nix/etc/supervisor/supervisord.conf create mode 100644 docker/all-in-one-nix/etc/tmpfiles.d/pgbouncer.conf create mode 100644 docker/all-in-one-nix/etc/vector/vector.yaml create mode 100755 docker/all-in-one-nix/healthcheck.sh create mode 100755 docker/all-in-one-nix/init/configure-admin-mgr.sh create mode 100755 docker/all-in-one-nix/init/configure-adminapi.sh create mode 100755 docker/all-in-one-nix/init/configure-autoshutdown.sh create mode 100755 docker/all-in-one-nix/init/configure-envoy.sh create mode 100755 docker/all-in-one-nix/init/configure-exporter.sh create mode 100755 docker/all-in-one-nix/init/configure-fail2ban.sh create mode 100755 docker/all-in-one-nix/init/configure-gotrue.sh create mode 100755 docker/all-in-one-nix/init/configure-kong.sh create mode 100755 docker/all-in-one-nix/init/configure-pg_egress_collect.sh create mode 100755 docker/all-in-one-nix/init/configure-pgbouncer.sh create mode 100755 docker/all-in-one-nix/init/configure-postgrest.sh create mode 100755 docker/all-in-one-nix/init/configure-vector.sh create mode 100755 docker/all-in-one-nix/init/start-kong.sh create mode 100644 docker/all-in-one-nix/opt/pg_egress_collect/pg_egress_collect.pl create mode 100644 docker/all-in-one-nix/opt/postgres_exporter/queries.yml create mode 100755 docker/all-in-one-nix/postgres-entrypoint.sh create mode 100755 docker/all-in-one-nix/run-logrotate.sh create mode 100755 docker/all-in-one-nix/shutdown.sh create mode 100755 ebssurrogate/scripts/chroot-bootstrap-nix.sh create mode 100755 ebssurrogate/scripts/surrogate-bootstrap-nix.sh diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml new file mode 100644 index 000000000..d25bfc793 --- /dev/null +++ b/.github/workflows/ami-release-nix.yml @@ -0,0 +1,169 @@ +name: Release AMI + +on: + push: + branches: + - develop + paths: + - '.github/workflows/ami-release-nix.yml' + - 'common-nix.vars.pkr.hcl' + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + include: + - runner: arm-runner + arch: arm64 + ubuntu_release: focal + ubuntu_version: 20.04 + mcpu: neoverse-n1 + runs-on: ${{ matrix.runner }} + timeout-minutes: 150 + permissions: + contents: write + packages: write + id-token: write + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Run checks if triggered manually + if: ${{ github.event_name == 'workflow_dispatch' }} + # Update `ci.yaml` too if changing constraints. + run: | + SUFFIX=$(sed -E 's/postgres-version = "[0-9\.]+(.*)"/\1/g' common-nix.vars.pkr.hcl) + if [[ -z $SUFFIX ]] ; then + echo "Version must include non-numeric characters if built manually." + exit 1 + fi + + # - id: args + # uses: mikefarah/yq@master + # with: + # cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' + # - run: docker context create builders + # - uses: docker/setup-buildx-action@v3 + # with: + # endpoint: builders + # - uses: docker/build-push-action@v5 + # with: + # build-args: | + # ${{ steps.args.outputs.result }} + # target: extensions + # tags: supabase/postgres:extensions + # platforms: linux/${{ matrix.arch }} + # outputs: type=tar,dest=/tmp/extensions.tar + # cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + # # No need to export extensions cache because latest depends on it + + # - name: Extract built packages + # run: | + # mkdir -p ansible/files/extensions + # tar xvf /tmp/extensions.tar -C ansible/files/extensions --strip-components 1 + # TODO remove this block as extensions are build in nix prior to this step + + - id: version + run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" + - name: Build Postgres deb + uses: docker/build-push-action@v5 + with: + file: docker/Dockerfile + target: pg-deb + build-args: | + ubuntu_release=${{ matrix.ubuntu_release }} + ubuntu_release_no=${{ matrix.ubuntu_version }} + postgresql_major=${{ steps.version.outputs.postgresql_major }} + postgresql_release=${{ steps.version.outputs.postgresql_release }} + CPPFLAGS=-mcpu=${{ matrix.mcpu }} + tags: supabase/postgres:deb + platforms: linux/${{ matrix.arch }} + outputs: type=tar,dest=/tmp/pg-deb.tar + cache-from: type=gha,scope=${{ github.ref_name }}-deb + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-deb + # - name: Extract Postgres deb + # run: | + # mkdir -p ansible/files/postgres + # tar xvf /tmp/pg-deb.tar -C ansible/files/postgres --strip-components 1 + #TODO remove this block as deb is build in nix prior to this step + + - name: Build AMI stage 1 + run: | + packer init amazon-arm64-nix.pkr.hcl + GIT_SHA=${{github.sha}} + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl + + - name: Build AMI stage 1 + run: | + packer init amazon-arm64-nix.pkr.hcl + GIT_SHA=${{github.sha}} + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl + + - name: Grab release version + id: process_release_version + run: | + VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl) + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: configure aws credentials - staging + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + aws-region: "us-east-1" + + - name: Upload software manifest to s3 staging + run: | + cd ansible + ansible-playbook -i localhost \ + -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ + -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ + manifest-playbook.yml + + # - name: Upload pg binaries to s3 staging + # run: | + # aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz + # #TODO look to see if this only pg binaries and if so, remove this as it is covered by nix build + # TODO deactivate this block to assure binaries from this file are not uploaded. This is covered by nix build + - name: configure aws credentials - prod + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.PROD_AWS_ROLE }} + aws-region: "us-east-1" + + - name: Upload software manifest to s3 prod + run: | + cd ansible + ansible-playbook -i localhost \ + -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ + -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ + manifest-playbook.yml + + # - name: Upload pg binaries to s3 prod + # run: | + # aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz + #TODO deactivate this block to assure binaries from this file are not uploaded. This is covered by nix build + + + - name: Create release + uses: softprops/action-gh-release@v1 + with: + name: ${{ steps.process_release_version.outputs.version }} + tag_name: ${{ steps.process_release_version.outputs.version }} + target_commitish: ${{github.sha}} + + - name: Slack Notification on Failure + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + SLACK_USERNAME: 'gha-failures-notifier' + SLACK_COLOR: 'danger' + SLACK_MESSAGE: 'Building Postgres AMI failed' + SLACK_FOOTER: '' + + - name: Cleanup resources on build cancellation + if: ${{ cancelled() }} + run: | + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {} diff --git a/.github/workflows/text-nix.yml b/.github/workflows/text-nix.yml new file mode 100644 index 000000000..89acf3328 --- /dev/null +++ b/.github/workflows/text-nix.yml @@ -0,0 +1,117 @@ +name: Test Database + +on: + push: + branches: + - develop + pull_request: + workflow_dispatch: + +jobs: + build: + strategy: + matrix: + include: + - runner: [self-hosted, X64] + arch: amd64 + - runner: arm-runner + arch: arm64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 180 + env: + POSTGRES_PORT: 5478 + POSTGRES_PASSWORD: password + steps: + - uses: actions/checkout@v3 + - id: args + uses: mikefarah/yq@master + with: + cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' + + - run: docker context create builders + - uses: docker/setup-buildx-action@v3 + with: + endpoint: builders + - uses: docker/build-push-action@v5 + with: + load: true + context: . + target: production + build-args: | + ${{ steps.args.outputs.result }} + tags: samrose/nix-experimental-postgresql-15-aarch64-linux:latest + cache-from: | + type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + type=gha,scope=${{ github.base_ref }}-latest-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} + + - name: Start Postgres + run: | + docker run --rm --pull=never \ + -e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \ + -p ${{ env.POSTGRES_PORT }}:5432 \ + --name supabase_postgres \ + -d supabase/postgres:latest + + - name: Install psql + run: | + sudo apt update + sudo apt install -y --no-install-recommends postgresql-client + + - name: Install pg_prove + run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP + env: + SHELL: /bin/bash + + - name: Wait for healthy database + run: | + count=0 + until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do + exit=$? + count=$((count + 1)) + if [ $count -ge "$retries" ]; then + echo "Retry $count/$retries exited $exit, no more retries left." + docker stop -t 2 "$container" + return $exit + fi + sleep 1; + done; + echo "$container container is healthy" + env: + retries: 20 + container: supabase_postgres + + - name: Run tests + run: pg_prove migrations/tests/test.sql + env: + PGHOST: localhost + PGPORT: ${{ env.POSTGRES_PORT }} + PGDATABASE: postgres + PGUSER: supabase_admin + PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} + + - name: Check migrations are idempotent + run: | + for sql in ./migrations/db/migrations/*.sql; do + echo "$0: running $sql" + psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -f "$sql" + done + env: + PGHOST: localhost + PGPORT: ${{ env.POSTGRES_PORT }} + PGDATABASE: postgres + PGUSER: supabase_admin + PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} + + schema: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: verify schema.sql is committed + run: | + docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit + if ! git diff --ignore-space-at-eol --exit-code --quiet migrations/schema.sql; then + echo "Detected uncommitted changes after build. See status below:" + git diff + exit 1 + fi diff --git a/.github/workflows/textinfra-nix.yml b/.github/workflows/textinfra-nix.yml new file mode 100644 index 000000000..9b60d852c --- /dev/null +++ b/.github/workflows/textinfra-nix.yml @@ -0,0 +1,145 @@ +name: Testinfra Integration Tests + +on: + pull_request: + workflow_dispatch: + +jobs: + test-all-in-one: + strategy: + matrix: + include: + - runner: [self-hosted, X64] + arch: amd64 + - runner: arm-runner + arch: arm64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v3 + + - run: docker context create builders + - uses: docker/setup-buildx-action@v3 + with: + endpoint: builders + + - name: Run aio integration tests + run: | + # TODO: use poetry for pkg mgmt + pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests + + + if ! pytest -vv testinfra/test_all_in_one.py; then + # display container logs if the test fails + + if [ -f testinfra-aio-container-logs.log ]; then + echo "AIO container logs:" + cat testinfra-aio-container-logs.log + fi + exit 1 + fi + + test-ami: + strategy: + matrix: + include: + - runner: arm-runner + arch: arm64 + ubuntu_release: focal + ubuntu_version: 20.04 + mcpu: neoverse-n1 + runs-on: ${{ matrix.runner }} + timeout-minutes: 150 + permissions: + contents: write + packages: write + id-token: write + + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - id: args + uses: mikefarah/yq@master + with: + cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' + + - run: docker context create builders + + - uses: docker/setup-buildx-action@v3 + with: + endpoint: builders + + - uses: docker/build-push-action@v5 + with: + build-args: | + ${{ steps.args.outputs.result }} + target: extensions + tags: supabase/postgres:extensions + platforms: linux/${{ matrix.arch }} + outputs: type=tar,dest=/tmp/extensions.tar + cache-from: | + type=gha,scope=${{ github.ref_name }}-extensions + type=gha,scope=${{ github.base_ref }}-extensions + type=gha,scope=develop-extensions + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-extensions + + - name: Extract built packages + run: | + mkdir -p ansible/files/extensions + tar xvf /tmp/extensions.tar -C ansible/files/extensions --strip-components 1 + + - id: version + run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" + + - name: Build Postgres deb + uses: docker/build-push-action@v5 + with: + file: docker/Dockerfile + target: pg-deb + build-args: | + ubuntu_release=${{ matrix.ubuntu_release }} + ubuntu_release_no=${{ matrix.ubuntu_version }} + postgresql_major=${{ steps.version.outputs.postgresql_major }} + postgresql_release=${{ steps.version.outputs.postgresql_release }} + CPPFLAGS=-mcpu=${{ matrix.mcpu }} + tags: supabase/postgres:deb + platforms: linux/${{ matrix.arch }} + outputs: type=tar,dest=/tmp/pg-deb.tar + cache-from: | + type=gha,scope=${{ github.ref_name }}-deb + type=gha,scope=${{ github.base_ref }}-deb + type=gha,scope=develop-deb + cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-deb + + - name: Extract Postgres deb + run: | + mkdir -p ansible/files/postgres + tar xvf /tmp/pg-deb.tar -C ansible/files/postgres --strip-components 1 + + # Packer doesn't support skipping registering the AMI for the ebssurrogate + # builder, so we register an AMI with a fixed name and run tests on an + # instance launched from that + # https://github.com/hashicorp/packer/issues/4899 + - name: Build AMI + run: | + packer init amazon-arm64.pkr.hcl + GIT_SHA=${{github.sha}} + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=ci-ami-test" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" amazon-arm64.pkr.hcl + + - name: Run tests + timeout-minutes: 10 + run: | + # TODO: use poetry for pkg mgmt + pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests + pytest -vv -s testinfra/test_ami.py + + - name: Cleanup resources on build cancellation + if: ${{ cancelled() }} + run: | + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} + + - name: Cleanup resources on build cancellation + if: ${{ always() }} + run: | + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} || true diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 559387b1e..2b1f0eb66 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -204,8 +204,8 @@ build { } provisioner "file" { - source = "ebssurrogate/scripts/chroot-bootstrap.sh" - destination = "/tmp/chroot-bootstrap.sh" + source = "ebssurrogate/scripts/chroot-bootstrap-nix.sh" + destination = "/tmp/chroot-bootstrap-nix.sh" } provisioner "file" { @@ -258,7 +258,7 @@ build { "POSTGRES_SUPABASE_VERSION=${var.postgres-version}" ] use_env_var_file = true - script = "ebssurrogate/scripts/surrogate-bootstrap.sh" + script = "ebssurrogate/scripts/surrogate-bootstrap-nix.sh" execute_command = "sudo -S sh -c '. {{.EnvVarFile}} && {{.Path}}'" start_retry_timeout = "5m" skip_clean = true diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl new file mode 100644 index 000000000..75b879061 --- /dev/null +++ b/common-nix.vars.pkr.hcl @@ -0,0 +1 @@ +postgres-version = "15.1.1.42" diff --git a/docker/all-in-one-nix/Dockerfile b/docker/all-in-one-nix/Dockerfile new file mode 100644 index 000000000..360a5fbf7 --- /dev/null +++ b/docker/all-in-one-nix/Dockerfile @@ -0,0 +1,293 @@ +ARG postgres_version=15.1.1.24 + +ARG pgbouncer_release=1.18.0 +ARG postgrest_release=10.1.2 +ARG gotrue_release=2.130.0 +ARG adminapi_release=0.63.5 +ARG adminmgr_release=0.19.0 +ARG vector_release=0.22.3 +ARG postgres_exporter_release=0.15.0 +ARG envoy_release=1.28.0 + +# Update `gateway-28` in the URL below if upgrading above v2.8.x. +ARG kong_release=2.8.1 + +FROM supabase/postgres:${postgres_version} as base +ARG TARGETARCH +ARG postgresql_major + +FROM base as builder +# Install build dependencies +RUN apt-get update && apt-get install -y \ + postgresql-server-dev-${postgresql_major} \ + build-essential \ + checkinstall \ + pkg-config \ + cmake \ + && rm -rf /var/lib/apt/lists/* + +#################### +# Install pgbouncer +#################### +FROM builder as pgbouncer-source +# Download and extract +ARG pgbouncer_release +ADD "https://www.pgbouncer.org/downloads/files/${pgbouncer_release}/pgbouncer-${pgbouncer_release}.tar.gz" /tmp/pgbouncer.tar.gz +RUN tar -xvf /tmp/pgbouncer.tar.gz -C /tmp && \ + rm -rf /tmp/pgbouncer.tar.gz +# Install build dependencies +RUN apt-get update && apt-get install -y \ + libevent-dev \ + && rm -rf /var/lib/apt/lists/* +# Build from source +WORKDIR /tmp/pgbouncer-${pgbouncer_release} +RUN ./configure --prefix=/usr/local +RUN make -j$(nproc) +# Create debian package +RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --requires=libevent-2.1-7 --nodoc + +FROM base as pgbouncer +# Download pre-built packages +RUN apt-get update && apt-get install -y --no-install-recommends --download-only \ + pgbouncer \ + && rm -rf /var/lib/apt/lists/* +RUN mv /var/cache/apt/archives/*.deb /tmp/ + +#################### +# Install PostgREST +#################### +FROM postgrest/postgrest:v${postgrest_release} as pgrst + +#################### +# Install GoTrue +#################### +FROM supabase/gotrue:v${gotrue_release} as gotrue + +#################### +# Install Envoy +#################### +FROM envoyproxy/envoy:v${envoy_release} as envoy + +#################### +# Install Kong +#################### +FROM base as kong +ARG kong_release +ADD "https://packages.konghq.com/public/gateway-28/deb/ubuntu/pool/focal/main/k/ko/kong_${kong_release}/kong_${kong_release}_${TARGETARCH}.deb" \ + /tmp/kong.deb + +#################### +# Install admin api +#################### +FROM base as adminapi +ARG adminapi_release +ADD "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-api/v${adminapi_release}/supabase-admin-api_${adminapi_release}_linux_${TARGETARCH}.tar.gz" /tmp/supabase-admin-api.tar.gz +RUN tar -xvf /tmp/supabase-admin-api.tar.gz -C /tmp && \ + rm -rf /tmp/supabase-admin-api.tar.gz + +#################### +# Install admin mgr +#################### +FROM base as adminmgr +ARG adminmgr_release +ADD "https://supabase-public-artifacts-bucket.s3.amazonaws.com/admin-mgr/v${adminmgr_release}/admin-mgr_${adminmgr_release}_linux_${TARGETARCH}.tar.gz" /tmp/admin-mgr.tar.gz +RUN tar -xvf /tmp/admin-mgr.tar.gz -C /tmp && \ + rm -rf /tmp/admin-mgr.tar.gz + +#################### +# Install Prometheus Exporter +#################### +FROM base as exporter +ARG postgres_exporter_release +ADD "https://github.com/prometheus-community/postgres_exporter/releases/download/v${postgres_exporter_release}/postgres_exporter-${postgres_exporter_release}.linux-${TARGETARCH}.tar.gz" /tmp/postgres_exporter.tar.gz +RUN tar -xvf /tmp/postgres_exporter.tar.gz -C /tmp --strip-components 1 && \ + rm -rf /tmp/postgres_exporter.tar.gz + +#################### +# Install vector +#################### +FROM base as vector +ARG vector_release +ADD "https://packages.timber.io/vector/${vector_release}/vector_${vector_release}-1_${TARGETARCH}.deb" /tmp/vector.deb + +#################### +# Install supervisord +#################### +FROM base as supervisor +# Download pre-built packages +RUN apt-get update -y && apt-get install -y --no-install-recommends --download-only \ + supervisor \ + && rm -rf /var/lib/apt/lists/* +RUN mv /var/cache/apt/archives/*.deb /tmp/ + +#################### +# Create the final image for production +#################### +FROM base as production + +# Copy dependencies from previous build stages +COPY --from=pgbouncer /tmp/*.deb /tmp/ +COPY --from=vector /tmp/*.deb /tmp/ +COPY --from=kong /tmp/*.deb /tmp/ +COPY --from=supervisor /tmp/*.deb /tmp/ + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + /tmp/*.deb \ + # For health check + curl \ + # For parsing init payload + jq \ + # Security tools + fail2ban \ + # sudo + sudo \ + vim-tiny \ + less \ + libnuma1 \ + logrotate \ + dumb-init \ + # pg_egress_collect deps + tcpdump libio-async-perl \ + && rm -rf /var/lib/apt/lists/* /tmp/* \ + && mkdir -p /dist \ + && mkdir -p /data/opt && chmod go+rwx /data/opt + +# Copy single binary dependencies +COPY --from=pgrst /bin/postgrest /dist/ +COPY --from=gotrue /usr/local/bin/auth /dist/gotrue +COPY --from=gotrue /usr/local/etc/auth /opt/gotrue/ +COPY --from=envoy /usr/local/bin/envoy /dist/ +COPY --from=adminapi /tmp/supabase-admin-api /dist/ +COPY --chown=root:root --from=adminmgr /tmp/admin-mgr /dist/ +COPY --from=exporter /tmp/postgres_exporter /opt/postgres_exporter/ +COPY docker/all-in-one/opt/postgres_exporter /opt/postgres_exporter/ + +# Configuring dangling symlinks for binaries +RUN ln -s /data/opt/supabase-admin-api /opt/supabase-admin-api \ + && ln -s /data/opt/postgrest /opt/postgrest \ + && ln -s /data/opt/gotrue /opt/gotrue/gotrue \ + && ln -s /data/opt/admin-mgr /usr/bin/admin-mgr + +# Scripts for adminapi +COPY ansible/files/admin_api_scripts /root +COPY --chown=adminapi:adminapi docker/all-in-one/etc/adminapi /etc/adminapi +COPY --chmod=644 docker/all-in-one/etc/sudoers.d /etc/sudoers.d/ + +# Script for pg_egress_collect +COPY --chown=adminapi:adminapi docker/all-in-one/opt/pg_egress_collect /opt/pg_egress_collect + +# Customizations for pgbouncer +COPY docker/all-in-one/etc/pgbouncer /etc/pgbouncer +COPY docker/all-in-one/etc/pgbouncer-custom /etc/pgbouncer-custom +COPY docker/all-in-one/etc/tmpfiles.d /etc/tmpfiles.d + +# Customizations for postgres +COPY --chown=postgres:postgres docker/all-in-one/etc/postgresql/pg_hba.conf /etc/postgresql/ +COPY --chown=postgres:postgres docker/all-in-one/etc/postgresql/logging.conf /etc/postgresql/ +COPY --chown=postgres:postgres docker/all-in-one/etc/postgresql-custom /etc/postgresql-custom +COPY --chown=postgres:postgres docker/all-in-one/etc/postgresql.schema.sql /etc/postgresql.schema.sql + +# Customizations for postgres_exporter +COPY --chown=postgres:postgres docker/all-in-one/opt/postgres_exporter/queries.yml /opt/postgres_exporter/queries.yml + +# Customizations for fail2ban +COPY docker/all-in-one/etc/fail2ban/filter.d /etc/fail2ban/filter.d/ +COPY docker/all-in-one/etc/fail2ban/jail.d /etc/fail2ban/jail.d/ + +# Customizations for postgrest +COPY --chown=postgrest:postgrest docker/all-in-one/etc/postgrest/bootstrap.sh /etc/postgrest/bootstrap.sh +COPY --chown=postgrest:postgrest docker/all-in-one/etc/postgrest/base.conf /etc/postgrest/base.conf +COPY --chown=postgrest:postgrest docker/all-in-one/etc/postgrest/generated.conf /etc/postgrest/generated.conf + +# Customizations for logrotate +COPY docker/all-in-one/etc/logrotate.d/walg.conf /etc/logrotate.d/walg.conf +COPY docker/all-in-one/etc/logrotate.d/postgresql.conf /etc/logrotate.d/postgresql.conf + +# Customizations for gotrue +COPY docker/all-in-one/etc/gotrue.env /etc/gotrue.env + +# Customizations for envoy +ARG envoy_release +ADD --chmod=755 --chown=envoy:envoy "https://raw.githubusercontent.com/envoyproxy/envoy/v${envoy_release}/restarter/hot-restarter.py" /opt/envoy-hot-restarter.py +COPY --chmod=775 --chown=envoy:envoy ansible/files/envoy_config/ /etc/envoy/ +COPY --chmod=755 --chown=envoy:envoy ansible/files/start-envoy.sh /opt/ + +# Customizations for kong +COPY docker/all-in-one/etc/kong/kong.conf /etc/kong/kong.conf +COPY docker/all-in-one/etc/kong/kong.yml /etc/kong/kong.yml + +# Customizations for vector +COPY --chown=vector:vector docker/all-in-one/etc/vector/vector.yaml /etc/vector/vector.yaml + +# Customizations for supervisor +COPY docker/all-in-one/etc/supervisor /etc/supervisor + +# Customizations for supa-shutdown +COPY --chown=adminapi:adminapi docker/all-in-one/etc/supa-shutdown /etc/supa-shutdown +COPY docker/all-in-one/configure-shim.sh /usr/local/bin/configure-shim.sh + +# Configure service ports +ENV PGRST_SERVER_PORT=3000 +ENV PGRST_ADMIN_SERVER_PORT=3001 +EXPOSE ${PGRST_SERVER_PORT} + +ENV GOTRUE_SITE_URL=http://localhost:${PGRST_SERVER_PORT} +ENV GOTRUE_API_PORT=9999 +EXPOSE ${GOTRUE_API_PORT} + +ENV ENVOY_HTTP_PORT=8000 +ENV ENVOY_HTTPS_PORT=8443 + +ENV KONG_HTTP_PORT=8000 +ENV KONG_HTTPS_PORT=8443 + +ENV HTTP_PORT=${ENVOY_HTTP_PORT:-KONG_HTTP_PORT} +ENV HTTP_PORT=${ENVOY_HTTPS_PORT:-KONG_HTTPS_PORT} +EXPOSE ${HTTP_PORT} ${HTTPS_PORT} + +ENV ADMIN_API_CERT_DIR=/etc/ssl/adminapi +ENV ADMIN_API_PORT=8085 +EXPOSE ${ADMIN_API_PORT} + +ENV PGBOUNCER_PORT=6543 +EXPOSE ${PGBOUNCER_PORT} + +ENV PGEXPORTER_PORT=9187 +EXPOSE ${PGEXPORTER_PORT} + +ENV VECTOR_API_PORT=9001 + +# Create system users +RUN useradd --create-home --shell /bin/bash postgrest && \ + useradd --create-home --shell /bin/bash gotrue && \ + useradd --create-home --shell /bin/bash envoy && \ + useradd --create-home --shell /bin/bash pgbouncer -G postgres,ssl-cert && \ + useradd --create-home --shell /bin/bash adminapi -G root,envoy,kong,pgbouncer,postgres,postgrest,wal-g && \ + usermod --append --shell /bin/bash -G postgres vector +RUN mkdir -p /etc/wal-g && \ + chown -R adminapi:adminapi /etc/wal-g && \ + chmod g+w /etc/wal-g +RUN mkdir -p /var/log/wal-g \ + && chown -R postgres:postgres /var/log/wal-g \ + && chmod +x /dist/admin-mgr \ + && chmod ug+s /dist/admin-mgr \ + && touch /etc/wal-g/config.json \ + && chown adminapi:adminapi /etc/wal-g/config.json \ + && echo '{"WALG_S3_PREFIX": "s3://foo/bar/"}' > /etc/wal-g/config.json +RUN chown -R adminapi:adminapi /etc/adminapi +RUN sed -i "s;#include = '/etc/postgresql-custom/generated-optimizations.conf';include = '/etc/postgresql-custom/generated-optimizations.conf';" /etc/postgresql/postgresql.conf + +# Add healthcheck and entrypoint scripts +COPY docker/all-in-one/healthcheck.sh /usr/local/bin/ +HEALTHCHECK --interval=3s --timeout=2s --start-period=4s --retries=10 CMD [ "healthcheck.sh" ] + +COPY docker/all-in-one/init /init +COPY docker/all-in-one/entrypoint.sh /usr/local/bin/ +COPY docker/all-in-one/postgres-entrypoint.sh /usr/local/bin/ +COPY docker/all-in-one/shutdown.sh /usr/local/bin/supa-shutdown.sh +COPY docker/all-in-one/run-logrotate.sh /usr/local/bin/run-logrotate.sh + +ENTRYPOINT [ "/usr/bin/dumb-init" ] + +CMD [ "entrypoint.sh"] diff --git a/docker/all-in-one-nix/README.md b/docker/all-in-one-nix/README.md new file mode 100644 index 000000000..72e120009 --- /dev/null +++ b/docker/all-in-one-nix/README.md @@ -0,0 +1,59 @@ +# Supabase All-in-One + +All Supabase backend services bundled in a single Docker image for quick local testing and edge deployment. + +## Build + +```bash +# cwd: repo root +docker build -f docker/all-in-one/Dockerfile -t supabase/all-in-one . +``` + +## Run + +```bash +docker run --rm -it \ + -e POSTGRES_PASSWORD=postgres \ + -e JWT_SECRET=super-secret-jwt-token-with-at-least-32-characters-long \ + -e ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE \ + -e SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q \ + -e ADMIN_API_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic3VwYWJhc2VfYWRtaW4iLCJpc3MiOiJzdXBhYmFzZS1kZW1vIiwiaWF0IjoxNjQxNzY5MjAwLCJleHAiOjE3OTk1MzU2MDB9.Y9mSNVuTw2TdfryoaqM5wySvwQemGGWfSe9ixcklVfM \ + -e DATA_VOLUME_MOUNTPOINT=/data \ + -e MACHINE_TYPE=shared_cpu_1x_512m \ + -p 5432:5432 \ + -p 8000:8000 \ + supabase/all-in-one +``` + +Use bind mount to start from an existing physical backup: `-v $(pwd)/data:/var/lib/postgresql/data` + +Alternatively, the container may be initialised using a payload tarball. + +```bash +docker run --rm \ + -e POSTGRES_PASSWORD=postgres \ + -e INIT_PAYLOAD_PRESIGNED_URL= \ + -p 5432:5432 \ + -p 8000:8000 \ + -it supabase/all-in-one +``` + +## Test + +```bash +curl -H "apikey: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE" \ + localhost:8000/rest/v1/ | jq +``` + +## TODO + +- [x] optimise admin config +- [x] propagate shutdown signals +- [x] add http health checks +- [x] generate dynamic JWT +- [ ] ufw / nftables +- [x] log rotation +- [x] egress metrics +- [x] vector +- [ ] apparmor +- [x] wal-g diff --git a/docker/all-in-one-nix/configure-shim.sh b/docker/all-in-one-nix/configure-shim.sh new file mode 100755 index 000000000..f42f1557a --- /dev/null +++ b/docker/all-in-one-nix/configure-shim.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +INITIAL_BINARY_PATH=$1 +SYMLINK_PATH=$2 + +SYMLINK_TARGET=$(readlink -m "$SYMLINK_PATH") + +if [ ! -f "$SYMLINK_TARGET" ]; then + cp "$INITIAL_BINARY_PATH" "$SYMLINK_TARGET" + + PERMS=$(stat -c "%a" "$INITIAL_BINARY_PATH") + chmod "$PERMS" "$SYMLINK_TARGET" + + OWNER_GROUP=$(stat -c "%u:%g" "$INITIAL_BINARY_PATH") + chown "$OWNER_GROUP" "$SYMLINK_TARGET" +fi diff --git a/docker/all-in-one-nix/entrypoint.sh b/docker/all-in-one-nix/entrypoint.sh new file mode 100755 index 000000000..56b58cc39 --- /dev/null +++ b/docker/all-in-one-nix/entrypoint.sh @@ -0,0 +1,304 @@ +#!/bin/bash +set -eou pipefail + +PG_CONF=/etc/postgresql/postgresql.conf +SUPERVISOR_CONF=/etc/supervisor/supervisord.conf + +export DATA_VOLUME_MOUNTPOINT=${DATA_VOLUME_MOUNTPOINT:-/data} +export CONFIGURED_FLAG_PATH=${CONFIGURED_FLAG_PATH:-$DATA_VOLUME_MOUNTPOINT/machine.configured} + +export MAX_IDLE_TIME_MINUTES=${MAX_IDLE_TIME_MINUTES:-5} + +# Ref: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746 +function retry { + # Pass 0 for unlimited retries + local retries=$1 + shift + + local start=$EPOCHSECONDS + local count=0 + until "$@"; do + exit=$? + # Reset count if service has been running for more than 2 minutes + local elapsed=$((EPOCHSECONDS - start)) + if [ $elapsed -gt 120 ]; then + count=0 + fi + # Exponential backoff up to n tries + local wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -ge "$retries" ] && [ "$retries" -gt 0 ]; then + echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + start=$EPOCHSECONDS + done + return 0 +} + +function configure_services { + # Start services after migrations are run + for file in /init/configure-*.sh; do + retry 0 "$file" + done +} + +function enable_swap { + fallocate -l 1G /mnt/swapfile + chmod 600 /mnt/swapfile + mkswap /mnt/swapfile + swapon /mnt/swapfile +} + +function push_lsn_checkpoint_file { + if [ "${PLATFORM_DEPLOYMENT:-}" != "true" ]; then + echo "Skipping push of LSN checkpoint file" + return + fi + + /usr/bin/admin-mgr lsn-checkpoint-push --immediately || echo "Failed to push LSN checkpoint" +} + +function graceful_shutdown { + echo "$(date): Received SIGINT. Shutting down." + + # Postgres ships the latest WAL file using archive_command during shutdown, in a blocking operation + # This is to ensure that the WAL file is shipped, just in case + sleep 0.2 + push_lsn_checkpoint_file +} + +function enable_autoshutdown { + sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/base-services/supa-shutdown.conf +} + +function enable_lsn_checkpoint_push { + sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/base-services/lsn-checkpoint-push.conf + sed -i "s/autorestart=.*/autorestart=true/" /etc/supervisor/base-services/lsn-checkpoint-push.conf +} + +function disable_fail2ban { + sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/fail2ban.conf + sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/fail2ban.conf +} + +function setup_postgres { + tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/postgresql.schema.sql + mv /etc/postgresql.schema.sql /docker-entrypoint-initdb.d/migrations/99-schema.sql + + tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/postgresql-custom/pgsodium_root.key + echo "include = '/etc/postgresql-custom/postgresql-platform-defaults.conf'" >> $PG_CONF + + # TODO (darora): walg enablement is temporarily performed here until changes from https://github.com/supabase/postgres/pull/639 get picked up + # other things will still be needed in the future (auth_delay config) + sed -i \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" \ + -e "s|shared_preload_libraries = '\(.*\)'|shared_preload_libraries = '\1, auth_delay'|" \ + -e "/# Automatically generated optimizations/i auth_delay.milliseconds = '3000'" \ + "${PG_CONF}" + + # Setup ssl certs + mkdir -p /etc/ssl/certs/postgres + tar -xzvf "$INIT_PAYLOAD_PATH" -C /etc/ssl/certs/postgres/ --strip-components 2 ./ssl/server.crt + tar -xzvf "$INIT_PAYLOAD_PATH" -C /etc/ssl/certs/postgres/ --strip-components 2 ./ssl/ca.crt + tar -xzvf "$INIT_PAYLOAD_PATH" -C /etc/ssl/private/ --strip-components 2 ./ssl/server.key + # tar -xzvf "$INIT_PAYLOAD_PATH" -C /etc/ssl/certs/postgres/ ./ssl/server-intermediate.srl + + PGSSLROOTCERT=/etc/ssl/certs/postgres/ca.crt + PGSSLCERT=/etc/ssl/certs/postgres/server.crt + PGSSLKEY=/etc/ssl/private/server.key + chown root:postgres $PGSSLROOTCERT $PGSSLKEY $PGSSLCERT + chmod 640 $PGSSLROOTCERT $PGSSLKEY $PGSSLCERT + + # Change ssl back to on in postgres.conf + sed -i -e "s|ssl = off|ssl = on|g" \ + -e "s|ssl_ca_file = ''|ssl_ca_file = '$PGSSLROOTCERT'|g" \ + -e "s|ssl_cert_file = ''|ssl_cert_file = '$PGSSLCERT'|g" \ + -e "s|ssl_key_file = ''|ssl_key_file = '$PGSSLKEY'|g" \ + $PG_CONF + + if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then + mkdir -p "${DATA_VOLUME_MOUNTPOINT}/opt" + /usr/local/bin/configure-shim.sh /dist/supabase-admin-api /opt/supabase-admin-api + /opt/supabase-admin-api optimize db --destination-config-file-path /etc/postgresql-custom/generated-optimizations.conf + + # Preserve postgresql configs across restarts + POSTGRESQL_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/postgresql-custom" + + mkdir -p "${POSTGRESQL_CUSTOM_DIR}" + + if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then + echo "Copying existing custom postgresql config from /etc/postgresql-custom to ${POSTGRESQL_CUSTOM_DIR}" + cp -R "/etc/postgresql-custom/." "${POSTGRESQL_CUSTOM_DIR}/" + fi + + rm -rf "/etc/postgresql-custom" + ln -s "${POSTGRESQL_CUSTOM_DIR}" "/etc/postgresql-custom" + chown -R postgres:postgres "/etc/postgresql-custom" + chown -R postgres:postgres "${POSTGRESQL_CUSTOM_DIR}" + chmod g+rx "${POSTGRESQL_CUSTOM_DIR}" + + # Preserve wal-g configs across restarts + WALG_CONF_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/wal-g" + mkdir -p "${WALG_CONF_DIR}" + + if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then + echo "Copying existing custom wal-g config from /etc/wal-g to ${WALG_CONF_DIR}" + cp -R "/etc/wal-g/." "${WALG_CONF_DIR}/" + fi + + rm -rf "/etc/wal-g" + ln -s "${WALG_CONF_DIR}" "/etc/wal-g" + chown -R adminapi:adminapi "/etc/wal-g" + chown -R adminapi:adminapi "${WALG_CONF_DIR}" + chmod g+rx "/etc/wal-g" + chmod g+rx "${WALG_CONF_DIR}" + fi +} + +function setup_credentials { + # Load credentials from init json + tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./tmp/init.json + export ANON_KEY=${ANON_KEY:-$(jq -r '.["anon_key"]' /tmp/init.json)} + export SERVICE_ROLE_KEY=${SERVICE_ROLE_KEY:-$(jq -r '.["service_key"]' /tmp/init.json)} + export ADMIN_API_KEY=${ADMIN_API_KEY:-$(jq -r '.["supabase_admin_key"]' /tmp/init.json)} + export JWT_SECRET=${JWT_SECRET:-$(jq -r '.["jwt_secret"]' /tmp/init.json)} +} + +function report_health { + if [ -z "${REPORTING_TOKEN:-}" ]; then + echo "Skipped health reporting: missing REPORTING_TOKEN" + exit 0 + fi + if [ -d "$ADMIN_API_CERT_DIR" ]; then + retry 10 curl -sSkf "https://localhost:$ADMIN_API_PORT/health-reporter/send" -X POST -H "apikey: $ADMIN_API_KEY" + else + retry 10 curl -sSf "http://localhost:$ADMIN_API_PORT/health-reporter/send" -X POST -H "apikey: $ADMIN_API_KEY" + fi +} + +function run_prelaunch_hooks { + if [ -f "/etc/postgresql-custom/supautils.conf" ]; then + sed -i -e 's/dblink, //' "/etc/postgresql-custom/supautils.conf" + fi +} + +function start_supervisor { + # Start health reporting + report_health & + + # Start supervisord + /usr/bin/supervisord -c $SUPERVISOR_CONF +} + +# Increase max number of open connections +ulimit -n 65536 + +# Update pgsodium root key +if [ "${PGSODIUM_ROOT_KEY:-}" ]; then + echo "${PGSODIUM_ROOT_KEY}" > /etc/postgresql-custom/pgsodium_root.key +fi + +# Update pgdata directory +if [ "${PGDATA_REAL:-}" ]; then + mkdir -p "${PGDATA_REAL}" + chown -R postgres:postgres "${PGDATA_REAL}" + chmod -R g+rx "${PGDATA_REAL}" +fi + +if [ "${PGDATA:-}" ]; then + if [ "${PGDATA_REAL:-}" ]; then + mkdir -p "$(dirname "${PGDATA}")" + rm -rf "${PGDATA}" + ln -s "${PGDATA_REAL}" "${PGDATA}" + chmod -R g+rx "${PGDATA}" + else + mkdir -p "$PGDATA" + chown postgres:postgres "$PGDATA" + fi + sed -i "s|data_directory = '.*'|data_directory = '$PGDATA'|g" $PG_CONF +fi + +# Download and extract init payload from s3 +export INIT_PAYLOAD_PATH=${INIT_PAYLOAD_PATH:-/tmp/payload.tar.gz} + +if [ "${INIT_PAYLOAD_PRESIGNED_URL:-}" ]; then + curl -fsSL "$INIT_PAYLOAD_PRESIGNED_URL" -o "/tmp/payload.tar.gz" || true + if [ -f "/tmp/payload.tar.gz" ] && [ "/tmp/payload.tar.gz" != "$INIT_PAYLOAD_PATH" ] ; then + mv "/tmp/payload.tar.gz" "$INIT_PAYLOAD_PATH" + fi +fi + +if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then + BASE_LOGS_FOLDER="${DATA_VOLUME_MOUNTPOINT}/logs" + + for folder in "postgresql" "services" "wal-g"; do + mkdir -p "${BASE_LOGS_FOLDER}/${folder}" + rm -rf "/var/log/${folder}" + ln -s "${BASE_LOGS_FOLDER}/${folder}" "/var/log/${folder}" + done + + chown -R postgres:postgres "${BASE_LOGS_FOLDER}" + + mkdir -p "${DATA_VOLUME_MOUNTPOINT}/etc/logrotate" +fi + +# Process init payload +if [ -f "$INIT_PAYLOAD_PATH" ]; then + setup_credentials + setup_postgres +else + echo "Skipped extracting init payload: $INIT_PAYLOAD_PATH does not exist" +fi + +mkdir -p /var/log/services + +SUPERVISOR_CONF=/etc/supervisor/supervisord.conf +find /etc/supervisor/ -type d -exec chmod 0770 {} + +find /etc/supervisor/ -type f -exec chmod 0660 {} + + +# Start services in the background +if [ "${POSTGRES_ONLY:-}" == "true" ]; then + sed -i "s| - postgrest| # - postgrest|g" /etc/adminapi/adminapi.yaml + sed -i "s|files = services/\*.conf base-services/\*.conf|files = base-services/\*.conf|g" $SUPERVISOR_CONF + /init/configure-adminapi.sh +else + sed -i "s| # - postgrest| - postgrest|g" /etc/adminapi/adminapi.yaml + sed -i "s|files = base-services/\*.conf|files = services/\*.conf base-services/\*.conf|g" $SUPERVISOR_CONF + configure_services +fi + +if [ "${AUTOSHUTDOWN_ENABLED:-}" == "true" ]; then + enable_autoshutdown +fi + +if [ "${ENVOY_ENABLED:-}" == "true" ]; then + sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/services/envoy.conf + sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/kong.conf + sed -i "s/kong/envoy/" /etc/supervisor/services/group.conf +fi + +if [ "${FAIL2BAN_DISABLED:-}" == "true" ]; then + disable_fail2ban +fi + +if [ "${GOTRUE_DISABLED:-}" == "true" ]; then + sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/gotrue.conf + sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/gotrue.conf +fi + +if [ "${PLATFORM_DEPLOYMENT:-}" == "true" ]; then + if [ "${SWAP_DISABLED:-}" != "true" ]; then + enable_swap + fi + enable_lsn_checkpoint_push + + trap graceful_shutdown SIGINT +fi + +touch "$CONFIGURED_FLAG_PATH" +run_prelaunch_hooks +start_supervisor +push_lsn_checkpoint_file diff --git a/docker/all-in-one-nix/etc/adminapi/adminapi.yaml b/docker/all-in-one-nix/etc/adminapi/adminapi.yaml new file mode 100644 index 000000000..6eaf643e1 --- /dev/null +++ b/docker/all-in-one-nix/etc/adminapi/adminapi.yaml @@ -0,0 +1,76 @@ +port: 8085 +host: 0.0.0.0 +ref: {{ .ProjectRef }} +jwt_secret: {{ .JwtSecret }} +metric_collectors: + - filesystem + - meminfo + - netdev + - loadavg + - cpu + - diskstats + - vmstat +node_exporter_additional_args: + - "--collector.filesystem.ignored-mount-points=^/(boot|sys|dev|run).*" + - "--collector.netdev.device-exclude=lo" +# cert_path: /etc/ssl/adminapi/server.crt +# key_path: /etc/ssl/adminapi/server.key +upstream_metrics_refresh_duration: 60s +pgbouncer_endpoints: + - "postgres://pgbouncer:{{ .PgbouncerPassword }}@localhost:6543/pgbouncer" +fail2ban_socket: /var/run/fail2ban/fail2ban.sock +upstream_metrics_sources: + - name: system + url: "https://localhost:8085/metrics" + labels_to_attach: + - name: supabase_project_ref + value: {{ .ProjectRef }} + - name: service_type + value: db + skip_tls_verify: true + - name: postgresql + url: "http://localhost:9187/metrics" + labels_to_attach: + - name: supabase_project_ref + value: {{ .ProjectRef }} + - name: service_type + value: postgresql + - name: gotrue + url: "http://localhost:9122/metrics" + labels_to_attach: + - name: supabase_project_ref + value: {{ .ProjectRef }} + - name: service_type + value: gotrue +monitoring: + disk_usage: + enabled: true +upgrades_config: + region: us-east-1 + s3_bucket_name: supabase-internal-artifacts-prod-bucket + common_prefix: upgrades + destination_dir: /tmp +firewall: + enabled: true + internal_ports: + - 9187 + - 8085 + - 9122 + privileged_ports: + - 22 + privileged_ports_allowlist: + - 0.0.0.0/0 + filtered_ports: + - 5432 + - 6543 + unfiltered_ports: + - 80 + - 443 + managed_rules_file: /etc/nftables/supabase_managed.conf +pg_egress_collect_path: /tmp/pg_egress_collect.txt +health_reporting: + api_url: {{ .SupabaseUrl }} + project_token: {{ .ReportingToken }} + check_services: + # - postgres + # - postgrest diff --git a/docker/all-in-one-nix/etc/fail2ban/filter.d/pgbouncer.conf b/docker/all-in-one-nix/etc/fail2ban/filter.d/pgbouncer.conf new file mode 100644 index 000000000..b2d59c1b3 --- /dev/null +++ b/docker/all-in-one-nix/etc/fail2ban/filter.d/pgbouncer.conf @@ -0,0 +1,2 @@ +[Definition] +failregex = ^.+@:.+error: password authentication failed$ diff --git a/docker/all-in-one-nix/etc/fail2ban/filter.d/postgresql.conf b/docker/all-in-one-nix/etc/fail2ban/filter.d/postgresql.conf new file mode 100644 index 000000000..4c708069d --- /dev/null +++ b/docker/all-in-one-nix/etc/fail2ban/filter.d/postgresql.conf @@ -0,0 +1,8 @@ +[Definition] +failregex = ^.*,.*,.*,.*,":.*password authentication failed for user.*$ +ignoreregex = ^.*,.*,.*,.*,"127\.0\.0\.1.*password authentication failed for user.*$ + ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_admin".*$ + ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_auth_admin".*$ + ^.*,.*,.*,.*,":.*password authentication failed for user ""supabase_storage_admin".*$ + ^.*,.*,.*,.*,":.*password authentication failed for user ""authenticator".*$ + ^.*,.*,.*,.*,":.*password authentication failed for user ""pgbouncer".*$ diff --git a/docker/all-in-one-nix/etc/fail2ban/jail.d/jail.local b/docker/all-in-one-nix/etc/fail2ban/jail.d/jail.local new file mode 100644 index 000000000..44e8210f1 --- /dev/null +++ b/docker/all-in-one-nix/etc/fail2ban/jail.d/jail.local @@ -0,0 +1,4 @@ +[DEFAULT] + +banaction = nftables-multiport +banaction_allports = nftables-allports diff --git a/docker/all-in-one-nix/etc/fail2ban/jail.d/pgbouncer.conf b/docker/all-in-one-nix/etc/fail2ban/jail.d/pgbouncer.conf new file mode 100644 index 000000000..c8b3c49c5 --- /dev/null +++ b/docker/all-in-one-nix/etc/fail2ban/jail.d/pgbouncer.conf @@ -0,0 +1,7 @@ +[pgbouncer] +enabled = true +port = 6543 +protocol = tcp +filter = pgbouncer +logpath = /var/log/services/pgbouncer.log +maxretry = 3 diff --git a/docker/all-in-one-nix/etc/fail2ban/jail.d/postgresql.conf b/docker/all-in-one-nix/etc/fail2ban/jail.d/postgresql.conf new file mode 100644 index 000000000..0ec1819d6 --- /dev/null +++ b/docker/all-in-one-nix/etc/fail2ban/jail.d/postgresql.conf @@ -0,0 +1,8 @@ +[postgresql] +enabled = true +port = 5432 +protocol = tcp +filter = postgresql +logpath = /var/log/postgresql/auth-failures.csv +maxretry = 3 +ignoreip = 192.168.0.0/16 172.17.1.0/20 diff --git a/docker/all-in-one-nix/etc/fail2ban/jail.d/sshd.local b/docker/all-in-one-nix/etc/fail2ban/jail.d/sshd.local new file mode 100644 index 000000000..703373833 --- /dev/null +++ b/docker/all-in-one-nix/etc/fail2ban/jail.d/sshd.local @@ -0,0 +1,3 @@ +[sshd] + +enabled = false diff --git a/docker/all-in-one-nix/etc/gotrue.env b/docker/all-in-one-nix/etc/gotrue.env new file mode 100644 index 000000000..eb1e1a3c6 --- /dev/null +++ b/docker/all-in-one-nix/etc/gotrue.env @@ -0,0 +1,9 @@ +API_EXTERNAL_URL=api_external_url +GOTRUE_API_HOST=gotrue_api_host +GOTRUE_SITE_URL=gotrue_site_url +GOTRUE_DB_DRIVER=postgres +GOTRUE_DB_DATABASE_URL=postgres://supabase_auth_admin@localhost/postgres?sslmode=disable +GOTRUE_DB_MIGRATIONS_PATH=/opt/gotrue/migrations +GOTRUE_JWT_ADMIN_ROLES=supabase_admin,service_role +GOTRUE_JWT_AUD=authenticated +GOTRUE_JWT_SECRET=gotrue_jwt_secret diff --git a/docker/all-in-one-nix/etc/kong/kong.conf b/docker/all-in-one-nix/etc/kong/kong.conf new file mode 100644 index 000000000..8c0c93649 --- /dev/null +++ b/docker/all-in-one-nix/etc/kong/kong.conf @@ -0,0 +1,35 @@ +database = off +declarative_config = /etc/kong/kong.yml + +# plugins defined in the dockerfile +plugins = request-transformer,cors,key-auth,basic-auth,http-log,ip-restriction,rate-limiting + +admin_listen = off +proxy_listen = 0.0.0.0:80 reuseport backlog=16384, 0.0.0.0:443 http2 ssl reuseport backlog=16834, [::]:80 reuseport backlog=16384, [::]:443 http2 ssl reuseport backlog=16348 + +nginx_http_log_format = custom_log '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time $request_length' +nginx_http_client_body_buffer_size = 512k +proxy_access_log = off +dns_stale_ttl = 60 +nginx_proxy_proxy_max_temp_file_size = 0 +nginx_proxy_proxy_buffer_size = 128k +nginx_proxy_proxy_buffers = 4 256k +nginx_proxy_proxy_busy_buffers_size = 256k +nginx_proxy_proxy_read_timeout = 120s +nginx_proxy_proxy_ssl_verify = off +nginx_http_gzip=on +nginx_http_gzip_comp_level=6 +nginx_http_gzip_min_length=256 +nginx_http_gzip_proxied=any +nginx_http_gzip_vary=on +nginx_http_gzip_types=text/plain application/xml application/openapi+json application/json + +# the upstream requests will be timed out after 60s idle anyway +# this ensures that we're not unnecessarily cycling them +upstream_keepalive_max_requests = 0 +# the pool size can be (and ought to be) scaled up on larger instances +upstream_keepalive_pool_size = 500 + +nginx_events_use = epoll +# can be tuned to be higher on larger boxes (4096 is totally fine) +nginx_events_worker_connections = 1024 diff --git a/docker/all-in-one-nix/etc/kong/kong.yml b/docker/all-in-one-nix/etc/kong/kong.yml new file mode 100644 index 000000000..53ad4baef --- /dev/null +++ b/docker/all-in-one-nix/etc/kong/kong.yml @@ -0,0 +1,88 @@ +# ############################################################################################## +# Updating this file also requires a corresponding update in worker/src/lib/config-utils/kong.ts +# ############################################################################################## +_format_version: '1.1' +services: + - { + name: auth-v1-open, + url: 'http://localhost:9999/verify', + routes: [{ name: auth-v1-open, strip_path: true, paths: [/auth/v1/verify] }], + plugins: [{ name: cors }], + } + - { + name: auth-v1-open-callback, + url: 'http://localhost:9999/callback', + routes: [{ name: auth-v1-open-callback, strip_path: true, paths: [/auth/v1/callback] }], + plugins: [{ name: cors }], + } + - { + name: auth-v1-open-authorize, + url: 'http://localhost:9999/authorize', + routes: [{ name: auth-v1-open-authorize, strip_path: true, paths: [/auth/v1/authorize] }], + plugins: [{ name: cors }], + } + - { + name: auth-v1-open-saml, + url: 'http://localhost:9999/sso/saml/', + routes: [{ name: auth-v1-open-saml, strip_path: true, paths: [/auth/v1/sso/saml/] }], + plugins: [{ name: cors }], + } + - { + name: auth-v1, + url: 'http://localhost:9999/', + routes: [{ name: auth-v1, strip_path: true, paths: [/auth/v1/] }], + plugins: [{ name: cors }, { name: key-auth, config: { hide_credentials: false } }], + } + - { + name: rest-v1-admin, + url: 'http://localhost:3001/', + routes: [{ name: rest-admin-v1, strip_path: true, paths: [/rest-admin/v1/] }], + plugins: [{ name: cors }, { name: key-auth, config: { hide_credentials: true } }], + } + - { + name: rest-v1, + url: 'http://localhost:3000/', + routes: [{ name: rest-v1, strip_path: true, paths: [/rest/v1/] }], + plugins: [{ name: cors }, { name: key-auth, config: { hide_credentials: true } }], + } + - { + name: graphql-v1, + url: 'http://localhost:3000/rpc/graphql', + routes: [{ name: graphql-v1, strip_path: true, paths: [/graphql/v1] }], + plugins: + [ + { name: cors }, + { name: key-auth, config: { hide_credentials: true } }, + { + name: request-transformer, + config: { add: { headers: [Content-Profile:graphql_public] } }, + }, + ], + } + - { + name: admin-v1, + url: 'https://localhost:8085/', + routes: [{ name: admin-v1, strip_path: true, paths: [/admin/v1/] }], + plugins: [{ name: cors }, { name: key-auth, config: { hide_credentials: false } }], + } + - { + name: admin-v1-user-routes, + url: 'https://localhost:8085/privileged', + routes: [{ name: admin-v1-user-routes, strip_path: true, paths: [/customer/v1/privileged] }], + plugins: [{ name: cors }, { name: basic-auth, config: { hide_credentials: false } }], + } + - { + name: admin-v1-metrics, + url: 'https://localhost:8085/metrics/aggregated', + routes: [{ name: admin-v1-metrics, strip_path: true, paths: [/supabase-internal/metrics] }], + plugins: [{ name: cors }, { name: ip-restriction, config: { allow: [10.0.0.0/8] } }], + } +consumers: + - { username: anon-key, keyauth_credentials: [{ key: anon_key }] } + - { username: service_role-key, keyauth_credentials: [{ key: service_key }] } + - { username: supabase-admin-key, keyauth_credentials: [{ key: supabase_admin_key }] } +basicauth_credentials: + - consumer: service_role-key + username: 'service_role' + password: service_key +plugins: [] diff --git a/docker/all-in-one-nix/etc/logrotate.d/postgresql.conf b/docker/all-in-one-nix/etc/logrotate.d/postgresql.conf new file mode 100644 index 000000000..6e2b8828a --- /dev/null +++ b/docker/all-in-one-nix/etc/logrotate.d/postgresql.conf @@ -0,0 +1,11 @@ +/var/log/postgresql/postgresql.csv { + size 50M + rotate 4 + compress + delaycompress + notifempty + missingok + postrotate + sudo -u postgres /usr/lib/postgresql/15/bin/pg_ctl -D /var/lib/postgresql/data logrotate + endscript +} diff --git a/docker/all-in-one-nix/etc/logrotate.d/walg.conf b/docker/all-in-one-nix/etc/logrotate.d/walg.conf new file mode 100644 index 000000000..49eeb59eb --- /dev/null +++ b/docker/all-in-one-nix/etc/logrotate.d/walg.conf @@ -0,0 +1,9 @@ +/var/log/wal-g/*.log { + size 50M + rotate 3 + copytruncate + delaycompress + compress + notifempty + missingok +} diff --git a/docker/all-in-one-nix/etc/pgbouncer-custom/custom-overrides.ini b/docker/all-in-one-nix/etc/pgbouncer-custom/custom-overrides.ini new file mode 100644 index 000000000..e69de29bb diff --git a/docker/all-in-one-nix/etc/pgbouncer-custom/generated-optimizations.ini b/docker/all-in-one-nix/etc/pgbouncer-custom/generated-optimizations.ini new file mode 100644 index 000000000..e69de29bb diff --git a/docker/all-in-one-nix/etc/pgbouncer-custom/ssl-config.ini b/docker/all-in-one-nix/etc/pgbouncer-custom/ssl-config.ini new file mode 100644 index 000000000..69a802500 --- /dev/null +++ b/docker/all-in-one-nix/etc/pgbouncer-custom/ssl-config.ini @@ -0,0 +1,4 @@ +client_tls_sslmode = allow +client_tls_ca_file = /etc/ssl/certs/postgres/ca.crt +client_tls_key_file = /etc/ssl/private/server.key +client_tls_cert_file = /etc/ssl/certs/postgres/server.crt diff --git a/docker/all-in-one-nix/etc/pgbouncer/pgbouncer.ini b/docker/all-in-one-nix/etc/pgbouncer/pgbouncer.ini new file mode 100644 index 000000000..5a36ac197 --- /dev/null +++ b/docker/all-in-one-nix/etc/pgbouncer/pgbouncer.ini @@ -0,0 +1,363 @@ +;;; +;;; PgBouncer configuration file +;;; + +;; database name = connect string +;; +;; connect string params: +;; dbname= host= port= user= password= auth_user= +;; client_encoding= datestyle= timezone= +;; pool_size= reserve_pool= max_db_connections= +;; pool_mode= connect_query= application_name= +[databases] +* = host=localhost auth_user=pgbouncer + +;; foodb over Unix socket +;foodb = + +;; redirect bardb to bazdb on localhost +;bardb = host=localhost dbname=bazdb + +;; access to dest database will go with single user +;forcedb = host=localhost port=300 user=baz password=foo client_encoding=UNICODE datestyle=ISO connect_query='SELECT 1' + +;; use custom pool sizes +;nondefaultdb = pool_size=50 reserve_pool=10 + +;; use auth_user with auth_query if user not present in auth_file +;; auth_user must exist in auth_file +; foodb = auth_user=bar + +;; fallback connect string +;* = host=testserver + +;; User-specific configuration +[users] + +;user1 = pool_mode=transaction max_user_connections=10 + +;; Configuration section +[pgbouncer] + +;;; +;;; Administrative settings +;;; + +pidfile = /var/run/pgbouncer/pgbouncer.pid + +;;; +;;; Where to wait for clients +;;; + +;; IP address or * which means all IPs +listen_addr = * +listen_port = 6543 + +;; Unix socket is also used for -R. +;; On Debian it should be /var/run/postgresql +unix_socket_dir = /tmp +;unix_socket_mode = 0777 +;unix_socket_group = + +;;; +;;; TLS settings for accepting clients +;;; + +;; disable, allow, require, verify-ca, verify-full +;client_tls_sslmode = disable + +;; Path to file that contains trusted CA certs +;client_tls_ca_file = + +;; Private key and cert to present to clients. +;; Required for accepting TLS connections from clients. +;client_tls_key_file = +;client_tls_cert_file = + +;; fast, normal, secure, legacy, +;client_tls_ciphers = fast + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;client_tls_protocols = secure + +;; none, auto, legacy +;client_tls_dheparams = auto + +;; none, auto, +;client_tls_ecdhcurve = auto + +;;; +;;; TLS settings for connecting to backend databases +;;; + +;; disable, allow, require, verify-ca, verify-full +;server_tls_sslmode = disable + +;; Path to that contains trusted CA certs +;server_tls_ca_file = + +;; Private key and cert to present to backend. +;; Needed only if backend server require client cert. +;server_tls_key_file = +;server_tls_cert_file = + +;; all, secure, tlsv1.0, tlsv1.1, tlsv1.2, tlsv1.3 +;server_tls_protocols = secure + +;; fast, normal, secure, legacy, +;server_tls_ciphers = fast + +;;; +;;; Authentication settings +;;; + +;; any, trust, plain, md5, cert, hba, pam +auth_type = scram-sha-256 +auth_file = /etc/pgbouncer/userlist.txt + +;; Path to HBA-style auth config +;auth_hba_file = + +;; Query to use to fetch password from database. Result +;; must have 2 columns - username and password hash. +auth_query = SELECT * FROM pgbouncer.get_auth($1) + +;;; +;;; Users allowed into database 'pgbouncer' +;;; + +;; comma-separated list of users who are allowed to change settings +admin_users = pgbouncer + +;; comma-separated list of users who are just allowed to use SHOW command +stats_users = pgbouncer + +;;; +;;; Pooler personality questions +;;; + +;; When server connection is released back to pool: +;; session - after client disconnects (default) +;; transaction - after transaction finishes +;; statement - after statement finishes +pool_mode = transaction + +;; Query for cleaning connection immediately after releasing from +;; client. No need to put ROLLBACK here, pgbouncer does not reuse +;; connections where transaction is left open. +;server_reset_query = DISCARD ALL + +;; Whether server_reset_query should run in all pooling modes. If it +;; is off, server_reset_query is used only for session-pooling. +;server_reset_query_always = 0 + +;; Comma-separated list of parameters to ignore when given in startup +;; packet. Newer JDBC versions require the extra_float_digits here. +ignore_startup_parameters = extra_float_digits + +;; When taking idle server into use, this query is run first. +;server_check_query = select 1 + +;; If server was used more recently that this many seconds ago, +; skip the check query. Value 0 may or may not run in immediately. +;server_check_delay = 30 + +;; Close servers in session pooling mode after a RECONNECT, RELOAD, +;; etc. when they are idle instead of at the end of the session. +;server_fast_close = 0 + +;; Use as application_name on server. +;application_name_add_host = 0 + +;; Period for updating aggregated stats. +;stats_period = 60 + +;;; +;;; Connection limits +;;; + +;; Total number of clients that can connect +;max_client_conn = 100 + +;; Default pool size. 20 is good number when transaction pooling +;; is in use, in session pooling it needs to be the number of +;; max clients you want to handle at any moment +default_pool_size = 15 + +;; Minimum number of server connections to keep in pool. +;min_pool_size = 0 + +; how many additional connection to allow in case of trouble +;reserve_pool_size = 0 + +;; If a clients needs to wait more than this many seconds, use reserve +;; pool. +;reserve_pool_timeout = 5 + +;; Maximum number of server connections for a database +;max_db_connections = 0 + +;; Maximum number of server connections for a user +;max_user_connections = 0 + +;; If off, then server connections are reused in LIFO manner +;server_round_robin = 0 + +;;; +;;; Logging +;;; + +;; Syslog settings +;syslog = 0 +;syslog_facility = daemon +;syslog_ident = pgbouncer + +;; log if client connects or server connection is made +;log_connections = 1 + +;; log if and why connection was closed +;log_disconnections = 1 + +;; log error messages pooler sends to clients +;log_pooler_errors = 1 + +;; write aggregated stats into log +;log_stats = 1 + +;; Logging verbosity. Same as -v switch on command line. +;verbose = 0 + +;;; +;;; Timeouts +;;; + +;; Close server connection if its been connected longer. +;server_lifetime = 3600 + +;; Close server connection if its not been used in this time. Allows +;; to clean unnecessary connections from pool after peak. +;server_idle_timeout = 600 + +;; Cancel connection attempt if server does not answer takes longer. +;server_connect_timeout = 15 + +;; If server login failed (server_connect_timeout or auth failure) +;; then wait this many second. +;server_login_retry = 15 + +;; Dangerous. Server connection is closed if query does not return in +;; this time. Should be used to survive network problems, _not_ as +;; statement_timeout. (default: 0) +;query_timeout = 0 + +;; Dangerous. Client connection is closed if the query is not +;; assigned to a server in this time. Should be used to limit the +;; number of queued queries in case of a database or network +;; failure. (default: 120) +;query_wait_timeout = 120 + +;; Dangerous. Client connection is closed if no activity in this +;; time. Should be used to survive network problems. (default: 0) +;client_idle_timeout = 0 + +;; Disconnect clients who have not managed to log in after connecting +;; in this many seconds. +;client_login_timeout = 60 + +;; Clean automatically created database entries (via "*") if they stay +;; unused in this many seconds. +; autodb_idle_timeout = 3600 + +;; Close connections which are in "IDLE in transaction" state longer +;; than this many seconds. +;idle_transaction_timeout = 0 + +;; How long SUSPEND/-R waits for buffer flush before closing +;; connection. +;suspend_timeout = 10 + +;;; +;;; Low-level tuning options +;;; + +;; buffer for streaming packets +;pkt_buf = 4096 + +;; man 2 listen +;listen_backlog = 128 + +;; Max number pkt_buf to process in one event loop. +;sbuf_loopcnt = 5 + +;; Maximum PostgreSQL protocol packet size. +;max_packet_size = 2147483647 + +;; Set SO_REUSEPORT socket option +;so_reuseport = 0 + +;; networking options, for info: man 7 tcp + +;; Linux: Notify program about new connection only if there is also +;; data received. (Seconds to wait.) On Linux the default is 45, on +;; other OS'es 0. +;tcp_defer_accept = 0 + +;; In-kernel buffer size (Linux default: 4096) +;tcp_socket_buffer = 0 + +;; whether tcp keepalive should be turned on (0/1) +;tcp_keepalive = 1 + +;; The following options are Linux-specific. They also require +;; tcp_keepalive=1. + +;; Count of keepalive packets +;tcp_keepcnt = 0 + +;; How long the connection can be idle before sending keepalive +;; packets +;tcp_keepidle = 0 + +;; The time between individual keepalive probes +;tcp_keepintvl = 0 + +;; How long may transmitted data remain unacknowledged before TCP +;; connection is closed (in milliseconds) +;tcp_user_timeout = 0 + +;; DNS lookup caching time +;dns_max_ttl = 15 + +;; DNS zone SOA lookup period +;dns_zone_check_period = 0 + +;; DNS negative result caching time +;dns_nxdomain_ttl = 15 + +;; Custom resolv.conf file, to set custom DNS servers or other options +;; (default: empty = use OS settings) +;resolv_conf = /etc/pgbouncer/resolv.conf + +;;; +;;; Random stuff +;;; + +;; Hackish security feature. Helps against SQL injection: when PQexec +;; is disabled, multi-statement cannot be made. +;disable_pqexec = 0 + +;; Config file to use for next RELOAD/SIGHUP +;; By default contains config file from command line. +;conffile + +;; Windows service name to register as. job_name is alias for +;; service_name, used by some Skytools scripts. +;service_name = pgbouncer +;job_name = pgbouncer + +;; Read additional config from other file +;%include /etc/pgbouncer/pgbouncer-other.ini + +%include /etc/pgbouncer-custom/generated-optimizations.ini +%include /etc/pgbouncer-custom/custom-overrides.ini +# %include /etc/pgbouncer-custom/ssl-config.ini diff --git a/docker/all-in-one-nix/etc/pgbouncer/userlist.txt b/docker/all-in-one-nix/etc/pgbouncer/userlist.txt new file mode 100644 index 000000000..e69de29bb diff --git a/docker/all-in-one-nix/etc/postgresql-custom/custom-overrides.conf b/docker/all-in-one-nix/etc/postgresql-custom/custom-overrides.conf new file mode 100644 index 000000000..e69de29bb diff --git a/docker/all-in-one-nix/etc/postgresql-custom/generated-optimizations.conf b/docker/all-in-one-nix/etc/postgresql-custom/generated-optimizations.conf new file mode 100644 index 000000000..e69de29bb diff --git a/docker/all-in-one-nix/etc/postgresql-custom/postgresql-platform-defaults.conf b/docker/all-in-one-nix/etc/postgresql-custom/postgresql-platform-defaults.conf new file mode 100644 index 000000000..e62a1de83 --- /dev/null +++ b/docker/all-in-one-nix/etc/postgresql-custom/postgresql-platform-defaults.conf @@ -0,0 +1,9 @@ +# these get imported _after_ the user specified overrides +row_security = on +wal_level = logical +max_wal_senders = 10 +max_replication_slots = 5 +log_connections = on +statement_timeout = 120000 +jit = off +pgaudit.log = 'ddl' diff --git a/docker/all-in-one-nix/etc/postgresql.schema.sql b/docker/all-in-one-nix/etc/postgresql.schema.sql new file mode 100644 index 000000000..475b09bb9 --- /dev/null +++ b/docker/all-in-one-nix/etc/postgresql.schema.sql @@ -0,0 +1,16 @@ +\set admin_pass `echo "${SUPABASE_ADMIN_PASSWORD:-$POSTGRES_PASSWORD}"` +\set pgrst_pass `echo "${AUTHENTICATOR_PASSWORD:-$POSTGRES_PASSWORD}"` +\set pgbouncer_pass `echo "${PGBOUNCER_PASSWORD:-$POSTGRES_PASSWORD}"` +\set auth_pass `echo "${SUPABASE_AUTH_ADMIN_PASSWORD:-$POSTGRES_PASSWORD}"` +\set storage_pass `echo "${SUPABASE_STORAGE_ADMIN_PASSWORD:-$POSTGRES_PASSWORD}"` +\set replication_pass `echo "${SUPABASE_REPLICATION_ADMIN_PASSWORD:-$POSTGRES_PASSWORD}"` +\set read_only_pass `echo "${SUPABASE_READ_ONLY_USER_PASSWORD:-$POSTGRES_PASSWORD}"` + +ALTER USER supabase_admin WITH PASSWORD :'admin_pass'; +ALTER USER authenticator WITH PASSWORD :'pgrst_pass'; +ALTER USER pgbouncer WITH PASSWORD :'pgbouncer_pass'; +ALTER USER supabase_auth_admin WITH PASSWORD :'auth_pass'; +ALTER USER supabase_storage_admin WITH PASSWORD :'storage_pass'; +ALTER USER supabase_replication_admin WITH PASSWORD :'replication_pass'; +ALTER ROLE supabase_read_only_user WITH PASSWORD :'read_only_pass'; +ALTER ROLE supabase_admin SET search_path TO "$user",public,auth,extensions; diff --git a/docker/all-in-one-nix/etc/postgresql/logging.conf b/docker/all-in-one-nix/etc/postgresql/logging.conf new file mode 100644 index 000000000..b8d64da51 --- /dev/null +++ b/docker/all-in-one-nix/etc/postgresql/logging.conf @@ -0,0 +1,33 @@ +# - Where to Log - + +log_destination = 'csvlog' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. + +# This is used when logging to stderr: +logging_collector = on # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +log_directory = '/var/log/postgresql' # directory where log files are written, + # can be absolute or relative to PGDATA +log_filename = 'postgresql.log' # log file name pattern, + # can include strftime() escapes +log_file_mode = 0640 # creation mode for log files, + # begin with 0 to use octal notation +log_rotation_age = 0 # Automatic rotation of logfiles will + # happen after that time. 0 disables. +log_rotation_size = 0 # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. diff --git a/docker/all-in-one-nix/etc/postgresql/pg_hba.conf b/docker/all-in-one-nix/etc/postgresql/pg_hba.conf new file mode 100755 index 000000000..9cafd4146 --- /dev/null +++ b/docker/all-in-one-nix/etc/postgresql/pg_hba.conf @@ -0,0 +1,94 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# non-SSL TCP/IP socket. Similarly, "hostgssenc" uses a +# GSSAPI-encrypted TCP/IP socket, while "hostnogssenc" uses a +# non-GSSAPI socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", +# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". +# Note that "password" sends passwords in clear text; "md5" or +# "scram-sha-256" are preferred since they send encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# "all", "sameuser", "samerole" or "replication" makes the name lose +# its special character, and just match a database or username with +# that name. +# +# This file is read on server startup and when the server receives a +# SIGHUP signal. If you edit the file on a running system, you have to +# SIGHUP the server for the changes to take effect, run "pg_ctl reload", +# or execute "SELECT pg_reload_conf()". +# +# Put your actual configuration here +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + +# TYPE DATABASE USER ADDRESS METHOD + +# trust local connections +local all supabase_admin scram-sha-256 +local all all peer map=supabase_map +host all all 127.0.0.1/32 trust +host all all ::1/128 trust + +# IPv4 external connections +host all all 10.0.0.0/8 scram-sha-256 +host all all 172.16.0.0/12 scram-sha-256 +host all all 192.168.0.0/16 scram-sha-256 +host all all 0.0.0.0/0 scram-sha-256 + +# IPv6 external connections +host all all ::0/0 scram-sha-256 diff --git a/docker/all-in-one-nix/etc/postgrest/base.conf b/docker/all-in-one-nix/etc/postgrest/base.conf new file mode 100644 index 000000000..e5120ede6 --- /dev/null +++ b/docker/all-in-one-nix/etc/postgrest/base.conf @@ -0,0 +1,7 @@ +server-port="pgrst_server_port" +admin-server-port="pgrst_admin_server_port" +db-schema="pgrst_db_schemas" +db-extra-search-path="pgrst_db_extra_search_path" +db-anon-role="pgrst_db_anon_role" +jwt-secret="pgrst_jwt_secret" +db-uri="postgres://authenticator@localhost:5432/postgres?application_name=postgrest" diff --git a/docker/all-in-one-nix/etc/postgrest/bootstrap.sh b/docker/all-in-one-nix/etc/postgrest/bootstrap.sh new file mode 100755 index 000000000..9ac21d201 --- /dev/null +++ b/docker/all-in-one-nix/etc/postgrest/bootstrap.sh @@ -0,0 +1,8 @@ +#! /usr/bin/env bash +set -euo pipefail +set -x + +cd "$(dirname "$0")" +cat $@ > merged.conf + +/opt/postgrest merged.conf diff --git a/docker/all-in-one-nix/etc/postgrest/generated.conf b/docker/all-in-one-nix/etc/postgrest/generated.conf new file mode 100644 index 000000000..e69de29bb diff --git a/docker/all-in-one-nix/etc/sudoers.d/adminapi b/docker/all-in-one-nix/etc/sudoers.d/adminapi new file mode 100644 index 000000000..eeff5fb9c --- /dev/null +++ b/docker/all-in-one-nix/etc/sudoers.d/adminapi @@ -0,0 +1,27 @@ +Cmnd_Alias ENVOY = /usr/bin/supervisorctl start services\:envoy, /usr/bin/supervisorctl stop services\:envoy, /usr/bin/supervisorctl restart services\:envoy, /usr/bin/supervisorctl status services\:envoy +Cmnd_Alias KONG = /usr/bin/supervisorctl start services\:kong, /usr/bin/supervisorctl stop services\:kong, /usr/bin/supervisorctl restart services\:kong, /usr/bin/supervisorctl status services\:kong +Cmnd_Alias POSTGREST = /usr/bin/supervisorctl start services\:postgrest, /usr/bin/supervisorctl stop services\:postgrest, /usr/bin/supervisorctl restart services\:postgrest, /usr/bin/supervisorctl status services\:postgrest +Cmnd_Alias GOTRUE = /usr/bin/supervisorctl start services\:gotrue, /usr/bin/supervisorctl stop services\:gotrue, /usr/bin/supervisorctl restart services\:gotrue, /usr/bin/supervisorctl status services\:gotrue +Cmnd_Alias PGBOUNCER = /usr/bin/supervisorctl start pgbouncer, /usr/bin/supervisorctl stop pgbouncer, /usr/bin/supervisorctl restart pgbouncer, /usr/bin/supervisorctl status pgbouncer + +%adminapi ALL= NOPASSWD: /root/grow_fs.sh +%adminapi ALL= NOPASSWD: /root/manage_readonly_mode.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/prepare.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/initiate.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/complete.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/check.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/common.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/pgsodium_getkey.sh +%adminapi ALL= NOPASSWD: /usr/bin/supervisorctl reread +%adminapi ALL= NOPASSWD: /usr/bin/supervisorctl update +%adminapi ALL= NOPASSWD: /usr/bin/supervisorctl restart postgresql +%adminapi ALL= NOPASSWD: /usr/bin/supervisorctl status postgresql +%adminapi ALL= NOPASSWD: /usr/bin/supervisorctl restart adminapi +%adminapi ALL= NOPASSWD: /usr/bin/supervisorctl restart services\:* +%adminapi ALL= NOPASSWD: /usr/sbin/nft -f /etc/nftables/supabase_managed.conf +%adminapi ALL= NOPASSWD: /usr/bin/admin-mgr +%adminapi ALL= NOPASSWD: ENVOY +%adminapi ALL= NOPASSWD: KONG +%adminapi ALL= NOPASSWD: POSTGREST +%adminapi ALL= NOPASSWD: GOTRUE +%adminapi ALL= NOPASSWD: PGBOUNCER diff --git a/docker/all-in-one-nix/etc/supa-shutdown/shutdown.conf b/docker/all-in-one-nix/etc/supa-shutdown/shutdown.conf new file mode 100644 index 000000000..384b9357f --- /dev/null +++ b/docker/all-in-one-nix/etc/supa-shutdown/shutdown.conf @@ -0,0 +1 @@ +SHUTDOWN_IDLE_TIME_MINUTES= diff --git a/docker/all-in-one-nix/etc/supervisor/base-services/adminapi.conf b/docker/all-in-one-nix/etc/supervisor/base-services/adminapi.conf new file mode 100644 index 000000000..06db75257 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/base-services/adminapi.conf @@ -0,0 +1,9 @@ +[program:adminapi] +command=/opt/supabase-admin-api +user=adminapi +autorestart=true +autostart=true +stdout_logfile=/var/log/services/adminapi.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=50 diff --git a/docker/all-in-one-nix/etc/supervisor/base-services/logrotate.conf b/docker/all-in-one-nix/etc/supervisor/base-services/logrotate.conf new file mode 100644 index 000000000..638f45828 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/base-services/logrotate.conf @@ -0,0 +1,10 @@ +[program:logrotate] +command=/usr/local/bin/run-logrotate.sh +autostart=true +autorestart=true +user=root +stdout_logfile=/var/log/services/logrotate.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=50 +environment=DATA_VOLUME_MOUNTPOINT="%(ENV_DATA_VOLUME_MOUNTPOINT)s" diff --git a/docker/all-in-one-nix/etc/supervisor/base-services/lsn-checkpoint-push.conf b/docker/all-in-one-nix/etc/supervisor/base-services/lsn-checkpoint-push.conf new file mode 100644 index 000000000..9c4b92d3c --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/base-services/lsn-checkpoint-push.conf @@ -0,0 +1,9 @@ +[program:lsn-checkpoint-push] +command=/usr/bin/admin-mgr lsn-checkpoint-push --watch +user=root +autorestart=false +autostart=false +stdout_logfile=/var/log/services/lsn-push.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=50 diff --git a/docker/all-in-one-nix/etc/supervisor/base-services/pg_egress_collect.conf b/docker/all-in-one-nix/etc/supervisor/base-services/pg_egress_collect.conf new file mode 100644 index 000000000..de166beab --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/base-services/pg_egress_collect.conf @@ -0,0 +1,9 @@ +[program:pg_egress_collect] +command=/bin/bash -c "tcpdump -s 128 -Q out -nn -tt -vv -p -l 'tcp and (port 5432 or port 6543)' | perl /opt/pg_egress_collect/pg_egress_collect.pl" +user=root +autorestart=true +autostart=true +stdout_logfile=/var/log/services/pg_egress_collect.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=50 diff --git a/docker/all-in-one-nix/etc/supervisor/base-services/postgresql.conf b/docker/all-in-one-nix/etc/supervisor/base-services/postgresql.conf new file mode 100644 index 000000000..99f224310 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/base-services/postgresql.conf @@ -0,0 +1,12 @@ +[program:postgresql] +command=/usr/local/bin/postgres-entrypoint.sh postgres -D /etc/postgresql +user=postgres +stopsignal=INT +autorestart=true +autostart=true +priority=1 +# Inherit env vars from https://github.com/supabase/postgres/blob/develop/Dockerfile#L800 +environment=POSTGRES_PASSWORD="%(ENV_POSTGRES_PASSWORD)s",POSTGRES_HOST="%(ENV_POSTGRES_HOST)s" +stdout_logfile=/var/log/postgresql/init.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB diff --git a/docker/all-in-one-nix/etc/supervisor/base-services/supa-shutdown.conf b/docker/all-in-one-nix/etc/supervisor/base-services/supa-shutdown.conf new file mode 100644 index 000000000..82f97ff82 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/base-services/supa-shutdown.conf @@ -0,0 +1,10 @@ +[program:supa-shutdown] +command=/usr/local/bin/supa-shutdown.sh +user=root +autorestart=true +autostart=false +stdout_logfile=/var/log/services/supa-shutdown.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=50 +environment=MAX_IDLE_TIME_MINUTES="%(ENV_MAX_IDLE_TIME_MINUTES)s" diff --git a/docker/all-in-one-nix/etc/supervisor/services/envoy.conf b/docker/all-in-one-nix/etc/supervisor/services/envoy.conf new file mode 100644 index 000000000..dafdd1d16 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/services/envoy.conf @@ -0,0 +1,9 @@ +[program:envoy] +command=/opt/envoy-hot-restarter.py /opt/start-envoy.sh +user=envoy +autorestart=true +autostart=false +stopasgroup=true +stdout_logfile=/var/log/services/envoy.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB diff --git a/docker/all-in-one-nix/etc/supervisor/services/exporter.conf b/docker/all-in-one-nix/etc/supervisor/services/exporter.conf new file mode 100644 index 000000000..7844f5137 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/services/exporter.conf @@ -0,0 +1,10 @@ +[program:exporter] +command=/opt/postgres_exporter/postgres_exporter --disable-settings-metrics --extend.query-path=/opt/postgres_exporter/queries.yml --disable-default-metrics --no-collector.locks --no-collector.replication --no-collector.replication_slot --no-collector.stat_bgwriter --no-collector.stat_database --no-collector.stat_user_tables --no-collector.statio_user_tables --no-collector.wal +user=root +autorestart=true +autostart=true +environment=DATA_SOURCE_NAME="host=localhost dbname=postgres sslmode=disable user=supabase_admin pg_stat_statements.track=none application_name=postgres_exporter" +stdout_logfile=/var/log/services/exporter.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=150 diff --git a/docker/all-in-one-nix/etc/supervisor/services/fail2ban.conf b/docker/all-in-one-nix/etc/supervisor/services/fail2ban.conf new file mode 100644 index 000000000..8000386dc --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/services/fail2ban.conf @@ -0,0 +1,9 @@ +[program:fail2ban] +command=/usr/bin/fail2ban-client -f start +user=root +autorestart=true +autostart=true +stdout_logfile=/var/log/services/fail2ban.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=200 diff --git a/docker/all-in-one-nix/etc/supervisor/services/gotrue.conf b/docker/all-in-one-nix/etc/supervisor/services/gotrue.conf new file mode 100644 index 000000000..8095cdec1 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/services/gotrue.conf @@ -0,0 +1,10 @@ +[program:gotrue] +directory=/opt/gotrue +command=/opt/gotrue/gotrue --config /etc/gotrue.env +user=gotrue +startretries=30 +autorestart=true +autostart=true +stdout_logfile=/var/log/services/gotrue.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB diff --git a/docker/all-in-one-nix/etc/supervisor/services/group.conf b/docker/all-in-one-nix/etc/supervisor/services/group.conf new file mode 100644 index 000000000..ef6673d59 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/services/group.conf @@ -0,0 +1,3 @@ +[group:services] +programs=gotrue,kong,postgrest +priority=100 diff --git a/docker/all-in-one-nix/etc/supervisor/services/kong.conf b/docker/all-in-one-nix/etc/supervisor/services/kong.conf new file mode 100644 index 000000000..afe0e9564 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/services/kong.conf @@ -0,0 +1,10 @@ +[program:kong] +command=/init/start-kong.sh +user=kong +autorestart=true +autostart=true +stopasgroup=true +environment=KONG_NGINX_DAEMON="off" +stdout_logfile=/var/log/services/kong.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB diff --git a/docker/all-in-one-nix/etc/supervisor/services/pgbouncer.conf b/docker/all-in-one-nix/etc/supervisor/services/pgbouncer.conf new file mode 100644 index 000000000..44fe93b5f --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/services/pgbouncer.conf @@ -0,0 +1,10 @@ +[program:pgbouncer] +command=/usr/sbin/pgbouncer /etc/pgbouncer/pgbouncer.ini +user=pgbouncer +stopsignal=INT +autorestart=true +autostart=true +stdout_logfile=/var/log/services/pgbouncer.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=150 diff --git a/docker/all-in-one-nix/etc/supervisor/services/postgrest.conf b/docker/all-in-one-nix/etc/supervisor/services/postgrest.conf new file mode 100644 index 000000000..04faa7f77 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/services/postgrest.conf @@ -0,0 +1,9 @@ +[program:postgrest] +command=/etc/postgrest/bootstrap.sh /etc/postgrest/generated.conf /etc/postgrest/base.conf +user=postgrest +autorestart=true +autostart=true +stopasgroup=true +stdout_logfile=/var/log/services/postgrest.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB diff --git a/docker/all-in-one-nix/etc/supervisor/supervisord.conf b/docker/all-in-one-nix/etc/supervisor/supervisord.conf new file mode 100644 index 000000000..d64f40f71 --- /dev/null +++ b/docker/all-in-one-nix/etc/supervisor/supervisord.conf @@ -0,0 +1,170 @@ +; Sample supervisor config file. +; +; For more information on the config file, please see: +; http://supervisord.org/configuration.html +; +; Notes: +; - Shell expansion ("~" or "$HOME") is not supported. Environment +; variables can be expanded using this syntax: "%(ENV_HOME)s". +; - Quotes around values are not supported, except in the case of +; the environment= options as shown below. +; - Comments must have a leading space: "a=b ;comment" not "a=b;comment". +; - Command will be truncated if it looks like a config file comment, e.g. +; "command=bash -c 'foo ; bar'" will truncate to "command=bash -c 'foo ". +; +; Warning: +; Paths throughout this example file use /tmp because it is available on most +; systems. You will likely need to change these to locations more appropriate +; for your system. Some systems periodically delete older files in /tmp. +; Notably, if the socket file defined in the [unix_http_server] section below +; is deleted, supervisorctl will be unable to connect to supervisord. + +[unix_http_server] +file=/tmp/supervisor.sock ; the path to the socket file +chmod=0760 ; socket file mode (default 0700) +chown=root:root ; socket file uid:gid owner +;username=user ; default is no username (open server) +;password=123 ; default is no password (open server) + +; Security Warning: +; The inet HTTP server is not enabled by default. The inet HTTP server is +; enabled by uncommenting the [inet_http_server] section below. The inet +; HTTP server is intended for use within a trusted environment only. It +; should only be bound to localhost or only accessible from within an +; isolated, trusted network. The inet HTTP server does not support any +; form of encryption. The inet HTTP server does not use authentication +; by default (see the username= and password= options to add authentication). +; Never expose the inet HTTP server to the public internet. + +;[inet_http_server] ; inet (TCP) server disabled by default +;port=127.0.0.1:9001 ; ip_address:port specifier, *:port for all iface +;username=user ; default is no username (open server) +;password=123 ; default is no password (open server) + +[supervisord] +logfile=/tmp/supervisord.log ; main log file; default $CWD/supervisord.log +logfile_maxbytes=50MB ; max main logfile bytes b4 rotation; default 50MB +logfile_backups=10 ; # of main logfile backups; 0 means none, default 10 +loglevel=info ; log level; default info; others: debug,warn,trace +pidfile=/tmp/supervisord.pid ; supervisord pidfile; default supervisord.pid +nodaemon=true ; start in foreground if true; default false +silent=false ; no logs to stdout if true; default false +minfds=1024 ; min. avail startup file descriptors; default 1024 +minprocs=200 ; min. avail process descriptors;default 200 +user=root ; setuid to this UNIX account at startup; recommended if root +;umask=022 ; process file creation umask; default 022 +;identifier=supervisor ; supervisord identifier, default is 'supervisor' +;directory=/tmp ; default is not to cd during start +;nocleanup=true ; don't clean up tempfiles at start; default false +;childlogdir=/tmp ; 'AUTO' child log dir, default $TEMP +;environment=KEY="value" ; key value pairs to add to environment +;strip_ansi=false ; strip ansi escape codes in logs; def. false + +; The rpcinterface:supervisor section must remain in the config file for +; RPC (supervisorctl/web interface) to work. Additional interfaces may be +; added by defining them in separate [rpcinterface:x] sections. + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +; The supervisorctl section configures how supervisorctl will connect to +; supervisord. configure it match the settings in either the unix_http_server +; or inet_http_server section. + +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket +;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket +;username=chris ; should be same as in [*_http_server] if set +;password=123 ; should be same as in [*_http_server] if set +;prompt=mysupervisor ; cmd line prompt (default "supervisor") +;history_file=~/.sc_history ; use readline history if available + +; The sample program section below shows all possible program subsection values. +; Create one or more 'real' program: sections to be able to control them under +; supervisor. + +;[program:theprogramname] +;command=/bin/cat ; the program (relative uses PATH, can take args) +;process_name=%(program_name)s ; process_name expr (default %(program_name)s) +;numprocs=1 ; number of processes copies to start (def 1) +;directory=/tmp ; directory to cwd to before exec (def no cwd) +;umask=022 ; umask for process (default None) +;priority=999 ; the relative start priority (default 999) +;autostart=true ; start at supervisord start (default: true) +;startsecs=1 ; # of secs prog must stay up to be running (def. 1) +;startretries=3 ; max # of serial start failures when starting (default 3) +;autorestart=unexpected ; when to restart if exited after running (def: unexpected) +;exitcodes=0 ; 'expected' exit codes used with autorestart (default 0) +;stopsignal=QUIT ; signal used to kill process (default TERM) +;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) +;stopasgroup=false ; send stop signal to the UNIX process group (default false) +;killasgroup=false ; SIGKILL the UNIX process group (def false) +;user=chrism ; setuid to this UNIX account to run the program +;redirect_stderr=true ; redirect proc stderr to stdout (default false) +;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO +;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10) +;stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) +;stdout_events_enabled=false ; emit events on stdout writes (default false) +;stdout_syslog=false ; send stdout to syslog with process name (default false) +;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO +;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10) +;stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) +;stderr_events_enabled=false ; emit events on stderr writes (default false) +;stderr_syslog=false ; send stderr to syslog with process name (default false) +;environment=A="1",B="2" ; process environment additions (def no adds) +;serverurl=AUTO ; override serverurl computation (childutils) + +; The sample eventlistener section below shows all possible eventlistener +; subsection values. Create one or more 'real' eventlistener: sections to be +; able to handle event notifications sent by supervisord. + +;[eventlistener:theeventlistenername] +;command=/bin/eventlistener ; the program (relative uses PATH, can take args) +;process_name=%(program_name)s ; process_name expr (default %(program_name)s) +;numprocs=1 ; number of processes copies to start (def 1) +;events=EVENT ; event notif. types to subscribe to (req'd) +;buffer_size=10 ; event buffer queue size (default 10) +;directory=/tmp ; directory to cwd to before exec (def no cwd) +;umask=022 ; umask for process (default None) +;priority=-1 ; the relative start priority (default -1) +;autostart=true ; start at supervisord start (default: true) +;startsecs=1 ; # of secs prog must stay up to be running (def. 1) +;startretries=3 ; max # of serial start failures when starting (default 3) +;autorestart=unexpected ; autorestart if exited after running (def: unexpected) +;exitcodes=0 ; 'expected' exit codes used with autorestart (default 0) +;stopsignal=QUIT ; signal used to kill process (default TERM) +;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) +;stopasgroup=false ; send stop signal to the UNIX process group (default false) +;killasgroup=false ; SIGKILL the UNIX process group (def false) +;user=chrism ; setuid to this UNIX account to run the program +;redirect_stderr=false ; redirect_stderr=true is not allowed for eventlisteners +;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO +;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10) +;stdout_events_enabled=false ; emit events on stdout writes (default false) +;stdout_syslog=false ; send stdout to syslog with process name (default false) +;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO +;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10) +;stderr_events_enabled=false ; emit events on stderr writes (default false) +;stderr_syslog=false ; send stderr to syslog with process name (default false) +;environment=A="1",B="2" ; process environment additions +;serverurl=AUTO ; override serverurl computation (childutils) + +; The sample group section below shows all possible group values. Create one +; or more 'real' group: sections to create "heterogeneous" process groups. + +;[group:thegroupname] +;programs=progname1,progname2 ; each refers to 'x' in [program:x] definitions +;priority=999 ; the relative start priority (default 999) + +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. + +[include] +files = base-services/*.conf diff --git a/docker/all-in-one-nix/etc/tmpfiles.d/pgbouncer.conf b/docker/all-in-one-nix/etc/tmpfiles.d/pgbouncer.conf new file mode 100644 index 000000000..d5d2cd49d --- /dev/null +++ b/docker/all-in-one-nix/etc/tmpfiles.d/pgbouncer.conf @@ -0,0 +1,2 @@ +# Directory for PostgreSQL sockets, lockfiles and stats tempfiles +d /run/pgbouncer 2775 pgbouncer postgres - - \ No newline at end of file diff --git a/docker/all-in-one-nix/etc/vector/vector.yaml b/docker/all-in-one-nix/etc/vector/vector.yaml new file mode 100644 index 000000000..8bcf867b8 --- /dev/null +++ b/docker/all-in-one-nix/etc/vector/vector.yaml @@ -0,0 +1,306 @@ +data_dir: /var/lib/vector +sources: + gotrue_log: + type: file + include: + - /var/log/services/gotrue.log + + postgrest_log: + type: file + include: + - /var/log/services/postgrest.log + + pgbouncer_log: + type: file + include: + - /var/log/services/pgbouncer.log + + pitr_log: + type: file + include: + - /var/log/wal-g/pitr.log + read_from: end + + postgres_log: + type: file + include: + - /var/log/postgresql/postgres*.csv + read_from: end + multiline: + start_pattern: '^20[0-9][0-9]-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9].[0-9]{3} UTC,"' + mode: halt_before + condition_pattern: '^20[0-9][0-9]-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9].[0-9]{3} UTC,"' + timeout_ms: 500 + +transforms: + csv_parse: + type: remap + inputs: + - postgres_log + source: |- + csv_data = parse_csv!(.message) + .metadata.parsed.timestamp = csv_data[0] + .metadata.parsed.user_name = csv_data[1] + .metadata.parsed.database_name = csv_data[2] + .metadata.parsed.process_id = to_int(csv_data[3]) ?? null + .metadata.parsed.connection_from = csv_data[4] + .metadata.parsed.session_id = csv_data[5] + .metadata.parsed.session_line_num = to_int(csv_data[6]) ?? null + .metadata.parsed.command_tag = csv_data[7] + .metadata.parsed.session_start_time = csv_data[8] + .metadata.parsed.virtual_transaction_id = csv_data[9] + .metadata.parsed.transaction_id = to_int(csv_data[10]) ?? null + .metadata.parsed.error_severity = csv_data[11] + .metadata.parsed.sql_state_code = csv_data[12] + .metadata.parsed.message = csv_data[13] + .metadata.parsed.detail = csv_data[14] + .metadata.parsed.hint = csv_data[15] + .metadata.parsed.internal_query = csv_data[16] + .metadata.parsed.internal_query_pos = to_int(csv_data[17]) ?? null + .metadata.parsed.context = csv_data[18] + .metadata.parsed.query = csv_data[19] + .metadata.parsed.query_pos = to_int(csv_data[20]) ?? null + .metadata.parsed.location = csv_data[21] + .metadata.parsed.application_name = csv_data[22] + .metadata.parsed.backend_type = csv_data[23] + .metadata.parsed.leader_pid = to_int(csv_data[24]) ?? null + .metadata.parsed.query_id = to_int(csv_data[25]) ?? null + + z_ts = replace!(.metadata.parsed.timestamp, " UTC", "Z") + iso8601_ts = replace(z_ts, " ", "T") + + .timestamp = iso8601_ts + + # Sends original csv log line duplicating data. Used for QA. + # .metadata.parsed_from = .message + + .message = del(.metadata.parsed.message) + .metadata.host = del(.host) + del(.file) + del(.source_type) + + drop_metrics: + type: filter + inputs: + - csv_parse + condition: > + .metadata.parsed.application_name != "postgres_exporter" && .metadata.parsed.application_name != "realtime_rls" && !contains!(.message, "disconnection: session time") + + add_project_ref: + type: add_fields + inputs: + - drop_metrics + fields: + project: {{ .ProjectRef }} + + auth_failures: + type: filter + inputs: + - postgres_log + condition: >- + contains!(.message, "password authentication failed for user") + + filter_pgbouncer_stats: + type: filter + inputs: + - pgbouncer_log + condition: >- + !starts_with!(.message, "stats:") && !starts_with!(.message, "kernel file descriptor limit") && !contains!(.message, "FIXME") + + filter_postgrest_stats: + type: filter + inputs: + - postgrest_log + condition: >- + !starts_with!(.message, "+") && !starts_with!(.message, "INFO:") && !contains!(.message, "Admin server listening") + + gotrue_to_object: + inputs: + - gotrue_log + type: remap + source: |2- + .project = "{{ .ProjectRef }}" + + .parsed, err = parse_json(.message) + if err == null { + .metadata = .parsed + .metadata.msg = .parsed.msg + .timestamp = del(.metadata.time) + } + del(.parsed) + .metadata.host = del(.host) + + del(.source_type) + del(.PRIORITY) + del(.SYSLOG_FACILITY) + del(.SYSLOG_IDENTIFIER) + del(._BOOT_ID) + del(._CAP_EFFECTIVE) + del(._CMDLINE) + del(._COMM) + del(._EXE) + del(._GID) + del(._MACHINE_ID) + del(._PID) + del(._SELINUX_CONTEXT) + del(._STREAM_ID) + del(._SYSTEMD_CGROUP) + del(._SYSTEMD_INVOCATION_ID) + del(._SYSTEMD_SLICE) + del(._SYSTEMD_UNIT) + del(._TRANSPORT) + del(._UID) + del(.__MONOTONIC_TIMESTAMP) + del(.__REALTIME_TIMESTAMP) + + postgrest_to_object: + inputs: + - filter_postgrest_stats + type: remap + source: |2- + .project = "{{ .ProjectRef }}" + + # removes timestamp embedded in log since Vector already sends it + .message = replace!(.message, r'^\d+/\w+/\d+:\d+:\d+:\d+\s\+\d+:\s', "") + .metadata.host = del(.host) + del(.source_type) + del(.PRIORITY) + del(.SYSLOG_FACILITY) + del(.SYSLOG_IDENTIFIER) + del(._BOOT_ID) + del(._CAP_EFFECTIVE) + del(._CMDLINE) + del(._COMM) + del(._EXE) + del(._GID) + del(._MACHINE_ID) + del(._PID) + del(._SELINUX_CONTEXT) + del(._STREAM_ID) + del(._SYSTEMD_CGROUP) + del(._SYSTEMD_INVOCATION_ID) + del(._SYSTEMD_SLICE) + del(._SYSTEMD_UNIT) + del(._TRANSPORT) + del(._UID) + del(.__MONOTONIC_TIMESTAMP) + del(.__REALTIME_TIMESTAMP) + + pgbouncer_to_object: + inputs: + - filter_pgbouncer_stats + type: remap + source: |2- + .project = "{{ .ProjectRef }}" + .metadata.host = del(.host) + del(.source_type) + del(.PRIORITY) + del(.SYSLOG_IDENTIFIER) + del(._BOOT_ID) + del(._CAP_EFFECTIVE) + del(._CMDLINE) + del(._COMM) + del(._EXE) + del(._GID) + del(._MACHINE_ID) + del(._PID) + del(._SELINUX_CONTEXT) + del(._SOURCE_REALTIME_TIMESTAMP) + del(._SYSTEMD_CGROUP) + del(._SYSTEMD_INVOCATION_ID) + del(._SYSTEMD_SLICE) + del(._SYSTEMD_UNIT) + del(._TRANSPORT) + del(._UID) + del(.__MONOTONIC_TIMESTAMP) + del(.__REALTIME_TIMESTAMP) + + pitr_to_object: + inputs: + - pitr_log + type: remap + source: |2- + .project = "{{ .ProjectRef }}" + + .parsed, err = parse_key_value(.message) + if err == null { + .metadata = .parsed + .metadata.host = del(.host) + .message = del(.metadata.msg) + .timestamp = del(.metadata.time) + } + + del(.parsed) + del(.source_type) + del(.file) + + filter_pitr_error: + inputs: + - pitr_to_object + type: filter + condition: > + .metadata.level != "info" + +sinks: + http_gotrue: + type: "http" + inputs: + - gotrue_to_object + encoding: + codec: "json" + method: "post" + compression: none + request: + retry_max_duration_secs: 10 + uri: "https://{{ .LogflareHost }}/logs?api_key={{ .ApiKey }}&source={{ .GotrueSource }}" + + http_postgrest: + type: http + inputs: + - postgrest_to_object + encoding: + codec: "json" + method: "post" + compression: none + request: + retry_max_duration_secs: 10 + uri: "https://{{ .LogflareHost }}/logs?api_key={{ .ApiKey }}&source={{ .PostgrestSource }}" + + http_pgbouncer: + type: http + inputs: + - pgbouncer_to_object + encoding: + codec: json + compression: none + uri: "https://{{ .LogflareHost }}/logs?api_key={{ .ApiKey }}&source={{ .PgbouncerSource }}" + + http_pitr_error: + type: http + inputs: + - filter_pitr_error + encoding: + codec: json + compression: none + uri: "https://{{ .LogflareHost }}/logs?api_key={{ .ApiKey }}&source={{ .PitrErrorsSource }}" + + http_postgres: + type: http + inputs: + - add_project_ref + encoding: + codec: "json" + method: "post" + compression: none + request: + retry_max_duration_secs: 10 + uri: "https://{{ .LogflareHost }}/logs?api_key={{ .ApiKey }}&source={{ .DbSource }}" + + file_postgres: + type: file + inputs: + - auth_failures + encoding: + codec: text + path: >- + /var/log/postgresql/auth-failures.csv diff --git a/docker/all-in-one-nix/healthcheck.sh b/docker/all-in-one-nix/healthcheck.sh new file mode 100755 index 000000000..039b461f1 --- /dev/null +++ b/docker/all-in-one-nix/healthcheck.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -eou pipefail + +# database up +pg_isready -U postgres -h localhost -p 5432 + +if [ -f "/tmp/init.json" ]; then + ADMIN_API_KEY=${ADMIN_API_KEY:-$(jq -r '.["supabase_admin_key"]' /tmp/init.json)} +fi + +# adminapi up +if [ -d "$ADMIN_API_CERT_DIR" ]; then + curl -sSkf "https://localhost:$ADMIN_API_PORT/health" -H "apikey: $ADMIN_API_KEY" +else + curl -sSf "http://localhost:$ADMIN_API_PORT/health" -H "apikey: $ADMIN_API_KEY" +fi + +if [ "${POSTGRES_ONLY:-}" ]; then + exit 0 +fi + +# postgrest up +curl -sSfI "http://localhost:$PGRST_ADMIN_SERVER_PORT/ready" + +# gotrue up +curl -sSf "http://localhost:$GOTRUE_API_PORT/health" + +if [ "${ENVOY_ENABLED:-}" == "true" ]; then + # envoy up + curl -sSfI "http://localhost:$ENVOY_HTTP_PORT/health" +else + # kong up + kong health +fi + +# pgbouncer up +printf \\0 > "/dev/tcp/localhost/$PGBOUNCER_PORT" + +# fail2ban up +fail2ban-client status + +# prometheus exporter up +curl -sSfI "http://localhost:$PGEXPORTER_PORT/metrics" + +# vector is up (if starting logflare) +# TODO: make this non-conditional once we set up local logflare for testinfra +if [ -n "${LOGFLARE_API_KEY:-}" ]; then + curl -sSfI "http://localhost:$VECTOR_API_PORT/health" +fi diff --git a/docker/all-in-one-nix/init/configure-admin-mgr.sh b/docker/all-in-one-nix/init/configure-admin-mgr.sh new file mode 100755 index 000000000..98ebf6c17 --- /dev/null +++ b/docker/all-in-one-nix/init/configure-admin-mgr.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -eou pipefail + +touch "/var/log/wal-g/pitr.log" +chown postgres:postgres "/var/log/wal-g/pitr.log" +chmod 0666 "/var/log/wal-g/pitr.log" + +/usr/local/bin/configure-shim.sh /dist/admin-mgr /usr/bin/admin-mgr diff --git a/docker/all-in-one-nix/init/configure-adminapi.sh b/docker/all-in-one-nix/init/configure-adminapi.sh new file mode 100755 index 000000000..f88c09160 --- /dev/null +++ b/docker/all-in-one-nix/init/configure-adminapi.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -eou pipefail + +ADMIN_API_CONF=/etc/adminapi/adminapi.yaml +touch /var/log/services/adminapi.log + +ADMINAPI_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/adminapi" + +/usr/local/bin/configure-shim.sh /dist/supabase-admin-api /opt/supabase-admin-api + +if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then + echo "init adminapi payload" + tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/adminapi/adminapi.yaml + chown adminapi:adminapi ./etc/adminapi/adminapi.yaml + + mkdir -p $ADMIN_API_CERT_DIR + tar -xzvf "$INIT_PAYLOAD_PATH" -C $ADMIN_API_CERT_DIR --strip-components 2 ./ssl/server.crt + tar -xzvf "$INIT_PAYLOAD_PATH" -C $ADMIN_API_CERT_DIR --strip-components 2 ./ssl/server.key + chown -R adminapi:root $ADMIN_API_CERT_DIR + chmod 700 -R $ADMIN_API_CERT_DIR +else + PROJECT_REF=${PROJECT_REF:-default} + PGBOUNCER_PASSWORD=${PGBOUNCER_PASSWORD:-$POSTGRES_PASSWORD} + SUPABASE_URL=${SUPABASE_URL:-https://api.supabase.io/system} + REPORTING_TOKEN=${REPORTING_TOKEN:-token} + + sed -i "s|{{ .JwtSecret }}|$JWT_SECRET|g" $ADMIN_API_CONF + sed -i "s|{{ .PgbouncerPassword }}|$PGBOUNCER_PASSWORD|g" $ADMIN_API_CONF + sed -i "s|{{ .ProjectRef }}|$PROJECT_REF|g" $ADMIN_API_CONF + sed -i "s|{{ .SupabaseUrl }}|$SUPABASE_URL|g" $ADMIN_API_CONF + sed -i "s|{{ .ReportingToken }}|$REPORTING_TOKEN|g" $ADMIN_API_CONF +fi + +# Allow adminapi to write to /etc and manage Postgres configs +chmod g+w /etc +chmod -R 0775 /etc/postgresql +chmod -R 0775 /etc/postgresql-custom + +# Update api port +sed -i "s|^port: .*$|port: ${ADMIN_API_PORT:-8085}|g" $ADMIN_API_CONF + +if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then + mkdir -p "${ADMINAPI_CUSTOM_DIR}" + if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then + echo "Copying existing custom adminapi config from /etc/adminapi to ${ADMINAPI_CUSTOM_DIR}" + cp -R "/etc/adminapi/." "${ADMINAPI_CUSTOM_DIR}/" + fi + + rm -rf "/etc/adminapi" + ln -s "${ADMINAPI_CUSTOM_DIR}" "/etc/adminapi" + chown -R adminapi:adminapi "/etc/adminapi" + + chown -R adminapi:adminapi "${ADMINAPI_CUSTOM_DIR}" + chmod g+wrx "${ADMINAPI_CUSTOM_DIR}" +fi diff --git a/docker/all-in-one-nix/init/configure-autoshutdown.sh b/docker/all-in-one-nix/init/configure-autoshutdown.sh new file mode 100755 index 000000000..66343e518 --- /dev/null +++ b/docker/all-in-one-nix/init/configure-autoshutdown.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -eou pipefail + +mkdir -p /etc/supa-shutdown + +AUTOSHUTDOWN_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/supa-shutdown" +if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then + mkdir -p "${AUTOSHUTDOWN_CUSTOM_DIR}" + + AUTOSHUTDOWN_CUSTOM_CONFIG_FILE_PATH="${AUTOSHUTDOWN_CUSTOM_DIR}/shutdown.conf" + if [ ! -f "${AUTOSHUTDOWN_CUSTOM_CONFIG_FILE_PATH}" ]; then + echo "Copying existing custom shutdown config from /etc/supa-shutdown to ${AUTOSHUTDOWN_CUSTOM_CONFIG_FILE_PATH}" + cp "/etc/supa-shutdown/shutdown.conf" "${AUTOSHUTDOWN_CUSTOM_CONFIG_FILE_PATH}" + fi + + rm -f "/etc/supa-shutdown/shutdown.conf" + ln -s "${AUTOSHUTDOWN_CUSTOM_CONFIG_FILE_PATH}" "/etc/supa-shutdown/shutdown.conf" + chmod g+wrx "${AUTOSHUTDOWN_CUSTOM_DIR}" + chown -R adminapi:adminapi "/etc/supa-shutdown/shutdown.conf" + chown -R adminapi:adminapi "${AUTOSHUTDOWN_CUSTOM_CONFIG_FILE_PATH}" +fi diff --git a/docker/all-in-one-nix/init/configure-envoy.sh b/docker/all-in-one-nix/init/configure-envoy.sh new file mode 100755 index 000000000..4d3382f8a --- /dev/null +++ b/docker/all-in-one-nix/init/configure-envoy.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -eou pipefail + +if [[ "${ENVOY_ENABLED:-}" != "true" ]]; then + exit +fi + +ENVOY_CDS_CONF=/etc/envoy/cds.yaml +ENVOY_LDS_CONF=/etc/envoy/lds.yaml +touch /var/log/services/envoy.log + +/usr/local/bin/configure-shim.sh /dist/envoy /opt/envoy + +if [[ -n "${DATA_VOLUME_MOUNTPOINT}" ]]; then + ENVOY_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/envoy" + mkdir -p "${ENVOY_CUSTOM_DIR}" + if [[ ! -f "${CONFIGURED_FLAG_PATH}" ]]; then + echo "Copying existing custom envoy config from /etc/envoy/ to ${ENVOY_CUSTOM_DIR}" + cp -R "/etc/envoy/." "${ENVOY_CUSTOM_DIR}/" + fi + + rm -rf "/etc/envoy" + ln -s "${ENVOY_CUSTOM_DIR}" "/etc/envoy" + chown -R envoy:envoy "/etc/envoy" + chmod -R g+w "/etc/envoy" + + chown -R envoy:envoy "${ENVOY_CUSTOM_DIR}" + chmod -R g+w "${ENVOY_CUSTOM_DIR}" + chmod g+rx "${ENVOY_CUSTOM_DIR}" +fi + +if [[ -f "${INIT_PAYLOAD_PATH:-}" ]]; then + echo "init envoy payload" + tar -xzvhf "${INIT_PAYLOAD_PATH}" -C / ./etc/envoy/ + chown -HR envoy:envoy /etc/envoy + chmod -HR g+w /etc/envoy +fi + +# Inject project specific configuration +# "c2VydmljZV9yb2xlOnNlcnZpY2Vfa2V5" is base64-encoded "service_role:service_key". +sed -i -e "s|anon_key|${ANON_KEY}|g" \ + -e "s|service_key|${SERVICE_ROLE_KEY}|g" \ + -e "s|supabase_admin_key|${ADMIN_API_KEY}|g" \ + -e "s|c2VydmljZV9yb2xlOnNlcnZpY2Vfa2V5|$(echo -n "service_role:${SERVICE_ROLE_KEY}" | base64 --wrap 0)|g" \ + "${ENVOY_LDS_CONF}" + +# Update Envoy ports +sed -i "s|port_value: 80$|port_value: ${ENVOY_HTTP_PORT}|g" "${ENVOY_LDS_CONF}" +sed -i "s|port_value: 443$|port_value: ${ENVOY_HTTPS_PORT}|g" "${ENVOY_LDS_CONF}" +sed -i "s|port_value: 3000$|port_value: ${PGRST_SERVER_PORT}|g" "${ENVOY_CDS_CONF}" +sed -i "s|port_value: 3001$|port_value: ${PGRST_ADMIN_SERVER_PORT}|g" "${ENVOY_CDS_CONF}" +sed -i "s|port_value: 8085$|port_value: ${ADMIN_API_PORT}|g" "${ENVOY_CDS_CONF}" +sed -i "s|port_value: 9999$|port_value: ${GOTRUE_API_PORT}|g" "${ENVOY_CDS_CONF}" diff --git a/docker/all-in-one-nix/init/configure-exporter.sh b/docker/all-in-one-nix/init/configure-exporter.sh new file mode 100755 index 000000000..93498c4e6 --- /dev/null +++ b/docker/all-in-one-nix/init/configure-exporter.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -eou pipefail + +touch /var/log/services/exporter.log + diff --git a/docker/all-in-one-nix/init/configure-fail2ban.sh b/docker/all-in-one-nix/init/configure-fail2ban.sh new file mode 100755 index 000000000..39b0a27a6 --- /dev/null +++ b/docker/all-in-one-nix/init/configure-fail2ban.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -eou pipefail + +mkdir -p /var/run/fail2ban +touch /var/log/services/fail2ban.log +touch /var/log/postgresql/auth-failures.csv diff --git a/docker/all-in-one-nix/init/configure-gotrue.sh b/docker/all-in-one-nix/init/configure-gotrue.sh new file mode 100755 index 000000000..e3d99f0e6 --- /dev/null +++ b/docker/all-in-one-nix/init/configure-gotrue.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -eou pipefail + +touch /var/log/services/gotrue.log + +GOTRUE_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/gotrue" +GOTRUE_CUSTOM_CONFIG_FILE_PATH="${DATA_VOLUME_MOUNTPOINT}/etc/gotrue/gotrue.env" + +/usr/local/bin/configure-shim.sh /dist/gotrue /opt/gotrue/gotrue + +if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then + mkdir -p "${GOTRUE_CUSTOM_DIR}" + chmod g+wrx "${GOTRUE_CUSTOM_DIR}" + chown adminapi:adminapi "${GOTRUE_CUSTOM_DIR}" + + if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then + echo "Copying existing GoTrue config from /etc/gotrue.env to ${GOTRUE_CUSTOM_CONFIG_FILE_PATH}" + cp "/etc/gotrue.env" "${GOTRUE_CUSTOM_CONFIG_FILE_PATH}" + fi + + rm -f "/etc/gotrue.env" + ln -s "${GOTRUE_CUSTOM_CONFIG_FILE_PATH}" "/etc/gotrue.env" + chown -R adminapi:adminapi "/etc/gotrue.env" + + chown -R adminapi:adminapi "${GOTRUE_CUSTOM_CONFIG_FILE_PATH}" + chmod g+rx "${GOTRUE_CUSTOM_CONFIG_FILE_PATH}" +fi + +if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then + echo "init gotrue payload" + tar -h --overwrite -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/gotrue.env + chown -R adminapi:adminapi /etc/gotrue.env +else + sed -i "s|api_external_url|${API_EXTERNAL_URL:-http://localhost}|g" /etc/gotrue.env + sed -i "s|gotrue_api_host|${GOTRUE_API_HOST:-0.0.0.0}|g" /etc/gotrue.env + sed -i "s|gotrue_site_url|$GOTRUE_SITE_URL|g" /etc/gotrue.env + sed -i "s|gotrue_jwt_secret|$JWT_SECRET|g" /etc/gotrue.env +fi diff --git a/docker/all-in-one-nix/init/configure-kong.sh b/docker/all-in-one-nix/init/configure-kong.sh new file mode 100755 index 000000000..110525d44 --- /dev/null +++ b/docker/all-in-one-nix/init/configure-kong.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -eou pipefail + +KONG_CONF=/etc/kong/kong.yml +KONG_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/kong" + +touch /var/log/services/kong.log + +if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then + echo "init kong payload" + # Setup ssl termination + tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/kong/ + chown -R adminapi:adminapi ./etc/kong/kong.yml + chown -R adminapi:adminapi ./etc/kong/*pem + echo "ssl_cipher_suite = intermediate" >> /etc/kong/kong.conf + echo "ssl_cert = /etc/kong/fullChain.pem" >> /etc/kong/kong.conf + echo "ssl_cert_key = /etc/kong/privKey.pem" >> /etc/kong/kong.conf +else + # Default gateway config + export KONG_DNS_ORDER=LAST,A,CNAME + export KONG_PROXY_ERROR_LOG=syslog:server=unix:/dev/log + export KONG_ADMIN_ERROR_LOG=syslog:server=unix:/dev/log +fi + +# Inject project specific configuration +sed -i -e "s|anon_key|$ANON_KEY|g" \ + -e "s|service_key|$SERVICE_ROLE_KEY|g" \ + -e "s|supabase_admin_key|$ADMIN_API_KEY|g" \ + $KONG_CONF + +# Update kong ports +sed -i "s|:80 |:$KONG_HTTP_PORT |g" /etc/kong/kong.conf +sed -i "s|:443 |:$KONG_HTTPS_PORT |g" /etc/kong/kong.conf + +if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then + mkdir -p "${KONG_CUSTOM_DIR}" + if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then + echo "Copying existing custom kong config from /etc/kong/kong.yml to ${KONG_CUSTOM_DIR}" + cp /etc/kong/kong.yml "${KONG_CUSTOM_DIR}/kong.yml" + fi + + rm -rf "/etc/kong/kong.yml" + ln -s "${KONG_CUSTOM_DIR}/kong.yml" "/etc/kong/kong.yml" + chown -R adminapi:adminapi "/etc/kong/kong.yml" + + chown -R adminapi:adminapi "${KONG_CUSTOM_DIR}" + chmod g+wrx "${KONG_CUSTOM_DIR}" +fi \ No newline at end of file diff --git a/docker/all-in-one-nix/init/configure-pg_egress_collect.sh b/docker/all-in-one-nix/init/configure-pg_egress_collect.sh new file mode 100755 index 000000000..c58f3635d --- /dev/null +++ b/docker/all-in-one-nix/init/configure-pg_egress_collect.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -eou pipefail + +PG_EGRESS_COLLECT_FILE=/tmp/pg_egress_collect.txt + +if [ "${DATA_VOLUME_MOUNTPOINT:-}" != "" ]; then + touch "${DATA_VOLUME_MOUNTPOINT}/pg_egress_collect.txt" + ln -s "${DATA_VOLUME_MOUNTPOINT}/pg_egress_collect.txt" $PG_EGRESS_COLLECT_FILE +fi diff --git a/docker/all-in-one-nix/init/configure-pgbouncer.sh b/docker/all-in-one-nix/init/configure-pgbouncer.sh new file mode 100755 index 000000000..ea30b9a2c --- /dev/null +++ b/docker/all-in-one-nix/init/configure-pgbouncer.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -eou pipefail + +touch /var/log/services/pgbouncer.log + +mkdir -p /var/run/pgbouncer +chown pgbouncer:postgres /var/run/pgbouncer + +PGBOUNCER_CONF=/etc/pgbouncer/pgbouncer.ini + +if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then + echo "init pgbouncer payload" + sed -i -E "s|^# (%include /etc/pgbouncer-custom/ssl-config.ini)$|\1|g" $PGBOUNCER_CONF + + tar -xzvf "$INIT_PAYLOAD_PATH" -C /etc/pgbouncer/ --strip-components 3 ./etc/pgbouncer/userlist.txt + chown -R pgbouncer:pgbouncer /etc/pgbouncer/userlist.txt +fi + +if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then + /opt/supabase-admin-api optimize pgbouncer --destination-config-file-path /etc/pgbouncer-custom/generated-optimizations.ini + + # Preserve pgbouncer configs across restarts + PGBOUNCER_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/pgbouncer" + PGBOUNCER_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/pgbouncer-custom" + + mkdir -p "${PGBOUNCER_DIR}" + mkdir -p "${PGBOUNCER_CUSTOM_DIR}" + + if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then + echo "Copying existing custom pgbouncer config from /etc/pgbouncer-custom to ${PGBOUNCER_CUSTOM_DIR}" + cp -R "/etc/pgbouncer-custom/." "${PGBOUNCER_CUSTOM_DIR}/" + cp -R "/etc/pgbouncer/." "${PGBOUNCER_DIR}/" + fi + + rm -rf "/etc/pgbouncer-custom" + ln -s "${PGBOUNCER_CUSTOM_DIR}" "/etc/pgbouncer-custom" + chown -R pgbouncer:pgbouncer "/etc/pgbouncer-custom" + chown -R pgbouncer:pgbouncer "${PGBOUNCER_CUSTOM_DIR}" + chmod -R g+rx "${PGBOUNCER_CUSTOM_DIR}" + + rm -rf "/etc/pgbouncer" + ln -s "${PGBOUNCER_DIR}" "/etc/pgbouncer" + chown -R pgbouncer:pgbouncer "/etc/pgbouncer" + chown -R pgbouncer:pgbouncer "${PGBOUNCER_DIR}" + chmod -R g+wrx "${PGBOUNCER_DIR}" +fi diff --git a/docker/all-in-one-nix/init/configure-postgrest.sh b/docker/all-in-one-nix/init/configure-postgrest.sh new file mode 100755 index 000000000..20f5a9902 --- /dev/null +++ b/docker/all-in-one-nix/init/configure-postgrest.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -eou pipefail + +touch /var/log/services/postgrest.log + +# Default in-database config +sed -i "s|pgrst_server_port|${PGRST_SERVER_PORT:-3000}|g" /etc/postgrest/base.conf +sed -i "s|pgrst_admin_server_port|${PGRST_ADMIN_SERVER_PORT:-3001}|g" /etc/postgrest/base.conf +sed -i "s|pgrst_db_schemas|${PGRST_DB_SCHEMAS:-public,storage,graphql_public}|g" /etc/postgrest/base.conf +sed -i "s|pgrst_db_extra_search_path|${PGRST_DB_SCHEMAS:-public,extensions}|g" /etc/postgrest/base.conf +sed -i "s|pgrst_db_anon_role|${PGRST_DB_ANON_ROLE:-anon}|g" /etc/postgrest/base.conf +sed -i "s|pgrst_jwt_secret|$JWT_SECRET|g" /etc/postgrest/base.conf + +/usr/local/bin/configure-shim.sh /dist/postgrest /opt/postgrest + +if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then + echo "init postgrest payload" + tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/postgrest/base.conf + chown -R postgrest:postgrest /etc/postgrest +fi + +if [ "${DATA_VOLUME_MOUNTPOINT}" ]; then + POSTGREST_CUSTOM_DIR="${DATA_VOLUME_MOUNTPOINT}/etc/postgrest" + mkdir -p "${POSTGREST_CUSTOM_DIR}" + if [ ! -f "${CONFIGURED_FLAG_PATH}" ]; then + echo "Copying existing custom PostgREST config from /etc/postgrest/ to ${POSTGREST_CUSTOM_DIR}" + cp -R "/etc/postgrest/." "${POSTGREST_CUSTOM_DIR}/" + fi + + rm -rf "/etc/postgrest" + ln -s "${POSTGREST_CUSTOM_DIR}" "/etc/postgrest" + chown -R postgrest:postgrest "/etc/postgrest" + + chown -R postgrest:postgrest "${POSTGREST_CUSTOM_DIR}" + chmod g+wrx "${POSTGREST_CUSTOM_DIR}" +fi + +PGRST_CONF=/etc/postgrest/generated.conf + +/opt/supabase-admin-api optimize postgrest --destination-config-file-path $PGRST_CONF +cat /etc/postgrest/base.conf >> $PGRST_CONF diff --git a/docker/all-in-one-nix/init/configure-vector.sh b/docker/all-in-one-nix/init/configure-vector.sh new file mode 100755 index 000000000..9177a0f94 --- /dev/null +++ b/docker/all-in-one-nix/init/configure-vector.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -eou pipefail + +VECTOR_CONF=/etc/vector/vector.yaml +touch /var/log/services/vector.log + +if [ -f "${INIT_PAYLOAD_PATH:-}" ]; then + echo "init vector payload" + tar -xzvf "$INIT_PAYLOAD_PATH" -C /etc/vector/ --strip-components 2 ./tmp/init.json + PROJECT_REF=$(jq -r '.["project_ref"]' /etc/vector/init.json) + LOGFLARE_DB_SOURCE=$(jq -r '.["logflare_db_source"]' /etc/vector/init.json) + LOGFLARE_GOTRUE_SOURCE=$(jq -r '.["logflare_gotrue_source"]' /etc/vector/init.json) + LOGFLARE_POSTGREST_SOURCE=$(jq -r '.["logflare_postgrest_source"]' /etc/vector/init.json) + LOGFLARE_PGBOUNCER_SOURCE=$(jq -r '.["logflare_pgbouncer_source"]' /etc/vector/init.json) + LOGFLARE_PITR_ERRORS_SOURCE=$(jq -r '.["logflare_pitr_errors_source"]' /etc/vector/init.json) + LOGFLARE_API_KEY=$(jq -r '.["logflare_api_key"]' /etc/vector/init.json) +fi + +# Exit early if not starting logflare +if [ -z "${LOGFLARE_API_KEY:-}" ]; then + echo "Skipped starting vector: missing LOGFLARE_API_KEY" + exit 0 +fi + +# Add vector to support both base-services and services config +cat < /etc/supervisor/services/vector.conf + +[program:vector] +command=/usr/bin/vector --config-yaml /etc/vector/vector.yaml +user=root +autorestart=true +stdout_logfile=/var/log/services/vector.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +priority=250 + +EOF + +VECTOR_API_PORT=${VECTOR_API_PORT:-9001} +PROJECT_REF=${PROJECT_REF:-default} +LOGFLARE_HOST=${LOGFLARE_HOST:-api.logflare.app} +LOGFLARE_DB_SOURCE=${LOGFLARE_DB_SOURCE:-postgres.logs} +LOGFLARE_GOTRUE_SOURCE=${LOGFLARE_GOTRUE_SOURCE:-gotrue.logs.prod} +LOGFLARE_POSTGREST_SOURCE=${LOGFLARE_POSTGREST_SOURCE:-postgREST.logs.prod} +LOGFLARE_PGBOUNCER_SOURCE=${LOGFLARE_PGBOUNCER_SOURCE:-pgbouncer.logs.prod} +LOGFLARE_PITR_ERRORS_SOURCE=${LOGFLARE_PITR_ERRORS_SOURCE:-pitr_errors.logs.prod} + +sed -i "s|{{ .ApiPort }}|$VECTOR_API_PORT|g" $VECTOR_CONF +sed -i "s|{{ .ProjectRef }}|$PROJECT_REF|g" $VECTOR_CONF +sed -i "s|{{ .LogflareHost }}|$LOGFLARE_HOST|g" $VECTOR_CONF +sed -i "s|{{ .ApiKey }}|$LOGFLARE_API_KEY|g" $VECTOR_CONF +sed -i "s|{{ .DbSource }}|$LOGFLARE_DB_SOURCE|g" $VECTOR_CONF +sed -i "s|{{ .GotrueSource }}|$LOGFLARE_GOTRUE_SOURCE|g" $VECTOR_CONF +sed -i "s|{{ .PostgrestSource }}|$LOGFLARE_POSTGREST_SOURCE|g" $VECTOR_CONF +sed -i "s|{{ .PgbouncerSource }}|$LOGFLARE_PGBOUNCER_SOURCE|g" $VECTOR_CONF +sed -i "s|{{ .PitrErrorsSource }}|$LOGFLARE_PITR_ERRORS_SOURCE|g" $VECTOR_CONF diff --git a/docker/all-in-one-nix/init/start-kong.sh b/docker/all-in-one-nix/init/start-kong.sh new file mode 100755 index 000000000..7418d26c4 --- /dev/null +++ b/docker/all-in-one-nix/init/start-kong.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -eou pipefail + +# In the event of a restart, properly stop any running kong instances first +# Confirmed by running /usr/local/bin/kong health +trap '/usr/local/bin/kong quit' EXIT +/usr/local/bin/kong start diff --git a/docker/all-in-one-nix/opt/pg_egress_collect/pg_egress_collect.pl b/docker/all-in-one-nix/opt/pg_egress_collect/pg_egress_collect.pl new file mode 100644 index 000000000..2acc98aa6 --- /dev/null +++ b/docker/all-in-one-nix/opt/pg_egress_collect/pg_egress_collect.pl @@ -0,0 +1,126 @@ +#!/usr/bin/env perl + +# This script receive tcpdump output through STDIN and does: +# +# 1. extract outgoing TCP packet length on the 1st non-loopback device port 5432 and 6543 +# 2. sum the length up to one minute +# 3. save the total length to file (default is /tmp/pg_egress_collect.txt) per minute +# +# Usage: +# +# tcpdump -s 128 -Q out -nn -tt -vv -p -l 'tcp and (port 5432 or port 6543)' | perl pg_egress_collect.pl -o /tmp/output.txt +# + +use POSIX; +use List::Util qw(sum); +use Getopt::Long 'HelpMessage'; +use IO::Async::Loop; +use IO::Async::Stream; +use IO::Async::Timer::Periodic; + +use strict; +use warnings; + +# total captured packets lenth in a time frame +my $captured_len = 0; + +# extract tcp packet length captured by tcpdump +# +# Sample input lines: +# +# 1674013833.940253 IP (tos 0x0, ttl 64, id 0, offset 0, flags [DF], proto TCP (6), length 60) +# 10.112.101.122.5432 > 220.235.16.223.62599: Flags [S.], cksum 0x5de3 (incorrect -> 0x63da), seq 2314200657, ack 2071735457, win 62643, options [mss 8961,sackOK,TS val 3358598837 ecr 1277499190,nop,wscale 7], length 0 +# 1674013833.989257 IP (tos 0x0, ttl 64, id 24975, offset 0, flags [DF], proto TCP (6), length 52) +# 10.112.101.122.5432 > 220.235.16.223.62599: Flags [.], cksum 0x5ddb (incorrect -> 0xa25b), seq 1, ack 9, win 490, options [nop,nop,TS val 3358598885 ecr 1277499232], length 0 +sub extract_packet_length { + my ($line) = @_; + + #print("debug: >> " . $line); + + if ($line =~ /^\s+\d+\.\d+\.\d+\.\d+\..*, length (\d+)$/) { + # extract tcp packet length and add it up + my $len = $1; + $captured_len += $len; + } +} + +# write total length to file +sub write_file { + my ($output) = @_; + + my $now = strftime "%F %T", localtime time; + print "[$now] write captured len $captured_len to $output\n"; + + open(my $fh, "+>", $output) or die "Could not open file '$output' $!"; + print $fh "$captured_len"; + close($fh) or die "Could not write file '$output' $!"; +} + +# main +sub main { + # get arguments + GetOptions( + "interval:i" => \(my $interval = 60), + "output:s" => \(my $output = "/tmp/pg_egress_collect.txt"), + "help" => sub { HelpMessage(0) }, + ) or HelpMessage(1); + + my $loop = IO::Async::Loop->new; + + # tcpdump extractor + my $extractor = IO::Async::Stream->new_for_stdin( + on_read => sub { + my ($self, $buffref, $eof) = @_; + + while($$buffref =~ s/^(.*\n)//) { + my $line = $1; + extract_packet_length($line); + } + + return 0; + }, + ); + + # schedule file writer per minute + my $writer = IO::Async::Timer::Periodic->new( + interval => $interval, + on_tick => sub { + write_file($output); + + # reset total captured length + $captured_len = 0; + }, + ); + $writer->start; + + print "pg_egress_collect started, egress data will be saved to $output at interval $interval seconds.\n"; + + $loop->add($extractor); + $loop->add($writer); + $loop->run; +} + +main(); + +__END__ + +=head1 NAME + +pg_egress_collect.pl - collect egress from tcpdump output, extract TCP packet length, aggregate in specified interval and write to output file. + +=head1 SYNOPSIS + +pg_egress_collect.pl [-i interval] [-o output] + +Options: + + -i, --interval interval + output file write interval, in seconds, default is 60 seconds + + -o, --output output + output file path, default is /tmp/pg_egress_collect.txt + + -h, --help + print this help message + +=cut diff --git a/docker/all-in-one-nix/opt/postgres_exporter/queries.yml b/docker/all-in-one-nix/opt/postgres_exporter/queries.yml new file mode 100644 index 000000000..83cfeed44 --- /dev/null +++ b/docker/all-in-one-nix/opt/postgres_exporter/queries.yml @@ -0,0 +1,287 @@ +pg_database: + master: true + cache_seconds: 60 + query: | + select sum(size_b) / (1024 * 1024) as size_mb + from (select sum(pg_database_size(pg_database.datname)) as size_b + from pg_database + union all + select sum(size) as size_b + from pg_ls_waldir()) s; + metrics: + - size_mb: + usage: "GAUGE" + description: "Disk space used by the database" + +pg_stat_bgwriter: + master: true + cache_seconds: 60 + query: | + select checkpoints_timed as checkpoints_timed_total, + checkpoints_req as checkpoints_req_total, + checkpoint_write_time as checkpoint_write_time_total, + checkpoint_sync_time as checkpoint_sync_time_total, + buffers_checkpoint as buffers_checkpoint_total, + buffers_clean as buffers_clean_total, + maxwritten_clean as maxwritten_clean_total, + buffers_backend as buffers_backend_total, + buffers_backend_fsync as buffers_backend_fsync_total, + buffers_alloc as buffers_alloc_total, + stats_reset + from pg_stat_bgwriter + metrics: + - checkpoints_timed_total: + usage: "COUNTER" + description: "Scheduled checkpoints performed" + - checkpoints_req_total: + usage: "COUNTER" + description: "Requested checkpoints performed" + - checkpoint_write_time_total: + usage: "COUNTER" + description: "Time spent writing checkpoint files to disk" + - checkpoint_sync_time_total: + usage: "COUNTER" + description: "Time spent synchronizing checkpoint files to disk" + - buffers_checkpoint_total: + usage: "COUNTER" + description: "Buffers written during checkpoints" + - buffers_clean_total: + usage: "COUNTER" + description: "Buffers written by bg writter" + - maxwritten_clean_total: + usage: "COUNTER" + description: "Number of times bg writer stopped a cleaning scan because it had written too many buffers" + - buffers_backend_total: + usage: "COUNTER" + description: "Buffers written directly by a backend" + - buffers_backend_fsync_total: + usage: "COUNTER" + description: "fsync calls executed by a backend directly" + - buffers_alloc_total: + usage: "COUNTER" + description: "Buffers allocated" + - stats_reset: + usage: "COUNTER" + description: "Most recent stat reset time" + +pg_stat_database: + master: true + cache_seconds: 60 + query: | + SELECT sum(numbackends) as num_backends, + sum(xact_commit) as xact_commit_total, + sum(xact_rollback) as xact_rollback_total, + sum(blks_read) as blks_read_total, + sum(blks_hit) as blks_hit_total, + sum(tup_returned) as tup_returned_total, + sum(tup_fetched) as tup_fetched_total, + sum(tup_inserted) as tup_inserted_total, + sum(tup_updated) as tup_updated_total, + sum(tup_deleted) as tup_deleted_total, + sum(conflicts) as conflicts_total, + sum(temp_files) as temp_files_total, + sum(temp_bytes) as temp_bytes_total, + sum(deadlocks) as deadlocks_total, + max(stats_reset) as most_recent_reset + FROM pg_stat_database + metrics: + - num_backends: + usage: "GAUGE" + description: "The number of active backends" + - xact_commit_total: + usage: "COUNTER" + description: "Transactions committed" + - xact_rollback_total: + usage: "COUNTER" + description: "Transactions rolled back" + - blks_read_total: + usage: "COUNTER" + description: "Number of disk blocks read" + - blks_hit_total: + usage: "COUNTER" + description: "Disk blocks found in buffer cache" + - tup_returned_total: + usage: "COUNTER" + description: "Rows returned by queries" + - tup_fetched_total: + usage: "COUNTER" + description: "Rows fetched by queries" + - tup_inserted_total: + usage: "COUNTER" + description: "Rows inserted" + - tup_updated_total: + usage: "COUNTER" + description: "Rows updated" + - tup_deleted_total: + usage: "COUNTER" + description: "Rows deleted" + - conflicts_total: + usage: "COUNTER" + description: "Queries canceled due to conflicts with recovery" + - temp_files_total: + usage: "COUNTER" + description: "Temp files created by queries" + - temp_bytes_total: + usage: "COUNTER" + description: "Temp data written by queries" + - deadlocks_total: + usage: "COUNTER" + description: "Deadlocks detected" + - most_recent_reset: + usage: "COUNTER" + description: "The most recent time one of the databases had its statistics reset" + +pg_stat_database_conflicts: + master: true + cache_seconds: 60 + query: | + SELECT sum(confl_tablespace) as confl_tablespace_total, + sum(confl_lock) as confl_lock_total, + sum(confl_snapshot) as confl_snapshot_total, + sum(confl_bufferpin) as confl_bufferpin_total, + sum(confl_deadlock) as confl_deadlock_total + from pg_stat_database_conflicts + metrics: + - confl_tablespace_total: + usage: "COUNTER" + description: "Queries cancelled due to dropped tablespaces" + - confl_lock_total: + usage: "COUNTER" + description: "Queries cancelled due to lock timeouts" + - confl_snapshot_total: + usage: "COUNTER" + description: "Queries cancelled due to old snapshots" + - confl_bufferpin_total: + usage: "COUNTER" + description: "Queries cancelled due to pinned buffers" + - confl_deadlock_total: + usage: "COUNTER" + description: "Queries cancelled due to deadlocks" + +pg_stat_statements: + master: true + cache_seconds: 60 + query: "SELECT sum(calls) as total_queries, sum(total_exec_time / 1000) as total_time_seconds FROM extensions.pg_stat_statements t1 JOIN pg_database t3 ON (t1.dbid=t3.oid)" + metrics: + - total_queries: + usage: "COUNTER" + description: "Number of times executed" + - total_time_seconds: + usage: "COUNTER" + description: "Total time spent, in seconds" + +auth_users: + master: true + cache_seconds: 60 + query: "select count(id) as user_count from auth.users" + metrics: + - user_count: + usage: "GAUGE" + description: "Number of users in the project db" + +realtime: + master: true + cache_seconds: 60 + query: "select count(1) as postgres_changes_total_subscriptions, count(distinct subscription_id) as postgres_changes_client_subscriptions from realtime.subscription" + metrics: + - postgres_changes_total_subscriptions: + usage: "GAUGE" + description: "Total subscription records listening for Postgres changes" + - postgres_changes_client_subscriptions: + usage: "GAUGE" + description: "Client subscriptions listening for Postgres changes" + +replication: + master: true + cache_seconds: 60 + query: "SELECT slot_name, pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) AS realtime_lag_bytes, active AS realtime_slot_status FROM pg_replication_slots WHERE slot_name LIKE ANY (ARRAY['realtime', 'realtime_rls', 'supabase_realtime_replication_slot%'])" + metrics: + - realtime_slot_name: + usage: "LABEL" + description: "Replication Slot Name for Realtime" + - realtime_lag_bytes: + usage: "GAUGE" + description: "Replication Lag for Realtime" + - realtime_slot_status: + usage: "GAUGE" + description: "Replication Slot Active Status" + +storage: + master: true + cache_seconds: 60 + query: "select sum(size) / (1024 * 1024) as storage_size_mb from storage.get_size_by_bucket()" + metrics: + - storage_size_mb: + usage: "GAUGE" + description: "The total size used for all storage buckets, in mb" + +supabase_usage_metrics: + # pg_stat_statements collects metrics from all databases on the cluster, so querying just the master db should be sufficient + master: true + cache_seconds: 60 + query: | + select sum(calls) as user_queries_total + from extensions.pg_stat_statements + where query <> 'SELECT version()' + and query <> 'BEGIN ISOLATION LEVEL READ COMMITTED READ ONLY' + and query <> 'COMMIT' + and query <> 'SET client_encoding = ''UTF8''' + and query <> 'SET client_min_messages TO WARNING' + and query <> 'LISTEN "ddl_command_end"' + and query <> 'LISTEN "pgrst"' + and query <> 'SELECT * FROM migrations ORDER BY id' + and query <> 'SELECT COUNT(*) = $1 FROM pg_publication WHERE pubname = $2' + and query <> 'SELECT COUNT(*) >= $1 FROM pg_replication_slots WHERE slot_name = $2' + and query <> 'SELECT EXISTS (SELECT schema_migrations.* FROM schema_migrations AS schema_migrations WHERE version = $1)' + and query <> 'SELECT current_setting($1)::integer, current_setting($2)' + and query <> 'SELECT pg_advisory_unlock($1)' + and query <> 'SELECT pg_try_advisory_lock($1)' + and query <> 'SELECT slot_name, pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) FROM pg_replication_slots' + and query <> 'SELECT typname::text, oid::int4, typarray::int4 FROM pg_type WHERE typname IN ($1,$2) ORDER BY typname' + and query <> 'select * from schema_migrations' + and query <> 'set local schema ''''' + and query <> 'SELECT SUM(pg_database_size(pg_database.datname)) / ($1 * $2) as size_mb FROM pg_database' + and query not like 'select set_config(%' + and query not like '%LATERAL (SELECT * FROM pg_namespace WHERE pg_namespace.oid = other.relnamespace) AS ns2%' + and query not like '%LEFT JOIN (pg_collation co JOIN pg_namespace nco ON co.collnamespace = nco.oid)%' + and query not like '%LEFT JOIN pg_description as d ON d.objoid = p.oid%' + and query not like '%LEFT JOIN pg_description as d on d.objoid = c.oid%' + and query not like '%-- CTE to replace information_schema.key_column_usage to remove owner limit%' + and query not like '%join pg_namespace sch on sch.oid = tbl.relnamespace%' + and query not like '%select setdatabase, unnest(setconfig) as setting from pg_catalog.pg_db_role_setting%' + and lower(trim(regexp_replace(regexp_replace(query, E'\n', ' ', 'g'), E'\\s+', ' ', 'g'))) not in + ('with rows as ( select id from net.http_request_queue order by id limit $1 ) delete from net.http_request_queue q using rows where q.id = rows.id returning q.id, q.method, q.url, timeout_milliseconds, array(select key || $2 || value from jsonb_each_text(q.headers)), q.body', + 'with rows as ( select ctid from net._http_response where created < now() - $1 order by created limit $2 ) delete from net._http_response r using rows where r.ctid = rows.ctid', + 'select exists ( select $2 from pg_catalog.pg_class c where c.relname = $1 and c.relkind = $3 )', + 'select description from pg_namespace n left join pg_description d on d.objoid = n.oid where n.nspname = $1', + 'select concat(schemaname, $1, tablename, $2, policyname) as policy from pg_policies order by 1 desc', + 'select concat(table_schema, $1, table_name) as table from information_schema.tables where table_schema not like $2 and table_schema <> $3 order by 1 desc', + 'select concat(conrelid::regclass, $1, conname) as fk from pg_constraint where contype = $2 order by 1 desc', + 'select datname from pg_database where datallowconn = $1 order by oid asc') + and query <> 'insert into schema_migrations (version) values ($1)' + -- temporarily included for older versions of pg_net + and query not like 'SELECT%FROM net.http_request_queue%' + and query not like 'DELETE FROM net.http_request_queue%' + and query not like '%source: project usage%' + metrics: + - user_queries_total: + usage: "COUNTER" + description: "The total number of user queries executed" + +pg_settings: + master: true + cache-seconds: 30 + query: "SELECT COUNT(*) as default_transaction_read_only FROM pg_settings WHERE name = 'default_transaction_read_only' AND setting = 'on';" + metrics: + - default_transaction_read_only: + usage: "GAUGE" + description: "Default transaction mode set to read only" + +pg_status: + master: true + cache-seconds: 60 + query: "SELECT CASE WHEN pg_is_in_recovery() = false THEN 0 ELSE 1 END as in_recovery" + metrics: + - in_recovery: + usage: "GAUGE" + description: "Database in recovery" diff --git a/docker/all-in-one-nix/postgres-entrypoint.sh b/docker/all-in-one-nix/postgres-entrypoint.sh new file mode 100755 index 000000000..04e447ea4 --- /dev/null +++ b/docker/all-in-one-nix/postgres-entrypoint.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash + +# Downloaded from https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh +# Changes needed to make adminapi able to read the recovery.signal file: +# -44: chmod 00700 "$PGDATA" || : +# +44: chmod 00750 "$PGDATA" || : +# +# We're already including the original file in the base postgres Docker image. + +set -Eeo pipefail + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + printf >&2 'error: both %s and %s are set (but are exclusive)\n' "$var" "$fileVar" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user +docker_create_db_directories() { + local user; user="$(id -u)" + + mkdir -p "$PGDATA" + # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) + chmod 00750 "$PGDATA" || : + + # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 + mkdir -p /var/run/postgresql || : + chmod 03775 /var/run/postgresql || : + + # Create the transaction log directory before initdb is run so the directory is owned by the correct user + if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then + mkdir -p "$POSTGRES_INITDB_WALDIR" + if [ "$user" = '0' ]; then + find "$POSTGRES_INITDB_WALDIR" \! -user postgres -exec chown postgres '{}' + + fi + chmod 700 "$POSTGRES_INITDB_WALDIR" + fi + + # allow the container to be started with `--user` + if [ "$user" = '0' ]; then + find "$PGDATA" \! -user postgres -exec chown postgres '{}' + + find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + + fi +} + +# initialize empty PGDATA directory with new database via 'initdb' +# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function +# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames +# this is also where the database user is created, specified by `POSTGRES_USER` env +docker_init_database_dir() { + # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary + # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html + local uid; uid="$(id -u)" + if ! getent passwd "$uid" &> /dev/null; then + # see if we can find a suitable "libnss_wrapper.so" (https://salsa.debian.org/sssd-team/nss-wrapper/-/commit/b9925a653a54e24d09d9b498a2d913729f7abb15) + local wrapper + for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do + if [ -s "$wrapper" ]; then + NSS_WRAPPER_PASSWD="$(mktemp)" + NSS_WRAPPER_GROUP="$(mktemp)" + export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + local gid; gid="$(id -g)" + printf 'postgres:x:%s:%s:PostgreSQL:%s:/bin/false\n' "$uid" "$gid" "$PGDATA" > "$NSS_WRAPPER_PASSWD" + printf 'postgres:x:%s:\n' "$gid" > "$NSS_WRAPPER_GROUP" + break + fi + done + fi + + if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then + set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@" + fi + + # --pwfile refuses to handle a properly-empty file (hence the "\n"): https://github.com/docker-library/postgres/issues/1025 + eval 'initdb --username="$POSTGRES_USER" --pwfile=<(printf "%s\n" "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' + + # unset/cleanup "nss_wrapper" bits + if [[ "${LD_PRELOAD:-}" == */libnss_wrapper.so ]]; then + rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" + unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + fi +} + +# print large warning if POSTGRES_PASSWORD is long +# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' +# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' +# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] +docker_verify_minimum_env() { + # check password first so we can output the warning before postgres + # messes it up + if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then + cat >&2 <<-'EOWARN' + + WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. + + This will not work if used via PGPASSWORD with "psql". + + https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) + https://github.com/docker-library/postgres/issues/507 + + EOWARN + fi + if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOE' + Error: Database is uninitialized and superuser password is not specified. + You must specify POSTGRES_PASSWORD to a non-empty value for the + superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". + + You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all + connections without a password. This is *not* recommended. + + See PostgreSQL documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + EOE + exit 1 + fi + if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then + cat >&2 <<-'EOWARN' + ******************************************************************************** + WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow + anyone with access to the Postgres port to access your database without + a password, even if POSTGRES_PASSWORD is set. See PostgreSQL + documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + In Docker's default configuration, this is effectively any other + container on the same system. + + It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace + it with "-e POSTGRES_PASSWORD=password" instead to set a password in + "docker run". + ******************************************************************************** + EOWARN + fi +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions and permissions +docker_process_init_files() { + # psql here for backwards compatibility "${psql[@]}" + psql=( docker_process_sql ) + + printf '\n' + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + printf '%s: running %s\n' "$0" "$f" + "$f" + else + printf '%s: sourcing %s\n' "$0" "$f" + . "$f" + fi + ;; + *.sql) printf '%s: running %s\n' "$0" "$f"; docker_process_sql -f "$f"; printf '\n' ;; + *.sql.gz) printf '%s: running %s\n' "$0" "$f"; gunzip -c "$f" | docker_process_sql; printf '\n' ;; + *.sql.xz) printf '%s: running %s\n' "$0" "$f"; xzcat "$f" | docker_process_sql; printf '\n' ;; + *.sql.zst) printf '%s: running %s\n' "$0" "$f"; zstd -dc "$f" | docker_process_sql; printf '\n' ;; + *) printf '%s: ignoring %s\n' "$0" "$f" ;; + esac + printf '\n' + done +} + +# Execute sql script, passed via stdin (or -f flag of pqsl) +# usage: docker_process_sql [psql-cli-args] +# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' +# ie: docker_process_sql -f my-file.sql +# ie: docker_process_sql > "$PGDATA/pg_hba.conf" +} + +# start socket-only postgresql server for setting up or running scripts +# all arguments will be passed along as arguments to `postgres` (via pg_ctl) +docker_temp_server_start() { + if [ "$1" = 'postgres' ]; then + shift + fi + + # internal start of server in order to allow setup using psql client + # does not listen on external TCP/IP and waits until start finishes + set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" \ + -o "$(printf '%q ' "$@")" \ + -w start +} + +# stop postgresql server after done setting up user and running scripts +docker_temp_server_stop() { + PGUSER="${PGUSER:-postgres}" \ + pg_ctl -D "$PGDATA" -m fast -w stop +} + +# check arguments for an option that would cause postgres to stop +# return true if there is one +_pg_want_help() { + local arg + for arg; do + case "$arg" in + # postgres --help | grep 'then exit' + # leaving out -C on purpose since it always fails and is unhelpful: + # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory + -'?'|--help|--describe-config|-V|--version) + return 0 + ;; + esac + done + return 1 +} + +_main() { + # if first arg looks like a flag, assume we want to run postgres server + if [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" + fi + + if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then + docker_setup_env + # setup data directories and permissions (when run as root) + docker_create_db_directories + if [ "$(id -u)" = '0' ]; then + # then restart script as postgres user + exec gosu postgres "$BASH_SOURCE" "$@" + fi + + # only run initialization on an empty data directory + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + docker_verify_minimum_env + + # check dir permissions to reduce likelihood of half-initialized database + ls /docker-entrypoint-initdb.d/ > /dev/null + + docker_init_database_dir + pg_setup_hba_conf "$@" + + # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless + # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + docker_temp_server_start "$@" + + docker_setup_db + docker_process_init_files /docker-entrypoint-initdb.d/* + + docker_temp_server_stop + unset PGPASSWORD + + cat <<-'EOM' + + PostgreSQL init process complete; ready for start up. + + EOM + else + cat <<-'EOM' + + PostgreSQL Database directory appears to contain a database; Skipping initialization + + EOM + fi + fi + + exec "$@" +} + +if ! _is_sourced; then + _main "$@" +fi diff --git a/docker/all-in-one-nix/run-logrotate.sh b/docker/all-in-one-nix/run-logrotate.sh new file mode 100755 index 000000000..40805f855 --- /dev/null +++ b/docker/all-in-one-nix/run-logrotate.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eou pipefail + +while true; do + sleep 1800 + /usr/sbin/logrotate /etc/logrotate.conf --state "${DATA_VOLUME_MOUNTPOINT}/etc/logrotate/logrotate.state" --verbose +done diff --git a/docker/all-in-one-nix/shutdown.sh b/docker/all-in-one-nix/shutdown.sh new file mode 100755 index 000000000..9f5beb250 --- /dev/null +++ b/docker/all-in-one-nix/shutdown.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# This script provides a method of shutting down the machine/container when the database has been idle +# for a certain amount of time (configurable via the MAX_IDLE_TIME_MINUTES env var) +# +# It checks for any active (non-idle) connections and for any connections which have been idle for more than MAX_IDLE_TIME_MINUTES. +# If there are no active connections and no idle connections, it then checks if the last disconnection event happened more than MAX_IDLE_TIME_MINUTES ago. +# +# If all of these conditions are met, then Postgres is shut down, allowing it to wrap up any pending transactions (such as WAL shippipng) and gracefully exit. +# To terminate the machine/container, a SIGTERM signal is sent to the top-level process (supervisord) which will then shut down all other processes and exit. + +DEFAULT_MAX_IDLE_TIME_MINUTES=${MAX_IDLE_TIME_MINUTES:-5} +CONFIG_FILE_PATH=${CONFIG_FILE_PATH:-/etc/supa-shutdown/shutdown.conf} + +run_sql() { + psql -h localhost -U supabase_admin -d postgres "$@" +} + +check_activity() { + pg_isready -h localhost > /dev/null 2>&1 || (echo "Postgres is not ready yet" && exit 1) + + QUERY=$(cat </dev/null || echo 0) + NOW=$(date +%s) + TIME_SINCE_LAST_DISCONNECT="$((NOW - LAST_DISCONNECT_TIME))" + + if [ $TIME_SINCE_LAST_DISCONNECT -gt "$((MAX_IDLE_TIME_MINUTES * 60))" ]; then + echo "$(date): No active connections for $MAX_IDLE_TIME_MINUTES minutes. Shutting down." + + supervisorctl stop postgresql + + # Postgres ships the latest WAL file using archive_command during shutdown, in a blocking operation + # This is to ensure that the WAL file is shipped, just in case + sleep 1 + + /usr/bin/admin-mgr lsn-checkpoint-push --immediately || echo "Failed to push LSN checkpoint" + + kill -s TERM "$(supervisorctl pid)" + fi +} + +# Wait for Postgres to be up +until pg_isready -h localhost > /dev/null 2>&1; + do sleep 3 +done + +# Enable logging of disconnections so the script can check when the last disconnection happened +run_sql -c "ALTER SYSTEM SET log_disconnections = 'on';" +run_sql -c "SELECT pg_reload_conf();" + +sleep $((DEFAULT_MAX_IDLE_TIME_MINUTES * 60)) +while true; do + if [ -f "$CONFIG_FILE_PATH" ]; then + source "$CONFIG_FILE_PATH" + + if [ -z "$SHUTDOWN_IDLE_TIME_MINUTES" ]; then + MAX_IDLE_TIME_MINUTES="$DEFAULT_MAX_IDLE_TIME_MINUTES" + else + MAX_IDLE_TIME_MINUTES="$SHUTDOWN_IDLE_TIME_MINUTES" + fi + else + MAX_IDLE_TIME_MINUTES="$DEFAULT_MAX_IDLE_TIME_MINUTES" + fi + + if [ "$MAX_IDLE_TIME_MINUTES" -gt 0 ] && [ "$MAX_IDLE_TIME_MINUTES" -lt 50000000 ]; then + check_activity + fi + + sleep 30 +done diff --git a/ebssurrogate/scripts/chroot-bootstrap-nix.sh b/ebssurrogate/scripts/chroot-bootstrap-nix.sh new file mode 100755 index 000000000..4b7ceaf8d --- /dev/null +++ b/ebssurrogate/scripts/chroot-bootstrap-nix.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# +# This script runs inside chrooted environment. It installs grub and its +# Configuration file. +# + +set -o errexit +set -o pipefail +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +export APT_OPTIONS="-oAPT::Install-Recommends=false \ + -oAPT::Install-Suggests=false \ + -oAcquire::Languages=none" + +if [ $(dpkg --print-architecture) = "amd64" ]; +then + ARCH="amd64"; +else + ARCH="arm64"; +fi + + + +function update_install_packages { + source /etc/os-release + + # Update APT with new sources + cat /etc/apt/sources.list + apt-get $APT_OPTIONS update && apt-get $APT_OPTIONS --yes dist-upgrade + + # Do not configure grub during package install + if [ "${ARCH}" = "amd64" ]; then + echo 'grub-pc grub-pc/install_devices_empty select true' | debconf-set-selections + echo 'grub-pc grub-pc/install_devices select' | debconf-set-selections + # Install various packages needed for a booting system + apt-get install -y \ + linux-aws \ + grub-pc \ + e2fsprogs + else + apt-get install -y e2fsprogs + fi + # Install standard packages + apt-get install -y \ + sudo \ + wget \ + cloud-init \ + acpid \ + ec2-hibinit-agent \ + ec2-instance-connect \ + hibagent \ + ncurses-term \ + ssh-import-id \ + + # apt upgrade + apt-get upgrade -y + + # Install OpenSSH and other packages + sudo add-apt-repository universe + apt-get update + apt-get install -y --no-install-recommends \ + openssh-server \ + git \ + ufw \ + cron \ + logrotate \ + fail2ban \ + locales \ + at \ + less \ + python3-systemd + + if [ "${ARCH}" = "arm64" ]; then + apt-get $APT_OPTIONS --yes install linux-aws initramfs-tools dosfstools + fi +} + +function setup_locale { +cat << EOF >> /etc/locale.gen +en_US.UTF-8 UTF-8 +EOF + +cat << EOF > /etc/default/locale +LANG="C.UTF-8" +LC_CTYPE="C.UTF-8" +EOF + localedef -i en_US -f UTF-8 en_US.UTF-8 +} + +function install_packages_for_build { + apt-get install -y --no-install-recommends linux-libc-dev \ + acl \ + magic-wormhole sysstat \ + build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libxslt-dev libssl-dev libsystemd-dev libpq-dev libxml2-utils uuid-dev xsltproc ssl-cert \ + gcc-10 g++-10 \ + libgeos-dev libproj-dev libgdal-dev libjson-c-dev libboost-all-dev libcgal-dev libmpfr-dev libgmp-dev cmake \ + libkrb5-dev \ + maven default-jre default-jdk \ + curl gpp apt-transport-https cmake libc++-dev libc++abi-dev libc++1 libglib2.0-dev libtinfo5 libc++abi1 ninja-build python \ + liblzo2-dev + + source /etc/os-release + + apt-get install -y --no-install-recommends llvm-11-dev clang-11 + # Mark llvm as manual to prevent auto removal + apt-mark manual libllvm11:arm64 +} + +function setup_apparmor { + apt-get install -y apparmor apparmor-utils auditd + + # Copy apparmor profiles + cp -rv /tmp/apparmor_profiles/* /etc/apparmor.d/ +} + +function setup_grub_conf_arm64 { +cat << EOF > /etc/default/grub +GRUB_DEFAULT=0 +GRUB_TIMEOUT=0 +GRUB_TIMEOUT_STYLE="hidden" +GRUB_DISTRIBUTOR="Supabase postgresql" +GRUB_CMDLINE_LINUX_DEFAULT="nomodeset console=tty1 console=ttyS0 ipv6.disable=0" +EOF +} + +# Install GRUB +function install_configure_grub { + if [ "${ARCH}" = "arm64" ]; then + apt-get $APT_OPTIONS --yes install cloud-guest-utils fdisk grub-efi-arm64 efibootmgr + setup_grub_conf_arm64 + rm -rf /etc/grub.d/30_os-prober + sleep 1 + fi + grub-install /dev/xvdf && update-grub +} + +# skip fsck for first boot +function disable_fsck { + touch /fastboot +} + +# Don't request hostname during boot but set hostname +function setup_hostname { + sed -i 's/gethostname()/ubuntu /g' /etc/dhcp/dhclient.conf + sed -i 's/host-name,//g' /etc/dhcp/dhclient.conf + echo "ubuntu" > /etc/hostname + chmod 644 /etc/hostname +} + +# Set options for the default interface +function setup_eth0_interface { +cat << EOF > /etc/netplan/eth0.yaml +network: + version: 2 + ethernets: + eth0: + dhcp4: true +EOF +} + +function disable_sshd_passwd_auth { + sed -i -E -e 's/^#?\s*PasswordAuthentication\s+(yes|no)\s*$/PasswordAuthentication no/g' \ + -e 's/^#?\s*ChallengeResponseAuthentication\s+(yes|no)\s*$/ChallengeResponseAuthentication no/g' \ + /etc/ssh/sshd_config +} + +function create_admin_account { + groupadd admin +} + +#Set default target as multi-user +function set_default_target { + rm -f /etc/systemd/system/default.target + ln -s /lib/systemd/system/multi-user.target /etc/systemd/system/default.target +} + +# Setup ccache +function setup_ccache { + apt-get install ccache -y + mkdir -p /tmp/ccache + export PATH=/usr/lib/ccache:$PATH + echo "PATH=$PATH" >> /etc/environment +} + +# Clear apt caches +function cleanup_cache { + apt-get clean +} + +update_install_packages +setup_locale +#install_packages_for_build +install_configure_grub +setup_apparmor +setup_hostname +create_admin_account +set_default_target +setup_eth0_interface +disable_sshd_passwd_auth +disable_fsck +#setup_ccache +cleanup_cache diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh new file mode 100755 index 000000000..8b7d6c238 --- /dev/null +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -0,0 +1,324 @@ +#!/usr/bin/env bash +# +# This script creates filesystem and setups up chrooted +# enviroment for further processing. It also runs +# ansible playbook and finally does system cleanup. +# +# Adapted from: https://github.com/jen20/packer-ubuntu-zfs + +set -o errexit +set -o pipefail +set -o xtrace + +if [ $(dpkg --print-architecture) = "amd64" ]; +then + ARCH="amd64"; +else + ARCH="arm64"; +fi + +function waitfor_boot_finished { + export DEBIAN_FRONTEND=noninteractive + + echo "args: ${ARGS}" + # Wait for cloudinit on the surrogate to complete before making progress + while [[ ! -f /var/lib/cloud/instance/boot-finished ]]; do + echo 'Waiting for cloud-init...' + sleep 1 + done +} + +function install_packages { + # Setup Ansible on host VM + apt-get update && sudo apt-get install software-properties-common -y + add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y + ansible-galaxy collection install community.general + + # Update apt and install required packages + apt-get update + apt-get install -y \ + gdisk \ + e2fsprogs \ + debootstrap \ + nvme-cli +} + +# Partition the new root EBS volume +function create_partition_table { + + if [ "${ARCH}" = "arm64" ]; then + parted --script /dev/xvdf \ + mklabel gpt \ + mkpart UEFI 1MiB 100MiB \ + mkpart ROOT 100MiB 100% + set 1 esp on \ + set 1 boot on + parted --script /dev/xvdf print + else + sgdisk -Zg -n1:0:4095 -t1:EF02 -c1:GRUB -n2:0:0 -t2:8300 -c2:EXT4 /dev/xvdf + fi + + sleep 2 +} + +function device_partition_mappings { + # NVMe EBS launch device mappings (symlinks): /dev/nvme*n* to /dev/xvd* + declare -A blkdev_mappings + for blkdev in $(nvme list | awk '/^\/dev/ { print $1 }'); do # /dev/nvme*n* + # Mapping info from disk headers + header=$(nvme id-ctrl --raw-binary "${blkdev}" | cut -c3073-3104 | tr -s ' ' | sed 's/ $//g' | sed 's!/dev/!!') + mapping="/dev/${header%%[0-9]}" # normalize sda1 => sda + + # Create /dev/xvd* device symlink + if [[ ! -z "$mapping" ]] && [[ -b "${blkdev}" ]] && [[ ! -L "${mapping}" ]]; then + ln -s "$blkdev" "$mapping" + + blkdev_mappings["$blkdev"]="$mapping" + fi + done + + create_partition_table + + # NVMe EBS launch device partition mappings (symlinks): /dev/nvme*n*p* to /dev/xvd*[0-9]+ + declare -A partdev_mappings + for blkdev in "${!blkdev_mappings[@]}"; do # /dev/nvme*n* + mapping="${blkdev_mappings[$blkdev]}" + + # Create /dev/xvd*[0-9]+ partition device symlink + for partdev in "${blkdev}"p*; do + partnum=${partdev##*p} + if [[ ! -L "${mapping}${partnum}" ]]; then + ln -s "${blkdev}p${partnum}" "${mapping}${partnum}" + + partdev_mappings["${blkdev}p${partnum}"]="${mapping}${partnum}" + fi + done + done +} + + +#Download and install latest e2fsprogs for fast_commit feature,if required. +function format_and_mount_rootfs { + mkfs.ext4 -m0.1 /dev/xvdf2 + + mount -o noatime,nodiratime /dev/xvdf2 /mnt + if [ "${ARCH}" = "arm64" ]; then + mkfs.fat -F32 /dev/xvdf1 + mkdir -p /mnt/boot/efi + sleep 2 + mount /dev/xvdf1 /mnt/boot/efi + fi + + mkfs.ext4 /dev/xvdh + mkdir -p /mnt/data + mount -o defaults,discard /dev/xvdh /mnt/data +} + +function create_swapfile { + fallocate -l 1G /mnt/swapfile + chmod 600 /mnt/swapfile + mkswap /mnt/swapfile +} + +function format_build_partition { + mkfs.ext4 -O ^has_journal /dev/xvdc +} +function pull_docker { + apt-get install -y docker.io + docker run -itd --name ccachedata "${DOCKER_IMAGE}:${DOCKER_IMAGE_TAG}" sh + docker exec -itd ccachedata mkdir -p /build/ccache +} + +# Create fstab +function create_fstab { + FMT="%-42s %-11s %-5s %-17s %-5s %s" +cat > "/mnt/etc/fstab" << EOF +$(printf "${FMT}" "# DEVICE UUID" "MOUNTPOINT" "TYPE" "OPTIONS" "DUMP" "FSCK") +$(findmnt -no SOURCE /mnt | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/", "ext4", "defaults,discard", "0", "1" ) }') +$(findmnt -no SOURCE /mnt/boot/efi | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/boot/efi", "vfat", "umask=0077", "0", "1" ) }') +$(findmnt -no SOURCE /mnt/data | xargs blkid -o export | awk -v FMT="${FMT}" '/^UUID=/ { printf(FMT, $0, "/data", "ext4", "defaults,discard", "0", "2" ) }') +$(printf "$FMT" "/swapfile" "none" "swap" "sw" "0" "0") +EOF + unset FMT +} + +function setup_chroot_environment { + UBUNTU_VERSION=$(lsb_release -cs) # 'focal' for Ubuntu 20.04 + + # Bootstrap Ubuntu into /mnt + debootstrap --arch ${ARCH} --variant=minbase "$UBUNTU_VERSION" /mnt + + # Update ec2-region + REGION=$(curl --silent --fail http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -E 's|[a-z]+$||g') + sed -i "s/REGION/${REGION}/g" /tmp/sources.list + cp /tmp/sources.list /mnt/etc/apt/sources.list + + if [ "${ARCH}" = "arm64" ]; then + create_fstab + fi + + # Create mount points and mount the filesystem + mkdir -p /mnt/{dev,proc,sys} + mount --rbind /dev /mnt/dev + mount --rbind /proc /mnt/proc + mount --rbind /sys /mnt/sys + + # Create build mount point and mount + mkdir -p /mnt/tmp + mount /dev/xvdc /mnt/tmp + chmod 777 /mnt/tmp + + # Copy apparmor profiles + chmod 644 /tmp/apparmor_profiles/* + cp -r /tmp/apparmor_profiles /mnt/tmp/ + + # Copy migrations + cp -r /tmp/migrations /mnt/tmp/ + + # Copy unit tests + cp -r /tmp/unit-tests /mnt/tmp/ + + # Copy the bootstrap script into place and execute inside chroot + cp /tmp/chroot-bootstrap-nix.sh /mnt/tmp/chroot-bootstrap-nix.sh + chroot /mnt /tmp/chroot-bootstrap-nix.sh + rm -f /mnt/tmp/chroot-bootstrap-nix.sh + echo "${POSTGRES_SUPABASE_VERSION}" > /mnt/root/supabase-release + + # Copy the nvme identification script into /sbin inside the chroot + mkdir -p /mnt/sbin + cp /tmp/ebsnvme-id /mnt/sbin/ebsnvme-id + chmod +x /mnt/sbin/ebsnvme-id + + # Copy the udev rules for identifying nvme devices into the chroot + mkdir -p /mnt/etc/udev/rules.d + cp /tmp/70-ec2-nvme-devices.rules \ + /mnt/etc/udev/rules.d/70-ec2-nvme-devices.rules + + #Copy custom cloud-init + rm -f /mnt/etc/cloud/cloud.cfg + cp /tmp/cloud.cfg /mnt/etc/cloud/cloud.cfg + + sleep 2 +} + +function download_ccache { + docker cp ccachedata:/build/ccache/. /mnt/tmp/ccache +} + +function execute_playbook { + +tee /etc/ansible/ansible.cfg < Date: Fri, 3 May 2024 10:23:01 -0400 Subject: [PATCH 018/115] chore: deactivate on PR for these while I refactor them, trigger manually only for now --- .github/workflows/testinfra.yml | 2 +- .github/workflows/text-nix.yml | 8 ++-- .github/workflows/textinfra-nix.yml | 62 ++++++++++++++--------------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/testinfra.yml b/.github/workflows/testinfra.yml index 9b60d852c..3d9ff465a 100644 --- a/.github/workflows/testinfra.yml +++ b/.github/workflows/testinfra.yml @@ -1,7 +1,7 @@ name: Testinfra Integration Tests on: - pull_request: + #pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/text-nix.yml b/.github/workflows/text-nix.yml index 89acf3328..6c5778c7f 100644 --- a/.github/workflows/text-nix.yml +++ b/.github/workflows/text-nix.yml @@ -1,10 +1,10 @@ name: Test Database on: - push: - branches: - - develop - pull_request: + # push: + # branches: + # - develop + # pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/textinfra-nix.yml b/.github/workflows/textinfra-nix.yml index 9b60d852c..f88cc6077 100644 --- a/.github/workflows/textinfra-nix.yml +++ b/.github/workflows/textinfra-nix.yml @@ -1,43 +1,43 @@ name: Testinfra Integration Tests on: - pull_request: + #pull_request: workflow_dispatch: jobs: - test-all-in-one: - strategy: - matrix: - include: - - runner: [self-hosted, X64] - arch: amd64 - - runner: arm-runner - arch: arm64 - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - steps: - - uses: actions/checkout@v3 - - - run: docker context create builders - - uses: docker/setup-buildx-action@v3 - with: - endpoint: builders - - - name: Run aio integration tests - run: | - # TODO: use poetry for pkg mgmt - pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests + # test-all-in-one: + # strategy: + # matrix: + # include: + # - runner: [self-hosted, X64] + # arch: amd64 + # - runner: arm-runner + # arch: arm64 + # runs-on: ${{ matrix.runner }} + # timeout-minutes: 30 + # steps: + # - uses: actions/checkout@v3 + + # - run: docker context create builders + # - uses: docker/setup-buildx-action@v3 + # with: + # endpoint: builders + + # - name: Run aio integration tests + # run: | + # # TODO: use poetry for pkg mgmt + # pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests - if ! pytest -vv testinfra/test_all_in_one.py; then - # display container logs if the test fails + # if ! pytest -vv testinfra/test_all_in_one.py; then + # # display container logs if the test fails - if [ -f testinfra-aio-container-logs.log ]; then - echo "AIO container logs:" - cat testinfra-aio-container-logs.log - fi - exit 1 - fi + # if [ -f testinfra-aio-container-logs.log ]; then + # echo "AIO container logs:" + # cat testinfra-aio-container-logs.log + # fi + # exit 1 + # fi test-ami: strategy: From 76f6606343e2e20d3a73b105a034d70f1fb39103 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 3 May 2024 13:12:00 -0400 Subject: [PATCH 019/115] feat: tying up loose ends on the creation and deployment to staging of the 2 stage ami --- .github/workflows/ami-release-nix.yml | 73 ++++++++++++++++----------- stage2-nix-psql.pkr.hcl | 22 +++++++- 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index d25bfc793..3864394ab 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -67,39 +67,28 @@ jobs: - id: version run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" - - name: Build Postgres deb - uses: docker/build-push-action@v5 - with: - file: docker/Dockerfile - target: pg-deb - build-args: | - ubuntu_release=${{ matrix.ubuntu_release }} - ubuntu_release_no=${{ matrix.ubuntu_version }} - postgresql_major=${{ steps.version.outputs.postgresql_major }} - postgresql_release=${{ steps.version.outputs.postgresql_release }} - CPPFLAGS=-mcpu=${{ matrix.mcpu }} - tags: supabase/postgres:deb - platforms: linux/${{ matrix.arch }} - outputs: type=tar,dest=/tmp/pg-deb.tar - cache-from: type=gha,scope=${{ github.ref_name }}-deb - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-deb + # - name: Build Postgres deb + # uses: docker/build-push-action@v5 + # with: + # file: docker/Dockerfile + # target: pg-deb + # build-args: | + # ubuntu_release=${{ matrix.ubuntu_release }} + # ubuntu_release_no=${{ matrix.ubuntu_version }} + # postgresql_major=${{ steps.version.outputs.postgresql_major }} + # postgresql_release=${{ steps.version.outputs.postgresql_release }} + # CPPFLAGS=-mcpu=${{ matrix.mcpu }} + # tags: supabase/postgres:deb + # platforms: linux/${{ matrix.arch }} + # outputs: type=tar,dest=/tmp/pg-deb.tar + # cache-from: type=gha,scope=${{ github.ref_name }}-deb + # cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-deb # - name: Extract Postgres deb # run: | # mkdir -p ansible/files/postgres # tar xvf /tmp/pg-deb.tar -C ansible/files/postgres --strip-components 1 #TODO remove this block as deb is build in nix prior to this step - - name: Build AMI stage 1 - run: | - packer init amazon-arm64-nix.pkr.hcl - GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl - - - name: Build AMI stage 1 - run: | - packer init amazon-arm64-nix.pkr.hcl - GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl - name: Grab release version id: process_release_version @@ -108,7 +97,7 @@ jobs: echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: configure aws credentials - staging - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} aws-region: "us-east-1" @@ -127,7 +116,7 @@ jobs: # #TODO look to see if this only pg binaries and if so, remove this as it is covered by nix build # TODO deactivate this block to assure binaries from this file are not uploaded. This is covered by nix build - name: configure aws credentials - prod - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} aws-region: "us-east-1" @@ -140,6 +129,32 @@ jobs: -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ manifest-playbook.yml + - name: Build AMI stage 1 + run: | + packer init amazon-arm64-nix.pkr.hcl + GIT_SHA=${{github.sha}} + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl + + - name: Set Environment Variables + id: set-env-vars + run: | + POSTGRES_VERSION=$(grep -oP '(?<=postgres-version = ").*(?=")' common-nix.vars.pkr.hcl) + echo "::set-output name=postgres-version::$POSTGRES_VERSION" + + - name: Get AMI Owner + id: get-ami-owner + run: | + POSTGRES_VERSION="${{ steps.set-env-vars.outputs.postgres-version }}" + AMI_NAME="supabase-postgres-$POSTGRES_VERSION-stage-1" + OWNER=$(aws ec2 describe-images --filters "Name=name,Values=$AMI_NAME" "Name=state,Values=available" --query 'Images[].OwnerId' --output text) + echo "::set-output name=ami-owner::$OWNER" + + - name: Build AMI stage 2 + run: | + packer init stage2-nix-psql.pkr.hcl + GIT_SHA=${{github.sha}} + packer build -var "ami-owner-id=${OWNER}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + # - name: Upload pg binaries to s3 prod # run: | # aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index bd8501acc..6851381b5 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -27,6 +27,26 @@ variable "postgres-version" { default = "" } +variable "git-head-version" { + type = string + default = "unknown" +} + +variable "packer-execution-id" { + type = string + default = "unknown" +} + +variable "force-deregister" { + type = bool + default = false +} + +variable "ami-owner-id" { + type = string + default = "" +} + packer { required_plugins { amazon = { @@ -47,7 +67,7 @@ source "amazon-ebs" "ubuntu" { virtualization-type = "hvm" } most_recent = true - owners = ["194568623217"] + owners = ["${var.ami-owner-id}"] } ssh_username = "ubuntu" ena_support = true From 92d96f0777c8f97a1caf3ed7d9a401d4e5ad61ab Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 3 May 2024 13:29:45 -0400 Subject: [PATCH 020/115] update the version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 75b879061..ec57e481c 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.42" +postgres-version = "15.6.1.1-nix-staged" From 6ce64521592e60a35fac960f53046d9ddeb53216 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 3 May 2024 14:47:33 -0400 Subject: [PATCH 021/115] fix: rename workflow --- .github/workflows/ami-release-nix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 3864394ab..9da727028 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -1,4 +1,4 @@ -name: Release AMI +name: Release AMI Nix on: push: From ebb97e0b1764fc2f0cd3c4bef757e3f0fc2954b5 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 3 May 2024 14:50:34 -0400 Subject: [PATCH 022/115] fix: try to get see the workflow appear as an option --- .github/workflows/ami-release-nix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 9da727028..ac6ebdf47 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -3,7 +3,7 @@ name: Release AMI Nix on: push: branches: - - develop + - sam/2-stage-ami-nix paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' From 51f05a379d47b7a8438661d32cfab2d9adc00c9e Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 3 May 2024 14:58:31 -0400 Subject: [PATCH 023/115] fix: narrow down the vars needed --- .github/workflows/ami-release-nix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index ac6ebdf47..23cce16e4 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -65,8 +65,8 @@ jobs: # tar xvf /tmp/extensions.tar -C ansible/files/extensions --strip-components 1 # TODO remove this block as extensions are build in nix prior to this step - - id: version - run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" + # - id: version + # run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" # - name: Build Postgres deb # uses: docker/build-push-action@v5 # with: From db2182137afb075272196aaa98f4aecf0e6b0efd Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 09:35:19 -0400 Subject: [PATCH 024/115] fix: use perms in gh runner for ami publishing prior to establishing s3 permissions --- .github/workflows/ami-release-nix.yml | 39 +++++++++------------------ 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 23cce16e4..7488e49da 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -89,7 +89,18 @@ jobs: # tar xvf /tmp/pg-deb.tar -C ansible/files/postgres --strip-components 1 #TODO remove this block as deb is build in nix prior to this step - + - name: Build AMI stage 1 + run: | + packer init amazon-arm64-nix.pkr.hcl + GIT_SHA=${{github.sha}} + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl + + - name: Build AMI stage 2 + run: | + packer init stage2-nix-psql.pkr.hcl + GIT_SHA=${{github.sha}} + packer build -var "ami-owner-id=${OWNER}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + - name: Grab release version id: process_release_version run: | @@ -128,32 +139,6 @@ jobs: -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ manifest-playbook.yml - - - name: Build AMI stage 1 - run: | - packer init amazon-arm64-nix.pkr.hcl - GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl - - - name: Set Environment Variables - id: set-env-vars - run: | - POSTGRES_VERSION=$(grep -oP '(?<=postgres-version = ").*(?=")' common-nix.vars.pkr.hcl) - echo "::set-output name=postgres-version::$POSTGRES_VERSION" - - - name: Get AMI Owner - id: get-ami-owner - run: | - POSTGRES_VERSION="${{ steps.set-env-vars.outputs.postgres-version }}" - AMI_NAME="supabase-postgres-$POSTGRES_VERSION-stage-1" - OWNER=$(aws ec2 describe-images --filters "Name=name,Values=$AMI_NAME" "Name=state,Values=available" --query 'Images[].OwnerId' --output text) - echo "::set-output name=ami-owner::$OWNER" - - - name: Build AMI stage 2 - run: | - packer init stage2-nix-psql.pkr.hcl - GIT_SHA=${{github.sha}} - packer build -var "ami-owner-id=${OWNER}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl # - name: Upload pg binaries to s3 prod # run: | From faf337a5cb93589ee05f7b44f9fc2c6eadf1f504 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 09:37:50 -0400 Subject: [PATCH 025/115] docs: add some notes about the workflow ordering --- .github/workflows/ami-release-nix.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 7488e49da..9d306f347 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -126,6 +126,12 @@ jobs: # aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz # #TODO look to see if this only pg binaries and if so, remove this as it is covered by nix build # TODO deactivate this block to assure binaries from this file are not uploaded. This is covered by nix build + + #Our self hosted github runner already has permissions to publish images + #but they're limited to only that; + #so if we want s3 access we'll need to config credentials with the below steps + # (which overwrites existing perms) after the ami build + - name: configure aws credentials - prod uses: aws-actions/configure-aws-credentials@v4 with: From 553faca37a6dd8dcf148a52b57f7d52c0ef5a671 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 11:41:58 -0400 Subject: [PATCH 026/115] fix: avoid passing around owner id as this is supported in the gh action --- stage2-nix-psql.pkr.hcl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 6851381b5..417903b94 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -42,11 +42,6 @@ variable "force-deregister" { default = false } -variable "ami-owner-id" { - type = string - default = "" -} - packer { required_plugins { amazon = { @@ -67,7 +62,7 @@ source "amazon-ebs" "ubuntu" { virtualization-type = "hvm" } most_recent = true - owners = ["${var.ami-owner-id}"] + owners = ["amazon", "self"] } ssh_username = "ubuntu" ena_support = true From 778ecf6fae093325e1df7affdc5843d30c35d9b2 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 11:54:44 -0400 Subject: [PATCH 027/115] chore: attempt to trigger ami build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index ec57e481c..8d11ca046 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.1-nix-staged" +postgres-version = "15.6.1.2-nix-staged" From 0fbfb0c93880ee8117d95a11b005878187d720a2 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 12:27:33 -0400 Subject: [PATCH 028/115] fix: don't pass in ami owner on packer build --- .github/workflows/ami-release-nix.yml | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 9d306f347..2065c2a18 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -99,7 +99,7 @@ jobs: run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "ami-owner-id=${OWNER}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 8d11ca046..6673e77ce 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.2-nix-staged" +postgres-version = "15.6.1.3-nix-staged" From b29029e4f3f99d4cc5f4774b3c0c6a1808deb871 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 15:24:26 -0400 Subject: [PATCH 029/115] fix: creator tags for packer --- .../scripts/surrogate-bootstrap-nix.sh | 2 +- scripts/nix-provision.sh | 2 +- stage2-nix-psql.pkr.hcl | 20 ++++++++++++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 8b7d6c238..0daa6e517 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -214,7 +214,7 @@ EOF # Run Ansible playbook #export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp - ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible-nix/playbook.yml -vvv $ARGS + ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible-nix/playbook.yml $ARGS } function update_systemd_services { diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index cac4f1451..1b7804e78 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -33,7 +33,7 @@ EOF # Run Ansible playbook #export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/tmp export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp - ansible-playbook /tmp/ansible-playbook/stage2/playbook.yml -vvv $ARGS + ansible-playbook /tmp/ansible-playbook/stage2/playbook.yml $ARGS } diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 417903b94..47f8f22a1 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -66,7 +66,25 @@ source "amazon-ebs" "ubuntu" { } ssh_username = "ubuntu" ena_support = true - + run_tags = { + creator = "packer" + appType = "postgres" + packerExecutionId = "${var.packer-execution-id}" + } + run_volume_tags = { + creator = "packer" + appType = "postgres" + } + snapshot_tags = { + creator = "packer" + appType = "postgres" + } + tags = { + creator = "packer" + appType = "postgres" + postgresVersion = "${var.postgres-version}" + sourceSha = "${var.git-head-version}" + } } build { From 2db86f7fe1bfcf6acf6b39633f4f955347909fa9 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 15:28:17 -0400 Subject: [PATCH 030/115] chore: bump version to trigger build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 6673e77ce..562e11004 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.3-nix-staged" +postgres-version = "15.6.1.4-nix-staged" From d6eaed017d6f80fd18fd894edfa879ac0da0beac Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 16:09:07 -0400 Subject: [PATCH 031/115] chore: bump to trigger --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 562e11004..f6a9be4f9 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.4-nix-staged" +postgres-version = "15.6.1.5-nix-staged" From 1f5c12ef7146ec7b5370a8cdb9391118ba92afe7 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 16:38:36 -0400 Subject: [PATCH 032/115] fix: fix ssh timeout on ami build --- common-nix.vars.pkr.hcl | 2 +- stage2-nix-psql.pkr.hcl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index f6a9be4f9..b163e4025 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.5-nix-staged" +postgres-version = "15.6.1.6-nix-staged" diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 47f8f22a1..9914bedfc 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -64,7 +64,11 @@ source "amazon-ebs" "ubuntu" { most_recent = true owners = ["amazon", "self"] } + + communicator = "ssh" + ssh_pty = true ssh_username = "ubuntu" + ssh_timeout = "5m" ena_support = true run_tags = { creator = "packer" From f409b025dedff45a401cae3df251c6dd5477dd4f Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 17:15:24 -0400 Subject: [PATCH 033/115] fix: ssh interface setting --- common-nix.vars.pkr.hcl | 2 +- stage2-nix-psql.pkr.hcl | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index b163e4025..c810c8082 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.6-nix-staged" +postgres-version = "15.6.1.7-nix-staged" diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 9914bedfc..1debb68be 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -69,7 +69,10 @@ source "amazon-ebs" "ubuntu" { ssh_pty = true ssh_username = "ubuntu" ssh_timeout = "5m" + ssh_interface = "public_ip" + ena_support = true + run_tags = { creator = "packer" appType = "postgres" From 2a9474b46b321af3546d5c26148729d02e0a97d0 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 18:05:38 -0400 Subject: [PATCH 034/115] fix: try increased timeout --- common-nix.vars.pkr.hcl | 2 +- stage2-nix-psql.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index c810c8082..5d5fe673e 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.7-nix-staged" +postgres-version = "15.6.1.8-nix-staged" diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 1debb68be..b531d2029 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -68,7 +68,7 @@ source "amazon-ebs" "ubuntu" { communicator = "ssh" ssh_pty = true ssh_username = "ubuntu" - ssh_timeout = "5m" + ssh_timeout = "10m" ssh_interface = "public_ip" ena_support = true From 376f37dbd483015e6f4dd1a6b4b30356ef7d692d Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 19:38:16 -0400 Subject: [PATCH 035/115] fix: revert timeout --- common-nix.vars.pkr.hcl | 2 +- stage2-nix-psql.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 5d5fe673e..57b8107c4 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.8-nix-staged" +postgres-version = "15.6.1.9-nix-staged" diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index b531d2029..1debb68be 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -68,7 +68,7 @@ source "amazon-ebs" "ubuntu" { communicator = "ssh" ssh_pty = true ssh_username = "ubuntu" - ssh_timeout = "10m" + ssh_timeout = "5m" ssh_interface = "public_ip" ena_support = true From 4747932c5ee52d741c7ed1dfd71ce5b9b2446d91 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 19:41:18 -0400 Subject: [PATCH 036/115] fix: try associate_public_ip_address = true --- common-nix.vars.pkr.hcl | 2 +- stage2-nix-psql.pkr.hcl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 57b8107c4..5f46554e8 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.9-nix-staged" +postgres-version = "15.6.1.10-nix-staged" diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 1debb68be..2c12ddd8b 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -69,7 +69,9 @@ source "amazon-ebs" "ubuntu" { ssh_pty = true ssh_username = "ubuntu" ssh_timeout = "5m" - ssh_interface = "public_ip" + + associate_public_ip_address = true + ena_support = true From 8706eb0fd21cfbd8142a3f345ac54c54b4f02052 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 20:18:41 -0400 Subject: [PATCH 037/115] chore: bump version to re-run build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 5f46554e8..4c4cf60a3 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.10-nix-staged" +postgres-version = "15.6.1.11-nix-staged" From 81cd22155cf1be1816985b87ab38a40525c31067 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 22:47:03 -0400 Subject: [PATCH 038/115] chore: advance version to trigger image build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 4c4cf60a3..25a1fa2df 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.11-nix-staged" +postgres-version = "15.6.1.12-nix-staged" From 7193125f12a86619cc950d27ddd135d10b5ecfc0 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 23:06:08 -0400 Subject: [PATCH 039/115] chore: bump version for rebuild attempt --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 25a1fa2df..aea608917 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.12-nix-staged" +postgres-version = "15.6.1.13-nix-staged" From 0ccedc298fa52dfa017e7fa9b89a00b8d50f04e3 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 6 May 2024 23:58:56 -0400 Subject: [PATCH 040/115] chore: bump version to try to run build in ci --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index aea608917..97ea4007e 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.13-nix-staged" +postgres-version = "15.6.1.14-nix-staged" From 046474f3dbec8716d102bd5d20458b57b202129c Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 7 May 2024 00:24:43 -0400 Subject: [PATCH 041/115] fix: stage 1 needs a distinct tag --- amazon-arm64-nix.pkr.hcl | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index 2b1f0eb66..d75557acf 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -164,7 +164,7 @@ source "amazon-ebssurrogate" "source" { tags = { creator = "packer" appType = "postgres" - postgresVersion = "${var.postgres-version}" + postgresVersion = "${var.postgres-version}-stage1" sourceSha = "${var.git-head-version}" } diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 97ea4007e..16fa93696 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.14-nix-staged" +postgres-version = "15.6.1.15-nix-staged" From fbe0983c1ab2ac6a0ffd5052f2a208947a549a86 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 7 May 2024 11:18:53 -0400 Subject: [PATCH 042/115] chore: bump version to trigger re-build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 16fa93696..423f90078 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.15-nix-staged" +postgres-version = "15.6.1.16-nix-staged" From a528f704dcd958eeffc0e41ae6e5de1a99f55cde Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 13 May 2024 12:04:28 -0400 Subject: [PATCH 043/115] fix: dealing with locales in stages --- .../postgresql_config/postgresql.service.j2 | 1 + .../tasks/stage2/stage2-setup-postgres.yml | 45 +++++++++++++++-- ansible-nix/tasks/test-image.yml | 50 ------------------- ebssurrogate/scripts/chroot-bootstrap-nix.sh | 17 ++++++- 4 files changed, 58 insertions(+), 55 deletions(-) delete mode 100644 ansible-nix/tasks/test-image.yml diff --git a/ansible-nix/files/postgresql_config/postgresql.service.j2 b/ansible-nix/files/postgresql_config/postgresql.service.j2 index ac44e0a45..41a0d0d80 100644 --- a/ansible-nix/files/postgresql_config/postgresql.service.j2 +++ b/ansible-nix/files/postgresql_config/postgresql.service.j2 @@ -18,6 +18,7 @@ TimeoutStartSec=86400 Restart=always RestartSec=5 OOMScoreAdjust=-1000 +EnvironmentFile=-/etc/environment.d/postgresql.env [Install] WantedBy=multi-user.target diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index fa45f6ea3..1b8b0fe11 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -39,6 +39,45 @@ #TODO include ls $(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale-archive +- name: Set permissions for postgresql.env + become: yes + file: + path: /etc/environment.d/postgresql.env + owner: postgres + group: postgres + mode: '0644' + +- name: Set ownership of /etc/ssl/private directory + become: yes + file: + path: /etc/ssl/private + owner: root + group: ssl-cert + state: directory + +- name: Set permissions for /etc/ssl/private directory + become: yes + file: + path: /etc/ssl/private + mode: '0750' + state: directory + + +- name: Set ownership of private key file + become: yes + file: + path: /etc/ssl/private/server.key + owner: root + group: ssl-cert + state: file + +- name: Set permissions for private key file + become: yes + file: + path: /etc/ssl/private/server.key + mode: '0640' + state: file + - name: Ensure /usr/lib/postgresql/bin directory exists file: path: /usr/lib/postgresql/bin @@ -182,12 +221,10 @@ LC_ALL: en_US.UTF-8 LC_CTYPE: en_US.UTF-8 LOCALE_ARCHIVE: /usr/lib/locale/locale-archive - # LOCALE_ARCHIVE: /usr/lib/locale/locale-archive - # LOCALE_ARCHIVE: /usr/lib/locale/locale-archive vars: ansible_command_timeout: 60 -# # Circumvents the following error: -# # "Timeout (12s) waiting for privilege escalation prompt" + # Circumvents the following error: + # "Timeout (12s) waiting for privilege escalation prompt" - name: copy PG systemd unit template: diff --git a/ansible-nix/tasks/test-image.yml b/ansible-nix/tasks/test-image.yml deleted file mode 100644 index cad503a39..000000000 --- a/ansible-nix/tasks/test-image.yml +++ /dev/null @@ -1,50 +0,0 @@ -- name: install pg_prove - apt: - pkg: - - libtap-parser-sourcehandler-pgtap-perl - -- name: Temporarily disable PG Sodium references in config - become: yes - become_user: postgres - shell: - cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf - -- name: Start Postgres Database to load all extensions. - become: yes - become_user: postgres - shell: - cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data start "-o -c config_file=/etc/postgresql/postgresql.conf" - -- name: Run Unit tests (with filename unit-test-*) on Postgres Database - shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql - register: retval - failed_when: retval.rc != 0 - -- name: Run migrations tests - shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql - register: retval - failed_when: retval.rc != 0 - args: - chdir: /tmp/migrations - -- name: Re-enable PG Sodium references in config - become: yes - become_user: postgres - shell: - cmd: mv /etc/postgresql/postgresql.conf.bak /etc/postgresql/postgresql.conf - -- name: Reset db stats - shell: /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();' - -- name: remove pg_prove - apt: - pkg: - - libtap-parser-sourcehandler-pgtap-perl - state: absent - autoremove: yes - -- name: Stop Postgres Database - become: yes - become_user: postgres - shell: - cmd: /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data stop diff --git a/ebssurrogate/scripts/chroot-bootstrap-nix.sh b/ebssurrogate/scripts/chroot-bootstrap-nix.sh index 4b7ceaf8d..cda6bd2aa 100755 --- a/ebssurrogate/scripts/chroot-bootstrap-nix.sh +++ b/ebssurrogate/scripts/chroot-bootstrap-nix.sh @@ -86,7 +86,21 @@ cat << EOF > /etc/default/locale LANG="C.UTF-8" LC_CTYPE="C.UTF-8" EOF - localedef -i en_US -f UTF-8 en_US.UTF-8 + locale-gen en_US.UTF-8 +} + +function setup_postgesql_env { + # Create the directory if it doesn't exist + sudo mkdir -p /etc/environment.d + + # Define the contents of the PostgreSQL environment file + cat </dev/null +LOCALE_ARCHIVE=/usr/lib/locale/locale-archive +LANG="en_US.UTF-8" +LANGUAGE="en_US.UTF-8" +LC_ALL="en_US.UTF-8" +LC_CTYPE="en_US.UTF-8" +EOF } function install_packages_for_build { @@ -191,6 +205,7 @@ function cleanup_cache { update_install_packages setup_locale +setup_postgesql_env #install_packages_for_build install_configure_grub setup_apparmor From 5cabc641dcd9b5bffc1f670edf6acf85a3c3a104 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 13 May 2024 12:05:08 -0400 Subject: [PATCH 044/115] chore: bump version to trigger the build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 423f90078..a3121cc5d 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.16-nix-staged" +postgres-version = "15.6.1.17-nix-staged" From 5cf08dde9826126209e9ce210576a00ce362c265 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 13 May 2024 12:56:04 -0400 Subject: [PATCH 045/115] fix: rm server.key settings --- .../tasks/stage2/stage2-setup-postgres.yml | 31 ------------------- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index 1b8b0fe11..dcbdd8080 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -47,37 +47,6 @@ group: postgres mode: '0644' -- name: Set ownership of /etc/ssl/private directory - become: yes - file: - path: /etc/ssl/private - owner: root - group: ssl-cert - state: directory - -- name: Set permissions for /etc/ssl/private directory - become: yes - file: - path: /etc/ssl/private - mode: '0750' - state: directory - - -- name: Set ownership of private key file - become: yes - file: - path: /etc/ssl/private/server.key - owner: root - group: ssl-cert - state: file - -- name: Set permissions for private key file - become: yes - file: - path: /etc/ssl/private/server.key - mode: '0640' - state: file - - name: Ensure /usr/lib/postgresql/bin directory exists file: path: /usr/lib/postgresql/bin diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index a3121cc5d..316a6424c 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.17-nix-staged" +postgres-version = "15.6.1.18-nix-staged" From 47f27037fa2b6f35c6b549ab181ed49b9cc98045 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 13 May 2024 14:32:34 -0400 Subject: [PATCH 046/115] fix: correct perms on the parnet directory for this key file --- ansible-nix/tasks/stage2/stage2-setup-postgres.yml | 6 ++++++ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index dcbdd8080..b1e7ea932 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -39,6 +39,12 @@ #TODO include ls $(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale-archive +- name: Set permissions to 750 for /etc/ssl/private + become: yes + file: + path: /etc/ssl/private + mode: '0750' + - name: Set permissions for postgresql.env become: yes file: diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 316a6424c..e301619c9 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.18-nix-staged" +postgres-version = "15.6.1.19-nix-staged" From 251ec25dac154277b196f5bdb19066604a15e3ce Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 13 May 2024 14:44:47 -0400 Subject: [PATCH 047/115] fix: also need perms on the parent dir for the key file --- ansible-nix/tasks/stage2/stage2-setup-postgres.yml | 4 +++- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index b1e7ea932..db422dc52 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -39,10 +39,12 @@ #TODO include ls $(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale-archive -- name: Set permissions to 750 for /etc/ssl/private +- name: Set ownership and permissions for /etc/ssl/private become: yes file: path: /etc/ssl/private + owner: root + group: postgres mode: '0750' - name: Set permissions for postgresql.env diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index e301619c9..c9ef2ac36 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.19-nix-staged" +postgres-version = "15.6.1.20-nix-staged" From 6aebe6dd5cb6ad4873e2ac9ea45fa1ff3400095e Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 14 May 2024 10:58:38 -0400 Subject: [PATCH 048/115] fix: sorting out the linux groups that the postgres user belongs to --- ansible-nix/tasks/setup-postgres.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 233e7c798..6fd796f99 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -18,8 +18,10 @@ - name: create users user: name: postgres - groups: postgres + groups: postgres, ssl-cert createhome: yes + append: yes + remove: users shell: /bin/bash # Set shell if needed, default is /bin/bash password: '' From 2fb3426a5549eb409525e31a55905d05d48ba827 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 14 May 2024 10:59:36 -0400 Subject: [PATCH 049/115] chore: bump version for build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index c9ef2ac36..490a7b36f 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.20-nix-staged" +postgres-version = "15.6.1.21-nix-staged" From caf2f1482cee478d32cb0ec499b6e31a4608fa54 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 14 May 2024 11:09:25 -0400 Subject: [PATCH 050/115] fix: jsut append: no to constrain groups --- ansible-nix/tasks/setup-postgres.yml | 3 +-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 6fd796f99..be8c84562 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -20,8 +20,7 @@ name: postgres groups: postgres, ssl-cert createhome: yes - append: yes - remove: users + append: no shell: /bin/bash # Set shell if needed, default is /bin/bash password: '' diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 490a7b36f..96cf147c9 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.21-nix-staged" +postgres-version = "15.6.1.22-nix-staged" From 741fde92bf33ff2ab400ff88c8e671add3e663ef Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 14 May 2024 11:31:35 -0400 Subject: [PATCH 051/115] get the group creation into the right place --- ansible-nix/tasks/setup-pgbouncer.yml | 5 ----- ansible-nix/tasks/setup-postgres.yml | 8 ++++++++ common-nix.vars.pkr.hcl | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ansible-nix/tasks/setup-pgbouncer.yml b/ansible-nix/tasks/setup-pgbouncer.yml index eee208961..4381ba24d 100644 --- a/ansible-nix/tasks/setup-pgbouncer.yml +++ b/ansible-nix/tasks/setup-pgbouncer.yml @@ -1,9 +1,4 @@ # PgBouncer -- name: create ssl-cert group - group: - name: ssl-cert - state: present - - name: PgBouncer - download & install dependencies apt: pkg: diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index be8c84562..dea88daa6 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -10,6 +10,14 @@ # TODO These lines appear to be installing and configuring https://launchpad.net/ubuntu/+source/postgresql-common # as far as I can see, we don't need this now. +- name: create ssl-cert group + group: + name: ssl-cert + state: present + +# the old method of installing from debian creates this group, but we must create it explicitly +# for the nix built version + - name: create postgres group group: name: postgres diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 96cf147c9..c6b3c31b0 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.22-nix-staged" +postgres-version = "15.6.1.23-nix-staged" From 7dedb19b65ce96c9c35e3df9c436714bfa5745ed Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 14 May 2024 17:12:48 -0400 Subject: [PATCH 052/115] fix: some programs are looking for items in /usr/bin --- ansible-nix/tasks/stage2/stage2-setup-postgres.yml | 8 ++++++++ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index db422dc52..dc8b3c79c 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -115,6 +115,14 @@ - "/home/postgres/.nix-profile/bin/*" become: yes +- name: Create symbolic links from /home/postgres/.nix-profile/bin to /usr/bin + file: + src: "{{ item }}" + dest: "/usr/bin/{{ item | basename }}" + state: link + with_fileglob: + - "/home/postgres/.nix-profile/bin/*" + become: yes - name: Create symbolic links from /home/postgres/.nix-profile/share/postgresql to /usr/lib/postgresql/share/postgresql file: diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index c6b3c31b0..8d6bd4284 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.23-nix-staged" +postgres-version = "15.6.1.24-nix-staged" From 8d218189eaee8148867825def9e00f4af300d943 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 14 May 2024 17:47:29 -0400 Subject: [PATCH 053/115] fix: just link psql --- ansible-nix/tasks/stage2/stage2-setup-postgres.yml | 6 ++---- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index dc8b3c79c..1180ddd4f 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -117,11 +117,9 @@ - name: Create symbolic links from /home/postgres/.nix-profile/bin to /usr/bin file: - src: "{{ item }}" - dest: "/usr/bin/{{ item | basename }}" + src: "/home/postgres/.nix-profile/bin/psql" + dest: "/usr/bin/psql" state: link - with_fileglob: - - "/home/postgres/.nix-profile/bin/*" become: yes - name: Create symbolic links from /home/postgres/.nix-profile/share/postgresql to /usr/lib/postgresql/share/postgresql diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 8d6bd4284..604654ef1 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.24-nix-staged" +postgres-version = "15.6.1.25-nix-staged" From b6c40f4822068ddb073caeb7d25c1171ca75ec7e Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 14 May 2024 23:19:40 -0400 Subject: [PATCH 054/115] feat: make sure postgres user not part of "users" group --- ansible-nix/tasks/setup-postgres.yml | 5 +++++ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index dea88daa6..af8a86053 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -32,6 +32,11 @@ shell: /bin/bash # Set shell if needed, default is /bin/bash password: '' +- name: Remove postgres user from users group + become: yes + command: gpasswd -d postgres users + + - name: Create relevant directories file: path: '{{ item }}' diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 604654ef1..5a9508cc8 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.25-nix-staged" +postgres-version = "15.6.1.26-nix-staged" From 6996c544316b6e43fe97d8d660e934cf6032a5c6 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 14 May 2024 23:43:32 -0400 Subject: [PATCH 055/115] fix: trying in second stage --- ansible-nix/tasks/stage2/stage2-setup-postgres.yml | 5 +++++ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index 1180ddd4f..96697f4f3 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -8,6 +8,11 @@ shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/sam/2-stage-ami-nix#pg_prove" +- name: Remove postgres user from users group + become: yes + command: gpasswd -d postgres users + + #TODO switch pg_prove sourcing to develop branch once PR is merged # - name: Generate en_US.UTF-8 locale diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 5a9508cc8..8220d2301 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.26-nix-staged" +postgres-version = "15.6.1.27-nix-staged" From fe206a3e64c6dca6bacb2236033e5c1bb4479528 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 14 May 2024 23:57:23 -0400 Subject: [PATCH 056/115] fix: move command to the right stage --- ansible-nix/tasks/setup-postgres.yml | 4 ---- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index af8a86053..006658dd3 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -32,10 +32,6 @@ shell: /bin/bash # Set shell if needed, default is /bin/bash password: '' -- name: Remove postgres user from users group - become: yes - command: gpasswd -d postgres users - - name: Create relevant directories file: diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 8220d2301..e2614ff37 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.27-nix-staged" +postgres-version = "15.6.1.28-nix-staged" From 4678636bba7b0ba1d1019e98c4e85008854e29f5 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 00:35:23 -0400 Subject: [PATCH 057/115] fix: try to remove group at the end of build --- ansible-nix/playbook.yml | 3 +++ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible-nix/playbook.yml b/ansible-nix/playbook.yml index e23f9652f..51a20cb0d 100644 --- a/ansible-nix/playbook.yml +++ b/ansible-nix/playbook.yml @@ -105,3 +105,6 @@ - name: Clean out build dependencies import_tasks: tasks/clean-build-dependencies.yml + - name: Remove postgres user from users group + become: yes + command: gpasswd -d postgres users diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index e2614ff37..9973b987d 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.28-nix-staged" +postgres-version = "15.6.1.29-nix-staged" From d6338b922af3e05d1d64b3b33bc93e0468c1a005 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 07:14:56 -0400 Subject: [PATCH 058/115] chore: check if postgres user member of "users" group --- ansible-nix/playbook.yml | 11 +++++++++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ansible-nix/playbook.yml b/ansible-nix/playbook.yml index 51a20cb0d..059c3cdc6 100644 --- a/ansible-nix/playbook.yml +++ b/ansible-nix/playbook.yml @@ -105,6 +105,13 @@ - name: Clean out build dependencies import_tasks: tasks/clean-build-dependencies.yml - - name: Remove postgres user from users group + + - name: Check if postgres user is part of users group + command: "id -nG postgres | grep -qw users" + register: check_user_group + ignore_errors: yes become: yes - command: gpasswd -d postgres users + + - name: Print result to Ansible log output + debug: + msg: "The postgres user is {{ 'not ' if check_user_group.rc != 0 else '' }}part of the users group" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 9973b987d..87527df72 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.29-nix-staged" +postgres-version = "15.6.1.30-nix-staged" From ec27e73fc4ad184b1662060733f2a37ce210ec6f Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 07:15:58 -0400 Subject: [PATCH 059/115] chore: bump version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 87527df72..7df19d654 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.30-nix-staged" +postgres-version = "15.6.1.31-nix-staged" From 73fa7f53c0820bcd34c96da645e85843e11b7f95 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 09:57:06 -0400 Subject: [PATCH 060/115] fix: do not try to remove yet, just check if postgres user is in users group --- ansible-nix/tasks/stage2/stage2-setup-postgres.yml | 4 ---- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index 96697f4f3..8c490add2 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -8,10 +8,6 @@ shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/sam/2-stage-ami-nix#pg_prove" -- name: Remove postgres user from users group - become: yes - command: gpasswd -d postgres users - #TODO switch pg_prove sourcing to develop branch once PR is merged diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 7df19d654..79e40d173 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.31-nix-staged" +postgres-version = "15.6.1.32-nix-staged" From a22335b0160ceb834826d00b3bf060f5a5f74a72 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 14:00:17 -0400 Subject: [PATCH 061/115] chore: try to see what group the user is part of at end of build --- ansible-nix/playbook.yml | 4 +++- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ansible-nix/playbook.yml b/ansible-nix/playbook.yml index 059c3cdc6..1234aaeb1 100644 --- a/ansible-nix/playbook.yml +++ b/ansible-nix/playbook.yml @@ -107,10 +107,12 @@ import_tasks: tasks/clean-build-dependencies.yml - name: Check if postgres user is part of users group - command: "id -nG postgres | grep -qw users" + shell: "id -nG postgres | grep -qw users" register: check_user_group ignore_errors: yes become: yes + args: + executable: /bin/bash - name: Print result to Ansible log output debug: diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 79e40d173..f137ba244 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.32-nix-staged" +postgres-version = "15.6.1.33-nix-staged" From 2cfe3941fa3ad50a0b0e5c7126171bdc416657db Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 14:30:53 -0400 Subject: [PATCH 062/115] chore: run user check on both stages --- ansible-nix/tasks/stage2/playbook.yml | 12 ++++++++++++ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ansible-nix/tasks/stage2/playbook.yml b/ansible-nix/tasks/stage2/playbook.yml index fb53402dc..332613e32 100644 --- a/ansible-nix/tasks/stage2/playbook.yml +++ b/ansible-nix/tasks/stage2/playbook.yml @@ -50,3 +50,15 @@ import_tasks: test-image.yml tags: - unit-tests + + - name: Check if postgres user is part of users group + shell: "id -nG postgres | grep -qw users" + register: check_user_group + ignore_errors: yes + become: yes + args: + executable: /bin/bash + + - name: Print result to Ansible log output + debug: + msg: "The postgres user is {{ 'not ' if check_user_group.rc != 0 else '' }}part of the users group" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index f137ba244..920bafb78 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.33-nix-staged" +postgres-version = "15.6.1.34-nix-staged" From ed0464f274e5e80065fb42b73760aa3d155917d3 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 15:31:04 -0400 Subject: [PATCH 063/115] feat: attempt to use command directly to create user and group --- ansible-nix/tasks/setup-postgres.yml | 35 ++++++++++++++++------------ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 006658dd3..87156f45f 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -18,21 +18,26 @@ # the old method of installing from debian creates this group, but we must create it explicitly # for the nix built version -- name: create postgres group - group: - name: postgres - state: present - -- name: create users - user: - name: postgres - groups: postgres, ssl-cert - createhome: yes - append: no - shell: /bin/bash # Set shell if needed, default is /bin/bash - password: '' - - +# - name: create postgres group +# group: +# name: postgres +# state: present + +# - name: create users +# user: +# name: postgres +# groups: postgres, ssl-cert +# createhome: yes +# append: no +# shell: /bin/bash # Set shell if needed, default is /bin/bash +# password: '' + +- name: create postgres user + shell: useradd -m -r -s /bin/bash postgres + args: + executable: /bin/bash + become: yes + - name: Create relevant directories file: path: '{{ item }}' diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 920bafb78..f1e60fb8f 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.34-nix-staged" +postgres-version = "15.6.1.35-nix-staged" From 774431512ce0484c0a697e5ea8e507061d498456 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 15:38:14 -0400 Subject: [PATCH 064/115] fix: user add command route --- ansible-nix/tasks/setup-postgres.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 87156f45f..8bc8057fa 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -18,10 +18,10 @@ # the old method of installing from debian creates this group, but we must create it explicitly # for the nix built version -# - name: create postgres group -# group: -# name: postgres -# state: present +- name: create postgres group + group: + name: postgres + state: present # - name: create users # user: @@ -33,11 +33,11 @@ # password: '' - name: create postgres user - shell: useradd -m -r -s /bin/bash postgres + shell: useradd -m -r -s /bin/bash -d /home/postgres postgres -G postgres,ssl-cert args: executable: /bin/bash become: yes - + - name: Create relevant directories file: path: '{{ item }}' From ff83fbac84a4bbc76f27dad6a0b1a93f400678a2 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 15:38:42 -0400 Subject: [PATCH 065/115] chore: bump version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index f1e60fb8f..843a5be67 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.35-nix-staged" +postgres-version = "15.6.1.36-nix-staged" From 26758c1467bdc42bd325b066fb28b0560f3c0864 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 15:52:35 -0400 Subject: [PATCH 066/115] fix: use correct flag on group add --- ansible-nix/tasks/setup-postgres.yml | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 8bc8057fa..785ae9117 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -33,7 +33,7 @@ # password: '' - name: create postgres user - shell: useradd -m -r -s /bin/bash -d /home/postgres postgres -G postgres,ssl-cert + shell: useradd -m -r -s /bin/bash -d /home/postgres postgres -g postgres,ssl-cert args: executable: /bin/bash become: yes diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 843a5be67..ec1eaca00 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.36-nix-staged" +postgres-version = "15.6.1.37-nix-staged" From 5df9ec8676c29c0c803545afd58604af54646642 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 16:02:52 -0400 Subject: [PATCH 067/115] chore: command formatting --- ansible-nix/tasks/setup-postgres.yml | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 785ae9117..589766950 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -33,7 +33,7 @@ # password: '' - name: create postgres user - shell: useradd -m -r -s /bin/bash -d /home/postgres postgres -g postgres,ssl-cert + shell: useradd -m -r -s /bin/bash -d /home/postgres postgres -g postgres -g ssl-cert args: executable: /bin/bash become: yes diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index ec1eaca00..1bb229c02 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.37-nix-staged" +postgres-version = "15.6.1.38-nix-staged" From 80af04e8638b3b18776a0fe32c8029545dac3269 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 17:15:58 -0400 Subject: [PATCH 068/115] fix: explicit group add for each group after user created --- ansible-nix/tasks/setup-postgres.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 589766950..6ea1c5766 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -33,11 +33,24 @@ # password: '' - name: create postgres user - shell: useradd -m -r -s /bin/bash -d /home/postgres postgres -g postgres -g ssl-cert + shell: useradd -m -r -s /bin/bash -d /home/postgres postgres args: executable: /bin/bash become: yes +- name: add postgres user to postgres group + shell: usermod -aG postgres postgres + args: + executable: /bin/bash + become: yes + +- name: add postgres user to ssl-cert group + shell: usermod -aG ssl-cert postgres + args: + executable: /bin/bash + become: yes + + - name: Create relevant directories file: path: '{{ item }}' From 4067a8217fee80451f4ea1fe702d57449209ca2a Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 17:19:18 -0400 Subject: [PATCH 069/115] chore: trigger build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 1bb229c02..f55d7e750 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.38-nix-staged" +postgres-version = "15.6.1.39-nix-staged" From 8ec33d5b770ea43e1fbaae01ebe5d4c7acd228f1 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 17:41:35 -0400 Subject: [PATCH 070/115] chore: trigger build --- ansible-nix/tasks/setup-postgres.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 6ea1c5766..95f3375f0 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -39,13 +39,13 @@ become: yes - name: add postgres user to postgres group - shell: usermod -aG postgres postgres + shell: usermod -a -G postgres postgres args: executable: /bin/bash become: yes - name: add postgres user to ssl-cert group - shell: usermod -aG ssl-cert postgres + shell: usermod -a -G ssl-cert postgres args: executable: /bin/bash become: yes diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index f55d7e750..5dae22a67 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.39-nix-staged" +postgres-version = "15.6.1.40-nix-staged" From d1cd8028316dd8ee78d5991d33c6bbecb7446942 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 18:50:55 -0400 Subject: [PATCH 071/115] fix: use this approach to prevent failure --- ansible-nix/tasks/setup-postgres.yml | 11 ++--------- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 95f3375f0..702082177 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -33,24 +33,17 @@ # password: '' - name: create postgres user - shell: useradd -m -r -s /bin/bash -d /home/postgres postgres + shell: useradd -m -r -s /bin/bash -d /home/postgres postgres -g postgres args: executable: /bin/bash become: yes - name: add postgres user to postgres group - shell: usermod -a -G postgres postgres + shell: usermod -a -g ssl-cert postgres args: executable: /bin/bash become: yes -- name: add postgres user to ssl-cert group - shell: usermod -a -G ssl-cert postgres - args: - executable: /bin/bash - become: yes - - - name: Create relevant directories file: path: '{{ item }}' diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 5dae22a67..9ddcaf6ad 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.40-nix-staged" +postgres-version = "15.6.1.41-nix-staged" From 4ff8846b36795346403e5dd4f5f8a0fb770b98d8 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 19:01:22 -0400 Subject: [PATCH 072/115] fix: correct flag --- ansible-nix/tasks/setup-postgres.yml | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 702082177..7cf9852bd 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -39,7 +39,7 @@ become: yes - name: add postgres user to postgres group - shell: usermod -a -g ssl-cert postgres + shell: usermod -a -G ssl-cert postgres args: executable: /bin/bash become: yes diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 9ddcaf6ad..3d71ef0ea 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.41-nix-staged" +postgres-version = "15.6.1.42-nix-staged" From 77bb52c9d40ce2b8ae64dfb19c11cd521857710b Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 15 May 2024 20:24:51 -0400 Subject: [PATCH 073/115] chore: trigger build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 3d71ef0ea..e8829b0d4 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.42-nix-staged" +postgres-version = "15.6.1.43-nix-staged" From 76a6a0177702ca0eb31365dbf0e177886fd27844 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 17 May 2024 10:40:53 -0400 Subject: [PATCH 074/115] fix: update vars to keep in sunc with dev branch --- ansible-nix/vars.yml | 12 ++++++------ ansible/vars.yml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible-nix/vars.yml b/ansible-nix/vars.yml index dcb11dbab..33526ea26 100644 --- a/ansible-nix/vars.yml +++ b/ansible-nix/vars.yml @@ -15,8 +15,8 @@ postgrest_release: "12.0.2" postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 -gotrue_release: 2.148.0 -gotrue_release_checksum: sha1:e3ab39c10605b6a00a953528022b563c8fa1ca33 +gotrue_release: 2.151.0 +gotrue_release_checksum: sha1:5a43a9879499d85714ba34356f62f5e8063b549d aws_cli_release: "2.2.7" @@ -129,14 +129,14 @@ pgroonga_release_checksum: sha256:885ff3878cc30e9030e5fc56d561bc8b66df3ede1562c9 wrappers_release: "0.3.0" -hypopg_release: "1.3.1" -hypopg_release_checksum: sha256:e7f01ee0259dc1713f318a108f987663d60f3041948c2ada57a94b469565ca8e +hypopg_release: "1.4.1" +hypopg_release_checksum: sha256:9afe6357fd389d8d33fad81703038ce520b09275ec00153c6c89282bcdedd6bc pg_repack_release: "1.5.0" pg_repack_release_checksum: sha256:9a14d6a95bfa29f856aa10538238622c1f351d38eb350b196c06720a878ccc52 -pgvector_release: "0.6.2" -pgvector_release_checksum: sha256:a11cc249a9f3f3d7b13069a1696f2915ac28991a72d7ba4e2bcfdceddbaeae49 +pgvector_release: "0.7.0" +pgvector_release_checksum: sha256:1b5503a35c265408b6eb282621c5e1e75f7801afc04eecb950796cfee2e3d1d8 pg_tle_release: "1.3.2" pg_tle_release_checksum: sha256:d04f72d88b21b954656609743560684ac42645b64a36c800d4d2f84d1f180de1 diff --git a/ansible/vars.yml b/ansible/vars.yml index d499b4fd2..33526ea26 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -15,8 +15,8 @@ postgrest_release: "12.0.2" postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 -gotrue_release: 2.150.0 -gotrue_release_checksum: sha1:d785c1a4aadcee88738c3e016ac12f9a7c3db858 +gotrue_release: 2.151.0 +gotrue_release_checksum: sha1:5a43a9879499d85714ba34356f62f5e8063b549d aws_cli_release: "2.2.7" From f2c5adb1f590bf71d3903be2a80cfd6c52b1506d Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 20 May 2024 20:29:13 -0400 Subject: [PATCH 075/115] chore: tmp install osquery --- ansible-nix/tasks/stage2/stage2-setup-postgres.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index 8c490add2..e3d9bbe23 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -8,6 +8,10 @@ shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/sam/2-stage-ami-nix#pg_prove" +- name: Install pg_prove from nix binary cache + become: yes + shell: | + sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#osquery" #TODO switch pg_prove sourcing to develop branch once PR is merged From 6075aca17e47fb4ce9ee55df20c6c38971de2ba7 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 20 May 2024 22:25:50 -0400 Subject: [PATCH 076/115] chore: bump to trigger build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index e8829b0d4..2b2b2ff28 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.43-nix-staged" +postgres-version = "15.6.1.44-nix-staged" From 53953ce4dd8b956d68c784e4b7286202ebb993ff Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 20 May 2024 22:32:54 -0400 Subject: [PATCH 077/115] chore: bump to trigger build --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 2b2b2ff28..0ccad3c02 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.44-nix-staged" +postgres-version = "15.6.1.45-nix-staged" From 9bd0f534d759587be4e0dd03eab37dd3454a40b8 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 21 May 2024 10:30:45 -0400 Subject: [PATCH 078/115] feat: tmp install osquery, check all permissions, remove script and osquery --- ansible-nix/files/permission_check.py | 183 ++++++++++++++++++ ansible-nix/playbook.yml | 28 +++ .../tasks/stage2/stage2-setup-postgres.yml | 5 - common-nix.vars.pkr.hcl | 2 +- 4 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 ansible-nix/files/permission_check.py diff --git a/ansible-nix/files/permission_check.py b/ansible-nix/files/permission_check.py new file mode 100644 index 000000000..a7175b63c --- /dev/null +++ b/ansible-nix/files/permission_check.py @@ -0,0 +1,183 @@ +import subprocess +import json +import sys + +# Expected groups for each user +expected_results = { + "postgres": [ + {"groupname": "postgres", "username": "postgres"}, + {"groupname": "ssl-cert", "username": "postgres"} + ], + "ubuntu": [ + {"groupname":"ubuntu","username":"ubuntu"}, + {"groupname":"adm","username":"ubuntu"}, + {"groupname":"dialout","username":"ubuntu"}, + {"groupname":"cdrom","username":"ubuntu"}, + {"groupname":"floppy","username":"ubuntu"}, + {"groupname":"sudo","username":"ubuntu"}, + {"groupname":"audio","username":"ubuntu"}, + {"groupname":"dip","username":"ubuntu"}, + {"groupname":"video","username":"ubuntu"}, + {"groupname":"plugdev","username":"ubuntu"}, + {"groupname":"lxd","username":"ubuntu"}, + {"groupname":"netdev","username":"ubuntu"} + ], + "root": [ + {"groupname":"root","username":"root"} + ], + "daemon": [ + {"groupname":"daemon","username":"daemon"} + ], + "bin": [ + {"groupname":"bin","username":"bin"} + ], + "sys": [ + {"groupname":"sys","username":"sys"} + ], + "sync": [ + {"groupname":"nogroup","username":"sync"} + ], + "games": [ + {"groupname":"games","username":"games"} + ], + "man": [ + {"groupname":"man","username":"man"} + ], + "lp": [ + {"groupname":"lp","username":"lp"} + ], + "mail": [ + {"groupname":"mail","username":"mail"} + ], + "news": [ + {"groupname":"news","username":"news"} + ], + "uucp": [ + {"groupname":"uucp","username":"uucp"} + ], + "proxy": [ + {"groupname":"proxy","username":"proxy"} + ], + "www-data": [ + {"groupname":"www-data","username":"www-data"} + ], + "backup": [ + {"groupname":"backup","username":"backup"} + ], + "list": [ + {"groupname":"list","username":"list"} + ], + "irc": [ + {"groupname":"irc","username":"irc"} + ], + "gnats": [ + {"groupname":"gnats","username":"gnats"} + ], + "nobody": [ + {"groupname":"nogroup","username":"nobody"} + ], + "systemd-network": [ + {"groupname":"systemd-network","username":"systemd-network"} + ], + "systemd-resolve": [ + {"groupname":"systemd-resolve","username":"systemd-resolve"} + ], + "systemd-timesync": [ + {"groupname":"systemd-timesync","username":"systemd-timesync"} + ], + "messagebus": [ + {"groupname":"messagebus","username":"messagebus"} + ], + "ec2-instance-connect": [ + {"groupname":"nogroup","username":"ec2-instance-connect"} + ], + "sshd": [ + {"groupname":"nogroup","username":"sshd"} + ], + "wal-g": [ + {"groupname":"wal-g","username":"wal-g"}, + {"groupname":"postgres","username":"wal-g"} + ], + "pgbouncer": [ + {"groupname":"pgbouncer","username":"pgbouncer"}, + {"groupname":"ssl-cert","username":"pgbouncer"}, + {"groupname":"postgres","username":"pgbouncer"} + ], + "gotrue": [ + {"groupname":"gotrue","username":"gotrue"} + ], + "envoy": [ + {"groupname":"envoy","username":"envoy"} + ], + "kong": [ + {"groupname":"kong","username":"kong"} + ], + "nginx": [ + {"groupname":"nginx","username":"nginx"} + ], + "vector": [ + {"groupname":"vector","username":"vector"}, + {"groupname":"adm","username":"vector"}, + {"groupname":"systemd-journal","username":"vector"}, + {"groupname":"postgres","username":"vector"} + ], + "adminapi": [ + {"groupname":"adminapi","username":"adminapi"}, + {"groupname":"root","username":"adminapi"}, + {"groupname":"systemd-journal","username":"adminapi"}, + {"groupname":"admin","username":"adminapi"}, + {"groupname":"postgres","username":"adminapi"}, + {"groupname":"pgbouncer","username":"adminapi"}, + {"groupname":"wal-g","username":"adminapi"}, + {"groupname":"postgrest","username":"adminapi"}, + {"groupname":"envoy","username":"adminapi"}, + {"groupname":"kong","username":"adminapi"}, + {"groupname":"vector","username":"adminapi"} + ], + "postgrest": [ + {"groupname":"postgrest","username":"postgrest"} + ], + "tcpdump": [ + {"groupname":"tcpdump","username":"tcpdump"} + ], + "systemd-coredump": [ + {"groupname":"systemd-coredump","username":"systemd-coredump"} + ] +} +# This program depends on osquery being installed on the system +# Function to run osquery +def run_osquery(query): + process = subprocess.Popen(['osqueryi', '--json', query], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + return output.decode('utf-8') + +def parse_json(json_str): + try: + return json.loads(json_str) + except json.JSONDecodeError as e: + print("Error decoding JSON:", e) + sys.exit(1) + +def compare_results(username, query_result): + expected_result = expected_results.get(username) + if expected_result is None: + print(f"No expected result defined for user '{username}'") + sys.exit(1) + + if query_result == expected_result: + print(f"The query result for user '{username}' matches the expected result.") + else: + print(f"The query result for user '{username}' does not match the expected result.") + print("Expected:", expected_result) + print("Got:", query_result) + sys.exit(1) + +# Define usernames for which you want to compare results +usernames = ["postgres", "ubuntu", "root", "daemon", "bin", "sys", "sync", "games","man","lp","mail","news","uucp","proxy","www-data","backup","list","irc","gnats","nobody","systemd-network","systemd-resolve","systemd-timesync","messagebus","ec2-instance-connect","sshd","wal-g","pgbouncer","gotrue","envoy","kong","nginx","vector","adminapi","postgrest","tcpdump","systemd-coredump"] + +# Iterate over usernames, run the query, and compare results +for username in usernames: + query = f"SELECT u.username, g.groupname FROM users u JOIN user_groups ug ON u.uid = ug.uid JOIN groups g ON ug.gid = g.gid WHERE u.username = '{username}';" + query_result = run_osquery(query) + parsed_result = parse_json(query_result) + compare_results(username, parsed_result) diff --git a/ansible-nix/playbook.yml b/ansible-nix/playbook.yml index 1234aaeb1..ab3d2b7b8 100644 --- a/ansible-nix/playbook.yml +++ b/ansible-nix/playbook.yml @@ -117,3 +117,31 @@ - name: Print result to Ansible log output debug: msg: "The postgres user is {{ 'not ' if check_user_group.rc != 0 else '' }}part of the users group" + + - name: Install osquery from nixpkgs binary cache + become: yes + shell: | + sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#osquery" + + - name: Install osquery permission check script + become: yes + copy: + src: files/permission_check.py + dest: /home/ubuntu/permission_check.py + mode: "0755" + + - name: Run osquery permission checks + become: yes + shell: | + sudo -u ubuntu bash -c "python3 /home/ubuntu/permission_check.py" + + - name: Remove osquery permission check script + become: yes + file: + path: /home/ubuntu/permission_check.py + state: absent + + - name: Remove osquery + become: yes + shell: | + sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile osquery" diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index e3d9bbe23..1180ddd4f 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -8,11 +8,6 @@ shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/sam/2-stage-ami-nix#pg_prove" -- name: Install pg_prove from nix binary cache - become: yes - shell: | - sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#osquery" - #TODO switch pg_prove sourcing to develop branch once PR is merged # - name: Generate en_US.UTF-8 locale diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 0ccad3c02..f02caa097 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.45-nix-staged" +postgres-version = "15.6.1.46-nix-staged" From e7d6b35042da6029c33cfa3691d8e7b17cb8c8c7 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 21 May 2024 10:57:04 -0400 Subject: [PATCH 079/115] chore: move commands to the proper stage --- ansible-nix/playbook.yml | 27 -------------------------- ansible-nix/tasks/stage2/playbook.yml | 28 +++++++++++++++++++++++++++ common-nix.vars.pkr.hcl | 2 +- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/ansible-nix/playbook.yml b/ansible-nix/playbook.yml index ab3d2b7b8..c1bb3b898 100644 --- a/ansible-nix/playbook.yml +++ b/ansible-nix/playbook.yml @@ -118,30 +118,3 @@ debug: msg: "The postgres user is {{ 'not ' if check_user_group.rc != 0 else '' }}part of the users group" - - name: Install osquery from nixpkgs binary cache - become: yes - shell: | - sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#osquery" - - - name: Install osquery permission check script - become: yes - copy: - src: files/permission_check.py - dest: /home/ubuntu/permission_check.py - mode: "0755" - - - name: Run osquery permission checks - become: yes - shell: | - sudo -u ubuntu bash -c "python3 /home/ubuntu/permission_check.py" - - - name: Remove osquery permission check script - become: yes - file: - path: /home/ubuntu/permission_check.py - state: absent - - - name: Remove osquery - become: yes - shell: | - sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile osquery" diff --git a/ansible-nix/tasks/stage2/playbook.yml b/ansible-nix/tasks/stage2/playbook.yml index 332613e32..82c3e24fc 100644 --- a/ansible-nix/tasks/stage2/playbook.yml +++ b/ansible-nix/tasks/stage2/playbook.yml @@ -62,3 +62,31 @@ - name: Print result to Ansible log output debug: msg: "The postgres user is {{ 'not ' if check_user_group.rc != 0 else '' }}part of the users group" + + - name: Install osquery from nixpkgs binary cache + become: yes + shell: | + sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#osquery" + + - name: Install osquery permission check script + become: yes + copy: + src: files/permission_check.py + dest: /home/ubuntu/permission_check.py + mode: "0755" + + - name: Run osquery permission checks + become: yes + shell: | + sudo -u ubuntu bash -c "python3 /home/ubuntu/permission_check.py" + + - name: Remove osquery permission check script + become: yes + file: + path: /home/ubuntu/permission_check.py + state: absent + + - name: Remove osquery + become: yes + shell: | + sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile osquery" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index f02caa097..969f41574 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.46-nix-staged" +postgres-version = "15.6.1.47-nix-staged" From 7d4dc0b655f87c453c3d35886ff9b7b5b139164e Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 21 May 2024 11:32:53 -0400 Subject: [PATCH 080/115] fix: source file from upload --- ansible-nix/tasks/stage2/playbook.yml | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible-nix/tasks/stage2/playbook.yml b/ansible-nix/tasks/stage2/playbook.yml index 82c3e24fc..4f4e6b5a4 100644 --- a/ansible-nix/tasks/stage2/playbook.yml +++ b/ansible-nix/tasks/stage2/playbook.yml @@ -71,7 +71,7 @@ - name: Install osquery permission check script become: yes copy: - src: files/permission_check.py + src: /tmp/ansible-playbook/files/permission_check.py dest: /home/ubuntu/permission_check.py mode: "0755" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 969f41574..69d735c46 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.47-nix-staged" +postgres-version = "15.6.1.48-nix-staged" From 4b17691ab84971b389fb58b1d7a990bb95f04ffb Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 21 May 2024 12:07:44 -0400 Subject: [PATCH 081/115] fix: try to run from /tmp dir --- ansible-nix/tasks/stage2/playbook.yml | 15 +-------------- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/ansible-nix/tasks/stage2/playbook.yml b/ansible-nix/tasks/stage2/playbook.yml index 4f4e6b5a4..c61389e4c 100644 --- a/ansible-nix/tasks/stage2/playbook.yml +++ b/ansible-nix/tasks/stage2/playbook.yml @@ -68,23 +68,10 @@ shell: | sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#osquery" - - name: Install osquery permission check script - become: yes - copy: - src: /tmp/ansible-playbook/files/permission_check.py - dest: /home/ubuntu/permission_check.py - mode: "0755" - - name: Run osquery permission checks become: yes shell: | - sudo -u ubuntu bash -c "python3 /home/ubuntu/permission_check.py" - - - name: Remove osquery permission check script - become: yes - file: - path: /home/ubuntu/permission_check.py - state: absent + sudo -u ubuntu bash -c "python3 /tmp/ansible-playbook/files/permission_check.py" - name: Remove osquery become: yes diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 69d735c46..d398eca6b 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.48-nix-staged" +postgres-version = "15.6.1.49-nix-staged" From 2fa9435fd23553b6532768e7babad93552d0617f Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 21 May 2024 12:53:19 -0400 Subject: [PATCH 082/115] fix: bring dep into path --- ansible-nix/tasks/stage2/playbook.yml | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible-nix/tasks/stage2/playbook.yml b/ansible-nix/tasks/stage2/playbook.yml index c61389e4c..cdbfc1e2e 100644 --- a/ansible-nix/tasks/stage2/playbook.yml +++ b/ansible-nix/tasks/stage2/playbook.yml @@ -71,7 +71,7 @@ - name: Run osquery permission checks become: yes shell: | - sudo -u ubuntu bash -c "python3 /tmp/ansible-playbook/files/permission_check.py" + sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && /usr/bin/python3 /tmp/ansible-playbook/files/permission_check.py" - name: Remove osquery become: yes diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index d398eca6b..d10ad68ab 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.49-nix-staged" +postgres-version = "15.6.1.50-nix-staged" From b3cd55a56b1c9db363c78bbff05ca19f3b603a38 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 21 May 2024 13:25:23 -0400 Subject: [PATCH 083/115] fix: typo --- ansible-nix/tasks/stage2/playbook.yml | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible-nix/tasks/stage2/playbook.yml b/ansible-nix/tasks/stage2/playbook.yml index cdbfc1e2e..aa27602ec 100644 --- a/ansible-nix/tasks/stage2/playbook.yml +++ b/ansible-nix/tasks/stage2/playbook.yml @@ -76,4 +76,4 @@ - name: Remove osquery become: yes shell: | - sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile osquery" + sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile remove osquery" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index d10ad68ab..4bc17d2ab 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.50-nix-staged" +postgres-version = "15.6.1.51-nix-staged" From b535cf3e191a5467a950eea471d884f7c2afe376 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 21 May 2024 14:12:08 -0400 Subject: [PATCH 084/115] feat: also check nixbld user groups --- ansible-nix/files/permission_check.py | 21 +++++++++++++++++++++ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/ansible-nix/files/permission_check.py b/ansible-nix/files/permission_check.py index a7175b63c..724acb10a 100644 --- a/ansible-nix/files/permission_check.py +++ b/ansible-nix/files/permission_check.py @@ -172,6 +172,24 @@ def compare_results(username, query_result): print("Got:", query_result) sys.exit(1) +def check_nixbld_users(): + query = """ + SELECT u.username, g.groupname + FROM users u + JOIN user_groups ug ON u.uid = ug.uid + JOIN groups g ON ug.gid = g.gid + WHERE u.username LIKE 'nixbld%'; + """ + query_result = run_osquery(query) + parsed_result = parse_json(query_result) + + for user in parsed_result: + if user['groupname'] != 'nixbld': + print(f"User '{user['username']}' is in group '{user['groupname']}' instead of 'nixbld'.") + sys.exit(1) + + print("All nixbld users are in the 'nixbld' group.") + # Define usernames for which you want to compare results usernames = ["postgres", "ubuntu", "root", "daemon", "bin", "sys", "sync", "games","man","lp","mail","news","uucp","proxy","www-data","backup","list","irc","gnats","nobody","systemd-network","systemd-resolve","systemd-timesync","messagebus","ec2-instance-connect","sshd","wal-g","pgbouncer","gotrue","envoy","kong","nginx","vector","adminapi","postgrest","tcpdump","systemd-coredump"] @@ -181,3 +199,6 @@ def compare_results(username, query_result): query_result = run_osquery(query) parsed_result = parse_json(query_result) compare_results(username, parsed_result) + +# Check if all nixbld users are in the nixbld group +check_nixbld_users() diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 4bc17d2ab..8ea36b069 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.51-nix-staged" +postgres-version = "15.6.1.52-nix-staged" From 0923ece32f49a4e1f1ed3de74d5f3da76cfae32d Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 22 May 2024 10:10:47 -0400 Subject: [PATCH 085/115] fix: merge conflict resolve --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9711dcd0b..fdffb4a1d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ result* #IDE -.vscode +.idea/ +.vscode/ From c4ba0809eaceddaec4556b2c0cd0a6a75a9537a9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 3 Jun 2024 16:00:29 +0000 Subject: [PATCH 086/115] chore: bump auth version to fix merge conflict --- ansible-nix/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible-nix/vars.yml b/ansible-nix/vars.yml index 33526ea26..74e5afe3b 100644 --- a/ansible-nix/vars.yml +++ b/ansible-nix/vars.yml @@ -15,8 +15,8 @@ postgrest_release: "12.0.2" postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 -gotrue_release: 2.151.0 -gotrue_release_checksum: sha1:5a43a9879499d85714ba34356f62f5e8063b549d +gotrue_release: 2.152.0 +gotrue_release_checksum: sha1:5f220e9f322cc13f93401ab927d32dd0d1f85b40 aws_cli_release: "2.2.7" From 09e31e49b158c6920829d63afd8ca2cf10ddd561 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 3 Jun 2024 16:03:26 +0000 Subject: [PATCH 087/115] chore: bump auth version --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 33526ea26..74e5afe3b 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -15,8 +15,8 @@ postgrest_release: "12.0.2" postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 -gotrue_release: 2.151.0 -gotrue_release_checksum: sha1:5a43a9879499d85714ba34356f62f5e8063b549d +gotrue_release: 2.152.0 +gotrue_release_checksum: sha1:5f220e9f322cc13f93401ab927d32dd0d1f85b40 aws_cli_release: "2.2.7" From 70a3f8ea883831fa781466b38ca82023409ff944 Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 6 Jun 2024 11:23:20 -0400 Subject: [PATCH 088/115] chore: cleaning up and triggering build --- .github/workflows/ami-release-nix.yml | 51 ++----------------- ansible-nix/tasks/setup-postgres.yml | 26 +--------- ansible-nix/tasks/stage2/optimizations.yml | 19 ------- ansible-nix/tasks/stage2/setup-extensions.yml | 24 --------- .../tasks/stage2/stage2-setup-postgres.yml | 39 +------------- common-nix.vars.pkr.hcl | 2 +- nix/ext/wrappers/default.nix | 2 - 7 files changed, 7 insertions(+), 156 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 2065c2a18..27da3b121 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -40,54 +40,9 @@ jobs: exit 1 fi - # - id: args - # uses: mikefarah/yq@master - # with: - # cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - # - run: docker context create builders - # - uses: docker/setup-buildx-action@v3 - # with: - # endpoint: builders - # - uses: docker/build-push-action@v5 - # with: - # build-args: | - # ${{ steps.args.outputs.result }} - # target: extensions - # tags: supabase/postgres:extensions - # platforms: linux/${{ matrix.arch }} - # outputs: type=tar,dest=/tmp/extensions.tar - # cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - # # No need to export extensions cache because latest depends on it - - # - name: Extract built packages - # run: | - # mkdir -p ansible/files/extensions - # tar xvf /tmp/extensions.tar -C ansible/files/extensions --strip-components 1 - # TODO remove this block as extensions are build in nix prior to this step - - # - id: version - # run: echo "${{ steps.args.outputs.result }}" | grep "postgresql" >> "$GITHUB_OUTPUT" - # - name: Build Postgres deb - # uses: docker/build-push-action@v5 - # with: - # file: docker/Dockerfile - # target: pg-deb - # build-args: | - # ubuntu_release=${{ matrix.ubuntu_release }} - # ubuntu_release_no=${{ matrix.ubuntu_version }} - # postgresql_major=${{ steps.version.outputs.postgresql_major }} - # postgresql_release=${{ steps.version.outputs.postgresql_release }} - # CPPFLAGS=-mcpu=${{ matrix.mcpu }} - # tags: supabase/postgres:deb - # platforms: linux/${{ matrix.arch }} - # outputs: type=tar,dest=/tmp/pg-deb.tar - # cache-from: type=gha,scope=${{ github.ref_name }}-deb - # cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-deb - # - name: Extract Postgres deb - # run: | - # mkdir -p ansible/files/postgres - # tar xvf /tmp/pg-deb.tar -C ansible/files/postgres --strip-components 1 - #TODO remove this block as deb is build in nix prior to this step + # extensions are build in nix prior to this step + # so we can just use the binaries from the nix store + # for postgres, extensions and wrappers - name: Build AMI stage 1 run: | diff --git a/ansible-nix/tasks/setup-postgres.yml b/ansible-nix/tasks/setup-postgres.yml index 7cf9852bd..0644f10dc 100644 --- a/ansible-nix/tasks/setup-postgres.yml +++ b/ansible-nix/tasks/setup-postgres.yml @@ -1,15 +1,3 @@ -# - name: Postgres - install commons -# apt: -# name: postgresql-common -# install_recommends: no - -# - name: Do not create main cluster -# shell: -# cmd: sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf -# -# TODO These lines appear to be installing and configuring https://launchpad.net/ubuntu/+source/postgresql-common -# as far as I can see, we don't need this now. - - name: create ssl-cert group group: name: ssl-cert @@ -23,15 +11,6 @@ name: postgres state: present -# - name: create users -# user: -# name: postgres -# groups: postgres, ssl-cert -# createhome: yes -# append: no -# shell: /bin/bash # Set shell if needed, default is /bin/bash -# password: '' - - name: create postgres user shell: useradd -m -r -s /bin/bash -d /home/postgres postgres -g postgres args: @@ -136,9 +115,8 @@ group: postgres # # Install extensions before init -# - name: Install Postgres extensions -# import_tasks: tasks/setup-docker.yml -# TODO resolve in new build +# (samrose) moved to tasks/stage2/setup-extensions.yml +# now called from stage2/stage2-setup-postgres.yml # init DB diff --git a/ansible-nix/tasks/stage2/optimizations.yml b/ansible-nix/tasks/stage2/optimizations.yml index 6e7052dc2..77ecb3697 100644 --- a/ansible-nix/tasks/stage2/optimizations.yml +++ b/ansible-nix/tasks/stage2/optimizations.yml @@ -1,17 +1,9 @@ -# - name: ensure services are stopped -# community.general.snap: -# name: amazon-ssm-agent -# state: absent -# TODO taking this out as machine at this stage reports -# that aws-ssm-agent is not installed at all - - name: ensure services are stopped and disabled for first boot systemd: enabled: no name: '{{ item }}' state: stopped with_items: - #- snapd - postgresql - pgbouncer - fail2ban @@ -24,17 +16,6 @@ pkg: - snapd -# - name: ensure services are stopped and disabled for first boot -# systemd: -# enabled: no -# name: '{{ item }}' -# state: stopped -# masked: yes -# with_items: -# - lvm2-monitor -# machine at this stage reports this service is stopped and disabled - - - name: disable man-db become: yes file: diff --git a/ansible-nix/tasks/stage2/setup-extensions.yml b/ansible-nix/tasks/stage2/setup-extensions.yml index df769dbb6..808b6e56a 100644 --- a/ansible-nix/tasks/stage2/setup-extensions.yml +++ b/ansible-nix/tasks/stage2/setup-extensions.yml @@ -1,21 +1,3 @@ -# - name: Copy extension packages -# copy: -# src: files/extensions/ -# dest: /tmp/extensions/ - -# # Builtin apt module does not support wildcard for deb paths -# - name: Install extensions -# shell: | -# set -e -# apt-get update -# apt-get install -y --no-install-recommends /tmp/extensions/*.deb -# TODO remove - -# The following tasks were moved to stage 2 from the original ansible/tasks/setup-docker.yml -# due to these configurations being required to be applied at this stage, after the database package -# has been installed and before the database is started. -# These tasks are required to be run before the database is started, as they modify the database configuration - - name: pg_cron - set cron.database_name become: yes lineinfile: @@ -78,9 +60,3 @@ path: /etc/postgresql/postgresql.conf regexp: "#include = '/etc/postgresql-custom/supautils.conf'" replace: "include = '/etc/postgresql-custom/supautils.conf'" - -# - name: Cleanup - extension packages -# file: -# path: /tmp/extensions -# state: absent -# TODO remove extensions are already packaged in the nix build diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index 1180ddd4f..2e1a0b0e4 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -2,43 +2,13 @@ become: yes shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/sam/2-stage-ami-nix#psql_15/bin" +#TODO (samrose) switch pg_prove sourcing to develop branch once PR is merged - name: Install pg_prove from nix binary cache become: yes shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/sam/2-stage-ami-nix#pg_prove" -#TODO switch pg_prove sourcing to develop branch once PR is merged - -# - name: Generate en_US.UTF-8 locale -# command: sudo localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 - - -# - name: Add LOCALE_ARCHIVE to .bashrc -# ansible.builtin.lineinfile: -# dest: "/home/postgres/.bashrc" -# line: 'export LOCALE_ARCHIVE=/usr/lib/locale/locale-archive' -# create: yes -# become: yes -# become_user: postgres - -# - name: Add LANG items to .bashrc -# ansible.builtin.lineinfile: -# dest: "/home/postgres/.bashrc" -# line: "{{ item }}" - -# loop: -# - 'export LANG="en_US.UTF-8"' -# - 'export LANGUAGE="en_US.UTF-8"' -# - 'export LC_ALL="en_US.UTF-8"' -# - 'export LANG="en_US.UTF-8"' -# - 'export LC_CTYPE="en_US.UTF-8"' -# become: yes -# become_user: postgres - - -#TODO include ls $(nix profile list | grep glibc-locales | tail -n 1 | cut -d ':' -f 2 | sed 's/^[ \t]*//')/lib/locale-archive - - name: Set ownership and permissions for /etc/ssl/private become: yes file: @@ -146,7 +116,6 @@ state: directory recurse: yes -# TODO check that these dirs were created correctly in final AMI - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir shell: > sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \ @@ -186,12 +155,6 @@ owner: postgres group: postgres -# - name: Set LANG environment variable -# environment: -# LANG: en_US.UTF-8 -# #become: yes -#source /home/postgres/.bashrc && - - name: Initialize the database become: yes become_user: postgres diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 8ea36b069..6b8e17bbd 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.52-nix-staged" +postgres-version = "15.6.1.53-nix-staged" diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index ec785d09d..c0113cdc6 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -30,8 +30,6 @@ buildPgrxExtension_0_11_3 rec { CARGO="${cargo}/bin/cargo"; cargoLock = { - #TODO when we move to newer versions this lockfile will need to be sourced - # from ${src}/Cargo.lock lockFile = "${src}/Cargo.lock"; outputHashes = { "clickhouse-rs-1.0.0-alpha.1" = "sha256-0zmoUo/GLyCKDLkpBsnLAyGs1xz6cubJhn+eVqMEMaw="; From ed9225dbb46dce3ca6952c4abd04cd3a1d528993 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 11 Jun 2024 10:40:12 -0400 Subject: [PATCH 089/115] chore: sync up go true versions --- ansible-nix/vars.yml | 4 ++-- ansible/vars.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible-nix/vars.yml b/ansible-nix/vars.yml index 74e5afe3b..81149f748 100644 --- a/ansible-nix/vars.yml +++ b/ansible-nix/vars.yml @@ -15,8 +15,8 @@ postgrest_release: "12.0.2" postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 -gotrue_release: 2.152.0 -gotrue_release_checksum: sha1:5f220e9f322cc13f93401ab927d32dd0d1f85b40 +gotrue_release: 2.153.0 +gotrue_release_checksum: sha1:334483141b8c03f24ebe7ecb24519bbb6b704cf4 aws_cli_release: "2.2.7" diff --git a/ansible/vars.yml b/ansible/vars.yml index 74e5afe3b..81149f748 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -15,8 +15,8 @@ postgrest_release: "12.0.2" postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 -gotrue_release: 2.152.0 -gotrue_release_checksum: sha1:5f220e9f322cc13f93401ab927d32dd0d1f85b40 +gotrue_release: 2.153.0 +gotrue_release_checksum: sha1:334483141b8c03f24ebe7ecb24519bbb6b704cf4 aws_cli_release: "2.2.7" From 5af81eb8e73408c98ed448cac93671e6314fa17c Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 11 Jun 2024 10:43:23 -0400 Subject: [PATCH 090/115] chore: rm TODO comments on done items --- .github/workflows/ami-release-nix.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 27da3b121..abc119bbc 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -76,11 +76,6 @@ jobs: -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ manifest-playbook.yml - # - name: Upload pg binaries to s3 staging - # run: | - # aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz - # #TODO look to see if this only pg binaries and if so, remove this as it is covered by nix build - # TODO deactivate this block to assure binaries from this file are not uploaded. This is covered by nix build #Our self hosted github runner already has permissions to publish images #but they're limited to only that; @@ -101,10 +96,6 @@ jobs: -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ manifest-playbook.yml - # - name: Upload pg binaries to s3 prod - # run: | - # aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz - #TODO deactivate this block to assure binaries from this file are not uploaded. This is covered by nix build - name: Create release From 72ccd63029c726ca0c0df9b3b4377e438a4229f8 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 11 Jun 2024 10:44:26 -0400 Subject: [PATCH 091/115] chore: trigger ami build with gotrue changes --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 6b8e17bbd..db6dc9093 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.53-nix-staged" +postgres-version = "15.6.1.54-nix-staged" From 3c8f683a0a0fe024d30aeb1ba66b42d3f9ee9496 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 11 Jun 2024 11:07:12 -0400 Subject: [PATCH 092/115] fix: syncing source of kong with develop branch --- ansible-nix/tasks/setup-kong.yml | 2 +- ansible/tasks/setup-kong.yml | 2 +- common-nix.vars.pkr.hcl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible-nix/tasks/setup-kong.yml b/ansible-nix/tasks/setup-kong.yml index 09d6d2b44..b34f96e78 100644 --- a/ansible-nix/tasks/setup-kong.yml +++ b/ansible-nix/tasks/setup-kong.yml @@ -12,7 +12,7 @@ - name: Kong - download deb package get_url: - url: "https://download.konghq.com/gateway-2.x-ubuntu-{{ kong_release_target }}/pool/all/k/kong/{{ kong_deb }}" + url: "https://packages.konghq.com/public/gateway-28/deb/ubuntu/pool/{{ kong_release_target }}/main/k/ko/kong_2.8.1/{{ kong_deb }}" dest: /tmp/kong.deb checksum: "{{ kong_deb_checksum }}" diff --git a/ansible/tasks/setup-kong.yml b/ansible/tasks/setup-kong.yml index 09d6d2b44..b34f96e78 100644 --- a/ansible/tasks/setup-kong.yml +++ b/ansible/tasks/setup-kong.yml @@ -12,7 +12,7 @@ - name: Kong - download deb package get_url: - url: "https://download.konghq.com/gateway-2.x-ubuntu-{{ kong_release_target }}/pool/all/k/kong/{{ kong_deb }}" + url: "https://packages.konghq.com/public/gateway-28/deb/ubuntu/pool/{{ kong_release_target }}/main/k/ko/kong_2.8.1/{{ kong_deb }}" dest: /tmp/kong.deb checksum: "{{ kong_deb_checksum }}" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index db6dc9093..369d1a6f5 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.54-nix-staged" +postgres-version = "15.6.1.55-nix-staged" From a2f566a885cb7a9050c8da7d27eb3f98dda6f444 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 21 May 2024 10:58:30 -0400 Subject: [PATCH 093/115] Sam/pljava nix (#976) * feat: package pljava * feat: pljava psql nix package * feat: package and smoke test pljava extension into the bundle run on openjdk11 * chore: cleanup nix expression --- flake.nix | 9 +++++-- nix/ext/pljava.nix | 51 ++++++++++++++++++++++++++++++++++++ nix/tests/postgresql.conf.in | 1 + nix/tests/prime.sql | 1 + nix/tools/run-server.sh.in | 8 +++--- 5 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 nix/ext/pljava.nix diff --git a/flake.nix b/flake.nix index 3f7c15b40..bb7fc2a0b 100644 --- a/flake.nix +++ b/flake.nix @@ -132,6 +132,7 @@ ./nix/ext/wrappers/default.nix ./nix/ext/supautils.nix ./nix/ext/plv8.nix + ./nix/ext/pljava.nix ]; #Where we import and build the orioledb extension, we add on our custom extensions @@ -418,7 +419,8 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ - --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' + --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ + --subst-var-by 'LIBJVM_LOCATION' '${pkgs.openjdk11}/lib/openjdk/lib/server/libjvm.so' chmod +x $out/bin/start-postgres-server ''; @@ -481,13 +483,16 @@ pkgs.runCommand "postgres-${pgpkg.version}-check-harness" { nativeBuildInputs = with pkgs; [ coreutils bash pgpkg pg_prove procps ]; + propagatedBuildInputs = with pkgs; [ openjdk11 ]; } '' export PGDATA=/tmp/pgdata mkdir -p $PGDATA initdb --locale=C substitute ${./nix/tests/postgresql.conf.in} $PGDATA/postgresql.conf \ - --subst-var-by PGSODIUM_GETKEY_SCRIPT "${./nix/tests/util/pgsodium_getkey.sh}" + --subst-var-by PGSODIUM_GETKEY_SCRIPT "${./nix/tests/util/pgsodium_getkey.sh}" \ + --subst-var-by PLJAVA_LIBJVM_LOCATION "${pkgs.openjdk11}/lib/openjdk/lib/server/libjvm.so" + postgres -k /tmp >logfile 2>&1 & sleep 2 diff --git a/nix/ext/pljava.nix b/nix/ext/pljava.nix new file mode 100644 index 000000000..16f8a59c3 --- /dev/null +++ b/nix/ext/pljava.nix @@ -0,0 +1,51 @@ +{ stdenv, lib, fetchFromGitHub, openssl, openjdk, maven, postgresql, libkrb5, makeWrapper, gcc, pkg-config, which }: + +maven.buildMavenPackage rec { + pname = "pljava"; + + version = "1.6.7"; + + src = fetchFromGitHub { + owner = "tada"; + repo = "pljava"; + rev = "V1_6_7"; + sha256 = "sha256-M17adSLsw47KZ2BoUwxyWkXKRD8TcexDAy61Yfw4fNU="; + + }; + + mvnParameters = "clean install -Dmaven.test.skip -DskipTests -Dmaven.javadoc.skip=true"; + mvnHash = "sha256-lcxRduh/nKcPL6YQIVTsNH0L4ga0LgJpQKgX5IPkRzs="; + + nativeBuildInputs = [ makeWrapper maven openjdk postgresql openssl postgresql gcc libkrb5 pkg-config ]; + buildInputs = [ stdenv.cc.cc.lib which]; + buildPhase = '' + export PATH=$(lib.makeBinPath [ postgresql ]):$PATH + + ''; + buildOffline = true; + + installPhase = '' + mkdir -p $out/pljavabuild + cp -r * $out/pljavabuild + mkdir -p $out/share/postgresql/extension/pljava + mkdir -p $out/share/postgresql/pljava + mkdir -p $out/lib + mkdir -p $out/etc + java -Dpgconfig=${postgresql}/bin/pg_config \ + -Dpgconfig.sharedir=$out/share \ + -Dpgconfig.sysconfdir=$out/etc/pljava.policy \ + -Dpgconfig.pkglibdir=$out/lib \ + -jar $out/pljavabuild/pljava-packaging/target/pljava-pg15.jar + cp $out/share/pljava/* $out/share/postgresql/extension/pljava + cp $out/share/pljava/* $out/share/postgresql/pljava + cp $out/share/extension/*.control $out/share/postgresql/extension + rm -r $out/pljavabuild + ''; + + meta = with lib; { + description = "PL/Java extension for PostgreSQL"; + homepage = https://github.com/tada/pljava; + license = licenses.bsd3; + maintainers = [ maintainers.samrose ]; # Update with actual maintainer info + }; +} diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index 4c5075aa1..22d1b93fc 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -793,3 +793,4 @@ shared_preload_libraries = 'auto_explain,pgsodium' # Add settings for extensions here pgsodium.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' +pljava.libjvm_location = '@PLJAVA_LIBJVM_LOCATION@' diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 084ad3b52..5a0e88f58 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -20,3 +20,4 @@ CREATE EXTENSION IF NOT EXISTS pg_graphql; CREATE EXTENSION IF NOT EXISTS pg_jsonschema; CREATE EXTENSION IF NOT EXISTS hypopg; CREATE EXTENSION IF NOT EXISTS index_advisor; +CREATE EXTENSION IF NOT EXISTS pljava; diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index c5b6eb040..b620a0f15 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -27,6 +27,7 @@ PGSQL_SUPERUSER=@PGSQL_SUPERUSER@ PSQL_CONF_FILE=@PSQL_CONF_FILE@ PGSODIUM_GETKEY_SCRIPT=@PGSODIUM_GETKEY@ PORTNO="${2:-@PGSQL_DEFAULT_PORT@}" +PLJAVA_LIBJVM_LOCATION=@LIBJVM_LOCATION@ DATDIR=$(mktemp -d) mkdir -p "$DATDIR" @@ -38,8 +39,9 @@ echo initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" --locale=C echo "NOTE: patching postgresql.conf files" -sed \ - "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ - $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" +echo "pljava libjvm location: $PLJAVA_LIBJVM_LOCATION" +sed -e "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ + -e "s#@PLJAVA_LIBJVM_LOCATION@#$PLJAVA_LIBJVM_LOCATION#g" \ + $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From 49cc9fe4069db9916451f98ae7b292af4e0e873f Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Tue, 4 Jun 2024 16:44:51 -0500 Subject: [PATCH 094/115] Bump supautils version (#989) * bump supautils version * chore: bump supautils everywhere --------- Co-authored-by: Bobbie Soedirgo --- Dockerfile | 2 +- ansible/vars.yml | 4 ++-- docker/orioledb/Dockerfile | 2 +- nix/ext/supautils.nix | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d7911686..37086d95d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ ARG hypopg_release=1.3.1 ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.0 +ARG supautils_release=2.2.1 ARG wal_g_release=2.0.1 #################### diff --git a/ansible/vars.yml b/ansible/vars.yml index 81149f748..97faa1664 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -82,8 +82,8 @@ timescaledb_release_checksum: sha256:883638f2e79d25ec88ee58f603f3c81c999b6364cb4 wal2json_release: "2_5" wal2json_release_checksum: sha256:b516653575541cf221b99cf3f8be9b6821f6dbcfc125675c85f35090f824f00e -supautils_release: "2.2.0" -supautils_release_checksum: sha256:0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5 +supautils_release: "2.2.1" +supautils_release_checksum: sha256:1a2d2b8fe604d38921ed9cf3a0d56dd142a274035d0dca17ad21cdc81ddd9569 pljava_release: master pljava_release_checksum: sha256:e99b1c52f7b57f64c8986fe6ea4a6cc09d78e779c1643db060d0ac66c93be8b6 diff --git a/docker/orioledb/Dockerfile b/docker/orioledb/Dockerfile index 90668f098..a17dc39dc 100644 --- a/docker/orioledb/Dockerfile +++ b/docker/orioledb/Dockerfile @@ -36,7 +36,7 @@ ARG hypopg_release=1.3.1 ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.0 +ARG supautils_release=2.2.1 ARG wal_g_release=2.0.1 #################### diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index eaf772766..9a32062b9 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "supautils"; - version = "2.2.0"; + version = "2.2.1"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-LEMhUM0woTwRcSWOU70izS5eWevoV8nnMUOEID+Nsdc="; + hash = "sha256-wSUEG0at00TPAoHv6+NMzuUE8mfW6fnHH0MNxvBdUiE="; }; installPhase = '' From e09976e6dd9ca4a5b4a892cc639c3140ebbaffc8 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 11 Jun 2024 15:16:14 -0400 Subject: [PATCH 095/115] chore: bump version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 369d1a6f5..e16860bad 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.55-nix-staged" +postgres-version = "15.6.1.56-nix-staged" From 26f193a3ca104d4f26feaf512712e32c0c866bc1 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 11 Jun 2024 15:17:37 -0400 Subject: [PATCH 096/115] chore: bump version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index e16860bad..29900939f 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.56-nix-staged" +postgres-version = "15.6.1.60-nix-staged" From f519287ac90feb06bfa13524c3584f54fe68a389 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 09:17:29 -0400 Subject: [PATCH 097/115] fix: resolve deps and congif for pljava --- ansible-nix/tasks/stage2/setup-extensions.yml | 7 +++++++ .../tasks/stage2/stage2-setup-postgres.yml | 21 +++++++++++++++++++ common-nix.vars.pkr.hcl | 2 +- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/ansible-nix/tasks/stage2/setup-extensions.yml b/ansible-nix/tasks/stage2/setup-extensions.yml index 808b6e56a..2ed3c3d72 100644 --- a/ansible-nix/tasks/stage2/setup-extensions.yml +++ b/ansible-nix/tasks/stage2/setup-extensions.yml @@ -1,3 +1,10 @@ +- name: pljava - libjvm.so location in postgresql.conf + become: yes + lineinfile: + path: /etc/postgresql/postgresql.conf + state: present + line: pljava.libjvm_location = '/home/postgres/.nix-profile/lib/openjdk/lib/server/libjvm.so' + - name: pg_cron - set cron.database_name become: yes lineinfile: diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index 2e1a0b0e4..1d4dee4b2 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -1,3 +1,8 @@ +- name: Install openjdk11 for pljava from nix binary cache + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11" + - name: Install Postgres from nix binary cache become: yes shell: | @@ -84,7 +89,23 @@ with_fileglob: - "/home/postgres/.nix-profile/bin/*" become: yes +sudo ln -s /home/postgres/.nix-profile/share/pljava /usr/lib/postgresql/share/postgresql/pljava +- name: Create symbolic links from /home/postgres/.nix-profile/bin to /usr/bin + file: + src: "/home/postgres/.nix-profile/bin/psql" + dest: "/usr/bin/psql" + state: link + become: yes +- name: Create symbolic links from /home/postgres/.nix-profile/share/pljava to /usr/lib/postgresql/share/postgresql/pljava + file: + src: "{{ item }}" + dest: "/usr/lib/postgresql/share/postgresql/pljava/{{ item | basename }}" + state: link + with_fileglob: + - "/home/postgres/.nix-profile/share/pljava/*" + become: yes + - name: Create symbolic links from /home/postgres/.nix-profile/bin to /usr/bin file: src: "/home/postgres/.nix-profile/bin/psql" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 29900939f..4b2594c79 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.60-nix-staged" +postgres-version = "15.6.1.61-nix-staged" From fe4ec8d30ace56826672b1a0223051b655f1196d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 09:51:16 -0400 Subject: [PATCH 098/115] fix: cleanup --- ansible-nix/tasks/stage2/stage2-setup-postgres.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index 1d4dee4b2..fd6c950f2 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -89,7 +89,7 @@ with_fileglob: - "/home/postgres/.nix-profile/bin/*" become: yes -sudo ln -s /home/postgres/.nix-profile/share/pljava /usr/lib/postgresql/share/postgresql/pljava + - name: Create symbolic links from /home/postgres/.nix-profile/bin to /usr/bin file: src: "/home/postgres/.nix-profile/bin/psql" @@ -105,7 +105,7 @@ sudo ln -s /home/postgres/.nix-profile/share/pljava /usr/lib/postgresql/share/po with_fileglob: - "/home/postgres/.nix-profile/share/pljava/*" become: yes - + - name: Create symbolic links from /home/postgres/.nix-profile/bin to /usr/bin file: src: "/home/postgres/.nix-profile/bin/psql" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 4b2594c79..1739d79a0 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.61-nix-staged" +postgres-version = "15.6.1.62-nix-staged" From fd609d60f29ea287ae45e96b467fda4c9ab53013 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 10:38:13 -0400 Subject: [PATCH 099/115] chore: ensure target dir exists --- ansible-nix/tasks/stage2/stage2-setup-postgres.yml | 7 +++++++ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index fd6c950f2..5b4cc4c80 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -73,6 +73,13 @@ owner: postgres group: postgres +- name: Ensure /usr/lib/postgresql/share/postgresql/pljava directory exists + file: + path: /usr/lib/postgresql/share/postgresql/pljava + state: directory + owner: postgres + group: postgres + - name: import pgsodium_getkey script template: src: /tmp/ansible-playbook/files/pgsodium_getkey_readonly.sh.j2 diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 1739d79a0..537318745 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.62-nix-staged" +postgres-version = "15.6.1.63-nix-staged" From ab558951ee6fd121566660ab264970702d6c5cfa Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 14:55:20 -0400 Subject: [PATCH 100/115] fix: It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 this commit disables the extension for now --- ansible-nix/tasks/stage2/setup-extensions.yml | 13 +++--- .../tasks/stage2/stage2-setup-postgres.yml | 40 ++++++++++--------- flake.nix | 2 +- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/ansible-nix/tasks/stage2/setup-extensions.yml b/ansible-nix/tasks/stage2/setup-extensions.yml index 2ed3c3d72..3f13d12e6 100644 --- a/ansible-nix/tasks/stage2/setup-extensions.yml +++ b/ansible-nix/tasks/stage2/setup-extensions.yml @@ -1,9 +1,10 @@ -- name: pljava - libjvm.so location in postgresql.conf - become: yes - lineinfile: - path: /etc/postgresql/postgresql.conf - state: present - line: pljava.libjvm_location = '/home/postgres/.nix-profile/lib/openjdk/lib/server/libjvm.so' +# - name: pljava - libjvm.so location in postgresql.conf +# become: yes +# lineinfile: +# path: /etc/postgresql/postgresql.conf +# state: present +# line: pljava.libjvm_location = '/home/postgres/.nix-profile/lib/openjdk/lib/server/libjvm.so' +# It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task - name: pg_cron - set cron.database_name become: yes diff --git a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml index 5b4cc4c80..3bc0fa91a 100644 --- a/ansible-nix/tasks/stage2/stage2-setup-postgres.yml +++ b/ansible-nix/tasks/stage2/stage2-setup-postgres.yml @@ -1,8 +1,8 @@ -- name: Install openjdk11 for pljava from nix binary cache - become: yes - shell: | - sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11" - +# - name: Install openjdk11 for pljava from nix binary cache +# become: yes +# shell: | +# sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11" +# It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task - name: Install Postgres from nix binary cache become: yes shell: | @@ -73,12 +73,13 @@ owner: postgres group: postgres -- name: Ensure /usr/lib/postgresql/share/postgresql/pljava directory exists - file: - path: /usr/lib/postgresql/share/postgresql/pljava - state: directory - owner: postgres - group: postgres +# - name: Ensure /usr/lib/postgresql/share/postgresql/pljava directory exists +# file: +# path: /usr/lib/postgresql/share/postgresql/pljava +# state: directory +# owner: postgres +# group: postgres +# It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task - name: import pgsodium_getkey script template: @@ -104,14 +105,15 @@ state: link become: yes -- name: Create symbolic links from /home/postgres/.nix-profile/share/pljava to /usr/lib/postgresql/share/postgresql/pljava - file: - src: "{{ item }}" - dest: "/usr/lib/postgresql/share/postgresql/pljava/{{ item | basename }}" - state: link - with_fileglob: - - "/home/postgres/.nix-profile/share/pljava/*" - become: yes +# - name: Create symbolic links from /home/postgres/.nix-profile/share/pljava to /usr/lib/postgresql/share/postgresql/pljava +# file: +# src: "{{ item }}" +# dest: "/usr/lib/postgresql/share/postgresql/pljava/{{ item | basename }}" +# state: link +# with_fileglob: +# - "/home/postgres/.nix-profile/share/pljava/*" +# become: yes +# It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task - name: Create symbolic links from /home/postgres/.nix-profile/bin to /usr/bin file: diff --git a/flake.nix b/flake.nix index bb7fc2a0b..c3a5b307a 100644 --- a/flake.nix +++ b/flake.nix @@ -132,7 +132,7 @@ ./nix/ext/wrappers/default.nix ./nix/ext/supautils.nix ./nix/ext/plv8.nix - ./nix/ext/pljava.nix + #./nix/ext/pljava.nix it was decided to remove pljava from the list of extensions at https://github.com/supabase/postgres/pull/690 therefore removing this task ]; #Where we import and build the orioledb extension, we add on our custom extensions From 91d0f670c1c0593bf494dfc848d39d66be4ef0ad Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 14:57:22 -0400 Subject: [PATCH 101/115] ore: bump version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 537318745..0593342df 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.63-nix-staged" +postgres-version = "15.6.1.64-nix-staged" From 92067bce688aad73fa53eee9ae38fb26951890f8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 14:59:02 -0400 Subject: [PATCH 102/115] chore: keep package but not configure for use in AMI --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index c3a5b307a..bb7fc2a0b 100644 --- a/flake.nix +++ b/flake.nix @@ -132,7 +132,7 @@ ./nix/ext/wrappers/default.nix ./nix/ext/supautils.nix ./nix/ext/plv8.nix - #./nix/ext/pljava.nix it was decided to remove pljava from the list of extensions at https://github.com/supabase/postgres/pull/690 therefore removing this task + ./nix/ext/pljava.nix ]; #Where we import and build the orioledb extension, we add on our custom extensions From ae517964a02a30658b386ca30732c577086fa027 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 16:03:30 -0400 Subject: [PATCH 103/115] chore: bump version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 0593342df..a0c368ce0 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.64-nix-staged" +postgres-version = "15.6.1.65-nix-staged" From 6f434999a0ce2d50676165945b4781ead70a173e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 16:57:45 -0400 Subject: [PATCH 104/115] chore: must totally deactivate pljava package --- common-nix.vars.pkr.hcl | 2 +- flake.nix | 8 +++----- nix/tests/postgresql.conf.in | 1 - nix/tests/prime.sql | 1 - nix/tools/run-server.sh.in | 2 -- 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index a0c368ce0..8429e55a7 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.65-nix-staged" +postgres-version = "15.6.1.66-nix-staged" diff --git a/flake.nix b/flake.nix index bb7fc2a0b..7378f41b2 100644 --- a/flake.nix +++ b/flake.nix @@ -132,7 +132,7 @@ ./nix/ext/wrappers/default.nix ./nix/ext/supautils.nix ./nix/ext/plv8.nix - ./nix/ext/pljava.nix + #./nix/ext/pljava.nix ]; #Where we import and build the orioledb extension, we add on our custom extensions @@ -419,8 +419,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ - --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ - --subst-var-by 'LIBJVM_LOCATION' '${pkgs.openjdk11}/lib/openjdk/lib/server/libjvm.so' + --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' chmod +x $out/bin/start-postgres-server ''; @@ -490,8 +489,7 @@ initdb --locale=C substitute ${./nix/tests/postgresql.conf.in} $PGDATA/postgresql.conf \ - --subst-var-by PGSODIUM_GETKEY_SCRIPT "${./nix/tests/util/pgsodium_getkey.sh}" \ - --subst-var-by PLJAVA_LIBJVM_LOCATION "${pkgs.openjdk11}/lib/openjdk/lib/server/libjvm.so" + --subst-var-by PGSODIUM_GETKEY_SCRIPT "${./nix/tests/util/pgsodium_getkey.sh}" postgres -k /tmp >logfile 2>&1 & diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index 22d1b93fc..4c5075aa1 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -793,4 +793,3 @@ shared_preload_libraries = 'auto_explain,pgsodium' # Add settings for extensions here pgsodium.getkey_script = '@PGSODIUM_GETKEY_SCRIPT@' -pljava.libjvm_location = '@PLJAVA_LIBJVM_LOCATION@' diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index 5a0e88f58..084ad3b52 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -20,4 +20,3 @@ CREATE EXTENSION IF NOT EXISTS pg_graphql; CREATE EXTENSION IF NOT EXISTS pg_jsonschema; CREATE EXTENSION IF NOT EXISTS hypopg; CREATE EXTENSION IF NOT EXISTS index_advisor; -CREATE EXTENSION IF NOT EXISTS pljava; diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index b620a0f15..2f11fd827 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -39,9 +39,7 @@ echo initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" --locale=C echo "NOTE: patching postgresql.conf files" -echo "pljava libjvm location: $PLJAVA_LIBJVM_LOCATION" sed -e "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ - -e "s#@PLJAVA_LIBJVM_LOCATION@#$PLJAVA_LIBJVM_LOCATION#g" \ $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From 1eccecac16600b09dec2930192c03a612f36a7e6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 17:36:55 -0400 Subject: [PATCH 105/115] chore: fixing merge conflict --- flake.nix | 28 ++++++++++------------------ nix/tools/run-server.sh.in | 2 ++ 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/flake.nix b/flake.nix index 7378f41b2..e83ffab1c 100644 --- a/flake.nix +++ b/flake.nix @@ -29,7 +29,6 @@ # it also serves as a base for importing the orioldb/postgres overlay to #build the orioledb postgres patched version of postgresql16 oriole_pkgs = import nixpkgs { - config = { allowUnfree = true; }; inherit system; overlays = [ # NOTE (aseipp): add any needed overlays here. in theory we could @@ -45,7 +44,6 @@ #This variable works the same as 'oriole_pkgs' but builds using the upstream #nixpkgs builds of postgresql 15 and 16 + the overlays listed below pkgs = import nixpkgs { - config = { allowUnfree = true; }; inherit system; overlays = [ # NOTE (aseipp): add any needed overlays here. in theory we could @@ -57,10 +55,8 @@ #(import ./nix/overlays/gdal-small.nix) ]; - }; - sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { }; # FIXME (aseipp): pg_prove is yet another perl program that needs # LOCALE_ARCHIVE set in non-NixOS environments. upstream this. once that's done, we @@ -132,7 +128,7 @@ ./nix/ext/wrappers/default.nix ./nix/ext/supautils.nix ./nix/ext/plv8.nix - #./nix/ext/pljava.nix + ./nix/ext/pljava.nix ]; #Where we import and build the orioledb extension, we add on our custom extensions @@ -141,8 +137,8 @@ #this var is a convenience setting to import the orioledb patched version of postgresql postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; + #postgis_override = pkgs.postgis_override; - # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other # tools to inspect what the contents of the install are: the PSQL @@ -309,7 +305,7 @@ ''; in nix2img.buildImage { - name = "samrose/nix-experimental-postgresql-${version}-${system}"; + name = "nix-experimental-postgresql-${version}-${system}"; tag = "latest"; nixUid = l.toInt uid; @@ -403,9 +399,7 @@ psql_15 = makePostgres "15"; #psql_16 = makePostgres "16"; #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; - pg_prove = pg_prove; - sfcgal = sfcgal; - + # Start a version of the server. start-server = let @@ -419,7 +413,8 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ - --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' + --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ + --subst-var-by 'LIBJVM_LOCATION' '${pkgs.openjdk11}/lib/openjdk/lib/server/libjvm.so' chmod +x $out/bin/start-postgres-server ''; @@ -458,7 +453,7 @@ mkdir -p $out/bin substitute ${./nix/tools/run-replica.sh.in} $out/bin/start-postgres-replica \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' + --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}'\ chmod +x $out/bin/start-postgres-replica ''; sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } '' @@ -489,7 +484,8 @@ initdb --locale=C substitute ${./nix/tests/postgresql.conf.in} $PGDATA/postgresql.conf \ - --subst-var-by PGSODIUM_GETKEY_SCRIPT "${./nix/tests/util/pgsodium_getkey.sh}" + --subst-var-by PGSODIUM_GETKEY_SCRIPT "${./nix/tests/util/pgsodium_getkey.sh}" \ + --subst-var-by PLJAVA_LIBJVM_LOCATION "${pkgs.openjdk11}/lib/openjdk/lib/server/libjvm.so" postgres -k /tmp >logfile 2>&1 & @@ -555,11 +551,7 @@ nix-update pg_prove shellcheck - ansible - ansible-lint - (packer.overrideAttrs (oldAttrs: { - version = "1.7.8"; - })) + basePackages.start-server basePackages.start-client basePackages.start-replica diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 2f11fd827..b620a0f15 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -39,7 +39,9 @@ echo initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" --locale=C echo "NOTE: patching postgresql.conf files" +echo "pljava libjvm location: $PLJAVA_LIBJVM_LOCATION" sed -e "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ + -e "s#@PLJAVA_LIBJVM_LOCATION@#$PLJAVA_LIBJVM_LOCATION#g" \ $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From 30e29ad9fb9dd73cec250b4841d87e5edf26847d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 17:39:11 -0400 Subject: [PATCH 106/115] chore: resolving merge conflicts --- flake.nix | 9 ++------- nix/tools/run-server.sh.in | 2 -- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index e83ffab1c..db2352520 100644 --- a/flake.nix +++ b/flake.nix @@ -128,7 +128,6 @@ ./nix/ext/wrappers/default.nix ./nix/ext/supautils.nix ./nix/ext/plv8.nix - ./nix/ext/pljava.nix ]; #Where we import and build the orioledb extension, we add on our custom extensions @@ -413,8 +412,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ - --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ - --subst-var-by 'LIBJVM_LOCATION' '${pkgs.openjdk11}/lib/openjdk/lib/server/libjvm.so' + --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' chmod +x $out/bin/start-postgres-server ''; @@ -477,16 +475,13 @@ pkgs.runCommand "postgres-${pgpkg.version}-check-harness" { nativeBuildInputs = with pkgs; [ coreutils bash pgpkg pg_prove procps ]; - propagatedBuildInputs = with pkgs; [ openjdk11 ]; } '' export PGDATA=/tmp/pgdata mkdir -p $PGDATA initdb --locale=C substitute ${./nix/tests/postgresql.conf.in} $PGDATA/postgresql.conf \ - --subst-var-by PGSODIUM_GETKEY_SCRIPT "${./nix/tests/util/pgsodium_getkey.sh}" \ - --subst-var-by PLJAVA_LIBJVM_LOCATION "${pkgs.openjdk11}/lib/openjdk/lib/server/libjvm.so" - + --subst-var-by PGSODIUM_GETKEY_SCRIPT "${./nix/tests/util/pgsodium_getkey.sh}" postgres -k /tmp >logfile 2>&1 & sleep 2 diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index b620a0f15..2f11fd827 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -39,9 +39,7 @@ echo initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" --locale=C echo "NOTE: patching postgresql.conf files" -echo "pljava libjvm location: $PLJAVA_LIBJVM_LOCATION" sed -e "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ - -e "s#@PLJAVA_LIBJVM_LOCATION@#$PLJAVA_LIBJVM_LOCATION#g" \ $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From 184dd2ccd474dcb370d04a873b5485f43347cdd2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 17:41:17 -0400 Subject: [PATCH 107/115] chore: resolve merge conflict --- nix/tools/run-server.sh.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 2f11fd827..b620a0f15 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -39,7 +39,9 @@ echo initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" --locale=C echo "NOTE: patching postgresql.conf files" +echo "pljava libjvm location: $PLJAVA_LIBJVM_LOCATION" sed -e "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ + -e "s#@PLJAVA_LIBJVM_LOCATION@#$PLJAVA_LIBJVM_LOCATION#g" \ $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From 161ef63c72c3a8babe2bace5ef031886de3a43b7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 17:43:20 -0400 Subject: [PATCH 108/115] chore: resolve merge conflict --- nix/tools/run-server.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index b620a0f15..e1f8680d9 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -39,9 +39,9 @@ echo initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" --locale=C echo "NOTE: patching postgresql.conf files" -echo "pljava libjvm location: $PLJAVA_LIBJVM_LOCATION" + sed -e "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ - -e "s#@PLJAVA_LIBJVM_LOCATION@#$PLJAVA_LIBJVM_LOCATION#g" \ + $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From a3013118ef83b5cca913a1db8b2ba5a6986f9f98 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 17:44:04 -0400 Subject: [PATCH 109/115] chore: resolve merge conflict --- nix/tools/run-server.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index e1f8680d9..b620a0f15 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -39,9 +39,9 @@ echo initdb -U "$PGSQL_SUPERUSER" -D "$DATDIR" --locale=C echo "NOTE: patching postgresql.conf files" - +echo "pljava libjvm location: $PLJAVA_LIBJVM_LOCATION" sed -e "s#@PGSODIUM_GETKEY_SCRIPT@#$PGSODIUM_GETKEY_SCRIPT#g" \ - + -e "s#@PLJAVA_LIBJVM_LOCATION@#$PLJAVA_LIBJVM_LOCATION#g" \ $PSQL_CONF_FILE > "$DATDIR/postgresql.conf" exec postgres -p "$PORTNO" -D "$DATDIR" -k /tmp From 319b724add98b2ffda37702691dff0b0554df316 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 17:44:58 -0400 Subject: [PATCH 110/115] chore: bump version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 8429e55a7..9cdd19022 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.66-nix-staged" +postgres-version = "15.6.1.67-nix-staged" From f0c49ee1832068f4a39aa1012ad24bb3fb709cf6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 18:04:46 -0400 Subject: [PATCH 111/115] chore: merge conflixts --- flake.nix | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index db2352520..02f789359 100644 --- a/flake.nix +++ b/flake.nix @@ -29,6 +29,7 @@ # it also serves as a base for importing the orioldb/postgres overlay to #build the orioledb postgres patched version of postgresql16 oriole_pkgs = import nixpkgs { + config = { allowUnfree = true; }; inherit system; overlays = [ # NOTE (aseipp): add any needed overlays here. in theory we could @@ -44,6 +45,7 @@ #This variable works the same as 'oriole_pkgs' but builds using the upstream #nixpkgs builds of postgresql 15 and 16 + the overlays listed below pkgs = import nixpkgs { + config = { allowUnfree = true; }; inherit system; overlays = [ # NOTE (aseipp): add any needed overlays here. in theory we could @@ -57,6 +59,7 @@ ]; }; + sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { }; # FIXME (aseipp): pg_prove is yet another perl program that needs # LOCALE_ARCHIVE set in non-NixOS environments. upstream this. once that's done, we @@ -304,7 +307,8 @@ ''; in nix2img.buildImage { - name = "nix-experimental-postgresql-${version}-${system}"; + #TODO (samrose) update this with the correct image name for supabase registry + name = "samrose/nix-experimental-postgresql-${version}-${system}"; tag = "latest"; nixUid = l.toInt uid; @@ -398,7 +402,8 @@ psql_15 = makePostgres "15"; #psql_16 = makePostgres "16"; #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; - + pg_prove = pg_prove; + sfcgal = sfcgal; # Start a version of the server. start-server = let @@ -546,6 +551,11 @@ nix-update pg_prove shellcheck + ansible + ansible-lint + (packer.overrideAttrs (oldAttrs: { + version = "1.7.8"; + })) basePackages.start-server basePackages.start-client From 2e5cc849ea876beca877ae2ba3493b23ef30445c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 18:24:26 -0400 Subject: [PATCH 112/115] chore: bump version --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 9cdd19022..4508be3d3 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.67-nix-staged" +postgres-version = "15.6.1.68-nix-staged" From 30bb692a17c4a99181d74c85ad7bbe1bebab23ce Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 12 Jun 2024 18:40:16 -0400 Subject: [PATCH 113/115] chore: fix gh action workflow so it checks out correct commit --- .github/workflows/nix-build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 93f6549e7..0d6336cd0 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -25,7 +25,11 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref || github.ref }} + fetch-depth: 0 + fetch-tags: true - name: aws-creds uses: aws-actions/configure-aws-credentials@v4 with: From 8e8c8b2bb574372a4401b5b132b8e1bf7ead180a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Jun 2024 10:13:33 -0400 Subject: [PATCH 114/115] Revert "Bump supautils version (#989)" This reverts commit 49cc9fe4069db9916451f98ae7b292af4e0e873f. --- Dockerfile | 2 +- ansible/vars.yml | 4 ++-- docker/orioledb/Dockerfile | 2 +- nix/ext/supautils.nix | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 37086d95d..4d7911686 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ ARG hypopg_release=1.3.1 ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.1 +ARG supautils_release=2.2.0 ARG wal_g_release=2.0.1 #################### diff --git a/ansible/vars.yml b/ansible/vars.yml index 97faa1664..81149f748 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -82,8 +82,8 @@ timescaledb_release_checksum: sha256:883638f2e79d25ec88ee58f603f3c81c999b6364cb4 wal2json_release: "2_5" wal2json_release_checksum: sha256:b516653575541cf221b99cf3f8be9b6821f6dbcfc125675c85f35090f824f00e -supautils_release: "2.2.1" -supautils_release_checksum: sha256:1a2d2b8fe604d38921ed9cf3a0d56dd142a274035d0dca17ad21cdc81ddd9569 +supautils_release: "2.2.0" +supautils_release_checksum: sha256:0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5 pljava_release: master pljava_release_checksum: sha256:e99b1c52f7b57f64c8986fe6ea4a6cc09d78e779c1643db060d0ac66c93be8b6 diff --git a/docker/orioledb/Dockerfile b/docker/orioledb/Dockerfile index a17dc39dc..90668f098 100644 --- a/docker/orioledb/Dockerfile +++ b/docker/orioledb/Dockerfile @@ -36,7 +36,7 @@ ARG hypopg_release=1.3.1 ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.1 +ARG supautils_release=2.2.0 ARG wal_g_release=2.0.1 #################### diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index 9a32062b9..eaf772766 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "supautils"; - version = "2.2.1"; + version = "2.2.0"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-wSUEG0at00TPAoHv6+NMzuUE8mfW6fnHH0MNxvBdUiE="; + hash = "sha256-LEMhUM0woTwRcSWOU70izS5eWevoV8nnMUOEID+Nsdc="; }; installPhase = '' From e72391df5917df169a3e8329a82a416580cbdda0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 24 Jun 2024 10:19:55 -0400 Subject: [PATCH 115/115] Revert "chore: sync up go true versions" This reverts commit ed9225dbb46dce3ca6952c4abd04cd3a1d528993. --- ansible-nix/vars.yml | 4 ++-- ansible/vars.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible-nix/vars.yml b/ansible-nix/vars.yml index 81149f748..74e5afe3b 100644 --- a/ansible-nix/vars.yml +++ b/ansible-nix/vars.yml @@ -15,8 +15,8 @@ postgrest_release: "12.0.2" postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 -gotrue_release: 2.153.0 -gotrue_release_checksum: sha1:334483141b8c03f24ebe7ecb24519bbb6b704cf4 +gotrue_release: 2.152.0 +gotrue_release_checksum: sha1:5f220e9f322cc13f93401ab927d32dd0d1f85b40 aws_cli_release: "2.2.7" diff --git a/ansible/vars.yml b/ansible/vars.yml index 81149f748..74e5afe3b 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -15,8 +15,8 @@ postgrest_release: "12.0.2" postgrest_arm_release_checksum: sha1:a08eaa2af548d44b4c8ea61b0223fb7019f5c768 postgrest_x86_release_checksum: sha1:40f65ded06b9de6567fbe2cd7a317196e22dd595 -gotrue_release: 2.153.0 -gotrue_release_checksum: sha1:334483141b8c03f24ebe7ecb24519bbb6b704cf4 +gotrue_release: 2.152.0 +gotrue_release_checksum: sha1:5f220e9f322cc13f93401ab927d32dd0d1f85b40 aws_cli_release: "2.2.7"