feat: support multiple versions of the http extension (#1664)

* feat: support multiple versions of the http extension

Build multiple versions of the http extension on different PostgreSQL versions.
Add test for the extensions and their upgrade on PostgreSQL 15 and 17.

* feat: http multiversion extension and pg_regress tests

* chore: bump to test

* tests: local mock http server for http regress tests

* fix: a better way to handle http server for tests

* chore: bump to release

---------

Co-authored-by: Sam Rose <[email protected]>

Author: jfroche

ansible/vars.yml

@@ -1,37 +1,38 @@
+---
 supabase_internal: true
 ebssurrogate_mode: true
 async_mode: true
 
 postgres_major:
   - "15"
   - "17"
-  - "orioledb-17"
+  - orioledb-17
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.5.1.029"
-  postgres17: "17.6.1.008"
-  postgres15: "15.14.1.008"
+  postgresorioledb-17: 17.5.1.030-orioledb
+  postgres17: 17.6.1.009
+  postgres15: 15.14.1.009
 
 # Non Postgres Extensions
-pgbouncer_release: "1.19.0"
+pgbouncer_release: 1.19.0
 pgbouncer_release_checksum: sha256:af0b05e97d0e1fd9ad45fe00ea6d2a934c63075f67f7e2ccef2ca59e3d8ce682
 
 # The checksum can be found under "Assets", in the GitHub release page for each version.
 # The binaries used are: ubuntu-aarch64 and linux-static.
 # https://github.com/PostgREST/postgrest/releases
-postgrest_release: "13.0.5"
+postgrest_release: 13.0.5
 postgrest_arm_release_checksum: sha256:7b4eafdaf76bc43b57f603109d460a838f89f949adccd02f452ca339f9a0a0d4
 postgrest_x86_release_checksum: sha256:05be2bd48abee6c1691fc7c5d005023466c6989e41a4fc7d1302b8212adb88b5
 
 gotrue_release: 2.179.0
 gotrue_release_checksum: sha1:e985fce00b2720b747e6a04420910015c4967121
 
-aws_cli_release: "2.23.11"
+aws_cli_release: 2.23.11
 
 salt_minion_version: 3007
 
-golang_version: "1.22.11"
+golang_version: 1.22.11
 golang_version_checksum:
   arm64: sha256:0fc88d966d33896384fbde56e9a8d80a305dc17a9f48f1832e061724b1719991
   amd64: sha256:9ebfcab26801fa4cf0627c6439db7a4da4d3c6766142a3dd83508240e4f21031
@@ -52,10 +53,10 @@ postgres_exporter_release_checksum:
   arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0
   amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84
 
-adminapi_release: 0.92.1
-adminmgr_release: 0.32.1
+adminapi_release: "0.92.1"
+adminmgr_release: "0.32.1"
 supabase_admin_agent_release: 1.4.38
 supabase_admin_agent_splay: 30
 
-vector_x86_deb: "https://packages.timber.io/vector/0.48.X/vector_0.48.0-1_amd64.deb"
-vector_arm_deb: "https://packages.timber.io/vector/0.48.X/vector_0.48.0-1_arm64.deb"
+vector_x86_deb: https://packages.timber.io/vector/0.48.X/vector_0.48.0-1_amd64.deb
+vector_arm_deb: https://packages.timber.io/vector/0.48.X/vector_0.48.0-1_arm64.deb

nix/checks.nix

@@ -163,11 +163,43 @@
                   which
                   getkey-script
                   supabase-groonga
+                  python3
+                  netcat
                 ];
               }
               ''
                 set -e
 
+                # Start HTTP mock server for http extension tests
+                # Use a build-specific directory for coordination
+                BUILD_TMP=$(mktemp -d)
+                HTTP_MOCK_PORT_FILE="$BUILD_TMP/http-mock-port"
+
+                echo "Starting HTTP mock server (will find free port)..."
+                HTTP_MOCK_PORT_FILE="$HTTP_MOCK_PORT_FILE" ${pkgs.python3}/bin/python3 ${./tests/http-mock-server.py} &
+                HTTP_MOCK_PID=$!
+
+                # Clean up on exit
+                trap "kill $HTTP_MOCK_PID 2>/dev/null || true; rm -rf '$BUILD_TMP'" EXIT
+
+                # Wait for server to start and write port file
+                for i in {1..10}; do
+                  if [ -f "$HTTP_MOCK_PORT_FILE" ]; then
+                    HTTP_MOCK_PORT=$(cat "$HTTP_MOCK_PORT_FILE")
+                    echo "HTTP mock server started on port $HTTP_MOCK_PORT"
+                    break
+                  fi
+                  sleep 1
+                done
+
+                if [ ! -f "$HTTP_MOCK_PORT_FILE" ]; then
+                  echo "Failed to start HTTP mock server"
+                  exit 1
+                fi
+
+                # Export the port for use in SQL tests
+                export HTTP_MOCK_PORT
+
                 #First we need to create a generic pg cluster for pgtap tests and run those
                 export GRN_PLUGINS_DIR=${pkgs.supabase-groonga}/lib/groonga/plugins
                 PGTAP_CLUSTER=$(mktemp -d)
@@ -228,6 +260,13 @@
                   pg_ctl -D "$PGTAP_CLUSTER" stop
                   exit 1
                 fi
+
+                # Create a table to store test configuration
+                psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -c "
+                  CREATE TABLE IF NOT EXISTS test_config (key TEXT PRIMARY KEY, value TEXT);
+                  INSERT INTO test_config (key, value) VALUES ('http_mock_port', '$HTTP_MOCK_PORT')
+                  ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;
+                "
                 SORTED_DIR=$(mktemp -d)
                 for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do
                   psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -f "${./tests/sql}/$t.sql" || true
@@ -261,6 +300,13 @@
                   exit 1
                 fi
 
+                # Create a table to store test configuration for pg_regress tests
+                psql -p ${pgPort} -h ${self.supabase.defaults.host} --no-password --username=supabase_admin -d postgres -c "
+                  CREATE TABLE IF NOT EXISTS test_config (key TEXT PRIMARY KEY, value TEXT);
+                  INSERT INTO test_config (key, value) VALUES ('http_mock_port', '$HTTP_MOCK_PORT')
+                  ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;
+                "
+
                 mkdir -p $out/regression_output
                 if ! pg_regress \
                   --use-existing \

nix/ext/pgsql-http.nix

@@ -1,39 +1,109 @@
 {
+  pkgs,
   lib,
   stdenv,
   fetchFromGitHub,
-  curl,
   postgresql,
+  curl,
 }:
+let
+  pname = "http";
 
-stdenv.mkDerivation rec {
-  pname = "pgsql-http";
-  version = "1.6.1";
+  # Load version configuration from external file
+  allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname};
 
-  buildInputs = [
-    curl
-    postgresql
-  ];
+  # Filter versions compatible with current PostgreSQL version
+  supportedVersions = lib.filterAttrs (
+    _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
+  ) allVersions;
 
-  src = fetchFromGitHub {
-    owner = "pramsey";
-    repo = pname;
-    rev = "refs/tags/v${version}";
-    hash = "sha256-C8eqi0q1dnshUAZjIsZFwa5FTYc7vmATF3vv2CReWPM=";
-  };
+  # Derived version information
+  versions = lib.naturalSort (lib.attrNames supportedVersions);
+  latestVersion = lib.last versions;
+  numberOfVersions = builtins.length versions;
+  packages = builtins.attrValues (
+    lib.mapAttrs (name: value: build name value.hash) supportedVersions
+  );
+
+  # Build function for individual versions
+  build =
+    version: hash:
+    stdenv.mkDerivation rec {
+      inherit pname version;
+
+      buildInputs = [
+        curl
+        postgresql
+      ];
+
+      src = fetchFromGitHub {
+        owner = "pramsey";
+        repo = "pgsql-http";
+        rev = "refs/tags/v${version}";
+        inherit hash;
+      };
+
+      installPhase = ''
+        runHook preInstall
 
-  installPhase = ''
-    mkdir -p $out/{lib,share/postgresql/extension}
+        mkdir -p $out/{lib,share/postgresql/extension}
+
+        # Install versioned library
+        install -Dm755 ${pname}${postgresql.dlSuffix} $out/lib/${pname}--${version}${postgresql.dlSuffix}
+
+        cp ${pname}--${version}.sql $out/share/postgresql/extension/${pname}--${version}.sql
+
+        # Create versioned control file with modified module path
+        sed -e "/^default_version =/d" \
+            -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}'|" \
+          ${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control
+
+        # For the latest version, create default control file and symlink and copy SQL upgrade scripts
+        if [[ "${version}" == "${latestVersion}" ]]; then
+          {
+            echo "default_version = '${version}'"
+            cat $out/share/postgresql/extension/${pname}--${version}.control
+          } > $out/share/postgresql/extension/${pname}.control
+          ln -sfn ${pname}--${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
+          cp *.sql $out/share/postgresql/extension
+        fi
+
+        runHook postInstall
+      '';
+
+      meta = with lib; {
+        description = "HTTP client for Postgres";
+        homepage = "https://github.com/pramsey/${pname}";
+        inherit (postgresql.meta) platforms;
+        license = licenses.postgresql;
+      };
+    };
+in
+pkgs.buildEnv {
+  name = pname;
+  paths = packages;
+
+  pathsToLink = [
+    "/lib"
+    "/share/postgresql/extension"
+  ];
+  postBuild = ''
+    # Verify all expected library files are present
+    expectedFiles=${toString (numberOfVersions + 1)}
+    actualFiles=$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)
 
-    cp *${postgresql.dlSuffix}      $out/lib
-    cp *.sql     $out/share/postgresql/extension
-    cp *.control $out/share/postgresql/extension
+    if [[ "$actualFiles" != "$expectedFiles" ]]; then
+      echo "Error: Expected $expectedFiles library files, found $actualFiles"
+      echo "Files found:"
+      ls -la $out/lib/${pname}*${postgresql.dlSuffix} || true
+      exit 1
+    fi
   '';
 
-  meta = with lib; {
-    description = "HTTP client for Postgres";
-    homepage = "https://github.com/pramsey/${pname}";
-    platforms = postgresql.meta.platforms;
-    license = licenses.postgresql;
+  passthru = {
+    inherit versions numberOfVersions;
+    pname = "${pname}-all";
+    version =
+      "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
   };
 }