feat: allow reconfiguring pg_net w/o a db restart

Otherwise changing a pg_net config requires a restart of the whole
database cluster.

Author: steve-chavez

README.md

@@ -137,32 +137,42 @@ create extension pg_net;
 
 ---
 
-# Extension Configuration:
+# Extension Configuration
 
 the extension creates 3 configurable variables:
 
 1. **pg_net.batch_size** _(default: 200)_: An integer that limits the max number of rows that the extension will process from _`net.http_request_queue`_ during each read
 2. **pg_net.ttl** _(default: 6 hours)_: An interval that defines the max time a row in the _`net.http_response`_ will live before being deleted
 3. **pg_net.database_name** _(default: 'postgres')_: A string that defines which database the extension is applied to
 
-All these variables can be viewed with the following command:
+All these variables can be viewed with the following commands:
 ```sql
-select  * from pg_settings WHERE name LIKE 'pg_net%'
+show pg_net.batch_size;
+show pg_net.ttl;
+show pg_net.database_name;
 ```
 
-The postgres.conf file can be found with the following SQL command:
-```sql
-SHOW config_file;
+You can change these by editing the `postgresql.conf` file (find it with `SHOW config_file;`) or with `ALTER SYSTEM`:
+
+```
+alter system set pg_net.ttl to '1 hour'
+alter system set pg_net.batch_size to 500;
+```
+
+Then, reload the settings and restart the `pg_net` background worker with:
+
 ```
+select net.worker_restart();
+```
+
+Note that doing `ALTER SYSTEM` requires SUPERUSER but on PostgreSQL >= 15, you can do:
 
-You can change the variables by adding any of the following line to your postgres.conf file
 ```
-pg_net.batch_size = <new integer>
-pg_net.ttl = '<new ttl interval>'
-pg_net.database_name = '<database name>'
+grant alter system on parameter pg_net.ttl to <role>;
+grant alter system on parameter pg_net.batch_size to <role>;
 ```
 
-After saving the file, you can execute `SELECT pg_reload_conf()` to update _postgres.conf_ for your database. If the extension does not respond to the update, it may be necessary to restart your database.
+To allow regular users to update `pg_net` settings.
 
 # Requests API
 

nix/pgScript.nix

@@ -22,7 +22,7 @@ let
 
     options="-F -c listen_addresses=\"\" -c log_min_messages=${logMin} -k $PGDATA"
 
-    ext_options="-c shared_preload_libraries=\"pg_net\" -c pg_net.ttl=\"4 seconds\""
+    ext_options="-c shared_preload_libraries=\"pg_net\""
 
     pg_ctl start -o "$options" -o "$ext_options"
 

sql/pg_net.sql

@@ -353,5 +353,14 @@ begin
 end;
 $$;
 
+create or replace function net.worker_restart() returns bool as $$
+  select pg_reload_conf();
+  select pg_terminate_backend(pid)
+  from pg_stat_activity
+  where backend_type ilike '%pg_net%';
+$$
+security definer
+language sql;
+
 grant all on schema net to postgres;
 grant all on all tables in schema net to postgres;

src/worker.c

@@ -36,7 +36,7 @@ PG_MODULE_MAGIC;
 
 static char *ttl = "6 hours";
 static int batch_size = 500;
-char* database_name = "postgres";
+static char* database_name = "postgres";
 
 void _PG_init(void);
 PGDLLEXPORT void worker_main(Datum main_arg) pg_attribute_noreturn();
@@ -181,6 +181,8 @@ worker_main(Datum main_arg)
 
 	BackgroundWorkerInitializeConnection(database_name, NULL, 0);
 
+	elog(INFO, "pg_net worker started with a config of: pg_net.ttl=%s, pg_net.batch_size=%d, pg_net.database_name=%s", ttl, batch_size, database_name);
+
 	while (!got_sigterm)
 	{
 		WaitLatch(&MyProc->procLatch,
@@ -430,7 +432,8 @@ worker_main(Datum main_arg)
 		CommitTransactionCommand();
 	}
 
-	proc_exit(0);
+	// causing a failure on exit will make the postmaster process restart the bg worker
+	proc_exit(EXIT_FAILURE);
 }
 
 void
@@ -454,7 +457,7 @@ _PG_init(void)
 	snprintf(worker.bgw_library_name, BGW_MAXLEN, "pg_net");
 	snprintf(worker.bgw_function_name, BGW_MAXLEN, "worker_main");
 	snprintf(worker.bgw_name, BGW_MAXLEN, "pg_net " EXTVERSION " worker");
-	worker.bgw_restart_time = 32;
+	worker.bgw_restart_time = 1;
 	worker.bgw_main_arg = (Datum) 0;
 	worker.bgw_notify_pid = 0;
 	RegisterBackgroundWorker(&worker);
@@ -477,7 +480,7 @@ _PG_init(void)
 								 NULL, NULL, NULL);
 
 	DefineCustomStringVariable("pg_net.database_name",
-								"database where the pg_net worker is connected",
+								"Database where pg_net tables are located",
 								NULL,
 								&database_name,
 								"postgres",

test/test_http_requests_deleted_after_ttl.py

@@ -5,6 +5,15 @@
 
 def test_http_requests_deleted_after_ttl(sess):
     """Check that http requests are deleted within a few seconds of their ttl"""
+
+    # commit to avoid "cannot run inside a transaction block" error, see https://stackoverflow.com/a/75757326/4692662
+    sess.execute(text("COMMIT"))
+    sess.execute(text("alter system set pg_net.ttl to '4 seconds'"))
+    sess.execute(text("select net.worker_restart()"))
+
+    # bg worker restarts after 1 second
+    time.sleep(1)
+
     # Create a request
     (request_id,) = sess.execute(text(
         """
@@ -42,3 +51,10 @@ def test_http_requests_deleted_after_ttl(sess):
     ).fetchone()
     # TODO an ERROR status doesn't seem correct here
     assert response[0] == "ERROR"
+
+    sess.execute(text("COMMIT"))
+    sess.execute(text("alter system reset pg_net.ttl"))
+    sess.execute(text("select net.worker_restart()"))
+
+    # wait until the worker has restarted to not affect other tests
+    time.sleep(1)

test/test_worker_error.py

@@ -16,12 +16,11 @@ def test_success_when_worker_is_up(sess):
 def test_http_get_error_when_worker_down(sess):
     """net.http_get returns an error when pg background worker is down"""
 
-    # kill the bg worker manually
     sess.execute(text("""
-        select pg_terminate_backend((select pid from pg_stat_activity where backend_type ilike '%pg_net%'));
+        select net.worker_restart();
     """))
 
-    time.sleep(1);
+    time.sleep(0.1)
 
     with pytest.raises(Exception) as execinfo:
         res = sess.execute(text(