test: add cloud test for reproducing timeouts

This uses NixOps wrapper scripts to reproduce
#86.

The steps are:

```
$ net-cloud-deploy
$ net-cloud-reproduce-timeouts

net-cloud-reproduce-timeouts
NOTICE:  Waiting until 10000 requests complete
NOTICE:  Stats: {"request_successes":8882,"request_failures":1118,"last_failure_error":"Timeout was reached"}
NOTICE:  Time taken: 00:01:44.341157
```

Then destroy the cloud setup with `net-cloud-destroy`

Author: steve-chavez

docs/contributing.md

@@ -72,28 +72,19 @@ $ sudo with-gdb -p 1145879
 
 ## Load Testing
 
-This will deploy a client and server on t3a.nano. You must have `default` setup in `.aws/credentials`.
+These are scripts that wrap NixOps to deploy an AWS cloud setup. You must have `default` setup in `.aws/credentials`.
 
 ```bash
-cd nix
-
-nixops create -d pg_net
-
-# will take a while
-nixops deploy -k -d pg_net --allow-reboot --confirm
+net-cloud-deploy
 ```
 
 Then you can connect on the client instance and do requests to the server instance through `pg_net`.
 
 ```bash
-cd nix
-
-nixops ssh -d pg_net client
+net-cloud-ssh
 
 psql -U postgres
 
-create extension pg_net;
-
 select net.http_get('http://server');
 # this the default welcome page of nginx on the server instance
 # "server" is already included to /etc/hosts, so `curl http://server` will give the same result
@@ -103,13 +94,10 @@ select net.http_get('http://server') from generate_series(1,1000);
 # run `top` on another shell(another `nixops ssh -d pg_net client`) to check the worker behavior
 ```
 
-To destroy the instances:
+To destroy the cloud setup:
 
 ```bash
-cd nix
-
-nixops destroy -d pg_net --confirm
-nixops delete -d pg_net
+net-cloud-destroy
 ```
 
 ## Documentation

nix/nixops.nix

@@ -115,16 +115,44 @@ in {
       initialScript = pkgs.writeText "init-sql-script" ''
         create extension pg_net;
 
-        alter system set pg_net.batch_size to 32000;
-
-        select net.worker_restart();
-
         create view pg_net_stats as
         select
           count(*) filter (where error_msg is null) as request_successes,
           count(*) filter (where error_msg is not null) as request_failures,
           (select error_msg from net._http_response where error_msg is not null order by id desc limit 1) as last_failure_error
         from net._http_response;
+
+        create or replace procedure repro_timeouts(number_of_requests int default 10000, url text default 'http://server/post') as ''$''$
+        declare
+          last_id bigint;
+          first_time timestamptz;
+          second_time timestamptz;
+          time_taken interval;
+        begin
+          delete from net._http_response;
+
+          with do_requests as (
+            select
+              net.http_post(url, jsonb_build_object('id', x, 'message', 'payload ' || x), headers:=jsonb_build_object('Content-Type', 'application/json')) as id
+            from generate_series (1, number_of_requests) x
+          )
+          select id, clock_timestamp() into last_id, first_time from do_requests offset number_of_requests - 1;
+
+          commit;
+
+          raise notice 'Waiting until % requests complete', number_of_requests;
+
+          perform net._await_response(last_id);
+
+          select clock_timestamp() into second_time;
+
+          select age(second_time, first_time) into time_taken;
+
+          raise notice 'Stats: %', (select to_json(x) from pg_net_stats x limit 1);
+
+          raise notice 'Time taken: %', time_taken;
+        end;
+        ''$''$ language plpgsql;
       '';
     };
 
@@ -136,22 +164,40 @@ in {
       pkgs.vegeta
       (
         pkgs.writeShellScriptBin "vegeta-bench" ''
+          set -euo pipefail
+
           # rate=0 means maximum rate subject to max-workers
           echo "GET http://server/pathological?status=200" | vegeta attack -rate=0 -duration=1s -max-workers=1 | tee results.bin | vegeta report
         ''
       )
       (
         pkgs.writeShellScriptBin "vegeta-bench-max-requests" ''
+          set -euo pipefail
+
           # rate=0 means maximum rate subject to max-workers
           echo "GET http://server/pathological?status=200" | vegeta attack -rate=0 -duration=10s -max-workers=50 | tee results.bin | vegeta report
         ''
       )
       (
-        pkgs.writeShellScriptBin "net-bench" ''
+        pkgs.writeShellScriptBin "psql-net-bench" ''
+          set -euo pipefail
+
+          psql -U postgres -c "TRUNCATE net._http_response; TRUNCATE net.http_request_queue;"
+          psql -U postgres -c "alter system set pg_net.batch_size to 32000;" # this just a high number
+          psql -U postgres -c "select net.worker_restart();"
           psql -U postgres -c "truncate net._http_response;"
           psql -U postgres -c "select net.http_get('http://server/pathological?status=200') from generate_series(1, 400);" > /dev/null
           sleep 2
           psql -U postgres -c "select * from pg_net_stats;"
+          psql -U postgres -c "alter system reset pg_net.batch_size;"
+          psql -U postgres -c "select net.worker_restart();"
+        ''
+      )
+      (
+        pkgs.writeShellScriptBin "psql-reproduce-timeouts" ''
+          set -euo pipefail
+
+          psql -U postgres -c "call repro_timeouts();"
         ''
       )
     ];

nix/nixopsScripts.nix

@@ -0,0 +1,59 @@
+{ nixops_unstable_minimal, writeShellScriptBin } :
+
+let
+  nixops = nixops_unstable_minimal.withPlugins (ps: [ ps.nixops-aws ]);
+  nixopsBin = "${nixops}/bin/nixops";
+  nixopsDeploy =
+    writeShellScriptBin "net-cloud-deploy"
+      ''
+        set -euo pipefail
+
+        cd nix
+
+        set +e && ${nixopsBin} info -d pg_net > /dev/null 2> /dev/null
+        info=$? && set -e
+
+        if test $info -eq 1
+        then
+          echo "Creating deployment..."
+          ${nixopsBin} create -d pg_net
+        fi
+
+        ${nixopsBin} deploy -k -d pg_net --allow-reboot --confirm
+      '';
+  nixopsSSH =
+    writeShellScriptBin ("net-cloud-ssh")
+      ''
+        set -euo pipefail
+
+        cd nix
+
+        ${nixopsBin} ssh -d pg_net client
+      '';
+  nixopsReproTimeouts =
+    writeShellScriptBin ("net-cloud-reproduce-timeouts")
+      ''
+        set -euo pipefail
+
+        cd nix
+
+        ${nixopsBin} ssh -d pg_net client psql-reproduce-timeouts
+      '';
+  nixopsDestroy =
+    writeShellScriptBin ("net-cloud-destroy")
+      ''
+        set -euo pipefail
+
+        cd nix
+
+        ${nixopsBin} destroy -d pg_net --confirm
+
+        ${nixopsBin} delete -d pg_net
+      '';
+in
+[
+  nixopsDeploy
+  nixopsSSH
+  nixopsReproTimeouts
+  nixopsDestroy
+]

shell.nix

@@ -17,6 +17,7 @@ mkShell {
       extAll = map (x: callPackage ./nix/pgScript.nix { postgresql = pgWithExt { pg = x;}; }) supportedPgVersions;
       nginxCustom = callPackage ./nix/nginxCustom.nix {};
       gdbScript = callPackage ./nix/gdbScript.nix {};
+      nixopsScripts = callPackage ./nix/nixopsScripts.nix {};
       pythonDeps = with python3Packages; [
         pytest
         psycopg2
@@ -30,8 +31,7 @@ mkShell {
       format.do format.doCheck
       nginxCustom.nginxScript
       gdbScript
-      (pkgs.nixops_unstable_minimal.withPlugins (ps: [ ps.nixops-aws ]))
-    ];
+    ] ++ nixopsScripts;
   shellHook = ''
     export HISTFILE=.history
   '';