nix: add new loadtest for errors

.github/workflows/test.yaml

@@ -120,7 +120,7 @@ jobs:
   loadtest:
     strategy:
       matrix:
-        kind: ['mixed', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
+        kind: ['mixed', 'errors', 'jwt-hs', 'jwt-hs-cache', 'jwt-hs-cache-worst', 'jwt-rsa', 'jwt-rsa-cache', 'jwt-rsa-cache-worst']
     name: Loadtest
     runs-on: ubuntu-24.04
     steps:

nix/tools/loadtest.nix

@@ -45,7 +45,7 @@ let
           "ARG_OPTIONAL_SINGLE([testdir], [t], [Directory to load tests and fixtures from], [./test/load])"
           "ARG_OPTIONAL_SINGLE([kind], [k], [Kind of loadtest], [mixed])"
           "ARG_OPTIONAL_SINGLE([method],, [HTTP method used for the jwt loadtests], [OPTIONS])"
-          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])"
+          "ARG_TYPE_GROUP_SET([KIND], [KIND], [kind], [mixed,errors,jwt-hs,jwt-hs-cache,jwt-hs-cache-worst,jwt-rsa,jwt-rsa-cache,jwt-rsa-cache-worst])"
           "ARG_TYPE_GROUP_SET([METHOD], [METHOD], [method], [OPTIONS,GET])"
           "ARG_OPTIONAL_SINGLE([monitor], [m], [Monitoring file], [./loadtest/result.csv])"
           "ARG_LEFTOVERS([additional vegeta arguments])"
@@ -152,11 +152,20 @@ let
             sh -c "cd \"$_arg_testdir\" && \
             ${runner} -targets targets.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
             ;;
+
+          # here we sleep purposefully to check how much memory does the schema cache consume in the final report
+          errors)
+            # shellcheck disable=SC2145
+            ${withTools.withPg} -f "$_arg_testdir"/errors.sql \
+            ${withTools.withPgrst} --timeout 2 --sleep 5 -m "$_arg_monitor" \
+            sh -c "cd \"$_arg_testdir\" && \
+            ${runner} -targets errors.http -output \"$abs_output\" \"''${_arg_leftovers[@]}\""
+            ;;
         esac
 
         ${vegeta}/bin/vegeta report -type=text "$_arg_output"
 
-        if [ "$_arg_kind" != "mixed" ]; then
+        if [ "$_arg_kind" != "errors" ]; then
           # fail in case 401 happened on jwt loadtests
           unauthorized_count="$(${vegeta}/bin/vegeta report -type=json "$_arg_output" \
             | ${jq}/bin/jq -r '.status_codes["401"] // 0')"

nix/tools/withTools.nix

@@ -132,9 +132,11 @@ let
         fi
 
         if test "$_arg_fixtures"; then
-          log "Loading fixtures under the postgres role..."
+          load_start=$SECONDS
+          >&2 printf "${commandName}: Loading fixtures under the postgres role..."
           psql -U postgres -v PGUSER="$PGUSER" -v ON_ERROR_STOP=1 -f "$_arg_fixtures" >> "$setuplog"
-          log "Done. Running command..."
+          load_end=$((SECONDS - load_start))
+          >&2 printf " done in %ss. Running command...\n" "$load_end"
         fi
 
         ("$_arg_command" "''${_arg_leftovers[@]}")
@@ -361,6 +363,8 @@ let
             "ARG_POSITIONAL_SINGLE([command], [Command to run])"
             "ARG_LEFTOVERS([command arguments])"
             "ARG_OPTIONAL_SINGLE([monitor], [m], [Enable CPU and memory monitoring of the PostgREST process and output to the designated file as markdown])"
+            "ARG_OPTIONAL_SINGLE([timeout], [t], [Maximum time to wait for PostgREST to be ready], [5])"
+            "ARG_OPTIONAL_SINGLE([sleep], [s],   [Sleep time after PostgREST is ready, this is useful for monitoring], [0])"
             "ARG_USE_ENV([PGRST_CMD], [], [PostgREST executable to run])"
           ];
         positionalCompletion = "_command";
@@ -412,17 +416,23 @@ let
         }
         trap cleanup EXIT
 
-        timeout -s TERM 5 ${waitForPgrstReady} || {
+        wait_start=$SECONDS
+        timeout -s TERM "$_arg_timeout" ${waitForPgrstReady} || {
           echo "timed out, output:"
           cat "$tmpdir"/run.log
           exit 1
         }
-        echo "done."
+        wait_duration=$((SECONDS - wait_start))
+        printf "done in %ss.\n" "$wait_duration"
 
         if [[ -n "$_arg_monitor" ]]; then
           ${monitorPid} "$pid" > "$_arg_monitor" &
         fi
 
+        if [[ -n "$_arg_sleep" ]]; then
+          sleep "$_arg_sleep"
+        fi
+
         ("$_arg_command" "''${_arg_leftovers[@]}")
       '';
 

test/load/errors.http

@@ -0,0 +1,15 @@
+# Misspelled relations
+GET http://postgrest/actoxs?actor=eq.1
+Prefer: tx=commit
+
+# Misspelled relations on embeds
+GET http://postgrest/actors?select=*,rolws(*,films(*))
+Prefer: tx=commit
+
+# Misspelled function names
+GET http://postgrest/rpc/call_em_x?name=John
+Prefer: tx=commit
+
+# Permission denied errors
+GET http://postgrest/actors_1
+Prefer: tx=commit

test/load/errors.sql

@@ -0,0 +1,7 @@
+\ir fixtures.sql
+
+SELECT format('CREATE TABLE test.actors_%s ();', n)
+FROM generate_series(1, 20000) n
+\gexec
+
+-- TODO add many function for fuzzy search (somehow this is making the loadtest start slow)