Run the three scripts nobody had run, and rewrite the handoff around it

A fourth pass, which was not a reading. The two year retention on
door_events could not run at all: the append-only trigger refused its own
specification's delete, so the table would have grown forever and three
readings of the code had not noticed. door_events now carries two triggers
rather than one, and the window lives in the trigger, which makes it the
authority on what may go. audit_log is unchanged and still refuses
everything.

scripts/backup.sh joined $PWD to a BACKUP_DIR that was already absolute and
wrote the caddy archive into this repository. Both defects were found by
typing the command.

The nightly job, the restore, and migrate across three files turned out to
be sound, and are now proved rather than assumed: planted rows on both sides
of every line the nightly draws, a dump taken and restored into a scratch
database, and three migrations applied in order then skipped on a second run.
Postgres notices no longer print as a raw object on every deploy.

HANDOFF section 6 is rewritten. Thirty nine defects across four passes is too
long a list to be read, so it is now the patterns that kept coming back, the
four things that were proved not to be defects, and the list of what has
still never been audited. The blow by blow is in the commits.

Author: virgilvox

HANDOFF.md

@@ -58,15 +58,34 @@ describe('the record', () => {
     assert.equal((await sql`select id from audit_log`).length, 1)
   })
 
-  test('door_events refuses update and delete, and names itself rather than audit_log', async () => {
+  test('door_events refuses a change, and names itself rather than audit_log', async () => {
     await sql`insert into door_events (controller_id, kind) values ('openaccess', 'entry')`
 
     await assert.rejects(
-      () => sql`delete from door_events`,
+      () => sql`update door_events set kind = 'denied'`,
       (error: Error) => error.message.includes('door_events is append only'),
     )
   })
 
+  test('door_events keeps two years, and the nightly job can take what is past it', async () => {
+    await sql`
+      insert into door_events (controller_id, kind, at) values
+        ('openaccess', 'entry', now() - interval '3 years'),
+        ('openaccess', 'entry', now() - interval '1 year')`
+
+    // The retention in scripts/nightly.sql is the only thing that removes a
+    // door event. A trigger refusing every delete makes it impossible to run
+    // and the table grow forever, which is what it did until somebody ran it.
+    await sql`delete from door_events where at < now() - interval '2 years'`
+    assert.equal((await sql`select id from door_events`).length, 1)
+
+    // And nothing else can, whatever it asks for.
+    await assert.rejects(
+      () => sql`delete from door_events`,
+      (error: Error) => error.message.includes('this table keeps two years'),
+    )
+  })
+
   test('every privileged route leaves exactly one row behind', async () => {
     const admin = await makeMember({ roles: ['admin', 'accountant', 'instructor'] })
     const subject = await makeMember()

api/src/audit.test.ts

@@ -1,6 +1,7 @@
 import postgres from 'postgres'
 
 import { config } from './config.ts'
+import { log } from './log.ts'
 
 /**
  * Columns are snake_case in SQL and camelCase in JavaScript, translated here so
@@ -12,6 +13,9 @@ import { config } from './config.ts'
  */
 export const sql = postgres(config.databaseUrl, {
   transform: { column: { to: postgres.fromCamel, from: postgres.toCamel } },
+  // Postgres notices print as a raw object on stderr otherwise, which in a log
+  // of one JSON line per event reads as something having gone wrong.
+  onnotice: (notice) => log({ evt: 'db_notice', message: notice.message ?? String(notice) }),
 })
 
 export type Sql = typeof sql

api/src/db.ts

@@ -157,11 +157,33 @@ begin
 end;
 $$ language plpgsql;
 
-create trigger audit_log_append_only   before update or delete on audit_log
+-- audit_log is kept forever, so nothing may change it and nothing may remove it.
+create trigger audit_log_append_only before update or delete on audit_log
   for each row execute function refuse_change();
-create trigger door_events_append_only before update or delete on door_events
+
+-- door_events is kept for two years, so it needs one thing audit_log does not:
+-- the nightly job has to be able to remove what is past that. A trigger that
+-- refuses every delete makes the retention in scripts/nightly.sql impossible to
+-- run and the table grow forever, which is what it did until somebody ran it.
+--
+-- The window lives here rather than in the job, so this is the authority on
+-- what may go. A job asking for a shorter one meets this and says so, rather
+-- than quietly taking more than it should.
+create function refuse_early_delete() returns trigger as $$
+begin
+  raise exception
+    'door event % is from % and this table keeps two years, so it was left alone. Only the nightly retention removes door events.',
+    OLD.id, OLD.at::date;
+end;
+$$ language plpgsql;
+
+create trigger door_events_no_change before update on door_events
   for each row execute function refuse_change();
 
+create trigger door_events_retention before delete on door_events
+  for each row when (OLD.at > now() - interval '2 years')
+  execute function refuse_early_delete();
+
 -- 4. The door ---------------------------------------------------------------
 
 -- Current state is upserted, never appended. The legacy system wrote a status

migrations/001_init.sql

@@ -13,13 +13,17 @@ set -eu
 stamp=$(date -u +%Y%m%dT%H%M%SZ)
 out=${BACKUP_DIR:-backups}
 mkdir -p "$out"
+# Resolved, because the docker mount below needs an absolute path and joining
+# $PWD to an already absolute BACKUP_DIR writes the archive somewhere nobody
+# asked for, inside this repository.
+here=$(cd "$out" && pwd)
 
 docker compose exec -T db pg_dump -U hsl -Fc hsl > "$out/hsl-$stamp.dump"
 echo "wrote $out/hsl-$stamp.dump"
 
 # Caddy's volume holds the TLS certificate and the ACME account key. Losing it
 # costs a new certificate rather than data, and it is cheap to keep.
-docker run --rm -v hsl-web_caddy:/data -v "$PWD/$out":/out alpine \
+docker run --rm -v hsl-web_caddy:/data -v "$here":/out alpine \
   tar czf "/out/caddy-$stamp.tar.gz" -C /data . 2>/dev/null || true
 
 find "$out" -name 'hsl-*.dump' -mtime +30 -delete

scripts/backup.sh

@@ -19,7 +19,10 @@ if (url === undefined || url === '') {
 }
 
 const directory = new URL('../migrations/', import.meta.url)
-const sql = postgres(url)
+// `create table if not exists` below raises a notice every run after the
+// first, and postgres.js prints one as a raw object, which on a deploy reads as
+// something having gone wrong.
+const sql = postgres(url, { onnotice: () => undefined })
 
 await sql`
   create table if not exists schema_migrations (