database/bulk-delete: add env vars and improve test

Author: haraldschilly

src/packages/database/postgres/bulk-delete.test.ts

@@ -19,7 +19,7 @@ describe("bulk delete", () => {
   test("deleting projects", async () => {
     const p = getPool();
     const project_id = uuid();
-    const N = 2000;
+    const N = 100000;
 
     // extra entry, which has to remain
     const other = uuid();
@@ -28,12 +28,12 @@ describe("bulk delete", () => {
       [other, uuid(), new Date()],
     );
 
-    for (let i = 0; i < N; i++) {
-      await p.query(
-        "INSERT INTO project_log (id, project_id, time) VALUES($1::UUID, $2::UUID, $3::TIMESTAMP)",
-        [uuid(), project_id, new Date()],
-      );
-    }
+    await p.query(
+      `INSERT INTO project_log (id, project_id, time)
+      SELECT gen_random_uuid(), $1::UUID, NOW() - interval '1 second' * g.n
+      FROM generate_series(1, $2) AS g(n)`,
+      [project_id, N],
+    );
 
     const num1 = await p.query(
       "SELECT COUNT(*)::INT as num FROM project_log WHERE project_id = $1",
@@ -45,16 +45,15 @@ describe("bulk delete", () => {
       table: "project_log",
       field: "project_id",
       value: project_id,
-      limit: 128,
     });
 
     // if this ever fails, the "ret.rowCount" value is inaccurate.
     // This must be replaced by "RETURNING 1" in the the query and a "SELECT COUNT(*) ..." and so.
     // (and not only here, but everywhere in the code base)
     expect(res.rowsDeleted).toEqual(N);
-    expect(res.durationS).toBeGreaterThan(0.01);
-    expect(res.totalPgTimeS).toBeGreaterThan(0.001);
-    expect(res.totalWaitS).toBeGreaterThan(0.001);
+    expect(res.durationS).toBeGreaterThan(0.1);
+    expect(res.totalPgTimeS).toBeGreaterThan(0.1);
+    expect(res.totalWaitS).toBeGreaterThan(0.1);
     expect((res.totalPgTimeS * 10) / res.totalWaitS).toBeGreaterThan(0.5);
 
     const num2 = await p.query(

src/packages/database/postgres/bulk-delete.ts

@@ -1,14 +1,30 @@
 import { escapeIdentifier } from "pg";
 
+import getLogger from "@cocalc/backend/logger";
+import { envToInt } from "@cocalc/backend/misc/env-to-number";
 import getPool from "@cocalc/database/pool";
 import { SCHEMA } from "@cocalc/util/schema";
 
+const log = getLogger("db:bulk-delete");
+const D = log.debug;
+
 type Field =
   | "project_id"
   | "account_id"
   | "target_project_id"
   | "source_project_id";
 
+const MAX_UTIL_PCT = envToInt("COCALC_DB_BULK_DELETE_MAX_UTIL_PCT", 10);
+// adjust the time limits: by default, we aim to keep the operation between 0.1 and 0.2 secs
+const MAX_TIME_TARGET_MS = envToInt(
+  "COCALC_DB_BULK_DELETE_MAX_TIME_TARGET_MS",
+  100,
+);
+const MAX_TARGET_S = MAX_TIME_TARGET_MS / 1000;
+const MIN_TARGET_S = MAX_TARGET_S / 2;
+const DEFAULT_LIMIT = envToInt("COCALC_DB_BULK_DELETE_DEFAULT_LIMIT", 16);
+const MAX_LIMIT = envToInt("COCALC_DB_BULK_DELETE_MAX_LIMIT", 32768);
+
 interface Opts {
   table: string; // e.g. project_log, etc.
   field: Field; // for now, we only support a few
@@ -38,8 +54,8 @@ WHERE ${ID} IN (
 }
 
 export async function bulkDelete(opts: Opts): Ret {
-  const { table, field, value, id = "id", maxUtilPct = 10 } = opts;
-  let { limit = 1024 } = opts;
+  const { table, field, value, id = "id", maxUtilPct = MAX_UTIL_PCT } = opts;
+  let { limit = DEFAULT_LIMIT } = opts;
   // assert table name is a key in SCHEMA
   if (!(table in SCHEMA)) {
     throw new Error(`table ${table} does not exist`);
@@ -63,18 +79,16 @@ export async function bulkDelete(opts: Opts): Ret {
     rowsDeleted += ret.rowCount ?? 0;
     totalPgTimeS += dt;
 
-    // adjust the limit: we aim to keep the operation between 0.1 and 0.2 secs
-    const next = dt > 0.2 ? limit / 2 : dt < 0.1 ? limit * 2 : limit;
-    limit = Math.max(1, Math.min(32768, Math.round(next)));
+    const next =
+      dt > MAX_TARGET_S ? limit / 2 : dt < MIN_TARGET_S ? limit * 2 : limit;
+    limit = Math.max(1, Math.min(MAX_LIMIT, Math.round(next)));
 
     // wait for a bit, but not more than 1 second ~ this aims for a max utilization of 10%
     const waitS = Math.min(1, dt * ((100 - maxUtilPct) / maxUtilPct));
     await new Promise((done) => setTimeout(done, 1000 * waitS));
     totalWaitS += waitS;
 
-    // console.log(
-    //   `deleted ${ret.rowCount} | dt=${dt} | wait=${waitS} | limit=${limit}`,
-    // );
+    D(`deleted ${ret.rowCount} | dt=${dt} | wait=${waitS} | limit=${limit}`);
 
     if (ret.rowCount === 0) break;
   }