btrfs snapshots: api for *users* create and delete snapshots

Author: williamstein

src/packages/backend/conat/test/files/file-server.test.ts

@@ -115,6 +115,17 @@ describe("create basic mocked file server and test it out", () => {
         project_id: string;
         name: string;
       }): Promise<void> => {},
+
+      updateSnapshots: async (_opts: {
+        project_id: string;
+        counts?: {
+          frequent?: number;
+          daily?: number;
+          weekly?: number;
+          monthly?: number;
+        };
+        limit?: number;
+      }): Promise<void> => {},
     });
   });
 

src/packages/backend/conat/test/project/jupyter/run-code.test.ts

@@ -266,7 +266,11 @@ describe("create mocked jupyter runner that does failover to backend output mana
   it("starts code running then closes the client, which causes output to have to be placed in the handler instead.", async () => {
     await client.run(cells);
     client.close();
-    await wait({ until: () => handler.messages.length >= 3 });
+    await wait({
+      until: () => {
+        return handler.messages.length >= 3;
+      },
+    });
     expect(handler.messages).toEqual([
       { id: "a", output: 50 },
       { id: "b", output: 100 },

src/packages/conat/core/client.ts

@@ -652,7 +652,7 @@ export class Client extends EventEmitter {
         .emitWithAck("wait-for-interest", { subject, timeout });
       return response;
     } catch (err) {
-      throw toConatError(err);
+      throw toConatError(err, { subject });
     }
   };
 
@@ -997,7 +997,7 @@ export class Client extends EventEmitter {
             });
           }
         } catch (err) {
-          throw toConatError(err);
+          throw toConatError(err, { subject });
         }
         if (response?.error) {
           throw new ConatError(response.error, { code: response.code });
@@ -1137,7 +1137,13 @@ export class Client extends EventEmitter {
           noThrow: true, // we're not catching this respond
           headers: {
             error,
-            error_attrs: { code: err.code },
+            error_attrs: {
+              code: err.code,
+              errno: err.errno,
+              path: err.path,
+              syscall: err.syscall,
+              subject: err.subject,
+            },
           },
         });
       }
@@ -1187,7 +1193,7 @@ export class Client extends EventEmitter {
       for await (const resp of sub) {
         if (resp.headers?.error) {
           yield new ConatError(`${resp.headers.error}`, {
-            code: resp.headers.code,
+            code: resp.headers.code as string | number,
           });
         } else {
           yield resp.data;
@@ -1350,7 +1356,7 @@ export class Client extends EventEmitter {
                 return response;
               }
             } catch (err) {
-              throw toConatError(err);
+              throw toConatError(err, { subject });
             }
           } else {
             return await this.conn.emitWithAck("publish", v);
@@ -1973,14 +1979,18 @@ function isEmpty(obj: object): boolean {
   return true;
 }
 
-function toConatError(socketIoError) {
+function toConatError(socketIoError, { subject }: { subject?: string } = {}) {
   // only errors are "disconnected" and a timeout
   const e = `${socketIoError}`;
   if (e.includes("disconnected")) {
     return e;
   } else {
-    return new ConatError(`timeout - ${e}`, {
-      code: 408,
-    });
+    return new ConatError(
+      `timeout - ${e}${subject ? " subject:" + subject : ""}`,
+      {
+        code: 408,
+        subject,
+      },
+    );
   }
 }

src/packages/conat/files/file-server.ts

@@ -100,6 +100,18 @@ export interface Fileserver {
     limit?: number;
   }) => Promise<void>;
   deleteSnapshot: (opts: { project_id: string; name: string }) => Promise<void>;
+  updateSnapshots: (opts: {
+    project_id: string;
+    counts?: {
+      frequent?: number;
+      daily?: number;
+      weekly?: number;
+      monthly?: number;
+    };
+    // global limit, same as with createSnapshot above; can prevent new snapshots from being
+    // made if counts are too large!
+    limit?: number;
+  }) => Promise<void>;
 }
 
 export interface Options extends Fileserver {

src/packages/conat/hub/api/projects.ts

@@ -15,6 +15,8 @@ export const projects = {
 
   createSnapshot: authFirstRequireAccount,
   deleteSnapshot: authFirstRequireAccount,
+  updateSnapshots: authFirstRequireAccount,
+  getSnapshotQuota: authFirstRequireAccount,
 };
 
 export type AddCollaborator =
@@ -126,4 +128,20 @@ export interface Projects {
     project_id: string;
     name: string;
   }) => Promise<void>;
+
+  updateSnapshots: (opts: {
+    account_id?: string;
+    project_id: string;
+    counts?: {
+      frequent?: number;
+      daily?: number;
+      weekly?: number;
+      monthly?: number;
+    };
+  }) => Promise<void>;
+
+  getSnapshotQuota: (opts: {
+    account_id?: string;
+    project_id: string;
+  }) => Promise<{ limit: number }>;
 }

src/packages/conat/persist/auth.ts

@@ -49,13 +49,13 @@ export function assertHasWritePermission({
   if (path.length > MAX_PATH_LENGTH) {
     throw new ConatError(
       `permission denied: path (of length ${path.length}) is too long (limit is '${MAX_PATH_LENGTH}' characters)`,
-      { code: 403 },
+      { code: 403, subject },
     );
   }
   if (path.startsWith("/") || path.endsWith("/")) {
     throw new ConatError(
       `permission denied: path '${path}' must not start or end with '/'`,
-      { code: 403 },
+      { code: 403, subject },
     );
   }
   const v = subject.split(".");
@@ -79,10 +79,10 @@ export function assertHasWritePermission({
       } else {
         throw new ConatError(
           `permission denied: subject '${subject}' does not grant write permission to path='${path}' since it is not under '${base}'`,
-          { code: 403 },
+          { code: 403, subject },
         );
       }
     }
   }
-  throw new ConatError(`invalid subject: '${subject}'`, { code: 403 });
+  throw new ConatError(`invalid subject: '${subject}'`, { code: 403, subject });
 }

src/packages/conat/persist/client.ts

@@ -143,7 +143,7 @@ class PersistStreamClient extends EventEmitter {
             });
             if (resp.headers?.error) {
               throw new ConatError(`${resp.headers?.error}`, {
-                code: resp.headers?.code,
+                code: resp.headers?.code as string | number,
               });
             }
             if (this.changefeeds.length == 0 || this.state != "ready") {
@@ -218,7 +218,7 @@ class PersistStreamClient extends EventEmitter {
     });
     if (resp.headers?.error) {
       throw new ConatError(`${resp.headers?.error}`, {
-        code: resp.headers?.code,
+        code: resp.headers?.code as string | number,
       });
     }
     // an iterator over any updates that are published.
@@ -384,7 +384,9 @@ class PersistStreamClient extends EventEmitter {
     let seq = 0; // next expected seq number for the sub (not the data)
     for await (const { data, headers } of sub) {
       if (headers?.error) {
-        throw new ConatError(`${headers.error}`, { code: headers.code });
+        throw new ConatError(`${headers.error}`, {
+          code: headers.code as string | number,
+        });
       }
       if (data == null || this.socket.state == "closed") {
         // done

src/packages/conat/util.ts

@@ -5,9 +5,14 @@ import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
 
 export class ConatError extends Error {
   code?: string | number;
-  constructor(mesg: string, { code }) {
+  subject?: string;
+  constructor(
+    mesg: string,
+    { code, subject }: { code?: string | number; subject?: string } = {},
+  ) {
     super(mesg);
     this.code = code;
+    this.subject = subject;
   }
 }
 

src/packages/file-server/btrfs/snapshots.ts

@@ -22,6 +22,9 @@ export const DEFAULT_SNAPSHOT_COUNTS = {
   monthly: 4,
 } as SnapshotCounts;
 
+// We have at least one snapshot for each interval, assuming
+// there are actual changes since the last snapshot, and at
+// most the listed number.
 export interface SnapshotCounts {
   frequent: number;
   daily: number;
@@ -32,9 +35,12 @@ export interface SnapshotCounts {
 export async function updateRollingSnapshots({
   snapshots,
   counts,
+  opts,
 }: {
   snapshots: SubvolumeSnapshots;
   counts?: Partial<SnapshotCounts>;
+  // options to create
+  opts?;
 }) {
   counts = { ...DEFAULT_SNAPSHOT_COUNTS, ...counts };
 
@@ -44,46 +50,71 @@ export async function updateRollingSnapshots({
     counts,
     changed,
   });
-  if (!changed) {
-    // definitely no data written since most recent snapshot, so nothing to do
-    return;
-  }
 
   // get exactly the iso timestamp snapshot names:
   const snapshotNames = (await snapshots.readdir()).filter((name) =>
     DATE_REGEXP.test(name),
   );
   snapshotNames.sort();
-  if (snapshotNames.length > 0) {
-    const age = Date.now() - new Date(snapshotNames.slice(-1)[0]).valueOf();
+  let needNewSnapshot = false;
+  if (changed) {
+    const timeSinceLastSnapshot =
+      snapshotNames.length == 0
+        ? 1e12 // infinitely old
+        : Date.now() - new Date(snapshotNames.slice(-1)[0]).valueOf();
     for (const key in SNAPSHOT_INTERVALS_MS) {
-      if (counts[key]) {
-        if (age < SNAPSHOT_INTERVALS_MS[key]) {
-          // no need to snapshot since there is already a sufficiently recent snapshot
-          logger.debug("updateRollingSnapshots: no need to snapshot", {
-            name: snapshots.subvolume.name,
-          });
-          return;
-        }
-        // counts[key] nonzero and snapshot is old enough so we'll be making a snapshot
+      if (counts[key] && timeSinceLastSnapshot > SNAPSHOT_INTERVALS_MS[key]) {
+        // there is NOT a sufficiently recent snapshot to satisfy the constraint
+        // of having at least one snapshot for the given interval.
+        needNewSnapshot = true;
         break;
       }
     }
   }
 
-  // make a new snapshot
-  const name = new Date().toISOString();
-  await snapshots.create(name);
+  // Regarding error reporting we try to do everything below and throw the
+  // create error or last delete error...
+
+  let createError: any = undefined;
+  if (changed && needNewSnapshot) {
+    // make a new snapshot -- but only bother
+    // definitely no data written since most recent snapshot, so nothing to do
+    const name = new Date().toISOString();
+    logger.debug(
+      "updateRollingSnapshots: creating snapshot of",
+      snapshots.subvolume.name,
+    );
+    try {
+      await snapshots.create(name, opts);
+      snapshotNames.push(name);
+    } catch (err) {
+      createError = err;
+    }
+  }
+
   // delete extra snapshots
-  snapshotNames.push(name);
   const toDelete = snapshotsToDelete({ counts, snapshots: snapshotNames });
-  for (const expired of toDelete) {
+  let deleteError: any = undefined;
+  for (const name of toDelete) {
     try {
-      await snapshots.delete(expired);
-    } catch {
-      // some snapshots can't be deleted, e.g., they were used for the last send.
+      logger.debug(
+        "updateRollingSnapshots: deleting snapshot of",
+        snapshots.subvolume.name,
+        name,
+      );
+      await snapshots.delete(name);
+    } catch (err) {
+      // ONLY report this if create doesn't error, to give both delete and create a chance to run.
+      deleteError = err;
     }
   }
+
+  if (createError) {
+    throw createError;
+  }
+  if (deleteError) {
+    throw deleteError;
+  }
 }
 
 function snapshotsToDelete({ counts, snapshots }): string[] {

src/packages/file-server/btrfs/subvolume-snapshots.ts

@@ -89,8 +89,8 @@ export class SubvolumeSnapshots {
   };
 
   // update the rolling snapshots schedule
-  update = async (counts?: Partial<SnapshotCounts>) => {
-    return await updateRollingSnapshots({ snapshots: this, counts });
+  update = async (counts?: Partial<SnapshotCounts>, opts?) => {
+    return await updateRollingSnapshots({ snapshots: this, counts, opts });
   };
 
   // has newly written changes since last snapshot