hub/delete-project: refactor, delete files, bugfixes

Author: haraldschilly

src/packages/backend/misc.ts

@@ -1,4 +1,6 @@
 import { createHash } from "crypto";
+
+import { projects } from "@cocalc/backend/data";
 import { is_valid_uuid_string } from "@cocalc/util/misc";
 
 /*
@@ -69,3 +71,8 @@ export function envForSpawn() {
   }
   return env;
 }
+
+// return the absolute home directory of given @project_id project on disk
+export function homePath(project_id: string): string {
+  return projects.replace("[project_id]", project_id);
+}

src/packages/database/postgres/delete-projects.ts

@@ -13,6 +13,7 @@ import { join } from "node:path";
 import { pathToFiles } from "@cocalc/backend/files/path-to-files";
 import getLogger from "@cocalc/backend/logger";
 import { newCounter } from "@cocalc/backend/metrics";
+import { homePath } from "@cocalc/backend/misc";
 import getPool from "@cocalc/database/pool";
 import { getServerSettings } from "@cocalc/database/settings";
 import { callback2 } from "@cocalc/util/async-utils";
@@ -82,7 +83,8 @@ export async function unlink_old_deleted_projects(
   db: PostgreSQL,
   age_d = 30,
 ): Promise<void> {
-  await callback2(db._query, {
+  const L = log.extend("unlink_old_deleted_projects").debug;
+  const { rowCount } = await callback2(db._query, {
     query: "UPDATE projects",
     set: { users: null },
     where: [
@@ -91,6 +93,7 @@ export async function unlink_old_deleted_projects(
       `last_edited <= NOW() - '${age_d} days'::INTERVAL`,
     ],
   });
+  L("unlinked projects:", rowCount);
 }
 
 const Q_CLEANUP_SYNCSTRINGS = `
@@ -184,6 +187,13 @@ export async function cleanup_old_projects_data(
       );
 
       if (on_prem) {
+        // we don't delete the central_log, it has its own expiration
+        // such an entry is good to have for reconstructing what really happened
+        db.log({
+          event: "delete_project",
+          value: { deleting: "files", project_id },
+        });
+
         L2(`delete all project files`);
         await deleteProjectFiles(L2, project_id);
 
@@ -199,6 +209,10 @@ export async function cleanup_old_projects_data(
 
       // This gets rid of all sorts of data in tables specific to the given project.
       delRows += await delete_associated_project_data(L2, project_id);
+      db.log({
+        event: "delete_project",
+        value: { deleting: "database", project_id },
+      });
 
       // now, that we're done with that project, mark it as state.state ->> 'deleted'
       // in addition to the flag "deleted = true". This also updates the state.time timestamp.
@@ -283,7 +297,8 @@ async function delete_associated_project_data(
 
 async function deleteProjectFiles(L2, project_id: string) {
   // TODO: this only works on-prem, and requires the project files to be mounted
-  const projects_root = process.env["MOUNTED_PROJECTS_ROOT"];
+  const projects_root =
+    process.env["MOUNTED_PROJECTS_ROOT"] ?? homePath(project_id);
   if (!projects_root) return;
   const project_dir = join(projects_root, project_id);
   try {
@@ -297,7 +312,7 @@ async function deleteProjectFiles(L2, project_id: string) {
     }
   } catch (err) {
     L2(
-      `not deleting project files: either it does not exist or is not accessible`,
+      `not deleting project files: either path does not exist or is not accessible`,
     );
   }
 }

src/packages/server/projects/control/multi-user.ts

@@ -16,6 +16,15 @@ This code is very similar to single-user.ts, except with some
 small modifications due to having to create and delete Linux users.
 */
 
+import getLogger from "@cocalc/backend/logger";
+import { getUid, homePath } from "@cocalc/backend/misc";
+import {
+  BaseProject,
+  CopyOptions,
+  getProject,
+  ProjectState,
+  ProjectStatus,
+} from "./base";
 import {
   chown,
   copyPath,
@@ -25,22 +34,12 @@ import {
   getEnvironment,
   getState,
   getStatus,
-  homePath,
   isProjectRunning,
   launchProjectDaemon,
   mkdir,
   setupDataPath,
   stopProjectProcesses,
 } from "./util";
-import {
-  BaseProject,
-  CopyOptions,
-  getProject,
-  ProjectStatus,
-  ProjectState,
-} from "./base";
-import getLogger from "@cocalc/backend/logger";
-import { getUid } from "@cocalc/backend/misc";
 
 const winston = getLogger("project-control:multi-user");
 
@@ -71,7 +70,7 @@ class Project extends BaseProject {
     const status = await getStatus(this.HOME);
     // TODO: don't include secret token in log message.
     winston.debug(
-      `got status of ${this.project_id} = ${JSON.stringify(status)}`
+      `got status of ${this.project_id} = ${JSON.stringify(status)}`,
     );
     this.saveStatusToDatabase(status);
     return status;
@@ -155,7 +154,7 @@ class Project extends BaseProject {
     await copyPath(
       opts,
       this.project_id,
-      opts.target_project_id ? getUid(opts.target_project_id) : undefined
+      opts.target_project_id ? getUid(opts.target_project_id) : undefined,
     );
     return "";
   }

src/packages/server/projects/control/single-user.ts

@@ -19,6 +19,7 @@ This is useful for:
 import { kill } from "process";
 
 import getLogger from "@cocalc/backend/logger";
+import { homePath } from "@cocalc/backend/misc";
 import {
   BaseProject,
   CopyOptions,
@@ -33,7 +34,6 @@ import {
   getProjectPID,
   getState,
   getStatus,
-  homePath,
   isProjectRunning,
   launchProjectDaemon,
   mkdir,

src/packages/server/projects/control/util.ts

@@ -1,19 +1,20 @@
-import { promisify } from "util";
-import { dirname, join, resolve } from "path";
-import { exec as exec0, spawn } from "child_process";
 import spawnAsync from "await-spawn";
-import * as fs from "fs";
-import { writeFile } from "fs/promises";
-import { projects, root, blobstore } from "@cocalc/backend/data";
-import { is_valid_uuid_string } from "@cocalc/util/misc";
-import { callback2 } from "@cocalc/util/async-utils";
-import getLogger from "@cocalc/backend/logger";
-import { CopyOptions, ProjectState, ProjectStatus } from "./base";
-import { getUid } from "@cocalc/backend/misc";
+import { exec as exec0, spawn } from "node:child_process";
+import * as fs from "node:fs";
+import { writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { promisify } from "node:util";
+
 import base_path from "@cocalc/backend/base-path";
+import { blobstore, root } from "@cocalc/backend/data";
+import getLogger from "@cocalc/backend/logger";
+import { getUid, homePath } from "@cocalc/backend/misc";
 import { db } from "@cocalc/database";
-import { getProject } from ".";
+import { callback2 } from "@cocalc/util/async-utils";
+import { is_valid_uuid_string } from "@cocalc/util/misc";
 import { pidFilename, pidUpdateIntervalMs } from "@cocalc/util/project-info";
+import { getProject } from ".";
+import { CopyOptions, ProjectState, ProjectStatus } from "./base";
 
 const logger = getLogger("project-control:util");
 
@@ -31,10 +32,6 @@ export function dataPath(HOME: string): string {
   return join(HOME, ".smc");
 }
 
-export function homePath(project_id: string): string {
-  return projects.replace("[project_id]", project_id);
-}
-
 export function getUsername(project_id: string): string {
   return project_id.split("-").join("");
 }

src/packages/util/db-schema/site-defaults.ts

@@ -653,10 +653,11 @@ export const site_settings_conf: SiteSettings = {
     to_val: split_iframe_comm_hosts,
     to_display: num_dns_hosts,
   },
-  delete_project_data : {
-    name :"Delete Project Data",
-    desc: "When a project has been marked as deleted, also actually delete associated data from the database and (for OnPrem) also its files.",
+  delete_project_data: {
+    name: "Delete Project Data",
+    desc: "When a project has been marked as deleted, also actually delete associated data from the database and (OnPrem only) also its files.",
     default: "no",
+    valid: only_booleans,
     to_val: to_bool,
   },
   email_enabled: {