Sync issues and workspace data when the issue properties like labels/modules/cycles etc are deleted from the project (#6165)

Author: SatishGandham

web/core/local-db/storage.sqlite.ts

@@ -294,7 +294,14 @@ export class Storage {
         log(`Project ${projectId} is loading, falling back to server`);
       }
       const issueService = new IssueService();
-      return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries, config);
+
+      // Ignore projectStatus if projectId is not provided
+      if (projectId) {
+        return await issueService.getIssuesFromServer(workspaceSlug, projectId, queries, config);
+      }
+      if (this.status !== "ready" && !rootStore.user.localDBEnabled) {
+        return;
+      }
     }
 
     const { cursor, group_by, sub_group_by } = queries;

web/core/local-db/utils/load-workspace.ts

@@ -1,4 +1,5 @@
-import { IEstimate, IEstimatePoint, IWorkspaceMember } from "@plane/types";
+import { difference } from "lodash";
+import { IEstimate, IEstimatePoint, IWorkspaceMember, TIssue } from "@plane/types";
 import { API_BASE_URL } from "@/helpers/common.helper";
 import { EstimateService } from "@/plane-web/services/project/estimate.service";
 import { CycleService } from "@/services/cycle.service";
@@ -7,6 +8,7 @@ import { ModuleService } from "@/services/module.service";
 import { ProjectStateService } from "@/services/project";
 import { WorkspaceService } from "@/services/workspace.service";
 import { persistence } from "../storage.sqlite";
+import { updateIssue } from "./load-issues";
 import {
   cycleSchema,
   estimatePointSchema,
@@ -103,6 +105,151 @@ export const getMembers = async (workspaceSlug: string) => {
   return objects;
 };
 
+const syncLabels = async (currentLabels: any) => {
+  const currentIdList = currentLabels.map((label: any) => label.id);
+  const existingLabels = await persistence.db.exec("SELECT id FROM labels;");
+
+  const existingIdList = existingLabels.map((label: any) => label.id);
+
+  const deletedIds = difference(existingIdList, currentIdList);
+
+  await syncIssuesWithDeletedLabels(deletedIds as string[]);
+};
+
+export const syncIssuesWithDeletedLabels = async (deletedLabelIds: string[]) => {
+  if (!deletedLabelIds.length) {
+    return;
+  }
+
+  // Ideally we should use recursion to fetch all the issues, but 10000 issues is more than enough for now.
+  const issues = await persistence.getIssues("", "", { labels: deletedLabelIds.join(","), cursor: "10000:0:0" }, {});
+  if (issues?.results && Array.isArray(issues.results)) {
+    const promises = issues.results.map(async (issue: TIssue) => {
+      const updatedIssue = {
+        ...issue,
+        label_ids: issue.label_ids.filter((id: string) => !deletedLabelIds.includes(id)),
+        is_local_update: 1,
+      };
+      // We should await each update because it uses a transaction. But transaction are handled in the query executor.
+      updateIssue(updatedIssue);
+    });
+    await Promise.all(promises);
+  }
+};
+
+const syncModules = async (currentModules: any) => {
+  const currentIdList = currentModules.map((module: any) => module.id);
+  const existingModules = await persistence.db.exec("SELECT id FROM modules;");
+  const existingIdList = existingModules.map((module: any) => module.id);
+  const deletedIds = difference(existingIdList, currentIdList);
+  await syncIssuesWithDeletedModules(deletedIds as string[]);
+};
+
+export const syncIssuesWithDeletedModules = async (deletedModuleIds: string[]) => {
+  if (!deletedModuleIds.length) {
+    return;
+  }
+
+  const issues = await persistence.getIssues("", "", { modules: deletedModuleIds.join(","), cursor: "10000:0:0" }, {});
+  if (issues?.results && Array.isArray(issues.results)) {
+    const promises = issues.results.map(async (issue: TIssue) => {
+      const updatedIssue = {
+        ...issue,
+        module_ids: issue.module_ids?.filter((id: string) => !deletedModuleIds.includes(id)) || [],
+        is_local_update: 1,
+      };
+      updateIssue(updatedIssue);
+    });
+    await Promise.all(promises);
+  }
+};
+
+const syncCycles = async (currentCycles: any) => {
+  const currentIdList = currentCycles.map((cycle: any) => cycle.id);
+  const existingCycles = await persistence.db.exec("SELECT id FROM cycles;");
+  const existingIdList = existingCycles.map((cycle: any) => cycle.id);
+  const deletedIds = difference(existingIdList, currentIdList);
+  await syncIssuesWithDeletedCycles(deletedIds as string[]);
+};
+
+export const syncIssuesWithDeletedCycles = async (deletedCycleIds: string[]) => {
+  if (!deletedCycleIds.length) {
+    return;
+  }
+
+  const issues = await persistence.getIssues("", "", { cycles: deletedCycleIds.join(","), cursor: "10000:0:0" }, {});
+  if (issues?.results && Array.isArray(issues.results)) {
+    const promises = issues.results.map(async (issue: TIssue) => {
+      const updatedIssue = {
+        ...issue,
+        cycle_id: null,
+        is_local_update: 1,
+      };
+      updateIssue(updatedIssue);
+    });
+    await Promise.all(promises);
+  }
+};
+
+const syncStates = async (currentStates: any) => {
+  const currentIdList = currentStates.map((state: any) => state.id);
+  const existingStates = await persistence.db.exec("SELECT id FROM states;");
+  const existingIdList = existingStates.map((state: any) => state.id);
+  const deletedIds = difference(existingIdList, currentIdList);
+  await syncIssuesWithDeletedStates(deletedIds as string[]);
+};
+
+export const syncIssuesWithDeletedStates = async (deletedStateIds: string[]) => {
+  if (!deletedStateIds.length) {
+    return;
+  }
+
+  const issues = await persistence.getIssues("", "", { states: deletedStateIds.join(","), cursor: "10000:0:0" }, {});
+  if (issues?.results && Array.isArray(issues.results)) {
+    const promises = issues.results.map(async (issue: TIssue) => {
+      const updatedIssue = {
+        ...issue,
+        state_id: null,
+        is_local_update: 1,
+      };
+      updateIssue(updatedIssue);
+    });
+    await Promise.all(promises);
+  }
+};
+
+const syncMembers = async (currentMembers: any) => {
+  const currentIdList = currentMembers.map((member: any) => member.id);
+  const existingMembers = await persistence.db.exec("SELECT id FROM members;");
+  const existingIdList = existingMembers.map((member: any) => member.id);
+  const deletedIds = difference(existingIdList, currentIdList);
+  await syncIssuesWithDeletedMembers(deletedIds as string[]);
+};
+
+export const syncIssuesWithDeletedMembers = async (deletedMemberIds: string[]) => {
+  if (!deletedMemberIds.length) {
+    return;
+  }
+
+  const issues = await persistence.getIssues(
+    "",
+    "",
+    { assignees: deletedMemberIds.join(","), cursor: "10000:0:0" },
+    {}
+  );
+  if (issues?.results && Array.isArray(issues.results)) {
+    const promises = issues.results.map(async (issue: TIssue) => {
+      const updatedIssue = {
+        ...issue,
+        assignee_ids: issue.assignee_ids.filter((id: string) => !deletedMemberIds.includes(id)),
+        is_local_update: 1,
+      };
+      updateIssue(updatedIssue);
+    });
+    await Promise.all(promises);
+  }
+};
+
 export const loadWorkSpaceData = async (workspaceSlug: string) => {
   if (!persistence.db || !persistence.db.exec) {
     return;
@@ -117,28 +264,45 @@ export const loadWorkSpaceData = async (workspaceSlug: string) => {
   promises.push(getMembers(workspaceSlug));
   const [labels, modules, cycles, states, estimates, members] = await Promise.all(promises);
 
+  // @todo: we don't need this manual sync here, when backend adds these changes to issue activity and updates the updated_at of the issue.
+  await syncLabels(labels);
+  await syncModules(modules);
+  await syncCycles(cycles);
+  await syncStates(states);
+  // TODO: Not handling sync estimates yet, as we don't know the new estimate point assigned.
+  // Backend should update the updated_at of the issue when estimate point is updated, or we should have realtime sync on the issues table.
+  // await syncEstimates(estimates);
+  await syncMembers(members);
+
   const start = performance.now();
+
   await persistence.db.exec("BEGIN;");
+  await persistence.db.exec("DELETE FROM labels WHERE 1=1;");
   await batchInserts(labels, "labels", labelSchema);
   await persistence.db.exec("COMMIT;");
 
   await persistence.db.exec("BEGIN;");
+  await persistence.db.exec("DELETE FROM modules WHERE 1=1;");
   await batchInserts(modules, "modules", moduleSchema);
   await persistence.db.exec("COMMIT;");
 
   await persistence.db.exec("BEGIN;");
+  await persistence.db.exec("DELETE FROM cycles WHERE 1=1;");
   await batchInserts(cycles, "cycles", cycleSchema);
   await persistence.db.exec("COMMIT;");
 
   await persistence.db.exec("BEGIN;");
+  await persistence.db.exec("DELETE FROM states WHERE 1=1;");
   await batchInserts(states, "states", stateSchema);
   await persistence.db.exec("COMMIT;");
 
   await persistence.db.exec("BEGIN;");
+  await persistence.db.exec("DELETE FROM estimate_points WHERE 1=1;");
   await batchInserts(estimates, "estimate_points", estimatePointSchema);
   await persistence.db.exec("COMMIT;");
 
   await persistence.db.exec("BEGIN;");
+  await persistence.db.exec("DELETE FROM members WHERE 1=1;");
   await batchInserts(members, "members", memberSchema);
   await persistence.db.exec("COMMIT;");
 

web/core/local-db/utils/query-constructor.ts

@@ -142,7 +142,11 @@ export const issueFilterQueryConstructor = (workspaceSlug: string, projectId: st
     `;
   });
 
-  sql += ` WHERE i.project_id = '${projectId}'    ${singleFilterConstructor(otherProps)} group by i.id  `;
+  sql += ` WHERE 1=1  `;
+  if (projectId) {
+    sql += ` AND i.project_id = '${projectId}'    `;
+  }
+  sql += ` ${singleFilterConstructor(otherProps)} group by i.id  `;
   sql += orderByString;
 
   // Add offset and paging to query

web/core/local-db/utils/query.utils.ts

@@ -161,8 +161,11 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => {
     if (otherProps.state_group) {
       sql += `LEFT JOIN states ON i.state_id = states.id `;
     }
-    sql += `WHERE i.project_id = '${projectId}'
-    `;
+    sql += `WHERE 1=1 `;
+    if (projectId) {
+      sql += ` AND i.project_id = '${projectId}'
+      `;
+    }
     sql += `${singleFilterConstructor(otherProps)}) 
     `;
     return sql;
@@ -212,8 +215,11 @@ export const getFilteredRowsForGrouping = (projectId: string, queries: any) => {
     `;
   }
 
-  sql += ` WHERE i.project_id = '${projectId}'
-  `;
+  sql += ` WHERE 1=1  `;
+  if (projectId) {
+    sql += ` AND i.project_id = '${projectId}'
+    `;
+  }
   sql += singleFilterConstructor(otherProps);
 
   sql += `)

web/core/store/cycle.store.ts

@@ -20,6 +20,7 @@ import { orderCycles, shouldFilterCycle, formatActiveCycle } from "@/helpers/cyc
 import { getDate } from "@/helpers/date-time.helper";
 import { DistributionUpdates, updateDistribution } from "@/helpers/distribution-update.helper";
 // services
+import { syncIssuesWithDeletedCycles } from "@/local-db/utils/load-workspace";
 import { CycleService } from "@/services/cycle.service";
 import { CycleArchiveService } from "@/services/cycle_archive.service";
 import { IssueService } from "@/services/issue";
@@ -675,6 +676,7 @@ export class CycleStore implements ICycleStore {
         delete this.cycleMap[cycleId];
         delete this.activeCycleIdMap[cycleId];
         if (this.rootStore.favorite.entityMap[cycleId]) this.rootStore.favorite.removeFavoriteFromStore(cycleId);
+        syncIssuesWithDeletedCycles([cycleId]);
       });
     });
 

web/core/store/label.store.ts

@@ -7,6 +7,7 @@ import { IIssueLabel, IIssueLabelTree } from "@plane/types";
 // helpers
 import { buildTree } from "@/helpers/array.helper";
 // services
+import { syncIssuesWithDeletedLabels } from "@/local-db/utils/load-workspace";
 import { IssueLabelService } from "@/services/issue";
 // store
 import { CoreRootStore } from "./root.store";
@@ -275,6 +276,7 @@ export class LabelStore implements ILabelStore {
       runInAction(() => {
         delete this.labelMap[labelId];
       });
+      syncIssuesWithDeletedLabels([labelId]);
     });
   };
 }

web/core/store/module.store.ts

@@ -10,6 +10,7 @@ import { IModule, ILinkDetails, TModulePlotType } from "@plane/types";
 import { DistributionUpdates, updateDistribution } from "@/helpers/distribution-update.helper";
 import { orderModules, shouldFilterModule } from "@/helpers/module.helper";
 // services
+import { syncIssuesWithDeletedModules } from "@/local-db/utils/load-workspace";
 import { ModuleService } from "@/services/module.service";
 import { ModuleArchiveService } from "@/services/module_archive.service";
 import { ProjectService } from "@/services/project";
@@ -438,6 +439,7 @@ export class ModulesStore implements IModuleStore {
       runInAction(() => {
         delete this.moduleMap[moduleId];
         if (this.rootStore.favorite.entityMap[moduleId]) this.rootStore.favorite.removeFavoriteFromStore(moduleId);
+        syncIssuesWithDeletedModules([moduleId]);
       });
     });
   };

web/core/store/state.store.ts

@@ -1,13 +1,14 @@
 import groupBy from "lodash/groupBy";
 import set from "lodash/set";
-import { makeObservable, observable, computed, action, runInAction } from "mobx";
+import { action, computed, makeObservable, observable, runInAction } from "mobx";
 import { computedFn } from "mobx-utils";
 // types
 import { IState } from "@plane/types";
 // helpers
 import { convertStringArrayToBooleanObject } from "@/helpers/array.helper";
 import { sortStates } from "@/helpers/state.helper";
 // plane web
+import { syncIssuesWithDeletedStates } from "@/local-db/utils/load-workspace";
 import { ProjectStateService } from "@/plane-web/services/project/project-state.service";
 import { RootStore } from "@/plane-web/store/root.store";
 
@@ -228,6 +229,7 @@ export class StateStore implements IStateStore {
     await this.stateService.deleteState(workspaceSlug, projectId, stateId).then(() => {
       runInAction(() => {
         delete this.stateMap[stateId];
+        syncIssuesWithDeletedStates([stateId]);
       });
     });
   };