ENG-1343 upload obsidian node schema to supabase

Author: trangdoan982

apps/obsidian/src/utils/conceptConversion.ts

@@ -1,10 +1,10 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import { TFile } from "obsidian";
-import { DiscourseNode } from "~/types";
-import { SupabaseContext } from "./supabaseContext";
-import { LocalConceptDataInput } from "@repo/database/inputTypes";
-import { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase";
-import { Json } from "@repo/database/dbTypes";
+import type { TFile } from "obsidian";
+import type { DiscourseNode } from "~/types";
+import type { SupabaseContext } from "./supabaseContext";
+import type { LocalConceptDataInput } from "@repo/database/inputTypes";
+import type { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase";
+import type { Json } from "@repo/database/dbTypes";
 
 /**
  * Get extra data (author, timestamps) from file metadata
@@ -34,15 +34,21 @@ export const discourseNodeSchemaToLocalConcept = ({
   accountLocalId: string;
 }): LocalConceptDataInput => {
   const now = new Date().toISOString();
+  const { description, template, id, name, ...otherData } = node;
   return {
     space_id: context.spaceId,
-    name: node.name,
-    source_local_id: node.id,
+    name: name,
+    source_local_id: id,
     is_schema: true,
     author_local_id: accountLocalId,
     created: now,
-    // TODO: get the template or any other info to put into literal_content jsonb
     last_modified: now,
+    description: description,
+    literal_content: {
+      label: name,
+      template: template,
+      source_data: otherData,
+    },
   };
 };
 
@@ -59,22 +65,19 @@ export const discourseNodeInstanceToLocalConcept = ({
   accountLocalId: string;
 }): LocalConceptDataInput => {
   const extraData = getNodeExtraData(nodeData.file, accountLocalId);
-  console.log(nodeData.frontmatter);
-  const concept = {
+  const { nodeInstanceId, nodeTypeId, ...otherData } = nodeData.frontmatter;
+  return {
     space_id: context.spaceId,
-    name: nodeData.file.basename,
-    source_local_id: nodeData.nodeInstanceId,
-    schema_represented_by_local_id: nodeData.nodeTypeId,
+    name: nodeData.file.path,
+    source_local_id: nodeInstanceId as string,
+    schema_represented_by_local_id: nodeTypeId as string,
     is_schema: false,
     literal_content: {
-      ...nodeData.frontmatter,
-    } as unknown as Json,
+      label: nodeData.file.basename,
+      source_data: otherData as unknown as Json,
+    },
     ...extraData,
   };
-  console.log(
-    `[discourseNodeInstanceToLocalConcept] Converting concept: source_local_id=${nodeData.nodeInstanceId}, name="${nodeData.file.basename}"`,
-  );
-  return concept;
 };
 
 export const relatedConcepts = (concept: LocalConceptDataInput): string[] => {
@@ -135,6 +138,7 @@ export const orderConceptsByDependency = (
       ...missing,
       ...orderConceptsRec(ordered, first, conceptById),
     ]);
+    if (missing.size > 0) console.error(`missing: ${[...missing]}`);
   }
   return { ordered, missing: Array.from(missing) };
 };

apps/obsidian/src/utils/syncDgNodesToSupabase.ts

@@ -1,19 +1,20 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import { Notice, TFile } from "obsidian";
-import { DGSupabaseClient } from "@repo/database/lib/client";
-import { Json } from "@repo/database/dbTypes";
+import type { DGSupabaseClient } from "@repo/database/lib/client";
+import type { Json } from "@repo/database/dbTypes";
 import {
   getSupabaseContext,
   getLoggedInClient,
-  SupabaseContext,
+  type SupabaseContext,
 } from "./supabaseContext";
 import { default as DiscourseGraphPlugin } from "~/index";
 import { upsertNodesToSupabaseAsContentWithEmbeddings } from "./upsertNodesAsContentWithEmbeddings";
 import {
   orderConceptsByDependency,
   discourseNodeInstanceToLocalConcept,
+  discourseNodeSchemaToLocalConcept,
 } from "./conceptConversion";
-import { LocalConceptDataInput } from "@repo/database/inputTypes";
+import type { LocalConceptDataInput } from "@repo/database/inputTypes";
 
 const DEFAULT_TIME = new Date("1970-01-01");
 export type ChangeType = "title" | "content";
@@ -101,7 +102,10 @@ const deleteNodesFromSupabase = async (
     if (conceptDeleteError) {
       result.success = false;
       result.errors.concept = conceptDeleteError;
-      console.error("Failed to delete concepts from Supabase:", conceptDeleteError);
+      console.error(
+        "Failed to delete concepts from Supabase:",
+        conceptDeleteError,
+      );
     }
 
     const { error: contentDeleteError } = await supabaseClient
@@ -195,7 +199,6 @@ const mergeChangeTypes = (
   return Array.from(merged);
 };
 
-
 /**
  * Step 1: Collect all discourse nodes from the vault
  * Filters markdown files that have nodeTypeId in frontmatter
@@ -466,12 +469,20 @@ export const createOrUpdateDiscourseEmbedding = async (
       throw new Error("accountLocalId not found in plugin settings");
     }
 
-    await syncChangedNodesToSupabase({
-      changedNodes: allNodeInstances,
+    await upsertNodesToSupabaseAsContentWithEmbeddings({
+      obsidianNodes: allNodeInstances,
+      supabaseClient,
+      context,
+      accountLocalId,
       plugin,
+    });
+
+    await convertDgToSupabaseConcepts({
+      nodesSince: allNodeInstances,
       supabaseClient,
       context,
       accountLocalId,
+      plugin,
     });
 
     console.debug("Sync completed successfully");
@@ -486,25 +497,34 @@ const convertDgToSupabaseConcepts = async ({
   supabaseClient,
   context,
   accountLocalId,
+  plugin,
 }: {
   nodesSince: ObsidianDiscourseNodeData[];
   supabaseClient: DGSupabaseClient;
   context: SupabaseContext;
   accountLocalId: string;
+  plugin: DiscourseGraphPlugin;
 }): Promise<void> => {
-  // TODO: handling schema (node types and relations) will be handled in the future by ENG-1181
-  // Schema upsert will need allNodeTypes parameter when enabled
+  const nodeTypes = plugin.settings.nodeTypes ?? [];
+
+  const nodesTypesToLocalConcepts = nodeTypes.map((nodeType) =>
+    discourseNodeSchemaToLocalConcept({
+      context,
+      node: nodeType,
+      accountLocalId,
+    }),
+  );
 
-  const nodeInstanceToLocalConcepts = nodesSince.map((node) => {
-    return discourseNodeInstanceToLocalConcept({
+  const nodeInstanceToLocalConcepts = nodesSince.map((node) =>
+    discourseNodeInstanceToLocalConcept({
       context,
       nodeData: node,
       accountLocalId,
-    });
-  });
+    }),
+  );
 
   const conceptsToUpsert: LocalConceptDataInput[] = [
-    // ...nodesTypesToLocalConcepts,
+    ...nodesTypesToLocalConcepts,
     ...nodeInstanceToLocalConcepts,
   ];
 
@@ -543,33 +563,29 @@ const syncChangedNodesToSupabase = async ({
   context: SupabaseContext;
   accountLocalId: string;
 }): Promise<void> => {
-  if (changedNodes.length === 0) {
-    console.debug("No nodes to sync");
-    return;
+  if (changedNodes.length > 0) {
+    await upsertNodesToSupabaseAsContentWithEmbeddings({
+      obsidianNodes: changedNodes,
+      supabaseClient,
+      context,
+      accountLocalId,
+      plugin,
+    });
   }
 
-  await upsertNodesToSupabaseAsContentWithEmbeddings({
-    obsidianNodes: changedNodes,
-    supabaseClient,
-    context,
-    accountLocalId,
-    plugin,
-  });
-
   // Only upsert concepts for nodes with title changes or new files
   // (concepts store the title, so content-only changes don't affect them)
   const nodesNeedingConceptUpsert = changedNodes.filter((node) =>
     node.changeTypes.includes("title"),
   );
 
-  if (nodesNeedingConceptUpsert.length > 0) {
-    await convertDgToSupabaseConcepts({
-      nodesSince: nodesNeedingConceptUpsert,
-      supabaseClient,
-      context,
-      accountLocalId,
-    });
-  }
+  await convertDgToSupabaseConcepts({
+    nodesSince: nodesNeedingConceptUpsert,
+    supabaseClient,
+    context,
+    accountLocalId,
+    plugin,
+  });
 };
 
 /**
@@ -635,10 +651,7 @@ export const syncSpecificFiles = async (
   const changeTypesByPath = new Map<string, ChangeType[]>();
   for (const filePath of filePaths) {
     const existing = changeTypesByPath.get(filePath) ?? [];
-    changeTypesByPath.set(
-      filePath,
-      mergeChangeTypes(existing, ["content"]),
-    );
+    changeTypesByPath.set(filePath, mergeChangeTypes(existing, ["content"]));
   }
 
   await syncDiscourseNodeChanges(plugin, changeTypesByPath);