eng-1344 f10b upload obsidian relations and their schemas

Author: maparent

apps/obsidian/src/utils/conceptConversion.ts

@@ -1,7 +1,13 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import type { TFile } from "obsidian";
-import type { DiscourseNode } from "~/types";
+import type DiscourseGraphPlugin from "~/index";
+import type {
+  DiscourseNode,
+  DiscourseRelation,
+  DiscourseRelationType,
+} from "~/types";
 import type { SupabaseContext } from "./supabaseContext";
+import type { DiscourseNodeInVault } from "./syncDgNodesToSupabase";
 import type { LocalConceptDataInput } from "@repo/database/inputTypes";
 import type { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase";
 import type { Json } from "@repo/database/dbTypes";
@@ -52,21 +58,157 @@ export const discourseNodeSchemaToLocalConcept = ({
   };
 };
 
+const STANDARD_ROLES = ["source", "destination"];
+
+export const discourseRelationTypeToLocalConcept = ({
+  context,
+  relationType,
+  accountLocalId,
+}: {
+  context: SupabaseContext;
+  relationType: DiscourseRelationType;
+  accountLocalId: string;
+}): LocalConceptDataInput => {
+  const now = new Date().toISOString();
+  const { id, label, complement, ...otherData } = relationType;
+  return {
+    space_id: context.spaceId,
+    name: label,
+    source_local_id: id,
+    is_schema: true,
+    author_local_id: accountLocalId,
+    created: now,
+    literal_content: {
+      label,
+      complement,
+      source_data: otherData,
+    } as unknown as Json,
+    last_modified: now,
+  };
+};
+
+export const discourseRelationSchemaToLocalConcept = ({
+  context,
+  relation,
+  accountLocalId,
+  nodeTypesById,
+  relationTypesById,
+}: {
+  context: SupabaseContext;
+  relation: DiscourseRelation;
+  accountLocalId: string;
+  nodeTypesById: Record<string, DiscourseNode>;
+  relationTypesById: Record<string, DiscourseRelationType>;
+}): LocalConceptDataInput => {
+  const sourceName =
+    nodeTypesById[relation.sourceId]?.name ?? relation.sourceId;
+  const destinationName =
+    nodeTypesById[relation.destinationId]?.name ?? relation.destinationId;
+  const relationType = relationTypesById[relation.relationshipTypeId];
+  if (!relationType)
+    throw new Error(`missing relation type ${relation.relationshipTypeId}`);
+  const { label, complement } = relationType;
+  const now = new Date().toISOString(); // TODO: Keep a date in the structure
+  // TODO: Create an Id for the triple
+  const compositeId = [
+    relation.relationshipTypeId,
+    relation.sourceId,
+    relation.destinationId,
+  ].join(":");
+
+  return {
+    space_id: context.spaceId,
+    name: `${sourceName} -${label}-> ${destinationName}`,
+    source_local_id: compositeId,
+    is_schema: true,
+    author_local_id: accountLocalId,
+    created: now,
+    last_modified: now,
+    literal_content: {
+      roles: STANDARD_ROLES,
+      label,
+      complement,
+    },
+    local_reference_content: {
+      relation_type: relation.relationshipTypeId,
+      source: relation.sourceId,
+      destination: relation.destinationId,
+    },
+  };
+};
+
 /**
  * Convert discourse node instance (file) to LocalConceptDataInput
  */
-export const discourseNodeInstanceToLocalConcept = ({
+export const discourseNodeInstanceToLocalConcepts = ({
+  plugin,
+  allNodesByName,
   context,
   nodeData,
   accountLocalId,
 }: {
+  plugin: DiscourseGraphPlugin;
+  allNodesByName: Record<string, DiscourseNodeInVault>;
   context: SupabaseContext;
   nodeData: ObsidianDiscourseNodeData;
   accountLocalId: string;
-}): LocalConceptDataInput => {
+}): LocalConceptDataInput[] => {
   const extraData = getNodeExtraData(nodeData.file, accountLocalId);
   const { nodeInstanceId, nodeTypeId, ...otherData } = nodeData.frontmatter;
-  return {
+  const response: LocalConceptDataInput[] = [];
+  for (const relType of plugin.settings.relationTypes) {
+    const rels = otherData[relType.id];
+    if (rels) {
+      delete otherData[relType.id];
+      const triples = plugin.settings.discourseRelations.filter(
+        (r) => r.relationshipTypeId === relType.id && r.sourceId === nodeTypeId,
+      );
+      if (!triples.length) {
+        // we're probably the target.
+        continue;
+      }
+      const tripleDestTypes = new Set(triples.map((rel) => rel.destinationId));
+      for (let rel of rels as string[]) {
+        if (rel.startsWith("[[") && rel.endsWith("]]"))
+          rel = rel.substring(2, rel.length - 2);
+        if (rel.endsWith(".md")) rel = rel.substring(0, rel.length - 3);
+        const target = allNodesByName[rel];
+        if (!target) {
+          console.error(`Could not find node name ${rel}`);
+          continue;
+        }
+        const targetTypeId = target.frontmatter.nodeTypeId as string;
+        const targetInstanceId = target.frontmatter.nodeInstanceId as string;
+        if (!tripleDestTypes.has(targetTypeId)) {
+          console.error(
+            `Found a relation of type ${relType.id} between ${nodeData.file.path} and ${rel} but no relation fits`,
+          );
+          continue;
+        }
+        const compositeSchemaId = [relType.id, nodeTypeId, targetTypeId].join(
+          ":",
+        );
+        const compositeInstanceId = [
+          relType.id,
+          nodeInstanceId as string,
+          targetInstanceId,
+        ].join(":");
+        response.push({
+          space_id: context.spaceId,
+          name: `[[${nodeData.file.basename}]] -${relType.label}-> [[${target.file.basename}]]`,
+          source_local_id: compositeInstanceId,
+          schema_represented_by_local_id: compositeSchemaId,
+          is_schema: false,
+          local_reference_content: {
+            source: nodeInstanceId as string,
+            destination: targetInstanceId,
+          },
+          ...extraData,
+        });
+      }
+    }
+  }
+  response.push({
     space_id: context.spaceId,
     name: nodeData.file.path,
     source_local_id: nodeInstanceId as string,
@@ -77,7 +219,8 @@ export const discourseNodeInstanceToLocalConcept = ({
       source_data: otherData as unknown as Json,
     },
     ...extraData,
-  };
+  });
+  return response;
 };
 
 export const relatedConcepts = (concept: LocalConceptDataInput): string[] => {

apps/obsidian/src/utils/syncDgNodesToSupabase.ts

@@ -11,8 +11,10 @@ import { default as DiscourseGraphPlugin } from "~/index";
 import { upsertNodesToSupabaseAsContentWithEmbeddings } from "./upsertNodesAsContentWithEmbeddings";
 import {
   orderConceptsByDependency,
-  discourseNodeInstanceToLocalConcept,
+  discourseNodeInstanceToLocalConcepts,
   discourseNodeSchemaToLocalConcept,
+  discourseRelationSchemaToLocalConcept,
+  discourseRelationTypeToLocalConcept,
 } from "./conceptConversion";
 import type { LocalConceptDataInput } from "@repo/database/inputTypes";
 
@@ -177,7 +179,7 @@ const getLastSyncTime = async (
   return new Date(data?.last_modified || DEFAULT_TIME);
 };
 
-type DiscourseNodeInVault = {
+export type DiscourseNodeInVault = {
   file: TFile;
   frontmatter: Record<string, unknown>;
   nodeTypeId: string;
@@ -203,7 +205,7 @@ const mergeChangeTypes = (
  * Step 1: Collect all discourse nodes from the vault
  * Filters markdown files that have nodeTypeId in frontmatter
  */
-const collectDiscourseNodesFromVault = async (
+export const collectDiscourseNodesFromVault = async (
   plugin: DiscourseGraphPlugin,
 ): Promise<DiscourseNodeInVault[]> => {
   const allFiles = plugin.app.vault.getMarkdownFiles();
@@ -504,8 +506,19 @@ const convertDgToSupabaseConcepts = async ({
   context: SupabaseContext;
   accountLocalId: string;
   plugin: DiscourseGraphPlugin;
+  convertRelations?: boolean;
 }): Promise<void> => {
   const nodeTypes = plugin.settings.nodeTypes ?? [];
+  const relationTypes = plugin.settings.relationTypes ?? [];
+  const discourseRelations = plugin.settings.discourseRelations ?? [];
+  const allNodes = await collectDiscourseNodesFromVault(plugin);
+  const allNodesByName = Object.fromEntries(
+    allNodes.map((n) => [n.file.basename, n]),
+  );
+
+  const nodeTypesById = Object.fromEntries(
+    nodeTypes.map((nodeType) => [nodeType.id, nodeType]),
+  );
 
   const nodesTypesToLocalConcepts = nodeTypes.map((nodeType) =>
     discourseNodeSchemaToLocalConcept({
@@ -515,21 +528,49 @@ const convertDgToSupabaseConcepts = async ({
     }),
   );
 
-  const nodeInstanceToLocalConcepts = nodesSince.map((node) =>
-    discourseNodeInstanceToLocalConcept({
+  const relationTypesById = Object.fromEntries(
+    relationTypes.map((relationType) => [relationType.id, relationType]),
+  );
+
+  const relationTypesToLocalConcepts = relationTypes.map((relationType) =>
+    discourseRelationTypeToLocalConcept({
+      context,
+      relationType,
+      accountLocalId,
+    }),
+  );
+
+  const discourseRelationsToLocalConcepts = discourseRelations.map((relation) =>
+    discourseRelationSchemaToLocalConcept({
       context,
-      nodeData: node,
+      relation,
       accountLocalId,
+      nodeTypesById,
+      relationTypesById,
     }),
   );
 
+  const nodeInstanceToLocalConcepts = nodesSince
+    .map((node) => {
+      return discourseNodeInstanceToLocalConcepts({
+        plugin,
+        allNodesByName,
+        context,
+        nodeData: node,
+        accountLocalId,
+      });
+    })
+    .flat();
+
   const conceptsToUpsert: LocalConceptDataInput[] = [
     ...nodesTypesToLocalConcepts,
+    ...relationTypesToLocalConcepts,
+    ...discourseRelationsToLocalConcepts,
     ...nodeInstanceToLocalConcepts,
   ];
 
   const { ordered } = orderConceptsByDependency(conceptsToUpsert);
-
+  console.log(ordered);
   const { error } = await supabaseClient.rpc("upsert_concepts", {
     data: ordered as Json,
     v_space_id: context.spaceId,