Skip to content

Commit 748ee78

Browse files
maparentmaparent
authored
ENG-1180 F7: Publish non-text assets referenced by a DG node (#708)
* eng-1180 publish non-text assets as blobs with file reference.
1 parent 6e5c9d4 commit 748ee78

File tree

10 files changed

+572
-27
lines changed

10 files changed

+572
-27
lines changed

‎apps/obsidian/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"@types/node": "^20",
2323
"@types/react": "catalog:obsidian",
2424
"@types/react-dom": "catalog:obsidian",
25+
"@types/mime-types": "3.0.1",
2526
"autoprefixer": "^10.4.21",
2627
"builtin-modules": "3.3.0",
2728
"dotenv": "^16.4.5",
@@ -41,10 +42,11 @@
4142
"@repo/utils": "workspace:*",
4243
"@supabase/supabase-js": "catalog:",
4344
"date-fns": "^4.1.0",
45+
"mime-types": "^3.0.1",
4446
"nanoid": "^4.0.2",
4547
"react": "catalog:obsidian",
4648
"react-dom": "catalog:obsidian",
4749
"tailwindcss-animate": "^1.0.7",
4850
"tldraw": "3.14.2"
4951
}
50-
}
52+
}

‎apps/obsidian/src/utils/publishNode.ts‎

Lines changed: 83 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import type { FrontMatterCache, TFile } from "obsidian";
22
import type { default as DiscourseGraphPlugin } from "~/index";
33
import { getLoggedInClient, getSupabaseContext } from "./supabaseContext";
4+
import { addFile } from "@repo/database/lib/files";
5+
import mime from "mime-types";
46

57
export const publishNode = async ({
68
plugin,
@@ -26,21 +28,88 @@ export const publishNode = async ({
2628
if (!myGroup) throw new Error("Cannot get group");
2729
const existingPublish =
2830
(frontmatter.publishedToGroups as undefined | string[]) || [];
29-
if (existingPublish.includes(myGroup)) return; // already published
30-
const publishResponse = await client.from("ResourceAccess").insert({
31-
/* eslint-disable @typescript-eslint/naming-convention */
32-
account_uid: myGroup,
33-
source_local_id: nodeId,
34-
space_id: spaceId,
35-
/* eslint-enable @typescript-eslint/naming-convention */
36-
});
31+
const idResponse = await client
32+
.from("Content")
33+
.select("last_modified")
34+
.eq("source_local_id", nodeId)
35+
.eq("space_id", spaceId)
36+
.eq("variant", "full")
37+
.maybeSingle();
38+
if (idResponse.error || !idResponse.data) {
39+
throw idResponse.error || new Error("no data while fetching node");
40+
}
41+
const lastModifiedDb = new Date(
42+
idResponse.data.last_modified + "Z",
43+
).getTime();
44+
const embeds = plugin.app.metadataCache.getFileCache(file)?.embeds ?? [];
45+
const attachments = embeds
46+
.map(({ link }) => {
47+
const attachment = plugin.app.metadataCache.getFirstLinkpathDest(
48+
link,
49+
file.path,
50+
);
51+
if (attachment === null) {
52+
console.warn("Could not find file for " + link);
53+
}
54+
return attachment;
55+
})
56+
.filter((a) => !!a);
57+
const lastModified = Math.max(
58+
file.stat.mtime,
59+
...attachments.map((a) => a.stat.mtime),
60+
);
61+
62+
if (existingPublish.includes(myGroup) && lastModified <= lastModifiedDb)
63+
return; // already published
64+
const publishResponse = await client.from("ResourceAccess").upsert(
65+
{
66+
/* eslint-disable @typescript-eslint/naming-convention */
67+
account_uid: myGroup,
68+
source_local_id: nodeId,
69+
space_id: spaceId,
70+
/* eslint-enable @typescript-eslint/naming-convention */
71+
},
72+
{ ignoreDuplicates: true },
73+
);
3774
if (publishResponse.error && publishResponse.error.code !== "23505")
3875
// 23505 is duplicate key, which counts as a success.
3976
throw publishResponse.error;
40-
await plugin.app.fileManager.processFrontMatter(
41-
file,
42-
(fm: Record<string, unknown>) => {
43-
fm.publishedToGroups = [...existingPublish, myGroup];
44-
},
45-
);
77+
78+
const existingFiles: string[] = [];
79+
for (const attachment of attachments) {
80+
const mimetype = mime.lookup(attachment.path) || "application/octet-stream";
81+
if (mimetype.startsWith("text/")) continue;
82+
existingFiles.push(attachment.path);
83+
const content = await plugin.app.vault.readBinary(attachment);
84+
await addFile({
85+
client,
86+
spaceId,
87+
sourceLocalId: nodeId,
88+
fname: attachment.path,
89+
mimetype,
90+
created: new Date(attachment.stat.ctime),
91+
lastModified: new Date(attachment.stat.mtime),
92+
content,
93+
});
94+
}
95+
let cleanupCommand = client
96+
.from("FileReference")
97+
.delete()
98+
.eq("space_id", spaceId)
99+
.eq("source_local_id", nodeId);
100+
if (existingFiles.length)
101+
cleanupCommand = cleanupCommand.notIn("filepath", [
102+
...new Set(existingFiles),
103+
]);
104+
const cleanupResult = await cleanupCommand;
105+
// do not fail on cleanup
106+
if (cleanupResult.error) console.error(cleanupResult.error);
107+
108+
if (!existingPublish.includes(myGroup))
109+
await plugin.app.fileManager.processFrontMatter(
110+
file,
111+
(fm: Record<string, unknown>) => {
112+
fm.publishedToGroups = [...existingPublish, myGroup];
113+
},
114+
);
46115
};

‎packages/database/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
"@repo/utils": "workspace:*",
4646
"@supabase/auth-js": "catalog:",
4747
"@supabase/functions-js": "catalog:",
48+
"@supabase/storage-js": "catalog:",
4849
"@supabase/supabase-js": "catalog:",
4950
"tslib": "2.5.1"
5051
},

‎packages/database/src/dbTypes.ts‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,70 @@ export type Database = {
511511
},
512512
]
513513
}
514+
file_gc: {
515+
Row: {
516+
filehash: string
517+
}
518+
Insert: {
519+
filehash: string
520+
}
521+
Update: {
522+
filehash?: string
523+
}
524+
Relationships: []
525+
}
526+
FileReference: {
527+
Row: {
528+
created: string
529+
filehash: string
530+
filepath: string
531+
last_modified: string
532+
source_local_id: string
533+
space_id: number
534+
variant: Database["public"]["Enums"]["ContentVariant"] | null
535+
}
536+
Insert: {
537+
created: string
538+
filehash: string
539+
filepath: string
540+
last_modified: string
541+
source_local_id: string
542+
space_id: number
543+
variant?: Database["public"]["Enums"]["ContentVariant"] | null
544+
}
545+
Update: {
546+
created?: string
547+
filehash?: string
548+
filepath?: string
549+
last_modified?: string
550+
source_local_id?: string
551+
space_id?: number
552+
variant?: Database["public"]["Enums"]["ContentVariant"] | null
553+
}
554+
Relationships: [
555+
{
556+
foreignKeyName: "FileReference_content_fkey"
557+
columns: ["space_id", "source_local_id", "variant"]
558+
isOneToOne: false
559+
referencedRelation: "Content"
560+
referencedColumns: ["space_id", "source_local_id", "variant"]
561+
},
562+
{
563+
foreignKeyName: "FileReference_content_fkey"
564+
columns: ["space_id", "source_local_id", "variant"]
565+
isOneToOne: false
566+
referencedRelation: "my_contents"
567+
referencedColumns: ["space_id", "source_local_id", "variant"]
568+
},
569+
{
570+
foreignKeyName: "FileReference_content_fkey"
571+
columns: ["space_id", "source_local_id", "variant"]
572+
isOneToOne: false
573+
referencedRelation: "my_contents_with_embedding_openai_text_embedding_3_small_1536"
574+
referencedColumns: ["space_id", "source_local_id", "variant"]
575+
},
576+
]
577+
}
514578
group_membership: {
515579
Row: {
516580
admin: boolean | null
@@ -1153,6 +1217,33 @@ export type Database = {
11531217
},
11541218
]
11551219
}
1220+
my_file_references: {
1221+
Row: {
1222+
created: string | null
1223+
filehash: string | null
1224+
filepath: string | null
1225+
last_modified: string | null
1226+
source_local_id: string | null
1227+
space_id: number | null
1228+
}
1229+
Insert: {
1230+
created?: string | null
1231+
filehash?: string | null
1232+
filepath?: string | null
1233+
last_modified?: string | null
1234+
source_local_id?: string | null
1235+
space_id?: number | null
1236+
}
1237+
Update: {
1238+
created?: string | null
1239+
filehash?: string | null
1240+
filepath?: string | null
1241+
last_modified?: string | null
1242+
source_local_id?: string | null
1243+
space_id?: number | null
1244+
}
1245+
Relationships: []
1246+
}
11561247
my_spaces: {
11571248
Row: {
11581249
id: number | null
@@ -1434,6 +1525,8 @@ export type Database = {
14341525
Returns: undefined
14351526
}
14361527
extract_references: { Args: { refs: Json }; Returns: number[] }
1528+
file_access: { Args: { hashvalue: string }; Returns: boolean }
1529+
file_exists: { Args: { hashvalue: string }; Returns: boolean }
14371530
generic_entity_access: {
14381531
Args: {
14391532
target_id: number
@@ -1890,3 +1983,4 @@ export const Constants = {
18901983
},
18911984
},
18921985
} as const
1986+

‎packages/database/src/lib/files.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import type { DGSupabaseClient } from "./client";
2+
3+
const ASSETS_BUCKET_NAME = "assets";
4+
5+
export const addFile = async ({
6+
client, spaceId, sourceLocalId, fname, mimetype, created, lastModified, content
7+
}:{
8+
client: DGSupabaseClient,
9+
spaceId: number,
10+
sourceLocalId: string,
11+
fname: string,
12+
mimetype: string,
13+
created: Date,
14+
lastModified: Date,
15+
content: ArrayBuffer
16+
}): Promise<void> => {
17+
// This assumes the content fits in memory.
18+
const uint8Array = new Uint8Array(content);
19+
const hashBuffer = await crypto.subtle.digest('SHA-256', uint8Array);
20+
const hashArray = Array.from(new Uint8Array(hashBuffer));
21+
const hashvalue = hashArray.map((h) => h.toString(16).padStart(2, '0')).join('');
22+
const lookForDup = await client.rpc("file_exists",{hashvalue})
23+
if (lookForDup.error) throw lookForDup.error;
24+
const exists = lookForDup.data;
25+
if (!exists) {
26+
// we should use upsert here for sync issues, but we get obscure rls errors.
27+
const uploadResult = await client.storage.from(ASSETS_BUCKET_NAME).upload(hashvalue, content, {contentType: mimetype});
28+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
29+
if (uploadResult.error && String((uploadResult.error as Record<string, any>).statusCode) !== "409")
30+
throw uploadResult.error;
31+
}
32+
// not doing an upsert because it does not update on conflict
33+
const frefResult = await client.from("FileReference").insert({
34+
/* eslint-disable @typescript-eslint/naming-convention */
35+
space_id: spaceId,
36+
source_local_id: sourceLocalId,
37+
last_modified: lastModified.toISOString(),
38+
/* eslint-enable @typescript-eslint/naming-convention */
39+
filepath: fname,
40+
filehash: hashvalue,
41+
created: created.toISOString()
42+
});
43+
44+
if (frefResult.error) {
45+
if (frefResult.error.code === "23505") {
46+
// 23505 is duplicate key, which means the file is already there, not an error
47+
const updateResult = await client.from("FileReference").update({
48+
// eslint-disable-next-line @typescript-eslint/naming-convention
49+
last_modified: lastModified.toISOString(),
50+
filehash: hashvalue,
51+
created: created.toISOString()
52+
}).eq("source_local_id", sourceLocalId).eq("space_id", spaceId).eq("filepath", fname);
53+
if (updateResult.error) throw updateResult.error;
54+
} else
55+
throw frefResult.error;
56+
}
57+
}

‎packages/database/supabase/config.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ schema_paths = [
5555
'./schemas/account.sql',
5656
'./schemas/content.sql',
5757
'./schemas/embedding.sql',
58+
'./schemas/assets.sql',
5859
'./schemas/concept.sql',
5960
'./schemas/contributor.sql',
6061
'./schemas/sync.sql',

0 commit comments

Comments
 (0)