Skip to content

Commit d79feec

Browse files
maparentmaparent
committed
eng-1180 WIP
1 parent a45aadf commit d79feec

File tree

10 files changed

+601
-27
lines changed

10 files changed

+601
-27
lines changed

‎apps/obsidian/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"@types/node": "^20",
2323
"@types/react": "catalog:obsidian",
2424
"@types/react-dom": "catalog:obsidian",
25+
"@types/mime-types": "3.0.1",
2526
"autoprefixer": "^10.4.21",
2627
"builtin-modules": "3.3.0",
2728
"dotenv": "^16.4.5",
@@ -41,10 +42,11 @@
4142
"@repo/utils": "workspace:*",
4243
"@supabase/supabase-js": "catalog:",
4344
"date-fns": "^4.1.0",
45+
"mime-types": "^3.0.1",
4446
"nanoid": "^4.0.2",
4547
"react": "catalog:obsidian",
4648
"react-dom": "catalog:obsidian",
4749
"tailwindcss-animate": "^1.0.7",
4850
"tldraw": "3.14.2"
4951
}
50-
}
52+
}

‎apps/obsidian/src/utils/publishNode.ts‎

Lines changed: 75 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import type { FrontMatterCache, TFile } from "obsidian";
22
import type { default as DiscourseGraphPlugin } from "~/index";
33
import { getLoggedInClient, getSupabaseContext } from "./supabaseContext";
4+
import { addFile } from "@repo/database/lib/files";
5+
import mime from "mime-types";
46

57
export const publishNode = async ({
68
plugin,
@@ -26,21 +28,80 @@ export const publishNode = async ({
2628
if (!myGroup) throw new Error("Cannot get group");
2729
const existingPublish =
2830
(frontmatter.publishedToGroups as undefined | string[]) || [];
29-
if (existingPublish.includes(myGroup)) return; // already published
30-
const publishResponse = await client.from("ContentAccess").insert({
31-
/* eslint-disable @typescript-eslint/naming-convention */
32-
account_uid: myGroup,
33-
source_local_id: nodeId,
34-
space_id: spaceId,
35-
/* eslint-enable @typescript-eslint/naming-convention */
36-
});
31+
const idResponse = await client
32+
.from("Content")
33+
.select("id,last_modified")
34+
.eq("source_local_id", nodeId)
35+
.eq("space_id", spaceId)
36+
.eq("variant", "full")
37+
.maybeSingle();
38+
if (idResponse.error || !idResponse.data) {
39+
throw idResponse.error || new Error("no data while fetching node");
40+
}
41+
const contentId = idResponse.data.id;
42+
const lastModifiedDb = new Date(idResponse.data.last_modified + "Z");
43+
if (
44+
existingPublish.includes(myGroup) &&
45+
file.stat.mtime <= lastModifiedDb.getTime()
46+
)
47+
return; // already published
48+
const publishResponse = await client.from("ContentAccess").upsert(
49+
{
50+
/* eslint-disable @typescript-eslint/naming-convention */
51+
account_uid: myGroup,
52+
source_local_id: nodeId,
53+
space_id: spaceId,
54+
/* eslint-enable @typescript-eslint/naming-convention */
55+
},
56+
{ ignoreDuplicates: true },
57+
);
3758
if (publishResponse.error && publishResponse.error.code !== "23505")
3859
// 23505 is duplicate key, which counts as a success.
3960
throw publishResponse.error;
40-
await plugin.app.fileManager.processFrontMatter(
41-
file,
42-
(fm: Record<string, unknown>) => {
43-
fm.publishedToGroups = [...existingPublish, myGroup];
44-
},
45-
);
61+
62+
const existingFiles: string[] = [];
63+
const embeds = plugin.app.metadataCache.getFileCache(file)?.embeds ?? [];
64+
for (const { link } of embeds) {
65+
const attachment = plugin.app.metadataCache.getFirstLinkpathDest(
66+
link,
67+
file.path,
68+
);
69+
if (attachment === null) {
70+
console.warn("Could not find file for " + link);
71+
continue;
72+
}
73+
const mimetype = mime.lookup(attachment.path) || "application/octet-stream";
74+
if (mimetype.startsWith("text/")) continue;
75+
existingFiles.push(attachment.path);
76+
const content = await plugin.app.vault.readBinary(attachment);
77+
await addFile({
78+
client,
79+
spaceId,
80+
contentId,
81+
fname: attachment.path,
82+
mimetype,
83+
created: new Date(attachment.stat.ctime),
84+
lastModified: new Date(attachment.stat.mtime),
85+
content,
86+
});
87+
}
88+
let cleanupCommand = client
89+
.from("FileReference")
90+
.delete()
91+
.eq("content_id", contentId);
92+
if (existingFiles.length)
93+
cleanupCommand = cleanupCommand.notIn("filepath", [
94+
...new Set(existingFiles),
95+
]);
96+
const cleanupResult = await cleanupCommand;
97+
// do not fail on cleanup
98+
if (cleanupResult.error) console.error(cleanupResult.error);
99+
100+
if (!existingPublish.includes(myGroup))
101+
await plugin.app.fileManager.processFrontMatter(
102+
file,
103+
(fm: Record<string, unknown>) => {
104+
fm.publishedToGroups = [...existingPublish, myGroup];
105+
},
106+
);
46107
};

‎packages/database/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
"@repo/utils": "workspace:*",
4646
"@supabase/auth-js": "catalog:",
4747
"@supabase/functions-js": "catalog:",
48+
"@supabase/storage-js": "catalog:",
4849
"@supabase/supabase-js": "catalog:",
4950
"tslib": "2.5.1"
5051
},

‎packages/database/src/dbTypes.ts‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -529,6 +529,81 @@ export type Database = {
529529
},
530530
]
531531
}
532+
file_gc: {
533+
Row: {
534+
filepath: string
535+
}
536+
Insert: {
537+
filepath: string
538+
}
539+
Update: {
540+
filepath?: string
541+
}
542+
Relationships: []
543+
}
544+
FileReference: {
545+
Row: {
546+
content_id: number
547+
created: string
548+
filehash: string
549+
filepath: string
550+
last_modified: string
551+
space_id: number
552+
}
553+
Insert: {
554+
content_id: number
555+
created: string
556+
filehash: string
557+
filepath: string
558+
last_modified: string
559+
space_id: number
560+
}
561+
Update: {
562+
content_id?: number
563+
created?: string
564+
filehash?: string
565+
filepath?: string
566+
last_modified?: string
567+
space_id?: number
568+
}
569+
Relationships: [
570+
{
571+
foreignKeyName: "FileReference_content_id_fkey"
572+
columns: ["content_id"]
573+
isOneToOne: false
574+
referencedRelation: "Content"
575+
referencedColumns: ["id"]
576+
},
577+
{
578+
foreignKeyName: "FileReference_content_id_fkey"
579+
columns: ["content_id"]
580+
isOneToOne: false
581+
referencedRelation: "my_contents"
582+
referencedColumns: ["id"]
583+
},
584+
{
585+
foreignKeyName: "FileReference_content_id_fkey"
586+
columns: ["content_id"]
587+
isOneToOne: false
588+
referencedRelation: "my_contents_with_embedding_openai_text_embedding_3_small_1536"
589+
referencedColumns: ["id"]
590+
},
591+
{
592+
foreignKeyName: "FileReference_space_id_fkey"
593+
columns: ["space_id"]
594+
isOneToOne: false
595+
referencedRelation: "my_spaces"
596+
referencedColumns: ["id"]
597+
},
598+
{
599+
foreignKeyName: "FileReference_space_id_fkey"
600+
columns: ["space_id"]
601+
isOneToOne: false
602+
referencedRelation: "Space"
603+
referencedColumns: ["id"]
604+
},
605+
]
606+
}
532607
group_membership: {
533608
Row: {
534609
admin: boolean | null
@@ -1153,6 +1228,69 @@ export type Database = {
11531228
},
11541229
]
11551230
}
1231+
my_file_references: {
1232+
Row: {
1233+
content_id: number | null
1234+
created: string | null
1235+
filehash: string | null
1236+
filepath: string | null
1237+
last_modified: string | null
1238+
space_id: number | null
1239+
}
1240+
Insert: {
1241+
content_id?: number | null
1242+
created?: string | null
1243+
filehash?: string | null
1244+
filepath?: string | null
1245+
last_modified?: string | null
1246+
space_id?: number | null
1247+
}
1248+
Update: {
1249+
content_id?: number | null
1250+
created?: string | null
1251+
filehash?: string | null
1252+
filepath?: string | null
1253+
last_modified?: string | null
1254+
space_id?: number | null
1255+
}
1256+
Relationships: [
1257+
{
1258+
foreignKeyName: "FileReference_content_id_fkey"
1259+
columns: ["content_id"]
1260+
isOneToOne: false
1261+
referencedRelation: "Content"
1262+
referencedColumns: ["id"]
1263+
},
1264+
{
1265+
foreignKeyName: "FileReference_content_id_fkey"
1266+
columns: ["content_id"]
1267+
isOneToOne: false
1268+
referencedRelation: "my_contents"
1269+
referencedColumns: ["id"]
1270+
},
1271+
{
1272+
foreignKeyName: "FileReference_content_id_fkey"
1273+
columns: ["content_id"]
1274+
isOneToOne: false
1275+
referencedRelation: "my_contents_with_embedding_openai_text_embedding_3_small_1536"
1276+
referencedColumns: ["id"]
1277+
},
1278+
{
1279+
foreignKeyName: "FileReference_space_id_fkey"
1280+
columns: ["space_id"]
1281+
isOneToOne: false
1282+
referencedRelation: "my_spaces"
1283+
referencedColumns: ["id"]
1284+
},
1285+
{
1286+
foreignKeyName: "FileReference_space_id_fkey"
1287+
columns: ["space_id"]
1288+
isOneToOne: false
1289+
referencedRelation: "Space"
1290+
referencedColumns: ["id"]
1291+
},
1292+
]
1293+
}
11561294
my_spaces: {
11571295
Row: {
11581296
id: number | null
@@ -1438,6 +1576,7 @@ export type Database = {
14381576
Returns: undefined
14391577
}
14401578
extract_references: { Args: { refs: Json }; Returns: number[] }
1579+
file_exists: { Args: { hashvalue: string }; Returns: boolean }
14411580
generic_entity_access: {
14421581
Args: {
14431582
target_id: number

‎packages/database/src/lib/files.ts‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import type { DGSupabaseClient } from "./client";
2+
3+
const ASSETS_BUCKET_NAME = "assets";
4+
5+
export const addFile = async ({
6+
client, spaceId, contentId, fname, mimetype, created, lastModified, content
7+
}:{
8+
client: DGSupabaseClient,
9+
spaceId: number,
10+
contentId: number,
11+
fname: string,
12+
mimetype: string,
13+
created: Date,
14+
lastModified: Date,
15+
content: ArrayBuffer
16+
}): Promise<void> => {
17+
// This assumes the content fits in memory.
18+
const uint8Array = new Uint8Array(content);
19+
const hashBuffer = await crypto.subtle.digest('SHA-256', uint8Array);
20+
const hashArray = Array.from(new Uint8Array(hashBuffer));
21+
const hashvalue = hashArray.map((h) => h.toString(16).padStart(2, '0')).join('');
22+
const lookForDup = await client.rpc("file_exists",{hashvalue})
23+
if (lookForDup.error) throw lookForDup.error;
24+
const exists = lookForDup.data;
25+
if (!exists) {
26+
// we should use upsert here for sync issues, but we get obscure rls errors.
27+
const uploadResult = await client.storage.from(ASSETS_BUCKET_NAME).upload(hashvalue, content, {contentType: mimetype});
28+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
29+
if (uploadResult.error && String((uploadResult.error as Record<string, any>).statusCode) !== "409")
30+
throw uploadResult.error;
31+
}
32+
// not doing an upsert because it does not update on conflict
33+
const frefResult = await client.from("FileReference").insert({
34+
/* eslint-disable @typescript-eslint/naming-convention */
35+
space_id: spaceId,
36+
content_id: contentId,
37+
last_modified: lastModified.toISOString(),
38+
/* eslint-enable @typescript-eslint/naming-convention */
39+
filepath: fname,
40+
filehash: hashvalue,
41+
created: created.toISOString()
42+
});
43+
44+
if (frefResult.error) {
45+
if (frefResult.error.code === "23505") {
46+
const updateResult = await client.from("FileReference").update({
47+
/* eslint-disable @typescript-eslint/naming-convention */
48+
space_id: spaceId,
49+
last_modified: lastModified.toISOString(),
50+
/* eslint-enable @typescript-eslint/naming-convention */
51+
filehash: hashvalue,
52+
created: created.toISOString()
53+
}).eq("content_id", contentId).eq("filepath", fname);
54+
if (updateResult.error) throw updateResult.error;
55+
} else
56+
throw frefResult.error;
57+
}
58+
}

‎packages/database/supabase/config.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ schema_paths = [
5555
'./schemas/account.sql',
5656
'./schemas/content.sql',
5757
'./schemas/embedding.sql',
58+
'./schemas/assets.sql',
5859
'./schemas/concept.sql',
5960
'./schemas/contributor.sql',
6061
'./schemas/sync.sql',

0 commit comments

Comments
 (0)