Skip to content

Commit fa2e614

Browse files
sid597sid597
authored
ENG-1074 Prod duplicate node alert on page, closes ENG-1095 (#564)
* duplicate node alert on page * add to node create dialog, only show small list, fix lint errors, fix bug to show in all open pages * remove duplication vector match shape * address review * optimise discourse node page observer, run isDiscourseNode only once * remove overlay from node dialog, remove usenoecontext, pass node from top of initialise discourse graph, create util * use handleTitleAddition .. only show when suggestive mode is enabled in a graph * address review suggestions
1 parent c6dca4a commit fa2e614

File tree

5 files changed

+320
-53
lines changed

5 files changed

+320
-53
lines changed

‎apps/roam/src/components/VectorDuplicateMatches.tsx‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
import React, { useEffect, useState, useMemo } from "react";
2+
import { Collapse, Spinner, Icon } from "@blueprintjs/core";
3+
import {
4+
findSimilarNodesVectorOnly as vectorSearch,
5+
type VectorMatch,
6+
} from "~/utils/hyde";
7+
import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle";
8+
import { DiscourseNode } from "~/utils/getDiscourseNodes";
9+
import extractContentFromTitle from "~/utils/extractContentFromTitle";
10+
import { handleTitleAdditions } from "~/utils/handleTitleAdditions";
11+
12+
export const VectorDuplicateMatches = ({
13+
pageTitle,
14+
text,
15+
limit = 15,
16+
node,
17+
}: {
18+
pageTitle?: string;
19+
text?: string;
20+
limit?: number;
21+
node: DiscourseNode;
22+
}) => {
23+
const [debouncedText, setDebouncedText] = useState(text);
24+
useEffect(() => {
25+
const handler = setTimeout(() => {
26+
setDebouncedText(text);
27+
}, 500);
28+
return () => {
29+
clearTimeout(handler);
30+
};
31+
}, [text]);
32+
33+
const [isOpen, setIsOpen] = useState(false);
34+
const [suggestionsLoading, setSuggestionsLoading] = useState(false);
35+
const [hasSearched, setHasSearched] = useState(false);
36+
const [suggestions, setSuggestions] = useState<VectorMatch[]>([]);
37+
38+
const searchText = extractContentFromTitle(pageTitle || "", node);
39+
const pageUid = getPageUidByPageTitle(searchText);
40+
const activeContext = useMemo(
41+
() =>
42+
text !== undefined
43+
? { searchText: debouncedText || "", pageUid: null }
44+
: { searchText, pageUid },
45+
[text, debouncedText, searchText, pageUid],
46+
);
47+
48+
useEffect(() => {
49+
setHasSearched(false);
50+
}, [activeContext?.searchText]);
51+
52+
useEffect(() => {
53+
let isCancelled = false;
54+
const fetchSuggestions = async () => {
55+
if (!isOpen || hasSearched) return;
56+
if (!activeContext || !activeContext.searchText.trim()) return;
57+
58+
const { searchText, pageUid } = activeContext;
59+
60+
setSuggestionsLoading(true);
61+
try {
62+
const raw = await vectorSearch({
63+
text: searchText,
64+
threshold: 0.3,
65+
limit,
66+
});
67+
const results: VectorMatch[] = raw.filter((candidate) => {
68+
const sameUid = !!pageUid && candidate.node.uid === pageUid;
69+
return !sameUid;
70+
});
71+
if (!isCancelled) {
72+
setSuggestions(results);
73+
setSuggestionsLoading(false);
74+
setHasSearched(true);
75+
}
76+
} catch (error: unknown) {
77+
console.error("Error fetching vector duplicates:", error);
78+
if (!isCancelled) {
79+
setSuggestionsLoading(false);
80+
}
81+
}
82+
};
83+
void fetchSuggestions();
84+
return () => {
85+
isCancelled = true;
86+
};
87+
}, [isOpen, hasSearched, activeContext, pageTitle, limit]);
88+
89+
const handleSuggestionClick = async (node: VectorMatch["node"]) => {
90+
await window.roamAlphaAPI.ui.rightSidebar.addWindow({
91+
window: {
92+
type: "outline",
93+
// @ts-expect-error - type definition mismatch
94+
// eslint-disable-next-line @typescript-eslint/naming-convention
95+
"block-uid": node.uid,
96+
},
97+
});
98+
};
99+
100+
if (!activeContext) {
101+
return null;
102+
}
103+
104+
const hasSuggestions = suggestions.length > 0;
105+
106+
return (
107+
<div className="my-2 rounded border border-gray-200">
108+
<div
109+
className="flex cursor-pointer items-center justify-between p-2"
110+
onClick={() => {
111+
setIsOpen(!isOpen);
112+
}}
113+
>
114+
<div className="flex items-center gap-2">
115+
<Icon icon={isOpen ? "chevron-down" : "chevron-right"} />
116+
<h5 className="m-0 font-semibold">Possible Duplicates</h5>
117+
</div>
118+
{hasSearched && !suggestionsLoading && hasSuggestions && (
119+
<span className="rounded-full bg-orange-500 px-2 py-0.5 text-xs text-white">
120+
{suggestions.length}
121+
</span>
122+
)}
123+
</div>
124+
125+
<Collapse isOpen={isOpen}>
126+
<div className="border-t border-gray-200 p-2">
127+
{suggestionsLoading && (
128+
<div className="ml-2 flex items-center gap-2 py-4">
129+
<Spinner size={20} />
130+
<span className="text-sm text-gray-600">
131+
Searching for duplicates...
132+
</span>
133+
</div>
134+
)}
135+
136+
{!suggestionsLoading && hasSearched && !hasSuggestions && (
137+
<p className="py-2 text-sm text-gray-600">No matches found.</p>
138+
)}
139+
140+
{!suggestionsLoading && hasSearched && hasSuggestions && (
141+
<ul className="flex flex-col gap-1">
142+
{suggestions.map((match) => (
143+
<li key={match.node.uid} className="flex items-start gap-2">
144+
<a
145+
onClick={() => {
146+
void handleSuggestionClick(match.node);
147+
}}
148+
className="min-w-0 flex-1 cursor-pointer break-words text-blue-600 opacity-70 hover:underline"
149+
>
150+
{match.node.text}
151+
</a>
152+
</li>
153+
))}
154+
</ul>
155+
)}
156+
</div>
157+
</Collapse>
158+
</div>
159+
);
160+
};
161+
162+
export const renderPossibleDuplicates = (
163+
h1: HTMLHeadingElement,
164+
title: string,
165+
node: DiscourseNode,
166+
) => {
167+
handleTitleAdditions(
168+
h1,
169+
<VectorDuplicateMatches pageTitle={title} node={node} />,
170+
);
171+
};

‎apps/roam/src/utils/extractContentFromTitle.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import getDiscourseNodeFormatExpression from "./getDiscourseNodeFormatExpression";
2+
3+
const extractContentFromTitle = (
4+
title: string,
5+
node: { format: string },
6+
): string => {
7+
if (!node.format) return title;
8+
const placeholderRegex = /{([\w\d-]+)}/g;
9+
const placeholders: string[] = [];
10+
let placeholderMatch: RegExpExecArray | null = null;
11+
while ((placeholderMatch = placeholderRegex.exec(node.format))) {
12+
placeholders.push(placeholderMatch[1]);
13+
}
14+
const expression = getDiscourseNodeFormatExpression(node.format);
15+
const expressionMatch = expression.exec(title);
16+
if (!expressionMatch || expressionMatch.length <= 1) {
17+
return title;
18+
}
19+
const contentIndex = placeholders.findIndex(
20+
(name) => name.toLowerCase() === "content",
21+
);
22+
if (contentIndex >= 0) {
23+
return expressionMatch[contentIndex + 1]?.trim() || title;
24+
}
25+
return expressionMatch[1]?.trim() || title;
26+
};
27+
28+
export default extractContentFromTitle;

‎apps/roam/src/utils/hyde.ts‎

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { getLoggedInClient, getSupabaseContext } from "./supabaseContext";
22
import { Result } from "./types";
33
import normalizePageTitle from "roamjs-components/queries/normalizePageTitle";
4+
import { render as renderToast } from "roamjs-components/components/Toast";
45
import findDiscourseNode from "./findDiscourseNode";
56
import { nextApiRoot } from "@repo/utils/execContext";
67
import { DiscourseNode } from "./getDiscourseNodes";
@@ -59,7 +60,7 @@ type EmbeddingFunc = (text: string) => Promise<EmbeddingVectorType>;
5960

6061
type SearchFunc = (params: {
6162
queryEmbedding: EmbeddingVectorType;
62-
indexData: CandidateNodeWithEmbedding[];
63+
indexData: Result[];
6364
}) => Promise<NodeSearchResult[]>;
6465

6566
const API_CONFIG = {
@@ -509,3 +510,62 @@ export const performHydeSearch = async ({
509510
}
510511
return [];
511512
};
513+
514+
export type VectorMatch = {
515+
node: Result;
516+
score: number;
517+
};
518+
519+
export const findSimilarNodesVectorOnly = async ({
520+
text,
521+
threshold = 0.4,
522+
limit = 15,
523+
}: {
524+
text: string;
525+
threshold?: number;
526+
limit?: number;
527+
}): Promise<VectorMatch[]> => {
528+
if (!text.trim()) {
529+
return [];
530+
}
531+
532+
try {
533+
const supabase = await getLoggedInClient();
534+
if (!supabase) return [];
535+
536+
const queryEmbedding = await createEmbedding(text);
537+
538+
const { data, error } = await supabase.rpc("match_content_embeddings", {
539+
query_embedding: JSON.stringify(queryEmbedding),
540+
match_threshold: threshold,
541+
match_count: limit,
542+
});
543+
544+
if (error) {
545+
console.error("Vector search failed:", error);
546+
throw error;
547+
}
548+
549+
if (!data || !Array.isArray(data)) return [];
550+
551+
const results: VectorMatch[] = data.map((item) => ({
552+
node: {
553+
uid: item.roam_uid,
554+
text: item.text_content,
555+
},
556+
score: item.similarity,
557+
}));
558+
559+
return results;
560+
} catch (error) {
561+
console.error("Error in vector-only similar nodes search:", error);
562+
renderToast({
563+
content: `Error in vector-only similar nodes search: ${
564+
error instanceof Error ? error.message : String(error)
565+
}`,
566+
intent: "danger",
567+
id: "vector-search-error",
568+
});
569+
return [];
570+
}
571+
};

‎apps/roam/src/utils/initializeObserversAndListeners.ts‎

Lines changed: 31 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
getPageTitleValueByHtmlElement,
55
} from "roamjs-components/dom";
66
import { createBlock } from "roamjs-components/writes";
7-
import { renderLinkedReferenceAdditions } from "~/utils/renderLinkedReferenceAdditions";
7+
import { renderDiscourseContextAndCanvasReferences } from "~/utils/renderLinkedReferenceAdditions";
88
import { createConfigObserver } from "roamjs-components/components/ConfigPage";
99
import {
1010
renderTldrawCanvas,
@@ -54,6 +54,9 @@ import { getUidAndBooleanSetting } from "./getExportSettings";
5454
import { getCleanTagText } from "~/components/settings/NodeConfig";
5555
import getPleasingColors from "@repo/utils/getPleasingColors";
5656
import { colord } from "colord";
57+
import { renderPossibleDuplicates } from "~/components/VectorDuplicateMatches";
58+
import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle";
59+
import findDiscourseNode from "./findDiscourseNode";
5760

5861
const debounce = (fn: () => void, delay = 250) => {
5962
let timeout: number;
@@ -85,24 +88,40 @@ export const initObservers = async ({
8588
const title = getPageTitleValueByHtmlElement(h1);
8689
const props = { title, h1, onloadArgs };
8790

91+
const isSuggestiveModeEnabled = getUidAndBooleanSetting({
92+
tree: getBasicTreeByParentUid(
93+
getPageUidByPageTitle(DISCOURSE_CONFIG_PAGE_TITLE),
94+
),
95+
text: "(BETA) Suggestive Mode Enabled",
96+
}).value;
97+
98+
const nodes = getDiscourseNodes();
99+
const node = findDiscourseNode({ title, nodes });
100+
const isDiscourseNode = node && node.backedBy !== "default";
101+
if (isDiscourseNode) {
102+
const uid = getPageUidByPageTitle(title);
103+
if (isSuggestiveModeEnabled) {
104+
renderPossibleDuplicates(h1, title, node);
105+
}
106+
const linkedReferencesDiv = document.querySelector(
107+
".rm-reference-main",
108+
) as HTMLDivElement;
109+
if (linkedReferencesDiv) {
110+
renderDiscourseContextAndCanvasReferences(
111+
linkedReferencesDiv,
112+
uid,
113+
onloadArgs,
114+
);
115+
}
116+
}
117+
88118
if (isNodeConfigPage(title)) renderNodeConfigPage(props);
89119
else if (isQueryPage(props)) renderQueryPage(props);
90120
else if (isCurrentPageCanvas(props)) renderTldrawCanvas(props);
91121
else if (isSidebarCanvas(props)) renderTldrawCanvasInSidebar(props);
92122
},
93123
});
94124

95-
// TODO: contains roam query: https://github.com/DiscourseGraphs/discourse-graph/issues/39
96-
const linkedReferencesObserver = createHTMLObserver({
97-
tag: "DIV",
98-
useBody: true,
99-
className: "rm-reference-main",
100-
callback: async (el) => {
101-
const div = el as HTMLDivElement;
102-
await renderLinkedReferenceAdditions(div, onloadArgs);
103-
},
104-
});
105-
106125
const queryBlockObserver = createButtonObserver({
107126
attribute: "query-block",
108127
render: (b) => renderQueryBlock(b, onloadArgs),
@@ -391,7 +410,6 @@ export const initObservers = async ({
391410
pageTitleObserver,
392411
queryBlockObserver,
393412
configPageObserver,
394-
linkedReferencesObserver,
395413
graphOverviewExportObserver,
396414
nodeTagPopupButtonObserver,
397415
leftSidebarObserver,

0 commit comments

Comments
 (0)