chat search -- quick proof of concept

Author: williamstein

src/packages/frontend/chat/filter-messages.ts

@@ -30,7 +30,7 @@ export function filterMessages({
     // no filters -- typical special case; waste now time.
     return messages;
   }
-  const searchData = getSearchData(messages);
+  const searchData = getSearchData({ messages, threads: true });
   let matchingRootTimes: Set<string>;
   if (filter) {
     matchingRootTimes = new Set<string>();
@@ -107,23 +107,37 @@ type SearchData = {
 
 const cache = new LRU<ChatMessages, SearchData>({ max: 25 });
 
-function getSearchData(messages): SearchData {
+export function getSearchData({
+  messages,
+  threads,
+}: {
+  messages: ChatMessages;
+  threads: boolean;
+}): SearchData {
   if (cache.has(messages)) {
     return cache.get(messages)!;
   }
   const data: SearchData = {};
   const userMap = redux.getStore("users").get("user_map");
-  for (const [time, message] of messages) {
+  for (let [time, message] of messages) {
+    if (typeof time != "string") {
+      // for typescript
+      time = `${time}`;
+    }
+    const messageTime = parseFloat(time);
+    const content = getContent(message, userMap);
+    if (!threads) {
+      data[time] = { content, newestTime: messageTime };
+      continue;
+    }
     let rootTime: string;
     if (message.get("reply_to")) {
       // non-root in thread
-      rootTime = `${new Date(message.get("reply_to")).valueOf()}`;
+      rootTime = `${new Date(message.get("reply_to")!).valueOf()}`;
     } else {
       // new root thread
       rootTime = time;
     }
-    const messageTime = parseFloat(time);
-    const content = getContent(message, userMap);
     if (data[rootTime] == null) {
       data[rootTime] = {
         content,

src/packages/frontend/frame-editors/chat-editor/editor.ts

@@ -14,6 +14,7 @@ import { createEditor } from "@cocalc/frontend/frame-editors/frame-tree/editor";
 import type { EditorDescription } from "@cocalc/frontend/frame-editors/frame-tree/types";
 import { terminal } from "@cocalc/frontend/frame-editors/terminal-editor/editor";
 import { time_travel } from "@cocalc/frontend/frame-editors/time-travel-editor/editor";
+import { search } from "./search";
 
 const chatroom: EditorDescription = {
   type: "chatroom",
@@ -39,6 +40,7 @@ const chatroom: EditorDescription = {
     "chatgpt",
     "scrollToBottom",
     "scrollToTop",
+    "show_search",
   ]),
   customizeCommands: {
     scrollToTop: {
@@ -59,13 +61,15 @@ const chatroom: EditorDescription = {
     "increase_font_size",
     "scrollToTop",
     "scrollToBottom",
+    "show_search",
   ]),
 } as const;
 
 const EDITOR_SPEC = {
   chatroom,
   terminal,
   time_travel,
+  search,
 } as const;
 
 export const Editor = createEditor({

src/packages/frontend/frame-editors/chat-editor/search.tsx

@@ -0,0 +1,98 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
+ *  License: MS-RSL – see LICENSE.md for details
+ */
+
+/*
+Full text search that is better than a simple filter.
+*/
+
+import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context";
+import type { EditorDescription } from "@cocalc/frontend/frame-editors/frame-tree/types";
+import { Button, Card, Input } from "antd";
+import { set } from "@cocalc/util/misc";
+import { useEffect, useMemo, useState } from "react";
+import { throttle } from "lodash";
+import useSearchIndex from "./use-search-index";
+import ShowError from "@cocalc/frontend/components/error";
+
+interface Props {
+  font_size: number;
+  desc;
+}
+
+function Search({ font_size, desc }: Props) {
+  const { project_id, path, actions, id } = useFrameContext();
+  const [search, setSearch] = useState<string>(desc.get("data-search") ?? "");
+  const [result, setResult] = useState<any>(null);
+  const saveSearch = useMemo(
+    () =>
+      throttle((search) => {
+        if (!actions.isClosed()) {
+          actions.set_frame_data({ id, search });
+        }
+      }, 250),
+    [project_id, path],
+  );
+
+  const { error, setError, index, doRefresh } = useSearchIndex();
+
+  useEffect(() => {
+    if (index == null) {
+      return;
+    }
+    if (!search.trim()) {
+      setResult([]);
+      return;
+    }
+    (async () => {
+      const result = await index.search({ term: search });
+      setResult(result);
+    })();
+  }, [search, index]);
+
+  return (
+    <div className="smc-vfill">
+      <Card
+        title={
+          <>
+            Search {path}
+            <Button
+              onClick={() => {
+                doRefresh();
+              }}
+              style={{ float: "right" }}
+            >
+              Refresh
+            </Button>
+          </>
+        }
+        style={{ fontSize: font_size }}
+      >
+        <ShowError error={error} setError={setError} />
+        <Input.Search
+          allowClear
+          placeholder="Search for messages..."
+          value={search}
+          onChange={(e) => {
+            const search = e.target.value ?? "";
+            setSearch(search);
+            saveSearch(search);
+          }}
+        />
+      </Card>
+      <pre className="smc-vfill" style={{ overflow: "auto" }}>
+        {JSON.stringify(result, undefined, 2)}
+      </pre>
+    </div>
+  );
+}
+
+export const search = {
+  type: "search",
+  short: "Search",
+  name: "Search",
+  icon: "comment",
+  commands: set(["decrease_font_size", "increase_font_size"]),
+  component: Search,
+} as EditorDescription;

src/packages/frontend/frame-editors/chat-editor/use-search-index.ts

@@ -0,0 +1,72 @@
+import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context";
+import { useEffect, useState } from "react";
+import { create, search, insertMultiple } from "@orama/orama";
+import { getSearchData } from "@cocalc/frontend/chat/filter-messages";
+import useCounter from "@cocalc/frontend/app-framework/counter-hook";
+
+export default function useSearchIndex() {
+  const { actions, project_id, path } = useFrameContext();
+  const [index, setIndex] = useState<null | SearchIndex>(null);
+  const [error, setError] = useState<string>("");
+  const { val: refresh, inc: doRefresh } = useCounter();
+
+  useEffect(() => {
+    (async () => {
+      try {
+        setError("");
+        const index = new SearchIndex({ actions });
+        await index.init();
+        setIndex(index);
+      } catch (err) {
+        setError(`${err}`);
+      }
+    })();
+  }, [project_id, path, refresh]);
+
+  return { index, error, doRefresh, setError };
+}
+
+class SearchIndex {
+  private actions;
+  private state: "init" | "ready" | "failed" = "init";
+  private error: Error | null = null;
+  private db;
+
+  constructor({ actions }) {
+    this.actions = actions;
+  }
+
+  getState = () => this.state;
+  getError = () => this.error;
+
+  search = async (query) => {
+    if (this.state != "ready") {
+      throw Error("index not ready");
+    }
+    return await search(this.db, query);
+  };
+
+  init = async () => {
+    this.db = await create({
+      schema: {
+        time: "number",
+        message: "string",
+      },
+    });
+
+    const messages = this.actions.store?.get("messages");
+    if (messages == null) {
+      return;
+    }
+    const searchData = getSearchData({ messages, threads: false });
+    const docs: { time: number; message: string }[] = [];
+    for (const time in searchData) {
+      docs.push({
+        time: parseInt(time),
+        message: searchData[time]?.content ?? "",
+      });
+    }
+    await insertMultiple(this.db, docs);
+    this.state = "ready";
+  };
+}