feat: full basic rag working

Author: lancewilhelm

app/components/Chat/Bubble/ChatBubble.vue

@@ -191,6 +191,16 @@ function resizeTextarea() {
             :model="message.model"
             :usage="message.usage"
           />
+          <ChatBubbleRetrievedKnowledge
+            v-if="
+              message.role === 'assistant' &&
+              message.knowledge &&
+              message.retrievedKnowledge &&
+              message.retrievedKnowledge.length > 0
+            "
+            :knowledge="message.knowledge"
+            :retrieved-knowledge="message.retrievedKnowledge"
+          />
           <div
             class="flex gap-2 text-(--main-color)"
             :class="isButtonRowVisible ? 'opacity-100' : 'opacity-0'"

app/components/Chat/Bubble/ChatBubbleRetrievedKnowledge.vue

@@ -0,0 +1,58 @@
+<script setup lang="ts">
+import type { KnowledgeDocumentResponse } from "~~/server/utils/db/rag";
+
+defineProps<{
+  knowledge: KnowledgeState;
+  retrievedKnowledge: KnowledgeDocumentResponse[];
+}>();
+const isModalOpen = ref(false);
+</script>
+
+<template>
+  <div class="flex items-center">
+    <Icon
+      name="lucide:library"
+      class="cursor-pointer text-(--main-color)"
+      @click="isModalOpen = true"
+    />
+  </div>
+
+  <!-- popup -->
+  <ModalWindow
+    :open="isModalOpen"
+    :shift-for-chat-list="true"
+    @close="isModalOpen = false"
+  >
+    <div class="flex flex-col gap-2 max-h-[80vh]">
+      <div class="flex items-center justify-between">
+        <div class="flex items-center gap-2">
+          <Icon name="lucide:library" class="text-(--main-color) scale-125" />
+          <div class="font-bold">{{ knowledge.name }}</div>
+        </div>
+        <Icon
+          name="lucide:x"
+          class="text-(--main-color) cursor-pointer"
+          @click="isModalOpen = false"
+        />
+      </div>
+      <div class="h-[1px] bg-(--sub-color) header-lines" />
+      <div class="flex flex-col gap-2 overflow-y-auto">
+        <div
+          v-for="doc in retrievedKnowledge"
+          :key="doc.id"
+          class="whitespace-pre-wrap"
+        >
+          <div class="font-bold text-(--main-color) font-mono">
+            {{ doc.source }} - {{ doc.chunkIndex }}
+          </div>
+          <div class="text-sm">
+            {{ doc.text }}
+          </div>
+          <div class="text-(--sub-color) text-xs font-mono">
+            {{ `distance: ${doc._distance}` }}
+          </div>
+        </div>
+      </div>
+    </div>
+  </ModalWindow>
+</template>

app/components/Chat/ChatInput.vue

@@ -49,6 +49,11 @@ const { width } = useWindowSize();
 // Handle file uploads
 const uploadedFiles = ref<MessageFile[]>([]);
 const globalSettingsStore = useGlobalSettingsStore();
+
+// Knowledge
+const knowledge = computed(() => {
+  return useKnowledgeStore().knowledge;
+});
 </script>
 
 <template>
@@ -126,7 +131,9 @@ const globalSettingsStore = useGlobalSettingsStore();
               }
             "
           />
-          <ChatInputKnowledge />
+          <ChatInputKnowledge
+            v-if="knowledge && Object.keys(knowledge).length"
+          />
         </div>
       </div>
       <div class="flex items-center chat-input-right">

app/pages/knowledge/[...id].vue

@@ -6,16 +6,144 @@ const routeId = computed(() => {
   return Array.isArray(route.params.id) ? route.params.id[0] : route.params.id;
 });
 
-const { knowledge, deleteKnowledge } = useKnowledgeStore();
+const { isLoading, knowledge, deleteKnowledge } = useKnowledgeStore();
 const k = computed(() => {
   if (!routeId.value) return null;
-  if (!knowledge[routeId.value]) navigateTo("/knowledge");
+  if (!knowledge[routeId.value] && !isLoading) navigateTo("/knowledge");
+  // getDocuments(knowledge[routeId.value]?.name);
   return knowledge[routeId.value];
 });
 
+// async function getDocuments(knowledgeName?: string) {
+//   if (!knowledgeName) return;
+//   const response = await $fetch("/api/knowledge/document", {
+//     method: "GET",
+//     params: {
+//       knowledgeName,
+//       type: "all",
+//     },
+//   });
+//   if (!response) {
+//     throw new Error("Failed to retrieve knowledge");
+//   }
+//   console.log("response:", response);
+// }
+
 const deleteKnowledgeModalVisible = ref(false);
 const deleteKnowledgeConfirmation = ref("");
 const deleteKnowledgeConfirmationRef = ref<HTMLElement | null>(null);
+const showAddDocuments = ref(false);
+const fileInputRef = ref<HTMLInputElement | null>(null);
+const dropZoneRef = ref<HTMLDivElement | null>(null);
+const pendingFiles = ref<File[]>([]);
+const uploadProgress = ref<{
+  fileName: string;
+  percent: number;
+} | null>(null);
+const uploadStatus = ref(""); // Add a ref to display upload status
+
+function onDrop(files: File[] | null) {
+  if (files && files.length > 0) {
+    // Filter for accepted types if needed, or rely on backend validation
+    pendingFiles.value.push(...files);
+  }
+}
+const { isOverDropZone } = useDropZone(dropZoneRef, {
+  onDrop,
+  // You might want to allow more types like '.pdf', etc.
+  multiple: true,
+});
+
+const triggerFileInput = () => {
+  if (fileInputRef.value) {
+    fileInputRef.value.click();
+  }
+};
+
+const handleFileChange = (event: Event) => {
+  const fileInput = event.target as HTMLInputElement;
+  if (fileInput.files && fileInput.files.length > 0) {
+    const files = Array.from(fileInput.files);
+    // Filter for accepted types if needed
+    pendingFiles.value.push(...files);
+  }
+};
+
+async function uploadFiles() {
+  if (!k.value) return;
+  uploadStatus.value = "Uploading files..."; // Set status to uploading
+
+  if (pendingFiles.value.length === 0) {
+    uploadStatus.value = "No files selected to upload.";
+    console.warn("No files to upload.");
+    return; // Stop the function if no files are present
+  }
+
+  const formData = new FormData();
+  formData.append("provider", k.value.provider);
+  formData.append("dbName", k.value.name);
+
+  // Loop through the array of files and append each one
+  // using the same key name ("documents").
+  for (const file of pendingFiles.value) {
+    formData.append("documents", file);
+  }
+
+  // Handle the file upload
+  try {
+    // $fetch will correctly handle the FormData body for a POST request
+    const response = await fetch("/api/knowledge/document", {
+      method: "POST",
+      body: formData,
+    });
+
+    if (!response.ok || !response.body) {
+      throw new Error(`Failed to create DB: ${response.statusText}`);
+    }
+
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    pendingFiles.value = []; // Clear the pending files after starting the upload
+
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+
+      // Decode the chunk and update the upload status
+      const chunk = decoder.decode(value);
+      const events = parseSSEChunk(chunk);
+      for (const event of events) {
+        if (event.eventType === "progress") {
+          uploadProgress.value = JSON.parse(event.data);
+        } else if (event.eventType === "error") {
+          uploadStatus.value = `Error: ${event.data}`;
+        } else if (event.eventType === "success") {
+          const data = JSON.parse(event.data);
+          const { updateKnowledge, fetchKnowledge } = useKnowledgeStore();
+          updateKnowledge(
+            data.id,
+            data.dbName,
+            data.provider,
+            data.createdAt,
+            data.updatedAt,
+            data.documents,
+            data.chunks,
+          );
+          await fetchKnowledge();
+          showAddDocuments.value = false;
+        }
+      }
+    }
+  } catch (error) {
+    console.error("Error uploading files:", error);
+    uploadStatus.value = `Error uploading files: ${error}`; // Display error message
+  } finally {
+    // Clear the file input value regardless of success or failure
+    if (fileInputRef.value) {
+      fileInputRef.value.value = "";
+    }
+  }
+}
 </script>
 <template>
   <div class="w-full h-full flex flex-col overflow-hidden">
@@ -46,22 +174,118 @@ const deleteKnowledgeConfirmationRef = ref<HTMLElement | null>(null);
           {{ k?.updatedAt && new Date(k?.updatedAt).toLocaleDateString() }}
           {{ k?.updatedAt && new Date(k?.updatedAt).toLocaleTimeString() }}
         </div>
-        <div>documents: {{ k?.details.documents }}</div>
-        <div>chunks: {{ k?.details.chunks }}</div>
-        <button
-          class="bg-(--error-color) rounded-lg"
-          @click="
-            () => {
-              deleteKnowledgeModalVisible = true;
-              nextTick(() => {
-                deleteKnowledgeConfirmationRef?.focus();
-              });
-            }
-          "
-        >
-          <Icon name="lucide:trash-2" class="scale-125" />
-          delete
-        </button>
+        <div>documents: {{ k?.documents }}</div>
+        <div>chunks: {{ k?.chunks }}</div>
+        <div class="flex gap-2">
+          <button
+            class="bg-(--sub-color) rounded-lg"
+            @click="showAddDocuments = !showAddDocuments"
+          >
+            <Icon name="lucide:file-plus" class="scale-125" />
+            add documents
+          </button>
+          <button
+            class="bg-(--error-color) rounded-lg"
+            @click="
+              () => {
+                deleteKnowledgeModalVisible = true;
+                nextTick(() => {
+                  deleteKnowledgeConfirmationRef?.focus();
+                });
+              }
+            "
+          >
+            <Icon name="lucide:trash-2" class="scale-125" />
+            delete
+          </button>
+        </div>
+        <div v-if="showAddDocuments">
+          <div class="h-[1px] bg-(--sub-color) my-4" />
+          <div class="flex flex-col gap-4">
+            <h4>1. add files</h4>
+            <div
+              class="grid grid-cols-[1fr_min-content_2fr] items-center gap-4 h-[200px]"
+            >
+              <div
+                class="h-full flex items-center justify-center gap-3 bg-(--sub-color) rounded-lg cursor-pointer"
+                @click="triggerFileInput"
+                @keydown.enter="triggerFileInput"
+                @keydown.space="triggerFileInput"
+              >
+                <Icon
+                  name="lucide:upload"
+                  class="text-(--main-color) scale-150"
+                />
+                browse local files
+              </div>
+              <input
+                ref="fileInputRef"
+                type="file"
+                multiple
+                accept=".txt,.pdf"
+                class="hidden"
+                @change="handleFileChange"
+              />
+              <div>or</div>
+              <div
+                ref="dropZoneRef"
+                class="h-full flex items-center justify-center bg-(--sub-alt-color) rounded-lg border-3 border-dashed"
+                :class="
+                  isOverDropZone
+                    ? 'border-(--main-color)'
+                    : 'border-(--sub-color)'
+                "
+              >
+                drag and drop files here
+              </div>
+            </div>
+          </div>
+          <div
+            v-if="pendingFiles.length"
+            class="h-[1px] bg-(--sub-color) my-4"
+          />
+          <div v-if="pendingFiles.length" class="flex flex-col gap-4">
+            <h4>2. review files</h4>
+            <div
+              v-for="file in pendingFiles"
+              :key="file.name"
+              class="flex justify-between"
+            >
+              {{ file.name }}
+              <Icon
+                name="lucide:x"
+                class="text-(--error-color) scale-150 cursor-pointer"
+                @click="
+                  () => {
+                    pendingFiles = pendingFiles.filter(
+                      (f) => f.name !== file.name,
+                    );
+                  }
+                "
+              />
+            </div>
+            <button
+              class="bg-(--main-color) text-(--bg-color) rounded-lg p-2"
+              @click="uploadFiles"
+            >
+              3. upload files and create database
+            </button>
+          </div>
+          <!-- Display the upload status -->
+          <div v-if="uploadProgress" class="flex gap-3 items-center">
+            <div>{{ uploadProgress.fileName }}</div>
+            <div class="w-[200px] h-2 bg-(--sub-color) rounded-full">
+              <div
+                class="h-full bg-(--main-color) rounded-full"
+                :style="{
+                  width: (uploadProgress.percent || 0) + '%',
+                }"
+              />
+            </div>
+            <div>{{ uploadProgress.percent || 0 }}%</div>
+          </div>
+          <div v-if="uploadStatus" class="mt-2 text-sm">{{ uploadStatus }}</div>
+        </div>
       </div>
     </div>
 
@@ -92,7 +316,7 @@ const deleteKnowledgeConfirmationRef = ref<HTMLElement | null>(null);
           @keyup.enter="
             async () => {
               if (!k?.id) return;
-              await deleteKnowledge(k?.id);
+              await deleteKnowledge(k?.id, k?.name);
               navigateTo('/knowledge');
             }
           "
@@ -108,7 +332,7 @@ const deleteKnowledgeConfirmationRef = ref<HTMLElement | null>(null);
           @click="
             async () => {
               if (!k?.id) return;
-              await deleteKnowledge(k?.id);
+              await deleteKnowledge(k?.id, k?.name);
               navigateTo('/knowledge');
             }
           "