implemented very simple priority-based deduplication

Author: adarshramiyer

core/autocomplete/templating/filtering.ts

@@ -130,6 +130,9 @@ export const getSnippets = (
   const finalSnippets = [];
   let remainingTokenCount = getRemainingTokenCount(helper);
 
+  // tracks already added filepaths for deduplication
+  const addedFilepaths = new Set<string>();
+
   // Process snippets in priority order
   for (const { key } of snippetOrder) {
     // Special handling for recentlyOpenedFiles
@@ -160,9 +163,10 @@ export const getSnippets = (
 
         if (remainingTokenCount >= snippetSize) {
           finalSnippets.push(snippet);
+          addedFilepaths.add(snippet.filepath);
           remainingTokenCount -= snippetSize;
         } else {
-          break; // Out of tokens
+          continue; // Not enough tokens, try again with next snippet
         }
       }
     } else {
@@ -171,7 +175,9 @@ export const getSnippets = (
         (snippet) =>
           !(snippet as AutocompleteCodeSnippet).filepath?.startsWith(
             "output:extension-output-Continue.continue",
-          ),
+          ) &&
+          ((snippet as AutocompleteCodeSnippet).filepath === undefined ||
+            !addedFilepaths.has((snippet as AutocompleteCodeSnippet).filepath)),
       );
 
       for (const snippet of snippetsToProcess) {
@@ -182,9 +188,14 @@ export const getSnippets = (
 
         if (remainingTokenCount >= snippetSize) {
           finalSnippets.push(snippet);
+
+          if ((snippet as AutocompleteCodeSnippet).filepath) {
+            addedFilepaths.add((snippet as AutocompleteCodeSnippet).filepath);
+          }
+
           remainingTokenCount -= snippetSize;
         } else {
-          break; // Out of tokens
+          continue; // Not enough tokens, try again with next snippet
         }
       }
     }

core/autocomplete/templating/formatOpenedFilesContext.ts

@@ -2,6 +2,7 @@ import { countTokens, pruneStringFromBottom } from "../../llm/countTokens";
 import {
   AutocompleteCodeSnippet,
   AutocompleteSnippet,
+  AutocompleteSnippetType,
 } from "../snippets/types";
 import { HelperVars } from "../util/HelperVars";
 
@@ -20,13 +21,23 @@ export function formatOpenedFilesContext(
   recentlyOpenedFilesSnippets: AutocompleteCodeSnippet[],
   remainingTokenCount: number,
   helper: HelperVars,
-  alreadyAddedSnippets: AutocompleteSnippet[], // TODO use this to deduplicate context
+  alreadyAddedSnippets: AutocompleteSnippet[],
   TOKEN_BUFFER: number,
 ): AutocompleteCodeSnippet[] {
   if (recentlyOpenedFilesSnippets.length === 0) {
     return [];
   }
 
+  // deduplication; if a snippet is already added, don't include it here
+  for (const snippet of alreadyAddedSnippets) {
+    if (snippet.type !== AutocompleteSnippetType.Code) {
+      continue;
+    }
+    recentlyOpenedFilesSnippets = recentlyOpenedFilesSnippets.filter(
+      (s) => s.filepath !== snippet.filepath,
+    );
+  }
+
   // Calculate how many full snippets would fit within the remaining token count
   let numSnippetsThatFit = 0;
   let totalTokens = 0;