Merge pull request #7302 from continuedev/dallin/prevent-unsafe-file-reads

feat: prevent unsafe file reads

Author: tomasz-stefaniak

.github/workflows/cli-pr-checks.yml

@@ -25,6 +25,14 @@ jobs:
           cache: "npm"
           cache-dependency-path: extensions/cli/package-lock.json
 
+      - name: Setup packages
+        uses: ./.github/actions/setup-packages
+      - name: Setup core component
+        uses: ./.github/actions/setup-component
+        with:
+          component: core
+          include-root: true
+
       - name: Install dependencies
         run: |
           cd extensions/cli
@@ -53,6 +61,14 @@ jobs:
           cache: "npm"
           cache-dependency-path: extensions/cli/package-lock.json
 
+      - name: Setup packages
+        uses: ./.github/actions/setup-packages
+      - name: Setup core component
+        uses: ./.github/actions/setup-component
+        with:
+          component: core
+          include-root: true
+
       - name: Install dependencies
         run: |
           cd extensions/cli

.vscode/settings.json

@@ -42,7 +42,8 @@
     "extensions/vscode/e2e/_output": true,
     "extensions/vscode/e2e/storage": true,
     "extensions/vscode/e2e/vsix": true,
-    "extensions/.continue-debug": true
+    "extensions/.continue-debug": true,
+    "extensions/cli/dist/**": true
     // "sync/**": true
   },
   "editor.formatOnSave": true,

binary/package-lock.json

@@ -6,12 +6,13 @@ import { DEFAULT_AUTOCOMPLETE_OPTS } from "../util/parameters.js";
 import { shouldCompleteMultiline } from "./classification/shouldCompleteMultiline.js";
 import { ContextRetrievalService } from "./context/ContextRetrievalService.js";
 
+import { isSecurityConcern } from "../indexing/ignore.js";
 import { BracketMatchingService } from "./filtering/BracketMatchingService.js";
 import { CompletionStreamer } from "./generation/CompletionStreamer.js";
 import { postprocessCompletion } from "./postprocessing/index.js";
 import { shouldPrefilter } from "./prefiltering/index.js";
-import { renderPromptWithTokenLimit } from "./templating/index.js";
 import { getAllSnippetsWithoutRace } from "./snippets/index.js";
+import { renderPromptWithTokenLimit } from "./templating/index.js";
 import { GetLspDefinitionsFunction } from "./types.js";
 import { AutocompleteDebouncer } from "./util/AutocompleteDebouncer.js";
 import { AutocompleteLoggingService } from "./util/AutocompleteLoggingService.js";
@@ -150,6 +151,10 @@ export class CompletionProvider {
         return undefined;
       }
 
+      if (isSecurityConcern(input.filepath)) {
+        return undefined;
+      }
+
       const options = await this._getAutocompleteOptions(llm);
 
       // Debounce

core/autocomplete/CompletionProvider.ts

@@ -4,7 +4,7 @@ import { IDE } from "../..";
 import {
   getGlobalContinueIgArray,
   getWorkspaceContinueIgArray,
-} from "../../indexing/ignore";
+} from "../../indexing/continueignore";
 import { getConfigJsonPath } from "../../util/paths";
 import { findUriInDirs } from "../../util/uri";
 import { HelperVars } from "../util/HelperVars";

core/autocomplete/prefiltering/index.ts

@@ -1,11 +1,8 @@
 import ignore from "ignore";
 
 import type { FileType, IDE, SlashCommand } from "../../..";
-import {
-  DEFAULT_IGNORE,
-  getGlobalContinueIgArray,
-  gitIgArrayFromFile,
-} from "../../../indexing/ignore";
+import { getGlobalContinueIgArray } from "../../../indexing/continueignore";
+import { DEFAULT_IGNORE, gitIgArrayFromFile } from "../../../indexing/ignore";
 import { renderChatMessage } from "../../../util/messageContent";
 import {
   findUriInDirs,

core/commands/slash/built-in-legacy/onboard.ts

@@ -6,6 +6,7 @@ import {
   ContextSubmenuItem,
   LoadSubmenuItemsArgs,
 } from "../../";
+import { isSecurityConcern } from "../../indexing/ignore";
 import { walkDirs } from "../../indexing/walkDir";
 import {
   getShortestUniqueRelativeUriPaths,
@@ -36,6 +37,21 @@ class FileContextProvider extends BaseContextProvider {
       await extras.ide.getWorkspaceDirs(),
     );
 
+    if (isSecurityConcern(relativePathOrBasename)) {
+      return [
+        {
+          description: last2Parts,
+          content:
+            "Content redacted, this file cannot be viewed for security reasons",
+          name: baseName,
+          uri: {
+            type: "file",
+            value: fileUri,
+          },
+        },
+      ];
+    }
+
     return [
       {
         name: baseName,

core/context/providers/FileContextProvider.ts

@@ -3,6 +3,7 @@ import {
   ContextProviderDescription,
   ContextProviderExtras,
 } from "../../index.js";
+import { isSecurityConcern } from "../../indexing/ignore.js";
 import { getUriDescription } from "../../util/uri.js";
 import { BaseContextProvider } from "../index.js";
 
@@ -27,10 +28,23 @@ class OpenFilesContextProvider extends BaseContextProvider {
 
     return await Promise.all(
       openFiles.map(async (filepath: string) => {
-        const content = await ide.readFile(filepath);
         const { relativePathOrBasename, last2Parts, baseName } =
           getUriDescription(filepath, workspaceDirs);
 
+        if (isSecurityConcern(filepath)) {
+          return {
+            description: last2Parts,
+            content:
+              "Content redacted, this file cannot be viewed for security reasons",
+            name: baseName,
+            uri: {
+              type: "file",
+              value: filepath,
+            },
+          };
+        }
+        const content = await ide.readFile(filepath);
+
         return {
           description: last2Parts,
           content: `\`\`\`${relativePathOrBasename}\n${content}\n\`\`\``,

core/context/providers/OpenFilesContextProvider.ts

@@ -3,6 +3,7 @@ import {
   ContextProviderDescription,
   ContextProviderExtras,
 } from "../../index.js";
+import { isSecurityConcern } from "../../indexing/ignore.js";
 import { getUriDescription } from "../../util/uri.js";
 import { BaseContextProvider } from "../index.js";
 
@@ -28,6 +29,14 @@ class ProblemsContextProvider extends BaseContextProvider {
           problem.filepath,
           workspaceDirs,
         );
+        if (isSecurityConcern(relativePathOrBasename)) {
+          return {
+            description: "Problems in current file",
+            content:
+              "Content was redacted because the file is detected as a potential security concern",
+            name: `Warnings in ${baseName}`,
+          };
+        }
         const content = await ide.readFile(problem.filepath);
         const lines = content.split("\n");
         const rangeContent = lines
@@ -40,7 +49,7 @@ class ProblemsContextProvider extends BaseContextProvider {
         return {
           description: "Problems in current file",
           content: `\`\`\`${relativePathOrBasename}\n${rangeContent}\n\`\`\`\n${problem.message}\n\n`,
-          name: `Warning in ${baseName}`,
+          name: `Warnings in ${baseName}`,
         };
       }),
     );

core/context/providers/ProblemsContextProvider.ts

@@ -0,0 +1,26 @@
+import fs from "fs";
+import { IDE } from "..";
+import { getGlobalContinueIgnorePath } from "../util/paths";
+import { gitIgArrayFromFile } from "./ignore";
+
+export const getGlobalContinueIgArray = () => {
+  const contents = fs.readFileSync(getGlobalContinueIgnorePath(), "utf8");
+  return gitIgArrayFromFile(contents);
+};
+
+export const getWorkspaceContinueIgArray = async (ide: IDE) => {
+  const dirs = await ide.getWorkspaceDirs();
+  return await dirs.reduce(
+    async (accPromise, dir) => {
+      const acc = await accPromise;
+      try {
+        const contents = await ide.readFile(`${dir}/.continueignore`);
+        return [...acc, ...gitIgArrayFromFile(contents)];
+      } catch (err) {
+        console.error(err);
+        return acc;
+      }
+    },
+    Promise.resolve([] as string[]),
+  );
+};