feat(cli, nvim): Add files subcommand for file manipulation (#230)

* feat(cli): add `files` subcommand for file manipulation

* docs(cli): mark `files` subcommand feature as completed

* tests(cli): fix test coverage for existing codebase

* test(cli): test for `files ls`

* tests(cli): add tests for `files rm`

* tests(cli): add tests for `files` subcommand

* feat(cli): make sure `files` commands honor `--project_root`

* docs(cli): document `files` subcommand in cli.md

* Auto generate docs

* feat(cli): remove empty collection after removing files

* tests(cli): fix broken tests due to wrong mocking

* refactor(cli): refactor `files ls` to reuse file listing logic

* feat(cli): add `files` subcommand to the LSP server

* feat(nvim): add `files_ls` tool to CodeCompanion

* fix(cli): improve `files rm` feedback and fix progress handling

* feat(nvim): add `files_rm` tool to CodeCompanion

* docs(cli): document pipe mode of `files ls` subcommand in cli.md

* Auto generate docs

* docs(cli): formatting

* Auto generate docs

---------

Co-authored-by: Davidyz <[email protected]>

Author: Davidyz

README.md

@@ -94,8 +94,7 @@ This project follows an adapted semantic versioning:
 - [x] implement some sort of project-root anchors (such as `.git` or a custom
   `.vectorcode.json`) that enhances automatic project-root detection.
   **Implemented project-level `.vectorcode/` and `.git` as root anchor**
-- [ ] ability to view and delete files in a collection (atm you can only `drop`
-  and `vectorise` again);
+- [x] ability to view and delete files in a collection;
 - [x] joint search (kinda, using codecompanion.nvim/MCP);
 - [x] Nix support (unofficial packages [here](https://search.nixos.org/packages?channel=unstable&from=0&size=50&sort=relevance&type=packages&query=vectorcode));
 - [ ] Query rewriting (#124).

doc/VectorCode-cli.txt

@@ -35,6 +35,7 @@ Table of Contents                           *VectorCode-cli-table-of-contents*
     - |VectorCode-cli-removing-a-collection|
     - |VectorCode-cli-checking-project-setup|
     - |VectorCode-cli-cleaning-up|
+    - |VectorCode-cli-inspecting-and-manupulating-files-in-an-indexed-project|
     - |VectorCode-cli-debugging-and-diagnosing|
 - |VectorCode-cli-shell-completion|
 - |VectorCode-cli-hardware-acceleration|
@@ -43,6 +44,7 @@ Table of Contents                           *VectorCode-cli-table-of-contents*
         - |VectorCode-cli-`vectorcode-query`|
         - |VectorCode-cli-`vectorcode-vectorise`|
         - |VectorCode-cli-`vectorcode-ls`|
+        - |VectorCode-cli-`vectorcode-files-ls`|
     - |VectorCode-cli-lsp-mode|
     - |VectorCode-cli-mcp-server|
     - |VectorCode-cli-writing-prompts|
@@ -556,6 +558,15 @@ For empty collections and collections for removed projects, you can use the
 `vectorcode clean` command to remove them at once.
 
 
+INSPECTING AND MANUPULATING FILES IN AN INDEXED PROJECT ~
+
+- `vectorcode files ls` prints a list of files that are indexed in the project.
+- `vectorcode files rm file1 file2` removes the embeddings that belong to the
+    specified files from the project.
+
+Both commands will honor the `--project_root` flag.
+
+
 DEBUGGING AND DIAGNOSING ~
 
 When something doesn’t work as expected, you can enable logging by setting
@@ -718,6 +729,11 @@ A JSON array of collection information of the following format will be printed:
 - `"num_files"`number of files that have been vectorised in the project.
 
 
+VECTORCODE FILES LS
+
+A JSON array of strings (the absolute paths to the files in the collection).
+
+
 LSP MODE ~
 
 There’s an experimental implementation of VectorCode CLI, which accepts

docs/cli.md

@@ -24,6 +24,7 @@
   * [Removing a Collection](#removing-a-collection)
   * [Checking Project Setup](#checking-project-setup)
   * [Cleaning up](#cleaning-up)
+  * [Inspecting and Manupulating Files in an Indexed Project](#inspecting-and-manupulating-files-in-an-indexed-project)
   * [Debugging and Diagnosing](#debugging-and-diagnosing)
 * [Shell Completion](#shell-completion)
 * [Hardware Acceleration](#hardware-acceleration)
@@ -32,6 +33,7 @@
     * [`vectorcode query`](#vectorcode-query)
     * [`vectorcode vectorise`](#vectorcode-vectorise)
     * [`vectorcode ls`](#vectorcode-ls)
+    * [`vectorcode files ls`](#vectorcode-files-ls)
   * [LSP Mode](#lsp-mode)
   * [MCP Server](#mcp-server)
   * [Writing Prompts](#writing-prompts)
@@ -507,6 +509,14 @@ some_message` and then getting an empty results.
 For empty collections and collections for removed projects, you can use the
 `vectorcode clean` command to remove them at once.
 
+### Inspecting and Manupulating Files in an Indexed Project
+
+- `vectorcode files ls` prints a list of files that are indexed in the project.
+- `vectorcode files rm file1 file2` removes the embeddings that belong to the 
+specified files from the project.
+
+Both commands will honor the `--project_root` flag.
+
 ### Debugging and Diagnosing
 
 When something doesn't work as expected, you can enable logging by setting the
@@ -647,6 +657,10 @@ A JSON array of collection information of the following format will be printed:
 - `"size"`: number of chunks stored in the database;
 - `"num_files"`: number of files that have been vectorised in the project.
 
+#### `vectorcode files ls`
+
+A JSON array of strings (the absolute paths to the files in the collection).
+
 ### LSP Mode
 
 There's an experimental implementation of VectorCode CLI, which accepts requests

lua/codecompanion/_extensions/vectorcode/init.lua

@@ -1,6 +1,6 @@
 ---@module "codecompanion"
 
----@alias sub_cmd "ls"|"query"|"vectorise"
+---@alias sub_cmd "ls"|"query"|"vectorise"|"files_ls"|"files_rm"
 
 ---@class VectorCode.CodeCompanion.ExtensionOpts
 --- A table where the keys are the subcommand name (`ls`, `query`, `vectorise`)
@@ -17,15 +17,29 @@ local use_lsp = vc_config.get_user_config().async_backend == "lsp"
 ---@type VectorCode.CodeCompanion.ExtensionOpts|{}
 local default_extension_opts = {
   tool_opts = {
-    ls = { use_lsp = use_lsp, requires_approval = false },
-    query = { use_lsp = use_lsp, requires_approval = false },
-    vectorise = { use_lsp = use_lsp, requires_approval = true },
+    ls = { use_lsp = use_lsp, requires_approval = false, include_in_toolbox = true },
+    query = { use_lsp = use_lsp, requires_approval = false, include_in_toolbox = true },
+    vectorise = {
+      use_lsp = use_lsp,
+      requires_approval = true,
+      include_in_toolbox = true,
+    },
+    files_ls = {
+      use_lsp = use_lsp,
+      requires_approval = false,
+      include_in_toolbox = false,
+    },
+    files_rm = {
+      use_lsp = use_lsp,
+      requires_approval = true,
+      include_in_toolbox = false,
+    },
   },
   tool_group = { enabled = true, collapse = true, extras = {} },
 }
 
 ---@type sub_cmd[]
-local valid_tools = { "ls", "query", "vectorise" }
+local valid_tools = { "ls", "query", "vectorise", "files_ls", "files_rm" }
 
 ---@type CodeCompanion.Extension
 local M = {
@@ -65,6 +79,9 @@ local M = {
     if opts.tool_group.enabled then
       local included_tools = vim
         .iter(valid_tools)
+        :filter(function(cmd_name)
+          return opts.tool_opts[cmd_name].include_in_toolbox
+        end)
         :map(function(s)
           return "vectorcode_" .. s
         end)

lua/vectorcode/integrations/codecompanion/files_ls_tool.lua

@@ -0,0 +1,86 @@
+---@module "codecompanion"
+
+local cc_common = require("vectorcode.integrations.codecompanion.common")
+
+---@param opts VectorCode.CodeCompanion.FilesLsToolOpts
+---@return CodeCompanion.Agent.Tool
+return function(opts)
+  local job_runner =
+    require("vectorcode.integrations.codecompanion.common").initialise_runner(
+      opts.use_lsp
+    )
+  local tool_name = "vectorcode_files_ls"
+  ---@type CodeCompanion.Agent.Tool|{}
+  return {
+    name = tool_name,
+    cmds = {
+      ---@param agent CodeCompanion.Agent
+      ---@param action {project_root: string}
+      ---@return nil|{ status: string, data: string }
+      function(agent, action, _, cb)
+        local args = { "files", "ls", "--pipe" }
+        if action ~= nil then
+          action.project_root = action.project_root
+            or vim.fs.root(0, { ".vectorcode", ".git" })
+          if action.project_root ~= nil then
+            action.project_root = vim.fs.normalize(action.project_root)
+            local stat = vim.uv.fs_stat(action.project_root)
+            if stat and stat.type == "directory" then
+              vim.list_extend(args, { "--project_root", action.project_root })
+            end
+          end
+        end
+        job_runner.run_async(args, function(result, error)
+          if vim.islist(result) and #result > 0 then
+            cb({ status = "success", data = result })
+          else
+            if type(error) == "table" then
+              error = cc_common.flatten_table_to_string(error)
+            end
+            cb({
+              status = "error",
+              data = error,
+            })
+          end
+        end, agent.chat.bufnr)
+      end,
+    },
+    schema = {
+      type = "function",
+      ["function"] = {
+        name = tool_name,
+        description = "Retrieve a list of files that have been added to the database for a given project.",
+        parameters = {
+          type = "object",
+          properties = {
+            project_root = {
+              type = "string",
+              description = "The project for which the indexed files will be listed. Leave this empty for the current project.",
+            },
+          },
+        },
+      },
+    },
+    output = {
+      ---@param agent CodeCompanion.Agent
+      ---@param stdout string[][]
+      success = function(_, agent, _, stdout)
+        stdout = stdout[1]
+        local user_message
+        for i, col in ipairs(stdout) do
+          if i == 1 then
+            user_message =
+              string.format("**VectorCode `files_ls` Tool**: Found %d files.", #stdout)
+          else
+            user_message = ""
+          end
+          agent.chat:add_tool_output(
+            agent.tool,
+            string.format("<path>%s</path>", col),
+            user_message
+          )
+        end
+      end,
+    },
+  }
+end

lua/vectorcode/integrations/codecompanion/files_rm_tool.lua

@@ -0,0 +1,112 @@
+---@module "codecompanion"
+
+local cc_common = require("vectorcode.integrations.codecompanion.common")
+
+---@alias FilesRmArgs { paths: string[], project_root: string }
+
+---@param opts VectorCode.CodeCompanion.FilesRmToolOpts
+---@return CodeCompanion.Agent.Tool
+return function(opts)
+  local tool_name = "vectorcode_files_rm"
+  local job_runner = cc_common.initialise_runner(opts.use_lsp)
+
+  ---@type CodeCompanion.Agent.Tool|{}
+  return {
+    name = tool_name,
+    schema = {
+      type = "function",
+      ["function"] = {
+        name = tool_name,
+        description = "Remove files from the VectorCode database. The files will remain in the file system.",
+        parameters = {
+          type = "object",
+          properties = {
+            paths = {
+              type = "array",
+              items = { type = "string" },
+              description = "Paths to the files to be removed from the database.",
+            },
+            project_root = {
+              type = "string",
+              description = "The project that the files belong to. Either use a path from the `vectorcode_ls` tool, or leave empty to use the current git project. If the user did not specify a path, use empty string for this parameter.",
+            },
+          },
+          required = { "paths", "project_root" },
+          additionalProperties = false,
+        },
+        strict = true,
+      },
+    },
+    cmds = {
+      ---@param agent CodeCompanion.Agent
+      ---@param action VectoriseToolArgs
+      ---@return nil|{ status: string, data: string }
+      function(agent, action, _, cb)
+        local args = { "files", "rm", "--pipe" }
+        local project_root = vim.fs.abspath(vim.fs.normalize(action.project_root or ""))
+        if project_root ~= "" then
+          local stat = vim.uv.fs_stat(project_root)
+          if stat and stat.type == "directory" then
+            vim.list_extend(args, { "--project_root", project_root })
+          else
+            return { status = "error", data = "Invalid path " .. project_root }
+          end
+        else
+          project_root = vim.fs.root(".", { ".vectorcode", ".git" }) or ""
+          if project_root == "" then
+            return {
+              status = "error",
+              data = "Please specify a project root. You may use the `vectorcode_ls` tool to find a list of existing projects.",
+            }
+          end
+        end
+        if project_root ~= "" then
+          action.project_root = project_root
+        end
+        vim.list_extend(
+          args,
+          vim
+            .iter(action.paths)
+            :filter(
+              ---@param item string
+              function(item)
+                local stat = vim.uv.fs_stat(item)
+                if stat and stat.type == "file" then
+                  return true
+                else
+                  return false
+                end
+              end
+            )
+            :totable()
+        )
+        job_runner.run_async(
+          args,
+          ---@param result VectoriseResult
+          function(result, error, code, _)
+            if result then
+              cb({ status = "success", data = result })
+            else
+              cb({ status = "error", data = { error = error, code = code } })
+            end
+          end,
+          agent.chat.bufnr
+        )
+      end,
+    },
+    output = {
+      ---@param self CodeCompanion.Agent.Tool
+      prompt = function(self, _)
+        return string.format(
+          "Remove %d files from VectorCode database?",
+          #self.args.paths
+        )
+      end,
+      ---@param self CodeCompanion.Agent.Tool
+      ---@param agent CodeCompanion.Agent
+      success = function(self, agent, _, _)
+        agent.chat:add_tool_output(self, "**VectorCode `files_rm` tool**: successful.")
+      end,
+    },
+  }
+end

lua/vectorcode/integrations/codecompanion/init.lua

@@ -35,7 +35,7 @@ return {
       }
     end),
 
-    ---@param subcommand "ls"|"query"|"vectorise"
+    ---@param subcommand sub_cmd
     ---@param opts VectorCode.CodeCompanion.ToolOpts
     ---@return CodeCompanion.Agent.Tool
     make_tool = function(subcommand, opts)

lua/vectorcode/types.lua

@@ -76,9 +76,15 @@
 --- Whether to use the LSP backend. Default: `false`
 ---@field use_lsp boolean?
 ---@field requires_approval boolean?
+--- Whether this tool should be included in `vectorcode_toolbox`
+---@field include_in_toolbox boolean?
 
 ---@class VectorCode.CodeCompanion.LsToolOpts: VectorCode.CodeCompanion.ToolOpts
 
+---@class VectorCode.CodeCompanion.FilesLsToolOpts: VectorCode.CodeCompanion.ToolOpts
+
+---@class VectorCode.CodeCompanion.FilesRmToolOpts: VectorCode.CodeCompanion.ToolOpts
+
 ---@class VectorCode.CodeCompanion.QueryToolOpts: VectorCode.CodeCompanion.ToolOpts
 --- Maximum number of results provided to the LLM.
 --- You may set this to a table to configure different values for document/chunk mode.