Major Refactor to Codebase Directory (#615)

**Existing Implementation:**
- Mirrors how `codebase.files` operates
- `dir.files` returns a recursive list of all source files
- `dir.subdirectories` returns a recursive list of all directories
- A reference between the file and directory is built during parse time
(files hold a reference to dir, and dirs hold references to files)

**New Implementation:**
- Mirrors more closely to the conceptual idea of a directory
- `dir.files` returns all top level source files
- `dir.files` now has a `extensions` argument just like codebase
- `dir.subdirectories` returns all top level directories
- Both `dir.files` and `dir.subdirectories` have a `recursive=True` flag
to get everything (similar to old behaviour)
- Introduces a `dir.tree` api to recursively list everything
- `dir.items` is now a list
- Directories and files are grabbed on the fly instead of being linked
during parse time (file grabs directories using ctx.get_directory on the
fly and vice versa)

Author: EdwardJXLi

src/codegen/extensions/tools/list_directory.py

@@ -140,13 +140,13 @@ def get_directory_info(dir_obj: Directory, current_depth: int) -> DirectoryInfo:
         """Helper function to get directory info recursively."""
         # Get direct files (always include files unless at max depth)
         all_files = []
-        for file in dir_obj.files:
+        for file in dir_obj.files(recursive=True):
             if file.directory == dir_obj:
                 all_files.append(file.filepath.split("/")[-1])
 
         # Get direct subdirectories
         subdirs = []
-        for subdir in dir_obj.subdirectories:
+        for subdir in dir_obj.subdirectories(recursive=True):
             # Only include direct descendants
             if subdir.parent == dir_obj:
                 if current_depth > 1 or current_depth == -1:

src/codegen/git/repo_operator/repo_operator.py

@@ -571,7 +571,11 @@ def delete_file(self, path: str) -> None:
     def get_filepaths_for_repo(self, ignore_list):
         # Get list of files to iterate over based on gitignore setting
         if self.repo_config.respect_gitignore:
-            filepaths = self.git_cli.git.ls_files().split("\n")
+            # ls-file flags:
+            # -c: show cached files
+            # -o: show other / untracked files
+            # --exclude-standard: exclude standard gitignore rules
+            filepaths = self.git_cli.git.ls_files("-co", "--exclude-standard").split("\n")
         else:
             filepaths = glob.glob("**", root_dir=self.repo_path, recursive=True, include_hidden=True)
             # Filter filepaths by ignore list.

src/codegen/sdk/codebase/codebase_context.py

@@ -46,7 +46,7 @@
     from codegen.sdk.core.dataclasses.usage import Usage
     from codegen.sdk.core.expressions import Expression
     from codegen.sdk.core.external_module import ExternalModule
-    from codegen.sdk.core.file import SourceFile
+    from codegen.sdk.core.file import File, SourceFile
     from codegen.sdk.core.interfaces.importable import Importable
     from codegen.sdk.core.node_id_factory import NodeId
     from codegen.sdk.core.parser import Parser
@@ -345,33 +345,15 @@ def prune_graph(self) -> None:
                 self.remove_node(module.node_id)
                 self._ext_module_idx.pop(module._idx_key, None)
 
-    def build_directory_tree(self, files: list[SourceFile]) -> None:
+    def build_directory_tree(self) -> None:
         """Builds the directory tree for the codebase"""
         # Reset and rebuild the directory tree
         self.directories = dict()
-        created_dirs = set()
-        for file in files:
-            directory = self.get_directory(file.path.parent, create_on_missing=True)
-            directory.add_file(file)
-            file._set_directory(directory)
-            created_dirs.add(file.path.parent)
-
-        def _dir_has_file(filepath):
-            gen = os.scandir(filepath)
-            while entry := next(gen, None):
-                if entry.is_file():
-                    return True
-            return False
-
-        for ctx in self.projects:
-            for rel_filepath in ctx.repo_operator.get_filepaths_for_repo(GLOBAL_FILE_IGNORE_LIST):
-                abs_filepath = self.to_absolute(rel_filepath)
-                if not abs_filepath.is_dir():
-                    abs_filepath = abs_filepath.parent
-
-                if abs_filepath not in created_dirs and self.is_subdir(abs_filepath) and _dir_has_file(abs_filepath):
-                    directory = self.get_directory(abs_filepath, create_on_missing=True)
-                    created_dirs.add(abs_filepath)
+
+        for file_path, _ in self.projects[0].repo_operator.iter_files(subdirs=self.projects[0].subdirectories, ignore_list=GLOBAL_FILE_IGNORE_LIST):
+            file_path = Path(file_path)
+            directory = self.get_directory(file_path.parent, create_on_missing=True)
+            directory._add_file(file_path.name)
 
     def get_directory(self, directory_path: PathLike, create_on_missing: bool = False, ignore_case: bool = False) -> Directory | None:
         """Returns the directory object for the given path, or None if the directory does not exist.
@@ -399,16 +381,16 @@ def get_directory(self, directory_path: PathLike, create_on_missing: bool = Fals
 
             # Base Case
             if str(absolute_path) == str(self.repo_path) or str(absolute_path) == str(parent_path):
-                root_directory = Directory(path=absolute_path, dirpath="", parent=None)
+                root_directory = Directory(ctx=self, path=absolute_path, dirpath="")
                 self.directories[absolute_path] = root_directory
                 return root_directory
 
             # Recursively create the parent directory
             parent = self.get_directory(parent_path, create_on_missing=True)
             # Create the directory
-            directory = Directory(path=absolute_path, dirpath=str(self.to_relative(absolute_path)), parent=parent)
+            directory = Directory(ctx=self, path=absolute_path, dirpath=str(self.to_relative(absolute_path)))
             # Add the directory to the parent
-            parent.add_subdirectory(directory)
+            parent._add_subdirectory(directory.name)
             # Add the directory to the tree
             self.directories[absolute_path] = directory
             return directory
@@ -514,7 +496,7 @@ def _process_diff_files(self, files_to_sync: Mapping[SyncType, list[Path]], incr
         # Step 6: Build directory tree
         logger.info("> Building directory tree")
         files = [f for f in sort_editables(self.get_nodes(NodeType.FILE), alphabetical=True, dedupe=False)]
-        self.build_directory_tree(files)
+        self.build_directory_tree()
 
         # Step 7: Build configs
         if self.config_parser is not None:
@@ -613,13 +595,20 @@ def get_file(self, file_path: os.PathLike, ignore_case: bool = False) -> SourceF
         if node_id is not None:
             return self.get_node(node_id)
         if ignore_case:
-            parent = self.to_absolute(file_path).parent
-            if parent == Path(self.repo_path):
-                for file in self.to_absolute(self.repo_path).iterdir():
-                    if str(file_path).lower() == str(self.to_absolute(file)).lower():
-                        return self.get_file(file, ignore_case=False)
-            if directory := self.get_directory(parent, ignore_case=ignore_case):
-                return directory.get_file(os.path.basename(file_path), ignore_case=ignore_case)
+            # Using `get_directory` so that the case insensitive lookup works
+            parent = self.get_directory(self.to_absolute(file_path).parent, ignore_case=ignore_case).path
+            for file in parent.iterdir():
+                if str(file_path).lower() == str(self.to_relative(file)).lower():
+                    return self.get_file(file, ignore_case=False)
+
+    def _get_raw_file_from_path(self, path: Path) -> File | None:
+        from codegen.sdk.core.file import File
+
+        try:
+            return File.from_content(path, self.io.read_text(path), self, sync=False)
+        except UnicodeDecodeError:
+            # Handle when file is a binary file
+            return File.from_content(path, self.io.read_bytes(path), self, sync=False, binary=True)
 
     def get_external_module(self, module: str, import_name: str) -> ExternalModule | None:
         node_id = self._ext_module_idx.get(module + "::" + import_name, None)

src/codegen/sdk/core/codebase.py

@@ -522,37 +522,24 @@ def get_file(self, filepath: str, *, optional: bool = False, ignore_case: bool =
         Raises:
             ValueError: If file not found and optional=False.
         """
-
-        def get_file_from_path(path: Path) -> File | None:
-            try:
-                return File.from_content(path, self.ctx.io.read_text(path), self.ctx, sync=False)
-            except UnicodeDecodeError:
-                # Handle when file is a binary file
-                return File.from_content(path, self.ctx.io.read_bytes(path), self.ctx, sync=False, binary=True)
-
         # Try to get the file from the graph first
         file = self.ctx.get_file(filepath, ignore_case=ignore_case)
         if file is not None:
             return file
+        # If the file is not in the graph, check the filesystem
         absolute_path = self.ctx.to_absolute(filepath)
-        if absolute_path.suffix in self.ctx.extensions and not self.ctx.io.file_exists(absolute_path):
-            return None
         if self.ctx.io.file_exists(absolute_path):
-            return get_file_from_path(absolute_path)
-        elif ignore_case:
-            parent = absolute_path.parent
-            if parent == Path(self.ctx.repo_path):
-                for file in self.ctx.to_absolute(self.ctx.repo_path).iterdir():
-                    if str(absolute_path).lower() == str(file).lower():
-                        return get_file_from_path(file)
-            else:
-                dir = self.ctx.get_directory(parent, ignore_case=ignore_case)
-                if dir is None:
-                    return None
-                for file in dir.path.iterdir():
-                    if str(absolute_path).lower() == str(file).lower():
-                        return get_file_from_path(file)
-        elif not optional:
+            return self.ctx._get_raw_file_from_path(absolute_path)
+        # If the file is not in the graph, check the filesystem
+        if absolute_path.parent.exists():
+            for file in absolute_path.parent.iterdir():
+                if ignore_case and str(absolute_path).lower() == str(file).lower():
+                    return self.ctx._get_raw_file_from_path(file)
+                elif not ignore_case and str(absolute_path) == str(file):
+                    return self.ctx._get_raw_file_from_path(file)
+
+        # If we get here, the file is not found
+        if not optional:
             msg = f"File {filepath} not found in codebase. Use optional=True to return None instead."
             raise ValueError(msg)
         return None