Merge branch 'develop' into api-reference-v0

Author: jemeza-codegen

.pre-commit-config.yaml

@@ -82,7 +82,7 @@ repos:
     hooks:
       - id: renovate-config-validator
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: "0.5.23"
+    rev: "0.5.24"
     hooks:
       - id: uv-lock
         entry: bash -c "uv lock --frozen"

README.md

@@ -1,7 +1,6 @@
 # Graph sitter
 
 [![Documentation](https://img.shields.io/badge/docs-docs.codegen.com-blue)](https://docs.codegen.com)
-[![Unit Tests](https://github.com/codegen-sh/codegen-sdk/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/codegen-sh/codegen-sdk/actions/workflows/unit-tests.yml)
 
 [Codegen](https://docs.codegen.com) is a python library for manipulating codebases.
 
@@ -42,7 +41,7 @@ We built Codegen backwards from real-world refactors performed on enterprise cod
 
 - **Natural mental model**: Write transforms that read like your thought process - "move this function", "rename this variable", "add this parameter". No more wrestling with ASTs or manual import management.
 
-- **Battle-tested on complex codebases**: Handle Python, TypeScript, and React codebases with millions of lines of code. Built and validated on refactors at companies like [Ramp](https://ramp.com).
+- **Battle-tested on complex codebases**: Handle Python, TypeScript, and React codebases with millions of lines of code.
 
 - **Built for advanced intelligences**: As AI developers become more sophisticated, they need expressive yet precise tools to manipulate code. Codegen provides a programmatic interface that both humans and AI can use to express complex transformations through code itself.
 

docs/building-with-codegen/files-and-directories.mdx

@@ -50,6 +50,32 @@ dir = file.directory
 exists = codebase.has_directory("path/to/dir")
 ```
 
+## Working with Non-Code Files (README, JSON, etc.)
+
+By default, Codegen focuses on source code files (Python, TypeScript, etc). However, you can access all files in your codebase, including documentation, configuration, and other non-code files like README.md, package.json, or .env:
+
+```python
+# Get all files in the codebase (including README, docs, config files)
+files = codebase.files(extensions="*")
+
+# Print files that are not source code (documentation, config, etc)
+for file in files:
+    if not file.filepath.endswith(('.py', '.ts', '.js')):
+        print(f"📄 Non-code file: {file.filepath}")
+```
+
+You can also filter for specific file types:
+
+```python
+# Get only markdown documentation files
+docs = codebase.files(extensions=[".md", ".mdx"])
+
+# Get configuration files
+config_files = codebase.files(extensions=[".json", ".yaml", ".toml"])
+```
+
+These APIs are similar for [`Directory`](../api-reference/core/Directory), which provides similar methods for accessing files and subdirectories.
+
 ## Raw Content and Metadata
 
 ```python

docs/prism.css

@@ -1,5 +1,24 @@
+/* Override font variables */
+:root {
+	--font-jetbrains-mono: Menlo, Monaco, "Courier New", monospace !important;
+	/* --font-jetbrains-mono: Consolas, Monaco, "Andale Mono", "Ubuntu Mono", monospace !important; */
+}
+
+pre[class*="language-"],
+code[class*="language-"],
+pre,
+code,
+kbd,
+samp {
+	font-family: Menlo, Monaco, "Andale Mono", "Ubuntu Mono", monospace !important;
+	--font-jetbrains-mono: Consolas, Monaco, "Andale Mono", "Ubuntu Mono",
+		monospace !important;
+	font-weight: 500 !important;
+}
+
 pre[class*="language-"] {
-	font-family: Consolas, Monaco, "Andale Mono", "Ubuntu Mono", monospace;
+	font-family: Menlo, Monaco, "Andale Mono", "Ubuntu Mono", monospace !important;
+	font-weight: 500 !important;
 	text-align: left;
 	white-space: pre;
 	word-spacing: normal;
@@ -52,7 +71,7 @@ pre[class*="language-"] {
 
 .language-python .language-bash {
 	background-color: #15141b !important;
-	font-family: Menlo, Monaco, Consolas, "Courier New", monospace !important;
+	font-family: Consolas, Monaco, "Andale Mono", "Ubuntu Mono", monospace !important;
 	font-weight: 600 !important;
 	font-size: 16px !important;
 }

src/codegen/__init__.py

@@ -0,0 +1,5 @@
+from codegen.cli.sdk.decorator import function
+from codegen.cli.sdk.functions import Function
+from codegen.sdk.core.codebase import Codebase
+
+__all__ = ["Codebase", "Function", "function"]

src/codegen/git/repo_operator/remote_repo_operator.py

@@ -24,12 +24,15 @@
 class RemoteRepoOperator(RepoOperator):
     """A wrapper around GitPython to make it easier to interact with a cloned lowside repo."""
 
+    # __init__ attributes
     repo_config: RepoConfig
     base_dir: str
+    github_type: GithubType
+
+    # lazy attributes
     _remote_git_repo: GitRepoClient | None = None
     _codeowners_parser: CodeOwnersParser | None = None
     _default_branch: str | None = None
-    bot_commit: bool = True
 
     # TODO: allow setting the access scope level of the lowside repo (currently it's always WRITE)
     def __init__(
@@ -38,9 +41,10 @@ def __init__(
         base_dir: str = "/tmp",
         setup_option: SetupOption = SetupOption.PULL_OR_CLONE,
         shallow: bool = True,
-        bot_commit: bool = True,
+        github_type: GithubType = GithubType.GithubEnterprise,
     ) -> None:
-        super().__init__(repo_config=repo_config, base_dir=base_dir, bot_commit=bot_commit)
+        super().__init__(repo_config=repo_config, base_dir=base_dir)
+        self.github_type = github_type
         self.setup_repo_dir(setup_option=setup_option, shallow=shallow)
 
     ####################################################################################################################
@@ -50,8 +54,7 @@ def __init__(
     @property
     def remote_git_repo(self) -> GitRepoClient:
         if not self._remote_git_repo:
-            # NOTE: local repo operator by default points at lowside (i.e. origin remote is lowside remote)
-            self._remote_git_repo = GitRepoClient(self.repo_config, github_type=GithubType.GithubEnterprise, access_scope=GithubScope.WRITE)
+            self._remote_git_repo = GitRepoClient(self.repo_config, github_type=self.github_type, access_scope=GithubScope.WRITE)
         return self._remote_git_repo
 
     @property
@@ -69,27 +72,31 @@ def codeowners_parser(self) -> CodeOwnersParser | None:
     ####################################################################################################################
     # SET UP
     ####################################################################################################################
+
     @override
     def pull_repo(self) -> None:
         """Pull the latest commit down to an existing local repo"""
-        pull_repo(repo=self.repo_config, path=self.base_dir)
+        pull_repo(repo=self.repo_config, path=self.base_dir, github_type=self.github_type)
+
+    def clone_repo(self, shallow: bool = True) -> None:
+        clone_repo(repo=self.repo_config, path=self.base_dir, shallow=shallow, github_type=self.github_type)
 
-    def clone_or_pull_repo(self) -> None:
+    def clone_or_pull_repo(self, shallow: bool = True) -> None:
         """If repo exists, pulls changes. otherwise, clones the repo."""
         # TODO(CG-7804): if repo is not valid we should delete it and re-clone. maybe we can create a pull_repo util + use the existing clone_repo util
         if self.repo_exists():
             self.clean_repo()
-        clone_or_pull_repo(self.repo_config, path=self.base_dir, shallow=True)
+        clone_or_pull_repo(self.repo_config, path=self.base_dir, shallow=shallow, github_type=self.github_type)
 
     def setup_repo_dir(self, setup_option: SetupOption = SetupOption.PULL_OR_CLONE, shallow: bool = True) -> None:
         os.makedirs(self.base_dir, exist_ok=True)
         os.chdir(self.base_dir)
         if setup_option is SetupOption.CLONE:
             # if repo exists delete, then clone, else clone
-            clone_repo(repo=self.repo_config, path=self.base_dir, shallow=shallow)
+            clone_repo(shallow=shallow)
         elif setup_option is SetupOption.PULL_OR_CLONE:
             # if repo exists, pull changes, else clone
-            self.clone_or_pull_repo()
+            self.clone_or_pull_repo(shallow=shallow)
         elif setup_option is SetupOption.SKIP:
             if not self.repo_exists():
                 logger.warning(f"Valid git repo does not exist at {self.repo_path}. Cannot skip setup with SetupOption.SKIP.")

src/codegen/sdk/core/autocommit/decorators.py

@@ -3,6 +3,7 @@
 from typing import TYPE_CHECKING, ParamSpec, TypeVar, Union, overload
 
 import wrapt
+
 from codegen.sdk.core.autocommit.constants import AutoCommitState, enabled
 from codegen.sdk.core.node_id_factory import NodeId
 

src/codegen/sdk/core/codebase.py

@@ -224,6 +224,8 @@ def files(self, *, extensions: list[str] | Literal["*"] | None = None) -> list[T
         if extensions is None:
             # Return all source files
             files = self.G.get_nodes(NodeType.FILE)
+        elif isinstance(extensions, str) and extensions != "*":
+            raise ValueError("extensions must be a list of extensions or '*'")
         else:
             files = []
             # Get all files with the specified extensions

tests/unit/codebase/file/test_file.py

@@ -59,6 +59,28 @@ def test_codebase_files(tmpdir) -> None:
         assert {f for f in codebase.files(extensions=[".bin"])} == {file3}
 
 
+@pytest.mark.skip("MDX editing is broken")
+def test_codebase_edit_mdx(tmpdir) -> None:
+    """Editing MDx seems broken currently - it will just prepend to the file"""
+    with get_codebase_session(tmpdir=tmpdir, files={"file1.mdx": "# Header", "file2.tsx": "console.log('hello, world!')"}) as codebase:
+        file = codebase.get_file("file1.mdx")
+        file.edit("NEW TEXT")
+        codebase.commit()
+        file = codebase.get_file("file1.mdx")
+        assert file.content == "NEW TEXT"
+
+
+@pytest.mark.skip("MDX replacing is broken")
+def test_codebase_replace_mdx(tmpdir) -> None:
+    """Editing MDx seems broken currently - it will just prepend to the file"""
+    with get_codebase_session(tmpdir=tmpdir, files={"file1.mdx": "# Header"}) as codebase:
+        file = codebase.get_file("file1.mdx")
+        file.replace("# Header", "NEW TEXT")
+        codebase.commit()
+        file = codebase.get_file("file1.mdx")
+        assert file.content == "NEW TEXT"
+
+
 @pytest.mark.skipif(sys.platform == "darwin", reason="macOS is case-insensitive")
 def test_file_extensions_ignore_case(tmpdir) -> None:
     with get_codebase_session(tmpdir=tmpdir, files={"file1.py": "print(123)", "file2.py": "print(456)", "file3.bin": b"\x89PNG", "file4": "Hello world!"}) as codebase:

tests/unit/test_imports.py

@@ -0,0 +1,16 @@
+import codegen
+from codegen import Codebase
+
+
+def test_codegen_imports():
+    # Test decorated function
+    @codegen.function(name="sample_codemod")
+    def run(codebase):
+        pass
+
+    # Test class
+    cls = codegen.Function
+    assert cls is not None
+
+    codebase = Codebase("./")
+    assert codebase is not None