Merge branch 'develop' of https://github.com/codegen-sh/codegen-sdk into daemon-run

Author: caroljung-cg

.codegen/.gitignore

@@ -8,12 +8,8 @@ jupyter/
 codegen-system-prompt.txt
 
 # Python cache files
-__pycache__/
+**/__pycache__/
 *.py[cod]
 *$py.class
 *.txt
 *.pyc
-
-# Keep codemods
-!codemods/
-!codemods/**

.codegen/codemods/no_link_backticks/no_link_backticks.py

@@ -2,7 +2,7 @@
 from codegen import Codebase
 
 
-@codegen.function("no-link-backticks")
+@codegen.function(name="no-link-backticks", subdirectories=["test/unit"])
 def run(codebase: Codebase):
     import re
 
@@ -12,6 +12,7 @@ def run(codebase: Codebase):
     # Iterate over all .mdx files in the codebase
     for file in codebase.files(extensions=["mdx"]):
         if file.extension == ".mdx":
+            print(f"Processing {file.path}")
             new_content = file.content
 
             # Find all markdown links with backticks in link text

.github/workflows/auto-release.yml

@@ -47,6 +47,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          token: ${{ secrets.GHA_TOKEN }}
 
       - name: Setup backend
         uses: ./.github/actions/setup-environment
@@ -66,7 +67,6 @@ jobs:
       - name: Push changes
         uses: ad-m/github-push-action@master
         with:
-          github_token: ${{ secrets.GHA_TOKEN }}
           branch: ${{ github.ref }}
 
   release:

.github/workflows/generate-docs.yml

@@ -16,22 +16,25 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          token: ${{ secrets.GHA_TOKEN }}
 
       - name: Setup backend
         uses: ./.github/actions/setup-environment
 
       - name: Generate API reference
         run: uv run python src/codegen/gscli/cli.py generate docs
 
+      - name: Generate System Prompt
+        run: uv run python src/codegen/gscli/cli.py generate system-prompt
+
       - name: Commit changes
         run: |
-          git config --local user.email "github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git add docs/
+          git config --local user.email ${{ secrets.DOCS_USER_EMAIL }}
+          git config --local user.name ${{ secrets.DOCS_USER_NAME }}
+          git add docs/ src/codegen/sdk/system-prompt.txt
           git diff --staged --quiet || git commit -m "docs: updated API reference"
 
       - name: Push changes
         uses: ad-m/github-push-action@master
         with:
-          github_token: ${{ secrets.GHA_TOKEN }}
           branch: ${{ github.ref }}

.github/workflows/test.yml

@@ -23,7 +23,6 @@ jobs:
   unit-tests:
     needs: access-check
     runs-on: ubuntu-latest-8
-    environment: testing
     steps:
       - uses: actions/checkout@v4
         with:
@@ -48,7 +47,6 @@ jobs:
     # TODO: re-enable when this check is a develop required check
     if: false
     runs-on: ubuntu-latest-32
-    environment: testing
     strategy:
       matrix:
         sync_graph: [ true, false ]
@@ -90,7 +88,6 @@ jobs:
     needs: access-check
     if: contains(github.event.pull_request.labels.*.name, 'parse-tests') || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest-32
-    environment: testing
     steps:
       - uses: actions/checkout@v4
         with:
@@ -161,7 +158,6 @@ jobs:
   integration-tests:
     needs: access-check
     runs-on: ubuntu-latest-16
-    environment: testing
     steps:
       - uses: actions/checkout@v4
         with:

codegen-examples/examples/swebench_agent_run/.env.template

@@ -2,3 +2,5 @@ OPENAI_API_KEY= # Your OpenAI API key
 ANTHROPIC_API_KEY= # Your Anthropic API key
 LANGSMITH_API_KEY= # Your Langsmith API key
 LANGCHAIN_TRACING_V2= # `true` for tracing, `false` for no tracing
+LANGCHAIN_PROJECT= # Your Langchain project
+RELACE_API= # Your Relace API key

codegen-examples/examples/swebench_agent_run/entry_point.py

@@ -1,10 +1,12 @@
 from codegen.extensions.swebench.utils import SweBenchExample
 from codegen.extensions.swebench.harness import run_agent_on_entry
 import modal
+import sys
+from codegen.sdk.core.codebase import Codebase
 
 image = (
     modal.Image.debian_slim(python_version="3.13")
-    .apt_install("git")
+    .apt_install(["git", "ripgrep"])
     .pip_install("fastapi[standard]")
     .copy_local_dir("../../../", "/root/codegen", ignore=[".venv", "**/.venv", "tests", "**/tests"])
     .run_commands("pip install -e /root/codegen")
@@ -17,3 +19,22 @@
 async def run_agent_modal(entry: SweBenchExample):
     """Modal function to process a single example from the SWE-bench dataset."""
     return run_agent_on_entry(entry)
+
+
+@app.cls(image=image, secrets=[modal.Secret.from_dotenv()], enable_memory_snapshot=True)
+class SwebenchAgentRun:
+    repo_full_name: str = modal.parameter()
+    commit: str = modal.parameter()
+    codebase: Codebase | None = None
+
+    @modal.enter(snap=True)
+    def load(self):
+        self.codebase = Codebase.from_repo(repo_full_name=self.repo_full_name, commit=self.commit, language="python")
+
+    @modal.exit()
+    def exit(self):
+        sys.exit(0)
+
+    @modal.method()
+    async def run(self, entry: SweBenchExample):
+        return run_agent_on_entry(entry, codebase=self.codebase)

codegen-examples/examples/swebench_agent_run/run_eval.py

@@ -6,16 +6,16 @@
 import modal
 import click
 from datetime import datetime
-from codegen.extensions.swebench.utils import SWEBenchDataset, get_swe_bench_example, get_swe_bench_examples
+from codegen.extensions.swebench.utils import SWEBenchDataset, SweBenchExample, get_swe_bench_examples
 from codegen.extensions.swebench.report import generate_report
 
 PREDS_DNAME = Path(__file__).parent / "predictions"
 LOG_DIR = Path(__file__).parent / "logs"
 
-run_agent_modal = modal.Function.lookup("swebench-agent-run", "run_agent_modal")
+SwebenchAgentRun = modal.Cls.from_name(app_name="swebench-agent-run", name="SwebenchAgentRun")
 
 
-async def process_batch(examples, batch_size=10):
+async def process_batch(examples: list[SweBenchExample], batch_size=10):
     """Process a batch of examples concurrently.
 
     Args:
@@ -31,7 +31,7 @@ async def process_batch(examples, batch_size=10):
         batch = examples[i : i + batch_size]
 
         # Create tasks for this batch
-        batch_tasks = [run_agent_modal.remote.aio(example) for example in batch]
+        batch_tasks = [SwebenchAgentRun(repo_full_name=example.repo, commit=example.base_commit).run.remote.aio(example) for example in batch]
 
         # Wait for all tasks in this batch to complete
         print(f"Processing batch {i // batch_size + 1}/{len(examples) // batch_size + 1} (examples {i + 1}-{min(i + batch_size, len(examples))})")
@@ -92,10 +92,7 @@ async def run_eval(use_existing_preds: str | None, dataset: str, length: int, in
     run_id = use_existing_preds or str(uuid.uuid4())
     predictions_dir = PREDS_DNAME / f"results_{run_id}"
     dataset = SWEBenchDataset(dataset)
-    if instance_id:
-        examples = [get_swe_bench_example(instance_id, dataset=dataset)]
-    else:
-        examples = get_swe_bench_examples(dataset=dataset, length=length)
+    examples = get_swe_bench_examples(dataset=dataset, length=length, instance_id=instance_id)
 
     try:
         if use_existing_preds is None:

codegen-examples/examples/swebench_agent_run/test.py

@@ -0,0 +1,14 @@
+from codegen import Codebase
+import modal
+
+image = modal.Image.debian_slim(python_version="3.13").apt_install("git").pip_install("fastapi[standard]").run_commands("pip install codegen")
+
+app = modal.App(name="codegen-examples", image=image, secrets=[modal.Secret.from_dotenv()])
+
+
+@app.function()
+def run_agent(AgentClass):
+    codebase = Codebase.from_repo(repo_full_name="pallets/flask")
+    agent = AgentClass(codebase)
+    agent.run(prompt="Tell me about the codebase and the files in it.")
+    return True

docs/api-reference/core/Class.mdx

@@ -11,7 +11,7 @@ import {HorizontalDivider} from '/snippets/HorizontalDivider.mdx';
 import {GithubLinkNote} from '/snippets/GithubLinkNote.mdx';
 import {Attribute} from '/snippets/Attribute.mdx';
 
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L53-L417" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L55-L419" />
 
 ### Inherits from
 [HasBlock](/api-reference/core/HasBlock), [Callable](/api-reference/core/Callable), [Expression](/api-reference/core/Expression), [Usable](/api-reference/core/Usable), [Symbol](/api-reference/core/Symbol), [Editable](/api-reference/core/Editable), [Importable](/api-reference/core/Importable), [HasName](/api-reference/core/HasName)
@@ -147,7 +147,7 @@ import {Attribute} from '/snippets/Attribute.mdx';
 <HorizontalDivider />
 ### <span className="text-primary">add_attribute</span>
 Adds an attribute to a class from another class.
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L358-L381" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L360-L383" />
 
 <ParameterWrapper>
 <Parameter
@@ -170,7 +170,7 @@ Adds an attribute to a class from another class.
 
 ### <span className="text-primary">add_attribute_from_source</span>
 Adds an attribute to a class from raw source code, placing it in a specific location
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L332-L355" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L334-L357" />
 
 <ParameterWrapper>
 <Parameter
@@ -244,7 +244,7 @@ Insert a keyword in the appropriate place before this symbol if it doesn't alrea
 
 ### <span className="text-primary">add_source</span>
 Add a block of source code to the bottom of a class definition.
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L318-L329" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L320-L331" />
 
 <ParameterWrapper>
 <Parameter
@@ -269,7 +269,7 @@ Find all ancestors of the node of the given type. Does not return itself
 
 ### <span className="text-primary">attributes</span>
 Retrieves all attributes from this Class including those from its superclasses up to a
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L272-L291" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L274-L293" />
 
 <ParameterWrapper>
 <Parameter
@@ -404,7 +404,7 @@ Adds a visual flag comment to the end of this Editable's source text.
 
 ### <span className="text-primary">get_attribute</span>
 Returns a specific attribute by name.
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L294-L311" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L296-L313" />
 
 <ParameterWrapper>
 <Parameter
@@ -421,7 +421,7 @@ Returns a specific attribute by name.
 
 ### <span className="text-primary">get_method</span>
 Returns a specific method by name from the class or any of its superclasses.
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L251-L268" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L253-L270" />
 
 <ParameterWrapper>
 <Parameter
@@ -446,7 +446,7 @@ Returns the name node of the object.
 
 ### <span className="text-primary">get_nested_class</span>
 Returns a nested class by name from the current class.
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L234-L248" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L236-L250" />
 
 <ParameterWrapper>
 <Parameter
@@ -514,7 +514,7 @@ Retrieves a parameter from the callable by its type.
 
 ### <span className="text-primary">get_parent_class</span>
 Returns the parent class node with the specified name.
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L111-L123" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L113-L125" />
 
 <ParameterWrapper>
 <Parameter
@@ -642,7 +642,7 @@ Inserts text before the current symbol node in the Abstract Syntax Tree.
 
 ### <span className="text-primary">is_subclass_of</span>
 Checks if the class inherits from a specified parent class.
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L138-L152" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L140-L154" />
 
 <ParameterWrapper>
 <Parameter
@@ -673,7 +673,7 @@ Check if this node is contained another node of the given class
 
 ### <span className="text-primary">methods</span>
 Retrieves all methods that exist on this Class, including methods from superclasses, with
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L203-L231" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L205-L233" />
 
 <ParameterWrapper>
 <Parameter
@@ -937,7 +937,7 @@ Sets the name of a code element.
 
 ### <span className="text-primary">subclasses</span>
 Returns all classes which subclass this class.
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L156-L167" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L158-L169" />
 
 <ParameterWrapper>
 <Parameter
@@ -954,7 +954,7 @@ Returns all classes which subclass this class.
 
 ### <span className="text-primary">superclasses</span>
 Returns a list of all classes that this class extends, up to max_depth.
-<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L77-L93" />
+<GithubLinkNote link="https://github.com/codegen-sh/codegen-sdk/blob/develop/src/codegen/sdk/core/class_definition.py#L79-L95" />
 
 <ParameterWrapper>
 <Parameter