Merge remote-tracking branch 'origin/develop' into oss-parsing

Author: clee-codegen

.codegen/.gitignore

@@ -4,16 +4,16 @@ examples/
 prompts/
 jupyter/
 .venv/
+.env
 codegen-system-prompt.txt
-*.txt
-*.pyc
 
 # Python cache files
 __pycache__/
 *.py[cod]
 *$py.class
+*.txt
+*.pyc
 
-# Keep config.toml and codemods
-!config.toml
+# Keep codemods
 !codemods/
 !codemods/**

.codegen/config.toml

@@ -9,7 +9,7 @@ runs:
   using: "composite"
   steps:
     - name: Install UV
-      uses: astral-sh/setup-uv@v5.2
+      uses: astral-sh/setup-uv@v5.3
       id: setup-uv
       with:
         enable-cache: true
@@ -26,5 +26,5 @@ runs:
     - name: Install codecov
       shell: bash
       run: |
-        uv tool install codecov-cli --python 3.10
+        uv tool install codecov-cli@10.0.1 --python 3.10
         uv tool update-shell

.github/actions/setup-environment/action.yml

@@ -54,7 +54,7 @@ jobs:
           ref: ${{ inputs.release-tag || github.event.pull_request.head.ref || github.ref }}
 
       - name: Install UV
-        uses: astral-sh/setup-uv@v5.2
+        uses: astral-sh/setup-uv@v5.3
         id: setup-uv
         with:
           enable-cache: false

.github/workflows/release.yml

@@ -141,7 +141,7 @@ jobs:
         timeout-minutes: 5
         env:
           GITHUB_WORKSPACE: $GITHUB_WORKSPACE
-          CODEGEN_SECRETS__GITHUB_TOKEN: ${{ secrets.GHA_PAT }}
+          GITHUB_TOKEN: ${{ secrets.GHA_PAT }}
         run: |
           uv run pytest \
             -n auto \

.github/workflows/unit-tests.yml

@@ -65,3 +65,9 @@ graph-sitter-types/typings/**
 coverage.json
 tests/integration/verified_codemods/codemod_data/repo_commits.json
 .benchmarks/*
+
+# SWE Bench results
+results.*.json
+codegen-examples/examples/swebench_agent_run/results/*
+codegen-examples/examples/swebench_agent_run/predictions/*
+codegen-examples/examples/swebench_agent_run/logs/*

.gitignore

@@ -0,0 +1,17 @@
+Please read two things before trying to install this codegen MCP server:
+
+1. [Readme](https://raw.githubusercontent.com/codegen-sh/codegen-sdk/refs/heads/develop/codegen-examples/examples/codegen-mcp-server/README.md)
+
+- This contains the instructions for setting up the mcp server
+
+2. [Documentation Index for Agents](https://docs.codegen.com/llms.txt)
+
+- This contains a list of urls to the documentation for topics related to creating codegen codemods
+
+Please use a CURL command to retrieve the contents of the readme and documentation and read both before starting to install the server.
+
+1. curl -sL "https://raw.githubusercontent.com/codegen-sh/codegen-sdk/refs/heads/develop/README.md" | cat
+
+1. curl -sL "https://docs.codegen.com/llms.txt" | cat
+
+This should give you all the context you need to successfully install and utilize the server.

codegen-examples/examples/codegen-mcp-server/llms-install.md

@@ -1,8 +1,9 @@
 import asyncio
 from dataclasses import dataclass, field
-from typing import Annotated, Optional, Dict, Any, List
-from mcp.server.fastmcp import FastMCP
+from typing import Annotated, Any, Dict, List, Optional
+
 from codegen import Codebase
+from mcp.server.fastmcp import FastMCP
 
 
 @dataclass
@@ -11,20 +12,16 @@ class CodebaseState:
 
     parse_task: Optional[asyncio.Future] = None
     parsed_codebase: Optional[Codebase] = None
-    codebase_path: Optional[str] = None
     log_buffer: List[str] = field(default_factory=list)
 
-    async def parse(self, path: str) -> Codebase:
+    def parse(self, path: str) -> Codebase:
         """Parse the codebase at the given path."""
         codebase = Codebase(path)
         self.parsed_codebase = codebase
-        self.codebase_path = path
         return codebase
 
     def reset(self) -> None:
         """Reset the state."""
-        if self.parsed_codebase:
-            self.parsed_codebase.reset()
         self.log_buffer.clear()
 
 
@@ -45,12 +42,24 @@ def capture_output(*args, **kwargs) -> None:
         state.log_buffer.append(str(arg))
 
 
+def update_codebase(future: asyncio.Future):
+    try:
+        result = future.result()
+        if result is not None:
+            state.parsed_codebase = result
+        else:
+            state.parsed_codebase = None
+    except Exception:
+        pass
+
+
 @mcp.tool(name="parse_codebase", description="Initiate codebase parsing")
 async def parse_codebase(codebase_path: Annotated[str, "path to the codebase to be parsed"]) -> Dict[str, str]:
     if not state.parse_task or state.parse_task.done():
         state.parse_task = asyncio.get_event_loop().run_in_executor(None, lambda: state.parse(codebase_path))
+        state.parse_task.add_done_callback(update_codebase)
         return {"message": "Codebase parsing initiated, this may take some time depending on the size of the codebase. Use the `check_parsing_status` tool to check if the parse has completed."}
-    return {"message": "Codebase is already being parsed."}
+    return {"message": "Codebase is already being parsed.", "status": "error"}
 
 
 @mcp.tool(name="check_parse_status", description="Check if codebase parsing has completed")
@@ -69,17 +78,19 @@ async def execute_codemod(codemod: Annotated[str, "The python codemod code to ex
 
     try:
         await state.parse_task
-        # TODO: Implement proper sandboxing for code execution
-        context = {
-            "codebase": state.parsed_codebase,
-            "print": capture_output,
-        }
-        exec(codemod, context)
+        if state.parsed_codebase is None:
+            return {"error": "Codebase path is not set."}
+        else:
+            # TODO: Implement proper sandboxing for code execution
+            context = {
+                "codebase": state.parsed_codebase,
+                "print": capture_output,
+            }
+            exec(codemod, context)
 
         logs = "\n".join(state.log_buffer)
-
         state.reset()
-        return {"message": "Codemod executed and codebase reset.", "logs": logs}
+        return {"message": "Codemod executed, view the logs for any output and your source code for any resulting updates.", "logs": logs}
     except Exception as e:
         return {"error": f"Error executing codemod: {str(e)}", "details": {"type": type(e).__name__, "message": str(e)}}
 

codegen-examples/examples/codegen-mcp-server/server.py

@@ -79,7 +79,7 @@ def cli():
 @cli.command()
 @click.argument("repo_name", required=False)
 @click.option("--query", "-q", default=None, help="Initial research query to start with.")
-def research(repo_name: Optional[str] = None, query: Optional[str] = None):
+def research(repo_name: Optional[str] = None, query: Optional[str] = None, thread_id: Optional[int] = 100):
     """[bold green]Start a code research session[/bold green]
 
     [blue]Arguments:[/blue]
@@ -107,7 +107,7 @@ def research(repo_name: Optional[str] = None, query: Optional[str] = None):
 
     # Initialize agent with research tools
     with console.status("[bold blue]Initializing research agent...[/bold blue]") as status:
-        agent = create_agent_with_tools(codebase=codebase, tools=tools, chat_history=[SystemMessage(content=RESEARCH_AGENT_PROMPT)], verbose=True)
+        agent = create_agent_with_tools(codebase=codebase, tools=tools, system_message=SystemMessage(content=RESEARCH_AGENT_PROMPT))
         status.update("[bold green]✓ Research agent ready![/bold green]")
 
     # Get initial query if not provided
@@ -136,11 +136,11 @@ def research(repo_name: Optional[str] = None, query: Optional[str] = None):
             try:
                 result = agent.invoke(
                     {"input": query},
-                    config={"configurable": {"session_id": "research"}},
+                    config={"configurable": {"thread_id": thread_id}},
                 )
                 # Display the result
                 console.print("\n[bold blue]📊 Research Findings:[/bold blue]")
-                console.print(Markdown(result["output"]))
+                console.print(Markdown(result["messages"][-1].content))
             except Exception as e:
                 console.print(f"\n[bold red]Error during research:[/bold red] {e}")
 

codegen-examples/examples/deep_code_research/run.py

@@ -29,11 +29,16 @@ from codegen.extensions.langchain import create_codebase_agent
 codebase = Codebase.from_repo("fastapi/fastapi")
 
 # Create the agent
-agent = create_codebase_agent(codebase=codebase, model_name="gpt-4", verbose=True)
+agent = create_codebase_agent(
+    codebase=codebase,
+    model_provider="anthropic",  # or "openai"
+    model_name="claude-3-5-sonnet-latest",  # or "gpt-4" for OpenAI
+    debug=True,
+)
 
 # Ask the agent to analyze code
-result = agent.invoke({"input": "What are the dependencies of the FastAPI class?", "config": {"configurable": {"session_id": "demo"}}})
-print(result["output"])
+result = agent.invoke({"input": "What are the dependencies of the FastAPI class?", "config": {"configurable": {"thread_id": "1"}}})
+print(result["messages"][-1].content)
 ```
 
 ## Installation
@@ -68,13 +73,13 @@ The agent can perform various code analysis and manipulation tasks:
 
 ```python
 # Analyze dependencies
-agent.invoke({"input": "What are the dependencies of the reveal_symbol function?", "config": {"configurable": {"session_id": "demo"}}})
+agent.invoke({"input": "What are the dependencies of the reveal_symbol function?", "config": {"configurable": {"thread_id": "1"}}})
 
 # Find usage patterns
-agent.invoke({"input": "Show me examples of dependency injection in the codebase", "config": {"configurable": {"session_id": "demo"}}})
+agent.invoke({"input": "Show me examples of dependency injection in the codebase", "config": {"configurable": {"thread_id": "1"}}})
 
 # Move code
-agent.invoke({"input": "Move the validate_email function to validation_utils.py", "config": {"configurable": {"session_id": "demo"}}})
+agent.invoke({"input": "Move the validate_email function to validation_utils.py", "config": {"configurable": {"thread_id": "1"}}})
 ```
 
 ## Learn More