CG-10610: System Prompt Generation

.pre-commit-config.yaml

@@ -99,7 +99,7 @@ repos:
         files: ''  # Check all files
       - id: generate-runner-imports
         name: Generate Runner Imports
-        entry: bash -c "uv run --frozen python -m codegen.gscli.main generate runner-imports src/codegen/shared/compilation/function_imports.py"
+        entry: bash -c "uv run --frozen python -m codegen.gscli.cli generate runner-imports src/codegen/shared/compilation/function_imports.py"
         language: system
         pass_filenames: false
         always_run: true

docs/snippets/links.mdx

@@ -6,4 +6,4 @@ export const CODEGEN_SDK_GITHUB_URL =
 export const CODEGEN_SDK_EXAMPLES_GITHUB_URL =
   "https://github.com/codegen-sh/codegen-examples";
 
-export const CODEGEN_SYSTEM_PROMPT = "https://gist.githubusercontent.com/jayhack/15681a2ceaccd726f19e6fdb3a44738b/raw/17c08054e3931b3b7fdf424458269c9e607541e8/codegen-system-prompt.txt"
+export const CODEGEN_SYSTEM_PROMPT = "https://gist.githubusercontent.com/codegen-team/708a870eec379d1f8086bd722f668978/raw/system-prompt.txt"

src/codegen/git/clients/gist_client.py

@@ -0,0 +1,116 @@
+from typing import Any, Optional
+from urllib.parse import urljoin
+
+import requests
+from requests.exceptions import RequestException
+
+
+class GistClientError(Exception):
+    """Base exception for GistClient errors."""
+
+    pass
+
+
+class GistAuthenticationError(GistClientError):
+    """Raised when authentication fails."""
+
+    pass
+
+
+class GistClient:
+    """A client for interacting with GitHub Gists API.
+
+    This client provides methods to read and update GitHub Gists using the GitHub API v3.
+    It supports both authenticated and unauthenticated requests, though some operations
+    require authentication.
+    """
+
+    def __init__(self, token: Optional[str] = None) -> None:
+        """Initialize the GistClient with your GitHub personal access token.
+
+        Args:
+            token (Optional[str]): GitHub personal access token with gist scope
+
+        Raises:
+            GistAuthenticationError: If the provided token is invalid
+        """
+        self.base_url = "https://api.github.com"
+        self.headers = {"Accept": "application/vnd.github.v3+json", "User-Agent": "GistClient"}
+
+        if token:
+            self.headers["Authorization"] = f"token {token}"
+
+        self.session = requests.Session()
+        self.session.headers.update(self.headers)
+
+    def _make_request(self, method: str, endpoint: str, **kwargs) -> dict[str, Any]:
+        """Make an HTTP request to the GitHub API.
+
+        Args:
+            method (str): HTTP method to use
+            endpoint (str): API endpoint to call
+            **kwargs: Additional arguments to pass to requests
+
+        Returns:
+            Dict[str, Any]: JSON response from the API
+
+        Raises:
+            GistClientError: If the request fails
+            GistAuthenticationError: If authentication fails
+        """
+        try:
+            url = urljoin(self.base_url, endpoint)
+            response = self.session.request(method, url, **kwargs)
+            response.raise_for_status()
+            return response.json()
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 401:
+                msg = "Invalid authentication token"
+                raise GistAuthenticationError(msg) from e
+            msg = f"GitHub API request failed: {e!s}"
+            raise GistClientError(msg) from e
+        except RequestException as e:
+            msg = f"Request failed: {e!s}"
+            raise GistClientError(msg) from e
+
+    def get_gist(self, gist_id: str) -> dict[str, Any]:
+        """Fetch a specific gist.
+
+        Args:
+            gist_id (str): The ID of the gist to fetch
+
+        Returns:
+            Dict[str, Any]: The gist data
+        """
+        return self._make_request("GET", f"/gists/{gist_id}")
+
+    def update_gist(self, gist_id: str, filename: str, content: str, description: Optional[str] = None) -> dict[str, Any]:
+        """Update a specific file in a gist.
+
+        Args:
+            gist_id (str): The ID of the gist to update
+            filename (str): The name of the file to update
+            content (str): The new content for the file
+            description (Optional[str]): New description for the gist
+
+        Returns:
+            Dict[str, Any]: The updated gist data
+
+        Raises:
+            GistAuthenticationError: If no authentication token was provided
+        """
+        if not self.headers.get("Authorization"):
+            msg = "Authentication token required to update a gist"
+            raise GistAuthenticationError(msg)
+
+        payload = {"files": {filename: {"content": content}}}
+        if description:
+            payload["description"] = description
+
+        return self._make_request("PATCH", f"/gists/{gist_id}", json=payload)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.session.close()

src/codegen/gscli/main.py → src/codegen/gscli/cli.py

@@ -1,23 +1,17 @@
 #!/bin/python
-import logging
 
 import click
 
 from codegen.gscli.generate.commands import generate
 
 
 @click.group()
-def cli() -> None:
+def main() -> None:
     pass
 
 
 # ============= Import all command groups =============
-cli.add_command(generate)
-
-
-def main() -> None:
-    logging.basicConfig(level=logging.INFO)
-    cli()
+main.add_command(generate)
 
 
 if __name__ == "__main__":

src/codegen/gscli/generate/commands.py

@@ -3,12 +3,17 @@
 import os
 import re
 import shutil
+from pathlib import Path
 
 import click
 from termcolor import colored
 
+from codegen.git.clients.gist_client import GistClient
+from codegen.gscli.generate.constants import SYSTEM_PROMPT_FILENAME, SYSTEM_PROMPT_GIST_ID
 from codegen.gscli.generate.runner_imports import _generate_runner_imports
+from codegen.gscli.generate.system_prompt import get_system_prompt
 from codegen.gscli.generate.utils import LanguageType, generate_builtins_file
+from codegen.sdk import __version__ as codegen_sdk_version
 from codegen.sdk.code_generation.codegen_sdk_codebase import get_codegen_sdk_codebase
 from codegen.sdk.code_generation.doc_utils.generate_docs_json import generate_docs_json
 from codegen.sdk.code_generation.mdx_docs_generation import render_mdx_page_for_class
@@ -104,9 +109,30 @@ def generate_docs(docs_dir: str) -> None:
     This will generate docs using the codebase locally, including any unstaged changes
     """
     generate_codegen_sdk_docs(docs_dir)
-    # generate_canonical_codemod_docs(docs_dir, codebase)
-    # generate_skills_docs(docs_dir)
-    # generate_guides(docs_dir)
+
+
+@generate.command()
+@click.option("--token", default=None, required=False, help="The GitHub token to use for updating the gist.")
+@click.option("--update-remote", default=False, required=False, is_flag=True, help="Update the system prompt in the gist.")
+@click.option("--filepath", default="./system-prompt.txt", required=False, help="The path to the file to write the system prompt to.")
+def system_prompt(token: str | None, update_remote: bool, filepath: str) -> None:
+    """Read the system prompt from the gist"""
+    with GistClient(token) as client:
+        if update_remote:
+            new_system_prompt = get_system_prompt()
+            print(f"Updating system prompt in gist {SYSTEM_PROMPT_GIST_ID} with filename {filepath}...")
+            description = f"System Prompt for Codegen SDK ({codegen_sdk_version})"
+            client.update_gist(SYSTEM_PROMPT_GIST_ID, SYSTEM_PROMPT_FILENAME, new_system_prompt, description)
+            print(f"Successfully updated system prompt in gist with filename {SYSTEM_PROMPT_FILENAME}.")
+            print(f"New description: {description}")
+        else:
+            print(f"Reading system prompt from gist {SYSTEM_PROMPT_GIST_ID}...")
+            gist = client.get_gist(SYSTEM_PROMPT_GIST_ID)
+            content = gist["files"][SYSTEM_PROMPT_FILENAME]["content"]
+            file_path = Path(filepath)
+            print(f"Writing system prompt to {filepath}...")
+            file_path.write_text(content)
+            print(f"Successfully wrote system prompt to {filepath}.")
 
 
 def get_snippet_pattern(target_name: str) -> str:
@@ -186,7 +212,3 @@ def generate_codegen_sdk_docs(docs_dir: str) -> None:
         json.dump(mint_data, mint_file, indent=2)
 
     print(colored("Updated mint.json with new page sets", "green"))
-
-
-if __name__ == "__main__":
-    generate()

src/codegen/gscli/generate/constants.py

@@ -0,0 +1,2 @@
+SYSTEM_PROMPT_GIST_ID = "708a870eec379d1f8086bd722f668978"
+SYSTEM_PROMPT_FILENAME = "system-prompt.txt"