CG-10491: Move skills testing + utils into tests/ (#96)

# Motivation
<!-- Why is this change necessary? -->

# Content
<!-- Please include a summary of the change -->
# Testing
<!-- How was the change tested? -->
# Please check the following before marking your PR as ready for review

- [x] I have added tests for my changes
- [x] I have updated the documentation or added new documentation as
needed
- [x] I have read and agree to the [Contributor License
Agreement](../CLA.md)

---------

Co-authored-by: codegen-bot <[email protected]>

Author: caroljung-cg

.circleci/config.yml

@@ -61,7 +61,7 @@ commands:
             if [ "<<parameters.extra_repos>>" = "true" ]; then
               EXTRA_REPOS_ARG="--extra-repos"
             fi
-            uv run --frozen gs codemod clone-repos ${EXTRA_REPOS_ARG} --token ${CODEGEN_BOT_GHE_TOKEN} --clean-cache
+            uv run --frozen python -m tests.shared.codemod.commands clone-repos ${EXTRA_REPOS_ARG} --token ${CODEGEN_BOT_GHE_TOKEN} --clean-cache
       - save_cache:
           paths:
             - $GITHUB_WORKSPACE
@@ -70,15 +70,15 @@ commands:
     steps:
       - run:
           command: |
-            uv run --frozen gs codemod fetch-verified-codemods --cli-api-key ${PROD_CLI_API_KEY}
+            uv run --frozen python -m tests.shared.codemod.commands fetch-verified-codemods --cli-api-key ${PROD_CLI_API_KEY}
   cache-verified-codemod-repos:
     steps:
       - restore_cache:
           keys:
             - v1-verified-codemod-repos-{{ checksum "tests/integration/verified_codemods/codemod_data/repo_commits.json" }}-{{.Environment.CIRCLE_NODE_INDEX}}-{{.Environment.CIRCLE_NODE_TOTAL}}
       - run:
           command: |
-            uv run --frozen gs codemod clone-repos --verified-codemod-repos --token ${CODEGEN_BOT_GHE_TOKEN}
+            uv run --frozen python -m tests.shared.codemod.commands clone-repos --verified-codemod-repos --token ${CODEGEN_BOT_GHE_TOKEN}
       - save_cache:
           paths:
             - $GITHUB_WORKSPACE

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
     hooks:
       - id: biome-check
         additional_dependencies: ["@biomejs/[email protected]"]
-        exclude: (src/codemods/eval)|(tests/unit/codegen/sdk/skills)|(tests/unit/codegen/sdk/output)|(tests/integration/verified_codemods)|(docs/samples)
+        exclude: (src/codemods/eval)|(tests/unit/skills/snapshots)|(tests/unit/codegen/sdk/output)|(tests/integration/verified_codemods)|(docs/samples)
 
   - repo: https://github.com/MarcoGorelli/cython-lint
     rev: v0.16.6

Dockerfile

@@ -34,7 +34,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv run pre-commit install-hooks
 FROM base-image AS extra-repos
 ARG CODEGEN_BOT_GHE_TOKEN=""
-RUN uv run gs codemod clone-repos --clean-cache --extra-repos --token ${CODEGEN_BOT_GHE_TOKEN}
+RUN uv run python -m tests.shared.codemod.commands clone-repos --clean-cache --extra-repos --token ${CODEGEN_BOT_GHE_TOKEN}
 FROM base-image AS oss-repos
 ARG CODEGEN_BOT_GHE_TOKEN=""
-RUN uv run gs codemod clone-repos --clean-cache --token ${CODEGEN_BOT_GHE_TOKEN}
+RUN uv run python -m tests.shared.codemod.commands clone-repos --clean-cache --token ${CODEGEN_BOT_GHE_TOKEN}

conftest.py

@@ -1,7 +1,7 @@
 import logging
 import os
 
-from codegen.sdk.testing.models import Size
+from tests.shared.codemod.models import Size
 
 
 def find_dirs_to_ignore(start_dir, prefix):

mypy.ini

@@ -46,7 +46,7 @@ exclude = (?x)(
           | scripts/.*
           | .*/codemods/.*/test_.*/
           | ^/expected/
-          | src/codegen/sdk/skills/*
+          | tests/unit/skills/snapshots/*
     )
 plugins = pydantic.mypy
 

pyproject.toml

@@ -23,19 +23,14 @@ dependencies = [
     "networkx>=3.4.1",
     "wrapt<2.0.0,>=1.16.0",
     "pyright<2.0.0,>=1.1.372",
-    "jsbeautifier<2.0.0,>=1.15.1",
     "watchfiles<1.1.0,>=1.0.0",
     "loguru<1.0.0,>=0.7.2",
     "rich<14.0.0,>=13.7.1",
-    "inflection<1.0.0,>=0.5.1",
     "pydantic<3.0.0,>=2.9.2",
     "docstring-parser<1.0,>=0.16",
     "plotly<6.0.0,>=5.24.0",
     "humanize<5.0.0,>=4.10.0",
-    "autoflake>=2.3.1",
-    "black>=24.8.0",
     "pytest-snapshot>=0.9.0",
-    "isort>=5.13.2",
     "anthropic==0.23.1",
     "pyjson5==1.6.8",
     "mini-racer>=0.12.4",
@@ -52,7 +47,6 @@ dependencies = [
     "hatchling>=1.25.0",
     "pyinstrument>=5.0.0",
     "pip>=24.3.1", # This is needed for some NPM/YARN/PNPM post-install scripts to work!
-    "emoji>=2.14.0",
     "rich-click>=1.8.5",
     "python-dotenv>=1.0.1",
     "giturlparse",
@@ -110,6 +104,12 @@ dev-dependencies = [
     "deptry>=0.22.0",
     "cibuildwheel[uv]>=2.22.0",
     "sybil[pytest]>=9.0.0",
+    "jsbeautifier<2.0.0,>=1.15.1",
+    "inflection<1.0.0,>=0.5.1",
+    "autoflake>=2.3.1",
+    "black>=24.8.0",
+    "isort>=5.13.2",
+    "emoji>=2.14.0",
 ]
 keyring-provider = "subprocess"
 #extra-index-url = ["https://[email protected]/pypi/codegen/simple/"]

src/codegen/gscli/generate/commands.py

@@ -5,17 +5,13 @@
 import shutil
 
 import click
-import inflection
 from termcolor import colored
 
 from codegen.gscli.generate.runner_imports import _generate_runner_imports
 from codegen.gscli.generate.utils import LanguageType, generate_builtins_file
 from codegen.sdk.code_generation.codegen_sdk_codebase import get_codegen_sdk_codebase
 from codegen.sdk.code_generation.doc_utils.generate_docs_json import generate_docs_json
-from codegen.sdk.code_generation.doc_utils.skills import format_all_skills
 from codegen.sdk.code_generation.mdx_docs_generation import render_mdx_page_for_class
-from codegen.sdk.python import PyClass
-from codegen.sdk.skills.core.utils import get_all_skills, get_guide_skills_dict
 
 logger = logging.getLogger(__name__)
 
@@ -113,26 +109,6 @@ def generate_docs(docs_dir: str) -> None:
     # generate_guides(docs_dir)
 
 
-def generate_guides(docs_dir: str):
-    """Updates code snippets in the guides with the latest skill implementations"""
-    guide_skills = get_guide_skills_dict()
-    for guide_relative_path in guide_skills:
-        guide_file_path = os.path.join(docs_dir, "api-reference", str(guide_relative_path) + ".mdx")
-
-        with open(guide_file_path) as f:
-            file_content = f.read()
-
-        for skill in guide_skills[guide_relative_path]:
-            # Perform the regex search and replace
-            target_name = inflection.camelize(skill.name)
-            pattern = get_snippet_pattern(target_name)
-            replacement = f"[//]: # (--{target_name}--)\n[//]: # (--{AUTO_GENERATED_COMMENT}--)\n\n{skill.generate_snippet()}"
-            file_content = re.sub(pattern, replacement, file_content, flags=re.DOTALL)
-
-        with open(guide_file_path, "w") as f:
-            f.write(file_content)
-
-
 def get_snippet_pattern(target_name: str) -> str:
     pattern = rf"\[//\]: # \(--{re.escape(target_name)}--\)\s*(?:\[//\]: # \(--{re.escape(AUTO_GENERATED_COMMENT)}--\)\s*)?"
     pattern += CODE_SNIPPETS_REGEX
@@ -210,29 +186,3 @@ def generate_codegen_sdk_docs(docs_dir: str) -> None:
         json.dump(mint_data, mint_file, indent=2)
 
     print(colored("Updated mint.json with new page sets", "green"))
-
-
-def generate_skills_docs(docs_dir: str) -> None:
-    """Generates docs for all skills"""
-    print(colored("Generating skills docs", "green"))
-    # Interim Modification: We need to refactor all skills
-    skills = [skill for skill in get_all_skills() if not skill.guide]
-    skills_doc = format_all_skills(skills)
-
-    # =====[ Write the skills docs to the file system ]=====
-    core_dir = os.path.join(docs_dir, "codebase-sdk", "examples")
-    file_path = os.path.join(core_dir, "skills.mdx")
-    print(colored(f"> Writing to {file_path}", "green"))
-    open(file_path, "w").write(skills_doc)
-
-
-def document_skill(skill: PyClass):
-    return "skill" in [decorator.name for decorator in skill.decorators]
-
-
-def filter_class(cls: PyClass):
-    for method in cls.methods:
-        for decorator in method.decorators:
-            if decorator.name == "skill_impl" and "external=True" in decorator.source:
-                return True
-    return False

src/codegen/gscli/main.py

@@ -3,7 +3,6 @@
 
 import click
 
-from codegen.gscli.codemod.commands import codemod
 from codegen.gscli.generate.commands import generate
 
 
@@ -13,7 +12,6 @@ def cli() -> None:
 
 
 # ============= Import all command groups =============
-cli.add_command(codemod)
 cli.add_command(generate)