fix: multi source runner

Author: clee-codegen

codegen-on-oss/codegen_on_oss/sources/__init__.py

@@ -1,13 +1,16 @@
 from .base import RepoSource, SourceSettings, all_sources
 from .csv_source import CSVInputSettings, CSVInputSource
 from .github_source import GithubSettings, GithubSource
+from .single_source import SingleSettings, SingleSource
 
 __all__ = [
     "CSVInputSettings",
     "CSVInputSource",
     "GithubSettings",
     "GithubSource",
     "RepoSource",
+    "SingleSettings",
+    "SingleSource",
     "SourceSettings",
     "all_sources",
 ]

codegen-on-oss/codegen_on_oss/sources/github_source.py

@@ -23,6 +23,7 @@ class GithubSettings(SourceSettings, env_prefix="GITHUB_"):
         # "dependency",
     ] = "stars"
     token: str | None = None
+    num_repos: int = 50
 
 
 class GithubSource(RepoSource[GithubSettings]):
@@ -37,8 +38,8 @@ class GithubSource(RepoSource[GithubSettings]):
     source_type: ClassVar[str] = "github"
     settings_cls: ClassVar[type[GithubSettings]] = GithubSettings
 
-    def __init__(self) -> None:
-        super().__init__()
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
         if self.settings.token is None:
             self.github_client = Github()
         else:

codegen-on-oss/input.csv

@@ -1,35 +1,2 @@
 repo_url, commit_hash
 https://github.com/JohnSnowLabs/spark-nlp.git,7d2bed7647bec3878362a4b7ebbe43befeabe2ba
-https://github.com/Lightning-AI/lightning.git,b313cd955bb7cc38bd899a023dfc43f4f194d50b
-https://github.com/PrefectHQ/prefect.git,3fa131368278cddd865802090b70c2ec5c971e8b
-https://github.com/apache/mxnet.git,b84609d3fc73d20929c114eab95faaa56e6c5ede
-https://github.com/celery/celery.git,7d5157343cc919b0f94d602ff1d2c037b38815a3
-https://github.com/conan-io/conan.git,8488bcbc051a0511f1bf1e92ff991dfcab4a5644
-https://github.com/conda/conda.git,b557176f293bb5b94ce6680524cd4e088ddffac2
-https://github.com/dagster-io/dagster.git,b3db2312329c0a809ed98b362da44c8ea9690a25
-https://github.com/docker/compose.git,d956ff13daacbb15905ac2877428e4e9cbcb65da
-https://github.com/explosion/spaCy.git,b3c46c315eb16ce644bddd106d31c3dd349f6bb2
-https://github.com/gitpython-developers/GitPython.git,e034cec8bbed5d21ca15f415a541c41c6cab0463
-https://github.com/google/jax.git,1079dc4477d41fd25397c8d0b78a32bdc5fa48da
-https://github.com/googleapis/google-cloud-python.git,04a1a30536c108b912ac45482496929fddd7beea
-https://github.com/huggingface/transformers.git,9f51dc25357bcde280a02b59e80b66248b018ca4
-https://github.com/ipython/ipython.git,aedea319404107bfec9f7c70356cbba68790c06e
-https://github.com/jupyterlab/jupyterlab.git,f15d555c89f3aa73bb16610d8056a1597bce6edc
-https://github.com/kubeflow/pipelines.git,8ca7ec1768f82f50b3b46606f39632eab11b8fe6
-https://github.com/mesonbuild/meson.git,1b54239a88261cbb679a342162d72632cd6b8093
-https://github.com/numpy/numpy.git,5566cc4375badc1a1f218c4a1bb8924abbf34618
-https://github.com/open-mmlab/mmdetection.git,cfd5d3a985b0249de009b67d04f37263e11cdf3d
-https://github.com/pandas-dev/pandas.git,d4dff29713faeb74931407ebd3e441253f883708
-https://github.com/pantsbuild/pants.git,6cc70e3245c968c006caf5697fd01cb5f8f54092
-https://github.com/pyca/cryptography.git,f95dda8bce1dac984ac3e0da188f66027184a69e
-https://github.com/pypa/pip.git,ce9451518f17cedf1efb3481526ae760bfdbe1a5
-https://github.com/python/typeshed.git,132456af62b1d15e966e490d9ba9235d2f6bbf2e
-https://github.com/ray-project/ray.git,094fde63cdce99bfe7ddca30d5a04c0759c86ffd
-https://github.com/scipy/scipy.git,0a934b1ffb1269a244cc007851632c466acdc5a0
-https://github.com/tensorflow/models.git,4a56eff0ad5ce23f64f7638167c3256b36da3b07
-https://github.com/tiangolo/fastapi.git,8c9c536c0a277125ca95c0d9ef19e2c6a39d1db8
-https://github.com/twisted/twisted.git,45dd522fcc218ebbc11afa1ed03c77768e7b8e44
-https://github.com/wagtail/wagtail.git,81171b1eaec3f30809f7ce27768bfb6c129e1d99
-https://github.com/ytdl-org/youtube-dl.git,711e72c292327674c4a0593fdbb83d6347738ec9
-https://github.com/DataDog/integrations-core.git,4329c4522f860bc90673d66313722c3b64b0a433
-https://github.com/vercel/next.js.git,20748811c66899fa8eaa221cab96a02a86bc0dd6

codegen-on-oss/modal_run.py

@@ -1,7 +1,6 @@
 import os
 import sys
 from pathlib import Path
-from typing import Literal
 
 import modal
 from loguru import logger
@@ -115,16 +114,13 @@ def parse_repo_on_modal(
 
 @parse_app.local_entrypoint()
 def main(
-    source: Literal["csv", "single", "github"] = "csv",
+    source: str = "csv",
     csv_file: str = "input.csv",
     single_url: str = "https://github.com/codegen-sh/codegen-sdk.git",
     single_commit: str | None = None,
-    github_language: Literal["python", "typescript"] = "python",
-    github_heuristic: Literal[
-        "stars",
-        "forks",
-        "updated",
-    ] = "stars",
+    github_language: str = "python",
+    github_heuristic: str = "stars",
+    github_num_repos: int = 50,
 ):
     """
     Main entrypoint for the parse app.
@@ -140,15 +136,18 @@ def main(
                 "CSV_FILE_PATH": f"/app/inputs/{input_path}",
             }
         case "single":
-            env = {
-                "SINGLE_URL": single_url,
-                "SINGLE_COMMIT": single_commit,
-            }
+            env = {"SINGLE_URL": single_url}
+            if single_commit:
+                env["SINGLE_COMMIT"] = single_commit
         case "github":
             env = {
                 "GITHUB_LANGUAGE": github_language,
                 "GITHUB_HEURISTIC": github_heuristic,
+                "GITHUB_NUM_REPOS": str(github_num_repos),
             }
+        case _:
+            msg = f"Invalid source: {source}"
+            raise ValueError(msg)
 
     return parse_repo_on_modal.remote(
         source=source,