Merge pull request #1 from jacksonthall22/jacksonthall22/ui-submodules

Web: add recurse submodules option

Author: jacksonthall22

src/gitingest/clone.py

@@ -27,6 +27,59 @@
 # Initialize logger for this module
 logger = get_logger(__name__)
 
+_ASKPASS_SCRIPT_NAME = "gitingest-askpass.sh"
+
+
+def _write_git_askpass_script(repo_path: Path) -> Path:
+    """Write a small askpass helper into ``.git`` that reads the token from env.
+
+    The script never embeds secrets; it prints the username and reads the token from
+    ``GITINGEST_GIT_PASSWORD``.
+    """
+    git_dir = repo_path / ".git"
+    git_dir.mkdir(parents=True, exist_ok=True)
+
+    askpass_path = git_dir / _ASKPASS_SCRIPT_NAME
+    askpass_path.write_text(
+        "#!/bin/sh\n"
+        'case "$1" in\n'
+        '  Username*) echo "x-access-token" ;;\n'
+        '  Password*) echo "${GITINGEST_GIT_PASSWORD:-}" ;;\n'
+        '  *) echo "" ;;\n'
+        "esac\n",
+        encoding="utf-8",
+    )
+    try:
+        askpass_path.chmod(0o700)
+    except OSError:
+        # Best-effort on platforms where chmod may be unsupported.
+        pass
+    return askpass_path
+
+
+def _configure_submodule_auth(repo: git.Repo, *, token: str | None, url: str, local_path: str) -> None:
+    """Disable interactive prompts and provide an askpass hook for private submodules."""
+    try:
+        repo.git.update_environment(GIT_TERMINAL_PROMPT="0")
+    except Exception:
+        # Best-effort: if the GitPython object doesn't support env updates, continue.
+        return
+
+    if not (token and is_github_host(url)):
+        return
+
+    try:
+        askpass_path = _write_git_askpass_script(Path(local_path))
+    except OSError:
+        logger.exception("Failed to write GIT_ASKPASS helper", extra={"local_path": local_path})
+        return
+
+    repo.git.update_environment(
+        GIT_ASKPASS=str(askpass_path),
+        GIT_TERMINAL_PROMPT="0",
+        GITINGEST_GIT_PASSWORD=token,
+    )
+
 
 @async_timeout(DEFAULT_TIMEOUT)
 async def clone_repo(config: CloneConfig, *, token: str | None = None) -> None:
@@ -169,6 +222,7 @@ async def _perform_post_clone_operations(
         # Update submodules
         if config.include_submodules:
             logger.info("Updating submodules")
+            _configure_submodule_auth(repo, token=token, url=url, local_path=local_path)
             repo.git.submodule("update", "--init", "--recursive", "--depth=1")
             logger.debug("Submodules updated successfully")
     except git.GitCommandError as exc:

src/server/models.py

@@ -29,6 +29,8 @@ class IngestRequest(BaseModel):
     ----------
     input_text : str
         The Git repository URL or slug to ingest.
+    include_submodules : bool
+        If ``True``, recursively clone and include Git submodules (default: ``False``).
     max_file_size : int
         Maximum file size slider position (0-500) for filtering files.
     pattern_type : PatternType
@@ -41,6 +43,7 @@ class IngestRequest(BaseModel):
     """
 
     input_text: str = Field(..., description="Git repository URL or slug to ingest")
+    include_submodules: bool = Field(default=False, description="Recursively clone and include Git submodules")
     max_file_size: int = Field(..., ge=1, le=MAX_FILE_SIZE_KB, description="File size in KB")
     pattern_type: PatternType = Field(default=PatternType.EXCLUDE, description="Pattern type for file filtering")
     pattern: str = Field(default="", description="Glob/regex pattern for file filtering")

src/server/query_processor.py

@@ -91,6 +91,7 @@ async def _check_s3_cache(
             repo_name=cast("str", query.repo_name),
             commit=query.commit,
             subpath=query.subpath,
+            include_submodules=query.include_submodules,
             include_patterns=query.include_patterns,
             ignore_patterns=query.ignore_patterns,
         )
@@ -170,6 +171,7 @@ def _store_digest_content(
             repo_name=cast("str", query.repo_name),
             commit=query.commit,
             subpath=query.subpath,
+            include_submodules=query.include_submodules,
             include_patterns=query.include_patterns,
             ignore_patterns=query.ignore_patterns,
         )
@@ -232,6 +234,7 @@ async def process_query(
     pattern_type: PatternType,
     pattern: str,
     token: str | None = None,
+    include_submodules: bool = False,
 ) -> IngestResponse:
     """Process a query by parsing input, cloning a repository, and generating a summary.
 
@@ -250,6 +253,8 @@ async def process_query(
         Pattern to include or exclude in the query, depending on the pattern type.
     token : str | None
         GitHub personal access token (PAT) for accessing private repositories.
+    include_submodules : bool
+        If ``True``, recursively clone and include Git submodules.
 
     Returns
     -------
@@ -272,6 +277,7 @@ async def process_query(
         return IngestErrorResponse(error=str(exc))
 
     query.url = cast("str", query.url)
+    query.include_submodules = include_submodules
     query.max_file_size = max_file_size * 1024  # Convert to bytes since we currently use KB in higher levels
     query.ignore_patterns, query.include_patterns = process_patterns(
         exclude_patterns=pattern if pattern_type == PatternType.EXCLUDE else None,

src/server/routers/ingest.py

@@ -42,6 +42,7 @@ async def api_ingest(
     """
     response = await _perform_ingestion(
         input_text=ingest_request.input_text,
+        include_submodules=ingest_request.include_submodules,
         max_file_size=ingest_request.max_file_size,
         pattern_type=ingest_request.pattern_type.value,
         pattern=ingest_request.pattern,
@@ -58,6 +59,7 @@ async def api_ingest_get(
     request: Request,  # noqa: ARG001 (unused-function-argument) # pylint: disable=unused-argument
     user: str,
     repository: str,
+    include_submodules: bool = False,
     max_file_size: int = DEFAULT_FILE_SIZE_KB,
     pattern_type: str = "exclude",
     pattern: str = "",
@@ -74,6 +76,7 @@ async def api_ingest_get(
     - **repository** (`str`): GitHub repository name
 
     **Query Parameters**
+    - **include_submodules** (`bool`, optional): Whether to recursively clone and include Git submodules (default: ``False``)
     - **max_file_size** (`int`, optional): Maximum file size in KB to include in the digest (default: 5120 KB)
     - **pattern_type** (`str`, optional): Type of pattern to use ("include" or "exclude", default: "exclude")
     - **pattern** (`str`, optional): Pattern to include or exclude in the query (default: "")
@@ -84,6 +87,7 @@ async def api_ingest_get(
     """
     response = await _perform_ingestion(
         input_text=f"{user}/{repository}",
+        include_submodules=include_submodules,
         max_file_size=max_file_size,
         pattern_type=pattern_type,
         pattern=pattern,

src/server/routers_utils.py

@@ -19,6 +19,7 @@
 
 async def _perform_ingestion(
     input_text: str,
+    include_submodules: bool,
     max_file_size: int,
     pattern_type: str,
     pattern: str,
@@ -33,6 +34,7 @@ async def _perform_ingestion(
 
         result = await process_query(
             input_text=input_text,
+            include_submodules=include_submodules,
             max_file_size=max_file_size,
             pattern_type=pattern_type,
             pattern=pattern,

src/server/s3_utils.py

@@ -63,6 +63,7 @@ def generate_s3_file_path(
     repo_name: str,
     commit: str,
     subpath: str,
+    include_submodules: bool,
     include_patterns: set[str] | None,
     ignore_patterns: set[str],
 ) -> str:
@@ -92,6 +93,8 @@ def generate_s3_file_path(
         Set of patterns specifying which files to include.
     ignore_patterns : set[str]
         Set of patterns specifying which files to exclude.
+    include_submodules : bool
+        Whether to recursively clone and include Git submodules.
 
     Returns
     -------
@@ -111,7 +114,8 @@ def generate_s3_file_path(
         raise ValueError(msg)
 
     # Create hash of exclude/include patterns for uniqueness
-    patterns_str = f"include:{sorted(include_patterns) if include_patterns else []}"
+    patterns_str = f"submodules:{int(include_submodules)}"
+    patterns_str += f"include:{sorted(include_patterns) if include_patterns else []}"
     patterns_str += f"exclude:{sorted(ignore_patterns)}"
     patterns_hash = hashlib.sha256(patterns_str.encode()).hexdigest()[:16]
     subpath_hash = hashlib.sha256(subpath.encode()).hexdigest()[:16]

src/server/templates/components/git_form.jinja

@@ -85,6 +85,23 @@
                 </div>
                 <!-- PAT checkbox with PAT field below -->
                 <div class="flex flex-col items-start w-full sm:col-span-2 lg:col-span-1 lg:row-span-2 lg:pt-3.5">
+                    <!-- Recurse submodules checkbox -->
+                    <div class="flex items-center space-x-2 mb-3">
+                        <label for="include_submodules"
+                               class="flex gap-2 text-gray-900 cursor-pointer">
+                            <div class="relative w-6 h-6">
+                                <input type="checkbox"
+                                       id="include_submodules"
+                                       name="include_submodules"
+                                       {% if include_submodules %}checked{% endif %}
+                                       class="cursor-pointer peer appearance-none w-full h-full rounded-sm border-[3px] border-current bg-white m-0 text-current shadow-[3px_3px_0_currentColor]" />
+                                <span class="absolute inset-0 w-3 h-3 m-auto scale-0 transition-transform duration-150 ease-in-out shadow-[inset_1rem_1rem_#FE4A60] bg-[CanvasText] origin-bottom-left peer-checked:scale-100"
+                                      style="clip-path: polygon(14% 44%, 0 65%, 50% 100%, 100% 16%, 80% 0%, 43% 62%)"></span>
+                            </div>
+                            Recurse submodules
+                        </label>
+                        <span class="badge-new">NEW</span>
+                    </div>
                     <!-- PAT checkbox -->
                     <div class="flex items-center space-x-2">
                         <label for="showAccessSettings"

src/static/js/utils.js

@@ -126,12 +126,14 @@ function collectFormData(form) {
     const json_data = {};
     const inputText = form.querySelector('[name="input_text"]');
     const token = form.querySelector('[name="token"]');
+    const includeSubmodules = form.querySelector('[name="include_submodules"]');
     const hiddenInput = document.getElementById('max_file_size_kb');
     const patternType = document.getElementById('pattern_type');
     const pattern = document.getElementById('pattern');
 
     if (inputText) {json_data.input_text = inputText.value;}
     if (token) {json_data.token = token.value;}
+    if (includeSubmodules) {json_data.include_submodules = includeSubmodules.checked;}
     if (hiddenInput) {json_data.max_file_size = hiddenInput.value;}
     if (patternType) {json_data.pattern_type = patternType.value;}
     if (pattern) {json_data.pattern = pattern.value;}

tests/server/test_include_submodules.py

@@ -0,0 +1,43 @@
+import pytest
+from pytest_mock import MockerFixture
+from unittest.mock import AsyncMock
+
+from gitingest.schemas.ingestion import IngestionQuery
+from src.server.models import PatternType
+
+
+@pytest.mark.asyncio
+async def test_process_query_forwards_include_submodules(mocker: MockerFixture) -> None:
+    query = IngestionQuery(
+        host="github.com",
+        user_name="octocat",
+        repo_name="Hello-World",
+        local_path="/tmp/gitingest/test-include-submodules",
+        url="https://github.com/octocat/Hello-World",
+        slug="octocat/Hello-World",
+        id="00000000-0000-0000-0000-000000000001",
+        branch="main",
+        commit="deadbeef",
+    )
+
+    mocker.patch("src.server.query_processor.parse_remote_repo", new_callable=AsyncMock, return_value=query)
+    clone_repo = mocker.patch("src.server.query_processor.clone_repo", new_callable=AsyncMock)
+    mocker.patch("src.server.query_processor.ingest_query", return_value=("summary", "tree", "content"))
+    mocker.patch("src.server.query_processor._store_digest_content")
+    mocker.patch("src.server.query_processor._cleanup_repository")
+    mocker.patch("src.server.query_processor._print_success")
+
+    from src.server.query_processor import process_query
+
+    await process_query(
+        input_text="https://github.com/octocat/Hello-World",
+        max_file_size=243,
+        pattern_type=PatternType.EXCLUDE,
+        pattern="",
+        token=None,
+        include_submodules=True,
+    )
+
+    assert clone_repo.call_count == 1
+    passed_config = clone_repo.call_args.args[0]
+    assert getattr(passed_config, "include_submodules") is True

tests/test_clone.py

@@ -205,6 +205,27 @@ async def test_clone_with_include_submodules(gitpython_mocks: dict) -> None:
     mock_repo.git.submodule.assert_called_with("update", "--init", "--recursive", "--depth=1")
 
 
+@pytest.mark.asyncio
+async def test_clone_with_private_submodules_uses_askpass(tmp_path: Path, gitpython_mocks: dict) -> None:
+    """Test cloning a repo with private submodules using a non-interactive auth helper.
+
+    Given ``include_submodules=True`` and a GitHub token:
+    When ``clone_repo`` is called,
+    Then it should configure GIT_ASKPASS so submodules can be cloned without a TTY prompt.
+    """
+    local_repo_path = tmp_path / "repo"
+    clone_config = CloneConfig(url=DEMO_URL, local_path=str(local_repo_path), branch="main", include_submodules=True)
+
+    await clone_repo(clone_config, token="token123")  # noqa: S106 (test-only)
+
+    askpass_path = local_repo_path / ".git" / "gitingest-askpass.sh"
+    assert askpass_path.exists()
+    assert "token123" not in askpass_path.read_text(encoding="utf-8")
+
+    mock_repo = gitpython_mocks["repo"]
+    assert any("GIT_ASKPASS" in kwargs for _, kwargs in mock_repo.git.update_environment.call_args_list)
+
+
 @pytest.mark.asyncio
 async def test_check_repo_exists_with_auth_token(mocker: MockerFixture) -> None:
     """Test ``check_repo_exists`` with authentication token.