refactor: streamline query processing by removing unnecessary cloning logic and enhancing ignore pattern handling

AbhiRam162105 · AbhiRam162105 · commit 24184fcda8a8 · 2025-02-03T22:50:19.000+05:30
diff --git a/src/gitingest/query_parser.py b/src/gitingest/query_parser.py
@@ -12,7 +12,7 @@
 from gitingest.config import MAX_FILE_SIZE, TMP_BASE_PATH
 from gitingest.exceptions import InvalidPatternError
 from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
-from gitingest.repository_clone import CloneConfig, _check_repo_exists, clone_repo, fetch_remote_branch_list
+from gitingest.repository_clone import _check_repo_exists, fetch_remote_branch_list
 
 HEX_DIGITS: set[str] = set(string.hexdigits)
 
@@ -113,23 +113,13 @@ async def parse_query(
         # Local path scenario
         parsed_query = _parse_path(source)
 
-    # Clone the repository if it's a URL
-    if parsed_query.url:
-        clone_config = CloneConfig(
-            url=parsed_query.url,
-            local_path=str(parsed_query.local_path),
-            commit=parsed_query.commit,
-            branch=parsed_query.branch,
-        )
-        await clone_repo(clone_config)
-
-        # Look for .gitingestignore file in the cloned repository
-        ignore_file_path = Path(parsed_query.local_path) / ".gitingestignore"
-        additional_ignore_patterns = parse_ignore_file(ignore_file_path)
-        if ignore_patterns:
-            ignore_patterns.update(additional_ignore_patterns)
-        else:
-            ignore_patterns = additional_ignore_patterns
+    # Look for .gitingestignore file in the local path
+    ignore_file_path = Path(parsed_query.local_path) / ".gitingestignore"
+    additional_ignore_patterns = parse_ignore_file(ignore_file_path)
+    if ignore_patterns:
+        ignore_patterns.update(additional_ignore_patterns)
+    else:
+        ignore_patterns = additional_ignore_patterns
 
     # Combine default ignore patterns + custom patterns
     ignore_patterns_set = DEFAULT_IGNORE_PATTERNS.copy()
diff --git a/src/server/query_processor.py b/src/server/query_processor.py
@@ -1,5 +1,6 @@
 """ Process a query by parsing input, cloning a repository, and generating a summary. """
 
+import os
 from functools import partial
 
 from fastapi import Request
@@ -46,43 +47,23 @@ async def process_query(
     _TemplateResponse
         Rendered template response containing the processed results or an error message.
 
-    Raises
-    ------
-    ValueError
-        If an invalid pattern type is provided.
     """
-    if pattern_type == "include":
-        include_patterns = pattern
-        exclude_patterns = None
-    elif pattern_type == "exclude":
-        exclude_patterns = pattern
-        include_patterns = None
-    else:
-        raise ValueError(f"Invalid pattern type: {pattern_type}")
-
+    include_patterns, exclude_patterns = validate_pattern_type(pattern_type, pattern)
     template = "index.jinja" if is_index else "git.jinja"
     template_response = partial(templates.TemplateResponse, name=template)
     max_file_size = log_slider_to_size(slider_position)
 
-    context = {
-        "request": request,
-        "repo_url": input_text,
-        "examples": EXAMPLE_REPOS if is_index else [],
-        "default_file_size": slider_position,
-        "pattern_type": pattern_type,
-        "pattern": pattern,
-    }
+    context = create_context(request, input_text, slider_position, pattern_type, pattern, is_index)
 
     try:
-        parsed_query: ParsedQuery = await parse_query(
+        parsed_query = await parse_query(
             source=input_text,
             max_file_size=max_file_size,
             from_web=True,
             include_patterns=include_patterns,
             ignore_patterns=exclude_patterns,
         )
-        if not parsed_query.url:
-            raise ValueError("The 'url' parameter is required.")
+        validate_parsed_query(parsed_query)
 
         clone_config = CloneConfig(
             url=parsed_query.url,
@@ -91,25 +72,18 @@ async def process_query(
             branch=parsed_query.branch,
         )
         await clone_repo(clone_config)
+
+        update_ignore_patterns(parsed_query, clone_config.local_path)
+
         summary, tree, content = run_ingest_query(parsed_query)
-        with open(f"{clone_config.local_path}.txt", "w", encoding="utf-8") as f:
-            f.write(tree + "\n" + content)
+        save_ingest_result(clone_config.local_path, tree, content)
+        content = filter_ignored_files(parsed_query, content)
+
     except Exception as e:
-        # hack to print error message when query is not defined
-        if "query" in locals() and parsed_query is not None and isinstance(parsed_query, dict):
-            _print_error(parsed_query["url"], e, max_file_size, pattern_type, pattern)
-        else:
-            print(f"{Colors.BROWN}WARN{Colors.END}: {Colors.RED}<-  {Colors.END}", end="")
-            print(f"{Colors.RED}{e}{Colors.END}")
-
-        context["error_message"] = f"Error: {e}"
+        handle_query_error(e, parsed_query, max_file_size, pattern_type, pattern, context)
         return template_response(context=context)
 
-    if len(content) > MAX_DISPLAY_SIZE:
-        content = (
-            f"(Files content cropped to {int(MAX_DISPLAY_SIZE / 1_000)}k characters, "
-            "download full ingest to see more)\n" + content[:MAX_DISPLAY_SIZE]
-        )
+    content = truncate_content(content)
 
     _print_success(
         url=parsed_query.url,
@@ -132,10 +106,207 @@ async def process_query(
     return template_response(context=context)
 
 
+def validate_pattern_type(pattern_type: str, pattern: str):
+    """
+    Ensure valid pattern type and return the corresponding include/exclude patterns.
+
+    Parameters
+    ----------
+    pattern_type : str
+        Specifies the type of pattern, either "include" or "exclude".
+    pattern : str
+        The pattern string to be included or excluded.
+
+    Returns
+    -------
+    tuple
+        A tuple containing either the include or exclude pattern.
+
+    Raises
+    ------
+    ValueError
+        If an invalid pattern type is provided.
+    """
+    if pattern_type == "include":
+        return pattern, None
+    if pattern_type == "exclude":
+        return None, pattern
+    raise ValueError(f"Invalid pattern type: {pattern_type}")
+
+
+def create_context(
+    request: Request, input_text: str, slider_position: int, pattern_type: str, pattern: str, is_index: bool
+) -> dict:
+    """
+    Prepare the context dictionary for rendering templates.
+
+    Parameters
+    ----------
+    request : Request
+        The HTTP request object.
+    input_text : str
+        The user-provided input text (Git repository URL or slug).
+    slider_position : int
+        The position of the slider, representing the maximum file size in the query.
+    pattern_type : str
+        Type of pattern to use, either "include" or "exclude".
+    pattern : str
+        The pattern string to include or exclude.
+    is_index : bool
+        Boolean flag indicating if the request is for the index page.
+
+    Returns
+    -------
+    dict
+        A dictionary containing template context data.
+    """
+    return {
+        "request": request,
+        "repo_url": input_text,
+        "examples": EXAMPLE_REPOS if is_index else [],
+        "default_file_size": slider_position,
+        "pattern_type": pattern_type,
+        "pattern": pattern,
+    }
+
+
+def validate_parsed_query(parsed_query: ParsedQuery):
+    """
+    Check if the parsed query contains a valid URL.
+
+    Parameters
+    ----------
+    parsed_query : ParsedQuery
+        The parsed query object containing repository information.
+
+    Raises
+    ------
+    ValueError
+        If the URL parameter is missing in the parsed query.
+    """
+    if not parsed_query.url:
+        raise ValueError("The 'url' parameter is required.")
+
+
+def update_ignore_patterns(parsed_query: ParsedQuery, local_path: str):
+    """
+    Load ignore patterns from `.gitingestignore` file if present.
+
+    Parameters
+    ----------
+    parsed_query : ParsedQuery
+        The parsed query object containing repository details.
+    local_path : str
+        The local path where the repository is cloned.
+    """
+    ignore_file_path = os.path.join(local_path, ".gitingestignore")
+    if os.path.exists(ignore_file_path):
+        with open(ignore_file_path, encoding="utf-8") as ignore_file:
+            additional_ignore_patterns = [
+                line.strip() for line in ignore_file if line.strip() and not line.startswith("#")
+            ]
+
+        if additional_ignore_patterns:
+            parsed_query.ignore_patterns = parsed_query.ignore_patterns or set()
+            parsed_query.ignore_patterns.update(additional_ignore_patterns)
+
+
+def save_ingest_result(local_path: str, tree: str, content: str):
+    """
+    Save the repository tree and file content to a text file.
+
+    Parameters
+    ----------
+    local_path : str
+        The local path where the repository is cloned.
+    tree : str
+        The repository tree structure.
+    content : str
+        The ingested file content.
+    """
+    with open(f"{local_path}.txt", "w", encoding="utf-8") as f:
+        f.write(tree + "\n" + content)
+
+
+def filter_ignored_files(parsed_query: ParsedQuery, content: str) -> str:
+    """
+    Remove ignored file patterns from content.
+
+    Parameters
+    ----------
+    parsed_query : ParsedQuery
+        The parsed query object containing ignore patterns.
+    content : str
+        The content to be filtered.
+
+    Returns
+    -------
+    str
+        The filtered content without ignored patterns.
+    """
+    if parsed_query.ignore_patterns:
+        content = "\n".join(
+            line
+            for line in content.splitlines()
+            if not any(ignored in line for ignored in parsed_query.ignore_patterns)
+        )
+    return content
+
+
+def handle_query_error(
+    e: Exception, parsed_query: ParsedQuery, max_file_size: int, pattern_type: str, pattern: str, context: dict
+):
+    """
+    Handle exceptions during query processing and log errors.
+
+    Parameters
+    ----------
+    e : Exception
+        The exception raised during processing.
+    parsed_query : ParsedQuery
+        The parsed query object.
+    max_file_size : int
+        The maximum file size allowed for the query, in bytes.
+    pattern_type : str
+        Specifies the type of pattern used.
+    pattern : str
+        The actual pattern string used.
+    context : dict
+        The template context dictionary.
+    """
+    if "query" in locals() and parsed_query is not None and isinstance(parsed_query, dict):
+        _print_error(parsed_query["url"], e, max_file_size, pattern_type, pattern)
+    else:
+        print(f"{Colors.BROWN}WARN{Colors.END}: {Colors.RED}<-  {Colors.END}{Colors.RED}{e}{Colors.END}")
+
+    context["error_message"] = f"Error: {e}"
+
+
+def truncate_content(content: str) -> str:
+    """
+    Truncate content if it exceeds the maximum display size.
+
+    Parameters
+    ----------
+    content : str
+        The content to be truncated.
+
+    Returns
+    -------
+    str
+        The truncated content, if applicable.
+    """
+    if len(content) > MAX_DISPLAY_SIZE:
+        content = (
+            f"(Files content cropped to {int(MAX_DISPLAY_SIZE / 1_000)}k characters, "
+            "download full ingest to see more)\n" + content[:MAX_DISPLAY_SIZE]
+        )
+    return content
+
+
 def _print_query(url: str, max_file_size: int, pattern_type: str, pattern: str) -> None:
     """
-    Print a formatted summary of the query details, including the URL, file size,
-    and pattern information, for easier debugging or logging.
+    Print a formatted summary of the query details.
 
     Parameters
     ----------
@@ -151,16 +322,16 @@ def _print_query(url: str, max_file_size: int, pattern_type: str, pattern: str)
     print(f"{Colors.WHITE}{url:<20}{Colors.END}", end="")
     if int(max_file_size / 1024) != 50:
         print(f" | {Colors.YELLOW}Size: {int(max_file_size/1024)}kb{Colors.END}", end="")
-    if pattern_type == "include" and pattern != "":
+    if pattern_type == "include" and pattern:
         print(f" | {Colors.YELLOW}Include {pattern}{Colors.END}", end="")
-    elif pattern_type == "exclude" and pattern != "":
+    elif pattern_type == "exclude" and pattern:
         print(f" | {Colors.YELLOW}Exclude {pattern}{Colors.END}", end="")
+    print()
 
 
 def _print_error(url: str, e: Exception, max_file_size: int, pattern_type: str, pattern: str) -> None:
     """
-    Print a formatted error message including the URL, file size, pattern details, and the exception encountered,
-    for debugging or logging purposes.
+    Print a formatted error message including details of the exception.
 
     Parameters
     ----------
@@ -182,8 +353,7 @@ def _print_error(url: str, e: Exception, max_file_size: int, pattern_type: str,
 
 def _print_success(url: str, max_file_size: int, pattern_type: str, pattern: str, summary: str) -> None:
     """
-    Print a formatted success message, including the URL, file size, pattern details, and a summary with estimated
-    tokens, for debugging or logging purposes.
+    Print a formatted success message, including estimated tokens.
 
     Parameters
     ----------
