Merge pull request #2155 from DerekMelchin/bug-python-api-ref-generator

Add Python API Reference Generator

Author: DerekMelchin

.github/workflows/update_python_api_reference.yml

@@ -0,0 +1,38 @@
+name: Update Python API Reference
+
+on:
+  schedule:
+    - cron: "0 23 * * 5" # Runs at 23:00 UTC every Friday
+  workflow_dispatch:  # Run on manual trigger
+
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Free space
+        run: df -h && rm -rf /opt/hostedtoolcache* && df -h
+
+      - name: Install dependencies (pip)
+        run: |
+          python -m pip install --upgrade pip
+          pip install quantconnect-stubs mkdocs
+
+      - name: Run the Python API Reference Generator
+        run: cd code-generators/python_api_reference && python main.py
+
+      - name: Build site with MkDocs
+        run: mkdocs build --clean
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_KEY }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET }}
+          aws-region: us-west-1
+
+      - name: Sync site to S3
+        run: aws s3 sync ./site s3://${{ secrets.AWS_BUCKET }}/python-api-reference --delete

code-generators/python_api_reference/ast_utils.py

@@ -0,0 +1,68 @@
+import ast
+from typing import List, Dict, Tuple
+
+def is_overload_decorator(dec: ast.expr) -> bool:
+    return (
+        (isinstance(dec, ast.Name) and dec.id == "overload") or 
+        (isinstance(dec, ast.Attribute) and dec.attr == "overload")
+    )
+
+def func_has_overload_decorator(fn: ast.FunctionDef) -> bool:
+    return any(is_overload_decorator(d) for d in fn.decorator_list)
+
+def walk_classes(node: ast.AST, qual_prefix: str = "") -> List[Tuple[str, ast.ClassDef]]:
+    """Walk an AST node and collect all class definitions with their 
+    qualified names.
+
+    Args:
+        node: The AST node to traverse (a Module or ClassDef).
+        qual_prefix: The qualified name prefix for nested classes 
+                     (internal use)
+
+    Returns:
+        List of tuples containing (qualified_name, class_definition) 
+        pairs.
+
+    Example:
+        For code like:
+        ```
+            class Outer:
+                class Inner:
+                    pass
+        ```
+        Returns: [("Outer", <ClassDef>), ("Outer.Inner", <ClassDef>)]
+    """
+    out: List[Tuple[str, ast.ClassDef]] = []
+    # Iterate through each class in the file.
+    for child in getattr(node, "body", []):
+        if isinstance(child, ast.ClassDef):
+            qualified_name = f"{qual_prefix}.{child.name}" if qual_prefix else child.name
+            # Add this class to the list of results.
+            out.append((qualified_name, child))
+            # Recurse, incase the class contains other class defintiions.
+            out.extend(walk_classes(child, qualified_name))
+    return out
+
+def group_methods_by_name(def_: ast.ClassDef) -> Dict[str, List[ast.FunctionDef]]:
+    """Create a dictionary where keys are method names and values are 
+    lists of function definitions with that name.
+    """
+    defs_by_method_name: Dict[str, List[ast.FunctionDef]] = {}
+    # Iterate through each method definition in the class body.
+    for node in def_.body:
+        if isinstance(node, ast.FunctionDef):
+            # Append the method defintiion to the list of definitions for
+            # this method name.
+            defs_by_method_name.setdefault(node.name, []).append(node)
+    return defs_by_method_name
+
+def format_bases(class_def: ast.ClassDef) -> str:
+    # If the class doesn't inherit from a base class, return an empty 
+    # string.
+    if not class_def.bases:
+        return ""
+    bases = [ast.unparse(node).strip() for node in class_def.bases]
+    return f"({', '.join(bases)})"
+
+def _indent(level: int) -> str:
+    return "    " * level

code-generators/python_api_reference/docstring.py

@@ -0,0 +1,134 @@
+import ast
+import re
+from typing import List, Tuple
+from collections import OrderedDict
+
+# Define some regex to help with parsing the docstrings.
+_PARAM_LINE_RE = re.compile(r"^\s*:param\s+([A-Za-z_]\w*)\s*:\s*(.*)$")
+_RET_LINE_RE = re.compile(r"^\s*:returns?\s*:\s*(.*)$")
+_STOP_BLOCK_RE = re.compile(r"^\s*:(param|type|returns?|rtype)\b")
+
+
+class Docstring:
+
+    def create_from_overloads(
+            self, overload_fns: List[ast.FunctionDef], class_name: str, 
+            method_name: str) -> str:
+        intro_blocks: List[List[str]] = []
+        seen_intro_keys = set()
+        params_union: "OrderedDict[str, str]" = OrderedDict()
+        returns_set: List[str] = []
+
+        # Iterate through each overload.
+        for fn in overload_fns:
+            # Get the docstring of the overload.
+            doc = (ast.get_docstring(fn) or "").strip()
+            # If there is no docstring in the overload, just continue.
+            if not doc:
+                continue
+            # Decompose the docstring into its 3 parts.
+            intro, param_lines, return_lines = self._split_doc(doc)
+            # If there is an introduction in the docstring...
+            if intro:
+                # Make the introduction look nice.
+                key = self._normalize_block(intro)
+                # Only save unique introduction text blocks.
+                if key and key not in seen_intro_keys:
+                    seen_intro_keys.add(key)
+                    intro_blocks.append(intro)
+            # Iterate through each parameter description in the 
+            # docstring.
+            for pl in param_lines:
+                m = _PARAM_LINE_RE.match(pl)
+                if not m:
+                    continue
+                # Get the parameter name and description.
+                name, desc = m.group(1), m.group(2).strip()
+                if name not in params_union:
+                    params_union[name] = desc
+                elif not params_union[name] and desc:
+                    params_union[name] = desc
+            # Iterate through each return description in the docstring.
+            for rl in return_lines:
+                m = _RET_LINE_RE.match(rl)
+                if not m:
+                    continue
+                desc = m.group(1).strip()
+                if desc and desc not in returns_set:
+                    returns_set.append(desc)
+        # Create an empty list to store the lines we'll write.
+        out_lines: List[str] = []
+        # If there was only one unique description we found, use it.
+        if len(intro_blocks) == 1:
+            out_lines.extend(intro_blocks[0])
+        # Otherwise, just list all of the unique descriptions.
+        elif len(intro_blocks) > 1:
+            out_lines.append("Signature descriptions:")
+            out_lines.append("")
+            for i, block in enumerate(intro_blocks):
+                first = True
+                for ln in block:
+                    if first:
+                        out_lines.append(f"- {ln}")
+                        first = False
+                    else:
+                        out_lines.append(f"  {ln}")
+                if i < len(intro_blocks) - 1:
+                    out_lines.append("")
+
+        # Write the parameter descriptions.
+        if params_union:
+            if out_lines:
+                out_lines.append("")
+            for name, desc in params_union.items():
+                out_lines.append(f":param {name}: {desc}")
+
+        # Write the return description.
+        if returns_set:
+            # If there was only one, use it.
+            if len(returns_set) == 1:
+                if out_lines:
+                    out_lines.append("")
+                out_lines.append(f":returns: {returns_set[0]}")
+            # Otherwise, list out all the cases.
+            else:
+                combined = "; ".join(f"Case {i+1}: [{desc}]" for i, desc in enumerate(returns_set))
+                if out_lines:
+                    out_lines.append("")
+                out_lines.append(f":returns: Depends on the signature used. {combined}")
+
+        return "\n".join(out_lines).rstrip()
+
+    def _split_doc(self, doc: str) -> Tuple[List[str], List[str], List[str]]:
+        # Split the docstring into lines.
+        lines = [ln.rstrip() for ln in doc.splitlines()]
+        # Create empty lists to store the intro, param descriptions, and
+        # return description.
+        intro: List[str] = []
+        param_lines: List[str] = []
+        return_lines: List[str] = []
+        in_intro = True
+        # Iterate through each line in the docstring.
+        for line in lines:
+            # Collect the intro description in the docstring.
+            if in_intro:
+                if not line.strip():
+                    in_intro = False
+                    continue
+                if _STOP_BLOCK_RE.match(line):
+                    in_intro = False
+                else:
+                    intro.append(line)
+                    continue
+            # Collect the :param: lines in the doctring.
+            if _PARAM_LINE_RE.match(line):
+                param_lines.append(line.strip())
+                continue
+            # Collect the :return: line in the docstring.
+            if _RET_LINE_RE.match(line):
+                return_lines.append(line.strip())
+        return intro, param_lines, return_lines
+
+    def _normalize_block(self, lines: List[str]) -> str:
+        text = "\n".join(lines).strip()
+        return "\n".join(" ".join(l.split()) for l in text.splitlines()).strip()