Move illuminator to utils and support json as input

Signed-off-by: Alina Ryan <[email protected]>

Author: alinaryan

notebooks/illuminator/.gitignore …b-knowledge/utils/illuminator/.gitignorenotebooks/illuminator/.gitignore renamed to notebooks/instructlab-knowledge/utils/illuminator/.gitignore notebooks/illuminator/.gitignore renamed to notebooks/instructlab-knowledge/utils/illuminator/.gitignore

@@ -35,9 +35,20 @@ pip install -r requirements.txt
 ```
 
 ## 🚀 Usage
-### Check a Single PDF or an Entire Folder of PDFs
+### Supports PDF and JSON
+Illuminator works with:
+- Raw PDF files (will convert using Docling)
+- Docling-generated JSON files (post-conversion documents)
+
+### Analyse a Single File
 ```
-python illuminator.py -f /path/to/pdf/document.pdf
+python illuminator.py -f /path/to/document.pdf
+python illuminator.py -f /path/to/document.json
+```
+
+### Analyze All Files in a Folder
+```
+python illuminator.py -f /path/to/folder/
 ```
 
 ### Save Results to a JSON File
@@ -89,4 +100,3 @@ python illuminator.py -f /path/to/pdf/folder/ -o results.json
 
 ## 🤝 Acknowledgments
 Built by Alina with ❤️ for better PDF conversion workflows!
-

notebooks/illuminator/README.md …ab-knowledge/utils/illuminator/README.mdnotebooks/illuminator/README.md renamed to notebooks/instructlab-knowledge/utils/illuminator/README.md notebooks/illuminator/README.md renamed to notebooks/instructlab-knowledge/utils/illuminator/README.md

@@ -1,5 +1,6 @@
 from docling.document_converter import DocumentConverter
-from typing import List, Tuple, Dict, Any, Union
+from docling.datamodel.document import DoclingDocument
+from typing import List, Tuple, Dict, Any, Union, Set
 from log_utils import logger
 import os
 
@@ -40,37 +41,37 @@ def summarize_tables(doc) -> Tuple[int, List[int]]:
 
     return num_tables, pages
 
-
-def analyze_pdf_with_docling(file_path) -> Dict[str, Union[int, List[Any], set]]:
+def convert_pdf_with_docling(file_path: str) -> DoclingDocument:
     """
-    Analyzes a PDF for merged table cells using the Docling converter,
-    and saves a Markdown version of the converted document.
+    Converts a PDF using Docling and saves a Markdown version of the document.
 
     Args:
         file_path: Path to the input PDF file.
 
     Returns:
-        A dictionary containing:
-            - Total number of tables.
-            - Set of pages with merged cells.
-            - List of merged cell details (page, position, spans, text).
-            - Total unique pages with tables.
+        The converted Docling Document object.
     """
     converter = DocumentConverter()
     result = converter.convert(file_path)
     doc = result.document
 
-    # ✅ Save Markdown output
+    # Save Markdown output
     markdown_text = doc.export_to_markdown()
     base_name = os.path.splitext(os.path.basename(file_path))[0]
     md_output_path = f"{base_name}.md"
-    with open(md_output_path, "w") as f:
+    with open(md_output_path, "w", encoding="utf-8") as f:
         f.write(markdown_text)
 
     logger.info(f"📝 Markdown saved to {md_output_path}")
+    return doc
+
+def analyze_docling_tables(doc_input: DoclingDocument) -> Dict[str, Union[int, List[dict], List[int], str]]:
+    """
+    Analyzes a Docling document (object or path to PDF/JSON file) for merged table cells.
+    """
+    doc_input
 
-    # ⬇️ Proceed with table analysis
-    table_count, table_pages_list = summarize_tables(doc)
+    table_count, table_pages_list = summarize_tables(doc_input)
     total_pages = len(set(table_pages_list)) or "Unknown"
 
     issues = {
@@ -80,7 +81,7 @@ def analyze_pdf_with_docling(file_path) -> Dict[str, Union[int, List[Any], set]]
         "page_count": total_pages
     }
 
-    for i, table_item in enumerate(doc.tables):
+    for i, table_item in enumerate(doc_input.tables):
         try:
             page_number = table_pages_list[i]
         except IndexError:
@@ -103,4 +104,4 @@ def analyze_pdf_with_docling(file_path) -> Dict[str, Union[int, List[Any], set]]
                         issues["merged_cell_pages"].add(page_number)
 
     issues["merged_cell_pages"] = sorted(issues["merged_cell_pages"])
-    return issues
+    return issues

notebooks/illuminator/analysis.py …-knowledge/utils/illuminator/analysis.pynotebooks/illuminator/analysis.py renamed to notebooks/instructlab-knowledge/utils/illuminator/analysis.py notebooks/illuminator/analysis.py renamed to notebooks/instructlab-knowledge/utils/illuminator/analysis.py

@@ -1,7 +1,7 @@
 # main.py
 import argparse
-from utils import get_pdf_files, save_results, generate_summary
-from analysis import analyze_pdf_with_docling
+from utils import get_supported_files, save_results, generate_summary
+from analysis import convert_pdf_with_docling, analyze_docling_tables
 from log_utils import logger
 
 def parse_args() -> argparse.Namespace:
@@ -27,25 +27,26 @@ def parse_args() -> argparse.Namespace:
     )
     return parser.parse_args()
 
-
 def main() -> None:
     """
     Main execution flow:
     - Parses arguments
-    - Loads and analyzes PDFs
+    - Loads and analyzes PDFs or JSONs
     - Generates and saves results
     """
     args = parse_args()
-    pdfs = get_pdf_files(args.file)
-    if not pdfs:
-        logger.error("❌ No PDFs found to process.")
+    files = get_supported_files(args.file)
+    if not files:
+        logger.error("❌ No input files found to process.")
         return
 
     all_results = {}
-    for path in pdfs:
+    for path in files:
         logger.info(f"\n🔍 Converting and analyzing: {path}\n")
         try:
-            result = analyze_pdf_with_docling(path)
+            # Use Docling to convert (PDF path)
+            doc = convert_pdf_with_docling(path)
+            result = analyze_docling_tables(doc)
             all_results[path] = result
         except Exception as e:
             logger.error(f"❌ Failed to process {path}: {e}")
@@ -54,4 +55,4 @@ def main() -> None:
     save_results(all_results, args.output)
 
 if __name__ == "__main__":
-    main()
+    main()

notebooks/illuminator/illuminator.py …owledge/utils/illuminator/illuminator.pynotebooks/illuminator/illuminator.py renamed to notebooks/instructlab-knowledge/utils/illuminator/illuminator.py notebooks/illuminator/illuminator.py renamed to notebooks/instructlab-knowledge/utils/illuminator/illuminator.py

@@ -6,24 +6,36 @@
 from log_utils import logger
 
 MAX_PREVIEW_LENGTH = 30  # Max characters shown from cell text in summary
+SUPPORTED_FILE_EXTENSIONS = [".pdf", ".json"]
 
-def get_pdf_files(path: str) -> List[str]:
+def get_supported_files(path: str) -> List[str]:
     """
-    Returns a list of PDF file paths from a given file or directory.
+    Returns a list of file paths from the given path that match supported extensions.
+    """
+    return [
+        os.path.join(path, f)
+        for f in os.listdir(path)
+        if os.path.isfile(os.path.join(path, f)) and any(f.endswith(ext) for ext in SUPPORTED_FILE_EXTENSIONS)
+    ]
+
+def get_supported_files(path: str, extensions: List[str] = [".pdf", ".json"]) -> List[str]:
+    """
+    Returns a list containing one or more files that are in SUPPORTED_FILE_EXTENSIONS
 
     Args:
-        path: Path to a single .pdf file or directory containing .pdf files.
+        path: Path to a single file or directory.
+        extensions: List of file extensions to include.
 
     Returns:
-        List of .pdf file paths.
+        List of matching file paths.
     """
-    if os.path.isfile(path) and path.endswith(".pdf"):
+    if os.path.isfile(path) and any(path.endswith(ext) for ext in extensions):
         return [path]
     elif os.path.isdir(path):
         return [
             os.path.join(path, f)
             for f in os.listdir(path)
-            if f.endswith(".pdf")
+            if any(f.endswith(ext) for ext in extensions)
         ]
     return []
 
@@ -114,4 +126,4 @@ def format_pages(pages) -> str:
         ranges.append(f"{start}")
     else:
         ranges.append(f"{start}-{pages[-1]}")
-    return ", ".join(ranges)
+    return ", ".join(ranges)