embed aiken docs to db; make github service flexible; assign alternate chunk title for non-title

Author: smutyala1at

apps/meshjs-rag/.gitignore

@@ -2,4 +2,5 @@
 .vscode/
 .venv/
 **/__pycache__/
-docs/
+docs/
+aiken-docs/

apps/meshjs-rag/app/api/v1/ingest.py

@@ -28,7 +28,7 @@ async def ingest_docs(credentials: HTTPAuthorizationCredentials = Depends(securi
       detail="You are not authorized"
     )
 
-  github = GithubService()
+  github = GithubService(owner="MeshJS", repo="mimir", doc_path="apps/docs/content/docs", output_path="docs")
   await github.download_docs()
 
   docs_dir = pathlib.Path(__file__).resolve().parents[3] / "docs"
@@ -106,4 +106,52 @@ async def ingest_packages(credentials: HTTPAuthorizationCredentials = Depends(se
 
   return {
     "message": "Ingestion process successful for package docs"
+  }
+
+
+@router.post("/aiken-docs")
+async def ingest_packages(credentials: HTTPAuthorizationCredentials = Depends(security), supabase: AsyncClient = Depends(get_db_client)):
+
+  token = credentials.credentials
+  if not token or token != os.getenv("ADMIN_KEY"):
+    raise HTTPException(
+      status_code=status.HTTP_401_UNAUTHORIZED,
+      detail="You're not authorized"
+    )
+  
+  github = GithubService(owner="aiken-lang", repo="site", doc_path="src/pages", output_path="aiken-docs")
+  await github.download_docs()
+  
+  aiken_docs_md_path = pathlib.Path(__file__).resolve().parents[3] / "aiken-docs"
+
+  try:
+    file_paths = get_docs_file_paths(aiken_docs_md_path)
+  except FileNotFoundError as e:
+    raise HTTPException(
+      status_code=status.HTTP_404_NOT_FOUND,
+      detail=f"The documents directory was not found: {e}"
+    )
+  except IOError as e:
+    raise HTTPException(
+      status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+      detail=f"An I/O error occurred while accessing the documents directory: {e}"
+    )
+
+  for relative_path in file_paths:
+    abs_path = aiken_docs_md_path / relative_path
+    try:
+      file_content = get_file_content(abs_path)
+      await process_docs_file_and_update_db(file_content, relative_path, supabase)
+    except (FileNotFoundError, IOError) as e:
+      print(f"Skipping file due to error: {e}")
+      continue
+
+    except Exception as e:
+      raise HTTPException(
+        status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        detail=f"An error occured during the file ingestion: {e}"
+      )
+
+  return {
+    "message": "Ingestion process successfully completed"
   }

apps/meshjs-rag/app/services/github.py

@@ -4,12 +4,12 @@
 import pathlib
 
 class GithubService:
-  def __init__(self):
+  def __init__(self, owner, repo, doc_path, output_path):
     self.base_url="https://api.github.com"
-    self.owner="smutyala1at"
-    self.repo="meshjs-docs"
-    self.doc_path="content/docs"
-    self.output_path="docs"
+    self.owner=owner
+    self.repo=repo
+    self.doc_path=doc_path
+    self.output_path=output_path
     self.token=os.getenv("GITHUB_TOKEN") or None
 
   def _get_headers(self):
@@ -28,7 +28,7 @@ async def _fetch_github_dir(self,client: httpx.AsyncClient, remote_path: str):
       response = await client.get(url)
       response.raise_for_status()
       return response.json()
-    except httpx.HTTPStatusException as e:
+    except httpx.HTTPError as e:
       print(f"Error fetching the directory '{remote_path}': HTTP status {e.response.status_code}")
       return None
     except httpx.RequestError as e:
@@ -81,6 +81,6 @@ async def download_docs(self):
 
 
 if __name__ == "__main__":
-  github = GithubService()
+  github = GithubService(owner="MeshJS", repo="mimir", doc_path="apps/docs/content/docs", output_path="docs")
   asyncio.run(github.download_docs())
   print("Successfully downloaded docs from github")

apps/meshjs-rag/app/utils/chunk_content.py

@@ -9,7 +9,7 @@ def chunk_content_by_h2(content: str) -> List[str]:
     chunks = []
     current_chunk = []
     for line in content.splitlines():
-      if line.startswith("## ") and line.strip()[-6:] != "[!toc]":
+      if line and line.startswith("## ") and not line.strip().endswith("[!toc]"):
         flush_chunk(current_chunk, chunks)
         current_chunk = [line]
       else:

apps/meshjs-rag/app/utils/process_chunks.py

@@ -30,7 +30,9 @@ async def process_chunks_and_update_db(
     chunk_title = title_extractor(chunk, idx, chunks)
 
     if not chunk_title:
-      continue
+      filepath = relative_path.split("/")
+      chunk_title = f"{filepath[-2]}/{filepath[-1]}_1" if len(filepath) > 1 else f"{relative_path}_1"
+
     current_chunk_data[chunk_title] = {
       "chunk": chunk,
       "chunk_id": idx,
@@ -65,9 +67,6 @@ async def process_chunks_and_update_db(
     existing = existing_records.get(key)
 
     if current and existing:
-      if current["checksum"] == existing["checksum"]:
-        print(f"Skipping the unchanged chunk: {chunk_title}")
-
       if current["checksum"] != existing["checksum"]:
         print(f"Updating chunk: {chunk_title}")
         try:
@@ -91,13 +90,16 @@ async def process_chunks_and_update_db(
 
       elif current["chunk_id"] != existing.get("chunk_id"):
         print(f"Updating chunk order for {chunk_title}")
-        updated_title = chunk_title.split("_")[0] + f"_{current["chunk_id"]}" if "_" in chunk_title else chunk_title
+        updated_title = chunk_title.split("_")[0] + f"_{current['chunk_id']}" if "_" in chunk_title else chunk_title
         await safe_db_operation(
           supabase.table("docs").update({"chunk_id": current["chunk_id"], "chunk_title": updated_title}).eq("id", existing["id"]).execute()
         )
         # reorder after update
         needs_reorder = True
 
+      else:
+        print(f"Skipping the unchanged chunk: {chunk_title}")
+
     elif current and not existing:
       print(f"New chunk {chunk_title}")
       try:

apps/meshjs-rag/app/utils/reorder_chunks.py

@@ -17,7 +17,7 @@ async def reorder_chunks(
             if chunk["chunk_id"] != new_idx:
                 updated_title = chunk["chunk_title"].split("_")[0] + f"_{new_idx}" if "_" in chunk["chunk_title"] else chunk["chunk_title"]
                 reorder_chunks.append(
-                    await safe_db_operation(
+                    safe_db_operation(
                     supabase.table("docs") \
                     .update({"chunk_id": new_idx, "chunk_title": updated_title}) \
                     .eq("id", chunk["id"]) \