LCORE-85: Enable links to referenced BYOK documents

Author: syedriko

byok/generate_embeddings_tool.py

@@ -7,6 +7,7 @@
 from typing import Callable, Dict
 
 import faiss
+import frontmatter
 import requests
 from llama_index.core import Settings, SimpleDirectoryReader, VectorStoreIndex
 from llama_index.core.llms.utils import resolve_llm
@@ -17,25 +18,27 @@
 from llama_index.readers.file.flat.base import FlatReader
 from llama_index.vector_stores.faiss import FaissVectorStore
 
-def get_file_title(file_path: str) -> str:
-    """Extract title from the plaintext doc file."""
-    title = ""
-    try:
-        with open(file_path, "r") as file:
-            title = file.readline().rstrip("\n").lstrip("# ")
-    except Exception:  # noqa: S110
-        pass
-    return title
-
-
 def file_metadata_func(file_path: str) -> Dict:
     """Populate the docs_url and title metadata elements with docs URL and the page's title.
 
     Args:
         file_path: str: file path in str
     """
+    title = file_path
     docs_url = file_path
-    title = get_file_title(file_path)
+    try:
+        with open(file_path, "r") as file:
+            first_line = file.readline()
+            if first_line.startswith("#"):
+                title = first_line.rstrip("\n").lstrip("# ")
+                docs_url = file_path
+            elif first_line.startswith("---"):
+                file.close()
+                post = frontmatter.load(file_path)
+                title = post['title']
+                docs_url = post['url']
+    except Exception:  # noqa: S110
+        pass
     msg = f"file_path: {file_path}, title: {title}, docs_url: {docs_url}"
     print(msg)
     return {"file_path": file_path, "title": title, "docs_url": docs_url}

byok/registries.conf

@@ -53,6 +53,7 @@ dependencies = [
     "llama-index-embeddings-huggingface",
     "llama-index-readers-file",
     "faiss-cpu",
+    "python-frontmatter",
     "aiohttp>=3.8.0",
     "beautifulsoup4>=4.10.0",
 ]

pdm.lock.cpu

@@ -158,9 +158,9 @@ httpcore==1.0.9 \
 httpx==0.28.1 \
     --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
     --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
-huggingface-hub[inference]==0.31.1 \
-    --hash=sha256:43f73124819b48b42d140cbc0d7a2e6bd15b2853b1b9d728d4d55ad1750cac5b \
-    --hash=sha256:492bb5f545337aa9e2f59b75ef4c5f535a371e8958a6ce90af056387e67f1180
+huggingface-hub[inference]==0.31.2 \
+    --hash=sha256:7053561376ed7f6ffdaecf09cc54d70dc784ac6315fa4bb9b93e19662b029675 \
+    --hash=sha256:8138cd52aa2326b4429bb00a4a1ba8538346b7b8a808cdce30acb6f1f1bdaeec
 idna==3.10 \
     --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
     --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
@@ -306,9 +306,9 @@ numpy==2.2.5 \
     --hash=sha256:b13f04968b46ad705f7c8a80122a42ae8f620536ea38cf4bdd374302926424dd \
     --hash=sha256:c42365005c7a6c42436a54d28c43fe0e01ca11eb2ac3cefe796c25a5f98e5e9b \
     --hash=sha256:f5045039100ed58fa817a6227a356240ea1b9a1bc141018864c306c1a16d4175
-openai==1.78.0 \
-    --hash=sha256:1ade6a48cd323ad8a7715e7e1669bb97a17e1a5b8a916644261aaef4bf284778 \
-    --hash=sha256:254aef4980688468e96cbddb1f348ed01d274d02c64c6c69b0334bf001fb62b3
+openai==1.78.1 \
+    --hash=sha256:7368bf147ca499804cc408fe68cdb6866a060f38dec961bbc97b04f9d917907e \
+    --hash=sha256:8b26b364531b100df1b961d03560042e5f5be11301d7d49a6cd1a2b9af824dca
 packaging==25.0 \
     --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \
     --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f
@@ -412,6 +412,9 @@ python-dateutil==2.9.0.post0 \
 python-dotenv==1.1.0 \
     --hash=sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5 \
     --hash=sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d
+python-frontmatter==1.1.0 \
+    --hash=sha256:335465556358d9d0e6c98bbeb69b1c969f2a4a21360587b9873bfc3b213407c1 \
+    --hash=sha256:7118d2bd56af9149625745c58c9b51fb67e8d1294a0c76796dafdc72c36e5f6d
 pytz==2025.2 \
     --hash=sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3 \
     --hash=sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00

pdm.lock.gpu

@@ -158,9 +158,9 @@ httpcore==1.0.9 \
 httpx==0.28.1 \
     --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \
     --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad
-huggingface-hub[inference]==0.31.1 \
-    --hash=sha256:43f73124819b48b42d140cbc0d7a2e6bd15b2853b1b9d728d4d55ad1750cac5b \
-    --hash=sha256:492bb5f545337aa9e2f59b75ef4c5f535a371e8958a6ce90af056387e67f1180
+huggingface-hub[inference]==0.31.2 \
+    --hash=sha256:7053561376ed7f6ffdaecf09cc54d70dc784ac6315fa4bb9b93e19662b029675 \
+    --hash=sha256:8138cd52aa2326b4429bb00a4a1ba8538346b7b8a808cdce30acb6f1f1bdaeec
 idna==3.10 \
     --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \
     --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3
@@ -355,9 +355,9 @@ nvidia-nvtx-cu12==12.4.127; platform_system == "Linux" and platform_machine == "
     --hash=sha256:641dccaaa1139f3ffb0d3164b4b84f9d253397e38246a4f2f36728b48566d485 \
     --hash=sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a \
     --hash=sha256:7959ad635db13edf4fc65c06a6e9f9e55fc2f92596db928d169c0bb031e88ef3
-openai==1.78.0 \
-    --hash=sha256:1ade6a48cd323ad8a7715e7e1669bb97a17e1a5b8a916644261aaef4bf284778 \
-    --hash=sha256:254aef4980688468e96cbddb1f348ed01d274d02c64c6c69b0334bf001fb62b3
+openai==1.78.1 \
+    --hash=sha256:7368bf147ca499804cc408fe68cdb6866a060f38dec961bbc97b04f9d917907e \
+    --hash=sha256:8b26b364531b100df1b961d03560042e5f5be11301d7d49a6cd1a2b9af824dca
 packaging==25.0 \
     --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \
     --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f
@@ -461,6 +461,9 @@ python-dateutil==2.9.0.post0 \
 python-dotenv==1.1.0 \
     --hash=sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5 \
     --hash=sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d
+python-frontmatter==1.1.0 \
+    --hash=sha256:335465556358d9d0e6c98bbeb69b1c969f2a4a21360587b9873bfc3b213407c1 \
+    --hash=sha256:7118d2bd56af9149625745c58c9b51fb67e8d1294a0c76796dafdc72c36e5f6d
 pytz==2025.2 \
     --hash=sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3 \
     --hash=sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00