Generalize product docs fetching from OpenShift to any product(s)

Author: max-svistunov

scripts/html_embeddings/README.md

@@ -28,17 +28,47 @@ Standard:
 ```bash
 # Generate embeddings for OpenShift 4.18
 python scripts/html_embeddings/generate_embeddings.py \
-  --version 4.18 \
+  --doc-url-slug "openshift_container_platform" \
+  --doc-url-version "4.18" \
   --output-dir ./vector_db \
   --model-dir ./embeddings_model
 ```
 
+Specify a full custom URL:
+
+```bash
+# Generate embeddings for RHEL 10
+python scripts/html_embeddings/generate_embeddings.py \
+  --doc-url "https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/10" \
+  --output-dir ./vector_db \
+  --model-dir ./embeddings_model
+```
+
+Use a configuration file:
+
+```bash
+# Generate embeddings for multiple products
+python scripts/html_embeddings/generate_embeddings.py \
+  --config-file ./docs_config.yaml
+```
+
+Example `docs_config.yaml`:
+```yaml
+products:
+  - slug: "openshift_container_platform"
+    version: "4.18"
+  - slug: "red_hat_enterprise_linux"
+    version: "9"
+  - url: "https://access.redhat.com/documentation/en-us/red_hat_ansible_automation_platform/2.4/html-single/red_hat_ansible_automation_platform_installation_guide"
+```
+
 Specify custom index name instead of the auto-generated one:
 
 ```bash
 # Generate embeddings for OpenShift 4.18
 python scripts/html_embeddings/generate_embeddings.py \
-  --version 4.18 \
+  --doc-url-slug "openshift_container_platform" \
+  --doc-url-version "4.18" \
   --output-dir ./vector_db \
   --index ocp-4.18 \
   --model-dir ./embeddings_model
@@ -47,9 +77,10 @@ python scripts/html_embeddings/generate_embeddings.py \
 Process only specific document and skip runbooks (good for quick testing):
 
 ```bash
-# Process only monitoring documentation
+# Process only observability documentation
 python scripts/html_embeddings/generate_embeddings.py \
-  --version 4.18 \
+  --doc-url-slug "openshift_container_platform" \
+  --doc-url-version "4.18" \
   --specific-doc observability_overview \
   --output-dir ./vector_db \
   --model-dir ./embeddings_model \
@@ -61,7 +92,8 @@ Use cached downloads:
 ```bash
 # Use previously downloaded files
 python scripts/html_embeddings/generate_embeddings.py \
-  --version 4.18 \
+  --doc-url-slug "openshift_container_platform" \
+  --doc-url-version "4.18" \
   --use-cached-downloads \
   --output-dir ./vector_db \
   --model-dir ./embeddings_model
@@ -72,7 +104,8 @@ Set a custom token limit (default is the same 380 as in Markdown-based chunking)
 ```bash
 # Set the token limit
 python generate_embeddings.py \
-  --version 4.18 \
+  --doc-url-slug "openshift_container_platform" \
+  --doc-url-version "4.18" \
   --chunk 380 \
   --output-dir ./vector_db \
   --model-dir ./embeddings_model
@@ -82,7 +115,10 @@ python generate_embeddings.py \
 
 ### Main arguments
 
-- `--version` - OpenShift version (required, e.g., "4.18")
+- `--doc-url`: The full URL to the documentation's html-single page.
+- `--doc-url-slug`: The product's documentation slug (e.g., 'openshift_container_platform').
+- `--doc-url-version`: The product version. Defaults to 'latest'. Used with --doc-url-slug.
+- `--config-file`: Path to a YAML or JSON configuration file specifying products to process.
 - `--index` - Index name (optional, e.g., "ocp-4.18")
 - `--output-dir` - Vector DB output directory (default: "./vector_db")
 - `--model-dir` - Embedding model directory (default: "./embeddings_model")

scripts/html_embeddings/chunk_html.py

@@ -20,6 +20,9 @@
 def chunk_html_documents(
     input_dir: Path,
     output_dir: Path,
+    product_slug: str,
+    product_version: str,
+    doc_url: Optional[str] = None,
     max_token_limit: int = 380,
     count_tag_tokens: bool = True,
     keep_siblings_together: bool = True,
@@ -31,6 +34,9 @@ def chunk_html_documents(
     Args:
         input_dir: Directory containing stripped HTML files
         output_dir: Directory to save chunked content
+        product_slug: Product URL slug
+        product_version: Product version
+        doc_url: The full URL to the documentation.
         max_token_limit: Maximum tokens per chunk
         count_tag_tokens: Whether to count HTML tags in token count
         keep_siblings_together: Keep sibling sections together when possible
@@ -75,22 +81,24 @@ def chunk_html_documents(
             logger.debug("Processing %s", html_file)
 
             # The doc name is the parent directory of the html file.
-            # The version is the parent of that directory.
             doc_name = html_file.parent.name
-            version = html_file.parent.parent.name
 
             # The main output_dir is the version dir, e.g., '.../chunks/4.18'.
             # We create the doc-specific subdirectory here.
             doc_specific_output_dir = output_dir / doc_name
 
             # Construct the source URL, which will be passed to the chunker.
-            source_url = f"https://docs.redhat.com/en/documentation/openshift_container_platform/{version}/html-single/{doc_name}/"
+            if doc_url:
+                source_url = doc_url
+            else:
+                source_url = f"https://docs.redhat.com/en/documentation/{product_slug}/{product_version}/html-single/{doc_name}/"
 
             success, chunk_count = chunk_single_html_file(
                 input_file=html_file.resolve(),
-                output_dir=doc_specific_output_dir, # Pass the new doc-specific dir
-                input_base_dir=base_dir_for_relative_paths.resolve(), # Pass the consistent version-level base path
+                output_dir=doc_specific_output_dir,
+                input_base_dir=base_dir_for_relative_paths.resolve(),
                 source_url=source_url,
+                product_slug=product_slug,
                 max_token_limit=max_token_limit,
                 count_tag_tokens=count_tag_tokens,
                 keep_siblings_together=keep_siblings_together,
@@ -133,7 +141,8 @@ def chunk_single_html_file(
     input_file: Path,
     output_dir: Path,
     input_base_dir: Path,
-    source_url: str, # Add source_url parameter
+    source_url: str,
+    product_slug: str,
     max_token_limit: int = 380,
     count_tag_tokens: bool = True,
     keep_siblings_together: bool = True,
@@ -147,6 +156,7 @@ def chunk_single_html_file(
         output_dir: Directory to save chunks
         input_base_dir: Base directory for input files (for relative path calculation)
         source_url: The public URL of the source document
+        product_slug: Product URL slug
         max_token_limit: Maximum tokens per chunk
         count_tag_tokens: Whether to count HTML tags
         keep_siblings_together: Keep sibling sections together
@@ -179,7 +189,7 @@ def chunk_single_html_file(
             return True, 0
 
         relative_path = input_file.relative_to(input_base_dir)
-        base_metadata = extract_metadata_from_path(relative_path)
+        base_metadata = extract_metadata_from_path(relative_path, product_slug)
 
         chunk_count = 0
         for i, chunk_obj in enumerate(chunks):
@@ -221,12 +231,13 @@ def chunk_single_html_file(
         return False, 0
 
 
-def extract_metadata_from_path(file_path: Path) -> Dict[str, Any]:
+def extract_metadata_from_path(file_path: Path, product_slug: str) -> Dict[str, Any]:
     """
     Extract metadata from file path.
 
     Args:
         file_path: Relative path to the file
+        product_slug: Product URL slug
 
     Returns:
         Dictionary with extracted metadata
@@ -253,7 +264,7 @@ def extract_metadata_from_path(file_path: Path) -> Dict[str, Any]:
         "doc_id": doc_id,
         "version": version,
         "file_path": str(file_path),
-        "doc_type": "openshift_documentation",
+        "doc_type": f"{product_slug}_documentation",
     }
 
 

scripts/html_embeddings/download_docs.py

@@ -21,6 +21,8 @@
 
 def download_documentation(
     version: str,
+    product_slug: str,
+    doc_url: Optional[str] = None,
     specific_doc: Optional[str] = None,
     output_dir: Path = Path("./downloads"),
     cache_existing: bool = True,
@@ -29,10 +31,12 @@ def download_documentation(
     fail_on_error: bool = False,
 ) -> bool:
     """
-    Download OpenShift documentation.
+    Download documentation.
 
     Args:
-        version: OpenShift version (e.g., "4.18")
+        version: Product version (e.g., "4.18")
+        product_slug: Product URL slug (e.g., "openshift_container_platform")
+        doc_url: The full URL to the documentation page.
         specific_doc: Optional specific document to download
         output_dir: Directory to save downloaded files
         cache_existing: Whether to use cached downloads
@@ -45,10 +49,12 @@ def download_documentation(
     """
     logger = logging.getLogger(__name__)
 
-    if specific_doc:
-        base_url = f"https://docs.redhat.com/en/documentation/openshift_container_platform/{version}/html-single/{specific_doc}"
+    if doc_url:
+        base_url = doc_url
+    elif specific_doc:
+        base_url = f"https://docs.redhat.com/en/documentation/{product_slug}/{version}/html-single/{specific_doc}"
     else:
-        base_url = f"https://docs.redhat.com/en/documentation/openshift_container_platform/{version}"
+        base_url = f"https://docs.redhat.com/en/documentation/{product_slug}/{version}"
 
     logger.info("Downloading from: %s", base_url)
     logger.info("Output directory: %s", output_dir)