feat(dedupe): add a deduplication script

Add a script capable of deduplicating assets used between multiple build
targets.

When attempting to play with sphinx's asset paths I noticed there was a
lot of logic around preserving relative links. It didn't seem like they
want to allow usage of an asset path outside of the build directory.

The WebSupport class seemed to allow some of that flexibility we were
looking for, but unfortunately it still assumes the conf.py is in the
source directory and has issues working with other Sphinx extensions
like ifconfig.

Given what we are doing seems to be a niche usecase, we can address this
with a little post processing using lxml filters.

Signed-off-by: Randolph Sapp <[email protected]>

Author: StaticRocket

bin/dedupe.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+
+"""Tool to deduplicate HTML assets for GitHub pages deployments
+
+SPDX-License-Identifier: MIT
+Copyright (C) 2025 Texas Instruments Incorporated - https://www.ti.com
+"""
+
+import logging
+
+from lxml import html
+from root_index import get_root_index, BUILD_PATH
+
+COMMON_PATHS = {"_images", "_downloads", "_static"}
+
+
+def _rewrite_wrapper(document, old_rel_path, new_rel_path, check_list):
+    """Wrapper to replace links using lxml rewrite_links. Defines a throwaway function to make
+    things faster.
+
+    :param document: lxml html document to operate on
+    :param old_rel_path: Pathlib path to the document root directory
+    :param new_rel_path: Pathlib path to the new common directory
+    :param check_list: Iterable of pathlib paths to check
+    """
+
+    def _update_link(link):
+        """Function to interact with lxml's rewrite_links
+
+        :param link: String link to rewrite
+        """
+        clean_link = link.strip()
+        if clean_link[:4] == "http":
+            return link
+
+        link_path = old_rel_path.joinpath(clean_link).resolve()
+        for check_path in check_list:
+            if link_path.is_relative_to(check_path):
+                logging.info("old link path: %s", link_path)
+                new_path = new_rel_path.joinpath(
+                    link_path.relative_to(check_path.parent)
+                )
+                logging.info("new link path: %s", new_path)
+                rel_path = new_path.relative_to(old_rel_path, walk_up=True)
+                logging.info("new rel path: %s", rel_path)
+                logging.info("---")
+                return rel_path.as_posix()
+
+        return link
+
+    document.rewrite_links(_update_link, resolve_base_href=False)
+
+
+def _move_files(old_rel_path, new_rel_path, check_list):
+    """Move the files that match the check_list from the old_rel_path root into new_rel_path.
+
+    :param old_rel_path: Pathlib path to the document root directory
+    :param new_rel_path: Pathlib path to the new common directory
+    :param check_list: Iterable of pathlib paths to check
+    """
+    for check_path in check_list:
+        operating_dir = old_rel_path.joinpath(check_path)
+        for path in operating_dir.glob("**/*"):
+            if not path.is_file():
+                continue
+            rel = path.relative_to(old_rel_path)
+            logging.info("moving file: %s", rel)
+            new = new_rel_path.joinpath(rel)
+            logging.info("destination: %s", new)
+            path.replace(new)
+            logging.info("---")
+
+        for empty_dir in sorted(operating_dir.glob("**/*"), reverse=True):
+            empty_dir.rmdir()
+
+
+def rewrite_paths(root_dir, common_dir):
+    """Rewrite the paths to move assets into a common_dir directory. This assumes:
+
+    1. Paths are already relative to the given root_dir
+    2. The root_dir resides under the common_dir
+
+    :param root_dir: Pathlib path to document root directory
+    :param common_dir: Pathlib path to new common_dir directory
+    """
+    check_list = {root_dir.joinpath(x).resolve() for x in COMMON_PATHS}
+    logging.info("rewriting paths")
+    for html_path in root_dir.glob("**/*.html"):
+        with html_path.open("r", encoding="utf-8") as file:
+            document = html.fromstring(file.read())
+
+        _rewrite_wrapper(
+            document, html_path.parent.resolve(), common_dir.resolve(), check_list
+        )
+
+        with html_path.open("wb") as file:
+            file.write(
+                html.tostring(
+                    document,
+                    encoding="utf-8",
+                    include_meta_content_type=True,
+                    doctype="<!DOCTYPE html>",
+                )
+            )
+
+    logging.info("moving the files")
+    _move_files(root_dir.resolve(), common_dir.resolve(), check_list)
+
+
+def main():
+    """Main processing loop"""
+    logging.basicConfig(level=logging.INFO)
+
+    root_list = []
+    for path in BUILD_PATH.glob("*/"):
+        index_path = get_root_index(path)
+        if index_path:
+            root_list.append(index_path.parent)
+            logging.info("found the following index: %s", index_path)
+
+    for path in root_list:
+        logging.info("working on the following document dir: %s", path)
+        rewrite_paths(path, BUILD_PATH)
+
+
+if __name__ == "__main__":
+    main()