sc-103646-add-sitemap-for-docs-pennylane-ai-lightning (#1331)

## Add sitemap generation for documentation

### Summary
- Added `sphinx-sitemap` extension to automatically generate a sitemap
for the PennyLane lightning documentation, improving SEO and to reduce
number of [404
pages](https://www.notion.so/xanaduai/404-URLs-on-PennyLane-Website-2a7bc6bd1764807ba9a4c145e9d15ee8?pvs=26&qid=1%3A95a102a1-752c-4dff-8481-3a834455b689%3A3)
showing up in Google Console.
- marks C++ API pages a `no-index` to prevent C++ pages from getting
indexed in Google.

### Changes
- Added `sphinx_sitemap` to Sphinx extensions in `doc/conf.py`
- Updated `html_baseurl` to use proper canonical URL with ReadTheDocs
fallback
- Added `sitemap_url_scheme` configuration for standard sitemap format
- Added `sphinx-sitemap` dependency to:
  - `doc/requirements.txt`
  - `pyproject.toml` (docs optional dependencies)
- Added `add_noindex_to_cpp_pages` to exclude C++ API pages
- Rest are just formatting changes by `black`

## Ticket
-
[sc-103646-add-sitemap-for-docs-pennylane-ai-lightning](https://app.shortcut.com/xanaduai/story/103646/add-sitemap-for-docs-pennylane-ai-lightning)
-
[sc-103649-add-no-index-to-c-library-pages-in-lightning](https://app.shortcut.com/xanaduai/story/103649/add-no-index-to-c-library-pages-in-lightning-docs?vc_group_by=day&ct_workflow=all&cf_workflow=500000005)

## Testing
- Check the sitemap
[here](https://xanaduai-pennylane--1331.com.readthedocs.build/projects/lightning/en/1331/sitemap.xml)
which includes list of all pages excluding C++ API ones.
- Screenshot below to confirm addition of `no-index` tag to tell google
to not index C++ API pages



<img width="1865" height="571" alt="Screenshot 2026-01-16 at 10 15
54 AM"
src="https://github.com/user-attachments/assets/1bafd238-25ae-49f7-ac3b-0bb2880389ff"
/>

---------

Co-authored-by: ringo-but-quantum <[email protected]>

Author: ashishks0522

.github/CHANGELOG.md

@@ -10,6 +10,9 @@
 
 <h3>Improvements 🛠</h3>
 
+- Added sitemap configuration and SEO improvements to documentation, including noindex meta tags for C++ API reference pages.
+  [(#1331)](https://github.com/PennyLaneAI/pennylane-lightning/pull/1331)
+
 - Added a C++ kernel for the `qml.PauliRot` simulation on `lightning.gpu`.
   [(#1329)](https://github.com/PennyLaneAI/pennylane-lightning/pull/1329)
 
@@ -46,6 +49,7 @@
 This release contains contributions from (in alphabetical order):
 
 Ali Asadi,
+Ashish Kanwar Singh
 Jeffrey Kam,
 Jake Zaia.
 

doc/conf.py

@@ -39,15 +39,25 @@ def obtain_cpp_files():
         print("The project directory structure is corrupted.")
         sys.exit(1)
 
-    exclude_dirs = [CPP_SOURCE_DIR.joinpath(exclude_dir) for exclude_dir in CPP_EXCLUDE_DIRS]
+    exclude_dirs = [
+        CPP_SOURCE_DIR.joinpath(exclude_dir) for exclude_dir in CPP_EXCLUDE_DIRS
+    ]
 
     p = subprocess.run(
-        [str(script_path), "--header-only", CPP_SOURCE_DIR, "--exclude-dirs", *exclude_dirs],
+        [
+            str(script_path),
+            "--header-only",
+            CPP_SOURCE_DIR,
+            "--exclude-dirs",
+            *exclude_dirs,
+        ],
         capture_output=True,
     )
     file_list = json.loads(p.stdout)
 
-    file_list = ["../" + str(Path(f).relative_to(PROJECT_SOURCE_DIR)) for f in file_list]
+    file_list = [
+        "../" + str(Path(f).relative_to(PROJECT_SOURCE_DIR)) for f in file_list
+    ]
     return file_list
 
 
@@ -63,8 +73,10 @@ def __getattr__(cls, name):
         return MagicMock()
 
 
-MOCK_MODULES = ["pennylane_lightning.lightning_qubit_ops",
-                "pennylane_lightning.lightning_qubit_ops.algorithms"]
+MOCK_MODULES = [
+    "pennylane_lightning.lightning_qubit_ops",
+    "pennylane_lightning.lightning_qubit_ops.algorithms",
+]
 
 mock = Mock()
 for mod_name in MOCK_MODULES:
@@ -94,6 +106,7 @@ def __getattr__(cls, name):
     "sphinx_automodapi.automodapi",
     "sphinx_automodapi.smart_resolver",
     "sphinxext.opengraph",
+    "sphinx_sitemap",
 ]
 
 # Open Graph metadata
@@ -106,9 +119,11 @@ def __getattr__(cls, name):
 ogp_image = "_static/pennylane_lightning.png"
 
 # The base URL with a proper language and version.
-html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "/")
+html_baseurl = os.environ.get(
+    "READTHEDOCS_CANONICAL_URL", "https://docs.pennylane.ai/projects/lightning/"
+)
 
-intersphinx_mapping = {'pennylane':("https://docs.pennylane.ai/en/stable/", None)}
+intersphinx_mapping = {"pennylane": ("https://docs.pennylane.ai/en/stable/", None)}
 
 autosummary_generate = True
 autosummary_imported_members = False
@@ -119,9 +134,7 @@ def __getattr__(cls, name):
 breathe_projects = {"Lightning-Qubit": "./doxyoutput/xml"}
 breathe_default_project = "Lightning-Qubit"
 
-mathjax_path = (
-    "https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-MML-AM_CHTML"
-)
+mathjax_path = "https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-MML-AM_CHTML"
 
 # Exhale extension
 # Setup the exhale extension
@@ -136,7 +149,9 @@ def __getattr__(cls, name):
     # TIP: if using the sphinx-bootstrap-theme, you need
     # "treeViewIsBootstrap": True,
     "exhaleExecutesDoxygen": True,
-    "exhaleDoxygenStdin": ("INPUT = " + " ".join(CPP_FILES) + "\nEXCLUDE_SYMBOLS = std::* "),
+    "exhaleDoxygenStdin": (
+        "INPUT = " + " ".join(CPP_FILES) + "\nEXCLUDE_SYMBOLS = std::* "
+    ),
     "afterTitleDescription": inspect.cleandoc(
         """
         The Pennylane Lightning C++ API is intended to be called from Python through Nanobind. Direct use of the C++ API is currently unsupported and is provided for reference only.
@@ -237,13 +252,42 @@ def __getattr__(cls, name):
 edit_on_github_project = "PennyLaneAI/pennylane-lightning"
 edit_on_github_branch = "master/doc"
 
+# -- Sitemap settings -----------------------------------------------------
+sitemap_url_scheme = "{link}"
+# Exclude C++ API pages from sitemap (reference only, not for public indexing)
+sitemap_excludes = ["api/*"]
+
+
+# -- Add noindex to C++ API pages -----------------------------------------
+def add_noindex_to_cpp_pages(
+    app, pagename, templatename, context, doctree
+):  # pylint: disable=unused-argument
+    """
+    Add noindex meta tag to C++ API pages generated by Exhale.
+
+    The C++ API is for reference only and should not be indexed by search engines.
+    """
+    if pagename.startswith("api/"):
+        metatags = context.get("metatags", "")
+        context["metatags"] = (
+            metatags + '<meta name="robots" content="noindex, nofollow" />\n'
+        )
+
+
+def setup(app):
+    """Sphinx application setup hook."""
+    app.connect("html-page-context", add_noindex_to_cpp_pages)
+
+
 # ============================================================
 
 # the order in which autodoc lists the documented members
 autodoc_member_order = "bysource"
 
 # inheritance_diagram graphviz attributes
-inheritance_node_attrs = dict(color="lightskyblue1", fillcolor="lightskyblue1", style="filled")
+inheritance_node_attrs = dict(
+    color="lightskyblue1", fillcolor="lightskyblue1", style="filled"
+)
 
 # autodoc_default_flags = ['members']
 autosummary_generate = True

doc/requirements.txt

@@ -20,4 +20,5 @@ sphinxcontrib-htmlhelp==2.0.6
 sphinxcontrib-serializinghtml==2.0.0
 pennylane-sphinx-theme
 sphinxext-opengraph
+sphinx-sitemap
 matplotlib

doc/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+
+Sitemap: https://docs.pennylane.ai/projects/lightning/sitemap.xml
+

pennylane_lightning/core/_version.py

@@ -16,4 +16,4 @@
 Version number (major.minor.patch[-label])
 """
 
-__version__ = "0.45.0-dev9"
+__version__ = "0.45.0-dev10"