docs: sync API reference updates with Docusaurus on release

.github/utils/pydoc-markdown.sh

@@ -1,10 +1,24 @@
 #!/bin/bash
 
+# Usage: ./pydoc-markdown.sh [CONFIG_PATH]
+#
+# Generate documentation from pydoc-markdown config files.
+#
+# Examples:
+#   ./pydoc-markdown.sh                    # Uses default path: ../config/*
+#   ./pydoc-markdown.sh ../config/api/*    # Uses custom path
+#   ./pydoc-markdown.sh /path/to/configs/* # Uses absolute path
+
 set -e   # Fails on any error in the following loop
+
+# Set default config path or use provided parameter
+CONFIG_PATH="${1:-../config/*}"
+
 cd docs/pydoc
 rm -rf temp && mkdir temp
 cd temp
-for file in ../config/* ; do
+echo "Processing config files in $CONFIG_PATH"
+for file in $CONFIG_PATH ; do
     echo "Converting $file..."
     pydoc-markdown "$file"
 done

.github/workflows/docusaurus_sync.yml

@@ -0,0 +1,98 @@
+name: Sync API reference with Docusaurus
+
+on:
+  push:
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+*"
+
+  # Manual trigger: Use this ONLY if the automatic API reference sync failed during a release.
+  # This will regenerate and sync the API reference to the Haystack repository.
+  # WARNING: Running this workflow when this repository has evolved significantly since the release
+  # will sync an incorrect API reference to Haystack.
+  workflow_dispatch:
+
+jobs:
+  generate-api-reference:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout this repo
+        uses: actions/checkout@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -U haystack-pydoc-tools
+
+      - name: Generate API reference
+        run: ./.github/utils/pydoc-markdown.sh "../config_docusaurus/*"
+
+      - name: Upload API reference artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: experimental-api-reference
+          path: docs/pydoc/temp
+          if-no-files-found: error
+          retention-days: 1
+          overwrite: true
+
+
+  sync-api-reference:
+    runs-on: ubuntu-latest
+    needs: generate-api-reference
+
+    steps:
+      - name: Checkout Haystack repo
+        uses: actions/checkout@v5
+        with:
+          repository: deepset-ai/haystack
+          ref: main
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.10"
+
+      - name: Download API reference artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: experimental-api-reference
+          path: temp_api_reference
+
+      - name: Sync API reference
+        run: |
+          # Function to sync generated API reference to a destination
+          sync_to_dest() {
+            echo "Syncing to $1"
+            mkdir -p "$1"
+            rsync -av --delete --exclude='.git/' "temp_api_reference/" "$1/"
+          }
+
+          # Sync to main reference
+          sync_to_dest "docs-website/reference/experiments-api"
+
+          # Sync to all versioned directories
+          if [ -d "docs-website/reference_versioned_docs" ]; then
+            for version_dir in "docs-website/reference_versioned_docs"/version-*; do
+              [ -d "$version_dir" ] && sync_to_dest "$version_dir/experiments-api"
+            done
+          fi
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
+          commit-message: "Sync Haystack Experimental API reference on Docusaurus"
+          branch: sync-docusaurus-api-reference-experimental
+          base: main
+          title: "docs: sync Haystack Experimental API reference on Docusaurus"
+          add-paths: |
+            docs-website
+          body: |
+            This PR syncs the Haystack Experimental API reference on Docusaurus. Just approve and merge it.

docs/pydoc/config_docusaurus/agents_api.yml

@@ -0,0 +1,35 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_experimental.components.agents.agent
+  - haystack_experimental.components.agents.human_in_the_loop.breakpoint
+  - haystack_experimental.components.agents.human_in_the_loop.dataclasses
+  - haystack_experimental.components.agents.human_in_the_loop.errors
+  - haystack_experimental.components.agents.human_in_the_loop.policies
+  - haystack_experimental.components.agents.human_in_the_loop.strategies
+  - haystack_experimental.components.agents.human_in_the_loop.types
+  - haystack_experimental.components.agents.human_in_the_loop.user_interfaces
+  search_path:
+  - ../../../
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: Tool-using agents with provider-agnostic chat model support.
+  id: experimental-agents-api
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: experimental_agents_api.md
+  title: Agents
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

docs/pydoc/config_docusaurus/chat_message_stores_api.yml

@@ -0,0 +1,28 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_experimental.chat_message_stores.in_memory
+  search_path:
+  - ../../../
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: Storage for the chat messages.
+  id: experimental-chatmessage-store-api
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: experimental_chatmessage_store_api.md
+  title: ChatMessage Store
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

docs/pydoc/config_docusaurus/generators_api.yml

@@ -0,0 +1,28 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_experimental.components.generators.chat.openai
+  search_path:
+  - ../../../
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: Enables text generation using LLMs.
+  id: experimental-generators-api
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: experimental_generators_api.md
+  title: Generators
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

docs/pydoc/config_docusaurus/preprocessors_api.yml

@@ -0,0 +1,28 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_experimental.components.preprocessors.embedding_based_document_splitter
+  search_path:
+  - ../../../
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: Pipelines wrapped as components.
+  id: experimental-preprocessors-api
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: experimental_preprocessors_api.md
+  title: Preprocessors
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

docs/pydoc/config_docusaurus/query_api.yml

@@ -0,0 +1,28 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_experimental.components.query.query_expander
+  search_path:
+  - ../../../
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: Components for query processing and expansion.
+  id: experimental-query-api
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: experimental_query_api.md
+  title: Query
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

docs/pydoc/config_docusaurus/retrievers_api.yml

@@ -0,0 +1,31 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_experimental.components.retrievers.chat_message_retriever
+  - haystack_experimental.components.retrievers.multi_query_embedding_retriever
+  - haystack_experimental.components.retrievers.multi_query_text_retriever
+  search_path:
+  - ../../../
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: Sweep through Document Stores and return a set of candidate documents
+    that are relevant to the query.
+  id: experimental-retrievers-api
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: experimental_retrievers_api.md
+  title: Retrievers
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

docs/pydoc/config_docusaurus/summarizers_api.yml

@@ -0,0 +1,28 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_experimental.components.summarizers.llm_summarizer
+  search_path:
+  - ../../../
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: Components that summarize texts into concise versions.
+  id: experimental-summarizers-api
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: experimental_summarizer_api.md
+  title: Summarizers
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

docs/pydoc/config_docusaurus/supercomponents_api.yml

@@ -0,0 +1,28 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_experimental.super_components.indexers.sentence_transformers_document_indexer
+  search_path:
+  - ../../../
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: Pipelines wrapped as components.
+  id: experimental-supercomponents-api
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: experimental_supercomponents_api.md
+  title: SuperComponents
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

docs/pydoc/config_docusaurus/writers_api.yml

@@ -0,0 +1,28 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_experimental.components.writers.chat_message_writer
+  search_path:
+  - ../../../
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: Writers for Haystack.
+  id: experimental-writers-api
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: experimental_writers_api.md
+  title: Writers
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer