diff --git a/.github/utils/pydoc-markdown.sh b/.github/utils/pydoc-markdown.sh index 670bd096..0344db5b 100755 --- a/.github/utils/pydoc-markdown.sh +++ b/.github/utils/pydoc-markdown.sh @@ -1,10 +1,24 @@ #!/bin/bash +# Usage: ./pydoc-markdown.sh [CONFIG_PATH] +# +# Generate documentation from pydoc-markdown config files. +# +# Examples: +# ./pydoc-markdown.sh # Uses default path: ../config/* +# ./pydoc-markdown.sh ../config/api/* # Uses custom path +# ./pydoc-markdown.sh /path/to/configs/* # Uses absolute path + set -e # Fails on any error in the following loop + +# Set default config path or use provided parameter +CONFIG_PATH="${1:-../config/*}" + cd docs/pydoc rm -rf temp && mkdir temp cd temp -for file in ../config/* ; do +echo "Processing config files in $CONFIG_PATH" +for file in $CONFIG_PATH ; do echo "Converting $file..." pydoc-markdown "$file" done diff --git a/.github/workflows/docusaurus_sync.yml b/.github/workflows/docusaurus_sync.yml new file mode 100644 index 00000000..dd2b03c6 --- /dev/null +++ b/.github/workflows/docusaurus_sync.yml @@ -0,0 +1,98 @@ +name: Sync API reference with Docusaurus + +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+*" + + # Manual trigger: Use this ONLY if the automatic API reference sync failed during a release. + # This will regenerate and sync the API reference to the Haystack repository. + # WARNING: Running this workflow when this repository has evolved significantly since the release + # will sync an incorrect API reference to Haystack. + workflow_dispatch: + +jobs: + generate-api-reference: + runs-on: ubuntu-latest + + steps: + - name: Checkout this repo + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.10" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -U haystack-pydoc-tools + + - name: Generate API reference + run: ./.github/utils/pydoc-markdown.sh "../config_docusaurus/*" + + - name: Upload API reference artifact + uses: actions/upload-artifact@v4 + with: + name: experimental-api-reference + path: docs/pydoc/temp + if-no-files-found: error + retention-days: 1 + overwrite: true + + + sync-api-reference: + runs-on: ubuntu-latest + needs: generate-api-reference + + steps: + - name: Checkout Haystack repo + uses: actions/checkout@v5 + with: + repository: deepset-ai/haystack + ref: main + token: ${{ secrets.HAYSTACK_BOT_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.10" + + - name: Download API reference artifact + uses: actions/download-artifact@v4 + with: + name: experimental-api-reference + path: temp_api_reference + + - name: Sync API reference + run: | + # Function to sync generated API reference to a destination + sync_to_dest() { + echo "Syncing to $1" + mkdir -p "$1" + rsync -av --delete --exclude='.git/' "temp_api_reference/" "$1/" + } + + # Sync to main reference + sync_to_dest "docs-website/reference/experiments-api" + + # Sync to all versioned directories + if [ -d "docs-website/reference_versioned_docs" ]; then + for version_dir in "docs-website/reference_versioned_docs"/version-*; do + [ -d "$version_dir" ] && sync_to_dest "$version_dir/experiments-api" + done + fi + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.HAYSTACK_BOT_TOKEN }} + commit-message: "Sync Haystack Experimental API reference on Docusaurus" + branch: sync-docusaurus-api-reference-experimental + base: main + title: "docs: sync Haystack Experimental API reference on Docusaurus" + add-paths: | + docs-website + body: | + This PR syncs the Haystack Experimental API reference on Docusaurus. Just approve and merge it. diff --git a/docs/pydoc/config_docusaurus/agents_api.yml b/docs/pydoc/config_docusaurus/agents_api.yml new file mode 100644 index 00000000..1dfc5661 --- /dev/null +++ b/docs/pydoc/config_docusaurus/agents_api.yml @@ -0,0 +1,35 @@ +loaders: +- ignore_when_discovered: + - __init__ + modules: + - haystack_experimental.components.agents.agent + - haystack_experimental.components.agents.human_in_the_loop.breakpoint + - haystack_experimental.components.agents.human_in_the_loop.dataclasses + - haystack_experimental.components.agents.human_in_the_loop.errors + - haystack_experimental.components.agents.human_in_the_loop.policies + - haystack_experimental.components.agents.human_in_the_loop.strategies + - haystack_experimental.components.agents.human_in_the_loop.types + - haystack_experimental.components.agents.human_in_the_loop.user_interfaces + search_path: + - ../../../ + type: haystack_pydoc_tools.loaders.CustomPythonLoader +processors: +- do_not_filter_modules: false + documented_only: true + expression: null + skip_empty_modules: true + type: filter +- type: smart +- type: crossref +renderer: + description: Tool-using agents with provider-agnostic chat model support. + id: experimental-agents-api + markdown: + add_member_class_prefix: false + add_method_class_prefix: true + classdef_code_block: false + descriptive_class_title: false + descriptive_module_title: true + filename: experimental_agents_api.md + title: Agents + type: haystack_pydoc_tools.renderers.DocusaurusRenderer diff --git a/docs/pydoc/config_docusaurus/chat_message_stores_api.yml b/docs/pydoc/config_docusaurus/chat_message_stores_api.yml new file mode 100644 index 00000000..007a5bc8 --- /dev/null +++ b/docs/pydoc/config_docusaurus/chat_message_stores_api.yml @@ -0,0 +1,28 @@ +loaders: +- ignore_when_discovered: + - __init__ + modules: + - haystack_experimental.chat_message_stores.in_memory + search_path: + - ../../../ + type: haystack_pydoc_tools.loaders.CustomPythonLoader +processors: +- do_not_filter_modules: false + documented_only: true + expression: null + skip_empty_modules: true + type: filter +- type: smart +- type: crossref +renderer: + description: Storage for the chat messages. + id: experimental-chatmessage-store-api + markdown: + add_member_class_prefix: false + add_method_class_prefix: true + classdef_code_block: false + descriptive_class_title: false + descriptive_module_title: true + filename: experimental_chatmessage_store_api.md + title: ChatMessage Store + type: haystack_pydoc_tools.renderers.DocusaurusRenderer diff --git a/docs/pydoc/config_docusaurus/generators_api.yml b/docs/pydoc/config_docusaurus/generators_api.yml new file mode 100644 index 00000000..98f32ce6 --- /dev/null +++ b/docs/pydoc/config_docusaurus/generators_api.yml @@ -0,0 +1,28 @@ +loaders: +- ignore_when_discovered: + - __init__ + modules: + - haystack_experimental.components.generators.chat.openai + search_path: + - ../../../ + type: haystack_pydoc_tools.loaders.CustomPythonLoader +processors: +- do_not_filter_modules: false + documented_only: true + expression: null + skip_empty_modules: true + type: filter +- type: smart +- type: crossref +renderer: + description: Enables text generation using LLMs. + id: experimental-generators-api + markdown: + add_member_class_prefix: false + add_method_class_prefix: true + classdef_code_block: false + descriptive_class_title: false + descriptive_module_title: true + filename: experimental_generators_api.md + title: Generators + type: haystack_pydoc_tools.renderers.DocusaurusRenderer diff --git a/docs/pydoc/config_docusaurus/preprocessors_api.yml b/docs/pydoc/config_docusaurus/preprocessors_api.yml new file mode 100644 index 00000000..e20882cd --- /dev/null +++ b/docs/pydoc/config_docusaurus/preprocessors_api.yml @@ -0,0 +1,28 @@ +loaders: +- ignore_when_discovered: + - __init__ + modules: + - haystack_experimental.components.preprocessors.embedding_based_document_splitter + search_path: + - ../../../ + type: haystack_pydoc_tools.loaders.CustomPythonLoader +processors: +- do_not_filter_modules: false + documented_only: true + expression: null + skip_empty_modules: true + type: filter +- type: smart +- type: crossref +renderer: + description: Pipelines wrapped as components. + id: experimental-preprocessors-api + markdown: + add_member_class_prefix: false + add_method_class_prefix: true + classdef_code_block: false + descriptive_class_title: false + descriptive_module_title: true + filename: experimental_preprocessors_api.md + title: Preprocessors + type: haystack_pydoc_tools.renderers.DocusaurusRenderer diff --git a/docs/pydoc/config_docusaurus/query_api.yml b/docs/pydoc/config_docusaurus/query_api.yml new file mode 100644 index 00000000..97c70710 --- /dev/null +++ b/docs/pydoc/config_docusaurus/query_api.yml @@ -0,0 +1,28 @@ +loaders: +- ignore_when_discovered: + - __init__ + modules: + - haystack_experimental.components.query.query_expander + search_path: + - ../../../ + type: haystack_pydoc_tools.loaders.CustomPythonLoader +processors: +- do_not_filter_modules: false + documented_only: true + expression: null + skip_empty_modules: true + type: filter +- type: smart +- type: crossref +renderer: + description: Components for query processing and expansion. + id: experimental-query-api + markdown: + add_member_class_prefix: false + add_method_class_prefix: true + classdef_code_block: false + descriptive_class_title: false + descriptive_module_title: true + filename: experimental_query_api.md + title: Query + type: haystack_pydoc_tools.renderers.DocusaurusRenderer diff --git a/docs/pydoc/config_docusaurus/retrievers_api.yml b/docs/pydoc/config_docusaurus/retrievers_api.yml new file mode 100644 index 00000000..db5e0ed5 --- /dev/null +++ b/docs/pydoc/config_docusaurus/retrievers_api.yml @@ -0,0 +1,31 @@ +loaders: +- ignore_when_discovered: + - __init__ + modules: + - haystack_experimental.components.retrievers.chat_message_retriever + - haystack_experimental.components.retrievers.multi_query_embedding_retriever + - haystack_experimental.components.retrievers.multi_query_text_retriever + search_path: + - ../../../ + type: haystack_pydoc_tools.loaders.CustomPythonLoader +processors: +- do_not_filter_modules: false + documented_only: true + expression: null + skip_empty_modules: true + type: filter +- type: smart +- type: crossref +renderer: + description: Sweep through Document Stores and return a set of candidate documents + that are relevant to the query. + id: experimental-retrievers-api + markdown: + add_member_class_prefix: false + add_method_class_prefix: true + classdef_code_block: false + descriptive_class_title: false + descriptive_module_title: true + filename: experimental_retrievers_api.md + title: Retrievers + type: haystack_pydoc_tools.renderers.DocusaurusRenderer diff --git a/docs/pydoc/config_docusaurus/summarizers_api.yml b/docs/pydoc/config_docusaurus/summarizers_api.yml new file mode 100644 index 00000000..61662cc8 --- /dev/null +++ b/docs/pydoc/config_docusaurus/summarizers_api.yml @@ -0,0 +1,28 @@ +loaders: +- ignore_when_discovered: + - __init__ + modules: + - haystack_experimental.components.summarizers.llm_summarizer + search_path: + - ../../../ + type: haystack_pydoc_tools.loaders.CustomPythonLoader +processors: +- do_not_filter_modules: false + documented_only: true + expression: null + skip_empty_modules: true + type: filter +- type: smart +- type: crossref +renderer: + description: Components that summarize texts into concise versions. + id: experimental-summarizers-api + markdown: + add_member_class_prefix: false + add_method_class_prefix: true + classdef_code_block: false + descriptive_class_title: false + descriptive_module_title: true + filename: experimental_summarizer_api.md + title: Summarizers + type: haystack_pydoc_tools.renderers.DocusaurusRenderer diff --git a/docs/pydoc/config_docusaurus/supercomponents_api.yml b/docs/pydoc/config_docusaurus/supercomponents_api.yml new file mode 100644 index 00000000..6b451dfb --- /dev/null +++ b/docs/pydoc/config_docusaurus/supercomponents_api.yml @@ -0,0 +1,28 @@ +loaders: +- ignore_when_discovered: + - __init__ + modules: + - haystack_experimental.super_components.indexers.sentence_transformers_document_indexer + search_path: + - ../../../ + type: haystack_pydoc_tools.loaders.CustomPythonLoader +processors: +- do_not_filter_modules: false + documented_only: true + expression: null + skip_empty_modules: true + type: filter +- type: smart +- type: crossref +renderer: + description: Pipelines wrapped as components. + id: experimental-supercomponents-api + markdown: + add_member_class_prefix: false + add_method_class_prefix: true + classdef_code_block: false + descriptive_class_title: false + descriptive_module_title: true + filename: experimental_supercomponents_api.md + title: SuperComponents + type: haystack_pydoc_tools.renderers.DocusaurusRenderer diff --git a/docs/pydoc/config_docusaurus/writers_api.yml b/docs/pydoc/config_docusaurus/writers_api.yml new file mode 100644 index 00000000..e615d77a --- /dev/null +++ b/docs/pydoc/config_docusaurus/writers_api.yml @@ -0,0 +1,28 @@ +loaders: +- ignore_when_discovered: + - __init__ + modules: + - haystack_experimental.components.writers.chat_message_writer + search_path: + - ../../../ + type: haystack_pydoc_tools.loaders.CustomPythonLoader +processors: +- do_not_filter_modules: false + documented_only: true + expression: null + skip_empty_modules: true + type: filter +- type: smart +- type: crossref +renderer: + description: Writers for Haystack. + id: experimental-writers-api + markdown: + add_member_class_prefix: false + add_method_class_prefix: true + classdef_code_block: false + descriptive_class_title: false + descriptive_module_title: true + filename: experimental_writers_api.md + title: Writers + type: haystack_pydoc_tools.renderers.DocusaurusRenderer diff --git a/haystack_experimental/components/query/query_expander.py b/haystack_experimental/components/query/query_expander.py index 0f273c48..c3febf3d 100644 --- a/haystack_experimental/components/query/query_expander.py +++ b/haystack_experimental/components/query/query_expander.py @@ -99,7 +99,7 @@ def __init__( If None, a default OpenAIChatGenerator with gpt-4.1-mini model is used. :param prompt_template: Custom [PromptBuilder](https://docs.haystack.deepset.ai/docs/promptbuilder) template for query expansion. The template should instruct the LLM to return a JSON response with the - structure: {"queries": ["query1", "query2", "query3"]}. The template should include 'query' and + structure: `{"queries": ["query1", "query2", "query3"]}`. The template should include 'query' and 'n_expansions' variables. :param n_expansions: Number of alternative queries to generate (default: 4). :param include_original_query: Whether to include the original query in the output.