toolhive-doc-mcp/sources.yaml.example at main · StacklokLabs/toolhive-doc-mcp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# Documentation sources configuration - Example
# Copy this file to sources.yaml and customize for your needs

sources:
  # Website sources - crawl and extract documentation from websites
  websites:
    - name: "Stacklok Toolhive Docs"
      url: "https://docs.stacklok.com/toolhive"
      path_prefix: "/toolhive"  # Limit crawling to this path
      enabled: true

  # GitHub repository sources - fetch markdown, YAML, and JSON files from specific repos
  github_repos:
    - name: "Stacklok Minder GitHub Docs"
      repo_owner: "stacklok"
      repo_name: "toolhive"
      branch: "main"  # Optional, defaults to default branch
      paths:
        - "docs/**/*.md"  # Glob patterns for markdown files
        - "README.md"
        - "examples/**/*.yaml"  # YAML example files
        - "examples/**/*.yml"  # YAML example files (alternative extension)
        - "examples/**/*.json"  # JSON example files
        - "**/*.yaml.example"  # Example files with .example suffix
        - "**/*.json.example"  # Example files with .example suffix
      enabled: true  # Set to true to enable

    - name: "Toolhive Registry"
      repo_owner: "stacklok"
      repo_name: "toolhive-registry"
      branch: "main"  # Optional, defaults to default branch
      paths:
        - "docs/**/*.md"  # Documentation files
        - "README.md"
        - "CONTRIBUTING.md"
        - "registry/**/*.yaml"  # Registry spec files
        - "registry/**/*.yml"  # Registry spec files (alternative extension)
      enabled: true  # Set to true to enable

# Fetching configuration (applies to all sources)
fetching:
  timeout: 30  # HTTP request timeout in seconds
  max_retries: 3  # Maximum retry attempts for failed requests
  concurrent_limit: 5  # Maximum concurrent HTTP requests
  delay_ms: 100  # Delay between requests in milliseconds
  max_depth: 5  # Maximum crawl depth for websites

# GitHub API configuration
github:
  # You can also set the GITHUB_TOKEN environment variable
  token: null
  api_url: "https://api.github.com"  # GitHub API base URL

# Background refresh configuration
refresh:
  enabled: true  # Enable/disable background refresh
  interval_hours: 24  # Refresh every 24 hours
  max_concurrent_jobs: 1  # Prevent overlapping refreshes