scraper.yml

name: Run scraper and update docs search
on:
  push:
    branches:
      - mintlify-iso

jobs:
  await-mintlify-deploy:
    runs-on: ubuntu-22.04
    steps:
      - name: Wait for Mintlify deployment
        run: |
          sleep 30

  scrape-docs:
    needs: await-mintlify-deploy
    runs-on: ubuntu-22.04
    name: scrape and push content on Meilisearch instance
    steps:
      - uses: actions/checkout@v4
      
      - name: Prepare config file
        run: |
          jq '.custom_settings.embedders.default.apiKey = "${{ secrets.OPENAI_API_KEY }}"' "${{ github.workspace }}/docs-scraper.config.json" > "${{ github.workspace }}/temp-config.json" && mv "${{ github.workspace }}/temp-config.json" "${{ github.workspace }}/docs-scraper.config.json"

      - name: Run docs-scraper
        env:
          HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }}
          API_KEY: ${{ secrets.MEILISEARCH_API_KEY }}
          CONFIG_FILE_PATH: ${{ github.workspace }}/docs-scraper.config.json
        run: |
          docker run -t --rm \
            -e MEILISEARCH_HOST_URL=$HOST_URL \
            -e MEILISEARCH_API_KEY=$API_KEY \
            -v $CONFIG_FILE_PATH:/docs-scraper/config.json \
            getmeili/docs-scraper:v0.12.8 pipenv run ./docs_scraper config.json