Create test-doc-generator.yml

Author: harshilp24

.github/workflows/test-doc-generator.yml

@@ -0,0 +1,131 @@
+name: Test Doc Generator
+
+on:
+  workflow_dispatch:
+
+jobs:
+  generate_docs:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout appsmith-docs
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.test_REPO_ACCESS_TOKEN }}
+
+    - name: Create exclusion list for test
+      run: |
+        cat > saas_exclusions.txt << EOF
+        EOF
+
+    - name: Ensure scripts directory exists
+      run: |
+        mkdir -p scripts
+        [ -f scripts/processed_files.txt ] || touch scripts/processed_files.txt
+        [ -f scripts/file_hashes.json ] || echo "{}" > scripts/file_hashes.json
+
+    - name: Fetch file list from test repo
+      id: fetch_files
+      run: |
+        curl -s --max-time 30 -H "Authorization: Bearer ${{ secrets.test_REPO_ACCESS_TOKEN }}" \
+             -H "Accept: application/vnd.github+json" \
+             https://api.github.com/repos/harshilp24/integration-resources-test/contents/Generic%20UQI%20Creation/uqi_configs \
+             -o response.json
+
+        jq -r '.[] | select(.type=="file") | [.name, .sha] | @tsv' response.json > latest_files_with_sha.txt
+        jq -r '.[] | select(.type=="file") | .name' response.json > latest_files.txt
+
+        echo "files_found=true" >> $GITHUB_ENV
+
+    - name: Identify new and modified files
+      id: detect_changes
+      run: |
+        PREV_HASHES=$(cat scripts/file_hashes.json)
+        NEW_FILES=$(comm -23 <(sort latest_files.txt) <(sort scripts/processed_files.txt) || true)
+        MODIFIED_FILES=""
+        while IFS=$'\t' read -r FILE_NAME FILE_SHA; do
+          PREV_SHA=$(echo "$PREV_HASHES" | jq -r --arg file "$FILE_NAME" '.[$file] // ""')
+          if [ -n "$PREV_SHA" ] && [ "$PREV_SHA" != "$FILE_SHA" ] && grep -q "^$FILE_NAME$" scripts/processed_files.txt; then
+            MODIFIED_FILES="$MODIFIED_FILES$FILE_NAME"$'\n'
+          fi
+        done < latest_files_with_sha.txt
+        { echo "$NEW_FILES"; echo "$MODIFIED_FILES"; } | grep -v "^$" > files_to_process.txt
+        if [ -s files_to_process.txt ]; then
+          echo "changes_found=true" >> $GITHUB_ENV
+        else
+          echo "changes_found=false" >> $GITHUB_ENV
+        fi
+
+    - name: Exit if no files to process
+      if: env.changes_found != 'true'
+      run: exit 0
+
+    - name: Process files from test repo
+      run: |
+        mkdir -p generated_docs
+        HASHES_JSON=$(cat scripts/file_hashes.json)
+        PROCESSED_COUNT=0
+        while IFS= read -r FILE_NAME; do
+          FILE_URL="https://raw.githubusercontent.com/harshilp24/integration-resources-test/main/Generic%20UQI%20Creation/uqi_configs/$FILE_NAME"
+          curl -sSL --max-time 30 "$FILE_URL" -o "input_file.json"
+
+          FILE_SHA=$(grep "$FILE_NAME" latest_files_with_sha.txt | cut -f2)
+          HASHES_JSON=$(echo "$HASHES_JSON" | jq --arg file "$FILE_NAME" --arg sha "$FILE_SHA" '.[$file] = $sha')
+
+          SYSTEM_PROMPT=$(cat .github/prompts/extract_prompt.txt)
+          USER_CONTENT=$(cat input_file.json)
+          PAYLOAD=$(jq -n --arg prompt "System: $SYSTEM_PROMPT\n\nUser: $USER_CONTENT" '{
+            model: "gpt-4.1",
+            prompt: $prompt,
+            max_tokens: 2000,
+            temperature: 0
+          }')
+          RESPONSE=$(curl -s https://api.openai.com/v1/completions \
+            -H "Authorization: Bearer ${{ secrets.test_OPENAI_API_KEY }}" \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD")
+          echo "$RESPONSE" | jq -r '.choices[0].text' > "extracted_info.md"
+
+          SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt)
+          EXTRACTED_CONTENT=$(cat extracted_info.md)
+          PAYLOAD=$(jq -n --arg prompt "System: $SYSTEM_PROMPT\n\nUser: $EXTRACTED_CONTENT" '{
+            model: "gpt-4.1",
+            prompt: $prompt,
+            max_tokens: 4000,
+            temperature: 0.3
+          }')
+          RESPONSE=$(curl -s https://api.openai.com/v1/completions \
+            -H "Authorization: Bearer ${{ secrets.test_OPENAI_API_KEY }}" \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD")
+          echo "$RESPONSE" | jq -r '.choices[0].text' > "generated_doc.md"
+
+          INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]')
+          FINAL_PATH="website/docs/connect-data/reference/${INTEGRATION}.md"
+          mkdir -p "$(dirname "$FINAL_PATH")"
+          cp "generated_doc.md" "$FINAL_PATH"
+          cp "generated_doc.md" "generated_docs/${INTEGRATION}.md"
+          echo "$FILE_NAME" >> scripts/processed_files.txt
+          PROCESSED_COUNT=$((PROCESSED_COUNT + 1))
+        done < files_to_process.txt
+
+        echo "$HASHES_JSON" > scripts/file_hashes.json
+        echo "processed_count=$PROCESSED_COUNT" >> $GITHUB_ENV
+        echo "content_generated=true" >> $GITHUB_ENV
+
+    - name: Commit and open PR
+      if: env.content_generated == 'true'
+      uses: peter-evans/create-pull-request@v5
+      with:
+        token: ${{ secrets.test_REPO_ACCESS_TOKEN }}
+        title: "test: generate integration docs from test repo"
+        commit-message: "test: generated docs from harshilp24/integration-resources-test"
+        branch: "test/docs-update-${{ github.run_id }}"
+        base: main
+        add-paths: |
+          website/docs/connect-data/reference/
+          scripts/processed_files.txt
+          scripts/file_hashes.json
+        body: |
+          ✅ Test PR: Generated integration documentation from your test repo.
+          Source: [harshilp24/integration-resources-test](https://github.com/harshilp24/integration-resources-test/tree/main/Generic%20UQI%20Creation/uqi_configs)