Update integration-doc-generator.yml

harshilp24 · web-flow · commit b1605198ba1c · 2025-05-26T14:51:07.000+05:30
diff --git a/.github/workflows/integration-doc-generator.yml b/.github/workflows/integration-doc-generator.yml
@@ -26,89 +26,124 @@ jobs:
       uses: actions/checkout@v4
       with:
         token: ${{ secrets.REPO_ACCESS_TOKEN }}  # Use a PAT with access to both repos
+        fetch-depth: 0  # Fetch all history to check last commit
 
     - name: Ensure scripts directory exists
       run: |
         mkdir -p scripts
         if [ ! -f scripts/processed_files.txt ]; then
           touch scripts/processed_files.txt
         fi
-        if [ ! -f scripts/file_hashes.json ]; then
-          echo "{}" > scripts/file_hashes.json
+        if [ ! -f scripts/last_commit_sha.txt ]; then
+          touch scripts/last_commit_sha.txt
+        fi
+
+    - name: Get last processed commit
+      id: last_commit
+      run: |
+        if [ -s scripts/last_commit_sha.txt ]; then
+          LAST_COMMIT=$(cat scripts/last_commit_sha.txt)
+          echo "last_commit=$LAST_COMMIT" >> $GITHUB_ENV
+          echo "Last processed commit: $LAST_COMMIT"
+        else
+          echo "No previous commit found, will process all files"
+          echo "last_commit=" >> $GITHUB_ENV
         fi
 
     - name: Fetch file list and metadata from integration-resources
       run: |
+        # Get the latest commit SHA from integration-resources repo
+        LATEST_COMMIT=$(curl -s -H "Authorization: Bearer ${{ secrets.REPO_ACCESS_TOKEN }}" \
+                        -H "Accept: application/vnd.github+json" \
+                        https://api.github.com/repos/appsmithorg/integration-resources/commits/main | \
+                        jq -r '.sha' )
+        
+        echo "Latest commit in integration-resources: $LATEST_COMMIT"
+        echo "latest_commit=$LATEST_COMMIT" >> $GITHUB_ENV
+        
+        # Get all files in the repo
         curl -s -H "Authorization: Bearer ${{ secrets.REPO_ACCESS_TOKEN }}" \
              -H "Accept: application/vnd.github+json" \
              https://api.github.com/repos/appsmithorg/integration-resources/contents/Generic%20UQI%20Creation/uqi_configs \
              -o response.json
 
-        # Validate it's an array (not an error message)
+        # Validate it's an array (not an error message )
         if ! jq 'type == "array"' response.json | grep -q true; then
           echo "❌ GitHub API did not return a file list. Possible error:"
           cat response.json
           exit 1
         fi
 
-        # Extract file names and their SHA hashes
-        jq -r '.[] | select(.type=="file") | [.name, .sha] | @tsv' response.json > latest_files_with_sha.txt
+        # Extract file names
         jq -r '.[] | select(.type=="file") | .name' response.json > latest_files.txt
 
-    - name: Identify new and modified files
+    - name: Identify changed files since last commit
       id: detect_changes
       run: |
-        # Load previous file hashes
-        PREV_HASHES=$(cat scripts/file_hashes.json)
-        
         # Force check all files if requested
         if [ "${{ github.event.inputs.force_check_all }}" == "true" ]; then
           echo "🔄 Force checking all files as requested"
           cat latest_files.txt > files_to_process.txt
           echo "files_found=true" >> $GITHUB_ENV
+        elif [ -z "${{ env.last_commit }}" ]; then
+          # No previous commit, process all files
+          cat latest_files.txt > files_to_process.txt
+          echo "files_found=true" >> $GITHUB_ENV
         else
-          # Find new files (not in processed_files.txt)
-          NEW_FILES=$(comm -23 <(sort latest_files.txt) <(sort scripts/processed_files.txt) || true)
-          
-          # Check for modified files (SHA changed)
-          MODIFIED_FILES=""
-          while IFS=$'\t' read -r FILE_NAME FILE_SHA; do
-            PREV_SHA=$(echo "$PREV_HASHES" | jq -r --arg file "$FILE_NAME" '.[$file] // ""')
-            if [ -n "$PREV_SHA" ] && [ "$PREV_SHA" != "$FILE_SHA" ] && grep -q "^$FILE_NAME$" scripts/processed_files.txt; then
-              echo "🔄 File modified: $FILE_NAME (SHA changed)"
-              MODIFIED_FILES="$MODIFIED_FILES$FILE_NAME"$'\n'
-            fi
-          done < latest_files_with_sha.txt
-          
-          # Combine new and modified files
-          { echo "$NEW_FILES"; echo "$MODIFIED_FILES"; } | grep -v "^$" > files_to_process.txt
-          
-          if [ -s files_to_process.txt ]; then
-            echo "🆕 Found files to process:"
+          # Get files changed since last commit
+          echo "🔍 Finding files changed since commit ${{ env.last_commit }}"
+          
+          # Get list of files changed between commits
+          CHANGED_FILES=$(curl -s -H "Authorization: Bearer ${{ secrets.REPO_ACCESS_TOKEN }}" \
+                         -H "Accept: application/vnd.github+json" \
+                         "https://api.github.com/repos/appsmithorg/integration-resources/compare/${{ env.last_commit }}...${{ env.latest_commit }}" | \
+                         jq -r '.files[] | select(.filename | startswith("Generic UQI Creation/uqi_configs/" )) | .filename | split("/") | last' | \
+                         grep -v "^$")
+          
+          if [ -n "$CHANGED_FILES" ]; then
+            echo "$CHANGED_FILES" > files_to_process.txt
+            echo "🆕 Found files changed since last commit:"
             cat files_to_process.txt
             echo "files_found=true" >> $GITHUB_ENV
           else
-            echo "✅ No new or modified files to process."
+            echo "✅ No files changed since last commit."
             echo "files_found=false" >> $GITHUB_ENV
           fi
         fi
         
+        # Check for existing documentation
+        if [ "${{ env.files_found }}" == "true" ]; then
+          echo "🔍 Checking for existing documentation..."
+          FILTERED_FILES=""
+          while IFS= read -r FILE_NAME; do
+            INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]')
+            DOC_PATH="website/docs/connect-data/reference/${INTEGRATION}.md"
+            
+            if [ -f "$DOC_PATH" ]; then
+              echo "🔄 Documentation exists for $INTEGRATION, will update"
+            fi
+            
+            FILTERED_FILES="${FILTERED_FILES}${FILE_NAME}"$'\n'
+          done < files_to_process.txt
+          
+          echo "$FILTERED_FILES" | grep -v "^$" > files_to_process.txt
+        fi
+        
         # Count files to process
         FILE_COUNT=$(wc -l < files_to_process.txt || echo "0")
         echo "file_count=$FILE_COUNT" >> $GITHUB_ENV
 
     - name: Exit if no files to process
-      if: env.files_found != 'true'
-      run: exit 0
+      if: env.files_found != 'true' || env.file_count == '0'
+      run: |
+        echo "No files to process. Exiting."
+        exit 0
 
     - name: Process files
       run: |
         # Create a directory for generated docs
         mkdir -p generated_docs
         
-        # Update file hashes JSON for tracking changes
-        HASHES_JSON=$(cat scripts/file_hashes.json)
-        
         # Process each file
         while IFS= read -r FILE_NAME; do
           echo "⏳ Processing: $FILE_NAME"
@@ -117,11 +152,7 @@ jobs:
           FILE_URL="https://raw.githubusercontent.com/appsmithorg/integration-resources/main/Generic%20UQI%20Creation/uqi_configs/$FILE_NAME"
           curl -sSL "$FILE_URL" -o "input_file.json"
           
-          # Update hash in our tracking JSON
-          FILE_SHA=$(grep "$FILE_NAME" latest_files_with_sha.txt | cut -f2)
-          HASHES_JSON=$(echo "$HASHES_JSON" | jq --arg file "$FILE_NAME" --arg sha "$FILE_SHA" '.[$file] = $sha')
-          
-          # Process with OpenAI API (using completion API, not chat)
+          # Process with OpenAI API (using completion API, not chat )
           echo "🧠 Extracting information with OpenAI API..."
           
           # Extract information using OpenAI API
@@ -144,7 +175,7 @@ jobs:
           # Generate documentation
           echo "📝 Generating documentation..."
           
-          SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt || echo "Generate comprehensive markdown documentation based on the extracted information.")
+          SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt || echo "Generate comprehensive markdown documentation based on the extracted information." )
           EXTRACTED_CONTENT=$(cat extracted_info.md)
           
           # Use OpenAI Completion API again
@@ -161,7 +192,7 @@ jobs:
             -d "$PAYLOAD" | jq -r '.choices[0].text' > "generated_doc.md"
           
           # Prepare final path
-          INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]')
+          INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]' )
           FINAL_PATH="website/docs/connect-data/reference/${INTEGRATION}.md"
           mkdir -p "$(dirname "$FINAL_PATH")"
           
@@ -179,11 +210,11 @@ jobs:
           echo "✅ Completed processing: $FILE_NAME"
         done < files_to_process.txt
         
-        # Save updated hashes
-        echo "$HASHES_JSON" > scripts/file_hashes.json
+        # Save the latest commit SHA for next run
+        echo "${{ env.latest_commit }}" > scripts/last_commit_sha.txt
 
     - name: Commit and open PR
-      if: env.files_found == 'true'
+      if: env.files_found == 'true' && env.file_count != '0'
       uses: peter-evans/create-pull-request@v5
       with:
         token: ${{ secrets.REPO_ACCESS_TOKEN }}
@@ -194,8 +225,8 @@ jobs:
         add-paths: |
           website/docs/connect-data/reference/
           scripts/processed_files.txt
-          scripts/file_hashes.json
+          scripts/last_commit_sha.txt
         body: |
           This PR adds or updates integration reference documentation for **${{ env.file_count }}** integrations.
           
-          Generated from the latest configuration files in the [integration-resources repository](https://github.com/appsmithorg/integration-resources/tree/main/Generic%20UQI%20Creation/uqi_configs).
+          Generated from files changed since commit ${{ env.last_commit || 'initial' }} in the [integration-resources repository](https://github.com/appsmithorg/integration-resources/tree/main/Generic%20UQI%20Creation/uqi_configs ).
