Skip to content

Commit 64b5e0e

Browse files
harshilp24harshilp24
authored
Update and rename poll-and-generate-doc.yml to integration-doc-generator.yml
1 parent b9e161a commit 64b5e0e

File tree

2 files changed

+201
-127
lines changed

2 files changed

+201
-127
lines changed

‎.github/workflows/integration-doc-generator.yml‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
name: Integration Doc Generator
2+
3+
on:
4+
# Manual trigger
5+
workflow_dispatch:
6+
inputs:
7+
force_check_all:
8+
description: 'Force check all files (not just new ones)'
9+
required: false
10+
default: 'false'
11+
type: choice
12+
options:
13+
- 'true'
14+
- 'false'
15+
16+
# Event-based trigger - when changes are pushed to the integration-resources repo
17+
repository_dispatch:
18+
types: [integration_resources_updated]
19+
20+
jobs:
21+
generate_docs:
22+
runs-on: ubuntu-latest
23+
24+
steps:
25+
- name: Checkout appsmith-docs
26+
uses: actions/checkout@v4
27+
with:
28+
token: ${{ secrets.REPO_ACCESS_TOKEN }} # Use a PAT with access to both repos
29+
30+
- name: Ensure scripts directory exists
31+
run: |
32+
mkdir -p scripts
33+
if [ ! -f scripts/processed_files.txt ]; then
34+
touch scripts/processed_files.txt
35+
fi
36+
if [ ! -f scripts/file_hashes.json ]; then
37+
echo "{}" > scripts/file_hashes.json
38+
fi
39+
40+
- name: Fetch file list and metadata from integration-resources
41+
run: |
42+
curl -s -H "Authorization: Bearer ${{ secrets.REPO_ACCESS_TOKEN }}" \
43+
-H "Accept: application/vnd.github+json" \
44+
https://api.github.com/repos/appsmithorg/integration-resources/contents/Generic%20UQI%20Creation/uqi_configs \
45+
-o response.json
46+
47+
# Validate it's an array (not an error message)
48+
if ! jq 'type == "array"' response.json | grep -q true; then
49+
echo "❌ GitHub API did not return a file list. Possible error:"
50+
cat response.json
51+
exit 1
52+
fi
53+
54+
# Extract file names and their SHA hashes
55+
jq -r '.[] | select(.type=="file") | [.name, .sha] | @tsv' response.json > latest_files_with_sha.txt
56+
jq -r '.[] | select(.type=="file") | .name' response.json > latest_files.txt
57+
58+
- name: Identify new and modified files
59+
id: detect_changes
60+
run: |
61+
# Load previous file hashes
62+
PREV_HASHES=$(cat scripts/file_hashes.json)
63+
64+
# Force check all files if requested
65+
if [ "${{ github.event.inputs.force_check_all }}" == "true" ]; then
66+
echo "🔄 Force checking all files as requested"
67+
cat latest_files.txt > files_to_process.txt
68+
echo "files_found=true" >> $GITHUB_ENV
69+
else
70+
# Find new files (not in processed_files.txt)
71+
NEW_FILES=$(comm -23 <(sort latest_files.txt) <(sort scripts/processed_files.txt) || true)
72+
73+
# Check for modified files (SHA changed)
74+
MODIFIED_FILES=""
75+
while IFS=$'\t' read -r FILE_NAME FILE_SHA; do
76+
PREV_SHA=$(echo "$PREV_HASHES" | jq -r --arg file "$FILE_NAME" '.[$file] // ""')
77+
if [ -n "$PREV_SHA" ] && [ "$PREV_SHA" != "$FILE_SHA" ] && grep -q "^$FILE_NAME$" scripts/processed_files.txt; then
78+
echo "🔄 File modified: $FILE_NAME (SHA changed)"
79+
MODIFIED_FILES="$MODIFIED_FILES$FILE_NAME"$'\n'
80+
fi
81+
done < latest_files_with_sha.txt
82+
83+
# Combine new and modified files
84+
{ echo "$NEW_FILES"; echo "$MODIFIED_FILES"; } | grep -v "^$" > files_to_process.txt
85+
86+
if [ -s files_to_process.txt ]; then
87+
echo "🆕 Found files to process:"
88+
cat files_to_process.txt
89+
echo "files_found=true" >> $GITHUB_ENV
90+
else
91+
echo "✅ No new or modified files to process."
92+
echo "files_found=false" >> $GITHUB_ENV
93+
fi
94+
fi
95+
96+
# Count files to process
97+
FILE_COUNT=$(wc -l < files_to_process.txt || echo "0")
98+
echo "file_count=$FILE_COUNT" >> $GITHUB_ENV
99+
100+
- name: Exit if no files to process
101+
if: env.files_found != 'true'
102+
run: exit 0
103+
104+
- name: Process files
105+
run: |
106+
# Create a directory for generated docs
107+
mkdir -p generated_docs
108+
109+
# Update file hashes JSON for tracking changes
110+
HASHES_JSON=$(cat scripts/file_hashes.json)
111+
112+
# Process each file
113+
while IFS= read -r FILE_NAME; do
114+
echo "⏳ Processing: $FILE_NAME"
115+
116+
# Download the file
117+
FILE_URL="https://raw.githubusercontent.com/appsmithorg/integration-resources/main/Generic%20UQI%20Creation/uqi_configs/$FILE_NAME"
118+
curl -sSL "$FILE_URL" -o "input_file.json"
119+
120+
# Update hash in our tracking JSON
121+
FILE_SHA=$(grep "$FILE_NAME" latest_files_with_sha.txt | cut -f2)
122+
HASHES_JSON=$(echo "$HASHES_JSON" | jq --arg file "$FILE_NAME" --arg sha "$FILE_SHA" '.[$file] = $sha')
123+
124+
# Process with OpenAI API (using completion API, not chat)
125+
echo "🧠 Extracting information with OpenAI API..."
126+
127+
# Extract information using OpenAI API
128+
SYSTEM_PROMPT=$(cat .github/prompts/extract_prompt.txt || echo "Extract the key information from this integration configuration file.")
129+
USER_CONTENT=$(cat input_file.json)
130+
131+
# Use OpenAI Completion API (not Chat API)
132+
PAYLOAD=$(jq -n --arg prompt "System: $SYSTEM_PROMPT\n\nUser: $USER_CONTENT" '{
133+
model: "gpt-4-turbo-preview",
134+
prompt: $prompt,
135+
max_tokens: 2000,
136+
temperature: 0
137+
}')
138+
139+
curl -s https://api.openai.com/v1/completions \
140+
-H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \
141+
-H "Content-Type: application/json" \
142+
-d "$PAYLOAD" | jq -r '.choices[0].text' > "extracted_info.md"
143+
144+
# Generate documentation
145+
echo "📝 Generating documentation..."
146+
147+
SYSTEM_PROMPT=$(cat .github/prompts/generate_prompt.txt || echo "Generate comprehensive markdown documentation based on the extracted information.")
148+
EXTRACTED_CONTENT=$(cat extracted_info.md)
149+
150+
# Use OpenAI Completion API again
151+
PAYLOAD=$(jq -n --arg prompt "System: $SYSTEM_PROMPT\n\nUser: $EXTRACTED_CONTENT" '{
152+
model: "gpt-4-turbo-preview",
153+
prompt: $prompt,
154+
max_tokens: 4000,
155+
temperature: 0.3
156+
}')
157+
158+
curl -s https://api.openai.com/v1/completions \
159+
-H "Authorization: Bearer ${{ secrets.OPENAI_API_KEY }}" \
160+
-H "Content-Type: application/json" \
161+
-d "$PAYLOAD" | jq -r '.choices[0].text' > "generated_doc.md"
162+
163+
# Prepare final path
164+
INTEGRATION=$(echo "$FILE_NAME" | sed 's/_uqi_config\.json//' | tr '[:upper:]' '[:lower:]')
165+
FINAL_PATH="website/docs/connect-data/reference/${INTEGRATION}.md"
166+
mkdir -p "$(dirname "$FINAL_PATH")"
167+
168+
# Copy to final location
169+
cp "generated_doc.md" "$FINAL_PATH"
170+
171+
# Also save to our generated_docs directory for the PR
172+
cp "generated_doc.md" "generated_docs/${INTEGRATION}.md"
173+
174+
# Mark as processed if it's a new file
175+
if ! grep -q "^$FILE_NAME$" scripts/processed_files.txt; then
176+
echo "$FILE_NAME" >> scripts/processed_files.txt
177+
fi
178+
179+
echo "✅ Completed processing: $FILE_NAME"
180+
done < files_to_process.txt
181+
182+
# Save updated hashes
183+
echo "$HASHES_JSON" > scripts/file_hashes.json
184+
185+
- name: Commit and open PR
186+
if: env.files_found == 'true'
187+
uses: peter-evans/create-pull-request@v5
188+
with:
189+
token: ${{ secrets.REPO_ACCESS_TOKEN }}
190+
title: "docs: add/update integration references"
191+
commit-message: "docs: add/update integration references"
192+
branch: "auto/docs-update-${{ github.run_id }}"
193+
base: main
194+
add-paths: |
195+
website/docs/connect-data/reference/
196+
scripts/processed_files.txt
197+
scripts/file_hashes.json
198+
body: |
199+
This PR adds or updates integration reference documentation for **${{ env.file_count }}** integrations.
200+
201+
Generated from the latest configuration files in the [integration-resources repository](https://github.com/appsmithorg/integration-resources/tree/main/Generic%20UQI%20Creation/uqi_configs).

‎.github/workflows/poll-and-generate-doc.yml‎

Lines changed: 0 additions & 127 deletions
This file was deleted.

0 commit comments

Comments
 (0)