Skip to content

Commit 6372de7

Browse files
MeelahMeMeelahMe
authored
Merge pull request #6527 from influxdata/jmercer/complete-plugin-sync-fix
Implements end-to-end automation for syncing plugin docs from influxdb3_plugins to docs-v2 with validation, transformation, and automated PR creation.
2 parents f084fe3 + 0e66ac6 commit 6372de7

File tree

3 files changed

+179
-28
lines changed

3 files changed

+179
-28
lines changed

‎.github/workflows/sync-plugins.yml‎

Lines changed: 103 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -80,11 +80,15 @@ jobs:
8080
token: ${{ secrets.GITHUB_TOKEN }}
8181

8282
- name: Checkout influxdb3_plugins (sparse)
83-
run: |
84-
git clone --filter=blob:none --sparse https://github.com/influxdata/influxdb3_plugins.git .ext/influxdb3_plugins
85-
cd .ext/influxdb3_plugins
86-
git sparse-checkout set influxdata/ scripts/
87-
git checkout ${{ steps.inputs.outputs.source_commit }}
83+
uses: actions/checkout@v4
84+
with:
85+
repository: influxdata/influxdb3_plugins
86+
token: ${{ secrets.PLUGINS_CONTENT_READ_TOKEN }}
87+
path: .ext/influxdb3_plugins
88+
sparse-checkout: |
89+
influxdata/
90+
scripts/
91+
ref: ${{ steps.inputs.outputs.source_commit }}
8892

8993
- name: Setup Node.js
9094
uses: actions/setup-node@v4
@@ -172,24 +176,109 @@ jobs:
172176
173177
core.setFailed('Plugin validation failed');
174178
179+
- name: Debug - Check file structure
180+
if: steps.validate.outputs.validation_passed == 'true'
181+
run: |
182+
echo "======================================"
183+
echo "DEBUG: Checking file structure"
184+
echo "======================================"
185+
echo ""
186+
187+
echo "1. Current working directory:"
188+
pwd
189+
echo ""
190+
191+
echo "2. List workspace root:"
192+
ls -la
193+
echo ""
194+
195+
echo "3. Check if .ext exists:"
196+
if [ -d ".ext" ]; then
197+
echo "✅ .ext directory exists"
198+
ls -la .ext/
199+
else
200+
echo "❌ .ext directory NOT found"
201+
fi
202+
echo ""
203+
204+
echo "4. Check if .ext/influxdb3_plugins exists:"
205+
if [ -d ".ext/influxdb3_plugins" ]; then
206+
echo "✅ .ext/influxdb3_plugins exists"
207+
ls -la .ext/influxdb3_plugins/
208+
else
209+
echo "❌ .ext/influxdb3_plugins NOT found"
210+
fi
211+
echo ""
212+
213+
echo "5. Check if influxdata directory exists:"
214+
if [ -d ".ext/influxdb3_plugins/influxdata" ]; then
215+
echo "✅ influxdata directory exists"
216+
ls -la .ext/influxdb3_plugins/influxdata/
217+
else
218+
echo "❌ influxdata directory NOT found"
219+
fi
220+
echo ""
221+
222+
echo "6. Check if basic_transformation exists:"
223+
if [ -d ".ext/influxdb3_plugins/influxdata/basic_transformation" ]; then
224+
echo "✅ basic_transformation directory exists"
225+
ls -la .ext/influxdb3_plugins/influxdata/basic_transformation/
226+
else
227+
echo "❌ basic_transformation directory NOT found"
228+
fi
229+
echo ""
230+
231+
echo "7. Now checking from script directory:"
232+
cd docs-v2/helper-scripts/influxdb3-plugins
233+
echo "Current directory:"
234+
pwd
235+
echo ""
236+
237+
echo "8. Trying to access with ../../../.ext/influxdb3_plugins:"
238+
if [ -d "../../../.ext/influxdb3_plugins" ]; then
239+
echo "✅ Can access via ../../../.ext/influxdb3_plugins"
240+
ls -la ../../../.ext/influxdb3_plugins/
241+
else
242+
echo "❌ Cannot access via ../../../.ext/influxdb3_plugins"
243+
fi
244+
echo ""
245+
246+
echo "9. Checking all possible paths from script dir:"
247+
echo "Checking ../.ext/influxdb3_plugins:"
248+
ls -la ../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
249+
echo ""
250+
echo "Checking ../../.ext/influxdb3_plugins:"
251+
ls -la ../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
252+
echo ""
253+
echo "Checking ../../../.ext/influxdb3_plugins:"
254+
ls -la ../../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
255+
echo ""
256+
echo "Checking ../../../../.ext/influxdb3_plugins:"
257+
ls -la ../../../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
258+
echo ""
259+
260+
echo "======================================"
261+
echo "DEBUG: Complete"
262+
echo "======================================"
263+
175264
- name: Transform plugin documentation
176265
if: steps.validate.outputs.validation_passed == 'true'
177266
run: |
178-
cd docs-v2
179-
180-
# Set PLUGIN_DIR for the transformation script
181-
export INFLUXDB3_PLUGINS_PATH=".ext/influxdb3_plugins"
182-
267+
cd docs-v2/helper-scripts/influxdb3-plugins
268+
269+
# Set path to plugins repo (relative from script directory)
270+
export INFLUXDB3_PLUGINS_PATH="../../../.ext/influxdb3_plugins"
271+
183272
# Run the transformation
184273
if [[ "${{ steps.inputs.outputs.plugins }}" == "all" ]]; then
185-
node helper-scripts/influxdb3-plugins/port_to_docs.js
274+
node port_to_docs.js # ← CHANGE: Script is in current dir now
186275
else
187276
# Transform specific plugins
188277
IFS=',' read -ra PLUGIN_ARRAY <<< "${{ steps.inputs.outputs.plugins }}"
189278
for plugin in "${PLUGIN_ARRAY[@]}"; do
190279
plugin=$(echo "$plugin" | xargs) # trim whitespace
191280
echo "Transforming plugin: $plugin"
192-
node helper-scripts/influxdb3-plugins/port_to_docs.js --plugin "$plugin"
281+
node port_to_docs.js --plugin "$plugin"
193282
done
194283
fi
195284
@@ -201,9 +290,11 @@ jobs:
201290
- name: Setup Playwright
202291
run: |
203292
cd docs-v2
293+
npm install playwright
204294
npx playwright install chromium
205295
206296
- name: Generate screenshots
297+
if: false # Disable screenshots for MVP
207298
id: screenshots
208299
run: |
209300
cd docs-v2

‎helper-scripts/influxdb3-plugins/docs_mapping.yaml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,79 +3,79 @@
33

44
plugins:
55
basic_transformation:
6-
source: ../../.ext/influxdb3_plugins/influxdata/basic_transformation/README.md
6+
source: ../../../.ext/influxdb3_plugins/influxdata/basic_transformation/README.md
77
target: ../../content/shared/influxdb3-plugins/plugins-library/official/basic-transformation.md
88
category: official
99
additional_sections:
1010
- schema_requirements
1111
- logging
1212

1313
downsampler:
14-
source: ../../.ext/influxdb3_plugins/influxdata/downsampler/README.md
14+
source: ../../../.ext/influxdb3_plugins/influxdata/downsampler/README.md
1515
target: ../../content/shared/influxdb3-plugins/plugins-library/official/downsampler.md
1616
category: official
1717
additional_sections:
1818
- schema_management
1919
- logging
2020

2121
forecast_error_evaluator:
22-
source: ../../.ext/influxdb3_plugins/influxdata/forecast_error_evaluator/README.md
22+
source: ../../../.ext/influxdb3_plugins/influxdata/forecast_error_evaluator/README.md
2323
target: ../../content/shared/influxdb3-plugins/plugins-library/official/forecast-error-evaluator.md
2424
category: official
2525
additional_sections:
2626
- logging
2727

2828
influxdb_to_iceberg:
29-
source: ../../.ext/influxdb3_plugins/influxdata/influxdb_to_iceberg/README.md
29+
source: ../../../.ext/influxdb3_plugins/influxdata/influxdb_to_iceberg/README.md
3030
target: ../../content/shared/influxdb3-plugins/plugins-library/official/influxdb-to-iceberg.md
3131
category: official
3232
additional_sections:
3333
- logging
3434

3535
mad_check:
36-
source: ../../.ext/influxdb3_plugins/influxdata/mad_check/README.md
36+
source: ../../../.ext/influxdb3_plugins/influxdata/mad_check/README.md
3737
target: ../../content/shared/influxdb3-plugins/plugins-library/official/mad-check.md
3838
category: official
3939
additional_sections:
4040
- logging
4141

4242
notifier:
43-
source: ../../.ext/influxdb3_plugins/influxdata/notifier/README.md
43+
source: ../../../.ext/influxdb3_plugins/influxdata/notifier/README.md
4444
target: ../../content/shared/influxdb3-plugins/plugins-library/official/notifier.md
4545
category: official
4646
additional_sections:
4747
- logging
4848

4949
prophet_forecasting:
50-
source: ../../.ext/influxdb3_plugins/influxdata/prophet_forecasting/README.md
50+
source: ../../../.ext/influxdb3_plugins/influxdata/prophet_forecasting/README.md
5151
target: ../../content/shared/influxdb3-plugins/plugins-library/official/prophet-forecasting.md
5252
category: official
5353
additional_sections:
5454
- logging
5555

5656
state_change:
57-
source: ../../.ext/influxdb3_plugins/influxdata/state_change/README.md
57+
source: ../../../.ext/influxdb3_plugins/influxdata/state_change/README.md
5858
target: ../../content/shared/influxdb3-plugins/plugins-library/official/state-change.md
5959
category: official
6060
additional_sections:
6161
- logging
6262

6363
stateless_adtk_detector:
64-
source: ../../.ext/influxdb3_plugins/influxdata/stateless_adtk_detector/README.md
64+
source: ../../../.ext/influxdb3_plugins/influxdata/stateless_adtk_detector/README.md
6565
target: ../../content/shared/influxdb3-plugins/plugins-library/official/stateless-adtk-detector.md
6666
category: official
6767
additional_sections:
6868
- logging
6969

7070
system_metrics:
71-
source: ../../.ext/influxdb3_plugins/influxdata/system_metrics/README.md
71+
source: ../../../.ext/influxdb3_plugins/influxdata/system_metrics/README.md
7272
target: ../../content/shared/influxdb3-plugins/plugins-library/official/system-metrics.md
7373
category: official
7474
additional_sections:
7575
- logging
7676

7777
threshold_deadman_checks:
78-
source: ../../.ext/influxdb3_plugins/influxdata/threshold_deadman_checks/README.md
78+
source: ../../../.ext/influxdb3_plugins/influxdata/threshold_deadman_checks/README.md
7979
target: ../../content/shared/influxdb3-plugins/plugins-library/official/threshold-deadman-checks.md
8080
category: official
8181
additional_sections:

‎helper-scripts/influxdb3-plugins/port_to_docs.js‎

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,28 +34,84 @@ function removeEmojiMetadata(content) {
3434
return content.replace(pattern, '');
3535
}
3636

37+
/**
38+
* Remove level 1 heading from content.
39+
*/
40+
function removeTitleHeading(content) {
41+
// Title is in frontmatter, remove H1 from content
42+
return content.replace(/^#\s+.+$\n*/m, '');
43+
}
44+
45+
/**
46+
* Remove standalone Description heading.
47+
*/
48+
function removeDescriptionHeading(content) {
49+
// Remove "## Description" heading when it appears alone on a line
50+
content = content.replace(/^##\s+Description\s*$/m, '');
51+
return content;
52+
}
53+
54+
/**
55+
* Expand common abbreviations for readability.
56+
*/
57+
function expandAbbreviations(content) {
58+
// Replace e.g., with "for example,"
59+
content = content.replace(/\be\.g\.,\s*/g, 'for example, ');
60+
// Replace i.e., with "that is,"
61+
content = content.replace(/\bi\.e\.,\s*/g, 'that is, ');
62+
return content;
63+
}
64+
65+
/**
66+
* Convert README TOML links to internal section links.
67+
*/
68+
function convertTomlReadmeLinks(content) {
69+
// If document has TOML configuration section, link to it instead of external README
70+
if (content.includes('## Using TOML Configuration Files')) {
71+
content = content.replace(
72+
/\[([^\]]*TOML[^\]]*)\]\(https:\/\/github\.com\/influxdata\/influxdb3_plugins\/blob\/master\/README\.md\)/gi,
73+
'[$1](#using-toml-configuration-files)'
74+
);
75+
}
76+
return content;
77+
}
78+
3779
/**
3880
* Convert relative links to GitHub URLs.
3981
*/
4082
function convertRelativeLinks(content, pluginName) {
4183
const baseUrl = `https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/${pluginName}/`;
84+
const rootUrl =
85+
'https://github.com/influxdata/influxdb3_plugins/blob/master/';
86+
87+
// Convert relative README links (../../README.md, ../README.md, etc.)
88+
content = content.replace(
89+
/\[([^\]]+)\]\((\.\.\/)+README\.md\)/g,
90+
`[$1](${rootUrl}README.md)`
91+
);
4292

4393
// Convert TOML file links
4494
content = content.replace(
45-
/\[([^\]]+\.toml)\]\(([^)]+\.toml)\)/g,
46-
(match, linkText, linkPath) => `[${linkText}](${baseUrl}${linkPath})`
95+
/\[([^\]]+\.toml)\]\(\.?\/?([^)]+\.toml)\)/g,
96+
(match, linkText, linkPath) => {
97+
const cleanPath = linkPath.replace(/^\.\//, '');
98+
return `[${linkText}](${baseUrl}${cleanPath})`;
99+
}
47100
);
48101

49102
// Convert Python file links
50103
content = content.replace(
51-
/\[([^\]]+\.py)\]\(([^)]+\.py)\)/g,
52-
(match, linkText, linkPath) => `[${linkText}](${baseUrl}${linkPath})`
104+
/\[([^\]]+\.py)\]\(\.?\/?([^)]+\.py)\)/g,
105+
(match, linkText, linkPath) => {
106+
const cleanPath = linkPath.replace(/^\.\//, '');
107+
return `[${linkText}](${baseUrl}${cleanPath})`;
108+
}
53109
);
54110

55111
// Convert main README reference
56112
content = content.replace(
57113
'[influxdb3_plugins/README.md](/README.md)',
58-
'[influxdb3_plugins/README.md](https://github.com/influxdata/influxdb3_plugins/blob/master/README.md)'
114+
`[influxdb3_plugins/README.md](${rootUrl}README.md)`
59115
);
60116

61117
return content;
@@ -214,7 +270,11 @@ Each downsampled record includes three additional metadata columns:
214270
function transformContent(content, pluginName, config) {
215271
// Apply transformations in order
216272
content = removeEmojiMetadata(content);
273+
content = removeTitleHeading(content);
274+
content = removeDescriptionHeading(content);
217275
content = convertRelativeLinks(content, pluginName);
276+
content = expandAbbreviations(content);
277+
content = convertTomlReadmeLinks(content);
218278
content = addProductShortcodes(content);
219279
content = enhanceOpeningParagraph(content);
220280
content = fixCodeBlockFormatting(content);

0 commit comments

Comments
 (0)