diff --git a/.github/workflows/sync-plugins.yml b/.github/workflows/sync-plugins.yml index d850122ee7..47de19a377 100644 --- a/.github/workflows/sync-plugins.yml +++ b/.github/workflows/sync-plugins.yml @@ -80,11 +80,15 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout influxdb3_plugins (sparse) - run: | - git clone --filter=blob:none --sparse https://github.com/influxdata/influxdb3_plugins.git .ext/influxdb3_plugins - cd .ext/influxdb3_plugins - git sparse-checkout set influxdata/ scripts/ - git checkout ${{ steps.inputs.outputs.source_commit }} + uses: actions/checkout@v4 + with: + repository: influxdata/influxdb3_plugins + token: ${{ secrets.PLUGINS_CONTENT_READ_TOKEN }} + path: .ext/influxdb3_plugins + sparse-checkout: | + influxdata/ + scripts/ + ref: ${{ steps.inputs.outputs.source_commit }} - name: Setup Node.js uses: actions/setup-node@v4 @@ -172,24 +176,109 @@ jobs: core.setFailed('Plugin validation failed'); + - name: Debug - Check file structure + if: steps.validate.outputs.validation_passed == 'true' + run: | + echo "======================================" + echo "DEBUG: Checking file structure" + echo "======================================" + echo "" + + echo "1. Current working directory:" + pwd + echo "" + + echo "2. List workspace root:" + ls -la + echo "" + + echo "3. Check if .ext exists:" + if [ -d ".ext" ]; then + echo "✅ .ext directory exists" + ls -la .ext/ + else + echo "❌ .ext directory NOT found" + fi + echo "" + + echo "4. Check if .ext/influxdb3_plugins exists:" + if [ -d ".ext/influxdb3_plugins" ]; then + echo "✅ .ext/influxdb3_plugins exists" + ls -la .ext/influxdb3_plugins/ + else + echo "❌ .ext/influxdb3_plugins NOT found" + fi + echo "" + + echo "5. Check if influxdata directory exists:" + if [ -d ".ext/influxdb3_plugins/influxdata" ]; then + echo "✅ influxdata directory exists" + ls -la .ext/influxdb3_plugins/influxdata/ + else + echo "❌ influxdata directory NOT found" + fi + echo "" + + echo "6. Check if basic_transformation exists:" + if [ -d ".ext/influxdb3_plugins/influxdata/basic_transformation" ]; then + echo "✅ basic_transformation directory exists" + ls -la .ext/influxdb3_plugins/influxdata/basic_transformation/ + else + echo "❌ basic_transformation directory NOT found" + fi + echo "" + + echo "7. Now checking from script directory:" + cd docs-v2/helper-scripts/influxdb3-plugins + echo "Current directory:" + pwd + echo "" + + echo "8. Trying to access with ../../../.ext/influxdb3_plugins:" + if [ -d "../../../.ext/influxdb3_plugins" ]; then + echo "✅ Can access via ../../../.ext/influxdb3_plugins" + ls -la ../../../.ext/influxdb3_plugins/ + else + echo "❌ Cannot access via ../../../.ext/influxdb3_plugins" + fi + echo "" + + echo "9. Checking all possible paths from script dir:" + echo "Checking ../.ext/influxdb3_plugins:" + ls -la ../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found" + echo "" + echo "Checking ../../.ext/influxdb3_plugins:" + ls -la ../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found" + echo "" + echo "Checking ../../../.ext/influxdb3_plugins:" + ls -la ../../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found" + echo "" + echo "Checking ../../../../.ext/influxdb3_plugins:" + ls -la ../../../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found" + echo "" + + echo "======================================" + echo "DEBUG: Complete" + echo "======================================" + - name: Transform plugin documentation if: steps.validate.outputs.validation_passed == 'true' run: | - cd docs-v2 - - # Set PLUGIN_DIR for the transformation script - export INFLUXDB3_PLUGINS_PATH=".ext/influxdb3_plugins" - + cd docs-v2/helper-scripts/influxdb3-plugins + + # Set path to plugins repo (relative from script directory) + export INFLUXDB3_PLUGINS_PATH="../../../.ext/influxdb3_plugins" + # Run the transformation if [[ "${{ steps.inputs.outputs.plugins }}" == "all" ]]; then - node helper-scripts/influxdb3-plugins/port_to_docs.js + node port_to_docs.js # ← CHANGE: Script is in current dir now else # Transform specific plugins IFS=',' read -ra PLUGIN_ARRAY <<< "${{ steps.inputs.outputs.plugins }}" for plugin in "${PLUGIN_ARRAY[@]}"; do plugin=$(echo "$plugin" | xargs) # trim whitespace echo "Transforming plugin: $plugin" - node helper-scripts/influxdb3-plugins/port_to_docs.js --plugin "$plugin" + node port_to_docs.js --plugin "$plugin" done fi @@ -201,9 +290,11 @@ jobs: - name: Setup Playwright run: | cd docs-v2 + npm install playwright npx playwright install chromium - name: Generate screenshots + if: false # Disable screenshots for MVP id: screenshots run: | cd docs-v2 diff --git a/helper-scripts/influxdb3-plugins/docs_mapping.yaml b/helper-scripts/influxdb3-plugins/docs_mapping.yaml index 35c0447b0a..3cb89e22bb 100644 --- a/helper-scripts/influxdb3-plugins/docs_mapping.yaml +++ b/helper-scripts/influxdb3-plugins/docs_mapping.yaml @@ -3,7 +3,7 @@ plugins: basic_transformation: - source: ../../.ext/influxdb3_plugins/influxdata/basic_transformation/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/basic_transformation/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/basic-transformation.md category: official additional_sections: @@ -11,7 +11,7 @@ plugins: - logging downsampler: - source: ../../.ext/influxdb3_plugins/influxdata/downsampler/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/downsampler/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/downsampler.md category: official additional_sections: @@ -19,63 +19,63 @@ plugins: - logging forecast_error_evaluator: - source: ../../.ext/influxdb3_plugins/influxdata/forecast_error_evaluator/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/forecast_error_evaluator/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/forecast-error-evaluator.md category: official additional_sections: - logging influxdb_to_iceberg: - source: ../../.ext/influxdb3_plugins/influxdata/influxdb_to_iceberg/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/influxdb_to_iceberg/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/influxdb-to-iceberg.md category: official additional_sections: - logging mad_check: - source: ../../.ext/influxdb3_plugins/influxdata/mad_check/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/mad_check/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/mad-check.md category: official additional_sections: - logging notifier: - source: ../../.ext/influxdb3_plugins/influxdata/notifier/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/notifier/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/notifier.md category: official additional_sections: - logging prophet_forecasting: - source: ../../.ext/influxdb3_plugins/influxdata/prophet_forecasting/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/prophet_forecasting/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/prophet-forecasting.md category: official additional_sections: - logging state_change: - source: ../../.ext/influxdb3_plugins/influxdata/state_change/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/state_change/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/state-change.md category: official additional_sections: - logging stateless_adtk_detector: - source: ../../.ext/influxdb3_plugins/influxdata/stateless_adtk_detector/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/stateless_adtk_detector/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/stateless-adtk-detector.md category: official additional_sections: - logging system_metrics: - source: ../../.ext/influxdb3_plugins/influxdata/system_metrics/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/system_metrics/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/system-metrics.md category: official additional_sections: - logging threshold_deadman_checks: - source: ../../.ext/influxdb3_plugins/influxdata/threshold_deadman_checks/README.md + source: ../../../.ext/influxdb3_plugins/influxdata/threshold_deadman_checks/README.md target: ../../content/shared/influxdb3-plugins/plugins-library/official/threshold-deadman-checks.md category: official additional_sections: diff --git a/helper-scripts/influxdb3-plugins/port_to_docs.js b/helper-scripts/influxdb3-plugins/port_to_docs.js index 590b45b0f2..324bf2bd83 100644 --- a/helper-scripts/influxdb3-plugins/port_to_docs.js +++ b/helper-scripts/influxdb3-plugins/port_to_docs.js @@ -34,28 +34,84 @@ function removeEmojiMetadata(content) { return content.replace(pattern, ''); } +/** + * Remove level 1 heading from content. + */ +function removeTitleHeading(content) { + // Title is in frontmatter, remove H1 from content + return content.replace(/^#\s+.+$\n*/m, ''); +} + +/** + * Remove standalone Description heading. + */ +function removeDescriptionHeading(content) { + // Remove "## Description" heading when it appears alone on a line + content = content.replace(/^##\s+Description\s*$/m, ''); + return content; +} + +/** + * Expand common abbreviations for readability. + */ +function expandAbbreviations(content) { + // Replace e.g., with "for example," + content = content.replace(/\be\.g\.,\s*/g, 'for example, '); + // Replace i.e., with "that is," + content = content.replace(/\bi\.e\.,\s*/g, 'that is, '); + return content; +} + +/** + * Convert README TOML links to internal section links. + */ +function convertTomlReadmeLinks(content) { + // If document has TOML configuration section, link to it instead of external README + if (content.includes('## Using TOML Configuration Files')) { + content = content.replace( + /\[([^\]]*TOML[^\]]*)\]\(https:\/\/github\.com\/influxdata\/influxdb3_plugins\/blob\/master\/README\.md\)/gi, + '[$1](#using-toml-configuration-files)' + ); + } + return content; +} + /** * Convert relative links to GitHub URLs. */ function convertRelativeLinks(content, pluginName) { const baseUrl = `https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/${pluginName}/`; + const rootUrl = + 'https://github.com/influxdata/influxdb3_plugins/blob/master/'; + + // Convert relative README links (../../README.md, ../README.md, etc.) + content = content.replace( + /\[([^\]]+)\]\((\.\.\/)+README\.md\)/g, + `[$1](${rootUrl}README.md)` + ); // Convert TOML file links content = content.replace( - /\[([^\]]+\.toml)\]\(([^)]+\.toml)\)/g, - (match, linkText, linkPath) => `[${linkText}](${baseUrl}${linkPath})` + /\[([^\]]+\.toml)\]\(\.?\/?([^)]+\.toml)\)/g, + (match, linkText, linkPath) => { + const cleanPath = linkPath.replace(/^\.\//, ''); + return `[${linkText}](${baseUrl}${cleanPath})`; + } ); // Convert Python file links content = content.replace( - /\[([^\]]+\.py)\]\(([^)]+\.py)\)/g, - (match, linkText, linkPath) => `[${linkText}](${baseUrl}${linkPath})` + /\[([^\]]+\.py)\]\(\.?\/?([^)]+\.py)\)/g, + (match, linkText, linkPath) => { + const cleanPath = linkPath.replace(/^\.\//, ''); + return `[${linkText}](${baseUrl}${cleanPath})`; + } ); // Convert main README reference content = content.replace( '[influxdb3_plugins/README.md](/README.md)', - '[influxdb3_plugins/README.md](https://github.com/influxdata/influxdb3_plugins/blob/master/README.md)' + `[influxdb3_plugins/README.md](${rootUrl}README.md)` ); return content; @@ -214,7 +270,11 @@ Each downsampled record includes three additional metadata columns: function transformContent(content, pluginName, config) { // Apply transformations in order content = removeEmojiMetadata(content); + content = removeTitleHeading(content); + content = removeDescriptionHeading(content); content = convertRelativeLinks(content, pluginName); + content = expandAbbreviations(content); + content = convertTomlReadmeLinks(content); content = addProductShortcodes(content); content = enhanceOpeningParagraph(content); content = fixCodeBlockFormatting(content);