Jmercer/complete plugin sync fix

.github/workflows/sync-plugins.yml

@@ -80,11 +80,15 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Checkout influxdb3_plugins (sparse)
-        run: |
-          git clone --filter=blob:none --sparse https://github.com/influxdata/influxdb3_plugins.git .ext/influxdb3_plugins
-          cd .ext/influxdb3_plugins
-          git sparse-checkout set influxdata/ scripts/
-          git checkout ${{ steps.inputs.outputs.source_commit }}
+        uses: actions/checkout@v4
+        with:
+          repository: influxdata/influxdb3_plugins
+          token: ${{ secrets.PLUGINS_CONTENT_READ_TOKEN }}
+          path: .ext/influxdb3_plugins
+          sparse-checkout: |
+              influxdata/
+              scripts/
+          ref: ${{ steps.inputs.outputs.source_commit }}
 
       - name: Setup Node.js
         uses: actions/setup-node@v4
@@ -172,24 +176,109 @@ jobs:
 
             core.setFailed('Plugin validation failed');
 
+      - name: Debug - Check file structure
+        if: steps.validate.outputs.validation_passed == 'true'
+        run: |
+          echo "======================================"
+          echo "DEBUG: Checking file structure"
+          echo "======================================"
+          echo ""
+
+          echo "1. Current working directory:"
+          pwd
+          echo ""
+
+          echo "2. List workspace root:"
+          ls -la
+          echo ""
+
+          echo "3. Check if .ext exists:"
+          if [ -d ".ext" ]; then
+            echo "✅ .ext directory exists"
+            ls -la .ext/
+          else
+            echo "❌ .ext directory NOT found"
+          fi
+          echo ""
+
+          echo "4. Check if .ext/influxdb3_plugins exists:"
+          if [ -d ".ext/influxdb3_plugins" ]; then
+            echo "✅ .ext/influxdb3_plugins exists"
+            ls -la .ext/influxdb3_plugins/
+          else
+            echo "❌ .ext/influxdb3_plugins NOT found"
+          fi
+          echo ""
+
+          echo "5. Check if influxdata directory exists:"
+          if [ -d ".ext/influxdb3_plugins/influxdata" ]; then
+            echo "✅ influxdata directory exists"
+            ls -la .ext/influxdb3_plugins/influxdata/
+          else
+            echo "❌ influxdata directory NOT found"
+          fi
+          echo ""
+
+          echo "6. Check if basic_transformation exists:"
+          if [ -d ".ext/influxdb3_plugins/influxdata/basic_transformation" ]; then
+            echo "✅ basic_transformation directory exists"
+            ls -la .ext/influxdb3_plugins/influxdata/basic_transformation/
+          else
+            echo "❌ basic_transformation directory NOT found"
+          fi
+          echo ""
+
+          echo "7. Now checking from script directory:"
+          cd docs-v2/helper-scripts/influxdb3-plugins
+          echo "Current directory:"
+          pwd
+          echo ""
+
+          echo "8. Trying to access with ../../../.ext/influxdb3_plugins:"
+          if [ -d "../../../.ext/influxdb3_plugins" ]; then
+            echo "✅ Can access via ../../../.ext/influxdb3_plugins"
+            ls -la ../../../.ext/influxdb3_plugins/
+          else
+            echo "❌ Cannot access via ../../../.ext/influxdb3_plugins"
+          fi
+          echo ""
+
+          echo "9. Checking all possible paths from script dir:"
+          echo "Checking ../.ext/influxdb3_plugins:"
+          ls -la ../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
+          echo ""
+          echo "Checking ../../.ext/influxdb3_plugins:"
+          ls -la ../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
+          echo ""
+          echo "Checking ../../../.ext/influxdb3_plugins:"
+          ls -la ../../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
+          echo ""
+          echo "Checking ../../../../.ext/influxdb3_plugins:"
+          ls -la ../../../../.ext/influxdb3_plugins/ 2>&1 || echo "❌ Not found"
+          echo ""
+
+          echo "======================================"
+          echo "DEBUG: Complete"
+          echo "======================================"    
+
       - name: Transform plugin documentation
         if: steps.validate.outputs.validation_passed == 'true'
         run: |
-          cd docs-v2
-
-          # Set PLUGIN_DIR for the transformation script
-          export INFLUXDB3_PLUGINS_PATH=".ext/influxdb3_plugins"
-
+          cd docs-v2/helper-scripts/influxdb3-plugins  
+              
+          # Set path to plugins repo (relative from script directory)
+          export INFLUXDB3_PLUGINS_PATH="../../../.ext/influxdb3_plugins" 
+              
           # Run the transformation
           if [[ "${{ steps.inputs.outputs.plugins }}" == "all" ]]; then
-            node helper-scripts/influxdb3-plugins/port_to_docs.js
+            node port_to_docs.js  # ← CHANGE: Script is in current dir now
           else
             # Transform specific plugins
             IFS=',' read -ra PLUGIN_ARRAY <<< "${{ steps.inputs.outputs.plugins }}"
             for plugin in "${PLUGIN_ARRAY[@]}"; do
               plugin=$(echo "$plugin" | xargs)  # trim whitespace
               echo "Transforming plugin: $plugin"
-              node helper-scripts/influxdb3-plugins/port_to_docs.js --plugin "$plugin"
+              node port_to_docs.js --plugin "$plugin"  
             done
           fi
 
@@ -201,9 +290,11 @@ jobs:
       - name: Setup Playwright
         run: |
           cd docs-v2
+          npm install playwright
           npx playwright install chromium
 
       - name: Generate screenshots
+        if: false  # Disable screenshots for MVP
         id: screenshots
         run: |
           cd docs-v2

helper-scripts/influxdb3-plugins/docs_mapping.yaml

@@ -3,79 +3,79 @@
 
 plugins:
   basic_transformation:
-    source: ../../.ext/influxdb3_plugins/influxdata/basic_transformation/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/basic_transformation/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/basic-transformation.md
     category: official
     additional_sections:
       - schema_requirements
       - logging
 
   downsampler:
-    source: ../../.ext/influxdb3_plugins/influxdata/downsampler/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/downsampler/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/downsampler.md
     category: official
     additional_sections:
       - schema_management
       - logging
 
   forecast_error_evaluator:
-    source: ../../.ext/influxdb3_plugins/influxdata/forecast_error_evaluator/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/forecast_error_evaluator/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/forecast-error-evaluator.md
     category: official
     additional_sections:
       - logging
 
   influxdb_to_iceberg:
-    source: ../../.ext/influxdb3_plugins/influxdata/influxdb_to_iceberg/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/influxdb_to_iceberg/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/influxdb-to-iceberg.md
     category: official
     additional_sections:
       - logging
 
   mad_check:
-    source: ../../.ext/influxdb3_plugins/influxdata/mad_check/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/mad_check/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/mad-check.md
     category: official
     additional_sections:
       - logging
 
   notifier:
-    source: ../../.ext/influxdb3_plugins/influxdata/notifier/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/notifier/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/notifier.md
     category: official
     additional_sections:
       - logging
 
   prophet_forecasting:
-    source: ../../.ext/influxdb3_plugins/influxdata/prophet_forecasting/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/prophet_forecasting/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/prophet-forecasting.md
     category: official
     additional_sections:
       - logging
 
   state_change:
-    source: ../../.ext/influxdb3_plugins/influxdata/state_change/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/state_change/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/state-change.md
     category: official
     additional_sections:
       - logging
 
   stateless_adtk_detector:
-    source: ../../.ext/influxdb3_plugins/influxdata/stateless_adtk_detector/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/stateless_adtk_detector/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/stateless-adtk-detector.md
     category: official
     additional_sections:
       - logging
 
   system_metrics:
-    source: ../../.ext/influxdb3_plugins/influxdata/system_metrics/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/system_metrics/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/system-metrics.md
     category: official
     additional_sections:
       - logging
 
   threshold_deadman_checks:
-    source: ../../.ext/influxdb3_plugins/influxdata/threshold_deadman_checks/README.md
+    source: ../../../.ext/influxdb3_plugins/influxdata/threshold_deadman_checks/README.md
     target: ../../content/shared/influxdb3-plugins/plugins-library/official/threshold-deadman-checks.md
     category: official
     additional_sections:

helper-scripts/influxdb3-plugins/port_to_docs.js

@@ -34,28 +34,84 @@ function removeEmojiMetadata(content) {
   return content.replace(pattern, '');
 }
 
+/**
+ * Remove level 1 heading from content.
+ */
+function removeTitleHeading(content) {
+  // Title is in frontmatter, remove H1 from content
+  return content.replace(/^#\s+.+$\n*/m, '');
+}
+
+/**
+ * Remove standalone Description heading.
+ */
+function removeDescriptionHeading(content) {
+  // Remove "## Description" heading when it appears alone on a line
+  content = content.replace(/^##\s+Description\s*$/m, '');
+  return content;
+}
+
+/**
+ * Expand common abbreviations for readability.
+ */
+function expandAbbreviations(content) {
+  // Replace e.g., with "for example,"
+  content = content.replace(/\be\.g\.,\s*/g, 'for example, ');
+  // Replace i.e., with "that is,"
+  content = content.replace(/\bi\.e\.,\s*/g, 'that is, ');
+  return content;
+}
+
+/**
+ * Convert README TOML links to internal section links.
+ */
+function convertTomlReadmeLinks(content) {
+  // If document has TOML configuration section, link to it instead of external README
+  if (content.includes('## Using TOML Configuration Files')) {
+    content = content.replace(
+      /\[([^\]]*TOML[^\]]*)\]\(https:\/\/github\.com\/influxdata\/influxdb3_plugins\/blob\/master\/README\.md\)/gi,
+      '[$1](#using-toml-configuration-files)'
+    );
+  }
+  return content;
+}
+
 /**
  * Convert relative links to GitHub URLs.
  */
 function convertRelativeLinks(content, pluginName) {
   const baseUrl = `https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/${pluginName}/`;
+  const rootUrl =
+    'https://github.com/influxdata/influxdb3_plugins/blob/master/';
+
+  // Convert relative README links (../../README.md, ../README.md, etc.)
+  content = content.replace(
+    /\[([^\]]+)\]\((\.\.\/)+README\.md\)/g,
+    `[$1](${rootUrl}README.md)`
+  );
 
   // Convert TOML file links
   content = content.replace(
-    /\[([^\]]+\.toml)\]\(([^)]+\.toml)\)/g,
-    (match, linkText, linkPath) => `[${linkText}](${baseUrl}${linkPath})`
+    /\[([^\]]+\.toml)\]\(\.?\/?([^)]+\.toml)\)/g,
+    (match, linkText, linkPath) => {
+      const cleanPath = linkPath.replace(/^\.\//, '');
+      return `[${linkText}](${baseUrl}${cleanPath})`;
+    }
   );
 
   // Convert Python file links
   content = content.replace(
-    /\[([^\]]+\.py)\]\(([^)]+\.py)\)/g,
-    (match, linkText, linkPath) => `[${linkText}](${baseUrl}${linkPath})`
+    /\[([^\]]+\.py)\]\(\.?\/?([^)]+\.py)\)/g,
+    (match, linkText, linkPath) => {
+      const cleanPath = linkPath.replace(/^\.\//, '');
+      return `[${linkText}](${baseUrl}${cleanPath})`;
+    }
   );
 
   // Convert main README reference
   content = content.replace(
     '[influxdb3_plugins/README.md](/README.md)',
-    '[influxdb3_plugins/README.md](https://github.com/influxdata/influxdb3_plugins/blob/master/README.md)'
+    `[influxdb3_plugins/README.md](${rootUrl}README.md)`
   );
 
   return content;
@@ -214,7 +270,11 @@ Each downsampled record includes three additional metadata columns:
 function transformContent(content, pluginName, config) {
   // Apply transformations in order
   content = removeEmojiMetadata(content);
+  content = removeTitleHeading(content);
+  content = removeDescriptionHeading(content);
   content = convertRelativeLinks(content, pluginName);
+  content = expandAbbreviations(content);
+  content = convertTomlReadmeLinks(content);
   content = addProductShortcodes(content);
   content = enhanceOpeningParagraph(content);
   content = fixCodeBlockFormatting(content);