docs(vrl): Automatically generate VRL function documentation

.github/actions/install-vdev/action.yml

@@ -20,7 +20,11 @@ runs:
       uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
       with:
         path: ~/.cargo/bin/vdev
-        key: ${{ runner.os }}-vdev-${{ hashFiles('vdev/**', 'Cargo.toml', 'Cargo.lock') }}
+        # ** WARNING - IMPORTANT **
+        # If updating the key, make sure to also update other usages of the key in this file
+        # and also in .github/actions/setup/action.yml
+        # ** WARNING - IMPORTANT **
+        key: ${{ runner.os }}-vdev-${{ hashFiles('vdev/**', 'lib/vector-vrl/**', 'Cargo.toml', 'Cargo.lock') }}
         restore-keys: |
           ${{ runner.os }}-vdev-
 
@@ -36,7 +40,11 @@ runs:
       uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
       with:
         path: ~/.cargo/bin/vdev
-        key: ${{ runner.os }}-vdev-${{ hashFiles('vdev/**', 'Cargo.toml', 'Cargo.lock') }}
+        # ** WARNING - IMPORTANT **
+        # If updating the key, make sure to also update other usages of the key in this file
+        # and also in .github/actions/setup/action.yml
+        # ** WARNING - IMPORTANT **
+        key: ${{ runner.os }}-vdev-${{ hashFiles('vdev/**', 'lib/vector-vrl/**', 'Cargo.toml', 'Cargo.lock') }}
 
     - name: Set VDEV environment variable
       shell: bash

.github/actions/setup/action.yml

@@ -90,7 +90,11 @@ runs:
       uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
       with:
         path: ~/.cargo/bin/vdev
-        key: ${{ runner.os }}-vdev-${{ hashFiles('vdev/**', 'Cargo.toml', 'Cargo.lock') }}
+        # ** WARNING - IMPORTANT **
+        # If updating the key, make sure to also update other usages of the key in this file
+        # and also in .github/actions/install-vdev/action.yml
+        # ** WARNING - IMPORTANT **
+        key: ${{ runner.os }}-vdev-${{ hashFiles('vdev/**', 'lib/vector-vrl/**', 'Cargo.toml', 'Cargo.lock') }}
         restore-keys: |
           ${{ runner.os }}-vdev-
         lookup-only: true
@@ -211,7 +215,7 @@ runs:
         EOF
 
     - name: Install protoc
-      if: ${{ inputs.protoc == 'true' }}
+      if: ${{ inputs.protoc == 'true' || env.VDEV_NEEDS_COMPILE == 'true' }}
       shell: bash
       run: |
         echo "Installing protoc"

.github/workflows/changes.yml

@@ -227,7 +227,11 @@ jobs:
           component_docs:
             - 'scripts/generate-component-docs.rb'
             - "vdev/**"
-            - 'website/cue/**/base/**.cue'
+            - 'website/cue/**/*.cue'
+            - 'docs/generated/**'
+            # If changes to the VRL sha is made the combined generated cue file will change which
+            # may cause issues
+            - 'Cargo.lock'
             - ".github/workflows/changes.yml"
           markdown:
             - '**/**.md'

.github/workflows/check_generated_vrl_docs.yml

@@ -0,0 +1,135 @@
+name: Check Generated VRL Docs Freshness
+
+on:
+  pull_request:
+  merge_group:
+    types: [checks_requested]
+  push:
+    branches:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
+env:
+  COMMIT_MESSAGE: "Update generated VRL docs"
+  COMMIT_AUTHOR: "github-actions[bot]"
+
+permissions:
+  contents: read
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      docs: ${{ steps.filter.outputs.docs }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            docs:
+              - "lib/vector-vrl/**"
+              - "vdev/**"
+              - "Cargo.lock"
+              - ".github/workflows/check_generated_vrl_docs.yml"
+
+  run-check-generated-vrl-docs:
+    needs: changes
+    if: needs.changes.outputs.docs == 'true'
+    runs-on: ubuntu-24.04-8core
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - uses: ./.github/actions/setup
+        with:
+          vdev: true
+          mold: false
+          cargo-cache: false
+
+      - name: Regenerate VRL docs
+        run: make generate-vector-vrl-docs
+
+      - name: Check for changes
+        id: check
+        run: |
+          git add docs/generated/
+          if git diff --cached --quiet; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Check last commit
+        if: steps.check.outputs.changed == 'true'
+        id: last-commit
+        run: |
+          MSG=$(git log -1 --pretty=%s)
+          AUTHOR=$(git log -1 --pretty=%an)
+          if [ "$MSG" = "$COMMIT_MESSAGE" ] && [ "$AUTHOR" = "$COMMIT_AUTHOR" ]; then
+            echo "is-auto=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "is-auto=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Commit and push
+        if: >
+          steps.check.outputs.changed == 'true'
+          && steps.last-commit.outputs.is-auto != 'true'
+          && github.event_name == 'pull_request'
+          && github.event.pull_request.head.repo.full_name == github.repository
+        id: push
+        continue-on-error: true
+        env:
+          HEAD_REF: ${{ github.head_ref }}
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git commit -m "$COMMIT_MESSAGE"
+          git push origin HEAD:refs/heads/$HEAD_REF
+
+      - name: Save PR number for comment workflow
+        if: >
+          steps.check.outputs.changed == 'true'
+          && steps.last-commit.outputs.is-auto != 'true'
+          && github.event_name == 'pull_request'
+          && steps.push.outcome != 'success'
+        run: |
+          mkdir -p /tmp/docs-check
+          echo "${{ github.event.pull_request.number }}" > /tmp/docs-check/pr-number
+
+      - name: Upload PR metadata
+        if: >
+          steps.check.outputs.changed == 'true'
+          && steps.last-commit.outputs.is-auto != 'true'
+          && github.event_name == 'pull_request'
+          && steps.push.outcome != 'success'
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: vrl-docs-check-pr
+          path: /tmp/docs-check/pr-number
+
+      - name: Fail if docs are out of date
+        if: steps.check.outputs.changed == 'true'
+        run: |
+          echo "docs/generated/ is out of date. Regenerate with: make generate-vector-vrl-docs"
+          exit 1
+
+  check-generated-vrl-docs:
+    if: always()
+    runs-on: ubuntu-latest
+    needs: run-check-generated-vrl-docs
+    steps:
+      - run: |
+          if [[ "${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
+            echo "One or more jobs failed or were cancelled"
+            exit 1
+          fi

.github/workflows/check_generated_vrl_docs_comment.yml

@@ -0,0 +1,62 @@
+name: Comment on PR (Generated VRL Docs)
+
+on:
+  workflow_run:
+    workflows: ["Check Generated VRL Docs Freshness"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.conclusion == 'failure'
+      && github.event.workflow_run.event == 'pull_request'
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Download PR metadata
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          name: vrl-docs-check-pr
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+
+      - name: Comment on PR
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          script: |
+            const fs = require('fs');
+            const prNumber = parseInt(fs.readFileSync('pr-number', 'utf8').trim(), 10);
+            if (isNaN(prNumber)) {
+              core.setFailed('Invalid PR number');
+              return;
+            }
+
+            const marker = '<!-- generated-vrl-docs-check -->';
+            const body = marker + '\nThe `docs/generated/` folder is out of date but I was unable to push the fix automatically.\n\nPlease run the following and commit the result:\n\n```\nmake generate-vector-vrl-docs\n```';
+
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+            });
+            const existing = comments.find(c => c.body.startsWith(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body,
+              });
+            }

.github/workflows/test.yml

@@ -150,8 +150,8 @@ jobs:
           markdownlint: true
       - run: make check-markdown
 
-  check-component-docs:
-    name: Check Component Docs
+  check-generated-docs:
+    name: Check Generated Docs
     runs-on: ubuntu-24.04-8core
     if: ${{ needs.changes.outputs.source == 'true' || needs.changes.outputs.component_docs == 'true' || needs.changes.outputs.test-yml == 'true' }}
     needs: changes
@@ -163,7 +163,7 @@ jobs:
           protoc: true
           cue: true
           libsasl2: true
-      - run: make check-component-docs
+      - run: make check-generated-docs
 
   check-rust-docs:
     name: Check Rust Docs
@@ -221,7 +221,7 @@ jobs:
       - check-licenses
       - check-docs
       - check-markdown
-      - check-component-docs
+      - check-generated-docs
       - check-rust-docs
       - test-vrl
       - build-vrl-playground

CONTRIBUTING.md

@@ -141,7 +141,7 @@ echo "Running pre-push checks..."
 make check-licenses
 make check-fmt
 make check-clippy
-make check-component-docs
+make check-generated-docs
 
 # Some other checks that in our experience rarely fail on PRs.
 make check-deny
@@ -292,7 +292,7 @@ cargo vdev check events
 cargo vdev check licenses
 # Vector's documentation for each component is generated from the comments attached to the Component structs and members.
 # Running this ensures that the generated docs are up to date.
-make check-component-docs
+make check-generated-docs
 # Generate the code documentation for the Vector project.
 # Run this to ensure the docs can be generated without errors (warnings are acceptable at the minute).
 cd rust-doc && make docs

LICENSE-3rdparty.csv

@@ -635,6 +635,7 @@ regex-automata,https://github.com/rust-lang/regex/tree/master/regex-automata,MIT
 regex-filtered,https://github.com/ua-parser/uap-rust,BSD-3-Clause,The regex-filtered Authors
 regex-lite,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant <jamslam@gmail.com>"
 regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant <jamslam@gmail.com>"
+relative-path,https://github.com/udoprog/relative-path,MIT OR Apache-2.0,John-John Tedro <udoprog@tedro.se>
 rend,https://github.com/djkoloski/rend,MIT,David Koloski <djkoloski@gmail.com>
 reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur <sean@seanmonstar.com>
 reqwest-middleware,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski <rodrigo.gryzinski@truelayer.com>