Merge branch 'main' into sentinel-lib

Author: davidcauchi

.github/workflows/docker-test.yaml

@@ -4,6 +4,8 @@ on:
     types: [submitted]
   pull_request:
     types: [labeled]
+    paths-ignore:
+      - 'tools/**'    
 
 jobs:
   eth_env:

.github/workflows/generate-go-docs.yaml

@@ -9,70 +9,215 @@ on:
 
 jobs:
   generate_docs_new_pr:
-    if: ${{ contains(github.event.issue.labels.*.name, 'generate_go_docs') }}
+    name: Generate Go Docs
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'generate_go_docs') }}
     runs-on: ubuntu-latest
+    environment: integration
     permissions:
+      id-token: write
       contents: read
-      pull-requests: write
-      repository-projects: read
 
     steps:
+      - name: Setup GitHub Token for reading Generate Go Doc Repo
+        id: setup-github-token-read
+        uses: smartcontractkit/.github/actions/setup-github-token@9e7cc0779934cae4a9028b8588c9adb64d8ce68c # [email protected]
+        with:
+          aws-role-arn: ${{ secrets.AWS_ROLE_ARN_READ_GENERATE_GO_DOC_REPO }}
+          aws-lambda-url: ${{ secrets.GATI_LAMBDA_TT_URL }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Configure git for private repository and install go tools
+        env:
+          GOPRIVATE: github.com/smartcontractkit/generate-go-function-docs
+        run: |
+          git config --global url."https://x-access-token:${{ steps.setup-github-token-read.outputs.access-token }}@github.com/".insteadOf "https://github.com/"
+          go install github.com/smartcontractkit/[email protected]
+          go install github.com/jmank88/[email protected]
+          go install golang.org/x/tools/gopls@latest
+
       - name: Checkout current branch
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Setup Git
-        run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"              
-
-      - name: Get PR details
-        id: pr-details
-        run: |
-          echo "pr_number=$(jq -r .pull_request.number <<< "$GITHUB_EVENT_PATH")" >> $GITHUB_OUTPUT
-          echo "pr_branch=$(jq -r .pull_request.head.ref <<< "$GITHUB_EVENT_PATH")" >> $GITHUB_OUTPUT
-          echo "base_commit=${{ github.event.pull_request.base.sha }}" >> $GITHUB_OUTPUT
-          echo "head_commit=${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT          
+      - name: Detect changes related to current PR
+        uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: changes
+        with:
+          filters: |
+            seth:
+              - 'seth/**/*.go'
+            wasp:
+              - 'wasp/**/*.go'
+            havoc:
+              - 'havoc/**/*.go'
+            lib:
+              - 'lib/**/*.go'
+            k8s-test-runner:
+              - 'k8s-test-runner/**/*.go'
+            framework:
+              - 'framework/**/*.go'
+            tools/asciitable:
+              - 'tools/asciitable/**/*.go'
+            tools/breakingchanges:
+              - 'tools/breakingchanges/**/*.go'
+            tools/citool:
+              - 'tools/citool/**/*.go'
+            tools/ecrimagefetcher:
+              - 'tools/ecrimagefetcher/**/*.go'
+            tools/envresolve:
+              - 'tools/envresolve/**/*.go'
+            tools/flakeguard:
+              - 'tools/flakeguard/**/*.go'
+            tools/ghlatestreleasechecker:
+              - 'tools/ghlatestreleasechecker/**/*.go'
+            tools/ghsecrets:
+              - 'tools/ghsecrets/**/*.go'
+            tools/gotestloghelper:
+              - 'tools/gotestloghelper/**/*.go'
+            tools/testlistgenerator:
+              - 'tools/testlistgenerator/**/*.go'
+            tools/workflowresultparser:
+              - 'tools/workflowresultparser/**/*.go'
+            # add more projects here
 
-      - name: Install go doc generator
+      - name: Find all go modules in the repository and filter the ones that changed
         shell: bash
+        id: go-modules
+        env:
+          FILTERS: ${{ steps.changes.outputs.changes }}
         run: |
-          go install github.com/smartcontractkit/generate-go-function-docs@latest
+          PATH=$PATH:$(go env GOPATH)/bin
+          export PATH
 
-      - name: Install gopls
-        shell: bash
-        run: |
-          go install golang.org/x/tools/gopls@latest
+          # Find all go projects
+          gomods_output=$(gomods 2>&1)
 
-      - name: Create a new branch
-        run: |
-          git checkout -b ${{ steps.pr-details.outputs.pr_branch }}-docs
+          # Extract the parent directories of go.mod files
+          parent_folders=$(echo "$gomods_output" | grep 'go\.mod$' | sed 's/\/go\.mod//' | sed 's/^[ \t]*//;s/[ \t]*$//' | xargs -n 1)
+
+          # Convert parent directories into a JSON matrix
+          echo "$parent_folders" | jq -R -s 'split("\n") | map(select(length > 0)) | map({folder: .})' > all_folders.json
+
+          # Filter the directories that did not changeß
+          jq --argjson filters "$FILTERS" 'map(select(.folder as $folder | $filters | index($folder)))' all_folders.json > filtered_folders.json
 
-      - name: Generate go docs
+          echo "Filtered folder List JSON"
+          cat filtered_folders.json
+
+          rm all_folders.json
+
+      - name: Generate go docs for changed projects
         shell: bash
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
-          generate-go-function-docs diff -b ${{ steps.pr-details.outputs.base_commit }} -c ${{ steps.pr-details.outputs.head_commit }} -g fake_slash
+          # Add go binary to PATH
+          PATH=$PATH:$(go env GOPATH)/bin
+          export PATH
+          cat filtered_folders.json | jq -c '.[]' | while read -r item; do
+            folder=$(echo "$item" | jq -r '.folder')
+            echo "Processing folder: $folder"
+            generate-go-function-docs diff -b ${{ github.event.pull_request.base.sha }} -c ${{ github.event.pull_request.head.sha }} --saveCosts --generator chatgpt --generatorSubType ${{ vars.GO_DOC_GEN_CHATGPT_MODEL }} --folder "$folder"
+          done
+          rm filtered_folders.json
 
-      - name: Make sure code still compiles
-        run: |
-          go mod tidy
-          go mod vendor
-          go build ./...
+      - name: Upload costs as artifact
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: generation-costs
+          path: ./costs
 
-      - name: Commit changes
-        run: |
-          git add .
-          git commit -m "Add automatically generated go documentation"
-          git push origin ${{ steps.pr-details.outputs.pr_branch }}-docs
+      - name: Remove costs before committing
+        shell: bash
+        run: rm -rf costs
+
+      - name: Setup GitHub Token for creating a new PR
+        id: setup-github-token-write
+        uses: smartcontractkit/.github/actions/setup-github-token@9e7cc0779934cae4a9028b8588c9adb64d8ce68c # [email protected]
+        with:
+          aws-role-arn: ${{ secrets.AWS_ROLE_ARN_CREATE_PR }}
+          aws-lambda-url: ${{ secrets.GATI_LAMBDA_TT_URL }}
+          aws-region: ${{ secrets.AWS_REGION }}
 
       - name: Create a new PR targeting current PR
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
+        id: create-pr
+        with:
+          token: ${{ steps.setup-github-token-write.outputs.access-token }}
+          branch: ${{ github.head_ref }}-docs
+          base: ${{ github.head_ref }}
+          title: "Go docs for PR#${{ github.event.pull_request.number }}"
+          body: "This PR contains automatically generated go documentation for the PR#${{ github.event.pull_request.number }}. Please review the changes."
+          commit-message: "[Bot] Add automatically generated go documentation"
+
+      - name: Find comment with PR link
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3.1.0
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body-includes: Go doc generation
+
+      - name: Create comment in the original PR with link to the new PR
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        if: steps.create-pr.outputs.pull-request-number != ''
+        with:
+          token: ${{ steps.setup-github-token-write.outputs.access-token }}
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          edit-mode: replace
+          body: |
+            ## Go doc generation
+            Hey @${{ github.actor }}, you can check generated Go function documentation [here](${{ steps.create-pr.outputs.pull-request-url }}). Please review them and merge to this PR once you're satisfied with them.
+
+      - name: Create comment in the original PR to notify about no new docs
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4.0.0
+        if: steps.create-pr.outputs.pull-request-number == ''
+        with:
+          token: ${{ steps.setup-github-token-write.outputs.access-token }}
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          edit-mode: replace
+          body: |
+            ## Go doc generation
+            Hey @${{ github.actor }}, no documentation was generated. Are you sure that you made changes to public functions without existing comments in your PR?
+
+      - name: Send Slack notification
+        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
+        if: failure()
+        id: slack
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          base: ${{ env.pr_branch }}
-          branch: ${{ steps.pr-details.outputs.pr_branch }}-docs
-          title: "Go docs for PR#${{ env.pr_number }}"
-          body: "This PR contains automatically generated go documentation for the PR#${{ env.pr_number }}. Please review the changes."
+          channel-id: 'C049X3353K2'
+          payload: |
+            {
+              "attachments": [
+                {
+                  "color": "#C62828",
+                  "blocks": [
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "Go doc generation - Failed :x:"
+                      }
+                    },
+                    {
+                        "type": "section",
+                        "text": {
+                          "type": "mrkdwn",
+                          "text": "<@U060CGGPY8H>, please have a look."
+                        }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "<${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.pull_request.number }}|PR#${{ github.event.pull_request.number }}> | <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run Details>"
+                      }
+                    }
+                  ]
+                }
+              ]
+            }

book/src/framework/components/img.png

@@ -10,3 +10,14 @@ rosetta error: Rosetta is only intended to run on Apple Silicon with a macOS hos
 #### Solution
 
 Update your docker to `Docker version 27.3.1, build ce12230`
+
+## Can't see any docker metrics with Cadvisor
+
+Cadvisor container can't mount some layers
+```
+E1209 13:14:49.908130       1 manager.go:1116] Failed to create existing container: /docker/aa40875c382af890861447fa8aaf6908978041b9077bb81029971d23929f8c4d: failed to identify the read-write layer ID for container "aa40875c382af890861447fa8aaf6908978041b9077bb81029971d23929f8c4d". - open /rootfs/var/lib/docker/image/overlayfs/layerdb/mounts/aa40875c382af890861447fa8aaf6908978041b9077bb81029971d23929f8c4d/mount-id: no such file or directory
+```
+
+Disable `containerd` images ( Settings -> General ), see [issue](https://github.com/google/cadvisor/pull/3569).
+
+![img.png](img.png)

book/src/framework/components/troubleshooting.md

@@ -2,10 +2,18 @@
 
 We use Prometheus to collect metrics of Chainlink nodes and other services.
 
+Check [Prometheus UI](http://localhost:9090/query).
+
 Check [Grafana](http://localhost:3000/explore?panes=%7B%22gGs%22:%7B%22datasource%22:%22PBFA97CFB590B2093%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22ctf%5C%22%7D%22,%22range%22:true,%22instant%22:true,%22datasource%22:%7B%22type%22:%22prometheus%22,%22uid%22:%22PBFA97CFB590B2093%22%7D,%22editorMode%22:%22code%22,%22legendFormat%22:%22__auto%22%7D%5D,%22range%22:%7B%22from%22:%22now-5m%22,%22to%22:%22now%22%7D%7D%7D&schemaVersion=1&orgId=1) example query.
 
 Queries:
 - All metrics of all containers
 ```json
 {job="ctf"}
-```
+```
+
+## Docker Resources
+
+We are using [cadvisor](https://github.com/google/cadvisor) to monitor resources of test [containers](http://localhost:3000/d/pMEd7m0Mz/cadvisor-exporter?orgId=1).
+
+Cadvisor UI can be found [here](http://localhost:8085/containers/)

book/src/framework/observability/metrics.md

@@ -0,0 +1,2 @@
+- Add cadvisor support to "ctf obs up"
+- Add cadvisor dashboard

framework/.changeset/v0.3.5.md

@@ -0,0 +1,2 @@
+- Add cadvisor support to obs stack
+- Add cadvisor dashboard

framework/.changeset/v0.3.6.md

@@ -0,0 +1,2 @@
+- Separate JobDistributor database
+- Wait when fund nodes on testnets

framework/.changeset/v0.3.7.md

@@ -11,3 +11,8 @@ scrape_configs:
       - source_labels: [ __meta_docker_port_private ]
         regex: '6688'
         action: keep
+  - job_name: cadvisor
+    scrape_interval: 10s
+    static_configs:
+      - targets:
+          - cadvisor:8080