Merge branch 'main' into tt-1772-wasp-o1-mini

Author: Tofel

.github/workflows/framework-golden-tests.yml

@@ -22,22 +22,23 @@ jobs:
             config: smoke.toml
             count: 1
             timeout: 10m
-          - name: TestLoad
-            config: load.toml
+          - name: TestUpgrade
+            config: upgrade.toml
             count: 1
             timeout: 10m
-          - name: TestChaos
-            config: chaos.toml
+          - name: TestPerformanceBaseline
+            config: performance_baseline.toml
             count: 1
             timeout: 10m
-          - name: TestUpgradeAll
-            config: upgrade_all.toml
+          - name: TestChaos
+            config: chaos.toml
             count: 1
             timeout: 10m
-          - name: TestUpgradeSome
-            config: upgrade_some.toml
+          - name: TestScalability
+            config: scalability.toml
             count: 1
             timeout: 10m
+    name: ${{ matrix.test.name }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
@@ -74,7 +75,7 @@ jobs:
             go-modules-${{ runner.os }}
       - name: Install dependencies
         run: go mod download
-      - name: Run Docker Component Tests
+      - name: Run System Tests
         if: steps.changes.outputs.src == 'true'
         env:
           CTF_CONFIGS: ${{ matrix.test.config }}

.github/workflows/framework.yml

@@ -5,7 +5,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-framework
   cancel-in-progress: true
 jobs:
-  test:
+  framework-component-tests:
     defaults:
       run:
         working-directory: framework
@@ -51,4 +51,4 @@ jobs:
       - name: Run Docker Component Tests
         if: steps.changes.outputs.src == 'true'
         run: |
-          go test -timeout 5m -v -count 1 -run TestDocker ./...
+          go test -timeout 2m -v -count 1 -run TestComponent ./...

.gitignore

@@ -46,9 +46,7 @@ suite/remote_runner_config.yaml
 *.csv
 
 # docs
-docs/_site/
-docs/.sass-cache/
-docs/Gemfile.lock
+book/docs/
 
 dist/
 
@@ -78,4 +76,4 @@ __debug*
 .tool-versions
 
 import_keys_test.go
-tag.py
+tag.py

book/README.md

@@ -1,15 +1,18 @@
-## MDBook
-Install [mdbook](https://github.com/rust-lang/mdBook)
-```
+# Chainlink Testing Framework Docs
+
+We use [mdBook](https://github.com/rust-lang/mdBook) for our docs. They can be found [here](https://smartcontractkit.github.io/chainlink-testing-framework/).
+
+## Development
+
+First [install Rust](https://doc.rust-lang.org/cargo/getting-started/installation.html), then [mdbook](https://github.com/rust-lang/mdBook).
+
+```sh
+# Install mdBook
 cargo install mdbook && \
 cargo install mdbook-alerts && \
 cargo install mdbook-cmdrun
-```
-
-Run it locally
-```
+# Run the mdBook
 make run
 ```
-Open your browser [here](http://localhost:9999/)
 
-GitHub hosted [version](https://smartcontractkit.github.io/chainlink-testing-framework/overview.html)
+Visit [localhost:9999](http://localhost:9999) to see your local version.

book/src/SUMMARY.md

@@ -14,9 +14,11 @@
     - [CLI](./framework/cli.md)
     - [Configuration](./framework/configuration.md)
     - [Test Configuration](./framework/test_configuration_overrides.md)
-    - [Components Persistence](framework/components/state.md)
+    - [Exposing Components](framework/components/state.md)
     - [Components Cleanup](framework/components/cleanup.md)
     - [Components Caching](framework/components/caching.md)
+    - [Mocking Services](framework/components/mocking.md)
+    - [Copying Files](framework/copying_files.md)
     - [External Environment](framework/components/external.md)
     - [Secrets]()
     - [Observability Stack](framework/observability/observability_stack.md)
@@ -39,7 +41,7 @@
       - [Chainlink]()
       - [RPC]()
       - [Loki]()
-- [Testing](framework/testing.md)
+- [Testing Maturity Model](framework/testing.md)
   - [Smoke]()
   - [Performance]()
   - [Chaos]()

book/src/framework/components/chainlink/node.md

@@ -10,17 +10,16 @@ Here we provide full configuration reference, if you want to copy and run it, pl
 ## Configuration
 ```toml
 [cl_node]
-  # Optional URL for fake data provider URL
-  # usually set up in test with local mock server
-  data_provider_url = "http://example.com"
 
   [cl_node.db]
     # PostgreSQL image version and tag
     image = "postgres:15.6"
     # Pulls the image every time if set to 'true', used like that in CI. Can be set to 'false' to speed up local runs
-    pull_image = true
+    pull_image = false
 
   [cl_node.node]
+    # custom ports that plugins may need to expose and map to the host machine
+    custom_ports = [14000, 14001]
     # A list of paths to capability binaries
     capabilities = ["./capability_1", "./capability_2"]
     # Default capabilities directory inside container
@@ -34,7 +33,7 @@ Here we provide full configuration reference, if you want to copy and run it, pl
     # Optional name for image we build, default is "ctftmp"
     docker_image_name = "ctftmp"
     # Pulls the image every time if set to 'true', used like that in CI. Can be set to 'false' to speed up local runs
-    pull_image = true
+    pull_image = false
     # Overrides Chainlink node TOML configuration
     # can be multiline, see example
     user_config_overrides = """
@@ -54,6 +53,10 @@ Here we provide full configuration reference, if you want to copy and run it, pl
     use_cache = true
     # Describes deployed or external Chainlink node
     [cl_node.out.node]
+      # API user name
+      api_auth_user = '[email protected]'
+      # API password
+      api_auth_password = 'fj293fbBnlQ!f9vNs'
       # Host Docker URLs the test uses
       # in case of using external component you can replace these URLs with another deployment
       p2p_url = "http://127.0.0.1:32812"

book/src/framework/components/chainlink/nodeset.md

@@ -52,22 +52,20 @@ Then configure NodeSet
   http_port_range_start = 10000
   # P2P API port range start, each new node get port incremented (host machine)
   p2p_port_range_start = 12000
-
+  
+  [nodeset.db]
+    # PostgreSQL image version and tag
+    image = "postgres:15.6"
+    # Pulls the image every time if set to 'true', used like that in CI. Can be set to 'false' to speed up local runs
+    pull_image = false
+    # PostgreSQL volume name
+    volume_name = ""
 
   [[nodeset.node_specs]]
-    # Optional URL for fake data provider URL
-    # usually set up in test with local mock server
-    data_provider_url = "http://example.com"
-
-    [nodeset.node_specs.db]
-      # PostgreSQL image version and tag
-      image = "postgres:15.6"
-      # Pulls the image every time if set to 'true', used like that in CI. Can be set to 'false' to speed up local runs
-      pull_image = true
-      # PostgreSQL volume name
-      volume_name = ""
 
     [nodeset.node_specs.node]
+      # custom ports that plugins may need to expose and map to the host machine
+      custom_ports = [14000, 14001]
       # A list of paths to capability binaries
       capabilities = ["./capability_1", "./capability_2"]
       # Default capabilities directory inside container
@@ -81,7 +79,7 @@ Then configure NodeSet
       # Optional name for image we build, default is "ctftmp"
       docker_image_name = "ctftmp"
       # Pulls the image every time if set to 'true', used like that in CI. Can be set to 'false' to speed up local runs
-      pull_image = true
+      pull_image = false
       # Overrides Chainlink node TOML configuration
       # can be multiline, see example
       user_config_overrides = """
@@ -106,6 +104,10 @@ Then configure NodeSet
 
       # Describes deployed or external Chainlink node
       [nodeset.out.cl_nodes.node]
+        # API user name
+        api_auth_user = '[email protected]'
+        # API password
+        api_auth_password = 'fj293fbBnlQ!f9vNs'
         # Host Docker URLs the test uses
         # in case of using external component you can replace these URLs with another deployment
         p2p_url = "http://127.0.0.1:32996"
@@ -130,15 +132,13 @@ package capabilities_test
 import (
 	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
-	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/fake"
 	ns "github.com/smartcontractkit/chainlink-testing-framework/framework/components/simple_node_set"
 	"github.com/stretchr/testify/require"
 	"testing"
 )
 
 type Config struct {
 	BlockchainA        *blockchain.Input `toml:"blockchain_a" validate:"required"`
-	MockerDataProvider *fake.Input       `toml:"data_provider" validate:"required"`
 	NodeSet            *ns.Input         `toml:"nodeset" validate:"required"`
 }
 
@@ -148,9 +148,7 @@ func TestMe(t *testing.T) {
 
 	bc, err := blockchain.NewBlockchainNetwork(in.BlockchainA)
 	require.NoError(t, err)
-	dp, err := fake.NewFakeDataProvider(in.MockerDataProvider)
-	require.NoError(t, err)
-	out, err := ns.NewSharedDBNodeSet(in.NodeSet, bc, dp.BaseURLDocker)
+	out, err := ns.NewSharedDBNodeSet(in.NodeSet, bc)
 	require.NoError(t, err)
 
 	t.Run("test something", func(t *testing.T) {

book/src/framework/components/external.md

@@ -20,23 +20,10 @@ For example, to integrate with remote `k8s` environment you can use `CTF_CONFIGS
       http_url = "http://127.0.0.1:8545"
       ws_url = "ws://127.0.0.1:8545"
 
-[contracts]
-
-  [contracts.out]
-    # set up your contracts
-    addresses = ["0x5fbdb2315678afecb367f032d93f642f64180aa3"]
-    use_cache = true
-
-[data_provider]
-  port = 9111
-
-  [data_provider.out]
-    # setup your data provider URLs
-    base_url_host = "http://localhost:9111"
-
 [nodeset]
 
   [[nodeset.node_specs]]
+  ...
 
   [nodeset.out]
     use_cache = true
@@ -45,6 +32,9 @@ For example, to integrate with remote `k8s` environment you can use `CTF_CONFIGS
       use_cache = true
 
       [nodeset.out.cl_nodes.node]
+        # set up your user/password for API authorization
+        api_auth_user = '[email protected]'
+        api_auth_password = 'fj293fbBnlQ!f9vNs'
         # set up each node URLs
         p2p_url = "http://127.0.0.1:12000"
         url = "http://127.0.0.1:10000"

book/src/framework/components/mocking.md

@@ -0,0 +1,14 @@
+# Mocking Services
+
+The framework aims to equip you with all the necessary tools to write end-to-end system-level tests, while still allowing the flexibility to mock third-party services that are not critical to your testing scope.
+
+## Configuration
+```toml
+[fake]
+  # port to start Gin server
+  port = 9111
+```
+
+## Usage
+
+See [full](https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/framework/examples/myproject/fake_test.go) example.

book/src/framework/components/state.md

@@ -1,4 +1,4 @@
-# Components Persistence
+# Exposing Components (Data and ports)
 
 We use static port ranges and volumes for all components to simplify Docker port management for developers.
 
@@ -9,6 +9,7 @@ When deploying a component, you can explicitly configure port ranges if the defa
 Defaults are:
 - [NodeSet](../components/chainlink/nodeset.md) (Node HTTP API): `10000..100XX`
 - [NodeSet](../components/chainlink/nodeset.md) (Node P2P API): `12000..120XX`
+- Shared `PostgreSQL` volume is called `postgresql_data`
 ```
 [nodeset]
   # HTTP API port range start, each new node get port incremented (host machine)
@@ -27,6 +28,7 @@ Defaults are:
 
 When you run `ctf d rm` database volume will be **removed**.
 
+
 <div class="warning">
 
 One node set is enough for any kind of testing, if you need more nodes consider extending your existing node set:
@@ -35,3 +37,22 @@ One node set is enough for any kind of testing, if you need more nodes consider
   nodes = 10
 ```
 </div>
+
+## Custom ports
+
+You can also define a custom set of ports for any node
+```toml
+[nodeset]
+  nodes = 5
+  override_mode = "each"
+  
+  [nodeset.db]
+    image = "postgres:15.6"
+
+  [[nodeset.node_specs]]
+
+    [nodeset.node_specs.node]
+      # here we defined 2 new ports to listen and mapped them to our host machine
+      custom_ports = [14000, 14001]
+      image = "public.ecr.aws/chainlink/chainlink:v2.16.0"
+```