Collect restart logs separately (#1437)

* read all logs even with restarts

* subdirs

* add test private keys to the lib, updated besu default image

* changeset

Author: skudasov

book/src/framework/components/blockchains/evm.md

@@ -72,5 +72,17 @@ Public: 0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266
 Private: ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
 ```
 
-Test keys for `Besu` can be found [here](https://besu.hyperledger.org/23.4.1/private-networks/reference/accounts-for-testing)
+For `Besu` keys are
+```
+Public: 0xfe3b557e8fb62b89f4916b721be55ceb828dbd73
+Private: 0x8f2a55949038a9610f50fb23b5883af3b4ecb3c3bb792cbcefbd1542c692be63
+
+Public: 0x627306090abaB3A6e1400e9345bC60c78a8BEf57
+Private: 0xc87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3
+
+Public: 0xf17f52151EbEF6C7334FAD080c5704D77216b732
+Private: 0xae6ae8e5ccbfb04590405997ee2d52d2b330726137b875053c36d94e974d162f
+```
+
+More docs for `Besu` can be found [here](https://besu.hyperledger.org/private-networks/reference/accounts-for-testing)
 

book/src/framework/configuration.md

@@ -7,9 +7,10 @@
 |         CTF_CONFIGS          | Path(s) to test config files. <br/>Can be more than one, ex.: smoke.toml,smoke_1.toml,smoke_2.toml.<br/>First filepath will hold all the merged values | Any valid TOML file path |                          -                          |     ✅     |
 |        CTF_LOG_LEVEL         |                                                                   Harness log level                                                                    | `info`, `debug`, `trace` |                       `info`                        |    🚫     |
 |      CTF_PROMTAIL_DEBUG      |                                    Set `true` if you are integrating with remote `Loki` push API to debug Promtail                                     |          `true`, `false` |                       `false`                       |    🚫     |
+|   CTF_IGNORE_CRITICAL_LOGS   |                                      Ignore all logs that has CRIT,FATAL or PANIC levels (Chainlink nodes only!)                                       |          `true`, `false` |                       `false`                       |    🚫     |
+|     CTF_CHAINLINK_IMAGE      |                                           Flag to override Chainlink Docker image in format $repository:$tag                                           |         $repository:$tag |                          -                          |    🚫     |
+|         CTF_JD_IMAGE         |                                                Job distributor service image in format $repository:$tag                                                |         $repository:$tag |                          -                          |    🚫     |
 |           LOKI_URL           |                                            URL to `Loki` push api, should be like`${host}/loki/api/v1/push`                                            |                      URL | `http://host.docker.internal:3030/loki/api/v1/push` |    🚫     |
 |        LOKI_TENANT_ID        |                                                Streams all components logs to `Loki`, see params below                                                 |                 `string` |                     `promtail`                      |    🚫     |
 |       LOKI_BASIC_AUTH        |                                                          Basic auth in format $user:$password                                                          |        `$user:$password` |                          -                          |    🚫     |
 |         RESTY_DEBUG          |                                                            Log all Resty client HTTP calls                                                             |          `true`, `false` |                       `false`                       |    🚫     |
-|   CTF_IGNORE_CRITICAL_LOGS   |                                      Ignore all logs that has CRIT,FATAL or PANIC levels (Chainlink nodes only!)                                       |          `true`, `false` |                       `false`                       |    🚫     |
-|     CTF_CHAINLINK_IMAGE      |                                           Flag to override Chainlink Docker image in format $repository:$tag                                           |         $repository:$tag |                          -                          |    🚫     |

framework/.changeset/v0.3.3.md

@@ -0,0 +1,3 @@
+- Upgrade Besu to 24.9.1
+- Expose default test private keys as framework constants
+- Collect all the logs even between restarts

framework/components/blockchain/besu.go

@@ -12,9 +12,15 @@ import (
 	"github.com/testcontainers/testcontainers-go/wait"
 )
 
+const (
+	DefaultBesuPrivateKey1 = "8f2a55949038a9610f50fb23b5883af3b4ecb3c3bb792cbcefbd1542c692be63"
+	DefaultBesuPrivateKey2 = "c87509a1c067bbde78beb793e6fa76530b6382a4c0241e5e4a9ec0a0f44dc0d3"
+	DefaultBesuPrivateKey3 = "ae6ae8e5ccbfb04590405997ee2d52d2b330726137b875053c36d94e974d162f"
+)
+
 func defaultBesu(in *Input) {
 	if in.Image == "" {
-		in.Image = "hyperledger/besu:22.1.0"
+		in.Image = "hyperledger/besu:24.9.1"
 	}
 	if in.ChainID == "" {
 		in.ChainID = "1337"

framework/components/blockchain/geth.go

@@ -51,6 +51,7 @@ const (
   }
 }
 `
+	DefaultGethPrivateKey = `ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80`
 )
 
 var initScript = `

framework/components/simple_node_set/reload.go

@@ -1,6 +1,9 @@
 package simple_node_set
 
 import (
+	"fmt"
+	"github.com/google/uuid"
+	"github.com/smartcontractkit/chainlink-testing-framework/framework"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/chaos"
 	"github.com/smartcontractkit/chainlink-testing-framework/framework/components/blockchain"
 	"time"
@@ -9,6 +12,10 @@ import (
 // UpgradeNodeSet updates nodes configuration TOML files
 // this API is discouraged, however, you can use it if nodes require restart or configuration updates, temporarily!
 func UpgradeNodeSet(in *Input, bc *blockchain.Output, wait time.Duration) (*Output, error) {
+	uniq := fmt.Sprintf("%s-%s", framework.DefaultCTFLogsDir, uuid.NewString()[0:4])
+	if err := framework.WriteAllContainersLogs(uniq); err != nil {
+		return nil, err
+	}
 	_, err := chaos.ExecPumba("rm --volumes=false re2:^node.*|ns-postgresql.*", wait)
 	if err != nil {
 		return nil, err

framework/config.go

@@ -130,9 +130,9 @@ func Load[X any](t *testing.T) (*X, error) {
 	t.Cleanup(func() {
 		err := Store[X](input)
 		require.NoError(t, err)
-		err = WriteAllContainersLogs()
+		err = WriteAllContainersLogs(DefaultCTFLogsDir)
 		require.NoError(t, err)
-		err = checkNodeLogErrors(DefaultCTFLogsDir)
+		err = checkAllNodeLogErrors()
 		require.NoError(t, err)
 	})
 	// TODO: not all the people have AWS access, sadly enough, uncomment when granted

framework/docker.go

@@ -21,7 +21,7 @@ import (
 )
 
 const (
-	DefaultCTFLogsDir = "logs"
+	DefaultCTFLogsDir = "logs/docker"
 )
 
 func IsDockerRunning() bool {
@@ -189,11 +189,11 @@ func isLocalToolDockerContainer(containerName string) bool {
 }
 
 // WriteAllContainersLogs writes all Docker container logs to the default logs directory
-func WriteAllContainersLogs() error {
+func WriteAllContainersLogs(dir string) error {
 	L.Info().Msg("Writing Docker containers logs")
-	if _, err := os.Stat(DefaultCTFLogsDir); os.IsNotExist(err) {
-		if err := os.MkdirAll(DefaultCTFLogsDir, 0755); err != nil {
-			return fmt.Errorf("failed to create directory %s: %w", DefaultCTFLogsDir, err)
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("failed to create directory %s: %w", dir, err)
 		}
 	}
 	provider, err := tc.NewDockerProvider()
@@ -220,7 +220,7 @@ func WriteAllContainersLogs() error {
 				L.Error().Err(err).Str("Container", containerName).Msg("failed to fetch logs for container")
 				return err
 			}
-			logFilePath := filepath.Join(DefaultCTFLogsDir, fmt.Sprintf("%s.log", containerName))
+			logFilePath := filepath.Join(dir, fmt.Sprintf("%s.log", containerName))
 			logFile, err := os.Create(logFilePath)
 			if err != nil {
 				L.Error().Err(err).Str("Container", containerName).Msg("failed to create container log file")

framework/examples/myproject/smoke.toml

@@ -2,7 +2,7 @@
 [blockchain_a]
   # choose "anvil", "geth" or "besu"
   # uncomment the second line for "anvil"
-  type = "geth"
+  type = "besu"
 #  docker_cmd_params = ["-b", "1"]
 
 [data_provider]

framework/logs.go

@@ -6,12 +6,44 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 )
 
 const (
 	EnvVarIgnoreCriticalLogs = "CTF_IGNORE_CRITICAL_LOGS"
 )
 
+func getLogDirectories() ([]string, error) {
+	logDirs := make([]string, 0)
+	currentDir, err := os.Getwd()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get current directory: %w", err)
+	}
+	entries, err := os.ReadDir(currentDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read directory: %w", err)
+	}
+	for _, entry := range entries {
+		if entry.IsDir() && strings.HasPrefix(entry.Name(), "logs-") {
+			logDirs = append(logDirs, filepath.Join(currentDir, entry.Name()))
+		}
+	}
+	return logDirs, nil
+}
+
+func checkAllNodeLogErrors() error {
+	dirs, err := getLogDirectories()
+	if err != nil {
+		return err
+	}
+	for _, dd := range dirs {
+		if err := checkNodeLogErrors(dd); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // checkNodeLogsErrors check Chainlink nodes logs for error levels
 func checkNodeLogErrors(dir string) error {
 	if os.Getenv(EnvVarIgnoreCriticalLogs) == "true" {