diff --git a/framework/.changeset/v0.7.7.md b/framework/.changeset/v0.7.7.md new file mode 100644 index 000000000..7507fcbd0 --- /dev/null +++ b/framework/.changeset/v0.7.7.md @@ -0,0 +1 @@ +- JD with a real DB dump \ No newline at end of file diff --git a/framework/components/jd/jd.go b/framework/components/jd/jd.go index 3c1480179..baec2b8d2 100644 --- a/framework/components/jd/jd.go +++ b/framework/components/jd/jd.go @@ -28,6 +28,7 @@ type Input struct { CSAEncryptionKey string `toml:"csa_encryption_key"` DockerFilePath string `toml:"docker_file"` DockerContext string `toml:"docker_ctx"` + JDSQLDumpPath string `toml:"jd_sql_dump_path"` DBInput *postgres.Input `toml:"db"` Out *Output `toml:"out"` } @@ -56,7 +57,7 @@ func defaults(in *Input) { func defaultJDDB() *postgres.Input { return &postgres.Input{ - Image: "postgres:12", + Image: "postgres:16", Port: 14000, Name: "jd-db", VolumeName: "jd", @@ -77,6 +78,7 @@ func NewJD(in *Input) (*Output, error) { if in.DBInput == nil { in.DBInput = defaultJDDB() } + in.DBInput.JDSQLDumpPath = in.JDSQLDumpPath pgOut, err := postgres.NewPostgreSQL(in.DBInput) if err != nil { return nil, err diff --git a/framework/components/postgres/postgres.go b/framework/components/postgres/postgres.go index c27cd2729..989d4bd84 100644 --- a/framework/components/postgres/postgres.go +++ b/framework/components/postgres/postgres.go @@ -22,6 +22,7 @@ const ( Port = "5432" ExposedStaticPort = 13000 Database = "chainlink" + JDDatabase = "job-distributor-db" DBVolumeName = "postgresql_data" ) @@ -32,6 +33,7 @@ type Input struct { VolumeName string `toml:"volume_name"` Databases int `toml:"databases"` JDDatabase bool `toml:"jd_database"` + JDSQLDumpPath string `toml:"jd_sql_dump_path"` PullImage bool `toml:"pull_image"` ContainerResources *framework.ContainerResources `toml:"resources"` Out *Output `toml:"out"` @@ -64,10 +66,23 @@ func NewPostgreSQL(in *Input) (*Output, error) { "CREATE EXTENSION pg_stat_statements;", ) } + sqlCommands = append(sqlCommands, "ALTER USER chainlink WITH SUPERUSER;") if in.JDDatabase { - sqlCommands = append(sqlCommands, "CREATE DATABASE jd;") + if in.JDSQLDumpPath != "" { + // if we have a full dump we replace RDS specific commands and apply it creating db and filling the tables + d, err := os.ReadFile(in.JDSQLDumpPath) + if err != nil { + return nil, fmt.Errorf("error reading JD dump file '%s': %v", in.JDSQLDumpPath, err) + } + // transaction_timeout is a custom RDS instruction, we must replace it + sqlMigration := strings.Replace(string(d), "SET transaction_timeout = 0;", "", -1) + sqlCommands = append(sqlCommands, sqlMigration) + sqlCommands = append(sqlCommands, "DELETE FROM public.csa_keypairs where id = 1;") + } else { + // if we don't have a dump we create an empty DB + sqlCommands = append(sqlCommands, fmt.Sprintf("CREATE DATABASE \"%s\";", JDDatabase)) + } } - sqlCommands = append(sqlCommands, "ALTER USER chainlink WITH SUPERUSER;") initSQL := strings.Join(sqlCommands, "\n") initFile, err := os.CreateTemp("", "init-*.sql") if err != nil { @@ -176,7 +191,7 @@ func NewPostgreSQL(in *Input) (*Output, error) { Password, containerName, Port, - "jd", + JDDatabase, ) o.JDUrl = fmt.Sprintf( "postgresql://%s:%s@%s:%d/%s?sslmode=disable", @@ -184,7 +199,7 @@ func NewPostgreSQL(in *Input) (*Output, error) { Password, host, portToExpose, - "jd", + JDDatabase, ) } return o, nil diff --git a/framework/examples/myproject/fork_plus_offchain.toml b/framework/examples/myproject/fork_plus_offchain.toml index 1484f78d7..4051ca2f1 100644 --- a/framework/examples/myproject/fork_plus_offchain.toml +++ b/framework/examples/myproject/fork_plus_offchain.toml @@ -1,20 +1,10 @@ -[blockchain_dst] - chain_id = "2337" -# docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate", "-b", "1"] - docker_cmd_params = ["-b", "1"] - port = "8545" - type = "anvil" - [blockchain_src] - chain_id = "3337" -# docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate", "-b", "1"] - docker_cmd_params = ["-b", "1"] - port = "8555" + chain_id = "1337" + docker_cmd_params = ["--steps-tracing", "--fork-block-number", "25335999", "--fork-url", "https://rpcs.cldev.sh/base/sepolia/archive", "--auto-impersonate"] +# docker_cmd_params = ["-b", "1", "--steps-tracing"] type = "anvil" -[contracts_dst] - [contracts_src] [[nodesets]] diff --git a/framework/examples/myproject/fork_plus_offchain_test.go b/framework/examples/myproject/fork_plus_offchain_test.go index f5f94982c..103a5ea01 100644 --- a/framework/examples/myproject/fork_plus_offchain_test.go +++ b/framework/examples/myproject/fork_plus_offchain_test.go @@ -18,11 +18,9 @@ import ( type CfgForkChainsOffChain struct { ContractsSrc *onchain.Input `toml:"contracts_src" validate:"required"` - ContractsDst *onchain.Input `toml:"contracts_dst" validate:"required"` BlockchainSrc *blockchain.Input `toml:"blockchain_src" validate:"required"` - BlockchainDst *blockchain.Input `toml:"blockchain_dst" validate:"required"` // off-chain components - NodeSet *ns.Input `toml:"nodeset" validate:"required"` + NodeSets []*ns.Input `toml:"nodesets" validate:"required"` } func TestOffChainAndFork(t *testing.T) { @@ -33,9 +31,6 @@ func TestOffChainAndFork(t *testing.T) { bcSrc, err := blockchain.NewBlockchainNetwork(in.BlockchainSrc) require.NoError(t, err) - bcDst, err := blockchain.NewBlockchainNetwork(in.BlockchainDst) - require.NoError(t, err) - // create configs for 2 EVM networks srcNetworkCfg, err := clnode.NewNetworkCfg(&clnode.EVMNetworkConfig{ MinIncomingConfirmations: 1, @@ -48,25 +43,11 @@ func TestOffChainAndFork(t *testing.T) { }, }, }, bcSrc) - dstNetworkConfig, err := clnode.NewNetworkCfg(&clnode.EVMNetworkConfig{ - MinIncomingConfirmations: 1, - MinContractPayment: "0.00001 link", - ChainID: bcSrc.ChainID, - EVMNodes: []*clnode.EVMNode{ - { - SendOnly: false, - Order: 100, - }, - }, - }, bcDst) - // override the configuration - in.NodeSet.NodeSpecs[0].Node.TestConfigOverrides = srcNetworkCfg + dstNetworkConfig + in.NodeSets[0].NodeSpecs[0].Node.TestConfigOverrides = srcNetworkCfg - // create a node set - _, err = ns.NewSharedDBNodeSet(in.NodeSet, bcSrc) + _, err = ns.NewSharedDBNodeSet(in.NodeSets[0], bcSrc) require.NoError(t, err) - // connect 2 clients scSrc, err := seth.NewClientBuilder(). WithRpcUrl(bcSrc.Nodes[0].ExternalWSUrl). WithGasPriceEstimations(true, 0, seth.Priority_Fast). @@ -74,40 +55,19 @@ func TestOffChainAndFork(t *testing.T) { WithPrivateKeys([]string{blockchain.DefaultAnvilPrivateKey}). Build() require.NoError(t, err) - scDst, err := seth.NewClientBuilder(). - WithRpcUrl(bcDst.Nodes[0].ExternalWSUrl). - WithGasPriceEstimations(true, 0, seth.Priority_Fast). - WithTracing(seth.TracingLevel_All, []string{seth.TraceOutput_Console}). - WithPrivateKeys([]string{blockchain.DefaultAnvilPrivateKey}). - Build() - require.NoError(t, err) - // deploy 2 example product contracts - // you should replace it with chainlink-deployments in.ContractsSrc.URL = bcSrc.Nodes[0].ExternalWSUrl contractsSrc, err := onchain.NewProductOnChainDeployment(scSrc, in.ContractsSrc) require.NoError(t, err) - in.ContractsDst.URL = bcDst.Nodes[0].ExternalWSUrl - contractsDst, err := onchain.NewProductOnChainDeployment(scDst, in.ContractsDst) - require.NoError(t, err) t.Run("test some contracts with fork", func(t *testing.T) { - // interact with a source chain i, err := testToken.NewBurnMintERC677(contractsSrc.Addresses[0], scSrc.Client) require.NoError(t, err) balance, err := i.BalanceOf(scSrc.NewCallOpts(), contractsSrc.Addresses[0]) require.NoError(t, err) fmt.Println(balance) - // interact with a destination chain - i, err = testToken.NewBurnMintERC677(contractsDst.Addresses[0], scDst.Client) - require.NoError(t, err) - balance, err = i.BalanceOf(scDst.NewCallOpts(), contractsDst.Addresses[0]) - require.NoError(t, err) - fmt.Println(balance) - // Use anvil methods, see https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/framework/rpc/rpc.go _ = rpc.New(bcSrc.Nodes[0].ExternalHTTPUrl, nil) - _ = rpc.New(bcDst.Nodes[0].ExternalHTTPUrl, nil) }) } diff --git a/framework/examples/myproject_cll/README.md b/framework/examples/myproject_cll/README.md index c6f93a7cd..d4d0f3256 100644 --- a/framework/examples/myproject_cll/README.md +++ b/framework/examples/myproject_cll/README.md @@ -11,5 +11,16 @@ docker build -t job-distributor:0.9.0 -f e2e/Dockerfile.e2e . Run the tests locally ``` -CTF_CONFIGS=jd.toml go test -v -count 1 -run TestJD +CTF_CONFIGS=jd.toml go test -v -run TestJD +``` + +## Job Distributor from staging dump +Get `.sql` dump from staging or production service and then run +``` +CTF_CONFIGS=jd_dump.toml go test -v run TestJD +``` + +## Connect to the database +``` +PGPASSWORD=thispasswordislongenough psql -h 0.0.0.0 -p 14000 -U chainlink -d job-distributor-db ``` \ No newline at end of file diff --git a/framework/examples/myproject_cll/jd.dump b/framework/examples/myproject_cll/jd.dump deleted file mode 100644 index 5ab211c09..000000000 Binary files a/framework/examples/myproject_cll/jd.dump and /dev/null differ diff --git a/framework/examples/myproject_cll/jd_dump.toml b/framework/examples/myproject_cll/jd_dump.toml new file mode 100644 index 000000000..001d27750 --- /dev/null +++ b/framework/examples/myproject_cll/jd_dump.toml @@ -0,0 +1,8 @@ +[blockchain] + type = "anvil" + chain_id = "1337" + docker_cmd_params = ["--fork-url", "wss://avalanche-fuji-c-chain-rpc.publicnode.com", "--auto-impersonate", "-b", "1"] + +[jd] + jd_sql_dump_path = "db.sql" + image = "job-distributor:0.9.0" diff --git a/framework/examples/myproject_cll/jd_test.go b/framework/examples/myproject_cll/jd_test.go index 81a8ae43e..96ded3351 100644 --- a/framework/examples/myproject_cll/jd_test.go +++ b/framework/examples/myproject_cll/jd_test.go @@ -24,21 +24,8 @@ func TestJD(t *testing.T) { jdOut, err := jd.NewJD(in.JD) require.NoError(t, err) - dc, err := framework.NewDockerClient() - require.NoError(t, err) - // find what to dump, RDS API here instead? - _, err = dc.ExecContainer(jdOut.DBContainerName, []string{"pg_dump", "-U", "chainlink", "-h", "localhost", "-p", "5432", "-d", "chainlink", "-F", "c", "-f", "jd.dump"}) - require.NoError(t, err) - - // copy your dump - err = dc.CopyFile(jdOut.DBContainerName, "jd.dump", "/") - require.NoError(t, err) - - // restore - _, err = dc.ExecContainer(jdOut.DBContainerName, []string{"pg_restore", "-U", "chainlink", "-d", "chainlink", "jd.dump"}) - require.NoError(t, err) - t.Run("test changesets with forked network and real JD state", func(t *testing.T) { + t.Run("test changesets with forked network/JD state", func(t *testing.T) { _ = bcOut _ = jdOut }) diff --git a/wasp/examples/simple_rps/main_test.go b/wasp/examples/simple_rps/main_test.go index b81625c40..53813fb4e 100644 --- a/wasp/examples/simple_rps/main_test.go +++ b/wasp/examples/simple_rps/main_test.go @@ -1,7 +1,6 @@ package main import ( - "os" "testing" "time" @@ -13,16 +12,16 @@ func TestGun(t *testing.T) { srv := wasp.NewHTTPMockServer(nil) srv.Run() - branch := os.Getenv("BRANCH") - commit := os.Getenv("COMMIT") + //branch := os.Getenv("BRANCH") + //commit := os.Getenv("COMMIT") // define labels for differentiate one run from another labels := map[string]string{ // check variables in dashboard/dashboard.go "go_test_name": "generator_healthcheck", "gen_name": "generator_healthcheck", - "branch": branch, - "commit": commit, + "branch": "test", + "commit": "test", } // create generator