From 79ddb2337ba810f38750554e229cf73836990e5e Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 10 Jul 2023 12:27:50 +1000 Subject: [PATCH 01/22] 'opera export graph' command stub --- cmd/opera/launcher/chaincmd.go | 16 ++++++++ cmd/opera/launcher/export_dag.go | 69 ++++++++++++++++++++++++++++++++ go.mod | 6 ++- go.sum | 64 +++++++++++++++++++++++++++-- 4 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 cmd/opera/launcher/export_dag.go diff --git a/cmd/opera/launcher/chaincmd.go b/cmd/opera/launcher/chaincmd.go index 00070b7e1..e880486ef 100644 --- a/cmd/opera/launcher/chaincmd.go +++ b/cmd/opera/launcher/chaincmd.go @@ -116,6 +116,22 @@ EVM export mode is configured with --export.evm.mode. opera export evm-keys Requires a first argument of the DB directory to write to. +`, + }, + { + Name: "graph", + Usage: "Export events DAG", + ArgsUsage: " [ ]", + Action: utils.MigrateFlags(exportDAGgraph), + Flags: []cli.Flag{ + DataDirFlag, + }, + Description: ` + opera export graph + +Requires a first argument of the file to write to. +Optional second and third arguments control the first and +last epoch to write `, }, }, diff --git a/cmd/opera/launcher/export_dag.go b/cmd/opera/launcher/export_dag.go new file mode 100644 index 000000000..cf5324f22 --- /dev/null +++ b/cmd/opera/launcher/export_dag.go @@ -0,0 +1,69 @@ +package launcher + +import ( + "io" + "os" + "strconv" + + "github.com/Fantom-foundation/lachesis-base/inter/idx" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + "gonum.org/v1/gonum/graph/encoding/dot" + "gopkg.in/urfave/cli.v1" + + "github.com/Fantom-foundation/go-opera/utils/dag" +) + +func exportDAGgraph(ctx *cli.Context) error { + if len(ctx.Args()) < 1 { + utils.Fatalf("This command requires an argument.") + } + + cfg := makeAllConfigs(ctx) + + rawDbs := makeDirectDBsProducer(cfg) + gdb := makeGossipStore(rawDbs, cfg) + defer gdb.Close() + + fn := ctx.Args().First() + + // Open the file handle and potentially wrap with a gzip stream + fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer fh.Close() + + var writer io.Writer = fh + + from := idx.Epoch(1) + if len(ctx.Args()) > 1 { + n, err := strconv.ParseUint(ctx.Args().Get(1), 10, 32) + if err != nil { + return err + } + from = idx.Epoch(n) + } + to := idx.Epoch(0) + if len(ctx.Args()) > 2 { + n, err := strconv.ParseUint(ctx.Args().Get(2), 10, 32) + if err != nil { + return err + } + to = idx.Epoch(n) + } + + graph := dag.Graph(gdb, from, to) + buf, err := dot.Marshal(graph, "DAG", "", "\t") + if err != nil { + utils.Fatalf("Export error: %v\n", err) + } + + log.Info("Exporting events DAG to file", "file", fn) + _, err = writer.Write(buf) + if err != nil { + return err + } + + return nil +} diff --git a/go.mod b/go.mod index 1f97835a5..c3dbc3e71 100644 --- a/go.mod +++ b/go.mod @@ -39,10 +39,12 @@ require ( go.uber.org/atomic v1.5.1 // indirect golang.org/x/crypto v0.7.0 golang.org/x/sys v0.6.0 - golang.org/x/tools v0.6.0 // indirect + golang.org/x/tools v0.7.0 // indirect gopkg.in/urfave/cli.v1 v1.20.0 ) +require gonum.org/v1/gonum v0.13.0 + require ( github.com/DataDog/zstd v1.4.5 // indirect github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect @@ -94,7 +96,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect - golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/sync v0.1.0 // indirect diff --git a/go.sum b/go.sum index 58e6e50e5..d3779cf8b 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -72,7 +74,10 @@ github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -100,6 +105,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -202,6 +209,7 @@ github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlK github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -223,6 +231,13 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/latin-modern v0.3.0/go.mod h1:ysEQXnuT/sCDOAONxC7ImeEDVINbltClhasMAqEtRK0= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.3.0/go.mod h1:jdJ+cqF+F4SUL2V+qxBth8fvBpBDS7yloUL5Fi8GTGY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -230,6 +245,9 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= +github.com/go-latex/latex v0.0.0-20230307184459-12ec69307ad9/go.mod h1:gWuR/CrFDDeVRFQwHPvsv9soJVB/iqymhuZQuJ3a9OM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= @@ -239,6 +257,8 @@ github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -308,8 +328,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -400,6 +421,7 @@ github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0t github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= @@ -516,6 +538,9 @@ github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHu github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -567,6 +592,8 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= @@ -680,6 +707,7 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -689,17 +717,30 @@ golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= +golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -721,8 +762,10 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -766,6 +809,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= @@ -840,6 +884,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -853,12 +898,14 @@ golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -870,6 +917,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -900,6 +948,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -933,8 +982,10 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -944,9 +995,14 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= +gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= From 3c2fde712d6e5eb558a028f38f3b94e6ea131534 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 10 Jul 2023 20:09:50 +1000 Subject: [PATCH 02/22] ./utils/dag impls graph --- utils/dag/dag.go | 16 ++++++ utils/dag/graph.go | 137 +++++++++++++++++++++++++++++++++++++++++++++ utils/dag/node.go | 101 +++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 utils/dag/dag.go create mode 100644 utils/dag/graph.go create mode 100644 utils/dag/node.go diff --git a/utils/dag/dag.go b/utils/dag/dag.go new file mode 100644 index 000000000..8c4682577 --- /dev/null +++ b/utils/dag/dag.go @@ -0,0 +1,16 @@ +package dag + +import ( + "github.com/Fantom-foundation/lachesis-base/inter/idx" + "gonum.org/v1/gonum/graph/encoding/dot" + + "github.com/Fantom-foundation/go-opera/gossip" +) + +func Graph(db *gossip.Store, from, to idx.Epoch) dot.Graph { + return &dagReader{ + db: db, + epochFrom: from, + epochTo: to, + } +} diff --git a/utils/dag/graph.go b/utils/dag/graph.go new file mode 100644 index 000000000..0bc6d9880 --- /dev/null +++ b/utils/dag/graph.go @@ -0,0 +1,137 @@ +package dag + +import ( + "github.com/Fantom-foundation/lachesis-base/hash" + "github.com/Fantom-foundation/lachesis-base/inter/idx" + "gonum.org/v1/gonum/graph" + + "github.com/Fantom-foundation/go-opera/gossip" + "github.com/Fantom-foundation/go-opera/inter" +) + +// dagReader implements dot.Graph over gossip.Store +type dagReader struct { + db *gossip.Store + epochFrom idx.Epoch + epochTo idx.Epoch +} + +func (g *dagReader) DOTID() string { + return "DAG" +} + +// Node returns the node with the given ID if it exists +// in the graph, and nil otherwise. +func (g *dagReader) Node(id int64) graph.Node { + e := g.db.GetEvent(id2event(id)) + return &dagNode{ + id: event2id(e.ID()), + event: e, + } +} + +// Nodes returns all the nodes in the graph. +// +// Nodes must not return nil. +func (g *dagReader) Nodes() graph.Nodes { + nn := &dagNodes{ + data: make(chan *inter.Event), + } + + go func() { + defer close(nn.data) + g.db.ForEachEvent(g.epochFrom, func(e *inter.EventPayload) bool { + if e.Epoch() > g.epochTo { + return false + } + + nn.data <- &e.Event + return true + }) + }() + + return nn +} + +// From returns all nodes that can be reached directly +// from the node with the given ID. +// +// From must not return nil. +func (g *dagReader) From(id int64) graph.Nodes { + nn := &dagNodes{ + data: make(chan *inter.Event), + } + + x := g.Node(id).(*dagNode).event + go func() { + defer close(nn.data) + for _, p := range x.Parents() { + n := g.Node(event2id(p)) + nn.data <- n.(*dagNode).event + } + }() + + return nn +} + +// HasEdgeBetween returns whether an edge exists between +// nodes with IDs xid and yid without considering direction. +func (g *dagReader) HasEdgeBetween(xid, yid int64) bool { + x := g.Node(xid).(*dagNode).event + y := g.Node(yid).(*dagNode).event + + for _, p := range x.Parents() { + if p == y.ID() { + return true + } + } + for _, p := range y.Parents() { + if p == x.ID() { + return true + } + } + + return false +} + +// Edge returns the edge from u to v, with IDs uid and vid, +// if such an edge exists and nil otherwise. The node v +// must be directly reachable from u as defined by the +// From method. +func (g *dagReader) Edge(uid, vid int64) graph.Edge { + u := g.Node(uid).(*dagNode) + v := g.Node(vid).(*dagNode) + + for _, p := range u.event.Parents() { + if p == v.event.ID() { + return &dagEdge{ + x: u, + y: v, + } + } + } + + return nil +} + +// -- + +var ( + id2hash = make(map[int64]hash.Event) +) + +func id2event(id int64) hash.Event { + return id2hash[id] +} + +func event2id(e hash.Event) int64 { + // NOTE: possible collision + var id int64 + for i := 0; i < 8; i++ { + id += int64(e[8+i] << (8 * i)) + } + + id2hash[id] = e + + return id +} diff --git a/utils/dag/node.go b/utils/dag/node.go new file mode 100644 index 000000000..a6f955c6a --- /dev/null +++ b/utils/dag/node.go @@ -0,0 +1,101 @@ +package dag + +import ( + "gonum.org/v1/gonum/graph" + "gonum.org/v1/gonum/graph/encoding" + + "github.com/Fantom-foundation/go-opera/inter" +) + +// Edge is a graph edge. In directed graphs, the direction of the +// edge is given from -> to, otherwise the edge is semantically +// unordered. +type dagEdge struct { + x, y *dagNode +} + +// From returns the from node of the edge. +func (e *dagEdge) From() graph.Node { + return e.x +} + +// To returns the to node of the edge. +func (e *dagEdge) To() graph.Node { + return e.y +} + +// ReversedEdge returns the edge reversal of the receiver +// if a reversal is valid for the data type. +// When a reversal is valid an edge of the same type as +// the receiver with nodes of the receiver swapped should +// be returned, otherwise the receiver should be returned +// unaltered. +func (e *dagEdge) ReversedEdge() graph.Edge { + return nil +} + +type dagNode struct { + id int64 + event *inter.Event +} + +func (n *dagNode) ID() int64 { + return n.id +} + +func (n *dagNode) Attributes() []encoding.Attribute { + return []encoding.Attribute{ + encoding.Attribute{ + Key: "ID", + Value: n.event.ID().String(), + }, + } +} + +type dagNodes struct { + data chan *inter.Event + current *inter.Event +} + +// Reset returns the iterator to its start position. +func (nn *dagNodes) Reset() { + panic("Not implemented yet") +} + +// Next advances the iterator and returns whether +// the next call to the item method will return a +// non-nil item. +// +// Next should be called prior to any call to the +// iterator's item retrieval method after the +// iterator has been obtained or reset. +// +// The order of iteration is implementation +// dependent. +func (nn *dagNodes) Next() bool { + nn.current = <-nn.data + return nn.current != nil +} + +// Node returns the current Node from the iterator. +func (nn *dagNodes) Node() graph.Node { + return &dagNode{ + id: event2id(nn.current.ID()), + event: nn.current, + } +} + +// Len returns the number of items remaining in the +// iterator. +// +// If the number of items in the iterator is unknown, +// too large to materialize or too costly to calculate +// then Len may return a negative value. +// In this case the consuming function must be able +// to operate on the items of the iterator directly +// without materializing the items into a slice. +// The magnitude of a negative length has +// implementation-dependent semantics. +func (nn *dagNodes) Len() int { + return -1 +} From e748b7e8b7e91509d705e056956fa84915df5adb Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 11 Jul 2023 11:17:42 +1000 Subject: [PATCH 03/22] dagNode stores only necessary fields --- utils/dag/graph.go | 43 ++++++++++++++++++++++++------------------- utils/dag/node.go | 33 +++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/utils/dag/graph.go b/utils/dag/graph.go index 0bc6d9880..9623c7f2d 100644 --- a/utils/dag/graph.go +++ b/utils/dag/graph.go @@ -24,10 +24,7 @@ func (g *dagReader) DOTID() string { // in the graph, and nil otherwise. func (g *dagReader) Node(id int64) graph.Node { e := g.db.GetEvent(id2event(id)) - return &dagNode{ - id: event2id(e.ID()), - event: e, - } + return event2node(e) } // Nodes returns all the nodes in the graph. @@ -35,17 +32,17 @@ func (g *dagReader) Node(id int64) graph.Node { // Nodes must not return nil. func (g *dagReader) Nodes() graph.Nodes { nn := &dagNodes{ - data: make(chan *inter.Event), + data: make(chan *dagNode), } go func() { defer close(nn.data) g.db.ForEachEvent(g.epochFrom, func(e *inter.EventPayload) bool { - if e.Epoch() > g.epochTo { + if g.epochTo >= g.epochFrom && e.Epoch() > g.epochTo { return false } - nn.data <- &e.Event + nn.data <- event2node(&e.Event) return true }) }() @@ -59,15 +56,15 @@ func (g *dagReader) Nodes() graph.Nodes { // From must not return nil. func (g *dagReader) From(id int64) graph.Nodes { nn := &dagNodes{ - data: make(chan *inter.Event), + data: make(chan *dagNode), } - x := g.Node(id).(*dagNode).event + x := g.Node(id).(*dagNode) go func() { defer close(nn.data) - for _, p := range x.Parents() { + for _, p := range x.parents { n := g.Node(event2id(p)) - nn.data <- n.(*dagNode).event + nn.data <- n.(*dagNode) } }() @@ -77,16 +74,16 @@ func (g *dagReader) From(id int64) graph.Nodes { // HasEdgeBetween returns whether an edge exists between // nodes with IDs xid and yid without considering direction. func (g *dagReader) HasEdgeBetween(xid, yid int64) bool { - x := g.Node(xid).(*dagNode).event - y := g.Node(yid).(*dagNode).event + x := g.Node(xid).(*dagNode) + y := g.Node(yid).(*dagNode) - for _, p := range x.Parents() { - if p == y.ID() { + for _, p := range x.parents { + if p == y.hash { return true } } - for _, p := range y.Parents() { - if p == x.ID() { + for _, p := range y.parents { + if p == x.hash { return true } } @@ -102,8 +99,8 @@ func (g *dagReader) Edge(uid, vid int64) graph.Edge { u := g.Node(uid).(*dagNode) v := g.Node(vid).(*dagNode) - for _, p := range u.event.Parents() { - if p == v.event.ID() { + for _, p := range u.parents { + if p == v.hash { return &dagEdge{ x: u, y: v, @@ -116,6 +113,14 @@ func (g *dagReader) Edge(uid, vid int64) graph.Edge { // -- +func event2node(e *inter.Event) *dagNode { + return &dagNode{ + id: event2id(e.ID()), + hash: e.ID(), + parents: e.Parents(), + } +} + var ( id2hash = make(map[int64]hash.Event) ) diff --git a/utils/dag/node.go b/utils/dag/node.go index a6f955c6a..3a5c70ba4 100644 --- a/utils/dag/node.go +++ b/utils/dag/node.go @@ -1,10 +1,9 @@ package dag import ( + "github.com/Fantom-foundation/lachesis-base/hash" "gonum.org/v1/gonum/graph" "gonum.org/v1/gonum/graph/encoding" - - "github.com/Fantom-foundation/go-opera/inter" ) // Edge is a graph edge. In directed graphs, the direction of the @@ -35,8 +34,10 @@ func (e *dagEdge) ReversedEdge() graph.Edge { } type dagNode struct { - id int64 - event *inter.Event + id int64 + hash hash.Event + parents hash.Events + isAtropos bool } func (n *dagNode) ID() int64 { @@ -44,17 +45,28 @@ func (n *dagNode) ID() int64 { } func (n *dagNode) Attributes() []encoding.Attribute { - return []encoding.Attribute{ + aa := []encoding.Attribute{ encoding.Attribute{ Key: "ID", - Value: n.event.ID().String(), + Value: n.hash.String(), }, } + + if n.isAtropos { + aa = append(aa, + encoding.Attribute{ + Key: "Role", + Value: "atropos", + }, + ) + } + + return aa } type dagNodes struct { - data chan *inter.Event - current *inter.Event + data chan *dagNode + current *dagNode } // Reset returns the iterator to its start position. @@ -79,10 +91,7 @@ func (nn *dagNodes) Next() bool { // Node returns the current Node from the iterator. func (nn *dagNodes) Node() graph.Node { - return &dagNode{ - id: event2id(nn.current.ID()), - event: nn.current, - } + return nn.current } // Len returns the number of items remaining in the From f5b62861aa9b6aae2b079f3c24b241d475aa3b7c Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 11 Jul 2023 18:19:46 +1000 Subject: [PATCH 04/22] dagLoader reads whole DAG into mem --- utils/dag/dag.go | 8 ++- utils/dag/loader.go | 139 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 utils/dag/loader.go diff --git a/utils/dag/dag.go b/utils/dag/dag.go index 8c4682577..585980318 100644 --- a/utils/dag/dag.go +++ b/utils/dag/dag.go @@ -8,9 +8,13 @@ import ( ) func Graph(db *gossip.Store, from, to idx.Epoch) dot.Graph { - return &dagReader{ + /* g:= &dagReader{ db: db, epochFrom: from, epochTo: to, - } + }*/ + + g := newDagLoader(db, from, to) + + return g } diff --git a/utils/dag/loader.go b/utils/dag/loader.go new file mode 100644 index 000000000..293be47a7 --- /dev/null +++ b/utils/dag/loader.go @@ -0,0 +1,139 @@ +package dag + +import ( + "github.com/Fantom-foundation/lachesis-base/hash" + "github.com/Fantom-foundation/lachesis-base/inter/idx" + "gonum.org/v1/gonum/graph" + + "github.com/Fantom-foundation/go-opera/gossip" + "github.com/Fantom-foundation/go-opera/inter" +) + +type dagLoader struct { + refs []hash.Event + nodes map[hash.Event]*dagNode +} + +func newDagLoader(db *gossip.Store, from, to idx.Epoch) *dagLoader { + g := &dagLoader{ + refs: make([]hash.Event, 0, 2000000), + nodes: make(map[hash.Event]*dagNode), + } + + db.ForEachEvent(from, func(e *inter.EventPayload) bool { + if to >= from && e.Epoch() > to { + return false + } + + id := len(g.refs) + g.refs = append(g.refs, e.ID()) + g.nodes[e.ID()] = &dagNode{ + id: int64(id), + hash: e.ID(), + parents: e.Parents(), + } + + return true + }) + + db.ForEachBlock(func(index idx.Block, block *inter.Block) { + node, exists := g.nodes[block.Atropos] + if exists { + node.isAtropos = true + } + }) + + return g +} + +func (g *dagLoader) DOTID() string { + return "DAG" +} + +// Node returns the node with the given ID if it exists +// in the graph, and nil otherwise. +func (g *dagLoader) Node(id int64) graph.Node { + hash := g.refs[id] + return g.nodes[hash] +} + +// Nodes returns all the nodes in the graph. +// +// Nodes must not return nil. +func (g *dagLoader) Nodes() graph.Nodes { + nn := &dagNodes{ + data: make(chan *dagNode), + } + + go func() { + defer close(nn.data) + + for _, e := range g.nodes { + nn.data <- e + } + }() + + return nn +} + +// From returns all nodes that can be reached directly +// from the node with the given ID. +// +// From must not return nil. +func (g *dagLoader) From(id int64) graph.Nodes { + nn := &dagNodes{ + data: make(chan *dagNode), + } + + h := g.refs[id] + x := g.nodes[h] + go func() { + defer close(nn.data) + for _, p := range x.parents { + n := g.nodes[p] + nn.data <- n + } + }() + + return nn +} + +// HasEdgeBetween returns whether an edge exists between +// nodes with IDs xid and yid without considering direction. +func (g *dagLoader) HasEdgeBetween(xid, yid int64) bool { + x := g.nodes[g.refs[xid]] + y := g.nodes[g.refs[yid]] + + for _, p := range x.parents { + if p == y.hash { + return true + } + } + for _, p := range y.parents { + if p == x.hash { + return true + } + } + + return false +} + +// Edge returns the edge from u to v, with IDs uid and vid, +// if such an edge exists and nil otherwise. The node v +// must be directly reachable from u as defined by the +// From method. +func (g *dagLoader) Edge(uid, vid int64) graph.Edge { + u := g.nodes[g.refs[uid]] + v := g.nodes[g.refs[vid]] + + for _, p := range u.parents { + if p == v.hash { + return &dagEdge{ + x: u, + y: v, + } + } + } + + return nil +} From d14217fc8227c4b39a19e208d007f50d06f8678a Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 11 Jul 2023 23:54:28 +1000 Subject: [PATCH 05/22] fix dot labels --- utils/dag/node.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/dag/node.go b/utils/dag/node.go index 3a5c70ba4..3dceed183 100644 --- a/utils/dag/node.go +++ b/utils/dag/node.go @@ -47,7 +47,7 @@ func (n *dagNode) ID() int64 { func (n *dagNode) Attributes() []encoding.Attribute { aa := []encoding.Attribute{ encoding.Attribute{ - Key: "ID", + Key: "label", Value: n.hash.String(), }, } @@ -55,8 +55,8 @@ func (n *dagNode) Attributes() []encoding.Attribute { if n.isAtropos { aa = append(aa, encoding.Attribute{ - Key: "Role", - Value: "atropos", + Key: "xlabel", + Value: "Atropos", }, ) } From 57f6c88b55bdcea32a0106122de0c8a5c4f31460 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 12 Jul 2023 00:00:11 +1000 Subject: [PATCH 06/22] dagLoader and dagReader impl graph.Directed --- utils/dag/graph.go | 27 +++++++++++++++++++++++++++ utils/dag/loader.go | 27 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/utils/dag/graph.go b/utils/dag/graph.go index 9623c7f2d..4c49dc981 100644 --- a/utils/dag/graph.go +++ b/utils/dag/graph.go @@ -71,6 +71,18 @@ func (g *dagReader) From(id int64) graph.Nodes { return nn } +// To returns all nodes that can reach directly +// to the node with the given ID. +// +// To must not return nil. +func (g *dagReader) To(id int64) graph.Nodes { + nn := &dagNodes{ + data: make(chan *dagNode), + } + close(nn.data) + return nn +} + // HasEdgeBetween returns whether an edge exists between // nodes with IDs xid and yid without considering direction. func (g *dagReader) HasEdgeBetween(xid, yid int64) bool { @@ -91,6 +103,21 @@ func (g *dagReader) HasEdgeBetween(xid, yid int64) bool { return false } +// HasEdgeFromTo returns whether an edge exists +// in the graph from u to v with IDs uid and vid. +func (g *dagReader) HasEdgeFromTo(uid, vid int64) bool { + u := g.Node(uid).(*dagNode) + v := g.Node(vid).(*dagNode) + + for _, p := range u.parents { + if p == v.hash { + return true + } + } + + return false +} + // Edge returns the edge from u to v, with IDs uid and vid, // if such an edge exists and nil otherwise. The node v // must be directly reachable from u as defined by the diff --git a/utils/dag/loader.go b/utils/dag/loader.go index 293be47a7..290ee7f65 100644 --- a/utils/dag/loader.go +++ b/utils/dag/loader.go @@ -98,6 +98,18 @@ func (g *dagLoader) From(id int64) graph.Nodes { return nn } +// To returns all nodes that can reach directly +// to the node with the given ID. +// +// To must not return nil. +func (g *dagLoader) To(id int64) graph.Nodes { + nn := &dagNodes{ + data: make(chan *dagNode), + } + close(nn.data) + return nn +} + // HasEdgeBetween returns whether an edge exists between // nodes with IDs xid and yid without considering direction. func (g *dagLoader) HasEdgeBetween(xid, yid int64) bool { @@ -118,6 +130,21 @@ func (g *dagLoader) HasEdgeBetween(xid, yid int64) bool { return false } +// HasEdgeFromTo returns whether an edge exists +// in the graph from u to v with IDs uid and vid. +func (g *dagLoader) HasEdgeFromTo(uid, vid int64) bool { + u := g.nodes[g.refs[uid]] + v := g.nodes[g.refs[vid]] + + for _, p := range u.parents { + if p == v.hash { + return true + } + } + + return false +} + // Edge returns the edge from u to v, with IDs uid and vid, // if such an edge exists and nil otherwise. The node v // must be directly reachable from u as defined by the From d4f7dba6769304eec3ca0513167cd63c2ec03ab1 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 26 Jul 2023 20:01:53 +1000 Subject: [PATCH 07/22] export DOT option of export events command --- cmd/opera/launcher/chaincmd.go | 4 +-- cmd/opera/launcher/export.go | 46 ++++++++++++++++++++------------ cmd/opera/launcher/export_dag.go | 14 ++++++++-- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/cmd/opera/launcher/chaincmd.go b/cmd/opera/launcher/chaincmd.go index e880486ef..182d46bc0 100644 --- a/cmd/opera/launcher/chaincmd.go +++ b/cmd/opera/launcher/chaincmd.go @@ -78,8 +78,8 @@ The import command imports EVM storage (trie nodes, code, preimages) from files. Requires a first argument of the file to write to. Optional second and third arguments control the first and -last epoch to write. If the file ends with .gz, the output will -be gzipped +last epoch to write. If the file ends with .gz, the output will be gzipped. +End the rest of file name with .dot to export events graph as DOT `, }, { diff --git a/cmd/opera/launcher/export.go b/cmd/opera/launcher/export.go index 7681f7fdf..11ff5cace 100644 --- a/cmd/opera/launcher/export.go +++ b/cmd/opera/launcher/export.go @@ -46,19 +46,14 @@ func exportEvents(ctx *cli.Context) error { fn := ctx.Args().First() - // Open the file handle and potentially wrap with a gzip stream + // Open the file handle + log.Info("Exporting events to file", "file", fn) fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) if err != nil { return err } defer fh.Close() - var writer io.Writer = fh - if strings.HasSuffix(fn, ".gz") { - writer = gzip.NewWriter(writer) - defer writer.(*gzip.Writer).Close() - } - from := idx.Epoch(1) if len(ctx.Args()) > 1 { n, err := strconv.ParseUint(ctx.Args().Get(1), 10, 32) @@ -76,22 +71,39 @@ func exportEvents(ctx *cli.Context) error { to = idx.Epoch(n) } - log.Info("Exporting events to file", "file", fn) - // Write header and version - _, err = writer.Write(append(eventsFileHeader, eventsFileVersion...)) - if err != nil { - return err + // Potentially wrap with a gzip stream + var writer io.Writer = fh + if strings.HasSuffix(fn, ".gz") { + fn = fn[:len(fn)-len(".gz")] + writer = gzip.NewWriter(writer) + defer writer.(*gzip.Writer).Close() } - err = exportTo(writer, gdb, from, to) - if err != nil { - utils.Fatalf("Export error: %v\n", err) + + switch { + // DOT format: + case strings.HasSuffix(fn, ".dot"): + err = exportDOT(writer, gdb, from, to) + if err != nil { + utils.Fatalf("Export DOT error: %v\n", err) + } + // RLP format: + default: + // Write header and version + _, err = writer.Write(append(eventsFileHeader, eventsFileVersion...)) + if err != nil { + return err + } + err = exportRLP(writer, gdb, from, to) + if err != nil { + utils.Fatalf("Export RLP error: %v\n", err) + } } return nil } -// exportTo writer the active chain. -func exportTo(w io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) { +// exportRLP writer the active chain. +func exportRLP(w io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) { start, reported := time.Now(), time.Time{} var ( diff --git a/cmd/opera/launcher/export_dag.go b/cmd/opera/launcher/export_dag.go index cf5324f22..ecc987e56 100644 --- a/cmd/opera/launcher/export_dag.go +++ b/cmd/opera/launcher/export_dag.go @@ -11,6 +11,7 @@ import ( "gonum.org/v1/gonum/graph/encoding/dot" "gopkg.in/urfave/cli.v1" + "github.com/Fantom-foundation/go-opera/gossip" "github.com/Fantom-foundation/go-opera/utils/dag" ) @@ -28,6 +29,7 @@ func exportDAGgraph(ctx *cli.Context) error { fn := ctx.Args().First() // Open the file handle and potentially wrap with a gzip stream + log.Info("Exporting events DAG to file", "file", fn) fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) if err != nil { return err @@ -53,13 +55,21 @@ func exportDAGgraph(ctx *cli.Context) error { to = idx.Epoch(n) } + err = exportDOT(writer, gdb, from, to) + if err != nil { + utils.Fatalf("Export DOT error: %v\n", err) + } + + return nil +} + +func exportDOT(writer io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) { graph := dag.Graph(gdb, from, to) buf, err := dot.Marshal(graph, "DAG", "", "\t") if err != nil { - utils.Fatalf("Export error: %v\n", err) + return err } - log.Info("Exporting events DAG to file", "file", fn) _, err = writer.Write(buf) if err != nil { return err From 2f009c3fc265c09a5a81f6ade4cf466e9c332e83 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 26 Jul 2023 20:20:44 +1000 Subject: [PATCH 08/22] rm unused --- cmd/opera/launcher/chaincmd.go | 16 ------- cmd/opera/launcher/export.go | 18 ++++++++ cmd/opera/launcher/export_dag.go | 79 -------------------------------- 3 files changed, 18 insertions(+), 95 deletions(-) delete mode 100644 cmd/opera/launcher/export_dag.go diff --git a/cmd/opera/launcher/chaincmd.go b/cmd/opera/launcher/chaincmd.go index 182d46bc0..008b22b3c 100644 --- a/cmd/opera/launcher/chaincmd.go +++ b/cmd/opera/launcher/chaincmd.go @@ -116,22 +116,6 @@ EVM export mode is configured with --export.evm.mode. opera export evm-keys Requires a first argument of the DB directory to write to. -`, - }, - { - Name: "graph", - Usage: "Export events DAG", - ArgsUsage: " [ ]", - Action: utils.MigrateFlags(exportDAGgraph), - Flags: []cli.Flag{ - DataDirFlag, - }, - Description: ` - opera export graph - -Requires a first argument of the file to write to. -Optional second and third arguments control the first and -last epoch to write `, }, }, diff --git a/cmd/opera/launcher/export.go b/cmd/opera/launcher/export.go index 11ff5cace..09e4be276 100644 --- a/cmd/opera/launcher/export.go +++ b/cmd/opera/launcher/export.go @@ -18,9 +18,11 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/status-im/keycard-go/hexutils" "github.com/syndtr/goleveldb/leveldb/opt" + "gonum.org/v1/gonum/graph/encoding/dot" "gopkg.in/urfave/cli.v1" "github.com/Fantom-foundation/go-opera/gossip" + "github.com/Fantom-foundation/go-opera/utils/dag" "github.com/Fantom-foundation/go-opera/utils/dbutil/autocompact" ) @@ -131,6 +133,22 @@ func exportRLP(w io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) { return } +// exportDOT writer the active chain. +func exportDOT(writer io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) { + graph := dag.Graph(gdb, from, to) + buf, err := dot.Marshal(graph, "DAG", "", "\t") + if err != nil { + return err + } + + _, err = writer.Write(buf) + if err != nil { + return err + } + + return nil +} + func exportEvmKeys(ctx *cli.Context) error { if len(ctx.Args()) < 1 { utils.Fatalf("This command requires an argument.") diff --git a/cmd/opera/launcher/export_dag.go b/cmd/opera/launcher/export_dag.go deleted file mode 100644 index ecc987e56..000000000 --- a/cmd/opera/launcher/export_dag.go +++ /dev/null @@ -1,79 +0,0 @@ -package launcher - -import ( - "io" - "os" - "strconv" - - "github.com/Fantom-foundation/lachesis-base/inter/idx" - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/log" - "gonum.org/v1/gonum/graph/encoding/dot" - "gopkg.in/urfave/cli.v1" - - "github.com/Fantom-foundation/go-opera/gossip" - "github.com/Fantom-foundation/go-opera/utils/dag" -) - -func exportDAGgraph(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { - utils.Fatalf("This command requires an argument.") - } - - cfg := makeAllConfigs(ctx) - - rawDbs := makeDirectDBsProducer(cfg) - gdb := makeGossipStore(rawDbs, cfg) - defer gdb.Close() - - fn := ctx.Args().First() - - // Open the file handle and potentially wrap with a gzip stream - log.Info("Exporting events DAG to file", "file", fn) - fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm) - if err != nil { - return err - } - defer fh.Close() - - var writer io.Writer = fh - - from := idx.Epoch(1) - if len(ctx.Args()) > 1 { - n, err := strconv.ParseUint(ctx.Args().Get(1), 10, 32) - if err != nil { - return err - } - from = idx.Epoch(n) - } - to := idx.Epoch(0) - if len(ctx.Args()) > 2 { - n, err := strconv.ParseUint(ctx.Args().Get(2), 10, 32) - if err != nil { - return err - } - to = idx.Epoch(n) - } - - err = exportDOT(writer, gdb, from, to) - if err != nil { - utils.Fatalf("Export DOT error: %v\n", err) - } - - return nil -} - -func exportDOT(writer io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) { - graph := dag.Graph(gdb, from, to) - buf, err := dot.Marshal(graph, "DAG", "", "\t") - if err != nil { - return err - } - - _, err = writer.Write(buf) - if err != nil { - return err - } - - return nil -} From 5af2739e417030ea41b2de8a09c534e5c7140753 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 11 Aug 2023 23:05:56 +1000 Subject: [PATCH 09/22] DAG reprocessing to restore lachesis roots --- cmd/opera/launcher/export.go | 12 +++- utils/dag/dag.go | 5 +- utils/dag/loader.go | 125 ++++++++++++++++++++++++++++++++--- utils/dag/node.go | 10 +++ 4 files changed, 136 insertions(+), 16 deletions(-) diff --git a/cmd/opera/launcher/export.go b/cmd/opera/launcher/export.go index 09e4be276..e137ab29e 100644 --- a/cmd/opera/launcher/export.go +++ b/cmd/opera/launcher/export.go @@ -22,6 +22,7 @@ import ( "gopkg.in/urfave/cli.v1" "github.com/Fantom-foundation/go-opera/gossip" + "github.com/Fantom-foundation/go-opera/integration" "github.com/Fantom-foundation/go-opera/utils/dag" "github.com/Fantom-foundation/go-opera/utils/dbutil/autocompact" ) @@ -84,7 +85,7 @@ func exportEvents(ctx *cli.Context) error { switch { // DOT format: case strings.HasSuffix(fn, ".dot"): - err = exportDOT(writer, gdb, from, to) + err = exportDOT(writer, gdb, cfg, from, to) if err != nil { utils.Fatalf("Export DOT error: %v\n", err) } @@ -134,8 +135,13 @@ func exportRLP(w io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) { } // exportDOT writer the active chain. -func exportDOT(writer io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) { - graph := dag.Graph(gdb, from, to) +func exportDOT(writer io.Writer, gdb *gossip.Store, cfg *config, from, to idx.Epoch) (err error) { + consensusCfg := integration.Configs{ + Lachesis: cfg.Lachesis, + VectorClock: cfg.VectorClock, + } + + graph := dag.Graph(gdb, consensusCfg, from, to) buf, err := dot.Marshal(graph, "DAG", "", "\t") if err != nil { return err diff --git a/utils/dag/dag.go b/utils/dag/dag.go index 585980318..036fe1a48 100644 --- a/utils/dag/dag.go +++ b/utils/dag/dag.go @@ -5,16 +5,17 @@ import ( "gonum.org/v1/gonum/graph/encoding/dot" "github.com/Fantom-foundation/go-opera/gossip" + "github.com/Fantom-foundation/go-opera/integration" ) -func Graph(db *gossip.Store, from, to idx.Epoch) dot.Graph { +func Graph(db *gossip.Store, cfg integration.Configs, from, to idx.Epoch) dot.Graph { /* g:= &dagReader{ db: db, epochFrom: from, epochTo: to, }*/ - g := newDagLoader(db, from, to) + g := newDagLoader(db, cfg, from, to) return g } diff --git a/utils/dag/loader.go b/utils/dag/loader.go index 290ee7f65..a236a1b3c 100644 --- a/utils/dag/loader.go +++ b/utils/dag/loader.go @@ -1,12 +1,22 @@ package dag import ( + "fmt" + + "github.com/Fantom-foundation/lachesis-base/abft" + "github.com/Fantom-foundation/lachesis-base/gossip/dagordering" "github.com/Fantom-foundation/lachesis-base/hash" + "github.com/Fantom-foundation/lachesis-base/inter/dag" "github.com/Fantom-foundation/lachesis-base/inter/idx" + "github.com/Fantom-foundation/lachesis-base/inter/pos" + "github.com/ethereum/go-ethereum/log" "gonum.org/v1/gonum/graph" "github.com/Fantom-foundation/go-opera/gossip" + "github.com/Fantom-foundation/go-opera/integration" "github.com/Fantom-foundation/go-opera/inter" + "github.com/Fantom-foundation/go-opera/utils/adapters/vecmt2dagidx" + "github.com/Fantom-foundation/go-opera/vecmt" ) type dagLoader struct { @@ -14,29 +24,116 @@ type dagLoader struct { nodes map[hash.Event]*dagNode } -func newDagLoader(db *gossip.Store, from, to idx.Epoch) *dagLoader { +func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *dagLoader { g := &dagLoader{ refs: make([]hash.Event, 0, 2000000), nodes: make(map[hash.Event]*dagNode), } - db.ForEachEvent(from, func(e *inter.EventPayload) bool { + store := abft.NewMemStore() + defer store.Close() + // ApplyGenesis() + store.SetEpochState(&abft.EpochState{ + Epoch: from, + }) + store.SetLastDecidedState(&abft.LastDecidedState{ + LastDecidedFrame: abft.FirstFrame - 1, + }) + + dagIndexer := vecmt.NewIndex(panics("Vector clock"), cfg.VectorClock) + orderer := abft.NewOrderer( + store, + &integration.GossipStoreAdapter{gdb}, + vecmt2dagidx.Wrap(dagIndexer), + panics("Lachesis"), + cfg.Lachesis) + + var ( + epoch idx.Epoch + vv = make(pos.ValidatorsBuilder, 60) + ee = make(map[hash.Event]dag.Event, 1000) + ) + err := orderer.Bootstrap(abft.OrdererCallbacks{ + ApplyAtropos: func(decidedFrame idx.Frame, atropos hash.Event) (sealEpoch *pos.Validators) { + return nil + }, + }) + if err != nil { + panic(err) + } + buffer := dagordering.New( + cfg.Opera.Protocol.DagProcessor.EventsBufferLimit, + dagordering.Callback{ + Process: func(e dag.Event) error { + fmt.Printf("<<< E(%d - %d) %s \n", e.Epoch(), e.Frame(), e.ID().String()) + err = dagIndexer.Add(e) + if err != nil { + panic(err) + } + dagIndexer.Flush() + orderer.Process(e) + + id := len(g.refs) + g.refs = append(g.refs, e.ID()) + g.nodes[e.ID()] = &dagNode{ + id: int64(id), + hash: e.ID(), + parents: e.Parents(), + } + return nil + }, + Released: func(e dag.Event, peer string, err error) { + // panic(err) + }, + Get: func(id hash.Event) dag.Event { + return ee[id] + }, + Exists: func(id hash.Event) bool { + _, ok := ee[id] + return ok + }, + }) + + gdb.ForEachEvent(from, func(e *inter.EventPayload) bool { + // current epoch is finished + // , so process accumulated events + if epoch < e.Epoch() { + epoch = e.Epoch() + + validators := vv.Build() + err := orderer.Reset(epoch, validators) + if err != nil { + panic(err) + } + + for _, e := range ee { + fmt.Printf(">>> E(%d - %d) %s \n", e.Epoch(), e.Frame(), e.ID().String()) + buffer.PushEvent(e, "") + } + + for f := idx.Frame(0); f <= store.GetLastDecidedFrame(); f++ { + rr := store.GetFrameRoots(f) + for _, r := range rr { + g.nodes[r.ID].isRoot = true + } + } + + // reset + vv = make(pos.ValidatorsBuilder, 60) + ee = make(map[hash.Event]dag.Event, 1000) + } + // break after last epoch if to >= from && e.Epoch() > to { return false } - - id := len(g.refs) - g.refs = append(g.refs, e.ID()) - g.nodes[e.ID()] = &dagNode{ - id: int64(id), - hash: e.ID(), - parents: e.Parents(), - } + // accumulate epoch's events and validators + ee[e.ID()] = e + vv.Set(e.Creator(), 1) return true }) - db.ForEachBlock(func(index idx.Block, block *inter.Block) { + gdb.ForEachBlock(func(index idx.Block, block *inter.Block) { node, exists := g.nodes[block.Atropos] if exists { node.isAtropos = true @@ -164,3 +261,9 @@ func (g *dagLoader) Edge(uid, vid int64) graph.Edge { return nil } + +func panics(name string) func(error) { + return func(err error) { + log.Crit(fmt.Sprintf("%s error", name), "err", err) + } +} diff --git a/utils/dag/node.go b/utils/dag/node.go index 3dceed183..6b3dd010e 100644 --- a/utils/dag/node.go +++ b/utils/dag/node.go @@ -37,6 +37,7 @@ type dagNode struct { id int64 hash hash.Event parents hash.Events + isRoot bool isAtropos bool } @@ -52,6 +53,15 @@ func (n *dagNode) Attributes() []encoding.Attribute { }, } + if n.isRoot { + aa = append(aa, + encoding.Attribute{ + Key: "role", + Value: "Root", + }, + ) + } + if n.isAtropos { aa = append(aa, encoding.Attribute{ From 7c960b8226acb18eecc278d9d2d67609f3dc4237 Mon Sep 17 00:00:00 2001 From: alex Date: Mon, 14 Aug 2023 23:47:43 +1000 Subject: [PATCH 10/22] tuning --- cmd/opera/launcher/export.go | 1 + utils/dag/loader.go | 27 +++++++++++++++++---------- utils/dag/node.go | 16 ++++++++-------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/cmd/opera/launcher/export.go b/cmd/opera/launcher/export.go index e137ab29e..4277a2e2f 100644 --- a/cmd/opera/launcher/export.go +++ b/cmd/opera/launcher/export.go @@ -137,6 +137,7 @@ func exportRLP(w io.Writer, gdb *gossip.Store, from, to idx.Epoch) (err error) { // exportDOT writer the active chain. func exportDOT(writer io.Writer, gdb *gossip.Store, cfg *config, from, to idx.Epoch) (err error) { consensusCfg := integration.Configs{ + Opera: cfg.Opera, Lachesis: cfg.Lachesis, VectorClock: cfg.VectorClock, } diff --git a/utils/dag/loader.go b/utils/dag/loader.go index a236a1b3c..a9b2e77ee 100644 --- a/utils/dag/loader.go +++ b/utils/dag/loader.go @@ -9,6 +9,7 @@ import ( "github.com/Fantom-foundation/lachesis-base/inter/dag" "github.com/Fantom-foundation/lachesis-base/inter/idx" "github.com/Fantom-foundation/lachesis-base/inter/pos" + "github.com/Fantom-foundation/lachesis-base/kvdb/memorydb" "github.com/ethereum/go-ethereum/log" "gonum.org/v1/gonum/graph" @@ -49,9 +50,10 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch cfg.Lachesis) var ( - epoch idx.Epoch - vv = make(pos.ValidatorsBuilder, 60) - ee = make(map[hash.Event]dag.Event, 1000) + epoch idx.Epoch + vv pos.ValidatorsBuilder // = make(pos.ValidatorsBuilder, 60) + ee map[hash.Event]dag.Event // = make(map[hash.Event]dag.Event, 1000) + processed map[hash.Event]dag.Event // = make(map[hash.Event]dag.Event, 1000) ) err := orderer.Bootstrap(abft.OrdererCallbacks{ ApplyAtropos: func(decidedFrame idx.Frame, atropos hash.Event) (sealEpoch *pos.Validators) { @@ -65,7 +67,7 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch cfg.Opera.Protocol.DagProcessor.EventsBufferLimit, dagordering.Callback{ Process: func(e dag.Event) error { - fmt.Printf("<<< E(%d - %d) %s \n", e.Epoch(), e.Frame(), e.ID().String()) + processed[e.ID()] = e err = dagIndexer.Add(e) if err != nil { panic(err) @@ -79,24 +81,26 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch id: int64(id), hash: e.ID(), parents: e.Parents(), + frame: e.Frame(), } return nil }, Released: func(e dag.Event, peer string, err error) { - // panic(err) + if err != nil { + panic(err) + } }, Get: func(id hash.Event) dag.Event { - return ee[id] + return processed[id] }, Exists: func(id hash.Event) bool { - _, ok := ee[id] + _, ok := processed[id] return ok }, }) gdb.ForEachEvent(from, func(e *inter.EventPayload) bool { - // current epoch is finished - // , so process accumulated events + // current epoch is finished, so process accumulated events if epoch < e.Epoch() { epoch = e.Epoch() @@ -105,9 +109,11 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch if err != nil { panic(err) } + dagIndexer.Reset(validators, memorydb.New(), func(id hash.Event) dag.Event { + return gdb.GetEvent(id) + }) for _, e := range ee { - fmt.Printf(">>> E(%d - %d) %s \n", e.Epoch(), e.Frame(), e.ID().String()) buffer.PushEvent(e, "") } @@ -121,6 +127,7 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch // reset vv = make(pos.ValidatorsBuilder, 60) ee = make(map[hash.Event]dag.Event, 1000) + processed = make(map[hash.Event]dag.Event, 1000) } // break after last epoch if to >= from && e.Epoch() > to { diff --git a/utils/dag/node.go b/utils/dag/node.go index 6b3dd010e..59057fd37 100644 --- a/utils/dag/node.go +++ b/utils/dag/node.go @@ -2,6 +2,7 @@ package dag import ( "github.com/Fantom-foundation/lachesis-base/hash" + "github.com/Fantom-foundation/lachesis-base/inter/idx" "gonum.org/v1/gonum/graph" "gonum.org/v1/gonum/graph/encoding" ) @@ -37,6 +38,7 @@ type dagNode struct { id int64 hash hash.Event parents hash.Events + frame idx.Frame isRoot bool isAtropos bool } @@ -53,20 +55,18 @@ func (n *dagNode) Attributes() []encoding.Attribute { }, } + var role string if n.isRoot { - aa = append(aa, - encoding.Attribute{ - Key: "role", - Value: "Root", - }, - ) + role = "Root" } - if n.isAtropos { + role = "Atropos" + } + if len(role) > 0 { aa = append(aa, encoding.Attribute{ Key: "xlabel", - Value: "Atropos", + Value: role, }, ) } From 7534344d3d276229e64c09c204313f49a4cec173 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 16 Aug 2023 14:07:27 +1000 Subject: [PATCH 11/22] using of gdb.GetHistoryEpochState() --- utils/dag/loader.go | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/utils/dag/loader.go b/utils/dag/loader.go index a9b2e77ee..42e289e1f 100644 --- a/utils/dag/loader.go +++ b/utils/dag/loader.go @@ -51,9 +51,7 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch var ( epoch idx.Epoch - vv pos.ValidatorsBuilder // = make(pos.ValidatorsBuilder, 60) - ee map[hash.Event]dag.Event // = make(map[hash.Event]dag.Event, 1000) - processed map[hash.Event]dag.Event // = make(map[hash.Event]dag.Event, 1000) + processed map[hash.Event]dag.Event ) err := orderer.Bootstrap(abft.OrdererCallbacks{ ApplyAtropos: func(decidedFrame idx.Frame, atropos hash.Event) (sealEpoch *pos.Validators) { @@ -102,40 +100,35 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch gdb.ForEachEvent(from, func(e *inter.EventPayload) bool { // current epoch is finished, so process accumulated events if epoch < e.Epoch() { + // data from restored abft store + for f := idx.Frame(0); f <= store.GetLastDecidedFrame(); f++ { + rr := store.GetFrameRoots(f) + for _, r := range rr { + g.nodes[r.ID].isRoot = true + } + } + + // reset to new epoch epoch = e.Epoch() - validators := vv.Build() - err := orderer.Reset(epoch, validators) + es := gdb.GetHistoryEpochState(epoch) + err := orderer.Reset(epoch, es.Validators) if err != nil { panic(err) } - dagIndexer.Reset(validators, memorydb.New(), func(id hash.Event) dag.Event { + dagIndexer.Reset(es.Validators, memorydb.New(), func(id hash.Event) dag.Event { return gdb.GetEvent(id) }) - for _, e := range ee { - buffer.PushEvent(e, "") - } - - for f := idx.Frame(0); f <= store.GetLastDecidedFrame(); f++ { - rr := store.GetFrameRoots(f) - for _, r := range rr { - g.nodes[r.ID].isRoot = true - } - } - - // reset - vv = make(pos.ValidatorsBuilder, 60) - ee = make(map[hash.Event]dag.Event, 1000) processed = make(map[hash.Event]dag.Event, 1000) } + // break after last epoch if to >= from && e.Epoch() > to { return false } - // accumulate epoch's events and validators - ee[e.ID()] = e - vv.Set(e.Creator(), 1) + // process epoch's event + buffer.PushEvent(e, "") return true }) From a256e6d2142456ba1db89ff042abea18bca7aa87 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 16 Aug 2023 14:51:42 +1000 Subject: [PATCH 12/22] more suitable breakpoint --- utils/dag/loader.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/dag/loader.go b/utils/dag/loader.go index 42e289e1f..df4986d0a 100644 --- a/utils/dag/loader.go +++ b/utils/dag/loader.go @@ -108,9 +108,14 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch } } - // reset to new epoch epoch = e.Epoch() + // break after last epoch + if to >= from && epoch > to { + return false + } + + // reset to new epoch es := gdb.GetHistoryEpochState(epoch) err := orderer.Reset(epoch, es.Validators) if err != nil { @@ -119,14 +124,9 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch dagIndexer.Reset(es.Validators, memorydb.New(), func(id hash.Event) dag.Event { return gdb.GetEvent(id) }) - processed = make(map[hash.Event]dag.Event, 1000) } - // break after last epoch - if to >= from && e.Epoch() > to { - return false - } // process epoch's event buffer.PushEvent(e, "") From 3585e9bb5c65ca87042f6f70a0c3d71f50f381ed Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 16 Aug 2023 16:02:33 +1000 Subject: [PATCH 13/22] iterate only epoch's blocks --- utils/dag/loader.go | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/utils/dag/loader.go b/utils/dag/loader.go index df4986d0a..6fd4bc478 100644 --- a/utils/dag/loader.go +++ b/utils/dag/loader.go @@ -16,6 +16,7 @@ import ( "github.com/Fantom-foundation/go-opera/gossip" "github.com/Fantom-foundation/go-opera/integration" "github.com/Fantom-foundation/go-opera/inter" + "github.com/Fantom-foundation/go-opera/inter/iblockproc" "github.com/Fantom-foundation/go-opera/utils/adapters/vecmt2dagidx" "github.com/Fantom-foundation/go-opera/vecmt" ) @@ -51,6 +52,7 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch var ( epoch idx.Epoch + prevBS *iblockproc.BlockState processed map[hash.Event]dag.Event ) err := orderer.Bootstrap(abft.OrdererCallbacks{ @@ -100,7 +102,11 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch gdb.ForEachEvent(from, func(e *inter.EventPayload) bool { // current epoch is finished, so process accumulated events if epoch < e.Epoch() { - // data from restored abft store + epoch = e.Epoch() + bs, es := gdb.GetHistoryBlockEpochState(epoch) + + // data from restored abft store: + for f := idx.Frame(0); f <= store.GetLastDecidedFrame(); f++ { rr := store.GetFrameRoots(f) for _, r := range rr { @@ -108,15 +114,25 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch } } - epoch = e.Epoch() + if prevBS != nil { + for n := prevBS.LastBlock.Idx + 1; n <= bs.LastBlock.Idx; n++ { + block := gdb.GetBlock(n) + node, exists := g.nodes[block.Atropos] + if exists { + node.isAtropos = true + } + } + } - // break after last epoch + // break after last epoch: if to >= from && epoch > to { return false } - // reset to new epoch - es := gdb.GetHistoryEpochState(epoch) + // reset to new epoch: + + prevBS = bs + processed = make(map[hash.Event]dag.Event, 1000) err := orderer.Reset(epoch, es.Validators) if err != nil { panic(err) @@ -124,7 +140,6 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch dagIndexer.Reset(es.Validators, memorydb.New(), func(id hash.Event) dag.Event { return gdb.GetEvent(id) }) - processed = make(map[hash.Event]dag.Event, 1000) } // process epoch's event @@ -133,13 +148,6 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch return true }) - gdb.ForEachBlock(func(index idx.Block, block *inter.Block) { - node, exists := g.nodes[block.Atropos] - if exists { - node.isAtropos = true - } - }) - return g } From c1d35454bdcefaaba128a65d6374da24063349ee Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 16 Aug 2023 17:44:51 +1000 Subject: [PATCH 14/22] readRestoredAbftStore() for last epoch --- utils/dag/loader.go | 107 ++++++++++++++++++++++++++------------------ 1 file changed, 63 insertions(+), 44 deletions(-) diff --git a/utils/dag/loader.go b/utils/dag/loader.go index 6fd4bc478..efb40934c 100644 --- a/utils/dag/loader.go +++ b/utils/dag/loader.go @@ -2,6 +2,7 @@ package dag import ( "fmt" + "math" "github.com/Fantom-foundation/lachesis-base/abft" "github.com/Fantom-foundation/lachesis-base/gossip/dagordering" @@ -32,29 +33,23 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch nodes: make(map[hash.Event]*dagNode), } - store := abft.NewMemStore() - defer store.Close() + cdb := abft.NewMemStore() + defer cdb.Close() // ApplyGenesis() - store.SetEpochState(&abft.EpochState{ + cdb.SetEpochState(&abft.EpochState{ Epoch: from, }) - store.SetLastDecidedState(&abft.LastDecidedState{ + cdb.SetLastDecidedState(&abft.LastDecidedState{ LastDecidedFrame: abft.FirstFrame - 1, }) dagIndexer := vecmt.NewIndex(panics("Vector clock"), cfg.VectorClock) orderer := abft.NewOrderer( - store, + cdb, &integration.GossipStoreAdapter{gdb}, vecmt2dagidx.Wrap(dagIndexer), panics("Lachesis"), cfg.Lachesis) - - var ( - epoch idx.Epoch - prevBS *iblockproc.BlockState - processed map[hash.Event]dag.Event - ) err := orderer.Bootstrap(abft.OrdererCallbacks{ ApplyAtropos: func(decidedFrame idx.Frame, atropos hash.Event) (sealEpoch *pos.Validators) { return nil @@ -63,6 +58,57 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch if err != nil { panic(err) } + + var ( + epoch idx.Epoch + prevBS *iblockproc.BlockState + processed map[hash.Event]dag.Event + ) + + readRestoredAbftStore := func() { + bs, _ := gdb.GetHistoryBlockEpochState(epoch) + + for f := idx.Frame(0); f <= cdb.GetLastDecidedFrame(); f++ { + rr := cdb.GetFrameRoots(f) + for _, r := range rr { + g.nodes[r.ID].isRoot = true + } + } + + if prevBS != nil { + + maxBlock := idx.Block(math.MaxUint64) + if bs != nil { + maxBlock = bs.LastBlock.Idx + } + + for n := prevBS.LastBlock.Idx + 1; n <= maxBlock; n++ { + block := gdb.GetBlock(n) + if block == nil { + break + } + node, exists := g.nodes[block.Atropos] + if exists { + node.isAtropos = true + } + } + } + + prevBS = bs + } + + resetToNewEpoch := func() { + validators := gdb.GetHistoryEpochState(epoch).Validators + processed = make(map[hash.Event]dag.Event, 1000) + err := orderer.Reset(epoch, validators) + if err != nil { + panic(err) + } + dagIndexer.Reset(validators, memorydb.New(), func(id hash.Event) dag.Event { + return gdb.GetEvent(id) + }) + } + buffer := dagordering.New( cfg.Opera.Protocol.DagProcessor.EventsBufferLimit, dagordering.Callback{ @@ -99,54 +145,27 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch }, }) + // process events gdb.ForEachEvent(from, func(e *inter.EventPayload) bool { // current epoch is finished, so process accumulated events if epoch < e.Epoch() { - epoch = e.Epoch() - bs, es := gdb.GetHistoryBlockEpochState(epoch) - - // data from restored abft store: - - for f := idx.Frame(0); f <= store.GetLastDecidedFrame(); f++ { - rr := store.GetFrameRoots(f) - for _, r := range rr { - g.nodes[r.ID].isRoot = true - } - } - - if prevBS != nil { - for n := prevBS.LastBlock.Idx + 1; n <= bs.LastBlock.Idx; n++ { - block := gdb.GetBlock(n) - node, exists := g.nodes[block.Atropos] - if exists { - node.isAtropos = true - } - } - } + readRestoredAbftStore() + epoch = e.Epoch() // break after last epoch: if to >= from && epoch > to { return false } - // reset to new epoch: - - prevBS = bs - processed = make(map[hash.Event]dag.Event, 1000) - err := orderer.Reset(epoch, es.Validators) - if err != nil { - panic(err) - } - dagIndexer.Reset(es.Validators, memorydb.New(), func(id hash.Event) dag.Event { - return gdb.GetEvent(id) - }) + resetToNewEpoch() } - // process epoch's event buffer.PushEvent(e, "") return true }) + epoch++ + readRestoredAbftStore() return g } From 54c86c7edf6cda698d07671ec7f3c216a8cffe12 Mon Sep 17 00:00:00 2001 From: alex Date: Wed, 16 Aug 2023 18:01:49 +1000 Subject: [PATCH 15/22] rm unused --- utils/dag/loader.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/utils/dag/loader.go b/utils/dag/loader.go index efb40934c..018489aa1 100644 --- a/utils/dag/loader.go +++ b/utils/dag/loader.go @@ -9,7 +9,6 @@ import ( "github.com/Fantom-foundation/lachesis-base/hash" "github.com/Fantom-foundation/lachesis-base/inter/dag" "github.com/Fantom-foundation/lachesis-base/inter/idx" - "github.com/Fantom-foundation/lachesis-base/inter/pos" "github.com/Fantom-foundation/lachesis-base/kvdb/memorydb" "github.com/ethereum/go-ethereum/log" "gonum.org/v1/gonum/graph" @@ -50,11 +49,7 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch vecmt2dagidx.Wrap(dagIndexer), panics("Lachesis"), cfg.Lachesis) - err := orderer.Bootstrap(abft.OrdererCallbacks{ - ApplyAtropos: func(decidedFrame idx.Frame, atropos hash.Event) (sealEpoch *pos.Validators) { - return nil - }, - }) + err := orderer.Bootstrap(abft.OrdererCallbacks{}) if err != nil { panic(err) } From 73b5caed1bc6d0bdfe45783534c6d00fe21768af Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 25 Aug 2023 13:21:23 +1000 Subject: [PATCH 16/22] renaming --- utils/dag/dag.go | 4 ++-- utils/dag/{node.go => dot.go} | 0 utils/dag/{loader.go => graph_inmem.go} | 24 +++++++++++++----------- utils/dag/{graph.go => graph_ondisk.go} | 18 +++++++++--------- 4 files changed, 24 insertions(+), 22 deletions(-) rename utils/dag/{node.go => dot.go} (100%) rename utils/dag/{loader.go => graph_inmem.go} (89%) rename utils/dag/{graph.go => graph_ondisk.go} (86%) diff --git a/utils/dag/dag.go b/utils/dag/dag.go index 036fe1a48..be3eca67c 100644 --- a/utils/dag/dag.go +++ b/utils/dag/dag.go @@ -9,13 +9,13 @@ import ( ) func Graph(db *gossip.Store, cfg integration.Configs, from, to idx.Epoch) dot.Graph { - /* g:= &dagReader{ + /* g:= &graphOnDisk{ db: db, epochFrom: from, epochTo: to, }*/ - g := newDagLoader(db, cfg, from, to) + g := readDagGraph(db, cfg, from, to) return g } diff --git a/utils/dag/node.go b/utils/dag/dot.go similarity index 100% rename from utils/dag/node.go rename to utils/dag/dot.go diff --git a/utils/dag/loader.go b/utils/dag/graph_inmem.go similarity index 89% rename from utils/dag/loader.go rename to utils/dag/graph_inmem.go index 018489aa1..1257c0a81 100644 --- a/utils/dag/loader.go +++ b/utils/dag/graph_inmem.go @@ -21,13 +21,15 @@ import ( "github.com/Fantom-foundation/go-opera/vecmt" ) -type dagLoader struct { +// graphInMem implements dot.Graph over inmem refs and nodes +type graphInMem struct { refs []hash.Event nodes map[hash.Event]*dagNode } -func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *dagLoader { - g := &dagLoader{ +// readDagGraph read gossip.Store into inmem dot.Graph +func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *graphInMem { + g := &graphInMem{ refs: make([]hash.Event, 0, 2000000), nodes: make(map[hash.Event]*dagNode), } @@ -165,13 +167,13 @@ func newDagLoader(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch return g } -func (g *dagLoader) DOTID() string { +func (g *graphInMem) DOTID() string { return "DAG" } // Node returns the node with the given ID if it exists // in the graph, and nil otherwise. -func (g *dagLoader) Node(id int64) graph.Node { +func (g *graphInMem) Node(id int64) graph.Node { hash := g.refs[id] return g.nodes[hash] } @@ -179,7 +181,7 @@ func (g *dagLoader) Node(id int64) graph.Node { // Nodes returns all the nodes in the graph. // // Nodes must not return nil. -func (g *dagLoader) Nodes() graph.Nodes { +func (g *graphInMem) Nodes() graph.Nodes { nn := &dagNodes{ data: make(chan *dagNode), } @@ -199,7 +201,7 @@ func (g *dagLoader) Nodes() graph.Nodes { // from the node with the given ID. // // From must not return nil. -func (g *dagLoader) From(id int64) graph.Nodes { +func (g *graphInMem) From(id int64) graph.Nodes { nn := &dagNodes{ data: make(chan *dagNode), } @@ -221,7 +223,7 @@ func (g *dagLoader) From(id int64) graph.Nodes { // to the node with the given ID. // // To must not return nil. -func (g *dagLoader) To(id int64) graph.Nodes { +func (g *graphInMem) To(id int64) graph.Nodes { nn := &dagNodes{ data: make(chan *dagNode), } @@ -231,7 +233,7 @@ func (g *dagLoader) To(id int64) graph.Nodes { // HasEdgeBetween returns whether an edge exists between // nodes with IDs xid and yid without considering direction. -func (g *dagLoader) HasEdgeBetween(xid, yid int64) bool { +func (g *graphInMem) HasEdgeBetween(xid, yid int64) bool { x := g.nodes[g.refs[xid]] y := g.nodes[g.refs[yid]] @@ -251,7 +253,7 @@ func (g *dagLoader) HasEdgeBetween(xid, yid int64) bool { // HasEdgeFromTo returns whether an edge exists // in the graph from u to v with IDs uid and vid. -func (g *dagLoader) HasEdgeFromTo(uid, vid int64) bool { +func (g *graphInMem) HasEdgeFromTo(uid, vid int64) bool { u := g.nodes[g.refs[uid]] v := g.nodes[g.refs[vid]] @@ -268,7 +270,7 @@ func (g *dagLoader) HasEdgeFromTo(uid, vid int64) bool { // if such an edge exists and nil otherwise. The node v // must be directly reachable from u as defined by the // From method. -func (g *dagLoader) Edge(uid, vid int64) graph.Edge { +func (g *graphInMem) Edge(uid, vid int64) graph.Edge { u := g.nodes[g.refs[uid]] v := g.nodes[g.refs[vid]] diff --git a/utils/dag/graph.go b/utils/dag/graph_ondisk.go similarity index 86% rename from utils/dag/graph.go rename to utils/dag/graph_ondisk.go index 4c49dc981..d446d87b3 100644 --- a/utils/dag/graph.go +++ b/utils/dag/graph_ondisk.go @@ -10,19 +10,19 @@ import ( ) // dagReader implements dot.Graph over gossip.Store -type dagReader struct { +type graphOnDisk struct { db *gossip.Store epochFrom idx.Epoch epochTo idx.Epoch } -func (g *dagReader) DOTID() string { +func (g *graphOnDisk) DOTID() string { return "DAG" } // Node returns the node with the given ID if it exists // in the graph, and nil otherwise. -func (g *dagReader) Node(id int64) graph.Node { +func (g *graphOnDisk) Node(id int64) graph.Node { e := g.db.GetEvent(id2event(id)) return event2node(e) } @@ -30,7 +30,7 @@ func (g *dagReader) Node(id int64) graph.Node { // Nodes returns all the nodes in the graph. // // Nodes must not return nil. -func (g *dagReader) Nodes() graph.Nodes { +func (g *graphOnDisk) Nodes() graph.Nodes { nn := &dagNodes{ data: make(chan *dagNode), } @@ -54,7 +54,7 @@ func (g *dagReader) Nodes() graph.Nodes { // from the node with the given ID. // // From must not return nil. -func (g *dagReader) From(id int64) graph.Nodes { +func (g *graphOnDisk) From(id int64) graph.Nodes { nn := &dagNodes{ data: make(chan *dagNode), } @@ -75,7 +75,7 @@ func (g *dagReader) From(id int64) graph.Nodes { // to the node with the given ID. // // To must not return nil. -func (g *dagReader) To(id int64) graph.Nodes { +func (g *graphOnDisk) To(id int64) graph.Nodes { nn := &dagNodes{ data: make(chan *dagNode), } @@ -85,7 +85,7 @@ func (g *dagReader) To(id int64) graph.Nodes { // HasEdgeBetween returns whether an edge exists between // nodes with IDs xid and yid without considering direction. -func (g *dagReader) HasEdgeBetween(xid, yid int64) bool { +func (g *graphOnDisk) HasEdgeBetween(xid, yid int64) bool { x := g.Node(xid).(*dagNode) y := g.Node(yid).(*dagNode) @@ -105,7 +105,7 @@ func (g *dagReader) HasEdgeBetween(xid, yid int64) bool { // HasEdgeFromTo returns whether an edge exists // in the graph from u to v with IDs uid and vid. -func (g *dagReader) HasEdgeFromTo(uid, vid int64) bool { +func (g *graphOnDisk) HasEdgeFromTo(uid, vid int64) bool { u := g.Node(uid).(*dagNode) v := g.Node(vid).(*dagNode) @@ -122,7 +122,7 @@ func (g *dagReader) HasEdgeFromTo(uid, vid int64) bool { // if such an edge exists and nil otherwise. The node v // must be directly reachable from u as defined by the // From method. -func (g *dagReader) Edge(uid, vid int64) graph.Edge { +func (g *graphOnDisk) Edge(uid, vid int64) graph.Edge { u := g.Node(uid).(*dagNode) v := g.Node(vid).(*dagNode) From be1930f94058b0dd6b6c27a629863be51d59533f Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 25 Aug 2023 14:37:45 +1000 Subject: [PATCH 17/22] dot coloring --- utils/dag/dot.go | 77 +++++++++++++++++++++------------------ utils/dag/graph_inmem.go | 36 +++++++++++------- utils/dag/graph_ondisk.go | 37 ++++++++----------- 3 files changed, 79 insertions(+), 71 deletions(-) diff --git a/utils/dag/dot.go b/utils/dag/dot.go index 59057fd37..2b2657a2d 100644 --- a/utils/dag/dot.go +++ b/utils/dag/dot.go @@ -2,25 +2,26 @@ package dag import ( "github.com/Fantom-foundation/lachesis-base/hash" + "github.com/Fantom-foundation/lachesis-base/inter/dag" "github.com/Fantom-foundation/lachesis-base/inter/idx" "gonum.org/v1/gonum/graph" "gonum.org/v1/gonum/graph/encoding" ) -// Edge is a graph edge. In directed graphs, the direction of the +// dotEdge is a graph edge. In directed graphs, the direction of the // edge is given from -> to, otherwise the edge is semantically // unordered. -type dagEdge struct { - x, y *dagNode +type dotEdge struct { + x, y *dotNode } // From returns the from node of the edge. -func (e *dagEdge) From() graph.Node { +func (e *dotEdge) From() graph.Node { return e.x } // To returns the to node of the edge. -func (e *dagEdge) To() graph.Node { +func (e *dotEdge) To() graph.Node { return e.y } @@ -30,53 +31,59 @@ func (e *dagEdge) To() graph.Node { // the receiver with nodes of the receiver swapped should // be returned, otherwise the receiver should be returned // unaltered. -func (e *dagEdge) ReversedEdge() graph.Edge { +func (e *dotEdge) ReversedEdge() graph.Edge { return nil } -type dagNode struct { - id int64 - hash hash.Event - parents hash.Events - frame idx.Frame - isRoot bool - isAtropos bool +// dotNode is a graph node. +type dotNode struct { + id int64 + hash hash.Event + parents hash.Events + frame idx.Frame + attrs map[string]string } -func (n *dagNode) ID() int64 { +func newDotNode(id int64, e dag.Event) *dotNode { + n := &dotNode{ + id: id, + hash: e.ID(), + parents: e.Parents(), + attrs: make(map[string]string, 10), + } + n.setAttr("label", n.hash.String()) + return n +} + +func (n *dotNode) ID() int64 { return n.id } -func (n *dagNode) Attributes() []encoding.Attribute { - aa := []encoding.Attribute{ - encoding.Attribute{ - Key: "label", - Value: n.hash.String(), - }, - } +func (n *dotNode) Attributes() []encoding.Attribute { + aa := make([]encoding.Attribute, 0, len(n.attrs)) - var role string - if n.isRoot { - role = "Root" - } - if n.isAtropos { - role = "Atropos" - } - if len(role) > 0 { + for k, v := range n.attrs { aa = append(aa, encoding.Attribute{ - Key: "xlabel", - Value: role, - }, - ) + Key: k, + Value: v, + }) } return aa } +func (n *dotNode) setAttr(key, val string) { + if val == "" { + delete(n.attrs, key) + return + } + n.attrs[key] = val +} + type dagNodes struct { - data chan *dagNode - current *dagNode + data chan *dotNode + current *dotNode } // Reset returns the iterator to its start position. diff --git a/utils/dag/graph_inmem.go b/utils/dag/graph_inmem.go index 1257c0a81..9e1208199 100644 --- a/utils/dag/graph_inmem.go +++ b/utils/dag/graph_inmem.go @@ -24,14 +24,14 @@ import ( // graphInMem implements dot.Graph over inmem refs and nodes type graphInMem struct { refs []hash.Event - nodes map[hash.Event]*dagNode + nodes map[hash.Event]*dotNode } // readDagGraph read gossip.Store into inmem dot.Graph func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *graphInMem { g := &graphInMem{ refs: make([]hash.Event, 0, 2000000), - nodes: make(map[hash.Event]*dagNode), + nodes: make(map[hash.Event]*dotNode), } cdb := abft.NewMemStore() @@ -68,7 +68,8 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch for f := idx.Frame(0); f <= cdb.GetLastDecidedFrame(); f++ { rr := cdb.GetFrameRoots(f) for _, r := range rr { - g.nodes[r.ID].isRoot = true + node := g.nodes[r.ID] + markAsRoot(node) } } @@ -86,7 +87,7 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch } node, exists := g.nodes[block.Atropos] if exists { - node.isAtropos = true + markAsAtropos(node) } } } @@ -120,12 +121,7 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch id := len(g.refs) g.refs = append(g.refs, e.ID()) - g.nodes[e.ID()] = &dagNode{ - id: int64(id), - hash: e.ID(), - parents: e.Parents(), - frame: e.Frame(), - } + g.nodes[e.ID()] = newDotNode(int64(id), e) return nil }, Released: func(e dag.Event, peer string, err error) { @@ -183,7 +179,7 @@ func (g *graphInMem) Node(id int64) graph.Node { // Nodes must not return nil. func (g *graphInMem) Nodes() graph.Nodes { nn := &dagNodes{ - data: make(chan *dagNode), + data: make(chan *dotNode), } go func() { @@ -203,7 +199,7 @@ func (g *graphInMem) Nodes() graph.Nodes { // From must not return nil. func (g *graphInMem) From(id int64) graph.Nodes { nn := &dagNodes{ - data: make(chan *dagNode), + data: make(chan *dotNode), } h := g.refs[id] @@ -225,7 +221,7 @@ func (g *graphInMem) From(id int64) graph.Nodes { // To must not return nil. func (g *graphInMem) To(id int64) graph.Nodes { nn := &dagNodes{ - data: make(chan *dagNode), + data: make(chan *dotNode), } close(nn.data) return nn @@ -276,7 +272,7 @@ func (g *graphInMem) Edge(uid, vid int64) graph.Edge { for _, p := range u.parents { if p == v.hash { - return &dagEdge{ + return &dotEdge{ x: u, y: v, } @@ -291,3 +287,15 @@ func panics(name string) func(error) { log.Crit(fmt.Sprintf("%s error", name), "err", err) } } + +func markAsRoot(n *dotNode) { + n.setAttr("xlabel", "root") + n.setAttr("style", "filled") + n.setAttr("fillcolor", "#FFFF00") +} + +func markAsAtropos(n *dotNode) { + n.setAttr("xlabel", "atropos") + n.setAttr("style", "filled") + n.setAttr("fillcolor", "#FF0000") +} diff --git a/utils/dag/graph_ondisk.go b/utils/dag/graph_ondisk.go index d446d87b3..daff459e3 100644 --- a/utils/dag/graph_ondisk.go +++ b/utils/dag/graph_ondisk.go @@ -24,7 +24,7 @@ func (g *graphOnDisk) DOTID() string { // in the graph, and nil otherwise. func (g *graphOnDisk) Node(id int64) graph.Node { e := g.db.GetEvent(id2event(id)) - return event2node(e) + return newDotNode(id, e) } // Nodes returns all the nodes in the graph. @@ -32,7 +32,7 @@ func (g *graphOnDisk) Node(id int64) graph.Node { // Nodes must not return nil. func (g *graphOnDisk) Nodes() graph.Nodes { nn := &dagNodes{ - data: make(chan *dagNode), + data: make(chan *dotNode), } go func() { @@ -42,7 +42,8 @@ func (g *graphOnDisk) Nodes() graph.Nodes { return false } - nn.data <- event2node(&e.Event) + id := event2id(e.ID()) + nn.data <- newDotNode(id, &e.Event) return true }) }() @@ -56,15 +57,15 @@ func (g *graphOnDisk) Nodes() graph.Nodes { // From must not return nil. func (g *graphOnDisk) From(id int64) graph.Nodes { nn := &dagNodes{ - data: make(chan *dagNode), + data: make(chan *dotNode), } - x := g.Node(id).(*dagNode) + x := g.Node(id).(*dotNode) go func() { defer close(nn.data) for _, p := range x.parents { n := g.Node(event2id(p)) - nn.data <- n.(*dagNode) + nn.data <- n.(*dotNode) } }() @@ -77,7 +78,7 @@ func (g *graphOnDisk) From(id int64) graph.Nodes { // To must not return nil. func (g *graphOnDisk) To(id int64) graph.Nodes { nn := &dagNodes{ - data: make(chan *dagNode), + data: make(chan *dotNode), } close(nn.data) return nn @@ -86,8 +87,8 @@ func (g *graphOnDisk) To(id int64) graph.Nodes { // HasEdgeBetween returns whether an edge exists between // nodes with IDs xid and yid without considering direction. func (g *graphOnDisk) HasEdgeBetween(xid, yid int64) bool { - x := g.Node(xid).(*dagNode) - y := g.Node(yid).(*dagNode) + x := g.Node(xid).(*dotNode) + y := g.Node(yid).(*dotNode) for _, p := range x.parents { if p == y.hash { @@ -106,8 +107,8 @@ func (g *graphOnDisk) HasEdgeBetween(xid, yid int64) bool { // HasEdgeFromTo returns whether an edge exists // in the graph from u to v with IDs uid and vid. func (g *graphOnDisk) HasEdgeFromTo(uid, vid int64) bool { - u := g.Node(uid).(*dagNode) - v := g.Node(vid).(*dagNode) + u := g.Node(uid).(*dotNode) + v := g.Node(vid).(*dotNode) for _, p := range u.parents { if p == v.hash { @@ -123,12 +124,12 @@ func (g *graphOnDisk) HasEdgeFromTo(uid, vid int64) bool { // must be directly reachable from u as defined by the // From method. func (g *graphOnDisk) Edge(uid, vid int64) graph.Edge { - u := g.Node(uid).(*dagNode) - v := g.Node(vid).(*dagNode) + u := g.Node(uid).(*dotNode) + v := g.Node(vid).(*dotNode) for _, p := range u.parents { if p == v.hash { - return &dagEdge{ + return &dotEdge{ x: u, y: v, } @@ -140,14 +141,6 @@ func (g *graphOnDisk) Edge(uid, vid int64) graph.Edge { // -- -func event2node(e *inter.Event) *dagNode { - return &dagNode{ - id: event2id(e.ID()), - hash: e.ID(), - parents: e.Parents(), - } -} - var ( id2hash = make(map[int64]hash.Event) ) From 133b6817f566c04a0cff6651a03d22641c3af1d4 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 25 Aug 2023 15:19:24 +1000 Subject: [PATCH 18/22] dot attributes --- utils/dag/dot.go | 63 ++++++++++++++++++++++------------------ utils/dag/graph_inmem.go | 22 ++++++++++++++ 2 files changed, 56 insertions(+), 29 deletions(-) diff --git a/utils/dag/dot.go b/utils/dag/dot.go index 2b2657a2d..6af7f1e80 100644 --- a/utils/dag/dot.go +++ b/utils/dag/dot.go @@ -3,7 +3,6 @@ package dag import ( "github.com/Fantom-foundation/lachesis-base/hash" "github.com/Fantom-foundation/lachesis-base/inter/dag" - "github.com/Fantom-foundation/lachesis-base/inter/idx" "gonum.org/v1/gonum/graph" "gonum.org/v1/gonum/graph/encoding" ) @@ -40,16 +39,15 @@ type dotNode struct { id int64 hash hash.Event parents hash.Events - frame idx.Frame - attrs map[string]string + attributer } func newDotNode(id int64, e dag.Event) *dotNode { n := &dotNode{ - id: id, - hash: e.ID(), - parents: e.Parents(), - attrs: make(map[string]string, 10), + id: id, + hash: e.ID(), + parents: e.Parents(), + attributer: attributer(make(map[string]string, 10)), } n.setAttr("label", n.hash.String()) return n @@ -59,28 +57,6 @@ func (n *dotNode) ID() int64 { return n.id } -func (n *dotNode) Attributes() []encoding.Attribute { - aa := make([]encoding.Attribute, 0, len(n.attrs)) - - for k, v := range n.attrs { - aa = append(aa, - encoding.Attribute{ - Key: k, - Value: v, - }) - } - - return aa -} - -func (n *dotNode) setAttr(key, val string) { - if val == "" { - delete(n.attrs, key) - return - } - n.attrs[key] = val -} - type dagNodes struct { data chan *dotNode current *dotNode @@ -125,3 +101,32 @@ func (nn *dagNodes) Node() graph.Node { func (nn *dagNodes) Len() int { return -1 } + +// -- + +// Attributer implements encoding.Attributer over kv-map +type attributer map[string]string + +// defines graph.Node or graph.Edge values that can +// specify graph attributes. +func (a attributer) Attributes() []encoding.Attribute { + aa := make([]encoding.Attribute, 0, len(a)) + + for k, v := range a { + aa = append(aa, + encoding.Attribute{ + Key: k, + Value: v, + }) + } + + return aa +} + +func (a attributer) setAttr(key, val string) { + if val == "" { + delete(a, key) + return + } + a[key] = val +} diff --git a/utils/dag/graph_inmem.go b/utils/dag/graph_inmem.go index 9e1208199..1db500231 100644 --- a/utils/dag/graph_inmem.go +++ b/utils/dag/graph_inmem.go @@ -12,6 +12,7 @@ import ( "github.com/Fantom-foundation/lachesis-base/kvdb/memorydb" "github.com/ethereum/go-ethereum/log" "gonum.org/v1/gonum/graph" + "gonum.org/v1/gonum/graph/encoding" "github.com/Fantom-foundation/go-opera/gossip" "github.com/Fantom-foundation/go-opera/integration" @@ -25,6 +26,10 @@ import ( type graphInMem struct { refs []hash.Event nodes map[hash.Event]*dotNode + attrs struct { + graph attributer + edge attributer + } } // readDagGraph read gossip.Store into inmem dot.Graph @@ -32,8 +37,17 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch g := &graphInMem{ refs: make([]hash.Event, 0, 2000000), nodes: make(map[hash.Event]*dotNode), + attrs: struct{ graph, edge attributer }{ + attributer(make(map[string]string, 10)), + attributer(make(map[string]string, 10)), + }, } + g.attrs.graph.setAttr("clusterrank", "local") + g.attrs.graph.setAttr("compound", "true") + g.attrs.graph.setAttr("newrank", "true") + g.attrs.graph.setAttr("ranksep", "0.05") + cdb := abft.NewMemStore() defer cdb.Close() // ApplyGenesis() @@ -167,6 +181,14 @@ func (g *graphInMem) DOTID() string { return "DAG" } +// DOTAttributers are graph.Graph values that specify top-level DOT attributes. +func (g *graphInMem) DOTAttributers() (graph, node, edge encoding.Attributer) { + graph = g.attrs.graph + node = attributer(make(map[string]string, 0)) // empty + edge = g.attrs.edge + return +} + // Node returns the node with the given ID if it exists // in the graph, and nil otherwise. func (g *graphInMem) Node(id int64) graph.Node { From f377d0879ea170c75fc78c15ff4d43a533337560 Mon Sep 17 00:00:00 2001 From: alex Date: Fri, 25 Aug 2023 16:15:48 +1000 Subject: [PATCH 19/22] validator's col as subgraph --- utils/dag/graph_inmem.go | 49 ++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/utils/dag/graph_inmem.go b/utils/dag/graph_inmem.go index 1db500231..1aa372b15 100644 --- a/utils/dag/graph_inmem.go +++ b/utils/dag/graph_inmem.go @@ -4,6 +4,8 @@ import ( "fmt" "math" + "gonum.org/v1/gonum/graph/encoding/dot" + "github.com/Fantom-foundation/lachesis-base/abft" "github.com/Fantom-foundation/lachesis-base/gossip/dagordering" "github.com/Fantom-foundation/lachesis-base/hash" @@ -24,6 +26,9 @@ import ( // graphInMem implements dot.Graph over inmem refs and nodes type graphInMem struct { + name string + subGraphs []dot.Graph + refs []hash.Event nodes map[hash.Event]*dotNode attrs struct { @@ -32,9 +37,9 @@ type graphInMem struct { } } -// readDagGraph read gossip.Store into inmem dot.Graph -func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *graphInMem { - g := &graphInMem{ +func newGraphInMem(name string) *graphInMem { + return &graphInMem{ + name: name, refs: make([]hash.Event, 0, 2000000), nodes: make(map[hash.Event]*dotNode), attrs: struct{ graph, edge attributer }{ @@ -42,7 +47,11 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch attributer(make(map[string]string, 10)), }, } +} +// readDagGraph read gossip.Store into inmem dot.Graph +func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *graphInMem { + g := newGraphInMem("DOT") g.attrs.graph.setAttr("clusterrank", "local") g.attrs.graph.setAttr("compound", "true") g.attrs.graph.setAttr("newrank", "true") @@ -71,6 +80,7 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch } var ( + graphCols = make(map[idx.ValidatorID]*graphInMem, 10) epoch idx.Epoch prevBS *iblockproc.BlockState processed map[hash.Event]dag.Event @@ -111,6 +121,18 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch resetToNewEpoch := func() { validators := gdb.GetHistoryEpochState(epoch).Validators + + for _, v := range validators.IDs() { + _, ok := graphCols[v] + if ok { + continue + } + col := newGraphInMem(fmt.Sprintf("validator-%d", v)) + col.attrs.graph.setAttr("style", "dotted") + graphCols[v] = col + g.subGraphs = append(g.subGraphs, col) + } + processed = make(map[hash.Event]dag.Event, 1000) err := orderer.Reset(epoch, validators) if err != nil { @@ -133,9 +155,7 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch dagIndexer.Flush() orderer.Process(e) - id := len(g.refs) - g.refs = append(g.refs, e.ID()) - g.nodes[e.ID()] = newDotNode(int64(id), e) + graphCols[e.Creator()].addDagEvent(e) return nil }, Released: func(e dag.Event, peer string, err error) { @@ -178,7 +198,12 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch } func (g *graphInMem) DOTID() string { - return "DAG" + return g.name +} + +// Structure returns subgraphs. +func (g *graphInMem) Structure() []dot.Graph { + return g.subGraphs } // DOTAttributers are graph.Graph values that specify top-level DOT attributes. @@ -189,6 +214,16 @@ func (g *graphInMem) DOTAttributers() (graph, node, edge encoding.Attributer) { return } +// TODO: global refs for subgraphs +func (g *graphInMem) addDagEvent(e dag.Event) (id int64) { + id = int64(len(g.refs)) + g.refs = append(g.refs, e.ID()) + n := newDotNode(id, e) + g.nodes[e.ID()] = n + + return +} + // Node returns the node with the given ID if it exists // in the graph, and nil otherwise. func (g *graphInMem) Node(id int64) graph.Node { From dcb48993c7e07b127b3b2c64ac43b6f447d5c4ef Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 31 Aug 2023 12:06:03 +1000 Subject: [PATCH 20/22] subgraphs: common nodes --- utils/dag/graph_inmem.go | 120 ++++++++++++++++++++++++++------------- 1 file changed, 81 insertions(+), 39 deletions(-) diff --git a/utils/dag/graph_inmem.go b/utils/dag/graph_inmem.go index 1aa372b15..11cb7b067 100644 --- a/utils/dag/graph_inmem.go +++ b/utils/dag/graph_inmem.go @@ -4,8 +4,6 @@ import ( "fmt" "math" - "gonum.org/v1/gonum/graph/encoding/dot" - "github.com/Fantom-foundation/lachesis-base/abft" "github.com/Fantom-foundation/lachesis-base/gossip/dagordering" "github.com/Fantom-foundation/lachesis-base/hash" @@ -15,6 +13,7 @@ import ( "github.com/ethereum/go-ethereum/log" "gonum.org/v1/gonum/graph" "gonum.org/v1/gonum/graph/encoding" + "gonum.org/v1/gonum/graph/encoding/dot" "github.com/Fantom-foundation/go-opera/gossip" "github.com/Fantom-foundation/go-opera/integration" @@ -24,14 +23,37 @@ import ( "github.com/Fantom-foundation/go-opera/vecmt" ) +type eventSeq []hash.Event + +func (s *eventSeq) event2id(e dag.Event) int64 { + id := int64(len(*s)) + *s = append(*s, e.ID()) + return id +} + +func (s *eventSeq) id2event(id int64) hash.Event { + return (*s)[id] +} + +type eventList map[hash.Event]*dotNode + +func (l *eventList) set(n *dotNode) { + (*l)[n.hash] = n +} + +func (l *eventList) get(h hash.Event) *dotNode { + return (*l)[h] +} + // graphInMem implements dot.Graph over inmem refs and nodes type graphInMem struct { name string - subGraphs []dot.Graph + subGraphs map[string]*graphInMem - refs []hash.Event - nodes map[hash.Event]*dotNode - attrs struct { + include map[int64]struct{} + refs *eventSeq + nodes *eventList + attrs struct { graph attributer edge attributer } @@ -39,9 +61,10 @@ type graphInMem struct { func newGraphInMem(name string) *graphInMem { return &graphInMem{ - name: name, - refs: make([]hash.Event, 0, 2000000), - nodes: make(map[hash.Event]*dotNode), + name: name, + include: make(map[int64]struct{}), + subGraphs: make(map[string]*graphInMem), + attrs: struct{ graph, edge attributer }{ attributer(make(map[string]string, 10)), attributer(make(map[string]string, 10)), @@ -52,6 +75,8 @@ func newGraphInMem(name string) *graphInMem { // readDagGraph read gossip.Store into inmem dot.Graph func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *graphInMem { g := newGraphInMem("DOT") + g.refs = &eventSeq{} + g.nodes = &eventList{} g.attrs.graph.setAttr("clusterrank", "local") g.attrs.graph.setAttr("compound", "true") g.attrs.graph.setAttr("newrank", "true") @@ -80,7 +105,6 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch } var ( - graphCols = make(map[idx.ValidatorID]*graphInMem, 10) epoch idx.Epoch prevBS *iblockproc.BlockState processed map[hash.Event]dag.Event @@ -92,7 +116,7 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch for f := idx.Frame(0); f <= cdb.GetLastDecidedFrame(); f++ { rr := cdb.GetFrameRoots(f) for _, r := range rr { - node := g.nodes[r.ID] + node := g.nodes.get(r.ID) markAsRoot(node) } } @@ -109,8 +133,8 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch if block == nil { break } - node, exists := g.nodes[block.Atropos] - if exists { + node := g.nodes.get(block.Atropos) + if node != nil { markAsAtropos(node) } } @@ -123,14 +147,7 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch validators := gdb.GetHistoryEpochState(epoch).Validators for _, v := range validators.IDs() { - _, ok := graphCols[v] - if ok { - continue - } - col := newGraphInMem(fmt.Sprintf("validator-%d", v)) - col.attrs.graph.setAttr("style", "dotted") - graphCols[v] = col - g.subGraphs = append(g.subGraphs, col) + _ = g.subGraph(v) } processed = make(map[hash.Event]dag.Event, 1000) @@ -155,7 +172,8 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch dagIndexer.Flush() orderer.Process(e) - graphCols[e.Creator()].addDagEvent(e) + col := g.subGraph(e.Creator()) + col.addDagEvent(e) return nil }, Released: func(e dag.Event, peer string, err error) { @@ -203,7 +221,27 @@ func (g *graphInMem) DOTID() string { // Structure returns subgraphs. func (g *graphInMem) Structure() []dot.Graph { - return g.subGraphs + res := make([]dot.Graph, 0, len(g.subGraphs)) + for _, sg := range g.subGraphs { + res = append(res, sg) + } + return res +} + +func (g *graphInMem) subGraph(v idx.ValidatorID) *graphInMem { + name := fmt.Sprintf("validator-%d", v) + sg, ok := g.subGraphs[name] + if !ok { + sg = newGraphInMem(name) + sg.refs = g.refs + sg.nodes = g.nodes + sg.attrs.graph.setAttr("label", name) + sg.attrs.graph.setAttr("sortv", fmt.Sprintf("%d", v)) + sg.attrs.graph.setAttr("style", "dotted") + g.subGraphs[name] = sg + + } + return sg } // DOTAttributers are graph.Graph values that specify top-level DOT attributes. @@ -214,12 +252,11 @@ func (g *graphInMem) DOTAttributers() (graph, node, edge encoding.Attributer) { return } -// TODO: global refs for subgraphs func (g *graphInMem) addDagEvent(e dag.Event) (id int64) { - id = int64(len(g.refs)) - g.refs = append(g.refs, e.ID()) + id = g.refs.event2id(e) n := newDotNode(id, e) - g.nodes[e.ID()] = n + g.nodes.set(n) + g.include[id] = struct{}{} return } @@ -227,8 +264,11 @@ func (g *graphInMem) addDagEvent(e dag.Event) (id int64) { // Node returns the node with the given ID if it exists // in the graph, and nil otherwise. func (g *graphInMem) Node(id int64) graph.Node { - hash := g.refs[id] - return g.nodes[hash] + if _, ok := g.include[id]; !ok { + return nil + } + hash := g.refs.id2event(id) + return g.nodes.get(hash) } // Nodes returns all the nodes in the graph. @@ -242,7 +282,9 @@ func (g *graphInMem) Nodes() graph.Nodes { go func() { defer close(nn.data) - for _, e := range g.nodes { + for id := range g.include { + h := g.refs.id2event(id) + e := g.nodes.get(h) nn.data <- e } }() @@ -259,12 +301,12 @@ func (g *graphInMem) From(id int64) graph.Nodes { data: make(chan *dotNode), } - h := g.refs[id] - x := g.nodes[h] + h := g.refs.id2event(id) + x := g.nodes.get(h) go func() { defer close(nn.data) for _, p := range x.parents { - n := g.nodes[p] + n := g.nodes.get(p) nn.data <- n } }() @@ -287,8 +329,8 @@ func (g *graphInMem) To(id int64) graph.Nodes { // HasEdgeBetween returns whether an edge exists between // nodes with IDs xid and yid without considering direction. func (g *graphInMem) HasEdgeBetween(xid, yid int64) bool { - x := g.nodes[g.refs[xid]] - y := g.nodes[g.refs[yid]] + x := g.nodes.get(g.refs.id2event(xid)) + y := g.nodes.get(g.refs.id2event(yid)) for _, p := range x.parents { if p == y.hash { @@ -307,8 +349,8 @@ func (g *graphInMem) HasEdgeBetween(xid, yid int64) bool { // HasEdgeFromTo returns whether an edge exists // in the graph from u to v with IDs uid and vid. func (g *graphInMem) HasEdgeFromTo(uid, vid int64) bool { - u := g.nodes[g.refs[uid]] - v := g.nodes[g.refs[vid]] + u := g.nodes.get(g.refs.id2event(uid)) + v := g.nodes.get(g.refs.id2event(vid)) for _, p := range u.parents { if p == v.hash { @@ -324,8 +366,8 @@ func (g *graphInMem) HasEdgeFromTo(uid, vid int64) bool { // must be directly reachable from u as defined by the // From method. func (g *graphInMem) Edge(uid, vid int64) graph.Edge { - u := g.nodes[g.refs[uid]] - v := g.nodes[g.refs[vid]] + u := g.nodes.get(g.refs.id2event(uid)) + v := g.nodes.get(g.refs.id2event(vid)) for _, p := range u.parents { if p == v.hash { From df5e0758209a02397415f3824a0e3f48f7ec8396 Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 31 Aug 2023 18:19:19 +1000 Subject: [PATCH 21/22] fail when using gonum.org/v1/gonum/graph --- utils/dag/dot.go | 9 +++- utils/dag/graph_inmem.go | 91 +++++++++++++++++++++++++++++++++------- 2 files changed, 84 insertions(+), 16 deletions(-) diff --git a/utils/dag/dot.go b/utils/dag/dot.go index 6af7f1e80..1b0978909 100644 --- a/utils/dag/dot.go +++ b/utils/dag/dot.go @@ -1,6 +1,10 @@ package dag import ( + "fmt" + + "github.com/Fantom-foundation/lachesis-base/inter/idx" + "github.com/Fantom-foundation/lachesis-base/hash" "github.com/Fantom-foundation/lachesis-base/inter/dag" "gonum.org/v1/gonum/graph" @@ -12,6 +16,7 @@ import ( // unordered. type dotEdge struct { x, y *dotNode + attributer } // From returns the from node of the edge. @@ -37,6 +42,7 @@ func (e *dotEdge) ReversedEdge() graph.Edge { // dotNode is a graph node. type dotNode struct { id int64 + creator idx.ValidatorID hash hash.Event parents hash.Events attributer @@ -45,11 +51,12 @@ type dotNode struct { func newDotNode(id int64, e dag.Event) *dotNode { n := &dotNode{ id: id, + creator: e.Creator(), hash: e.ID(), parents: e.Parents(), attributer: attributer(make(map[string]string, 10)), } - n.setAttr("label", n.hash.String()) + n.setAttr("label", fmt.Sprintf("%s\n%d", n.hash.String(), e.Creator())) return n } diff --git a/utils/dag/graph_inmem.go b/utils/dag/graph_inmem.go index 11cb7b067..0a0cc678c 100644 --- a/utils/dag/graph_inmem.go +++ b/utils/dag/graph_inmem.go @@ -25,9 +25,9 @@ import ( type eventSeq []hash.Event -func (s *eventSeq) event2id(e dag.Event) int64 { +func (s *eventSeq) event2id(h hash.Event) int64 { id := int64(len(*s)) - *s = append(*s, e.ID()) + *s = append(*s, h) return id } @@ -48,6 +48,7 @@ func (l *eventList) get(h hash.Event) *dotNode { // graphInMem implements dot.Graph over inmem refs and nodes type graphInMem struct { name string + global bool subGraphs map[string]*graphInMem include map[int64]struct{} @@ -75,12 +76,14 @@ func newGraphInMem(name string) *graphInMem { // readDagGraph read gossip.Store into inmem dot.Graph func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *graphInMem { g := newGraphInMem("DOT") + g.global = true g.refs = &eventSeq{} g.nodes = &eventList{} g.attrs.graph.setAttr("clusterrank", "local") g.attrs.graph.setAttr("compound", "true") g.attrs.graph.setAttr("newrank", "true") g.attrs.graph.setAttr("ranksep", "0.05") + g.attrs.edge.setAttr("constraint", "true") cdb := abft.NewMemStore() defer cdb.Close() @@ -172,8 +175,9 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch dagIndexer.Flush() orderer.Process(e) - col := g.subGraph(e.Creator()) - col.addDagEvent(e) + sg := g.subGraph(e.Creator()) + sg.addDagEvent(e) + return nil }, Released: func(e dag.Event, peer string, err error) { @@ -229,17 +233,19 @@ func (g *graphInMem) Structure() []dot.Graph { } func (g *graphInMem) subGraph(v idx.ValidatorID) *graphInMem { - name := fmt.Sprintf("validator-%d", v) + name := fmt.Sprintf("cluster%d", v) + label := fmt.Sprintf("validator-%d", v) sg, ok := g.subGraphs[name] if !ok { sg = newGraphInMem(name) sg.refs = g.refs sg.nodes = g.nodes - sg.attrs.graph.setAttr("label", name) + sg.attrs.graph.setAttr("label", label) sg.attrs.graph.setAttr("sortv", fmt.Sprintf("%d", v)) sg.attrs.graph.setAttr("style", "dotted") + sg.attrs.edge.setAttr("constraint", "true") g.subGraphs[name] = sg - + sg.addPseudoNode(label) } return sg } @@ -253,11 +259,38 @@ func (g *graphInMem) DOTAttributers() (graph, node, edge encoding.Attributer) { } func (g *graphInMem) addDagEvent(e dag.Event) (id int64) { - id = g.refs.event2id(e) - n := newDotNode(id, e) - g.nodes.set(n) + n := g.nodes.get(e.ID()) + if n == nil { + id = g.refs.event2id(e.ID()) + n = newDotNode(id, e) + g.nodes.set(n) + } else { + id = n.id + } + g.include[id] = struct{}{} + return +} +func (g *graphInMem) addPseudoNode(name string) (id int64) { + h := hash.BytesToEvent([]byte(name)) + n := g.nodes.get(h) + if n == nil { + id = g.refs.event2id(h) + n = &dotNode{ + id: id, + hash: h, + attributer: attributer(make(map[string]string, 10)), + } + n.setAttr("label", name) + n.setAttr("style", "invis") + n.setAttr("width", "0") + g.nodes.set(n) + } else { + id = n.id + } + + g.include[id] = struct{}{} return } @@ -303,11 +336,14 @@ func (g *graphInMem) From(id int64) graph.Nodes { h := g.refs.id2event(id) x := g.nodes.get(h) + go func() { defer close(nn.data) for _, p := range x.parents { - n := g.nodes.get(p) - nn.data <- n + y := g.nodes.get(p) + if g.hasEdge(x, y) { + nn.data <- y + } } }() @@ -331,6 +367,9 @@ func (g *graphInMem) To(id int64) graph.Nodes { func (g *graphInMem) HasEdgeBetween(xid, yid int64) bool { x := g.nodes.get(g.refs.id2event(xid)) y := g.nodes.get(g.refs.id2event(yid)) + if !g.hasEdge(x, y) { + return false + } for _, p := range x.parents { if p == y.hash { @@ -351,6 +390,9 @@ func (g *graphInMem) HasEdgeBetween(xid, yid int64) bool { func (g *graphInMem) HasEdgeFromTo(uid, vid int64) bool { u := g.nodes.get(g.refs.id2event(uid)) v := g.nodes.get(g.refs.id2event(vid)) + if !g.hasEdge(u, v) { + return false + } for _, p := range u.parents { if p == v.hash { @@ -368,19 +410,38 @@ func (g *graphInMem) HasEdgeFromTo(uid, vid int64) bool { func (g *graphInMem) Edge(uid, vid int64) graph.Edge { u := g.nodes.get(g.refs.id2event(uid)) v := g.nodes.get(g.refs.id2event(vid)) + if !g.hasEdge(u, v) { + return nil + } for _, p := range u.parents { if p == v.hash { - return &dotEdge{ + e := &dotEdge{ x: u, y: v, } + return e } } return nil } +func (g *graphInMem) hasEdge(x, y *dotNode) bool { + if _, ok := g.include[x.id]; !ok && !g.global { + return false + } + if _, ok := g.include[y.id]; !ok && !g.global { + return false + } + + if g.global && x.creator == y.creator { + return false + } + + return true +} + func panics(name string) func(error) { return func(err error) { log.Crit(fmt.Sprintf("%s error", name), "err", err) @@ -388,13 +449,13 @@ func panics(name string) func(error) { } func markAsRoot(n *dotNode) { - n.setAttr("xlabel", "root") + // n.setAttr("xlabel", "root") n.setAttr("style", "filled") n.setAttr("fillcolor", "#FFFF00") } func markAsAtropos(n *dotNode) { - n.setAttr("xlabel", "atropos") + // n.setAttr("xlabel", "atropos") n.setAttr("style", "filled") n.setAttr("fillcolor", "#FF0000") } From 982dc680906dadf1b5f8f997d0f7ac288e853f7d Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 31 Aug 2023 21:34:11 +1000 Subject: [PATCH 22/22] github.com/tmc/dot instead of gonum.org/v1/gonum/graph --- cmd/opera/launcher/export.go | 7 +- go.mod | 2 - go.sum | 45 --- utils/dag/dag.go | 10 +- utils/dag/dot.go | 139 --------- utils/dag/dot/LICENSE.md | 7 + utils/dag/dot/README.md | 16 + utils/dag/dot/dot.go | 531 ++++++++++++++++++++++++++++++++++ utils/dag/dot/dot_test.go | 151 ++++++++++ utils/dag/dot/example_test.go | 34 +++ utils/dag/graph_inmem.go | 382 ++++++------------------ utils/dag/graph_ondisk.go | 162 ----------- 12 files changed, 823 insertions(+), 663 deletions(-) delete mode 100644 utils/dag/dot.go create mode 100644 utils/dag/dot/LICENSE.md create mode 100644 utils/dag/dot/README.md create mode 100644 utils/dag/dot/dot.go create mode 100644 utils/dag/dot/dot_test.go create mode 100644 utils/dag/dot/example_test.go delete mode 100644 utils/dag/graph_ondisk.go diff --git a/cmd/opera/launcher/export.go b/cmd/opera/launcher/export.go index 4277a2e2f..5751b7c08 100644 --- a/cmd/opera/launcher/export.go +++ b/cmd/opera/launcher/export.go @@ -18,7 +18,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/status-im/keycard-go/hexutils" "github.com/syndtr/goleveldb/leveldb/opt" - "gonum.org/v1/gonum/graph/encoding/dot" "gopkg.in/urfave/cli.v1" "github.com/Fantom-foundation/go-opera/gossip" @@ -143,12 +142,8 @@ func exportDOT(writer io.Writer, gdb *gossip.Store, cfg *config, from, to idx.Ep } graph := dag.Graph(gdb, consensusCfg, from, to) - buf, err := dot.Marshal(graph, "DAG", "", "\t") - if err != nil { - return err - } - _, err = writer.Write(buf) + _, err = writer.Write([]byte(graph.String())) if err != nil { return err } diff --git a/go.mod b/go.mod index c3dbc3e71..c21bea709 100644 --- a/go.mod +++ b/go.mod @@ -43,8 +43,6 @@ require ( gopkg.in/urfave/cli.v1 v1.20.0 ) -require gonum.org/v1/gonum v0.13.0 - require ( github.com/DataDog/zstd v1.4.5 // indirect github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect diff --git a/go.sum b/go.sum index d3779cf8b..d630269a6 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,6 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= -git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= @@ -74,10 +72,7 @@ github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= -github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -105,8 +100,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -209,7 +202,6 @@ github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlK github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -231,13 +223,6 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= -github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= -github.com/go-fonts/latin-modern v0.3.0/go.mod h1:ysEQXnuT/sCDOAONxC7ImeEDVINbltClhasMAqEtRK0= -github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= -github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= -github.com/go-fonts/liberation v0.3.0/go.mod h1:jdJ+cqF+F4SUL2V+qxBth8fvBpBDS7yloUL5Fi8GTGY= -github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -245,9 +230,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= -github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= -github.com/go-latex/latex v0.0.0-20230307184459-12ec69307ad9/go.mod h1:gWuR/CrFDDeVRFQwHPvsv9soJVB/iqymhuZQuJ3a9OM= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= @@ -257,8 +239,6 @@ github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= -github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -421,7 +401,6 @@ github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0t github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= @@ -538,9 +517,6 @@ github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHu github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= -github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= -github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -592,8 +568,6 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= -github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= @@ -717,7 +691,6 @@ golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -730,17 +703,6 @@ golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZ golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4= -golang.org/x/image v0.6.0/go.mod h1:MXLdDR43H7cDJq5GEGXEVeeNhPgi+YYEQ2pC1byI1x0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -884,7 +846,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -948,7 +909,6 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -995,14 +955,9 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= -gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= -gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= -gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= diff --git a/utils/dag/dag.go b/utils/dag/dag.go index be3eca67c..0630322f2 100644 --- a/utils/dag/dag.go +++ b/utils/dag/dag.go @@ -2,19 +2,13 @@ package dag import ( "github.com/Fantom-foundation/lachesis-base/inter/idx" - "gonum.org/v1/gonum/graph/encoding/dot" "github.com/Fantom-foundation/go-opera/gossip" "github.com/Fantom-foundation/go-opera/integration" + "github.com/Fantom-foundation/go-opera/utils/dag/dot" ) -func Graph(db *gossip.Store, cfg integration.Configs, from, to idx.Epoch) dot.Graph { - /* g:= &graphOnDisk{ - db: db, - epochFrom: from, - epochTo: to, - }*/ - +func Graph(db *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *dot.Graph { g := readDagGraph(db, cfg, from, to) return g diff --git a/utils/dag/dot.go b/utils/dag/dot.go deleted file mode 100644 index 1b0978909..000000000 --- a/utils/dag/dot.go +++ /dev/null @@ -1,139 +0,0 @@ -package dag - -import ( - "fmt" - - "github.com/Fantom-foundation/lachesis-base/inter/idx" - - "github.com/Fantom-foundation/lachesis-base/hash" - "github.com/Fantom-foundation/lachesis-base/inter/dag" - "gonum.org/v1/gonum/graph" - "gonum.org/v1/gonum/graph/encoding" -) - -// dotEdge is a graph edge. In directed graphs, the direction of the -// edge is given from -> to, otherwise the edge is semantically -// unordered. -type dotEdge struct { - x, y *dotNode - attributer -} - -// From returns the from node of the edge. -func (e *dotEdge) From() graph.Node { - return e.x -} - -// To returns the to node of the edge. -func (e *dotEdge) To() graph.Node { - return e.y -} - -// ReversedEdge returns the edge reversal of the receiver -// if a reversal is valid for the data type. -// When a reversal is valid an edge of the same type as -// the receiver with nodes of the receiver swapped should -// be returned, otherwise the receiver should be returned -// unaltered. -func (e *dotEdge) ReversedEdge() graph.Edge { - return nil -} - -// dotNode is a graph node. -type dotNode struct { - id int64 - creator idx.ValidatorID - hash hash.Event - parents hash.Events - attributer -} - -func newDotNode(id int64, e dag.Event) *dotNode { - n := &dotNode{ - id: id, - creator: e.Creator(), - hash: e.ID(), - parents: e.Parents(), - attributer: attributer(make(map[string]string, 10)), - } - n.setAttr("label", fmt.Sprintf("%s\n%d", n.hash.String(), e.Creator())) - return n -} - -func (n *dotNode) ID() int64 { - return n.id -} - -type dagNodes struct { - data chan *dotNode - current *dotNode -} - -// Reset returns the iterator to its start position. -func (nn *dagNodes) Reset() { - panic("Not implemented yet") -} - -// Next advances the iterator and returns whether -// the next call to the item method will return a -// non-nil item. -// -// Next should be called prior to any call to the -// iterator's item retrieval method after the -// iterator has been obtained or reset. -// -// The order of iteration is implementation -// dependent. -func (nn *dagNodes) Next() bool { - nn.current = <-nn.data - return nn.current != nil -} - -// Node returns the current Node from the iterator. -func (nn *dagNodes) Node() graph.Node { - return nn.current -} - -// Len returns the number of items remaining in the -// iterator. -// -// If the number of items in the iterator is unknown, -// too large to materialize or too costly to calculate -// then Len may return a negative value. -// In this case the consuming function must be able -// to operate on the items of the iterator directly -// without materializing the items into a slice. -// The magnitude of a negative length has -// implementation-dependent semantics. -func (nn *dagNodes) Len() int { - return -1 -} - -// -- - -// Attributer implements encoding.Attributer over kv-map -type attributer map[string]string - -// defines graph.Node or graph.Edge values that can -// specify graph attributes. -func (a attributer) Attributes() []encoding.Attribute { - aa := make([]encoding.Attribute, 0, len(a)) - - for k, v := range a { - aa = append(aa, - encoding.Attribute{ - Key: k, - Value: v, - }) - } - - return aa -} - -func (a attributer) setAttr(key, val string) { - if val == "" { - delete(a, key) - return - } - a[key] = val -} diff --git a/utils/dag/dot/LICENSE.md b/utils/dag/dot/LICENSE.md new file mode 100644 index 000000000..99daa6bda --- /dev/null +++ b/utils/dag/dot/LICENSE.md @@ -0,0 +1,7 @@ +Copyright (C) 2012 Travis Cline + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/utils/dag/dot/README.md b/utils/dag/dot/README.md new file mode 100644 index 000000000..bc40901cc --- /dev/null +++ b/utils/dag/dot/README.md @@ -0,0 +1,16 @@ +dot +=== + +dot language support for Go + + +See http://godoc.org/github.com/tmc/dot + +Todo: + +* parser +* regression suite + +The pydot library used as a reference + +License: MIT diff --git a/utils/dag/dot/dot.go b/utils/dag/dot/dot.go new file mode 100644 index 000000000..0c3ec2429 --- /dev/null +++ b/utils/dag/dot/dot.go @@ -0,0 +1,531 @@ +/* +Package dot implements an API to produce Graphviz dot language output. + +Basic Graph creation: + + g := dot.NewGraph("G") + g.SetType(dot.DIGRAPH) + ... + g.AddEdge(dot.NewNode("A"), dot.NewNode("B")) + ... + fmt.Sprint(g) +*/ +package dot + +import ( + "errors" + "fmt" + "regexp" + "sort" + "strings" +) + +var AttributeError = errors.New("Invalid Attribute") + +var graphAttributes = []string{"Damping", "K", "URL", "aspect", "bb", "bgcolor", + "center", "charset", "clusterrank", "colorscheme", "comment", "compound", + "concentrate", "defaultdist", "dim", "dimen", "diredgeconstraints", + "dpi", "epsilon", "esep", "fontcolor", "fontname", "fontnames", + "fontpath", "fontsize", "id", "label", "labeljust", "labelloc", + "landscape", "layers", "layersep", "layout", "levels", "levelsgap", + "lheight", "lp", "lwidth", "margin", "maxiter", "mclimit", "mindist", + "mode", "model", "mosek", "nodesep", "nojustify", "normalize", "nslimit", + "nslimit1", "ordering", "orientation", "outputorder", "overlap", + "overlap_scaling", "pack", "packmode", "pad", "page", "pagedir", + "quadtree", "quantum", "rankdir", "ranksep", "ratio", "remincross", + "repulsiveforce", "resolution", "root", "rotate", "searchsize", "sep", + "showboxes", "style", "size", "smoothing", "sortv", "splines", "start", + "stylesheet", "target", "truecolor", "viewport", "voro_margin", + "rank", "newrank"} + +var edgeAttributes = []string{"URL", "arrowhead", "arrowsize", "arrowtail", + "color", "colorscheme", "comment", "constraint", "decorate", "dir", + "edgeURL", "edgehref", "edgetarget", "edgetooltip", "fontcolor", + "fontname", "fontsize", "headURL", "headclip", "headhref", "headlabel", + "headport", "headtarget", "headtooltip", "href", "id", "label", + "labelURL", "labelangle", "labeldistance", "labelfloat", "labelfontcolor", + "labelfontname", "labelfontsize", "labelhref", "labeltarget", + "labeltooltip", "layer", "len", "lhead", "lp", "ltail", "minlen", + "nojustify", "penwidth", "pos", "samehead", "sametail", "showboxes", + "style", "tailURL", "tailclip", "tailhref", "taillabel", "tailport", + "tailtarget", "tailtooltip", "target", "tooltip", "weight", + // for subgraphs + "rank"} + +var nodeAttributes = []string{"URL", "color", "colorscheme", "comment", + "distortion", "fillcolor", "fixedsize", "fontcolor", "fontname", + "fontsize", "group", "height", "id", "image", "imagescale", "label", + "labelloc", "layer", "margin", "nojustify", "orientation", "penwidth", + "peripheries", "pin", "pos", "rects", "regular", "root", "samplepoints", + "shape", "shapefile", "showboxes", "sides", "skew", "sortv", "style", + "target", "tooltip", "vertices", "width", "z", + // The following are attributes dot2tex + "texlbl", "texmode"} + +var clusterAttributes = []string{"K", "URL", "bgcolor", "color", "colorscheme", + "fillcolor", "fontcolor", "fontname", "fontsize", "label", "labeljust", + "labelloc", "lheight", "lp", "lwidth", "nojustify", "pencolor", + "penwidth", "peripheries", "sortv", "style", "target", "tooltip"} + +var dotKeywords = []string{"graph", "subgraph", "digraph", "node", "edge", "strict"} + +type GraphType int + +const ( + DIGRAPH GraphType = iota + GRAPH + SUBGRAPH +) + +// Fields common to all graph object types +type common struct { + _type string + name string + attributes map[string]string + sequence int + parentGraph *Graph +} + +type GraphObject interface { + Type() string + Get(string) string + Set(string, string) error + GetParentGraph() *Graph + SetParentGraph(g *Graph) + Sequence() int +} + +type graphObjects []GraphObject + +func (gol graphObjects) Len() int { + return len(gol) +} + +func (gol graphObjects) Less(i, j int) bool { + return gol[i].Sequence() < gol[j].Sequence() +} + +func (gol graphObjects) Swap(i, j int) { + gol[i], gol[j] = gol[j], gol[i] +} + +type Graph struct { + common + nodeAttributes map[string]string + edgeAttributes map[string]string + sameRank [][]string + strict bool + graphType GraphType + supressDisconnected bool + simplify bool + currentChildSequence int + nodes map[string][]*Node + edges map[string][]*Edge + subgraphs map[string][]*SubGraph +} + +func NewGraph(name string) *Graph { + g := &Graph{ + common: common{ + _type: "graph", + name: name, + attributes: make(map[string]string, 0), + }, + nodeAttributes: make(map[string]string), + edgeAttributes: make(map[string]string), + sameRank: make([][]string, 0), + nodes: make(map[string][]*Node, 0), + edges: make(map[string][]*Edge, 0), + subgraphs: make(map[string][]*SubGraph, 0), + currentChildSequence: 1, + } + g.SetParentGraph(g) + return g +} + +type SubGraph struct { + Graph +} + +func NewSubgraph(name string) *SubGraph { + result := &SubGraph{ + Graph: *NewGraph(name), + } + result._type = "subgraph" + result.graphType = SUBGRAPH + return result +} + +func indexInSlice(slice []string, toFind string) int { + for i, v := range slice { + if v == toFind { + return i + } + } + return -1 +} + +var alreadyQuotedRegex = regexp.MustCompile("^\".+\"$") +var validIdentifierRegexWithPort = regexp.MustCompile("^[_a-zA-Z][a-zA-Z0-9_,:\"]*[a-zA-Z0-9_,\"]+$") +var validIdentifierRegex = regexp.MustCompile("^[_a-zA-Z][a-zA-Z0-9_,]*$") + +func needsQuotes(s string) bool { + if indexInSlice(dotKeywords, s) != -1 { + return false + } + if alreadyQuotedRegex.MatchString(s) { + return false + } + if validIdentifierRegexWithPort.MatchString(s) || validIdentifierRegex.MatchString(s) { + return false + } + + return true +} + +func QuoteIfNecessary(s string) (result string) { + if needsQuotes(s) { + s = strings.Replace(s, "\"", "\\\"", -1) + s = strings.Replace(s, "\n", "\\n", -1) + s = strings.Replace(s, "\r", "\\r", -1) + s = "\"" + s + "\"" + } + return s +} + +func validAttribute(attributeCollection []string, attributeName string) bool { + return indexInSlice(attributeCollection, attributeName) != -1 +} + +func validGraphAttribute(attributeName string) bool { + return validAttribute(graphAttributes, attributeName) +} + +func validNodeAttribute(attributeName string) bool { + return validAttribute(nodeAttributes, attributeName) +} + +func sortedKeys(sourceMap map[string]string) []string { + keys := make([]string, 0, len(sourceMap)) + for k, _ := range sourceMap { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +//////////////////////////////////////////////////////////////////////////////////////// + +func (gt GraphType) String() string { + if gt == DIGRAPH { + return "digraph" + } else if gt == GRAPH { + return "graph" + } else if gt == SUBGRAPH { + return "subgraph" + } + return "(invalid)" +} + +func (c *common) Type() string { + return c._type +} + +func (c *common) GetParentGraph() *Graph { + return c.parentGraph +} + +func (c *common) SetParentGraph(g *Graph) { + c.parentGraph = g +} + +func (c *common) Sequence() int { + return c.sequence +} + +func (c *common) Get(attributeName string) string { + return c.attributes[attributeName] +} + +func (c *common) Set(attributeName, attributeValue string) error { + c.attributes[attributeName] = attributeValue + return nil +} + +func setAttribute(validAttributes []string, attributes map[string]string, attributeName, attributeValue string) error { + if validAttribute(validAttributes, attributeName) { + attributes[attributeName] = attributeValue + return nil + } + return AttributeError +} + +func (g *Graph) Set(attributeName, attributeValue string) error { + return setAttribute(graphAttributes, g.common.attributes, attributeName, attributeValue) +} + +func (g *Graph) SetGlobalNodeAttr(attributeName, attributeValue string) error { + return setAttribute(nodeAttributes, g.nodeAttributes, attributeName, attributeValue) +} + +func (g *Graph) SetGlobalEdgeAttr(attributeName, attributeValue string) error { + return setAttribute(edgeAttributes, g.edgeAttributes, attributeName, attributeValue) +} + +func (n *Node) Set(attributeName, attributeValue string) error { + return setAttribute(nodeAttributes, n.common.attributes, attributeName, attributeValue) +} + +func (e *Edge) Set(attributeName, attributeValue string) error { + return setAttribute(edgeAttributes, e.common.attributes, attributeName, attributeValue) +} + +func (c *common) setSequence(sequence int) { + c.sequence = sequence +} + +// SameRank enforces alignment of the given nodes +func (g *Graph) SameRank(nodes []string) { + g.sameRank = append(g.sameRank, nodes) +} + +// Set the type of the graph, valid values are GRAPH or DIGRAPH +func (g *Graph) SetType(t GraphType) { + g.graphType = t + // @todo consider disallowing setting type to SUBGRAPH +} + +func (c common) Name() string { + return c.name +} + +func (g *Graph) GetRoot() (result *Graph) { + result = g + for parent := g.GetParentGraph(); parent != result; parent = parent.GetParentGraph() { + result = parent + } + return result +} + +func (g *Graph) getNextSequenceNumber() (next int) { + next = g.currentChildSequence + g.currentChildSequence += 1 + return +} +func (g *Graph) AddNode(n *Node) { + name := n.Name() + if _, ok := g.nodes[name]; !ok { + g.nodes[name] = make([]*Node, 0) + } + n.setSequence(g.getNextSequenceNumber()) + n.SetParentGraph(g.GetParentGraph()) + g.nodes[name] = append(g.nodes[name], n) +} + +func (g *Graph) AddEdge(e *Edge) { + name := e.Name() + if _, ok := g.edges[name]; !ok { + g.edges[name] = make([]*Edge, 0) + } + e.setSequence(g.getNextSequenceNumber()) + e.SetParentGraph(g.GetParentGraph()) + g.edges[name] = append(g.edges[name], e) +} + +func (g *Graph) AddSubgraph(sg *SubGraph) { + name := sg.Name() + if _, ok := g.subgraphs[name]; !ok { + g.subgraphs[name] = make([]*SubGraph, 0) + } + sg.setSequence(g.getNextSequenceNumber()) + g.subgraphs[name] = append(g.subgraphs[name], sg) +} + +func (g *Graph) GetSubgraphs() (result []*SubGraph) { + result = make([]*SubGraph, 0) + for _, sgs := range g.subgraphs { + for _, sg := range sgs { + result = append(result, sg) + } + } + return result +} + +func (g Graph) String() string { + var parts []string + if g.strict { + parts = append(parts, "strict ") + } + if g.name == "" { + parts = append(parts, "{\n") + } else { + parts = append(parts, fmt.Sprintf("%s %s {\n", g.graphType, QuoteIfNecessary(g.name))) + } + + if len(g.attributes) > 0 { + attrs := make([]string, 0, len(g.attributes)) + for _, key := range sortedKeys(g.attributes) { + attrs = append(attrs, " "+key+"="+QuoteIfNecessary(g.attributes[key])) + } + if len(attrs) > 0 { + parts = append(parts, "graph [\n") + parts = append(parts, strings.Join(attrs, ";\n")) + parts = append(parts, ";\n];\n") + } + } + + if len(g.nodeAttributes) > 0 { + attrs := make([]string, 0, len(g.nodeAttributes)) + for _, key := range sortedKeys(g.nodeAttributes) { + attrs = append(attrs, " "+key+"="+QuoteIfNecessary(g.nodeAttributes[key])) + } + if len(attrs) > 0 { + parts = append(parts, "node [\n") + parts = append(parts, strings.Join(attrs, ";\n")) + parts = append(parts, ";\n];\n") + } + } + + if len(g.edgeAttributes) > 0 { + attrs := make([]string, 0, len(g.edgeAttributes)) + for _, key := range sortedKeys(g.edgeAttributes) { + attrs = append(attrs, " "+key+"="+QuoteIfNecessary(g.edgeAttributes[key])) + } + if len(attrs) > 0 { + parts = append(parts, "edge [\n") + parts = append(parts, strings.Join(attrs, ";\n")) + parts = append(parts, ";\n];\n") + } + } + + objectList := make(graphObjects, 0) + + for _, nodes := range g.nodes { + for _, node := range nodes { + objectList = append(objectList, node) + } + } + for _, edges := range g.edges { + for _, edge := range edges { + objectList = append(objectList, edge) + } + } + for _, subgraphs := range g.subgraphs { + for _, subgraph := range subgraphs { + objectList = append(objectList, subgraph) + } + } + sort.Sort(objectList) + + for _, obj := range objectList { + //@todo type-based decision making re: supressDisconnected and simplify + //switch o := obj.(type) { + //case *Node: + //} + parts = append(parts, fmt.Sprintf("%s\n", obj)) + } + + for _, nodes := range g.sameRank { + parts = append(parts, fmt.Sprintf("{ rank=same %s }", strings.Join(nodes, " "))) + } + + parts = append(parts, "}\n") + return strings.Join(parts, "") +} + +type Node struct { + common +} + +func NewNode(name string) *Node { + return &Node{ + common{ + name: name, + attributes: make(map[string]string, 0), + }, + } +} + +func (n Node) String() string { + + name := QuoteIfNecessary(n.name) + + parts := make([]string, 0) + + attrs := make([]string, 0) + for _, key := range sortedKeys(n.attributes) { + value := n.attributes[key] + if key == "label" && len(value) > 4 && value[0] == '<' && value[len(value)-1] == '>' { + attrs = append(attrs, key+"="+value) + } else { + attrs = append(attrs, key+"="+QuoteIfNecessary(value)) + } + } + if len(attrs) > 0 { + parts = append(parts, strings.Join(attrs, ", ")) + } + + //@todo don't print if node is empty + if len(parts) > 0 { + name += " [" + strings.Join(parts, ", ") + "]" + } + + return name + ";" +} + +type Edge struct { + common + points [2]*Node +} + +func NewEdge(src, dst *Node) *Edge { + return &Edge{ + common{ + _type: "edge", + attributes: make(map[string]string, 0), + }, + [2]*Node{src, dst}, + } +} + +func (e Edge) Source() *Node { + return e.points[0] +} + +func (e Edge) Destination() *Node { + return e.points[1] +} + +func (e Edge) String() string { + src, dst := e.Source(), e.Destination() + parts := make([]string, 0) + + parts = append(parts, QuoteIfNecessary(src.Name())) + + parent := e.GetParentGraph() + if parent != nil && parent.GetRoot() != nil && parent.GetRoot().graphType == DIGRAPH { + parts = append(parts, "->") + } else { + parts = append(parts, "--") + } + parts = append(parts, QuoteIfNecessary(dst.Name())) + + attrs := make([]string, 0) + for _, key := range sortedKeys(e.attributes) { + attrs = append(attrs, key+"="+QuoteIfNecessary(e.attributes[key])) + } + if len(attrs) > 0 { + parts = append(parts, " [") + parts = append(parts, strings.Join(attrs, ", ")) + parts = append(parts, "]") + } + + return strings.Join(parts, " ") +} + +func init() { + sort.Strings(graphAttributes) + sort.Strings(nodeAttributes) + sort.Strings(edgeAttributes) + sort.Strings(clusterAttributes) +} diff --git a/utils/dag/dot/dot_test.go b/utils/dag/dot/dot_test.go new file mode 100644 index 000000000..431fd0db8 --- /dev/null +++ b/utils/dag/dot/dot_test.go @@ -0,0 +1,151 @@ +package dot_test + +import ( + "fmt" + "testing" + + "github.com/Fantom-foundation/go-opera/utils/dag/dot" +) + +func TestQuotingIfNecessary(t *testing.T) { + cases := map[string]string{ + "foo": "foo", + "\"foo\"": "\"foo\"", + "foo bar": "\"foo bar\"", + "Allen, C.": "\"Allen, C.\"", + } + + for input, expected := range cases { + if dot.QuoteIfNecessary(input) != expected { + t.Errorf("'%s' != '%s'", dot.QuoteIfNecessary(input), expected) + } + } +} + +func TestGraphPrinting(t *testing.T) { + g1 := dot.NewGraph("foo") + expected1 := "digraph foo {\n}\n" + g2 := dot.NewGraph("foo bar") + expected2 := "digraph \"foo bar\" {\n}\n" + + if fmt.Sprint(g1) != expected1 { + t.Errorf("'%s' != '%s'", fmt.Sprint(g1), expected1) + } + if fmt.Sprint(g2) != expected2 { + t.Errorf("'%s' != '%s'", fmt.Sprint(g2), expected2) + } +} + +func TestCreateSimpleGraphWithNode(t *testing.T) { + g := dot.NewGraph("Test") + + expected := "digraph Test {\n}\n" + if fmt.Sprint(g) != expected { + t.Errorf("'%s' != '%s'", fmt.Sprint(g), expected) + } + g.SetType(dot.GRAPH) + + expected = "graph Test {\n}\n" + if fmt.Sprint(g) != expected { + t.Errorf("'%s' != '%s'", fmt.Sprint(g), expected) + } + g.SetType(dot.DIGRAPH) + + node := dot.NewNode("legend") + node.Set("shape", "box") + g.AddNode(node) + node.Set("label", "value with spaces") + + node = dot.NewNode("html") + node.Set("shape", "plain") + node.Set("label", "<bold>") + g.AddNode(node) + + expected = "digraph Test {\nlegend [label=\"value with spaces\", shape=box];\nhtml [label=<bold>, shape=plain];\n}\n" + if fmt.Sprint(g) != expected { + t.Errorf("'%s' != '%s'", fmt.Sprint(g), expected) + } +} + +func TestCreateSimpleNode(t *testing.T) { + node := dot.NewNode("nodename") + node.Set("shape", "box") + node.Set("label", "mine") + + expected := "nodename [label=mine, shape=box];" + if fmt.Sprint(node) != expected { + t.Errorf("'%s' != '%s'", fmt.Sprint(node), expected) + } +} + +func TestGraphAttributeSetting(t *testing.T) { + g := dot.NewGraph("Test") + if g.Set("label", "foo") != nil { + t.Error("Error setting value on g", g) + } + g.Set("Damping", "x") + if g.Set("this_does_not_exist", "and_should_error") != dot.AttributeError { + t.Error("Did not get godot.AttributeError when setting invalid attribute on g", g) + } +} + +func TestSubGraphs(t *testing.T) { + g := dot.NewGraph("G") + s := dot.NewSubgraph("SG") + + subgraphs := make([]*dot.SubGraph, 0) + if subgraphs = g.GetSubgraphs(); len(subgraphs) != 0 { + t.Error("Non-empty subgraphs returned:", subgraphs) + } + g.AddSubgraph(s) + if g.GetSubgraphs()[0].Name() != s.Name() { + t.Error(g.GetSubgraphs()[0].Name(), " != ", s.Name()) + } + + expected := `digraph G { +subgraph SG { +} + +} +` + + if fmt.Sprint(g) != expected { + t.Errorf("'%s' != '%s'", g, expected) + } +} + +func TestEdgeAddition(t *testing.T) { + simple_graph := `digraph G { +graph [ + label="this is a graph"; +]; +a -> b +} +` + g := dot.NewGraph("G") + g.Set("label", "this is a graph") + a, b := dot.NewNode("a"), dot.NewNode("b") + e := dot.NewEdge(a, b) + g.AddEdge(e) + + if fmt.Sprint(g) != simple_graph { + t.Errorf("'%s' != '%s'", g, simple_graph) + } + +} + +func TestQuoting(t *testing.T) { + g := dot.NewGraph("G") + a, b := dot.NewNode("192.168.1.1"), dot.NewNode("192.168.1.2") + e := dot.NewEdge(a, b) + g.AddEdge(e) + + expected := `digraph G { +"192.168.1.1" -> "192.168.1.2" +} +` + if fmt.Sprint(g) != expected { + t.Errorf("'%s' != '%s'", g, expected) + } + +} diff --git a/utils/dag/dot/example_test.go b/utils/dag/dot/example_test.go new file mode 100644 index 000000000..3f2dd9f66 --- /dev/null +++ b/utils/dag/dot/example_test.go @@ -0,0 +1,34 @@ +package dot_test + +import ( + "fmt" + + "github.com/Fantom-foundation/go-opera/utils/dag/dot" +) + +func ExampleNewGraph() { + g := dot.NewGraph("G") + g.Set("label", "Example graph") + n1, n2 := dot.NewNode("Node1"), dot.NewNode("Node2") + + n1.Set("color", "sienna") + + g.AddNode(n1) + g.AddNode(n2) + + e := dot.NewEdge(n1, n2) + e.Set("dir", "both") + g.AddEdge(e) + + fmt.Println(g) + // Output: + // digraph G { + // graph [ + // label="Example graph"; + // ]; + // Node1 [color=sienna]; + // Node2; + // Node1 -> Node2 [ dir=both ] + // } + // +} diff --git a/utils/dag/graph_inmem.go b/utils/dag/graph_inmem.go index 0a0cc678c..84a03beae 100644 --- a/utils/dag/graph_inmem.go +++ b/utils/dag/graph_inmem.go @@ -3,6 +3,7 @@ package dag import ( "fmt" "math" + "strings" "github.com/Fantom-foundation/lachesis-base/abft" "github.com/Fantom-foundation/lachesis-base/gossip/dagordering" @@ -11,79 +12,19 @@ import ( "github.com/Fantom-foundation/lachesis-base/inter/idx" "github.com/Fantom-foundation/lachesis-base/kvdb/memorydb" "github.com/ethereum/go-ethereum/log" - "gonum.org/v1/gonum/graph" - "gonum.org/v1/gonum/graph/encoding" - "gonum.org/v1/gonum/graph/encoding/dot" "github.com/Fantom-foundation/go-opera/gossip" "github.com/Fantom-foundation/go-opera/integration" "github.com/Fantom-foundation/go-opera/inter" "github.com/Fantom-foundation/go-opera/inter/iblockproc" "github.com/Fantom-foundation/go-opera/utils/adapters/vecmt2dagidx" + "github.com/Fantom-foundation/go-opera/utils/dag/dot" "github.com/Fantom-foundation/go-opera/vecmt" ) -type eventSeq []hash.Event - -func (s *eventSeq) event2id(h hash.Event) int64 { - id := int64(len(*s)) - *s = append(*s, h) - return id -} - -func (s *eventSeq) id2event(id int64) hash.Event { - return (*s)[id] -} - -type eventList map[hash.Event]*dotNode - -func (l *eventList) set(n *dotNode) { - (*l)[n.hash] = n -} - -func (l *eventList) get(h hash.Event) *dotNode { - return (*l)[h] -} - -// graphInMem implements dot.Graph over inmem refs and nodes -type graphInMem struct { - name string - global bool - subGraphs map[string]*graphInMem - - include map[int64]struct{} - refs *eventSeq - nodes *eventList - attrs struct { - graph attributer - edge attributer - } -} - -func newGraphInMem(name string) *graphInMem { - return &graphInMem{ - name: name, - include: make(map[int64]struct{}), - subGraphs: make(map[string]*graphInMem), - - attrs: struct{ graph, edge attributer }{ - attributer(make(map[string]string, 10)), - attributer(make(map[string]string, 10)), - }, - } -} - // readDagGraph read gossip.Store into inmem dot.Graph -func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *graphInMem { - g := newGraphInMem("DOT") - g.global = true - g.refs = &eventSeq{} - g.nodes = &eventList{} - g.attrs.graph.setAttr("clusterrank", "local") - g.attrs.graph.setAttr("compound", "true") - g.attrs.graph.setAttr("newrank", "true") - g.attrs.graph.setAttr("ranksep", "0.05") - g.attrs.edge.setAttr("constraint", "true") +func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch) *dot.Graph { + // 0. Set gossip data: cdb := abft.NewMemStore() defer cdb.Close() @@ -107,6 +48,22 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch panic(err) } + // 1. Set dot.Graph data: + + g := dot.NewGraph("DOT") + g.Set("clusterrank", "local") + g.Set("compound", "true") + g.Set("newrank", "true") + g.Set("ranksep", "0.05") + g.SetGlobalEdgeAttr("constraint", "true") + + var ( + nodes = make(map[hash.Event]*dot.Node) + subGraphs = make(map[idx.ValidatorID]*dot.SubGraph) + ) + + // 2. Set event processor data: + var ( epoch idx.Epoch prevBS *iblockproc.BlockState @@ -119,8 +76,8 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch for f := idx.Frame(0); f <= cdb.GetLastDecidedFrame(); f++ { rr := cdb.GetFrameRoots(f) for _, r := range rr { - node := g.nodes.get(r.ID) - markAsRoot(node) + n := nodes[r.ID] + markAsRoot(n) } } @@ -131,15 +88,16 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch maxBlock = bs.LastBlock.Idx } - for n := prevBS.LastBlock.Idx + 1; n <= maxBlock; n++ { - block := gdb.GetBlock(n) + for b := prevBS.LastBlock.Idx + 1; b <= maxBlock; b++ { + block := gdb.GetBlock(b) if block == nil { break } - node := g.nodes.get(block.Atropos) - if node != nil { - markAsAtropos(node) + n := nodes[block.Atropos] + if n == nil { + continue } + markAsAtropos(n) } } @@ -150,7 +108,22 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch validators := gdb.GetHistoryEpochState(epoch).Validators for _, v := range validators.IDs() { - _ = g.subGraph(v) + if _, ok := subGraphs[v]; ok { + continue + } + group := groupName(v) + sg := dot.NewSubgraph(fmt.Sprintf("cluster%d", len(subGraphs))) + sg.Set("label", group) + sg.Set("sortv", fmt.Sprintf("%d", v)) + sg.Set("style", "dotted") + + pseudoNode := dot.NewNode(group) + pseudoNode.Set("style", "invis") + pseudoNode.Set("width", "0") + sg.AddNode(pseudoNode) + + subGraphs[v] = sg + g.AddSubgraph(sg) } processed = make(map[hash.Event]dag.Event, 1000) @@ -175,8 +148,21 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch dagIndexer.Flush() orderer.Process(e) - sg := g.subGraph(e.Creator()) - sg.addDagEvent(e) + name := fmt.Sprintf("%s\n%d", e.ID().String(), e.Creator()) + n := dot.NewNode(name) + sg := subGraphs[e.Creator()] + sg.AddNode(n) + nodes[e.ID()] = n + + for _, h := range e.Parents() { + p := nodes[h] + ref := dot.NewEdge(n, p) + if processed[h].Creator() == e.Creator() { + sg.AddEdge(ref) + } else { + g.AddEdge(ref) + } + } return nil }, @@ -194,7 +180,8 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch }, }) - // process events + // 3. Iterate over events: + gdb.ForEachEvent(from, func(e *inter.EventPayload) bool { // current epoch is finished, so process accumulated events if epoch < e.Epoch() { @@ -216,230 +203,19 @@ func readDagGraph(gdb *gossip.Store, cfg integration.Configs, from, to idx.Epoch epoch++ readRestoredAbftStore() - return g -} - -func (g *graphInMem) DOTID() string { - return g.name -} - -// Structure returns subgraphs. -func (g *graphInMem) Structure() []dot.Graph { - res := make([]dot.Graph, 0, len(g.subGraphs)) - for _, sg := range g.subGraphs { - res = append(res, sg) - } - return res -} - -func (g *graphInMem) subGraph(v idx.ValidatorID) *graphInMem { - name := fmt.Sprintf("cluster%d", v) - label := fmt.Sprintf("validator-%d", v) - sg, ok := g.subGraphs[name] - if !ok { - sg = newGraphInMem(name) - sg.refs = g.refs - sg.nodes = g.nodes - sg.attrs.graph.setAttr("label", label) - sg.attrs.graph.setAttr("sortv", fmt.Sprintf("%d", v)) - sg.attrs.graph.setAttr("style", "dotted") - sg.attrs.edge.setAttr("constraint", "true") - g.subGraphs[name] = sg - sg.addPseudoNode(label) - } - return sg -} - -// DOTAttributers are graph.Graph values that specify top-level DOT attributes. -func (g *graphInMem) DOTAttributers() (graph, node, edge encoding.Attributer) { - graph = g.attrs.graph - node = attributer(make(map[string]string, 0)) // empty - edge = g.attrs.edge - return -} - -func (g *graphInMem) addDagEvent(e dag.Event) (id int64) { - n := g.nodes.get(e.ID()) - if n == nil { - id = g.refs.event2id(e.ID()) - n = newDotNode(id, e) - g.nodes.set(n) - } else { - id = n.id - } - - g.include[id] = struct{}{} - return -} - -func (g *graphInMem) addPseudoNode(name string) (id int64) { - h := hash.BytesToEvent([]byte(name)) - n := g.nodes.get(h) - if n == nil { - id = g.refs.event2id(h) - n = &dotNode{ - id: id, - hash: h, - attributer: attributer(make(map[string]string, 10)), - } - n.setAttr("label", name) - n.setAttr("style", "invis") - n.setAttr("width", "0") - g.nodes.set(n) - } else { - id = n.id - } - - g.include[id] = struct{}{} - return -} - -// Node returns the node with the given ID if it exists -// in the graph, and nil otherwise. -func (g *graphInMem) Node(id int64) graph.Node { - if _, ok := g.include[id]; !ok { - return nil - } - hash := g.refs.id2event(id) - return g.nodes.get(hash) -} + // 4. Result -// Nodes returns all the nodes in the graph. -// -// Nodes must not return nil. -func (g *graphInMem) Nodes() graph.Nodes { - nn := &dagNodes{ - data: make(chan *dotNode), - } - - go func() { - defer close(nn.data) - - for id := range g.include { - h := g.refs.id2event(id) - e := g.nodes.get(h) - nn.data <- e - } - }() - - return nn -} - -// From returns all nodes that can be reached directly -// from the node with the given ID. -// -// From must not return nil. -func (g *graphInMem) From(id int64) graph.Nodes { - nn := &dagNodes{ - data: make(chan *dotNode), - } - - h := g.refs.id2event(id) - x := g.nodes.get(h) - - go func() { - defer close(nn.data) - for _, p := range x.parents { - y := g.nodes.get(p) - if g.hasEdge(x, y) { - nn.data <- y - } - } - }() - - return nn -} - -// To returns all nodes that can reach directly -// to the node with the given ID. -// -// To must not return nil. -func (g *graphInMem) To(id int64) graph.Nodes { - nn := &dagNodes{ - data: make(chan *dotNode), - } - close(nn.data) - return nn -} - -// HasEdgeBetween returns whether an edge exists between -// nodes with IDs xid and yid without considering direction. -func (g *graphInMem) HasEdgeBetween(xid, yid int64) bool { - x := g.nodes.get(g.refs.id2event(xid)) - y := g.nodes.get(g.refs.id2event(yid)) - if !g.hasEdge(x, y) { - return false - } - - for _, p := range x.parents { - if p == y.hash { - return true - } - } - for _, p := range y.parents { - if p == x.hash { - return true - } - } - - return false -} - -// HasEdgeFromTo returns whether an edge exists -// in the graph from u to v with IDs uid and vid. -func (g *graphInMem) HasEdgeFromTo(uid, vid int64) bool { - u := g.nodes.get(g.refs.id2event(uid)) - v := g.nodes.get(g.refs.id2event(vid)) - if !g.hasEdge(u, v) { - return false - } - - for _, p := range u.parents { - if p == v.hash { - return true - } - } - - return false -} - -// Edge returns the edge from u to v, with IDs uid and vid, -// if such an edge exists and nil otherwise. The node v -// must be directly reachable from u as defined by the -// From method. -func (g *graphInMem) Edge(uid, vid int64) graph.Edge { - u := g.nodes.get(g.refs.id2event(uid)) - v := g.nodes.get(g.refs.id2event(vid)) - if !g.hasEdge(u, v) { - return nil - } - - for _, p := range u.parents { - if p == v.hash { - e := &dotEdge{ - x: u, - y: v, - } - return e - } - } - - return nil -} - -func (g *graphInMem) hasEdge(x, y *dotNode) bool { - if _, ok := g.include[x.id]; !ok && !g.global { - return false - } - if _, ok := g.include[y.id]; !ok && !g.global { - return false - } - - if g.global && x.creator == y.creator { - return false + // NOTE: github.com/tmc/dot renders subgraphs not in the ordering that specified + // so we introduce pseudo nodes and edges to work around + groups := make([]string, 0, len(subGraphs)) + for v := range subGraphs { + groups = append(groups, groupName(v)) } + g.SameRank([]string{ + "\"" + strings.Join(groups, `" -> "`) + "\" [style = invis, constraint = true];", + }) - return true + return g } func panics(name string) func(error) { @@ -448,14 +224,18 @@ func panics(name string) func(error) { } } -func markAsRoot(n *dotNode) { +func markAsRoot(n *dot.Node) { // n.setAttr("xlabel", "root") - n.setAttr("style", "filled") - n.setAttr("fillcolor", "#FFFF00") + n.Set("style", "filled") + n.Set("fillcolor", "#FFFF00") } -func markAsAtropos(n *dotNode) { +func markAsAtropos(n *dot.Node) { // n.setAttr("xlabel", "atropos") - n.setAttr("style", "filled") - n.setAttr("fillcolor", "#FF0000") + n.Set("style", "filled") + n.Set("fillcolor", "#FF0000") +} + +func groupName(v idx.ValidatorID) string { + return fmt.Sprintf("host-%d", v) } diff --git a/utils/dag/graph_ondisk.go b/utils/dag/graph_ondisk.go deleted file mode 100644 index daff459e3..000000000 --- a/utils/dag/graph_ondisk.go +++ /dev/null @@ -1,162 +0,0 @@ -package dag - -import ( - "github.com/Fantom-foundation/lachesis-base/hash" - "github.com/Fantom-foundation/lachesis-base/inter/idx" - "gonum.org/v1/gonum/graph" - - "github.com/Fantom-foundation/go-opera/gossip" - "github.com/Fantom-foundation/go-opera/inter" -) - -// dagReader implements dot.Graph over gossip.Store -type graphOnDisk struct { - db *gossip.Store - epochFrom idx.Epoch - epochTo idx.Epoch -} - -func (g *graphOnDisk) DOTID() string { - return "DAG" -} - -// Node returns the node with the given ID if it exists -// in the graph, and nil otherwise. -func (g *graphOnDisk) Node(id int64) graph.Node { - e := g.db.GetEvent(id2event(id)) - return newDotNode(id, e) -} - -// Nodes returns all the nodes in the graph. -// -// Nodes must not return nil. -func (g *graphOnDisk) Nodes() graph.Nodes { - nn := &dagNodes{ - data: make(chan *dotNode), - } - - go func() { - defer close(nn.data) - g.db.ForEachEvent(g.epochFrom, func(e *inter.EventPayload) bool { - if g.epochTo >= g.epochFrom && e.Epoch() > g.epochTo { - return false - } - - id := event2id(e.ID()) - nn.data <- newDotNode(id, &e.Event) - return true - }) - }() - - return nn -} - -// From returns all nodes that can be reached directly -// from the node with the given ID. -// -// From must not return nil. -func (g *graphOnDisk) From(id int64) graph.Nodes { - nn := &dagNodes{ - data: make(chan *dotNode), - } - - x := g.Node(id).(*dotNode) - go func() { - defer close(nn.data) - for _, p := range x.parents { - n := g.Node(event2id(p)) - nn.data <- n.(*dotNode) - } - }() - - return nn -} - -// To returns all nodes that can reach directly -// to the node with the given ID. -// -// To must not return nil. -func (g *graphOnDisk) To(id int64) graph.Nodes { - nn := &dagNodes{ - data: make(chan *dotNode), - } - close(nn.data) - return nn -} - -// HasEdgeBetween returns whether an edge exists between -// nodes with IDs xid and yid without considering direction. -func (g *graphOnDisk) HasEdgeBetween(xid, yid int64) bool { - x := g.Node(xid).(*dotNode) - y := g.Node(yid).(*dotNode) - - for _, p := range x.parents { - if p == y.hash { - return true - } - } - for _, p := range y.parents { - if p == x.hash { - return true - } - } - - return false -} - -// HasEdgeFromTo returns whether an edge exists -// in the graph from u to v with IDs uid and vid. -func (g *graphOnDisk) HasEdgeFromTo(uid, vid int64) bool { - u := g.Node(uid).(*dotNode) - v := g.Node(vid).(*dotNode) - - for _, p := range u.parents { - if p == v.hash { - return true - } - } - - return false -} - -// Edge returns the edge from u to v, with IDs uid and vid, -// if such an edge exists and nil otherwise. The node v -// must be directly reachable from u as defined by the -// From method. -func (g *graphOnDisk) Edge(uid, vid int64) graph.Edge { - u := g.Node(uid).(*dotNode) - v := g.Node(vid).(*dotNode) - - for _, p := range u.parents { - if p == v.hash { - return &dotEdge{ - x: u, - y: v, - } - } - } - - return nil -} - -// -- - -var ( - id2hash = make(map[int64]hash.Event) -) - -func id2event(id int64) hash.Event { - return id2hash[id] -} - -func event2id(e hash.Event) int64 { - // NOTE: possible collision - var id int64 - for i := 0; i < 8; i++ { - id += int64(e[8+i] << (8 * i)) - } - - id2hash[id] = e - - return id -}