Always hash storage diff contract addresses

- Enables syncing Geth and Parity diffs with same transformer lookup
- Maybe worth always hashing the storage key so we don't need a hashed
  and not-hashed version in the key lookups?

Author: Rob Mulholand

cmd/composeAndExecute.go

@@ -188,15 +188,15 @@ func composeAndExecute() {
 			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
 			payloadChan := make(chan statediff.Payload)
 			storageFetcher := fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan)
-			sw := watcher.NewGethStorageWatcher(storageFetcher, &db)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
 			sw.AddTransformers(ethStorageInitializers)
 			wg.Add(1)
 			go watchEthStorage(&sw, &wg)
 		default:
 			log.Debug("fetching storage diffs from csv")
 			tailer := fs.FileTailer{Path: storageDiffsPath}
 			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-			sw := watcher.NewCsvStorageWatcher(storageFetcher, &db)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
 			sw.AddTransformers(ethStorageInitializers)
 			wg.Add(1)
 			go watchEthStorage(&sw, &wg)

cmd/execute.go

@@ -132,15 +132,15 @@ func execute() {
 			stateDiffStreamer := streamer.NewStateDiffStreamer(rpcClient)
 			payloadChan := make(chan statediff.Payload)
 			storageFetcher := fetcher.NewGethRpcStorageFetcher(&stateDiffStreamer, payloadChan)
-			sw := watcher.NewGethStorageWatcher(storageFetcher, &db)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
 			sw.AddTransformers(ethStorageInitializers)
 			wg.Add(1)
 			go watchEthStorage(&sw, &wg)
 		default:
 			log.Debug("fetching storage diffs from csv")
 			tailer := fs.FileTailer{Path: storageDiffsPath}
 			storageFetcher := fetcher.NewCsvTailStorageFetcher(tailer)
-			sw := watcher.NewCsvStorageWatcher(storageFetcher, &db)
+			sw := watcher.NewStorageWatcher(storageFetcher, &db)
 			sw.AddTransformers(ethStorageInitializers)
 			wg.Add(1)
 			go watchEthStorage(&sw, &wg)

go.mod

@@ -14,7 +14,6 @@ require (
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
 	github.com/go-sql-driver/mysql v1.4.1 // indirect
 	github.com/golang/protobuf v1.3.2 // indirect
-	github.com/google/uuid v1.0.0 // indirect
 	github.com/howeyc/fsnotify v0.9.0 // indirect
 	github.com/hpcloud/tail v1.0.0
 	github.com/huin/goupnp v1.0.0 // indirect

libraries/shared/factories/storage/transformer.go

@@ -18,17 +18,17 @@ package storage
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
+	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type Transformer struct {
-	Address    common.Address
-	Mappings   storage.Mappings
-	Repository Repository
+	HashedAddress common.Hash
+	Mappings      storage.Mappings
+	Repository    repository.StorageRepository
 }
 
 func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.StorageTransformer {
@@ -37,8 +37,8 @@ func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.Stora
 	return transformer
 }
 
-func (transformer Transformer) ContractAddress() common.Address {
-	return transformer.Address
+func (transformer Transformer) KeccakContractAddress() common.Hash {
+	return transformer.HashedAddress
 }
 
 func (transformer Transformer) Execute(diff utils.StorageDiff) error {

libraries/shared/factories/storage/transformer_test.go

@@ -18,9 +18,9 @@ package storage_test
 
 import (
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
@@ -38,17 +38,17 @@ var _ = Describe("Storage transformer", func() {
 		mappings = &mocks.MockMappings{}
 		repository = &mocks.MockStorageRepository{}
 		t = storage.Transformer{
-			Address:    common.Address{},
-			Mappings:   mappings,
-			Repository: repository,
+			HashedAddress: common.Hash{},
+			Mappings:      mappings,
+			Repository:    repository,
 		}
 	})
 
 	It("returns the contract address being watched", func() {
-		fakeAddress := common.HexToAddress("0x12345")
-		t.Address = fakeAddress
+		fakeAddress := common.BytesToHash(crypto.Keccak256(common.FromHex("0x12345")))
+		t.HashedAddress = fakeAddress
 
-		Expect(t.ContractAddress()).To(Equal(fakeAddress))
+		Expect(t.KeccakContractAddress()).To(Equal(fakeAddress))
 	})
 
 	It("looks up metadata for storage key", func() {
@@ -73,11 +73,11 @@ var _ = Describe("Storage transformer", func() {
 		fakeBlockNumber := 123
 		fakeBlockHash := "0x67890"
 		fakeRow := utils.StorageDiff{
-			Contract:     common.Address{},
-			BlockHash:    common.HexToHash(fakeBlockHash),
-			BlockHeight:  fakeBlockNumber,
-			StorageKey:   common.Hash{},
-			StorageValue: rawValue.Hash(),
+			KeccakOfContractAddress: common.Hash{},
+			BlockHash:               common.HexToHash(fakeBlockHash),
+			BlockHeight:             fakeBlockNumber,
+			StorageKey:              common.Hash{},
+			StorageValue:            rawValue.Hash(),
 		}
 
 		err := t.Execute(fakeRow)
@@ -121,11 +121,11 @@ var _ = Describe("Storage transformer", func() {
 		It("passes the decoded data items to the repository", func() {
 			mappings.Metadata = fakeMetadata
 			fakeRow := utils.StorageDiff{
-				Contract:     common.Address{},
-				BlockHash:    common.HexToHash(fakeBlockHash),
-				BlockHeight:  fakeBlockNumber,
-				StorageKey:   common.Hash{},
-				StorageValue: rawValue.Hash(),
+				KeccakOfContractAddress: common.Hash{},
+				BlockHash:               common.HexToHash(fakeBlockHash),
+				BlockHeight:             fakeBlockNumber,
+				StorageKey:              common.Hash{},
+				StorageValue:            rawValue.Hash(),
 			}
 
 			err := t.Execute(fakeRow)

libraries/shared/fetcher/csv_tail_storage_fetcher.go

@@ -17,12 +17,10 @@
 package fetcher
 
 import (
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-
+	"github.com/sirupsen/logrus"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/fs"
+	"strings"
 )
 
 type CsvTailStorageFetcher struct {
@@ -38,9 +36,9 @@ func (storageFetcher CsvTailStorageFetcher) FetchStorageDiffs(out chan<- utils.S
 	if tailErr != nil {
 		errs <- tailErr
 	}
-	log.Debug("fetching storage diffs...")
+	logrus.Debug("fetching storage diffs...")
 	for line := range t.Lines {
-		diff, parseErr := utils.FromStrings(strings.Split(line.Text, ","))
+		diff, parseErr := utils.FromParityCsvRow(strings.Split(line.Text, ","))
 		if parseErr != nil {
 			errs <- parseErr
 		} else {

libraries/shared/fetcher/csv_tail_storage_fetcher_test.go

@@ -18,17 +18,15 @@ package fetcher_test
 
 import (
 	"fmt"
-	"strings"
-	"time"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/hpcloud/tail"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"strings"
+	"time"
 )
 
 var _ = Describe("Csv Tail Storage Fetcher", func() {
@@ -61,7 +59,7 @@ var _ = Describe("Csv Tail Storage Fetcher", func() {
 		go storageFetcher.FetchStorageDiffs(diffsChannel, errorsChannel)
 		mockTailer.Lines <- line
 
-		expectedRow, err := utils.FromStrings(strings.Split(line.Text, ","))
+		expectedRow, err := utils.FromParityCsvRow(strings.Split(line.Text, ","))
 		Expect(err).NotTo(HaveOccurred())
 		Expect(<-diffsChannel).To(Equal(expectedRow))
 		close(done)

libraries/shared/fetcher/geth_rpc_storage_fetcher.go

@@ -16,7 +16,6 @@ package fetcher
 
 import (
 	"fmt"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/sirupsen/logrus"
@@ -61,13 +60,7 @@ func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD
 			logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
 			for _, storage := range account.Storage {
 				logrus.Trace("adding storage diff to out channel")
-				out <- utils.StorageDiff{
-					KeccakOfContractAddress: common.BytesToHash(account.Key),
-					BlockHash:               stateDiff.BlockHash,
-					BlockHeight:             int(stateDiff.BlockNumber.Int64()),
-					StorageKey:              common.BytesToHash(storage.Key),
-					StorageValue:            common.BytesToHash(storage.Value),
-				}
+				out <- utils.FromGethStateDiff(account, stateDiff, storage)
 			}
 		}
 	}

libraries/shared/mocks/storage_transformer.go

@@ -18,25 +18,24 @@ package mocks
 
 import (
 	"github.com/ethereum/go-ethereum/common"
-
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 type MockStorageTransformer struct {
-	Address    common.Address
-	ExecuteErr error
-	PassedDiff utils.StorageDiff
+	KeccakOfAddress common.Hash
+	ExecuteErr      error
+	PassedDiff      utils.StorageDiff
 }
 
 func (transformer *MockStorageTransformer) Execute(diff utils.StorageDiff) error {
 	transformer.PassedDiff = diff
 	return transformer.ExecuteErr
 }
 
-func (transformer *MockStorageTransformer) ContractAddress() common.Address {
-	return transformer.Address
+func (transformer *MockStorageTransformer) KeccakContractAddress() common.Hash {
+	return transformer.KeccakOfAddress
 }
 
 func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) transformer.StorageTransformer {

libraries/shared/storage/storage_queue.go

@@ -38,7 +38,7 @@ func NewStorageQueue(db *postgres.DB) StorageQueue {
 func (queue StorageQueue) Add(diff utils.StorageDiff) error {
 	_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
 		block_hash, block_height, storage_key, storage_value) VALUES
-		($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, diff.Contract.Bytes(), diff.BlockHash.Bytes(),
+		($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, diff.KeccakOfContractAddress.Bytes(), diff.BlockHash.Bytes(),
 		diff.BlockHeight, diff.StorageKey.Bytes(), diff.StorageValue.Bytes())
 	return err
 }