Skip to content
This repository was archived by the owner on Aug 31, 2021. It is now read-only.

Commit 62e1378

Browse files
author
Rob Mulholand
authored
Merge pull request #163 from vulcanize/vdb-929-storage-key-lookup-cleanup
(VDB-929) Minimize storage key lookup bespoke code
2 parents b767531 + b8fec5e commit 62e1378

File tree

12 files changed

+412
-98
lines changed

12 files changed

+412
-98
lines changed

libraries/shared/factories/storage/README.md

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,36 +31,52 @@ The storage transformer depends on contract-specific implementations of code cap
3131

3232
```golang
3333
func (transformer Transformer) Execute(row shared.StorageDiffRow) error {
34-
metadata, lookupErr := transformer.Mappings.Lookup(row.StorageKey)
34+
metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
3535
if lookupErr != nil {
3636
return lookupErr
3737
}
38-
value, decodeErr := shared.Decode(row, metadata)
38+
value, decodeErr := utils.Decode(diff, metadata)
3939
if decodeErr != nil {
4040
return decodeErr
4141
}
42-
return transformer.Repository.Create(row.BlockHeight, row.BlockHash.Hex(), metadata, value)
42+
return transformer.Repository.Create(diff.BlockHeight, diff.BlockHash.Hex(), metadata, value)
4343
}
4444
```
4545

4646
## Custom Code
4747

4848
In order to watch an additional smart contract, a developer must create three things:
4949

50-
1. Mappings - specify how to identify keys in the contract's storage trie.
50+
1. StorageKeysLoader - identify keys in the contract's storage trie, providing metadata to describe how associated values should be decoded.
5151
1. Repository - specify how to persist a parsed version of the storage value matching the recognized storage key.
5252
1. Instance - create an instance of the storage transformer that uses your mappings and repository.
5353

54-
### Mappings
54+
### StorageKeysLoader
55+
56+
A `StorageKeysLoader` is used by the `StorageKeysLookup` object on a storage transformer.
5557

5658
```golang
57-
type Mappings interface {
58-
Lookup(key common.Hash) (shared.StorageValueMetadata, error)
59+
type KeysLoader interface {
60+
LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
5961
SetDB(db *postgres.DB)
6062
}
6163
```
6264

63-
A contract-specific implementation of the mappings interface enables the storage transformer to fetch metadata associated with a storage key.
65+
When a key is not found, the lookup object refreshes its known keys by calling the loader.
66+
67+
```golang
68+
func (lookup *keysLookup) refreshMappings() error {
69+
var err error
70+
lookup.mappings, err = lookup.loader.LoadMappings()
71+
if err != nil {
72+
return err
73+
}
74+
lookup.mappings = utils.AddHashedKeys(lookup.mappings)
75+
return nil
76+
}
77+
```
78+
79+
A contract-specific implementation of the loader enables the storage transformer to fetch metadata associated with a storage key.
6480

6581
Storage metadata contains: the name of the variable matching the storage key, a raw version of any keys associated with the variable (if the variable is a mapping), and the variable's type.
6682

@@ -72,7 +88,7 @@ type StorageValueMetadata struct {
7288
}
7389
```
7490

75-
Keys are only relevant if the variable is a mapping. For example, in the following Solidity code:
91+
The `Keys` field on the metadata is only relevant if the variable is a mapping. For example, in the following Solidity code:
7692

7793
```solidity
7894
pragma solidity ^0.4.0;
@@ -85,7 +101,7 @@ contract Contract {
85101

86102
The metadata for variable `x` would not have any associated keys, but the metadata for a storage key associated with `y` would include the address used to specify that key's index in the mapping.
87103

88-
The `SetDB` function is required for the mappings to connect to the database.
104+
The `SetDB` function is required for the storage key loader to connect to the database.
89105
A database connection may be desired when keys in a mapping variable need to be read from log events (e.g. to lookup what addresses may exist in `y`, above).
90106

91107
### Repository
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// VulcanizeDB
2+
// Copyright © 2019 Vulcanize
3+
4+
// This program is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Affero General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// This program is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Affero General Public License for more details.
13+
14+
// You should have received a copy of the GNU Affero General Public License
15+
// along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package storage
18+
19+
import (
20+
"github.com/ethereum/go-ethereum/common"
21+
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
22+
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
23+
)
24+
25+
type KeysLoader interface {
26+
LoadMappings() (map[common.Hash]utils.StorageValueMetadata, error)
27+
SetDB(db *postgres.DB)
28+
}
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// VulcanizeDB
2+
// Copyright © 2019 Vulcanize
3+
4+
// This program is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Affero General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// This program is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Affero General Public License for more details.
13+
14+
// You should have received a copy of the GNU Affero General Public License
15+
// along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package storage
18+
19+
import (
20+
"github.com/ethereum/go-ethereum/common"
21+
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
22+
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
23+
)
24+
25+
type KeysLookup interface {
26+
Lookup(key common.Hash) (utils.StorageValueMetadata, error)
27+
SetDB(db *postgres.DB)
28+
}
29+
30+
type keysLookup struct {
31+
loader KeysLoader
32+
mappings map[common.Hash]utils.StorageValueMetadata
33+
}
34+
35+
func NewKeysLookup(loader KeysLoader) KeysLookup {
36+
return &keysLookup{loader: loader, mappings: make(map[common.Hash]utils.StorageValueMetadata)}
37+
}
38+
39+
func (lookup *keysLookup) Lookup(key common.Hash) (utils.StorageValueMetadata, error) {
40+
metadata, ok := lookup.mappings[key]
41+
if !ok {
42+
refreshErr := lookup.refreshMappings()
43+
if refreshErr != nil {
44+
return metadata, refreshErr
45+
}
46+
metadata, ok = lookup.mappings[key]
47+
if !ok {
48+
return metadata, utils.ErrStorageKeyNotFound{Key: key.Hex()}
49+
}
50+
}
51+
return metadata, nil
52+
}
53+
54+
func (lookup *keysLookup) refreshMappings() error {
55+
var err error
56+
lookup.mappings, err = lookup.loader.LoadMappings()
57+
if err != nil {
58+
return err
59+
}
60+
lookup.mappings = utils.AddHashedKeys(lookup.mappings)
61+
return nil
62+
}
63+
64+
func (lookup *keysLookup) SetDB(db *postgres.DB) {
65+
lookup.loader.SetDB(db)
66+
}
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// VulcanizeDB
2+
// Copyright © 2019 Vulcanize
3+
4+
// This program is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Affero General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// This program is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Affero General Public License for more details.
13+
14+
// You should have received a copy of the GNU Affero General Public License
15+
// along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package storage_test
18+
19+
import (
20+
"github.com/ethereum/go-ethereum/common"
21+
"github.com/ethereum/go-ethereum/crypto"
22+
. "github.com/onsi/ginkgo"
23+
. "github.com/onsi/gomega"
24+
"github.com/vulcanize/vulcanizedb/libraries/shared/factories/storage"
25+
"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
26+
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
27+
"github.com/vulcanize/vulcanizedb/pkg/fakes"
28+
"github.com/vulcanize/vulcanizedb/test_config"
29+
)
30+
31+
var _ = Describe("Storage keys lookup", func() {
32+
var (
33+
fakeMetadata = utils.GetStorageValueMetadata("name", map[utils.Key]string{}, utils.Uint256)
34+
lookup storage.KeysLookup
35+
loader *mocks.MockStorageKeysLoader
36+
)
37+
38+
BeforeEach(func() {
39+
loader = &mocks.MockStorageKeysLoader{}
40+
lookup = storage.NewKeysLookup(loader)
41+
})
42+
43+
Describe("Lookup", func() {
44+
Describe("when key not found", func() {
45+
It("refreshes keys", func() {
46+
loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
47+
_, err := lookup.Lookup(fakes.FakeHash)
48+
49+
Expect(err).NotTo(HaveOccurred())
50+
Expect(loader.LoadMappingsCallCount).To(Equal(1))
51+
})
52+
53+
It("returns error if refreshing keys fails", func() {
54+
loader.LoadMappingsError = fakes.FakeError
55+
56+
_, err := lookup.Lookup(fakes.FakeHash)
57+
58+
Expect(err).To(HaveOccurred())
59+
Expect(err).To(MatchError(fakes.FakeError))
60+
})
61+
})
62+
63+
Describe("when key found", func() {
64+
BeforeEach(func() {
65+
loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
66+
_, err := lookup.Lookup(fakes.FakeHash)
67+
Expect(err).NotTo(HaveOccurred())
68+
Expect(loader.LoadMappingsCallCount).To(Equal(1))
69+
})
70+
71+
It("does not refresh keys", func() {
72+
_, err := lookup.Lookup(fakes.FakeHash)
73+
74+
Expect(err).NotTo(HaveOccurred())
75+
Expect(loader.LoadMappingsCallCount).To(Equal(1))
76+
})
77+
})
78+
79+
It("returns metadata for loaded static key", func() {
80+
loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
81+
82+
metadata, err := lookup.Lookup(fakes.FakeHash)
83+
84+
Expect(err).NotTo(HaveOccurred())
85+
Expect(metadata).To(Equal(fakeMetadata))
86+
})
87+
88+
It("returns metadata for hashed version of key (accommodates keys emitted from Geth)", func() {
89+
loader.StorageKeyMappings = map[common.Hash]utils.StorageValueMetadata{fakes.FakeHash: fakeMetadata}
90+
91+
hashedKey := common.BytesToHash(crypto.Keccak256(fakes.FakeHash.Bytes()))
92+
metadata, err := lookup.Lookup(hashedKey)
93+
94+
Expect(err).NotTo(HaveOccurred())
95+
Expect(metadata).To(Equal(fakeMetadata))
96+
})
97+
98+
It("returns key not found error if key not found", func() {
99+
_, err := lookup.Lookup(fakes.FakeHash)
100+
101+
Expect(err).To(HaveOccurred())
102+
Expect(err).To(MatchError(utils.ErrStorageKeyNotFound{Key: fakes.FakeHash.Hex()}))
103+
})
104+
})
105+
106+
Describe("SetDB", func() {
107+
It("sets the db on the loader", func() {
108+
lookup.SetDB(test_config.NewTestDB(test_config.NewTestNode()))
109+
110+
Expect(loader.SetDBCalled).To(BeTrue())
111+
})
112+
})
113+
})

libraries/shared/factories/storage/transformer.go

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,19 @@ package storage
1818

1919
import (
2020
"github.com/ethereum/go-ethereum/common"
21-
"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
2221
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
2322
"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
2423
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
2524
)
2625

2726
type Transformer struct {
28-
HashedAddress common.Hash
29-
Mappings storage.Mappings
30-
Repository Repository
27+
HashedAddress common.Hash
28+
StorageKeysLookup KeysLookup
29+
Repository Repository
3130
}
3231

3332
func (transformer Transformer) NewTransformer(db *postgres.DB) transformer.StorageTransformer {
34-
transformer.Mappings.SetDB(db)
33+
transformer.StorageKeysLookup.SetDB(db)
3534
transformer.Repository.SetDB(db)
3635
return transformer
3736
}
@@ -41,7 +40,7 @@ func (transformer Transformer) KeccakContractAddress() common.Hash {
4140
}
4241

4342
func (transformer Transformer) Execute(diff utils.StorageDiff) error {
44-
metadata, lookupErr := transformer.Mappings.Lookup(diff.StorageKey)
43+
metadata, lookupErr := transformer.StorageKeysLookup.Lookup(diff.StorageKey)
4544
if lookupErr != nil {
4645
return lookupErr
4746
}

libraries/shared/factories/storage/transformer_test.go

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,18 +28,18 @@ import (
2828

2929
var _ = Describe("Storage transformer", func() {
3030
var (
31-
mappings *mocks.MockMappings
32-
repository *mocks.MockStorageRepository
33-
t storage.Transformer
31+
storageKeysLookup *mocks.MockStorageKeysLookup
32+
repository *mocks.MockStorageRepository
33+
t storage.Transformer
3434
)
3535

3636
BeforeEach(func() {
37-
mappings = &mocks.MockMappings{}
37+
storageKeysLookup = &mocks.MockStorageKeysLookup{}
3838
repository = &mocks.MockStorageRepository{}
3939
t = storage.Transformer{
40-
HashedAddress: common.Hash{},
41-
Mappings: mappings,
42-
Repository: repository,
40+
HashedAddress: common.Hash{},
41+
StorageKeysLookup: storageKeysLookup,
42+
Repository: repository,
4343
}
4444
})
4545

@@ -53,11 +53,11 @@ var _ = Describe("Storage transformer", func() {
5353
It("looks up metadata for storage key", func() {
5454
t.Execute(utils.StorageDiff{})
5555

56-
Expect(mappings.LookupCalled).To(BeTrue())
56+
Expect(storageKeysLookup.LookupCalled).To(BeTrue())
5757
})
5858

5959
It("returns error if lookup fails", func() {
60-
mappings.LookupErr = fakes.FakeError
60+
storageKeysLookup.LookupErr = fakes.FakeError
6161

6262
err := t.Execute(utils.StorageDiff{})
6363

@@ -67,7 +67,7 @@ var _ = Describe("Storage transformer", func() {
6767

6868
It("creates storage row with decoded data", func() {
6969
fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
70-
mappings.Metadata = fakeMetadata
70+
storageKeysLookup.Metadata = fakeMetadata
7171
rawValue := common.HexToAddress("0x12345")
7272
fakeBlockNumber := 123
7373
fakeBlockHash := "0x67890"
@@ -91,7 +91,7 @@ var _ = Describe("Storage transformer", func() {
9191
It("returns error if creating row fails", func() {
9292
rawValue := common.HexToAddress("0x12345")
9393
fakeMetadata := utils.StorageValueMetadata{Type: utils.Address}
94-
mappings.Metadata = fakeMetadata
94+
storageKeysLookup.Metadata = fakeMetadata
9595
repository.CreateErr = fakes.FakeError
9696

9797
err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})
@@ -118,7 +118,7 @@ var _ = Describe("Storage transformer", func() {
118118
}
119119

120120
It("passes the decoded data items to the repository", func() {
121-
mappings.Metadata = fakeMetadata
121+
storageKeysLookup.Metadata = fakeMetadata
122122
fakeRow := utils.StorageDiff{
123123
HashedAddress: common.Hash{},
124124
BlockHash: common.HexToHash(fakeBlockHash),
@@ -140,7 +140,7 @@ var _ = Describe("Storage transformer", func() {
140140
})
141141

142142
It("returns error if creating a row fails", func() {
143-
mappings.Metadata = fakeMetadata
143+
storageKeysLookup.Metadata = fakeMetadata
144144
repository.CreateErr = fakes.FakeError
145145

146146
err := t.Execute(utils.StorageDiff{StorageValue: rawValue.Hash()})

0 commit comments

Comments
 (0)