Merge pull request #158 from vulcanize/missed_marked_checked_headers

Fix for issue #146

Author: i-norden

integration_test/contract_watcher_full_transformer_test.go

@@ -18,7 +18,6 @@ package integration
 
 import (
 	"fmt"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"math/rand"
 	"strings"
 	"time"
@@ -27,6 +26,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/helpers/test_helpers"

integration_test/contract_watcher_header_sync_transformer_test.go

@@ -40,8 +40,8 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
 	var blockChain core.BlockChain
 	var headerRepository repositories.HeaderRepository
 	var headerID int64
-	var ensAddr = strings.ToLower(constants.EnsContractAddress)
-	var tusdAddr = strings.ToLower(constants.TusdContractAddress)
+	var ensAddr = strings.ToLower(constants.EnsContractAddress)   // 0x314159265dd8dbb310642f98f50c066173c1259b
+	var tusdAddr = strings.ToLower(constants.TusdContractAddress) // 0x8dd5fbce2f6a956c3022ba3663759011dd51e73e
 
 	BeforeEach(func() {
 		db, blockChain = test_helpers.SetupDBandBC()
@@ -377,6 +377,42 @@ var _ = Describe("contractWatcher headerSync transformer", func() {
 			Expect(transferLog.Value).To(Equal("2800000000000000000000"))
 		})
 
+		It("Marks header checked for a contract that has no logs at that header", func() {
+			t := transformer.NewTransformer(test_helpers.ENSandTusdConfig, blockChain, db)
+			err = t.Init()
+			Expect(err).ToNot(HaveOccurred())
+			err = t.Execute()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(t.Start).To(Equal(int64(6885702)))
+
+			newOwnerLog := test_helpers.HeaderSyncNewOwnerLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.newowner_event", ensAddr)).StructScan(&newOwnerLog)
+			Expect(err).ToNot(HaveOccurred())
+			transferLog := test_helpers.HeaderSyncTransferLog{}
+			err = db.QueryRowx(fmt.Sprintf("SELECT * FROM header_%s.transfer_event", tusdAddr)).StructScan(&transferLog)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(transferLog.HeaderID).ToNot(Equal(newOwnerLog.HeaderID))
+
+			type checkedHeader struct {
+				ID       int64 `db:"id"`
+				HeaderID int64 `db:"header_id"`
+				NewOwner int64 `db:"newowner_0x314159265dd8dbb310642f98f50c066173c1259b"`
+				Transfer int64 `db:"transfer_0x8dd5fbce2f6a956c3022ba3663759011dd51e73e"`
+			}
+
+			transferCheckedHeader := new(checkedHeader)
+			err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", transferLog.HeaderID).StructScan(transferCheckedHeader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(transferCheckedHeader.Transfer).To(Equal(int64(1)))
+			Expect(transferCheckedHeader.NewOwner).To(Equal(int64(1)))
+
+			newOwnerCheckedHeader := new(checkedHeader)
+			err = db.QueryRowx("SELECT * FROM public.checked_headers WHERE header_id = $1", newOwnerLog.HeaderID).StructScan(newOwnerCheckedHeader)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(newOwnerCheckedHeader.NewOwner).To(Equal(int64(1)))
+			Expect(newOwnerCheckedHeader.Transfer).To(Equal(int64(1)))
+		})
+
 		It("Keeps track of contract-related hashes and addresses while transforming event data if they need to be used for later method polling", func() {
 			var testConf config.ContractConfig
 			testConf = test_helpers.ENSandTusdConfig

libraries/shared/repository/repository.go

@@ -17,7 +17,6 @@
 package converter
 
 import (
-	"errors"
 	"fmt"
 	"math/big"
 	"strconv"
@@ -32,22 +31,24 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
-// Converter is used to convert watched event logs to
+// ConverterInterface is used to convert watched event logs to
 // custom logs containing event input name => value maps
 type ConverterInterface interface {
 	Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error)
 	Update(info *contract.Contract)
 }
 
+// Converter is the underlying struct for the ConverterInterface
 type Converter struct {
 	ContractInfo *contract.Contract
 }
 
+// Update configures the converter for a specific contract
 func (c *Converter) Update(info *contract.Contract) {
 	c.ContractInfo = info
 }
 
-// Convert the given watched event log into a types.Log for the given event
+// Convert converts the given watched event log into a types.Log for the given event
 func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (*types.Log, error) {
 	boundContract := bind.NewBoundContract(common.HexToAddress(c.ContractInfo.Address), c.ContractInfo.ParsedAbi, nil, nil, nil)
 	values := make(map[string]interface{})
@@ -88,14 +89,14 @@ func (c *Converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
 			b := input.(byte)
 			strValues[fieldName] = string(b)
 		default:
-			return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input))
+			return nil, fmt.Errorf("error: unhandled abi type %T", input)
 		}
 	}
 
 	// Only hold onto logs that pass our address filter, if any
 	if c.ContractInfo.PassesEventFilter(strValues) {
 		eventLog := &types.Log{
-			Id:     watchedEvent.LogID,
+			ID:     watchedEvent.LogID,
 			Values: strValues,
 			Block:  watchedEvent.BlockNumber,
 			Tx:     watchedEvent.TxHash,

libraries/shared/repository/repository_test.go

@@ -18,11 +18,12 @@ package retriever
 
 import (
 	"database/sql"
+
 	"github.com/vulcanize/vulcanizedb/libraries/shared/repository"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
-// Block retriever is used to retrieve the first block for a given contract and the most recent block
+// BlockRetriever is used to retrieve the first block for a given contract and the most recent block
 // It requires a vDB synced database with blocks, transactions, receipts, and logs
 type BlockRetriever interface {
 	RetrieveFirstBlock(contractAddr string) (int64, error)
@@ -33,13 +34,15 @@ type blockRetriever struct {
 	db *postgres.DB
 }
 
-func NewBlockRetriever(db *postgres.DB) (r *blockRetriever) {
+// NewBlockRetriever returns a new BlockRetriever
+func NewBlockRetriever(db *postgres.DB) BlockRetriever {
 	return &blockRetriever{
 		db: db,
 	}
 }
 
-// Try both methods of finding the first block, with the receipt method taking precedence
+// RetrieveFirstBlock fetches the block number for the earliest block in the db
+// Tries both methods of finding the first block, with the receipt method taking precedence
 func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error) {
 	i, err := r.retrieveFirstBlockFromReceipts(contractAddr)
 	if err != nil {
@@ -55,7 +58,7 @@ func (r *blockRetriever) RetrieveFirstBlock(contractAddr string) (int64, error)
 // For some contracts the contract creation transaction receipt doesn't have the contract address so this doesn't work (e.g. Sai)
 func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (int64, error) {
 	var firstBlock int64
-	addressId, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr)
+	addressID, getAddressErr := repository.GetOrCreateAddress(r.db, contractAddr)
 	if getAddressErr != nil {
 		return firstBlock, getAddressErr
 	}
@@ -66,7 +69,7 @@ func (r *blockRetriever) retrieveFirstBlockFromReceipts(contractAddr string) (in
                            WHERE contract_address_id = $1
 		                   ORDER BY block_id ASC
 					       LIMIT 1)`,
-		addressId,
+		addressID,
 	)
 
 	return firstBlock, err
@@ -84,7 +87,7 @@ func (r *blockRetriever) retrieveFirstBlockFromLogs(contractAddr string) (int64,
 	return int64(firstBlock), err
 }
 
-// Method to retrieve the most recent block in vDB
+// RetrieveMostRecentBlock retrieves the most recent block number in vDB
 func (r *blockRetriever) RetrieveMostRecentBlock() (int64, error) {
 	var lastBlock int64
 	err := r.db.Get(

pkg/contract_watcher/full/converter/converter.go

@@ -17,10 +17,11 @@
 package retriever_test
 
 import (
+	"strings"
+
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"strings"
 
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/full/retriever"
 	"github.com/vulcanize/vulcanizedb/pkg/contract_watcher/shared/constants"

pkg/contract_watcher/full/retriever/block_retriever.go

@@ -17,12 +17,12 @@
 package retriever_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
 )
 
 func TestRetriever(t *testing.T) {

pkg/contract_watcher/full/retriever/block_retriever_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
 )
 
+// Transformer is the top level struct for transforming watched contract data
 // Requires a fully synced vDB and a running eth node (or infura)
 type Transformer struct {
 	// Database interfaces
@@ -60,7 +61,7 @@ type Transformer struct {
 	LastBlock int64
 }
 
-// Transformer takes in config for blockchain, database, and network id
+// NewTransformer takes in contract config, blockchain, and database, and returns a new Transformer
 func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.DB) *Transformer {
 	return &Transformer{
 		Poller:                     poller.NewPoller(BC, DB, types.FullSync),
@@ -75,6 +76,7 @@ func NewTransformer(con config.ContractConfig, BC core.BlockChain, DB *postgres.
 	}
 }
 
+// Init initializes the transformer
 // Use after creating and setting transformer
 // Loops over all of the addr => filter sets
 // Uses parser to pull event info from abi
@@ -167,6 +169,7 @@ func (tr *Transformer) Init() error {
 	return nil
 }
 
+// Execute runs the transformation processes
 // Iterates through stored, initialized contract objects
 // Iterates through contract's event filters, grabbing watched event logs
 // Uses converter to convert logs into custom log type
@@ -227,6 +230,7 @@ func (tr *Transformer) Execute() error {
 	return nil
 }
 
+// GetConfig returns the transformers config; satisfies the transformer interface
 func (tr *Transformer) GetConfig() config.ContractConfig {
 	return tr.Config
 }

pkg/contract_watcher/full/retriever/retriever_suite_test.go

@@ -17,12 +17,12 @@
 package transformer_test
 
 import (
-	"github.com/sirupsen/logrus"
 	"io/ioutil"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
 )
 
 func TestTransformer(t *testing.T) {