increase file write buffer size

Author: i-norden

statediff/indexer/database/file/writer.go

@@ -31,9 +31,9 @@ import (
 )
 
 var (
-	nullHash         = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
-	pipeSize         = 65336 // min(linuxPipeSize, macOSPipeSize)
-	collatedStmtSize = pipeSize * 16
+	nullHash        = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000")
+	pipeSize        = 65336 // min(linuxPipeSize, macOSPipeSize)
+	writeBufferSize = pipeSize * 16 * 48
 )
 
 // SQLWriter writes sql statements to a file
@@ -54,7 +54,7 @@ func NewSQLWriter(wc io.WriteCloser) *SQLWriter {
 	return &SQLWriter{
 		wc:            wc,
 		stmts:         make(chan []byte),
-		collatedStmt:  make([]byte, collatedStmtSize),
+		collatedStmt:  make([]byte, writeBufferSize),
 		flushChan:     make(chan struct{}),
 		flushFinished: make(chan struct{}),
 		quitChan:      make(chan struct{}),
@@ -74,7 +74,7 @@ func (sqw *SQLWriter) Loop() {
 			select {
 			case stmt := <-sqw.stmts:
 				l = len(stmt)
-				if l+sqw.collationIndex+1 > collatedStmtSize {
+				if sqw.collationIndex+l > writeBufferSize {
 					if err := sqw.flush(); err != nil {
 						panic(fmt.Sprintf("error writing sql stmts buffer to file: %v", err))
 					}

statediff/indexer/ipld/eth_header.go

@@ -20,13 +20,13 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/common"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // EthHeader (eth-block, codec 0x90), represents an ethereum block header

statediff/indexer/ipld/eth_header_test.go

@@ -9,10 +9,11 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/core/types"
 	block "github.com/ipfs/go-block-format"
 	node "github.com/ipfs/go-ipld-format"
 	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 func TestBlockBodyRlpParsing(t *testing.T) {

statediff/indexer/ipld/eth_log.go

@@ -3,11 +3,12 @@ package ipld
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // EthLog (eth-log, codec 0x9a), represents an ethereum block header

statediff/indexer/ipld/eth_log_trie.go

@@ -3,10 +3,11 @@ package ipld
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 // EthLogTrie (eth-tx-trie codec 0x9p) represents

statediff/indexer/ipld/eth_parser.go

@@ -23,11 +23,12 @@ import (
 	"io"
 	"io/ioutil"
 
+	"github.com/ipfs/go-cid"
+	"github.com/multiformats/go-multihash"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ipfs/go-cid"
-	"github.com/multiformats/go-multihash"
 )
 
 // FromBlockRLP takes an RLP message representing

statediff/indexer/ipld/eth_parser_test.go

@@ -21,10 +21,11 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
-	"github.com/stretchr/testify/require"
 )
 
 type kind string

statediff/indexer/ipld/eth_receipt.go

@@ -21,10 +21,11 @@ import (
 	"fmt"
 	"strconv"
 
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 type EthReceipt struct {

statediff/indexer/ipld/eth_tx.go

@@ -22,11 +22,12 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
 	mh "github.com/multiformats/go-multihash"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
 )
 
 // EthTx (eth-tx codec 0x93) represents an ethereum transaction

statediff/indexer/ipld/trie_node.go

@@ -20,9 +20,10 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ipfs/go-cid"
 	node "github.com/ipfs/go-ipld-format"
+
+	"github.com/ethereum/go-ethereum/rlp"
 )
 
 const (