Skip to content

Commit 1b84a0e

Browse files
authored
Merge branch 'staging' into error-log-stderr
2 parents 9eb1980 + d34e2f0 commit 1b84a0e

File tree

8 files changed

+209
-46
lines changed

8 files changed

+209
-46
lines changed

core/client/init_node.go

Lines changed: 90 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,10 +136,98 @@ func Init(ctx context.Context, cfg conf.Config) error {
136136
return err
137137
}
138138

139+
// Check if a node is online
140+
isNodeOnline := func(nodeURL string) bool {
141+
reqCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
142+
defer cancel()
143+
req, err := util.NewHTTPGetRequestContext(reqCtx, nodeURL)
144+
if err != nil {
145+
return false
146+
}
147+
res, err := req.Get()
148+
return err == nil && res.StatusCode == http.StatusOK
149+
}
150+
151+
// Filter online nodes concurrently
152+
filterOnlineNodes := func(nodes []string) []string {
153+
var onlineNodes []string
154+
var wg sync.WaitGroup
155+
var mu sync.Mutex
156+
157+
for _, node := range nodes {
158+
wg.Add(1)
159+
go func(url string) {
160+
defer wg.Done()
161+
if isNodeOnline(url) {
162+
mu.Lock()
163+
onlineNodes = append(onlineNodes, url)
164+
mu.Unlock()
165+
} else {
166+
logging.Debug("Node offline during initialization", zap.String("url", url))
167+
}
168+
}(node)
169+
}
170+
wg.Wait()
171+
return onlineNodes
172+
}
173+
174+
onlineMiners := filterOnlineNodes(network.Miners)
175+
onlineSharders := filterOnlineNodes(network.Sharders)
176+
139177
reqMiners := util.MaxInt(3, int(math.Ceil(float64(cfg.MinSubmit)*float64(len(network.Miners))/100)))
140-
sharders := NewHolder(network.Sharders, util.MinInt(len(network.Sharders), util.MaxInt(cfg.SharderConsensous, conf.DefaultSharderConsensous)))
178+
reqSharders := util.MinInt(len(network.Sharders), util.MaxInt(cfg.SharderConsensous, conf.DefaultSharderConsensous))
179+
180+
if len(onlineMiners) < reqMiners {
181+
logging.Debug("Not enough online miners found, supplementing with random miners",
182+
zap.Int("online", len(onlineMiners)),
183+
zap.Int("required", reqMiners))
184+
185+
onlineMinersMap := make(map[string]struct{})
186+
for _, miner := range onlineMiners {
187+
onlineMinersMap[miner] = struct{}{}
188+
}
189+
190+
offlineMiners := make([]string, 0)
191+
for _, miner := range network.Miners {
192+
if _, exists := onlineMinersMap[miner]; !exists {
193+
offlineMiners = append(offlineMiners, miner)
194+
}
195+
}
196+
197+
needed := reqMiners - len(onlineMiners)
198+
if needed > 0 && len(offlineMiners) > 0 {
199+
randomOfflineMiners := util.GetRandom(offlineMiners, needed)
200+
onlineMiners = append(onlineMiners, randomOfflineMiners...)
201+
}
202+
}
203+
204+
if len(onlineSharders) < reqSharders {
205+
logging.Debug("Not enough online sharders found, supplementing with random sharders",
206+
zap.Int("online", len(onlineSharders)),
207+
zap.Int("required", reqSharders))
208+
209+
onlineShardersMap := make(map[string]struct{})
210+
for _, sharder := range onlineSharders {
211+
onlineShardersMap[sharder] = struct{}{}
212+
}
213+
214+
offlineSharders := make([]string, 0)
215+
for _, sharder := range network.Sharders {
216+
if _, exists := onlineShardersMap[sharder]; !exists {
217+
offlineSharders = append(offlineSharders, sharder)
218+
}
219+
}
220+
221+
needed := reqSharders - len(onlineSharders)
222+
if needed > 0 && len(offlineSharders) > 0 {
223+
randomOfflineSharders := util.GetRandom(offlineSharders, needed)
224+
onlineSharders = append(onlineSharders, randomOfflineSharders...)
225+
}
226+
}
227+
228+
sharders := NewHolder(onlineSharders, reqSharders)
141229
nodeClient = &Node{
142-
stableMiners: util.GetRandom(network.Miners, reqMiners),
230+
stableMiners: onlineMiners,
143231
sharders: sharders,
144232
network: network,
145233
clientCtx: ctx,

core/client/set.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ type InitSdkOptions struct {
4646
AddWallet bool
4747
TxnFee *int
4848
MinConfirmation *int
49-
MinSubmit *int
5049
ConfirmationChainLength *int
50+
MinSubmit *int
5151
SharderConsensous *int
5252
ZboxHost string
5353
ZboxAppType string
@@ -287,6 +287,7 @@ func PublicKey(clients ...string) string {
287287
}
288288
return client.wallets[clients[0]].ClientKey
289289
}
290+
290291
return client.wallet.ClientKey
291292
}
292293

go.mod

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@ require (
2929
github.com/uptrace/bunrouter v1.0.20
3030
go.dedis.ch/kyber/v3 v3.1.0
3131
go.uber.org/zap v1.24.0
32-
golang.org/x/crypto v0.21.0
32+
golang.org/x/crypto v0.26.0
3333
golang.org/x/image v0.14.0
34-
golang.org/x/sync v0.7.0
34+
golang.org/x/sync v0.8.0
3535
gopkg.in/cheggaaa/pb.v1 v1.0.28
3636
gopkg.in/natefinch/lumberjack.v2 v2.2.1
3737
gopkg.in/yaml.v2 v2.4.0
@@ -107,8 +107,8 @@ require (
107107
github.com/x448/float16 v0.8.4 // indirect
108108
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
109109
go.dedis.ch/fixbuf v1.0.3 // indirect
110-
golang.org/x/sys v0.18.0 // indirect
111-
golang.org/x/text v0.14.0 // indirect
110+
golang.org/x/sys v0.23.0 // indirect
111+
golang.org/x/text v0.17.0 // indirect
112112
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
113113
gopkg.in/ini.v1 v1.67.0 // indirect
114114
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
@@ -117,13 +117,13 @@ require (
117117
require (
118118
github.com/btcsuite/btcd/btcutil v1.1.3
119119
github.com/hack-pad/safejs v0.1.1
120-
github.com/klauspost/compress v1.17.0 // indirect
120+
github.com/klauspost/compress v1.17.9 // indirect
121121
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
122122
github.com/remeh/sizedwaitgroup v1.0.0
123123
github.com/yusufpapurcu/wmi v1.2.2 // indirect
124124
go.uber.org/atomic v1.10.0 // indirect
125125
go.uber.org/multierr v1.9.0 // indirect
126-
golang.org/x/net v0.23.0 // indirect
126+
golang.org/x/net v0.28.0 // indirect
127127
golang.org/x/time v0.3.0 // indirect
128128
)
129129

go.sum

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -334,8 +334,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
334334
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
335335
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
336336
github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
337-
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
338-
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
337+
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
338+
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
339339
github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
340340
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
341341
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
@@ -579,8 +579,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
579579
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
580580
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
581581
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
582-
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
583-
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
582+
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
583+
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
584584
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
585585
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
586586
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -659,8 +659,8 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
659659
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
660660
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
661661
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
662-
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
663-
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
662+
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
663+
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
664664
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
665665
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
666666
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -681,8 +681,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
681681
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
682682
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
683683
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
684-
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
685-
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
684+
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
685+
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
686686
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
687687
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
688688
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -734,17 +734,17 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc
734734
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
735735
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
736736
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
737-
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
738-
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
737+
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
738+
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
739739
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
740740
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
741741
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
742742
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
743743
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
744744
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
745745
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
746-
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
747-
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
746+
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
747+
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
748748
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
749749
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
750750
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

zboxcore/sdk/allocation.go

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,16 @@ import (
4444
)
4545

4646
var (
47-
noBLOBBERS = errors.New("", "No Blobbers set in this allocation")
48-
notInitialized = errors.New("sdk_not_initialized", "Please call InitStorageSDK Init and use GetAllocation to get the allocation object")
49-
IsWasm = false
50-
MultiOpBatchSize = 50
51-
RepairBatchSize = 50
52-
Workdir string
53-
logChanMap = make(map[string]chan logEntry)
54-
logMapMutex = &sync.Mutex{}
47+
noBLOBBERS = errors.New("", "No Blobbers set in this allocation")
48+
notInitialized = errors.New("sdk_not_initialized", "Please call InitStorageSDK Init and use GetAllocation to get the allocation object")
49+
IsWasm = false
50+
MultiOpBatchSize = 50
51+
RepairBatchSize = 50
52+
Workdir string
53+
logChanMap = make(map[string]chan logEntry)
54+
logMapMutex = &sync.Mutex{}
55+
LogBlobberMonitoringFileSize = int64(0)
56+
LogBlobberMonitoringChan = make(chan BlobberMonitoring)
5557
)
5658

5759
const (
@@ -80,6 +82,19 @@ var GetFileInfo = func(localpath string) (os.FileInfo, error) {
8082
return sys.Files.Stat(localpath)
8183
}
8284

85+
func SetBlobberMonitoringFileSize(val int64) {
86+
LogBlobberMonitoringFileSize = val
87+
}
88+
89+
type BlobberMonitoring struct {
90+
BlobberId string `json:"blobber_id"`
91+
Operation string `json:"operation"`
92+
FileType string `json:"file_type"`
93+
FileSize int64 `json:"file_size"`
94+
TimeSpent int64 `json:"time_spent"`
95+
Count int `json:"count"`
96+
}
97+
8398
// BlobberAllocationStats represents the blobber allocation statistics.
8499
type BlobberAllocationStats struct {
85100
BlobberID string
@@ -3434,3 +3449,7 @@ func writeLogEntry(blobberURL string, log logEntry) {
34343449
logChan := getLogChan(blobberURL)
34353450
logChan <- log
34363451
}
3452+
3453+
func addBlobberMonitoringLog(log BlobberMonitoring) {
3454+
LogBlobberMonitoringChan <- log
3455+
}

zboxcore/sdk/blockdownloadworker.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,18 @@ func (req *BlockDownloadRequest) downloadBlobberBlock(fastClient *fasthttp.Clien
188188
}
189189
return errors.New("response_error", string(respBuf))
190190
}
191+
192+
if LogBlobberMonitoringFileSize > int64(0) {
193+
blobberMonitoringlog := BlobberMonitoring{
194+
BlobberId: req.blobber.ID,
195+
Operation: "download",
196+
TimeSpent: timeTaken,
197+
FileSize: LogBlobberMonitoringFileSize,
198+
Count: 1,
199+
}
200+
addBlobberMonitoringLog(blobberMonitoringlog)
201+
}
202+
191203
entry := logEntry{
192204
OpType: "download",
193205
TimeTaken: timeTaken,

zboxcore/sdk/chunked_upload_blobber.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,10 @@ func (sb *ChunkedUploadBlobber) sendUploadRequest(
8989
err, shouldContinue = func() (err error, shouldContinue bool) {
9090
resp := fasthttp.AcquireResponse()
9191
defer fasthttp.ReleaseResponse(resp)
92+
now := time.Now()
9293
err = zboxutil.FastHttpClient.DoTimeout(req, resp, su.uploadTimeOut)
9394
fasthttp.ReleaseRequest(req)
95+
timeTaken := time.Since(now).Milliseconds()
9496
if err != nil {
9597
logger.Logger.Error("Upload : ", err, " baseurl: ", sb.blobber.Baseurl)
9698
if errors.Is(err, fasthttp.ErrConnectionClosed) || errors.Is(err, syscall.EPIPE) {
@@ -99,6 +101,17 @@ func (sb *ChunkedUploadBlobber) sendUploadRequest(
99101
return fmt.Errorf("Error while doing reqeust. Error %s", err), false
100102
}
101103

104+
if LogBlobberMonitoringFileSize > int64(0) {
105+
blobberMonitoringlog := BlobberMonitoring{
106+
BlobberId: sb.blobber.ID,
107+
Operation: "upload",
108+
TimeSpent: timeTaken,
109+
FileSize: LogBlobberMonitoringFileSize,
110+
Count: 1,
111+
}
112+
addBlobberMonitoringLog(blobberMonitoringlog)
113+
}
114+
102115
if resp.StatusCode() == http.StatusOK {
103116
return
104117
}
@@ -139,6 +152,7 @@ func (sb *ChunkedUploadBlobber) sendUploadRequest(
139152

140153
break
141154
}
155+
142156
return err
143157
})
144158
}

0 commit comments

Comments
 (0)