Skip to content

Commit b7c77ce

Browse files
committed
fix: lotus-shed: make finality calculator calculate properly
1 parent 408bb06 commit b7c77ce

File tree

3 files changed

+40
-71
lines changed

3 files changed

+40
-71
lines changed

cmd/lotus-shed/finality.go

Lines changed: 32 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,10 @@ import (
77
"os"
88
"strconv"
99

10-
"github.com/dreading/gospecfunc/bessel"
1110
"github.com/filecoin-project/lotus/build"
11+
skellampmf "github.com/rvagg/go-skellam-pmf"
1212
"github.com/urfave/cli/v2"
1313
"golang.org/x/exp/constraints"
14-
"gonum.org/v1/gonum/stat/distuv"
1514
)
1615

1716
var finalityCmd = &cli.Command{
@@ -25,6 +24,10 @@ var finalityCmd = &cli.Command{
2524
&cli.StringFlag{
2625
Name: "input",
2726
},
27+
&cli.IntFlag{
28+
Name: "target",
29+
Usage: "target epoch for which finality is calculated",
30+
},
2831
},
2932
ArgsUsage: "[inputFile]",
3033
Action: func(cctx *cli.Context) error {
@@ -49,14 +52,15 @@ var finalityCmd = &cli.Command{
4952
return err
5053
}
5154

52-
blocksPerEpoch := 5.0 // Expected number of blocks per epoch
53-
byzantineFraction := 0.3 // Upper bound on the fraction of malicious nodes in the network
54-
currentEpoch := len(chain) - 1 // Current epoch (end of history)
55-
targetEpoch := currentEpoch - 30 // Target epoch for which finality is calculated
55+
blocksPerEpoch := 5.0 // Expected number of blocks per epoch
56+
byzantineFraction := 0.3 // Upper bound on the fraction of malicious nodes in the network
57+
currentEpoch := len(chain) - 1 // Current epoch (end of history)
58+
// targetEpoch := currentEpoch - 30 // Target epoch for which finality is calculated
59+
targetEpoch := cctx.Int("target")
5660

5761
finality := FinalityCalcValidator(chain, blocksPerEpoch, byzantineFraction, currentEpoch, targetEpoch)
5862

59-
fmt.Fprintf(cctx.App.Writer, "Finality probability: %f\n", finality)
63+
fmt.Fprintf(cctx.App.Writer, "Finality=%v @ %d for chain len=%d\n", finality, targetEpoch, currentEpoch)
6064

6165
return nil
6266
},
@@ -88,15 +92,14 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
8892
sumExpectedAdversarialBlocksI += rateMaliciousBlocks
8993
sumChainBlocksI += chain[i-1]
9094
// Poisson(k=k, lambda=sum(f*e))
91-
prLi := distuv.Poisson{Lambda: sumExpectedAdversarialBlocksI}.Prob(float64(k + sumChainBlocksI))
95+
prLi := poissonProb(sumExpectedAdversarialBlocksI, float64(k+sumChainBlocksI))
9296
prL[k] = math.Max(prL[k], prLi)
9397

94-
// Break if prL[k] becomes negligible
95-
if k > 1 && prL[k] < negligibleThreshold && prL[k] < prL[k-1] {
96-
maxKL = k
97-
prL = prL[:k+1]
98-
break
99-
}
98+
}
99+
if k > 1 && prL[k] < negligibleThreshold && prL[k] < prL[k-1] {
100+
maxKL = k
101+
prL = prL[:k+1]
102+
break
100103
}
101104
}
102105

@@ -108,7 +111,7 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
108111

109112
// Calculate Pr(B=k) for each value of k
110113
for k := 0; k <= maxKB; k++ {
111-
prB[k] = distuv.Poisson{Lambda: float64(currentEpoch-targetEpoch) * rateMaliciousBlocks}.Prob(float64(k))
114+
prB[k] = poissonProb(float64(currentEpoch-targetEpoch)*rateMaliciousBlocks, float64(k))
112115

113116
// Break if prB[k] becomes negligible
114117
if k > 1 && prB[k] < negligibleThreshold && prB[k] < prB[k-1] {
@@ -119,11 +122,11 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
119122
}
120123

121124
// Compute M
122-
prHgt0 := 1 - distuv.Poisson{Lambda: rateHonestBlocks}.Prob(0)
125+
prHgt0 := 1 - poissonProb(rateHonestBlocks, 0)
123126

124127
expZ := 0.0
125128
for k := 0; k < int(4*blocksPerEpoch); k++ {
126-
pmf := distuv.Poisson{Lambda: rateMaliciousBlocks}.Prob(float64(k))
129+
pmf := poissonProb(rateMaliciousBlocks, float64(k))
127130
expZ += ((rateHonestBlocks + float64(k)) / math.Pow(2, float64(k))) * pmf
128131
}
129132

@@ -132,7 +135,7 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
132135
prM := make([]float64, maxKM+1)
133136
for k := 0; k <= maxKM; k++ {
134137
for i := maxIM; i > 0; i-- {
135-
probMI := SkellamPMF(k, float64(i)*rateMaliciousBlocks, float64(i)*ratePublicChain)
138+
probMI := skellampmf.SkellamPMF(k, float64(i)*rateMaliciousBlocks, float64(i)*ratePublicChain)
136139

137140
// Break if probMI becomes negligible
138141
if probMI < negligibleThreshold && probMI < prM[k] {
@@ -185,6 +188,17 @@ func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFractio
185188

186189
return math.Min(prError, 1.0)
187190
}
191+
func poissonProb(lambda float64, x float64) float64 {
192+
return math.Exp(poissonLogProb(lambda, x))
193+
}
194+
195+
func poissonLogProb(lambda float64, x float64) float64 {
196+
if x < 0 || math.Floor(x) != x {
197+
return math.Inf(-1)
198+
}
199+
lg, _ := math.Lgamma(math.Floor(x) + 1)
200+
return x*math.Log(lambda) - lambda - lg
201+
}
188202

189203
func sum[T constraints.Integer | constraints.Float](s []T) T {
190204
var total T
@@ -210,46 +224,3 @@ func min(a, b int) int {
210224
}
211225
return b
212226
}
213-
214-
// SkellamPMF calculates the probability mass function (PMF) of a Skellam distribution.
215-
//
216-
// The Skellam distribution is the probability distribution of the difference
217-
// of two independent Poisson random variables.
218-
//
219-
// Arguments:
220-
// * k - The difference of two Poisson random variables.
221-
// * mu1 - The expected value of the first Poisson distribution.
222-
// * mu2 - The expected value of the second Poisson distribution.
223-
//
224-
// Returns:
225-
// * A float64 representing the PMF of the Skellam distribution at k.
226-
func SkellamPMF(k int, mu1 float64, mu2 float64) float64 {
227-
// Based on https://github.com/jsoares/rusty-skellam/blob/main/src/lib.rs
228-
229-
// Return NaN if parameters outside range
230-
if math.IsNaN(mu1) || mu1 <= 0 || math.IsNaN(mu2) || mu2 <= 0 {
231-
return math.NaN()
232-
}
233-
234-
// Parameterise and compute the Modified Bessel function of the first kind
235-
nu := float64(k)
236-
z := complex(2.0*math.Sqrt(mu1*mu2), 0)
237-
besselResult := bessel.I(nu, z)
238-
239-
// Compute the pmf
240-
return math.Exp(-(mu1 + mu2)) * math.Pow(mu1/mu2, nu/2.0) * real(besselResult)
241-
}
242-
243-
/*
244-
func main() {
245-
seed := rand.NewSource(1)
246-
random := rand.New(seed)
247-
chain := make([]int, 1000)
248-
for i := range chain {
249-
chain[i] = random.Intn(5) + 1
250-
}
251-
252-
errorProbability := FinalityCalcValidator(chain, 5.0, 0.3, 1000, 900)
253-
fmt.Printf("Error probability: %f\n", errorProbability)
254-
}
255-
*/

go.mod

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module github.com/filecoin-project/lotus
22

3-
go 1.21
3+
go 1.21.11
44

55
retract v1.14.0 // Accidentally force-pushed tag, use v1.14.1+ instead.
66

@@ -42,7 +42,7 @@ require (
4242
github.com/filecoin-project/go-jsonrpc v0.3.2
4343
github.com/filecoin-project/go-padreader v0.0.1
4444
github.com/filecoin-project/go-paramfetch v0.0.4
45-
github.com/filecoin-project/go-state-types v0.14.0-dev
45+
github.com/filecoin-project/go-state-types v0.14.0-rc1
4646
github.com/filecoin-project/go-statemachine v1.0.3
4747
github.com/filecoin-project/go-statestore v0.2.0
4848
github.com/filecoin-project/go-storedcounter v0.1.0
@@ -123,6 +123,7 @@ require (
123123
github.com/puzpuzpuz/xsync/v2 v2.4.0
124124
github.com/raulk/clock v1.1.0
125125
github.com/raulk/go-watchdog v1.3.0
126+
github.com/rvagg/go-skellam-pmf v0.0.0
126127
github.com/samber/lo v1.39.0
127128
github.com/stretchr/testify v1.9.0
128129
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
@@ -146,6 +147,7 @@ require (
146147
go.uber.org/multierr v1.11.0
147148
go.uber.org/zap v1.27.0
148149
golang.org/x/crypto v0.23.0
150+
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
149151
golang.org/x/net v0.25.0
150152
golang.org/x/sync v0.7.0
151153
golang.org/x/sys v0.20.0
@@ -181,7 +183,6 @@ require (
181183
github.com/dgraph-io/ristretto v0.1.1 // indirect
182184
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
183185
github.com/drand/kyber-bls12381 v0.3.1 // indirect
184-
github.com/dreading/gospecfunc v0.0.0-20191105042551-e794f60da5c3 // indirect
185186
github.com/elastic/go-windows v1.0.0 // indirect
186187
github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect
187188
github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect
@@ -318,7 +319,6 @@ require (
318319
go.uber.org/dig v1.17.1 // indirect
319320
go.uber.org/mock v0.4.0 // indirect
320321
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
321-
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
322322
golang.org/x/mod v0.17.0 // indirect
323323
golang.org/x/text v0.15.0 // indirect
324324
gonum.org/v1/gonum v0.15.0 // indirect

go.sum

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -216,8 +216,6 @@ github.com/drand/kyber v1.3.0 h1:TVd7+xoRgKQ4Ck1viNLPFy6IWhuZM36Bq6zDXD8Asls=
216216
github.com/drand/kyber v1.3.0/go.mod h1:f+mNHjiGT++CuueBrpeMhFNdKZAsy0tu03bKq9D5LPA=
217217
github.com/drand/kyber-bls12381 v0.3.1 h1:KWb8l/zYTP5yrvKTgvhOrk2eNPscbMiUOIeWBnmUxGo=
218218
github.com/drand/kyber-bls12381 v0.3.1/go.mod h1:H4y9bLPu7KZA/1efDg+jtJ7emKx+ro3PU7/jWUVt140=
219-
github.com/dreading/gospecfunc v0.0.0-20191105042551-e794f60da5c3 h1:oOp1la+wHlyd3ODqW2CbFj8w6Lod4gPMzHFbD0rbp88=
220-
github.com/dreading/gospecfunc v0.0.0-20191105042551-e794f60da5c3/go.mod h1:lkytgpljbGOM3VZj4Fm7FkGy/oUInQFklbkHBVAvJEg=
221219
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
222220
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
223221
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
@@ -293,9 +291,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go
293291
github.com/filecoin-project/go-state-types v0.1.0/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
294292
github.com/filecoin-project/go-state-types v0.1.6/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
295293
github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
296-
github.com/filecoin-project/go-state-types v0.13.1/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY=
297-
github.com/filecoin-project/go-state-types v0.14.0-dev h1:bDwq1S28D7EC/uDmKU8vvNcdFw/YDsNq09pe3zeV5h4=
298-
github.com/filecoin-project/go-state-types v0.14.0-dev/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY=
294+
github.com/filecoin-project/go-state-types v0.14.0-rc1 h1:kWBGX/uqZmYotYMNmw+R/fIuot/k0KMcEtB7PKFy1SQ=
295+
github.com/filecoin-project/go-state-types v0.14.0-rc1/go.mod h1:cHpOPup9H1g2T29dKHAjC2sc7/Ef5ypjuW9A3I+e9yY=
299296
github.com/filecoin-project/go-statemachine v1.0.3 h1:N07o6alys+V1tNoSTi4WuuoeNC4erS/6jE74+NsgQuk=
300297
github.com/filecoin-project/go-statemachine v1.0.3/go.mod h1:jZdXXiHa61n4NmgWFG4w8tnqgvZVHYbJ3yW7+y8bF54=
301298
github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI=
@@ -1192,6 +1189,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
11921189
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
11931190
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
11941191
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
1192+
github.com/rvagg/go-skellam-pmf v0.0.0 h1:117SdhgSlbIWHtteOQHIm4TdxlmM3niE9mJhY6iJ37E=
1193+
github.com/rvagg/go-skellam-pmf v0.0.0/go.mod h1:8vgJzn0nYi+wCYggFMFtpn70AgqEO889eatpNoxM/kI=
11951194
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
11961195
github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA=
11971196
github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
@@ -1757,7 +1756,6 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
17571756
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
17581757
golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
17591758
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
1760-
golang.org/x/tools v0.0.0-20191022213345-0bbdf54effa2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
17611759
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
17621760
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
17631761
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

0 commit comments

Comments
 (0)