Skip to content

Commit fe9513e

Browse files
committed
Merge branch 'yurii/6493-follower-core' of https://github.com/onflow/flow-go into yurii/6493-badger-cleaner
2 parents 062c54c + f681cd4 commit fe9513e

File tree

191 files changed

+6526
-2584
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

191 files changed

+6526
-2584
lines changed

.github/workflows/builds.yml

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# This workflow is used to build and push one-off images for specific node types. This is useful
2+
# when deploying hotfixes or any time a change is not needed for all node roles.
3+
name: Build Node Docker Images
4+
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
tag:
9+
type: string
10+
description: 'Git tag/commit'
11+
required: true
12+
docker_tag:
13+
type: string
14+
description: 'Docker tag'
15+
required: true
16+
# GHA doesn't support multi-selects, so simulating it with one boolean for each option
17+
build_access:
18+
type: boolean
19+
description: 'Access'
20+
required: false
21+
build_collection:
22+
type: boolean
23+
description: 'Collection'
24+
required: false
25+
build_consensus:
26+
type: boolean
27+
description: 'Consensus'
28+
required: false
29+
build_execution:
30+
type: boolean
31+
description: 'Execution'
32+
required: false
33+
build_verification:
34+
type: boolean
35+
description: 'Verification'
36+
required: false
37+
build_observer:
38+
type: boolean
39+
description: 'Observer'
40+
required: false
41+
include_without_netgo:
42+
type: boolean
43+
description: 'Build `without_netgo` images'
44+
required: false
45+
46+
jobs:
47+
# matrix_builder generates a matrix that includes the roles selected in the input
48+
matrix_builder:
49+
name: Setup build jobs
50+
runs-on: ubuntu-latest
51+
outputs:
52+
matrix: ${{ steps.generate.outputs.matrix }}
53+
steps:
54+
- id: generate
55+
run: |
56+
roles=()
57+
if [[ "${{ inputs.build_access }}" = "true" ]]; then
58+
roles+=( "access" )
59+
fi
60+
if [[ "${{ inputs.build_collection }}" = "true" ]]; then
61+
roles+=( "collection" )
62+
fi
63+
if [[ "${{ inputs.build_consensus }}" = "true" ]]; then
64+
roles+=( "consensus" )
65+
fi
66+
if [[ "${{ inputs.build_execution }}" = "true" ]]; then
67+
roles+=( "execution" )
68+
fi
69+
if [[ "${{ inputs.build_verification }}" = "true" ]]; then
70+
roles+=( "verification" )
71+
fi
72+
if [[ "${{ inputs.build_observer }}" = "true" ]]; then
73+
roles+=( "observer" )
74+
fi
75+
rolesJSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${roles[@]}")
76+
echo "matrix={\"role\":$(echo $rolesJSON)}" >> $GITHUB_OUTPUT
77+
docker-push:
78+
name: ${{ matrix.role }} images
79+
runs-on: ubuntu-latest
80+
needs: matrix_builder
81+
82+
# setup jobs for each role
83+
strategy:
84+
fail-fast: false
85+
matrix: ${{ fromJson(needs.matrix_builder.outputs.matrix) }}
86+
87+
steps:
88+
- name: Setup Go
89+
uses: actions/setup-go@v2
90+
with:
91+
go-version: '1.19'
92+
- name: Checkout repo
93+
uses: actions/checkout@v2
94+
with:
95+
ref: ${{ inputs.tag }}
96+
# Provide Google Service Account credentials to Github Action, allowing interaction with the Google Container Registry
97+
# Logging in as [email protected]
98+
- name: Docker login
99+
uses: docker/login-action@v1
100+
with:
101+
registry: gcr.io
102+
username: _json_key
103+
password: ${{ secrets.GCR_SERVICE_KEY }}
104+
105+
- name: Build/Push ${{ matrix.role }} images
106+
env:
107+
IMAGE_TAG: ${{ inputs.docker_tag }}
108+
run: |
109+
make docker-build-${{ matrix.role }} docker-push-${{ matrix.role }}
110+
111+
- name: Build/Push ${{ matrix.role }} without_netgo images
112+
if: ${{ inputs.include_without_netgo }}
113+
env:
114+
IMAGE_TAG: ${{ inputs.docker_tag }}
115+
run: |
116+
make docker-build-${{ matrix.role }}-without-netgo docker-push-${{ matrix.role }}-without-netgo

.github/workflows/ci.yml

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,14 @@ jobs:
115115
with:
116116
go-version: ${{ env.GO_VERSION }}
117117
cache: true
118+
- name: Setup tests (${{ matrix.targets.name }}
119+
run: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools
118120
- name: Run tests (${{ matrix.targets.name }})
119121
uses: nick-fields/retry@v2
120122
with:
121123
timeout_minutes: 25
122124
max_attempts: 3
123-
command: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" ci
125+
command: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test
124126

125127
# TODO(rbtz): re-enable when we fix exisiting races.
126128
#env:
@@ -162,17 +164,17 @@ jobs:
162164
with:
163165
go-version: ${{ env.GO_VERSION }}
164166
cache: true
167+
- name: Setup tests (${{ matrix.name }})
168+
run: make ${{ matrix.make1 }}
165169
- name: Run tests (${{ matrix.name }})
166170
env:
167171
RACE_DETECTOR: ${{ matrix.race }}
168172
uses: nick-fields/retry@v2
169173
with:
170174
timeout_minutes: 25
171175
max_attempts: ${{ matrix.retries }}
172-
# run `make1` target before running `make2` target inside each module's root
173-
command: |
174-
make ${{ matrix.make1 }}
175-
VERBOSE=1 make -C ${{ matrix.name }} ${{ matrix.make2 }}
176+
# run `make2` target inside each module's root
177+
command: VERBOSE=1 make -C ${{ matrix.name }} ${{ matrix.make2 }}
176178
- name: Upload coverage report
177179
uses: codecov/codecov-action@v3
178180
with:

.github/workflows/test-monitor-flaky.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ jobs:
7272
TEST_FLAKY: true
7373
JSON_OUTPUT: true
7474
RACE_DETECTOR: 1
75+
- name: Print test results
76+
run: cat test-output
7577
- name: Process test results
7678
run: cat test-output | go run tools/test_monitor/level1/process_summary1_results.go
7779
env:

.github/workflows/test-monitor-regular-skipped.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ jobs:
7676
command: ./tools/test_monitor/run-tests.sh
7777
env:
7878
JSON_OUTPUT: true
79+
- name: Print test results
80+
run: cat test-output
7981
- name: Process test results
8082
run: cat test-output | go run tools/test_monitor/level1/process_summary1_results.go
8183
env:

.github/workflows/tools.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
1+
# This workflow is used to build and upload the node bootstrapping tools
12
name: Build Tools
23

34
on:
45
workflow_dispatch:
56
inputs:
67
tag:
7-
description: 'Tagged commit to build tools against'
8+
description: 'Tag/commit'
89
required: true
910
type: string
1011
promote:
11-
description: 'Should this build be promoted to the official boot-tools?'
12+
description: 'Promote to official boot-tools?'
1213
required: false
1314
type: boolean
1415

CODEOWNERS

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@
88
/engine/consensus/** @AlexHentschel @durkmurder @jordanschalm
99

1010
# Execution Stream
11-
/cmd/execution/** @m4ksio @ramtinms
12-
/engine/execution/** @m4ksio @ramtinms
11+
/cmd/execution/** @ramtinms
12+
/engine/execution/** @ramtinms
1313

1414
# Access Stream
15-
/cmd/access/** @vishalchangrani
16-
/engine/access/** @vishalchangrani
15+
/access/** @peterargue
16+
/cmd/access/** @peterargue
17+
/cmd/observer/** @peterargue
18+
/engine/access/** @peterargue
1719

1820
# Verification Stream
1921
/cmd/verification/** @ramtinms @yhassanzadeh13
@@ -22,19 +24,19 @@
2224
/integration/tests/verification @ramtinms @yhassanzadeh13
2325

2426
# Ledger Stream
25-
/ledger/** @ramtinms @m4ksio @AlexHentschel
27+
/ledger/** @ramtinms @AlexHentschel
2628

2729
# FVM Stream
2830
/fvm/** @ramtinms @janezpodhostnik @pattyshack
2931

3032
# Networking Stream
31-
/network/** @yhassanzadeh13 @vishalchangrani
33+
/network/** @yhassanzadeh13
3234

3335
# Cryptography Stream
3436
/crypto/** @tarakby
3537

3638
# Bootstrap and transit scripts
37-
/cmd/bootstrap/** @vishalchangrani
39+
/cmd/bootstrap/** @zhangchiqing
3840

3941
# Dev Tools Stream
4042
.github/workflows/** @gomisha
@@ -48,3 +50,7 @@
4850
/module/profiler/** @SaveTheRbtz @pattyshack
4951
/module/trace/** @SaveTheRbtz @pattyshack
5052
/module/tracer.go @SaveTheRbtz @pattyshack
53+
54+
# Execution Sync
55+
/module/executiondatasync/** @peterargue
56+
/module/state_synchronization/** @peterargue

cmd/access/node_builder/access_node_builder.go

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ import (
5252
"github.com/onflow/flow-go/module/executiondatasync/execution_data"
5353
finalizer "github.com/onflow/flow-go/module/finalizer/consensus"
5454
"github.com/onflow/flow-go/module/id"
55-
"github.com/onflow/flow-go/module/mempool/queue"
5655
"github.com/onflow/flow-go/module/mempool/stdmap"
5756
"github.com/onflow/flow-go/module/metrics"
5857
"github.com/onflow/flow-go/module/metrics/unstaked"
@@ -67,7 +66,6 @@ import (
6766
"github.com/onflow/flow-go/network/p2p/cache"
6867
"github.com/onflow/flow-go/network/p2p/connection"
6968
"github.com/onflow/flow-go/network/p2p/dht"
70-
"github.com/onflow/flow-go/network/p2p/distributor"
7169
"github.com/onflow/flow-go/network/p2p/middleware"
7270
"github.com/onflow/flow-go/network/p2p/p2pbuilder"
7371
"github.com/onflow/flow-go/network/p2p/subscription"
@@ -335,7 +333,7 @@ func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuild
335333
builder.Validator,
336334
builder.SyncCore,
337335
node.Tracer,
338-
followereng.WithComplianceOptions(modulecompliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold)),
336+
modulecompliance.WithSkipNewProposalsThreshold(node.ComplianceConfig.SkipNewProposalsThreshold),
339337
)
340338
if err != nil {
341339
return nil, fmt.Errorf("could not create follower core: %w", err)
@@ -708,12 +706,7 @@ func (builder *FlowAccessNodeBuilder) InitIDProviders() {
708706
}
709707
builder.IDTranslator = translator.NewHierarchicalIDTranslator(idCache, translator.NewPublicNetworkIDTranslator())
710708

711-
heroStoreOpts := []queue.HeroStoreConfigOption{queue.WithHeroStoreSizeLimit(builder.DisallowListNotificationCacheSize)}
712-
if builder.HeroCacheMetricsEnable {
713-
collector := metrics.DisallowListNotificationQueueMetricFactory(builder.MetricsRegisterer)
714-
heroStoreOpts = append(heroStoreOpts, queue.WithHeroStoreCollector(collector))
715-
}
716-
builder.NodeDisallowListDistributor = distributor.DefaultDisallowListNotificationDistributor(builder.Logger, heroStoreOpts...)
709+
builder.NodeDisallowListDistributor = cmd.BuildDisallowListNotificationDisseminator(builder.DisallowListNotificationCacheSize, builder.MetricsRegisterer, builder.Logger, builder.MetricsEnabled)
717710

718711
// The following wrapper allows to disallow-list byzantine nodes via an admin command:
719712
// the wrapper overrides the 'Ejected' flag of disallow-listed nodes to true

cmd/bootstrap/cmd/key.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/spf13/cobra"
1414

1515
"github.com/onflow/flow-go/crypto"
16+
1617
model "github.com/onflow/flow-go/model/bootstrap"
1718
"github.com/onflow/flow-go/model/flow"
1819
)
@@ -47,31 +48,31 @@ func init() {
4748
&flagNetworkSeed,
4849
"networking-seed",
4950
[]byte{},
50-
fmt.Sprintf("hex encoded networking seed (min %d bytes)", crypto.KeyGenSeedMinLenECDSAP256))
51+
fmt.Sprintf("hex encoded networking seed (min %d bytes)", crypto.KeyGenSeedMinLen))
5152
keyCmd.Flags().BytesHexVar(
5253
&flagStakingSeed,
5354
"staking-seed",
5455
[]byte{},
55-
fmt.Sprintf("hex encoded staking seed (min %d bytes)", crypto.KeyGenSeedMinLenBLSBLS12381))
56+
fmt.Sprintf("hex encoded staking seed (min %d bytes)", crypto.KeyGenSeedMinLen))
5657
keyCmd.Flags().BytesHexVar(
5758
&flagMachineSeed,
5859
"machine-seed",
5960
[]byte{},
60-
fmt.Sprintf("hex encoded machine account seed (min %d bytes)", crypto.KeyGenSeedMinLenECDSAP256))
61+
fmt.Sprintf("hex encoded machine account seed (min %d bytes)", crypto.KeyGenSeedMinLen))
6162
}
6263

6364
// keyCmdRun generate the node staking key, networking key and node information
6465
func keyCmdRun(_ *cobra.Command, _ []string) {
6566

6667
// generate private key seeds if not specified via flag
6768
if len(flagNetworkSeed) == 0 {
68-
flagNetworkSeed = GenerateRandomSeed(crypto.KeyGenSeedMinLenECDSAP256)
69+
flagNetworkSeed = GenerateRandomSeed(crypto.KeyGenSeedMinLen)
6970
}
7071
if len(flagStakingSeed) == 0 {
71-
flagStakingSeed = GenerateRandomSeed(crypto.KeyGenSeedMinLenBLSBLS12381)
72+
flagStakingSeed = GenerateRandomSeed(crypto.KeyGenSeedMinLen)
7273
}
7374
if len(flagMachineSeed) == 0 {
74-
flagMachineSeed = GenerateRandomSeed(crypto.KeyGenSeedMinLenECDSAP256)
75+
flagMachineSeed = GenerateRandomSeed(crypto.KeyGenSeedMinLen)
7576
}
7677

7778
// validate inputs

cmd/bootstrap/cmd/keys.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ package cmd
33
import (
44
"fmt"
55

6-
"github.com/onflow/flow-go/cmd/bootstrap/utils"
76
"github.com/onflow/flow-go/crypto/hash"
7+
8+
"github.com/onflow/flow-go/cmd/bootstrap/utils"
89
"github.com/onflow/flow-go/model/flow/order"
910

1011
"github.com/onflow/flow-go/crypto"
12+
1113
model "github.com/onflow/flow-go/model/bootstrap"
1214
"github.com/onflow/flow-go/model/encodable"
1315
"github.com/onflow/flow-go/model/flow"
@@ -27,14 +29,14 @@ func genNetworkAndStakingKeys() []model.NodeInfo {
2729
log.Debug().Msg("all node addresses are unique")
2830

2931
log.Debug().Msgf("will generate %v networking keys for nodes in config", nodes)
30-
networkKeys, err := utils.GenerateNetworkingKeys(nodes, GenerateRandomSeeds(nodes, crypto.KeyGenSeedMinLenECDSAP256))
32+
networkKeys, err := utils.GenerateNetworkingKeys(nodes, GenerateRandomSeeds(nodes, crypto.KeyGenSeedMinLen))
3133
if err != nil {
3234
log.Fatal().Err(err).Msg("cannot generate networking keys")
3335
}
3436
log.Info().Msgf("generated %v networking keys for nodes in config", nodes)
3537

3638
log.Debug().Msgf("will generate %v staking keys for nodes in config", nodes)
37-
stakingKeys, err := utils.GenerateStakingKeys(nodes, GenerateRandomSeeds(nodes, crypto.KeyGenSeedMinLenBLSBLS12381))
39+
stakingKeys, err := utils.GenerateStakingKeys(nodes, GenerateRandomSeeds(nodes, crypto.KeyGenSeedMinLen))
3840
if err != nil {
3941
log.Fatal().Err(err).Msg("cannot generate staking keys")
4042
}

cmd/bootstrap/cmd/machine_account_key.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@ import (
44
"fmt"
55
"path"
66

7-
"github.com/onflow/flow-go/cmd/bootstrap/utils"
87
"github.com/onflow/flow-go/crypto"
98

9+
"github.com/onflow/flow-go/cmd/bootstrap/utils"
10+
1011
"github.com/spf13/cobra"
1112

1213
model "github.com/onflow/flow-go/model/bootstrap"
@@ -24,7 +25,7 @@ var machineAccountKeyCmd = &cobra.Command{
2425
func init() {
2526
rootCmd.AddCommand(machineAccountKeyCmd)
2627

27-
machineAccountKeyCmd.Flags().BytesHexVar(&flagMachineSeed, "seed", GenerateRandomSeed(crypto.KeyGenSeedMinLenECDSAP256), fmt.Sprintf("hex encoded machine account seed (min %d bytes)", crypto.KeyGenSeedMinLenECDSAP256))
28+
machineAccountKeyCmd.Flags().BytesHexVar(&flagMachineSeed, "seed", GenerateRandomSeed(crypto.KeyGenSeedMinLen), fmt.Sprintf("hex encoded machine account seed (min %d bytes)", crypto.KeyGenSeedMinLen))
2829
}
2930

3031
// machineAccountKeyRun generate a machine account key and writes it to a default file path.

0 commit comments

Comments
 (0)