Merge branch 'master' into leo/refactor-index-cluster-height

Author: zhangchiqing

.github/workflows/ci.yml

@@ -321,6 +321,8 @@ jobs:
           username: ${{ vars.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
         if: ${{ (github.event_name == 'merge_group' || (github.event.pull_request && (github.event.pull_request.author_association == 'MEMBER' || github.event.pull_request.author_association == 'COLLABORATOR'))) }}
+        # this docker auth is exclusively for higher rate limits. continue unauthenticated if it fails
+        continue-on-error: true
 
 
       - name: Docker build

.golangci.yml

@@ -3,10 +3,22 @@ run:
 
 linters-settings:
   goimports:
+    # enforced by linter
     # put imports beginning with prefix after 3rd-party packages;
     # it's a comma-separated list of prefixes
     local-prefixes: github.com/onflow/flow-go/
 
+  # used to generate canonical import ordering
+  # not enforced by linter
+  gci:
+    sections:
+      - standard                           # 1. standard library
+      - default                            # 2. external packages
+      - prefix(github.com/onflow/)         # 3. org packages
+      - prefix(github.com/onflow/flow-go/) # 4. project packages
+    skip-generated: true
+    custom-order: true
+
   gosec:
     # To select a subset of rules to run.
     # Available rules: https://github.com/securego/gosec#available-rules

.mockery.yaml

@@ -0,0 +1,113 @@
+dir: "{{.InterfaceDir}}/mock"
+outpkg: "mock"
+filename: "{{.InterfaceName | snakecase}}.go"
+mockname: "{{.InterfaceName}}"
+
+all: True
+with-expecter: False
+include-auto-generated: False
+disable-func-mocks: True
+fail-on-missing: True
+
+# Suppress warnings
+issue-845-fix: True
+disable-version-string: True
+resolve-type-alias: False
+
+packages:
+  github.com/onflow/flow-go/access:
+  github.com/onflow/flow-go/access/validator:
+  github.com/onflow/flow-go/cmd/util/ledger/reporters:
+  github.com/onflow/flow-go/consensus/hotstuff:
+    config:
+      dir: "consensus/hotstuff/mocks"
+      outpkg: "mocks"
+  github.com/onflow/flow-go/engine/access/ingestion/tx_error_messages:
+  github.com/onflow/flow-go/engine/access/rest/common/models:
+  github.com/onflow/flow-go/engine/access/rest/websockets:
+  github.com/onflow/flow-go/engine/access/rest/websockets/data_providers:
+  github.com/onflow/flow-go/engine/access/rpc/backend/accounts/provider:
+  github.com/onflow/flow-go/engine/access/rpc/backend/events/provider:
+  github.com/onflow/flow-go/engine/access/rpc/backend/node_communicator:
+  github.com/onflow/flow-go/engine/access/rpc/backend/transactions/error_messages:
+  github.com/onflow/flow-go/engine/access/rpc/backend/transactions/provider:
+  github.com/onflow/flow-go/engine/access/rpc/backend/transactions/retrier:
+  github.com/onflow/flow-go/engine/access/rpc/connection:
+  github.com/onflow/flow-go/engine/access/state_stream:
+  github.com/onflow/flow-go/engine/access/subscription:
+  github.com/onflow/flow-go/engine/access/subscription/tracker:
+  github.com/onflow/flow-go/engine/access/wrapper:
+    config:
+      dir: "engine/access/mock"
+  github.com/onflow/flow-go/engine/collection:
+  github.com/onflow/flow-go/engine/collection/epochmgr:
+  github.com/onflow/flow-go/engine/collection/rpc:
+  github.com/onflow/flow-go/engine/common/follower:
+    interfaces:
+      complianceCore:
+        config:
+          exported: true
+          mockname:  "{{.InterfaceName | firstUpper}}"
+  github.com/onflow/flow-go/engine/common/follower/cache:
+  github.com/onflow/flow-go/engine/consensus:
+  github.com/onflow/flow-go/engine/consensus/approvals:
+  github.com/onflow/flow-go/engine/execution:
+  github.com/onflow/flow-go/engine/execution/computation:
+  github.com/onflow/flow-go/engine/execution/computation/computer:
+  github.com/onflow/flow-go/engine/execution/computation/query:
+  github.com/onflow/flow-go/engine/execution/ingestion/uploader:
+  github.com/onflow/flow-go/engine/execution/provider:
+  github.com/onflow/flow-go/engine/execution/state:
+  github.com/onflow/flow-go/engine/protocol:
+  github.com/onflow/flow-go/engine/verification/fetcher:
+  github.com/onflow/flow-go/fvm:
+  github.com/onflow/flow-go/fvm/environment:
+  github.com/onflow/flow-go/fvm/storage/snapshot:
+  github.com/onflow/flow-go/insecure:
+  github.com/onflow/flow-go/ledger:
+  github.com/onflow/flow-go/model/fingerprint:
+  github.com/onflow/flow-go/module:
+  github.com/onflow/flow-go/module/component:
+  github.com/onflow/flow-go/module/execution:
+  github.com/onflow/flow-go/module/executiondatasync/execution_data:
+  github.com/onflow/flow-go/module/executiondatasync/optimistic_sync:
+    config:
+      all: False
+    interfaces:
+      Core:
+  github.com/onflow/flow-go/module/executiondatasync/tracker:
+  github.com/onflow/flow-go/module/forest:
+  github.com/onflow/flow-go/module/mempool:
+  github.com/onflow/flow-go/module/mempool/consensus/mock_interfaces:
+    config:
+      dir: "module/mempool/consensus/mock"
+  github.com/onflow/flow-go/module/state_synchronization:
+  github.com/onflow/flow-go/module/state_synchronization/requester:
+  github.com/onflow/flow-go/network:
+  github.com/onflow/flow-go/network/alsp:
+  github.com/onflow/flow-go/network/p2p:
+  github.com/onflow/flow-go/state/cluster:
+  github.com/onflow/flow-go/state/protocol:
+  github.com/onflow/flow-go/state/protocol/events:
+  github.com/onflow/flow-go/state/protocol/protocol_state:
+  github.com/onflow/flow-go/state/protocol/protocol_state/mock_interfaces:
+    config:
+      dir: "state/protocol/protocol_state/mock"
+  github.com/onflow/flow-go/state/protocol/protocol_state/epochs:
+  github.com/onflow/flow-go/state/protocol/protocol_state/epochs/mock_interfaces:
+    config:
+      dir: "state/protocol/protocol_state/epochs/mock"
+  github.com/onflow/flow-go/storage:
+
+  # external libraries
+  github.com/onflow/flow-go-sdk/access:
+    config:
+      dir: "integration/benchmark/mock"
+    interfaces:
+      Client:
+
+  github.com/onflow/crypto:
+    config:
+      dir: "module/mock"
+    interfaces:
+      PublicKey:

AGENTS.md

@@ -51,13 +51,21 @@ This file provides guidance to AI Agents when working with code in this reposito
 
 ### Code Quality and Generation
 - `make lint` - Run linter (includes tidy and custom checks)
-- `make lint -e LINT_PATH=./path/to/lint/...` - Run linter for a specific module
+- `make fix-new` - Run linter for files changed since master
 - `make fix-lint` - Automatically fix linting issues
+- `make fix-lint-new` - Automatically fix linting issues for files changed since master
+- `make fix-imports` - Automatically fix imports
+- `make fix-imports-new` - Automatically fix imports for files changed since master
+- `make vet` - Run go vet
+- `make vet-new` - Run go vet for files changed since master
 - `make generate` - Run all code generators (proto, mocks, fvm wrappers)
 - `make generate-mocks` - Generate mocks for unit tests
 - `make generate-proto` - Generate protobuf stubs
 - `make tidy` - Run go mod tidy
 
+`lint`, `vet`, `fix-lint`, and `fix-imports` support passing `LINT_PATH`, which sets the path used by golangci-lint
+- `make lint -e LINT_PATH=./path/to/lint/...` - Run linter for a specific module
+
 ### Dependency Management
 - `make install-tools` - Install all required development tools
 - `make install-mock-generators` - Install mock generation tools

Makefile

@@ -77,8 +77,7 @@ unittest-main:
 .PHONY: install-mock-generators
 install-mock-generators:
 	cd ${GOPATH}; \
-    go install github.com/vektra/mockery/[email protected]; \
-    go install github.com/golang/mock/[email protected];
+    go install github.com/vektra/mockery/[email protected];
 
 .PHONY: install-tools
 install-tools: check-go-version install-mock-generators
@@ -154,83 +153,8 @@ generate-fvm-env-wrappers:
 
 .PHONY: generate-mocks
 generate-mocks: install-mock-generators
-	mockery --name '(Connector|PingInfoProvider)' --dir=network/p2p --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
-	CGO_CFLAGS=$(CRYPTO_FLAG) mockgen -destination=network/mocknetwork/mock_network.go -package=mocknetwork github.com/onflow/flow-go/network EngineRegistry
-	mockery --name=ExecutionDataStore --dir=module/executiondatasync/execution_data --case=underscore --output="./module/executiondatasync/execution_data/mock" --outpkg="mock"
-	mockery --name=Downloader --dir=module/executiondatasync/execution_data --case=underscore --output="./module/executiondatasync/execution_data/mock" --outpkg="mock"
-	mockery --name='.*' --dir=integration/benchmark/mocksiface --case=underscore --output="integration/benchmark/mock" --outpkg="mock"
-	mockery --name '(ExecutionDataRequester|IndexReporter)' --dir=module/state_synchronization --case=underscore --output="./module/state_synchronization/mock" --outpkg="state_synchronization"
-	mockery --name 'ExecutionState' --dir=engine/execution/state --case=underscore --output="engine/execution/state/mock" --outpkg="mock"
-	mockery --name 'BlockComputer' --dir=engine/execution/computation/computer --case=underscore --output="engine/execution/computation/computer/mock" --outpkg="mock"
-	mockery --name 'ComputationManager' --dir=engine/execution/computation --case=underscore --output="engine/execution/computation/mock" --outpkg="mock"
-	mockery --name '.*' --dir=engine/execution/computation/query --case=underscore --output="engine/execution/computation/query/mock" --outpkg="mock"
-	mockery --name 'EpochComponentsFactory' --dir=engine/collection/epochmgr --case=underscore --output="engine/collection/epochmgr/mock" --outpkg="mock"
-	mockery --name '.*' --dir=engine/execution/ --case=underscore --output="engine/execution/mock" --outpkg="mock"
-	mockery --name 'Backend' --dir=engine/collection/rpc --case=underscore --output="engine/collection/rpc/mock" --outpkg="mock"
-	mockery --name 'ProviderEngine' --dir=engine/execution/provider --case=underscore --output="engine/execution/provider/mock" --outpkg="mock"
-	mockery --name '.*' --dir=state/cluster --case=underscore --output="state/cluster/mock" --outpkg="mock"
-	mockery --name '.*' --dir=module --case=underscore --output="./module/mock" --outpkg="mock"
-	mockery --name '.*' --dir=module/mempool --case=underscore --output="./module/mempool/mock" --outpkg="mempool"
-	mockery --name '.*' --dir=module/component --case=underscore --output="./module/component/mock" --outpkg="component"
-	mockery --name '.*' --dir=network --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
-	mockery --name '.*' --dir=storage --case=underscore --output="./storage/mock" --outpkg="mock"
-	mockery --name '.*' --dir="state/protocol" --case=underscore --output="state/protocol/mock" --outpkg="mock"
-	mockery --name '.*' --dir="state/protocol/events" --case=underscore --output="./state/protocol/events/mock" --outpkg="mock"
-	mockery --name '.*' --dir="state/protocol/protocol_state" --case=underscore --output="state/protocol/protocol_state/mock" --outpkg="mock"
-	mockery --name '.*' --dir="state/protocol/protocol_state/epochs" --case=underscore --output="state/protocol/protocol_state/epochs/mock" --outpkg="mock"
-	mockery --name '.*' --dir=engine/execution/computation/computer --case=underscore --output="./engine/execution/computation/computer/mock" --outpkg="mock"
-	mockery --name '.*' --dir=engine/execution/state --case=underscore --output="./engine/execution/state/mock" --outpkg="mock"
-	mockery --name '.*' --dir=engine/collection --case=underscore --output="./engine/collection/mock" --outpkg="mock"
-	mockery --name 'complianceCore' --dir=engine/common/follower --exported --case=underscore --output="./engine/common/follower/mock" --outpkg="mock"
-	mockery --name '.*' --dir=engine/common/follower/cache --case=underscore --output="./engine/common/follower/cache/mock" --outpkg="mock"
-	mockery --name '.*' --dir=engine/consensus --case=underscore --output="./engine/consensus/mock" --outpkg="mock"
-	mockery --name '.*' --dir=engine/consensus/approvals --case=underscore --output="./engine/consensus/approvals/mock" --outpkg="mock"
-	rm -rf ./fvm/mock
-	mockery --name '.*' --dir=fvm --case=underscore --output="./fvm/mock" --outpkg="mock"
-	rm -rf ./fvm/environment/mock
-	mockery --name '.*' --dir=fvm/environment --case=underscore --output="./fvm/environment/mock" --outpkg="mock"
-	mockery --name '.*' --dir=ledger --case=underscore --output="./ledger/mock" --outpkg="mock"
-	mockery --name 'ViolationsConsumer' --dir=network --case=underscore --output="./network/mocknetwork" --outpkg="mocknetwork"
-	mockery --name '.*' --dir=network/p2p/ --case=underscore --output="./network/p2p/mock" --outpkg="mockp2p"
-	mockery --name '.*' --dir=network/alsp --case=underscore --output="./network/alsp/mock" --outpkg="mockalsp"
-	mockery --name 'Vertex' --dir="./module/forest" --case=underscore --output="./module/forest/mock" --outpkg="mock"
-	mockery --name '.*' --dir="./consensus/hotstuff" --case=underscore --output="./consensus/hotstuff/mocks" --outpkg="mocks"
-	mockery --name '.*' --dir="./engine/access/wrapper" --case=underscore --output="./engine/access/mock" --outpkg="mock"
-	mockery --name 'API' --dir="./access" --case=underscore --output="./access/mock" --outpkg="mock"
-	mockery --name 'Blocks' --dir="./access/validator" --case=underscore --output="./access/validator/mock" --outpkg="mock"
-	mockery --name 'API' --dir="./engine/protocol" --case=underscore --output="./engine/protocol/mock" --outpkg="mock"
-	mockery --name '.*' --dir="./engine/access/state_stream" --case=underscore --output="./engine/access/state_stream/mock" --outpkg="mock"
-	mockery --name 'BlockTracker' --dir="./engine/access/subscription/tracker" --case=underscore --output="./engine/access/subscription/tracker/mock"  --outpkg="mock"
-	mockery --name 'ExecutionDataTracker' --dir="./engine/access/subscription/tracker" --case=underscore --output="./engine/access/subscription/tracker/mock"  --outpkg="mock"
-	mockery --name 'DataProvider' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
-	mockery --name 'DataProviderFactory' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
-	mockery --name 'LinkGenerator' --dir="./engine/access/rest/common/models" --case=underscore --output="./engine/access/rest/common/models/mock"  --outpkg="mock"
-	mockery --name 'WebsocketConnection' --dir="./engine/access/rest/websockets" --case=underscore --output="./engine/access/rest/websockets/mock"  --outpkg="mock"
-	mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/connection" --case=underscore --output="./engine/access/rpc/connection/mock" --outpkg="mock"
-	mockery --name 'Communicator' --dir="./engine/access/rpc/backend/node_communicator" --case=underscore --output="./engine/access/rpc/backend/node_communicator/mock" --outpkg="mock"
-	mockery --name 'AccountProvider' --dir="./engine/access/rpc/backend/accounts/provider" --case=underscore --output="./engine/access/rpc/backend/accounts/provider/mock" --outpkg="mock"
-	mockery --name 'EventProvider' --dir="./engine/access/rpc/backend/events/provider" --case=underscore --output="./engine/access/rpc/backend/events/provider/mock" --outpkg="mock"
-	mockery --name 'TransactionProvider' --dir="./engine/access/rpc/backend/transactions/provider" --case=underscore --output="./engine/access/rpc/backend/transactions/provider/mock" --outpkg="mock"
-	mockery --name 'Provider' --dir="./engine/access/rpc/backend/transactions/error_messages" --case=underscore --output="./engine/access/rpc/backend/transactions/error_messages/mock" --outpkg="mock"
-	mockery --name 'TransactionSender' --dir="./engine/access/rpc/backend/transactions/retrier" --case=underscore --output="./engine/access/rpc/backend/transactions/retrier/mock" --outpkg="mock"
-	mockery --name 'Retrier' --dir="./engine/access/rpc/backend/transactions/retrier" --case=underscore --output="./engine/access/rpc/backend/transactions/retrier/mock" --outpkg="mock"
-	mockery --name '.*' --dir=model/fingerprint --case=underscore --output="./model/fingerprint/mock" --outpkg="mock"
-	mockery --name 'ExecForkActor' --structname 'ExecForkActorMock' --dir=module/mempool/consensus/mock/ --case=underscore --output="./module/mempool/consensus/mock/" --outpkg="mock"
-	mockery --name '.*' --dir=engine/verification/fetcher/ --case=underscore --output="./engine/verification/fetcher/mock" --outpkg="mockfetcher"
-	mockery --name '.*' --dir=./cmd/util/ledger/reporters --case=underscore --output="./cmd/util/ledger/reporters/mock" --outpkg="mock"
-	mockery --name 'Storage' --dir=module/executiondatasync/tracker --case=underscore --output="module/executiondatasync/tracker/mock" --outpkg="mocktracker"
-	mockery --name 'ScriptExecutor' --dir=module/execution --case=underscore --output="module/execution/mock" --outpkg="mock"
-	mockery --name 'StorageSnapshot' --dir=fvm/storage/snapshot --case=underscore --output="fvm/storage/snapshot/mock" --outpkg="mock"
-	mockery --name 'Core' --dir=module/executiondatasync/optimistic_sync --case=underscore --output="module/executiondatasync/optimistic_sync/mock" --outpkg="mock"
-	mockery --name 'Requester' --dir=engine/access/ingestion/tx_error_messages --case=underscore --output="engine/access/ingestion/tx_error_messages/mock" --outpkg="mock"
-	mockery --name 'ExecutionDataRequester' --dir=module/state_synchronization/requester --case=underscore --output="module/state_synchronization/requester/mock" --outpkg="mock"
-
-	#temporarily make insecure/ a non-module to allow mockery to create mocks
-	mv insecure/go.mod insecure/go2.mod
-	if [ -f go.work ]; then mv go.work go2.work; fi
-	mockery --name '.*' --dir=insecure/ --case=underscore --output="./insecure/mock"  --outpkg="mockinsecure"
-	mv insecure/go2.mod insecure/go.mod
-	if [ -f go2.work ]; then mv go2.work go.work; fi
+	mockery --config .mockery.yaml
+	cd insecure; mockery --config .mockery.yaml
 
 # this ensures there is no unused dependency being added by accident
 .PHONY: tidy
@@ -246,15 +170,39 @@ tools/custom-gcl: tools/structwrite .custom-gcl.yml
 	golangci-lint custom
 
 .PHONY: lint
-lint: tidy tools/custom-gcl
+lint: tools/custom-gcl
 	# revive -config revive.toml -exclude storage/ledger/trie ./...
 	./tools/custom-gcl run -v $(or $(LINT_PATH),./...)
 
+.PHONY: lint-new
+lint-new: tools/custom-gcl
+	./tools/custom-gcl run -v --new-from-rev=master
+
 .PHONY: fix-lint
-fix-lint:
+fix-lint: tools/custom-gcl
 	# revive -config revive.toml -exclude storage/ledger/trie ./...
 	./tools/custom-gcl run -v --fix $(or $(LINT_PATH),./...)
 
+.PHONY: fix-lint-new
+fix-lint-new: tools/custom-gcl
+	./tools/custom-gcl run -v --fix --new-from-rev=master
+
+.PHONY: fix-imports
+fix-imports: tools/custom-gcl
+	./tools/custom-gcl run --enable-only=gci --fix $(or $(LINT_PATH),./...)
+
+.PHONY: fix-imports-new
+fix-imports-new: tools/custom-gcl
+	./tools/custom-gcl run --enable-only=gci --fix --new-from-rev=master
+
+.PHONY: vet
+vet: tools/custom-gcl
+	./tools/custom-gcl run --enable-only=govet $(or $(LINT_PATH),./...)
+
+.PHONY: vet-new
+vet-new: tools/custom-gcl
+	./tools/custom-gcl run --enable-only=govet --new-from-rev=master
+
 # Runs unit tests with different list of packages as passed by CI so they run in parallel
 .PHONY: ci
 ci: install-tools test