Merge branch 'master' into leo/db-ops

Author: zhangchiqing

.github/workflows/builds.yml

@@ -43,6 +43,10 @@ on:
         type: boolean
         description: 'Build amd64 `without_adx` and `without_netgo_without_adx` images, and arm64 images'
         required: false
+      private_build:
+        type: boolean
+        description: 'Build private images'
+        required: false
 
 jobs:
   # matrix_builder generates a matrix that includes the roles selected in the input
@@ -109,16 +113,30 @@ jobs:
         credentials_json: ${{ secrets.GCR_SERVICE_KEY_SECRET }}
     - name: Set up Google Cloud SDK
       uses: google-github-actions/setup-gcloud@v1
-    - name: Authenticate docker with gcloud
+
+    - name: Authenticate Docker with gcloud
       run: |
-        gcloud auth configure-docker
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          gcloud auth configure-docker us-central1-docker.pkg.dev
+        else
+          gcloud auth configure-docker
+        fi
+
+    - name: Set CONTAINER_REGISTRY
+      id: set-registry
+      run: |
+        if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then
+          echo "CONTAINER_REGISTRY=${{ vars.PRIVATE_REGISTRY }}" >> $GITHUB_ENV
+        else
+          echo "CONTAINER_REGISTRY=${{ vars.PUBLIC_REGISTRY }}" >> $GITHUB_ENV
+        fi
 
     - name: Build/Push ${{ matrix.role }} amd64 images with adx (default)
       env:
         IMAGE_TAG: ${{ inputs.docker_tag }}
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx
+        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx CONTAINER_REGISTRY=$CONTAINER_REGISTRY
 
     - name: Build/Push ${{ matrix.role }} amd64 images without netgo and without adx, arm64 images
       if: ${{ inputs.include_alternative_builds }}
@@ -128,7 +146,7 @@ jobs:
       run: |
         make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx \
           docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx \
-          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm
+          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm CONTAINER_REGISTRY=$CONTAINER_REGISTRY
 
 
         

.github/workflows/ci.yml

@@ -55,13 +55,19 @@ jobs:
         # https://github.com/golangci/golangci-lint-action/issues/244
         skip-cache: true
 
-
   tidy:
     name: Tidy
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+
+      - name: Setup private build environment
+        if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+        uses: ./actions/private-setup
+        with:
+          cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
       - name: Setup Go
         uses: actions/setup-go@v4
         timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -139,6 +145,13 @@ jobs:
     steps:
     - name: Checkout repo
       uses: actions/checkout@v3
+
+    - name: Setup private build environment
+      if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+      uses: ./actions/private-setup
+      with:
+        cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
     - name: Setup Go
       uses: actions/setup-go@v4
       timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -178,6 +191,13 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+
+      - name: Setup private build environment
+        if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+        uses: ./actions/private-setup
+        with:
+          cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
       - name: Setup Go
         uses: actions/setup-go@v4
         timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -216,6 +236,13 @@ jobs:
         with:
           # all tags are needed for integration tests
           fetch-depth: 0
+
+      - name: Setup private build environment
+        if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+        uses: ./actions/private-setup
+        with:
+          cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
       - name: Setup Go
         uses: actions/setup-go@v4
         timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -255,9 +282,17 @@ jobs:
         targets: ${{ fromJSON(needs.create-integration-dynamic-test-matrix.outputs.dynamic-matrix)}}
     ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
     runs-on: ${{ matrix.targets.runner }}
+
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+      
+      - name: Setup private build environment
+        if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+        uses: ./actions/private-setup
+        with:
+          cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
       - name: Setup Go
         uses: actions/setup-go@v4
         timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
@@ -348,6 +383,13 @@ jobs:
       with:
           # all tags are needed for integration tests
           fetch-depth: 0
+
+    - name: Setup private build environment
+      if: ${{ vars.PRIVATE_BUILDS_SUPPORTED == 'true' }} 
+      uses: ./actions/private-setup
+      with:
+        cadence_deploy_key: ${{ secrets.CADENCE_DEPLOY_KEY }}
+
     - name: Setup Go
       uses: actions/setup-go@v4
       timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time

Makefile

@@ -26,7 +26,7 @@ IMAGE_TAG_ARM := $(IMAGE_TAG)-arm
 # Name of the cover profile
 COVER_PROFILE := coverage.txt
 # Disable go sum database lookup for private repos
-GOPRIVATE=github.com/dapperlabs/*
+GOPRIVATE=github.com/onflow/*-internal
 # OS
 UNAME := $(shell uname)
 
@@ -138,8 +138,8 @@ endif
 
 .PHONY: generate-openapi
 generate-openapi:
-	swagger-codegen generate -l go -i https://raw.githubusercontent.com/onflow/flow/master/openapi/access.yaml -D packageName=models,modelDocs=false,models -o engine/access/rest/models;
-	go fmt ./engine/access/rest/models
+	swagger-codegen generate -l go -i https://raw.githubusercontent.com/onflow/flow/master/openapi/access.yaml -D packageName=models,modelDocs=false,models -o engine/access/rest/http/models;
+	go fmt ./engine/access/rest/http/models
 
 .PHONY: generate
 generate: generate-proto generate-mocks generate-fvm-env-wrappers
@@ -203,10 +203,11 @@ generate-mocks: install-mock-generators
 	mockery --name 'API' --dir="./engine/protocol" --case=underscore --output="./engine/protocol/mock" --outpkg="mock"
 	mockery --name '.*' --dir="./engine/access/state_stream" --case=underscore --output="./engine/access/state_stream/mock" --outpkg="mock"
 	mockery --name 'BlockTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock"  --outpkg="mock"
+	mockery --name 'DataProvider' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
+	mockery --name 'DataProviderFactory' --dir="./engine/access/rest/websockets/data_providers" --case=underscore --output="./engine/access/rest/websockets/data_providers/mock"  --outpkg="mock"
 	mockery --name 'ExecutionDataTracker' --dir="./engine/access/subscription" --case=underscore --output="./engine/access/subscription/mock"  --outpkg="mock"
 	mockery --name 'ConnectionFactory' --dir="./engine/access/rpc/connection" --case=underscore --output="./engine/access/rpc/connection/mock" --outpkg="mock"
 	mockery --name 'Communicator' --dir="./engine/access/rpc/backend" --case=underscore --output="./engine/access/rpc/backend/mock" --outpkg="mock"
-
 	mockery --name '.*' --dir=model/fingerprint --case=underscore --output="./model/fingerprint/mock" --outpkg="mock"
 	mockery --name 'ExecForkActor' --structname 'ExecForkActorMock' --dir=module/mempool/consensus/mock/ --case=underscore --output="./module/mempool/consensus/mock/" --outpkg="mock"
 	mockery --name '.*' --dir=engine/verification/fetcher/ --case=underscore --output="./engine/verification/fetcher/mock" --outpkg="mockfetcher"
@@ -857,7 +858,8 @@ docker-all-tools: tool-util tool-remove-execution-fork
 
 PHONY: docker-build-util
 docker-build-util:
-	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \
+	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/util --build-arg GOARCH=$(GOARCH) --build-arg VERSION=$(IMAGE_TAG) --build-arg CGO_FLAG=$(DISABLE_ADX) --target production \
+		--secret id=cadence_deploy_key,env=CADENCE_DEPLOY_KEY --build-arg GOPRIVATE=$(GOPRIVATE) \
 		-t "$(CONTAINER_REGISTRY)/util:latest"  \
 		-t "$(CONTAINER_REGISTRY)/util:$(IMAGE_TAG)" .
 

access/handler.go

@@ -1066,7 +1066,7 @@ func (h *Handler) SubscribeBlocksFromStartBlockID(request *access.SubscribeBlock
 	}
 
 	sub := h.api.SubscribeBlocksFromStartBlockID(stream.Context(), startBlockID, blockStatus)
-	return subscription.HandleSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
+	return subscription.HandleRPCSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
 }
 
 // SubscribeBlocksFromStartHeight handles subscription requests for blocks started from block height.
@@ -1093,7 +1093,7 @@ func (h *Handler) SubscribeBlocksFromStartHeight(request *access.SubscribeBlocks
 	}
 
 	sub := h.api.SubscribeBlocksFromStartHeight(stream.Context(), request.GetStartBlockHeight(), blockStatus)
-	return subscription.HandleSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
+	return subscription.HandleRPCSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
 }
 
 // SubscribeBlocksFromLatest handles subscription requests for blocks started from latest sealed block.
@@ -1120,7 +1120,7 @@ func (h *Handler) SubscribeBlocksFromLatest(request *access.SubscribeBlocksFromL
 	}
 
 	sub := h.api.SubscribeBlocksFromLatest(stream.Context(), blockStatus)
-	return subscription.HandleSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
+	return subscription.HandleRPCSubscription(sub, h.handleBlocksResponse(stream.Send, request.GetFullBlockResponse(), blockStatus))
 }
 
 // handleBlocksResponse handles the subscription to block updates and sends
@@ -1179,7 +1179,7 @@ func (h *Handler) SubscribeBlockHeadersFromStartBlockID(request *access.Subscrib
 	}
 
 	sub := h.api.SubscribeBlockHeadersFromStartBlockID(stream.Context(), startBlockID, blockStatus)
-	return subscription.HandleSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
+	return subscription.HandleRPCSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
 }
 
 // SubscribeBlockHeadersFromStartHeight handles subscription requests for block headers started from block height.
@@ -1206,7 +1206,7 @@ func (h *Handler) SubscribeBlockHeadersFromStartHeight(request *access.Subscribe
 	}
 
 	sub := h.api.SubscribeBlockHeadersFromStartHeight(stream.Context(), request.GetStartBlockHeight(), blockStatus)
-	return subscription.HandleSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
+	return subscription.HandleRPCSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
 }
 
 // SubscribeBlockHeadersFromLatest handles subscription requests for block headers started from latest sealed block.
@@ -1233,7 +1233,7 @@ func (h *Handler) SubscribeBlockHeadersFromLatest(request *access.SubscribeBlock
 	}
 
 	sub := h.api.SubscribeBlockHeadersFromLatest(stream.Context(), blockStatus)
-	return subscription.HandleSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
+	return subscription.HandleRPCSubscription(sub, h.handleBlockHeadersResponse(stream.Send))
 }
 
 // handleBlockHeadersResponse handles the subscription to block updates and sends
@@ -1293,7 +1293,7 @@ func (h *Handler) SubscribeBlockDigestsFromStartBlockID(request *access.Subscrib
 	}
 
 	sub := h.api.SubscribeBlockDigestsFromStartBlockID(stream.Context(), startBlockID, blockStatus)
-	return subscription.HandleSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
+	return subscription.HandleRPCSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
 }
 
 // SubscribeBlockDigestsFromStartHeight handles subscription requests for lightweight blocks started from block height.
@@ -1320,7 +1320,7 @@ func (h *Handler) SubscribeBlockDigestsFromStartHeight(request *access.Subscribe
 	}
 
 	sub := h.api.SubscribeBlockDigestsFromStartHeight(stream.Context(), request.GetStartBlockHeight(), blockStatus)
-	return subscription.HandleSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
+	return subscription.HandleRPCSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
 }
 
 // SubscribeBlockDigestsFromLatest handles subscription requests for lightweight block started from latest sealed block.
@@ -1347,7 +1347,7 @@ func (h *Handler) SubscribeBlockDigestsFromLatest(request *access.SubscribeBlock
 	}
 
 	sub := h.api.SubscribeBlockDigestsFromLatest(stream.Context(), blockStatus)
-	return subscription.HandleSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
+	return subscription.HandleRPCSubscription(sub, h.handleBlockDigestsResponse(stream.Send))
 }
 
 // handleBlockDigestsResponse handles the subscription to block updates and sends
@@ -1433,17 +1433,21 @@ func (h *Handler) SendAndSubscribeTransactionStatuses(
 	sub := h.api.SubscribeTransactionStatuses(ctx, &tx, request.GetEventEncodingVersion())
 
 	messageIndex := counters.NewMonotonousCounter(0)
-	return subscription.HandleSubscription(sub, func(txResults []*TransactionResult) error {
+	return subscription.HandleRPCSubscription(sub, func(txResults []*TransactionResult) error {
 		for i := range txResults {
-			value := messageIndex.Increment()
+			index := messageIndex.Value()
+			if ok := messageIndex.Set(index + 1); !ok {
+				return status.Errorf(codes.Internal, "message index already incremented to %d", messageIndex.Value())
+			}
 
 			err = stream.Send(&access.SendAndSubscribeTransactionStatusesResponse{
 				TransactionResults: TransactionResultToMessage(txResults[i]),
-				MessageIndex:       value,
+				MessageIndex:       index,
 			})
 			if err != nil {
 				return rpc.ConvertError(err, "could not send response", codes.Internal)
 			}
+
 		}
 
 		return nil

actions/private-setup/action.yml

@@ -0,0 +1,29 @@
+name: "Private Build Setup"
+description: "Checks and configures the environment for building private dependencies"
+inputs:
+  cadence_deploy_key:
+    description: "Deploy Key for Private Cadence Repo"
+    required: true
+  go_private_value:
+    description: "The value for GOPRIVATE"
+    required: false
+    default: "github.com/onflow/*-internal"
+runs:
+  using: "composite"
+  steps:
+    - name: Load deploy key
+      uses: webfactory/[email protected]
+      with:
+        ssh-private-key: "${{ inputs.cadence_deploy_key }}"
+        known-hosts: "github.com"
+
+    - name: Configure git for SSH
+      shell: bash
+      run: |
+        git config --global url."[email protected]:".insteadOf "https://github.com/"
+
+    - name: Configure GOPRIVATE env  
+      shell: bash
+      run: |
+        echo "GOPRIVATE=${{ inputs.go_private_value }}" >> $GITHUB_ENV
+

admin/README.md

@@ -115,3 +115,8 @@ curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"
 curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "protocol-snapshot"}'
 curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "protocol-snapshot", "data": { "blocks-to-skip": 10 }}'
 ```
+
+### To backfill transaction error messages
+```
+curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "backfill-tx-error-messages", "data": { "start-height": 340, "end-height": 343, "execution-node-ids":["ec7b934df29248d574ae1cc33ae77f22f0fcf96a79e009224c46374d1837824e", "8cbdc8d24a28899a33140cb68d4146cd6f2f6c18c57f54c299f26351d126919e"] }}'
+```