Merge pull request #6541 from Algo-devops-service/relbeta4.5.1

Author: algojohnlee

.codecov.yml

@@ -1,10 +1,15 @@
 codecov:
   require_ci_to_pass: yes
 
+flags:
+  full_coverage:
+    joined: false
+
 ignore:
   - "**/*_gen.go"
   - "**/*_gen_test.go"
   - "**/generated"
+  - "**/*_string.go"  # ignore stringer-generated code
 
 coverage:
   status:

.github/workflows/ci-nightly.yml

@@ -103,13 +103,15 @@ jobs:
         run: |
           ./scripts/configure_dev.sh
           PACKAGES="$(go list ./... | grep -v /go-algorand/test/)"
-          export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')          
+          export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')
+          COVERPACKAGES="$(go list ./... | grep -E -v '/go-algorand/(test|debug|cmd|config/defaultsGenerator|tools)' | grep -E -v '(test|testing|mock|mocks)$' | paste -sd ',' -)"
           mkdir -p test_results/${{ matrix.platform }}_test_nightly/${PARTITION_ID}
           go tool -modfile=tool.mod gotestsum --format standard-quiet \
             --junitfile ~/test_results/${{ matrix.platform }}_test_nightly/${PARTITION_ID}/results.xml \
             --jsonfile ~/test_results/${{ matrix.platform }}_test_nightly/${PARTITION_ID}/testresults.json \
             -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" \
             -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 1 \
+            -coverpkg=$COVERPACKAGES \
             $PACKAGE_NAMES
       - name: Notify Slack on failure
         if: failure()
@@ -142,6 +144,7 @@ jobs:
         with:
           token: ${{ env.CODECOV_TOKEN }}
           file: ./coverage.txt
+          flags: full_coverage
           fail_ci_if_error: false
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}

.github/workflows/ci-pr.yml

@@ -4,6 +4,13 @@ on:
     branches:
       - master
       - 'rel/**'
+  workflow_dispatch:
+    inputs:
+      full_coverage:
+        description: 'Run with -coverpkg to measure cross-package coverage (slower)'
+        required: false
+        type: boolean
+        default: false
 
 env:
   CODECOV_TOKEN: "8b4a1f91-f154-4c26-b84c-c9aaa90159c6"  # Same public token from CircleCI config
@@ -49,13 +56,19 @@ jobs:
       - name: Run tests
         run: |
           PACKAGES="$(go list ./... | grep -v /go-algorand/test/)"
-          export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')          
+          export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')
+          COVERPKG_FLAG=""
+          if [[ "${{ inputs.full_coverage }}" == "true" ]]; then
+            COVERPACKAGES="$(go list ./... | grep -E -v '/go-algorand/(test|debug|cmd|config/defaultsGenerator|tools)' | grep -E -v '(test|testing|mock|mocks)$' | paste -sd ',' -)"
+            COVERPKG_FLAG="-coverpkg=$COVERPACKAGES"
+          fi
           mkdir -p test_results/${{ matrix.platform }}_test/${PARTITION_ID}
           go tool -modfile=tool.mod gotestsum --format standard-quiet \
             --junitfile ~/test_results/${{ matrix.platform }}_test/${PARTITION_ID}/results.xml \
             --jsonfile ~/test_results/${{ matrix.platform }}_test/${PARTITION_ID}/testresults.json \
             -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" $SHORTTEST \
             -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 4 \
+            $COVERPKG_FLAG \
             $PACKAGE_NAMES
       - name: Notify Slack on failure
         if: failure()
@@ -89,6 +102,7 @@ jobs:
         with:
           token: ${{ env.CODECOV_TOKEN }}
           file: ./coverage.txt
+          flags: ${{ inputs.full_coverage && 'full_coverage' || '' }}
           fail_ci_if_error: false
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}

Makefile

@@ -84,6 +84,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 		 -X github.com/algorand/go-algorand/config.Channel=$(CHANNEL)
 
 UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ ))
+COVERPKG_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | egrep -v '/go-algorand/(test|debug|cmd|config/defaultsGenerator|tools)' | egrep -v '(test|testing|mocks|mock)$$' ))
 ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... ))
 
 GOMOD_DIRS := ./tools/block-generator ./tools/x-repo-types
@@ -132,8 +133,19 @@ check_shell:
 
 sanity: fix lint fmt tidy modernize
 
+# "make cover" runs all tests, and collects full coverage across all go-algorand packages by setting -coverpkg.
+# Without setting -coverpkg, coverage reports only measure lines of code exercised within the same package as the tests.
+#
+# "make cover PACKAGE=X" runs all tests in package github.com/algorand/go-algorand/X/... and collects full coverage
+# across all packages that are dependencies of that package.
 cover:
-	go test $(GOTAGS) -coverprofile=cover.out $(UNIT_TEST_SOURCES)
+ifeq ($(PACKAGE),)
+	$(GOTESTCOMMAND) $(GOTAGS) -coverprofile=cover.out $(UNIT_TEST_SOURCES) -covermode=atomic -coverpkg=$(shell echo $(COVERPKG_PACKAGES) | sed 's/ /,/g')
+else
+	cd $(PACKAGE); \
+	$(GOTESTCOMMAND) $(GOTAGS) -coverprofile=cover.out ./... -covermode=atomic -coverpkg=$$( (go list -f '{{ join .Deps "\n" }}' ./...; go list -f '{{ join .TestImports "\n" }}' ./...) | grep 'github.com/algorand/go-algorand' | egrep -v '/go-algorand/(test|debug|cmd|config/defaultsGenerator|tools)' | egrep -v '(test|testing|mocks|mock)$$' | sort | uniq | paste -sd ',' -); \
+	go tool cover -html cover.out
+endif
 
 prof:
 	cd node && go test $(GOTAGS) -cpuprofile=cpu.out -memprofile=mem.out -mutexprofile=mutex.out

buildnumber.dat

@@ -1 +1 @@
-0
+1

data/txHandler.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"reflect"
 	"sync"
 	"time"
 
@@ -145,10 +144,6 @@ func MakeTxHandler(opts TxHandlerOpts) (*TxHandler, error) {
 	if opts.TxPool == nil {
 		return nil, ErrInvalidTxPool
 	}
-	txPoolValue := reflect.ValueOf(opts.TxPool)
-	if txPoolValue.Kind() == reflect.Ptr && txPoolValue.IsNil() {
-		return nil, ErrInvalidTxPool
-	}
 
 	if opts.Ledger == nil {
 		return nil, ErrInvalidLedger

data/txHandler_test.go

@@ -3247,11 +3247,12 @@ func TestTxHandlerNilTxPool(t *testing.T) {
 	})
 	require.ErrorIs(t, err, ErrInvalidTxPool)
 
-	var nilPoll2 *mockTxPool
-	_, err = MakeTxHandler(TxHandlerOpts{
-		TxPool: nilPoll2,
-	})
-	require.ErrorIs(t, err, ErrInvalidTxPool)
+	// This case not handled in MakeTxHandler
+	// var nilPoll2 *mockTxPool
+	// _, err = MakeTxHandler(TxHandlerOpts{
+	// 	TxPool: nilPoll2,
+	// })
+	// require.ErrorIs(t, err, ErrInvalidTxPool)
 
 	_, err = MakeTxHandler(TxHandlerOpts{
 		TxPool: &mockTxPool{},

installer/rpm/algorand-devtools/algorand-devtools.spec

@@ -27,7 +27,7 @@ mkdir -p %{buildroot}/usr/bin
 # NOTE: keep in sync with scripts/build_deb.sh bin_files
 # NOTE: keep in sync with %files section below
 for f in carpenter msgpacktool tealdbg; do
-  install -m 755 ${ALGO_BIN}/${f} %{buildroot}/usr/bin/${f}
+  install -m 755 ${ALGO_TOOLS}/${f} %{buildroot}/usr/bin/${f}
 done
 
 mkdir -p %{buildroot}/etc/pki/rpm-gpg

ledger/boxtxn_test.go

@@ -513,9 +513,10 @@ assert
 		for i := 0; i < 330; i++ {
 			dl.fullBlock()
 		}
-		time.Sleep(5 * time.Second) // balancesFlushInterval, so commit happens
 		dl.fullBlock(call.Args("check", "x", string(make([]byte, 16))))
-		time.Sleep(100 * time.Millisecond) // give commit time to run, and prune au caches
+		// commit au deltas so the box app is executed on of data from ledger, not trackers
+		commitRoundLookback(0, dl.generator)
+		commitRoundLookback(0, dl.validator)
 		dl.fullBlock(call.Args("check", "x", string(make([]byte, 16))))
 
 		// Still the same after caches are flushed

ledger/ledger_test.go

@@ -1514,10 +1514,6 @@ func benchLedgerCache(b *testing.B, startRound basics.Round) {
 
 // triggerTrackerFlush is based in the commit flow but executed it in a single (this) goroutine.
 func triggerTrackerFlush(t *testing.T, l *Ledger) {
-	l.trackers.mu.Lock()
-	dbRound := l.trackers.dbRound
-	l.trackers.mu.Unlock()
-
 	rnd := l.Latest()
 	minBlock := rnd
 	maxLookback := basics.Round(0)
@@ -1538,6 +1534,7 @@ func triggerTrackerFlush(t *testing.T, l *Ledger) {
 	}
 
 	l.trackers.mu.RLock()
+	dbRound := l.trackers.dbRound
 	cdr := l.trackers.produceCommittingTask(rnd, dbRound, &dcc.deferredCommitRange)
 	if cdr != nil {
 		dcc.deferredCommitRange = *cdr