Skip to content

Commit 288524f

Browse files
committed
multi: add itest-parallel make target
1 parent 3652b3e commit 288524f

File tree

8 files changed

+264
-21
lines changed

8 files changed

+264
-21
lines changed

.github/workflows/main.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ jobs:
278278
go-version: '${{ env.GO_VERSION }}'
279279

280280
- name: run itest
281-
run: make itest cover=1
281+
run: make itest-parallel
282282

283283
- name: Zip log files on failure
284284
if: ${{ failure() }}
@@ -321,7 +321,7 @@ jobs:
321321
go-version: '${{ env.GO_VERSION }}'
322322

323323
- name: run itest
324-
run: make itest dbbackend=postgres cover=1
324+
run: make itest-parallel dbbackend=postgres
325325

326326
- name: Zip log files on failure
327327
if: ${{ failure() }}

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ cmd/tapd/tapd
2121
/itest/.backendlogs/*
2222
/itest/.minerlogs/*
2323
/itest/tapd-itest
24+
/itest/itest.test
2425

2526
/docs/examples/basic-price-oracle/basic-price-oracle
2627

Makefile

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,10 @@ build-itest:
102102
@$(call print, "Building itest lnd.")
103103
CGO_ENABLED=0 $(GOBUILD) -mod=mod -tags="$(ITEST_TAGS)" -o itest/lnd-itest $(DEV_LDFLAGS) $(LND_PKG)/cmd/lnd
104104

105+
build-itest-binary:
106+
@$(call print, "Building itest binary for ${backend} backend.")
107+
CGO_ENABLED=0 $(GOTEST) -v $(ITEST_COVERAGE) ./itest -tags="$(ITEST_TAGS)" -c -o itest/itest.test
108+
105109
build-loadtest:
106110
CGO_ENABLED=0 $(GOTEST) -c -tags="$(LOADTEST_TAGS)" -o loadtest $(PKG)/itest/loadtest
107111

@@ -194,16 +198,25 @@ itest: build-itest itest-only
194198

195199
itest-trace: build-itest itest-only-trace
196200

197-
itest-only: aperture-dir
201+
itest-only: aperture-dir clean-itest-logs
198202
@$(call print, "Running integration tests with ${backend} backend.")
199-
rm -rf itest/regtest; date
203+
date
200204
$(GOTEST) ./itest -v $(ITEST_COVERAGE) -tags="$(ITEST_TAGS)" $(TEST_FLAGS) $(ITEST_FLAGS) -btcdexec=./btcd-itest -logdir=regtest
201205

202-
itest-only-trace: aperture-dir
206+
itest-only-trace: aperture-dir clean-itest-logs
203207
@$(call print, "Running integration tests with ${backend} backend.")
204208
rm -rf itest/regtest; date
205209
$(GOTEST) ./itest -v -tags="$(ITEST_TAGS)" $(TEST_FLAGS) $(ITEST_FLAGS) -loglevel=trace -btcdexec=./btcd-itest -logdir=regtest
206210

211+
itest-parallel: aperture-dir clean-itest-logs build-itest build-itest-binary
212+
@$(call print, "Running integration tests in parallel with ${backend} backend.")
213+
date
214+
scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS)
215+
$(COLLECT_ITEST_COVERAGE)
216+
217+
clean-itest-logs:
218+
rm -rf itest/regtest
219+
207220
aperture-dir:
208221
ifeq ($(UNAME_S),Linux)
209222
mkdir -p $$HOME/.aperture

itest/integration_test.go

Lines changed: 144 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,64 @@ package itest
55
import (
66
"context"
77
"flag"
8+
"fmt"
89
"testing"
910

1011
"github.com/lightninglabs/taproot-assets/taprpc"
1112
"github.com/lightningnetwork/lnd/lntest"
1213
"github.com/stretchr/testify/require"
14+
"golang.org/x/exp/rand"
1315
)
1416

15-
var optionalTests = flag.Bool("optional", false, "if true, the optional test"+
16-
"list will be used")
17+
const (
18+
// defaultSplitTranches is the default number of tranches we split the
19+
// test cases into.
20+
defaultSplitTranches uint = 1
21+
22+
// defaultRunTranche is the default index of the test cases tranche that
23+
// we run.
24+
defaultRunTranche uint = 0
25+
)
26+
27+
var (
28+
optionalTests = flag.Bool(
29+
"optional", false, "if true, the optional test list will be "+
30+
"used",
31+
)
32+
33+
// testCasesSplitParts is the number of tranches the test cases should
34+
// be split into. By default this is set to 1, so no splitting happens.
35+
// If this value is increased, then the -runtranche flag must be
36+
// specified as well to indicate which part should be run in the current
37+
// invocation.
38+
testCasesSplitTranches = flag.Uint(
39+
"splittranches", defaultSplitTranches, "split the test cases "+
40+
"in this many tranches and run the tranche at "+
41+
"0-based index specified by the -runtranche flag",
42+
)
43+
44+
// shuffleSeedFlag is the source of randomness used to shuffle the test
45+
// cases. If not specified, the test cases won't be shuffled.
46+
shuffleSeedFlag = flag.Uint64(
47+
"shuffleseed", 0, "if set, shuffles the test cases using this "+
48+
"as the source of randomness",
49+
)
50+
51+
// testCasesRunTranche is the 0-based index of the split test cases
52+
// tranche to run in the current invocation.
53+
testCasesRunTranche = flag.Uint(
54+
"runtranche", defaultRunTranche, "run the tranche of the "+
55+
"split test cases with the given (0-based) index",
56+
)
57+
)
1758

1859
// TestTaprootAssetsDaemon performs a series of integration tests amongst a
1960
// programmatically driven set of participants, namely a Taproot Assets daemon
2061
// and a universe server.
2162
func TestTaprootAssetsDaemon(t *testing.T) {
63+
// Get the test cases to be run in this tranche.
64+
testCases, trancheIndex, trancheOffset := getTestCaseSplitTranche()
65+
2266
// Switch to the list of optional test cases with the '-optional' flag.
2367
testList := testCases
2468
if *optionalTests {
@@ -43,9 +87,16 @@ func TestTaprootAssetsDaemon(t *testing.T) {
4387
lndHarness.CleanShutDown()
4488
})
4589

90+
// Get the current block height.
91+
height := lndHarness.CurrentHeight()
92+
4693
t.Logf("Running %v integration tests", len(testList))
47-
for _, testCase := range testList {
48-
success := t.Run(testCase.name, func(t1 *testing.T) {
94+
for idx, testCase := range testList {
95+
name := fmt.Sprintf("tranche%02d/%02d-of-%d/%s",
96+
trancheIndex, trancheOffset+uint(idx)+1,
97+
len(allTestCases), testCase.name)
98+
99+
success := t.Run(name, func(t1 *testing.T) {
49100
// Create a new LND node for use with the universe
50101
// server.
51102
t.Log("Starting universe server LND node")
@@ -90,6 +141,11 @@ func TestTaprootAssetsDaemon(t *testing.T) {
90141
return
91142
}
92143
}
144+
145+
//nolint:forbidigo
146+
fmt.Printf("=========> tranche %v finished, tested %d cases, mined "+
147+
"blocks: %d\n", trancheIndex, len(testCases),
148+
lndHarness.CurrentHeight()-height)
93149
}
94150

95151
// testGetInfo tests the GetInfo RPC call.
@@ -115,3 +171,87 @@ func testGetInfo(t *harnessTest) {
115171
// Ensure the response matches the expected response.
116172
require.Equal(t.t, resp, respCli)
117173
}
174+
175+
// maybeShuffleTestCases shuffles the test cases if the flag `shuffleseed` is
176+
// set and not 0. In parallel tests we want to shuffle the test cases so they
177+
// are executed in a random order. This is done to even out the blocks mined in
178+
// each test tranche so they can run faster.
179+
//
180+
// NOTE: Because the parallel tests are initialized with the same seed (job
181+
// ID), they will always have the same order.
182+
func maybeShuffleTestCases() {
183+
// Exit if not set.
184+
if shuffleSeedFlag == nil {
185+
return
186+
}
187+
188+
// Exit if set to 0.
189+
if *shuffleSeedFlag == 0 {
190+
return
191+
}
192+
193+
// Init the seed and shuffle the test cases.
194+
rand.Seed(*shuffleSeedFlag)
195+
rand.Shuffle(len(allTestCases), func(i, j int) {
196+
allTestCases[i], allTestCases[j] =
197+
allTestCases[j], allTestCases[i]
198+
})
199+
}
200+
201+
// createIndices divides the number of test cases into pairs of indices that
202+
// specify the start and end of a tranche.
203+
func createIndices(numCases, numTranches uint) [][2]uint {
204+
// Calculate base value and remainder.
205+
base := numCases / numTranches
206+
remainder := numCases % numTranches
207+
208+
// Generate indices.
209+
indices := make([][2]uint, numTranches)
210+
start := uint(0)
211+
212+
for i := uint(0); i < numTranches; i++ {
213+
end := start + base
214+
if i < remainder {
215+
// Add one for the remainder.
216+
end++
217+
}
218+
indices[i] = [2]uint{start, end}
219+
start = end
220+
}
221+
222+
return indices
223+
}
224+
225+
// getTestCaseSplitTranche returns the sub slice of the test cases that should
226+
// be run as the current split tranche as well as the index and slice offset of
227+
// the tranche.
228+
func getTestCaseSplitTranche() ([]*testCase, uint, uint) {
229+
numTranches := defaultSplitTranches
230+
if testCasesSplitTranches != nil {
231+
numTranches = *testCasesSplitTranches
232+
}
233+
runTranche := defaultRunTranche
234+
if testCasesRunTranche != nil {
235+
runTranche = *testCasesRunTranche
236+
}
237+
238+
// There's a special flake-hunt mode where we run the same test multiple
239+
// times in parallel. In that case the tranche index is equal to the
240+
// thread ID, but we need to actually run all tests for the regex
241+
// selection to work.
242+
threadID := runTranche
243+
if numTranches == 1 {
244+
runTranche = 0
245+
}
246+
247+
// Shuffle the test cases if the `shuffleseed` flag is set.
248+
maybeShuffleTestCases()
249+
250+
numCases := uint(len(allTestCases))
251+
indices := createIndices(numCases, numTranches)
252+
index := indices[runTranche]
253+
trancheOffset, trancheEnd := index[0], index[1]
254+
255+
return allTestCases[trancheOffset:trancheEnd], threadID,
256+
trancheOffset
257+
}

itest/test_list_on_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ package itest
44

55
import "github.com/lightninglabs/taproot-assets/proof"
66

7-
var testCases = []*testCase{
7+
var allTestCases = []*testCase{
88
{
99
name: "psbt stxo exclusion proofs",
1010
test: testPsbtSTXOExclusionProofs,

make/testing_flags.mk

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,14 @@ RPC_TAGS = autopilotrpc chainrpc invoicesrpc peersrpc routerrpc signrpc verrpc w
44
LOG_TAGS =
55
TEST_FLAGS =
66
ITEST_FLAGS = -logoutput
7-
ITEST_COVERAGE =
7+
ITEST_COVERAGE = -cover -coverpkg=./... -coverprofile=itest/coverage.txt
8+
COLLECT_ITEST_COVERAGE = go tool covdata textfmt -i=itest/regtest/cover -o itest/coverage.txt
89
COVER_PKG = $$(go list -deps -tags="$(DEV_TAGS)" ./... | grep '$(PKG)' | grep -v taprpc)
910
COVER_HTML = go tool cover -html=coverage.txt -o coverage.html
1011
POSTGRES_START_DELAY = 5
12+
NUM_ITEST_TRANCHES = 8
13+
ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES)
14+
SHUFFLE_SEED = 0
1115

1216
GOLIST := go list -tags="$(DEV_TAGS)" -deps $(PKG)/... | grep '$(PKG)'| grep -v '/vendor/'
1317
GOLISTCOVER := $(shell go list -tags="$(DEV_TAGS)" -deps -f '{{.ImportPath}}' ./... | grep '$(PKG)' | sed -e 's/^$(ESCPKG)/./')
@@ -17,6 +21,22 @@ ifneq ($(with-rpc),)
1721
DEV_TAGS += $(RPC_TAGS)
1822
endif
1923

24+
# Scale the number of parallel running itest tranches.
25+
ifneq ($(tranches),)
26+
NUM_ITEST_TRANCHES = $(tranches)
27+
ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES)
28+
endif
29+
30+
# Give the ability to run the same tranche multiple times at the same time.
31+
ifneq ($(parallel),)
32+
ITEST_PARALLELISM = $(parallel)
33+
endif
34+
35+
# Set the seed for shuffling the test cases.
36+
ifneq ($(shuffleseed),)
37+
SHUFFLE_SEED = $(shuffleseed)
38+
endif
39+
2040
# If specific package is being unit tested, construct the full name of the
2141
# subpackage.
2242
ifneq ($(pkg),)
@@ -34,7 +54,7 @@ endif
3454

3555
# Define the integration test.run filter if the icase argument was provided.
3656
ifneq ($(icase),)
37-
TEST_FLAGS += -test.run="TestTaprootAssetsDaemon/$(icase)"
57+
TEST_FLAGS += -test.run="TestTaprootAssetsDaemon/tranche.*/.*-of-.*/$(icase)"
3858
endif
3959

4060
# Don't delete the data directories of nodes.
@@ -65,11 +85,6 @@ ifneq ($(tags),)
6585
DEV_TAGS += ${tags}
6686
endif
6787

68-
# Enable integration test coverage (requires Go >= 1.20.0).
69-
ifneq ($(cover),)
70-
ITEST_COVERAGE = -coverprofile=itest/coverage.txt -coverpkg=./...
71-
endif
72-
7388
# Define the log tags that will be applied only when running unit tests. If none
7489
# are provided, we default to "nolog" which will be silent.
7590
ifneq ($(log),)
@@ -128,8 +143,5 @@ endif
128143
# Construct the integration test command with the added build flags.
129144
ITEST_TAGS := $(DEV_TAGS) $(RPC_TAGS) integration itest $(backend)
130145

131-
# Construct the coverage test command path with the added build flags.
132-
GOACC := $(GOACC_BIN) $(COVER_PKG) -- -tags="$(DEV_TAGS) $(LOG_TAGS)" $(TEST_FLAGS)
133-
134146
# Construct the load test command with the added build flags.
135-
LOADTEST_TAGS := $(DEV_TAGS) $(RPC_TAGS) loadtest
147+
LOADTEST_TAGS := $(DEV_TAGS) $(RPC_TAGS) loadtest

scripts/itest_parallel.sh

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
3+
# Get all the variables.
4+
PROCESSES=$1
5+
TRANCHES=$2
6+
SHUFFLE_SEED=$3
7+
8+
# Here we also shift 3 times and get the rest of our flags to pass on in $@.
9+
shift 3
10+
11+
# Create a variable to hold the final exit code.
12+
exit_code=0
13+
14+
# Run commands using xargs in parallel and capture their PIDs
15+
pids=()
16+
for ((i=0; i<PROCESSES; i++)); do
17+
scripts/itest_part.sh $i $TRANCHES $SHUFFLE_SEED $@ &
18+
pids+=($!)
19+
done
20+
21+
22+
# Wait for the processes created by xargs to finish.
23+
for pid in "${pids[@]}"; do
24+
wait $pid
25+
26+
# Once finished, grab its exit code.
27+
current_exit_code=$?
28+
29+
# Overwrite the exit code if current itest doesn't return 0.
30+
if [ $current_exit_code -ne 0 ]; then
31+
# Only write the exit code of the first failing itest.
32+
if [ $exit_code -eq 0 ]; then
33+
exit_code=$current_exit_code
34+
fi
35+
fi
36+
done
37+
38+
# Exit with the exit code of the first failing itest or 0.
39+
exit $exit_code

0 commit comments

Comments
 (0)