Skip to content

Commit fa97f8f

Browse files
authored
Merge pull request #1641 from lightninglabs/itest-parallel
itest: run in parallel, fix local flake
2 parents 97baec2 + 95ac6e3 commit fa97f8f

File tree

12 files changed

+284
-57
lines changed

12 files changed

+284
-57
lines changed

.github/workflows/main.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ jobs:
278278
go-version: '${{ env.GO_VERSION }}'
279279

280280
- name: run itest
281-
run: make itest cover=1
281+
run: make itest-parallel
282282

283283
- name: Zip log files on failure
284284
if: ${{ failure() }}
@@ -321,7 +321,7 @@ jobs:
321321
go-version: '${{ env.GO_VERSION }}'
322322

323323
- name: run itest
324-
run: make itest dbbackend=postgres cover=1
324+
run: make itest-parallel dbbackend=postgres
325325

326326
- name: Zip log files on failure
327327
if: ${{ failure() }}

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ cmd/tapd/tapd
2121
/itest/.backendlogs/*
2222
/itest/.minerlogs/*
2323
/itest/tapd-itest
24+
/itest/itest.test
2425

2526
/docs/examples/basic-price-oracle/basic-price-oracle
2627

Makefile

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,10 @@ build-itest:
102102
@$(call print, "Building itest lnd.")
103103
CGO_ENABLED=0 $(GOBUILD) -mod=mod -tags="$(ITEST_TAGS)" -o itest/lnd-itest $(DEV_LDFLAGS) $(LND_PKG)/cmd/lnd
104104

105+
build-itest-binary:
106+
@$(call print, "Building itest binary for ${backend} backend.")
107+
CGO_ENABLED=0 $(GOTEST) -v $(ITEST_COVERAGE) ./itest -tags="$(ITEST_TAGS)" -c -o itest/itest.test
108+
105109
build-loadtest:
106110
CGO_ENABLED=0 $(GOTEST) -c -tags="$(LOADTEST_TAGS)" -o loadtest $(PKG)/itest/loadtest
107111

@@ -194,16 +198,25 @@ itest: build-itest itest-only
194198

195199
itest-trace: build-itest itest-only-trace
196200

197-
itest-only: aperture-dir
201+
itest-only: aperture-dir clean-itest-logs
198202
@$(call print, "Running integration tests with ${backend} backend.")
199-
rm -rf itest/regtest; date
203+
date
200204
$(GOTEST) ./itest -v $(ITEST_COVERAGE) -tags="$(ITEST_TAGS)" $(TEST_FLAGS) $(ITEST_FLAGS) -btcdexec=./btcd-itest -logdir=regtest
201205

202-
itest-only-trace: aperture-dir
206+
itest-only-trace: aperture-dir clean-itest-logs
203207
@$(call print, "Running integration tests with ${backend} backend.")
204208
rm -rf itest/regtest; date
205209
$(GOTEST) ./itest -v -tags="$(ITEST_TAGS)" $(TEST_FLAGS) $(ITEST_FLAGS) -loglevel=trace -btcdexec=./btcd-itest -logdir=regtest
206210

211+
itest-parallel: aperture-dir clean-itest-logs build-itest build-itest-binary
212+
@$(call print, "Running integration tests in parallel with ${backend} backend.")
213+
date
214+
scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(SHUFFLE_SEED) $(TEST_FLAGS) $(ITEST_FLAGS)
215+
$(COLLECT_ITEST_COVERAGE)
216+
217+
clean-itest-logs:
218+
rm -rf itest/regtest
219+
207220
aperture-dir:
208221
ifeq ($(UNAME_S),Linux)
209222
mkdir -p $$HOME/.aperture

itest/integration_test.go

Lines changed: 144 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,64 @@ package itest
55
import (
66
"context"
77
"flag"
8+
"fmt"
89
"testing"
910

1011
"github.com/lightninglabs/taproot-assets/taprpc"
1112
"github.com/lightningnetwork/lnd/lntest"
1213
"github.com/stretchr/testify/require"
14+
"golang.org/x/exp/rand"
1315
)
1416

15-
var optionalTests = flag.Bool("optional", false, "if true, the optional test"+
16-
"list will be used")
17+
const (
18+
// defaultSplitTranches is the default number of tranches we split the
19+
// test cases into.
20+
defaultSplitTranches uint = 1
21+
22+
// defaultRunTranche is the default index of the test cases tranche that
23+
// we run.
24+
defaultRunTranche uint = 0
25+
)
26+
27+
var (
28+
optionalTests = flag.Bool(
29+
"optional", false, "if true, the optional test list will be "+
30+
"used",
31+
)
32+
33+
// testCasesSplitParts is the number of tranches the test cases should
34+
// be split into. By default this is set to 1, so no splitting happens.
35+
// If this value is increased, then the -runtranche flag must be
36+
// specified as well to indicate which part should be run in the current
37+
// invocation.
38+
testCasesSplitTranches = flag.Uint(
39+
"splittranches", defaultSplitTranches, "split the test cases "+
40+
"in this many tranches and run the tranche at "+
41+
"0-based index specified by the -runtranche flag",
42+
)
43+
44+
// shuffleSeedFlag is the source of randomness used to shuffle the test
45+
// cases. If not specified, the test cases won't be shuffled.
46+
shuffleSeedFlag = flag.Uint64(
47+
"shuffleseed", 0, "if set, shuffles the test cases using this "+
48+
"as the source of randomness",
49+
)
50+
51+
// testCasesRunTranche is the 0-based index of the split test cases
52+
// tranche to run in the current invocation.
53+
testCasesRunTranche = flag.Uint(
54+
"runtranche", defaultRunTranche, "run the tranche of the "+
55+
"split test cases with the given (0-based) index",
56+
)
57+
)
1758

1859
// TestTaprootAssetsDaemon performs a series of integration tests amongst a
1960
// programmatically driven set of participants, namely a Taproot Assets daemon
2061
// and a universe server.
2162
func TestTaprootAssetsDaemon(t *testing.T) {
63+
// Get the test cases to be run in this tranche.
64+
testCases, trancheIndex, trancheOffset := getTestCaseSplitTranche()
65+
2266
// Switch to the list of optional test cases with the '-optional' flag.
2367
testList := testCases
2468
if *optionalTests {
@@ -43,9 +87,16 @@ func TestTaprootAssetsDaemon(t *testing.T) {
4387
lndHarness.CleanShutDown()
4488
})
4589

90+
// Get the current block height.
91+
height := lndHarness.CurrentHeight()
92+
4693
t.Logf("Running %v integration tests", len(testList))
47-
for _, testCase := range testList {
48-
success := t.Run(testCase.name, func(t1 *testing.T) {
94+
for idx, testCase := range testList {
95+
name := fmt.Sprintf("tranche%02d/%02d-of-%d/%s",
96+
trancheIndex, trancheOffset+uint(idx)+1,
97+
len(allTestCases), testCase.name)
98+
99+
success := t.Run(name, func(t1 *testing.T) {
49100
// Create a new LND node for use with the universe
50101
// server.
51102
t.Log("Starting universe server LND node")
@@ -90,6 +141,11 @@ func TestTaprootAssetsDaemon(t *testing.T) {
90141
return
91142
}
92143
}
144+
145+
//nolint:forbidigo
146+
fmt.Printf("=========> tranche %v finished, tested %d cases, mined "+
147+
"blocks: %d\n", trancheIndex, len(testCases),
148+
lndHarness.CurrentHeight()-height)
93149
}
94150

95151
// testGetInfo tests the GetInfo RPC call.
@@ -115,3 +171,87 @@ func testGetInfo(t *harnessTest) {
115171
// Ensure the response matches the expected response.
116172
require.Equal(t.t, resp, respCli)
117173
}
174+
175+
// maybeShuffleTestCases shuffles the test cases if the flag `shuffleseed` is
176+
// set and not 0. In parallel tests we want to shuffle the test cases so they
177+
// are executed in a random order. This is done to even out the blocks mined in
178+
// each test tranche so they can run faster.
179+
//
180+
// NOTE: Because the parallel tests are initialized with the same seed (job
181+
// ID), they will always have the same order.
182+
func maybeShuffleTestCases() {
183+
// Exit if not set.
184+
if shuffleSeedFlag == nil {
185+
return
186+
}
187+
188+
// Exit if set to 0.
189+
if *shuffleSeedFlag == 0 {
190+
return
191+
}
192+
193+
// Init the seed and shuffle the test cases.
194+
rand.Seed(*shuffleSeedFlag)
195+
rand.Shuffle(len(allTestCases), func(i, j int) {
196+
allTestCases[i], allTestCases[j] =
197+
allTestCases[j], allTestCases[i]
198+
})
199+
}
200+
201+
// createIndices divides the number of test cases into pairs of indices that
202+
// specify the start and end of a tranche.
203+
func createIndices(numCases, numTranches uint) [][2]uint {
204+
// Calculate base value and remainder.
205+
base := numCases / numTranches
206+
remainder := numCases % numTranches
207+
208+
// Generate indices.
209+
indices := make([][2]uint, numTranches)
210+
start := uint(0)
211+
212+
for i := uint(0); i < numTranches; i++ {
213+
end := start + base
214+
if i < remainder {
215+
// Add one for the remainder.
216+
end++
217+
}
218+
indices[i] = [2]uint{start, end}
219+
start = end
220+
}
221+
222+
return indices
223+
}
224+
225+
// getTestCaseSplitTranche returns the sub slice of the test cases that should
226+
// be run as the current split tranche as well as the index and slice offset of
227+
// the tranche.
228+
func getTestCaseSplitTranche() ([]*testCase, uint, uint) {
229+
numTranches := defaultSplitTranches
230+
if testCasesSplitTranches != nil {
231+
numTranches = *testCasesSplitTranches
232+
}
233+
runTranche := defaultRunTranche
234+
if testCasesRunTranche != nil {
235+
runTranche = *testCasesRunTranche
236+
}
237+
238+
// There's a special flake-hunt mode where we run the same test multiple
239+
// times in parallel. In that case the tranche index is equal to the
240+
// thread ID, but we need to actually run all tests for the regex
241+
// selection to work.
242+
threadID := runTranche
243+
if numTranches == 1 {
244+
runTranche = 0
245+
}
246+
247+
// Shuffle the test cases if the `shuffleseed` flag is set.
248+
maybeShuffleTestCases()
249+
250+
numCases := uint(len(allTestCases))
251+
indices := createIndices(numCases, numTranches)
252+
index := indices[runTranche]
253+
trancheOffset, trancheEnd := index[0], index[1]
254+
255+
return allTestCases[trancheOffset:trancheEnd], threadID,
256+
trancheOffset
257+
}

itest/send_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1835,7 +1835,7 @@ func testSendNoCourierUniverseImport(t *harnessTest) {
18351835
bobLnd := t.lndHarness.NewNodeWithCoins("Bob", nil)
18361836
secondTapd := setupTapdHarness(
18371837
t.t, t, bobLnd, t.universeServer, func(p *tapdHarnessParams) {
1838-
p.proofCourier = &proof.MockProofCourier{}
1838+
p.proofCourier = proof.NewMockProofCourier()
18391839
},
18401840
)
18411841
defer func() {

itest/tapd_harness.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
lfn "github.com/lightningnetwork/lnd/fn/v2"
3232
"github.com/lightningnetwork/lnd/lnrpc"
3333
"github.com/lightningnetwork/lnd/lntest/node"
34+
"github.com/lightningnetwork/lnd/lntest/port"
3435
"github.com/lightningnetwork/lnd/lntest/wait"
3536
"github.com/lightningnetwork/lnd/macaroons"
3637
"github.com/stretchr/testify/require"
@@ -199,10 +200,10 @@ func newTapdHarness(t *testing.T, ht *harnessTest, cfg tapdConfig,
199200
}
200201

201202
tapCfg.RpcConf.RawRPCListeners = []string{
202-
fmt.Sprintf("127.0.0.1:%d", nextAvailablePort()),
203+
fmt.Sprintf("127.0.0.1:%d", port.NextAvailablePort()),
203204
}
204205
tapCfg.RpcConf.RawRESTListeners = []string{
205-
fmt.Sprintf("127.0.0.1:%d", nextAvailablePort()),
206+
fmt.Sprintf("127.0.0.1:%d", port.NextAvailablePort()),
206207
}
207208

208209
// Update the config with the lnd node's connection info.
@@ -289,6 +290,11 @@ func newTapdHarness(t *testing.T, ht *harnessTest, cfg tapdConfig,
289290
typedProofCourier.ListenAddr,
290291
)
291292

293+
case *proof.MockProofCourier:
294+
finalCfg.DefaultProofCourierAddr = fmt.Sprintf(
295+
"%s://%s", proof.MockCourierType, "dummyhost:1234",
296+
)
297+
292298
default:
293299
finalCfg.DefaultProofCourierAddr = ""
294300
}

itest/test_harness.go

Lines changed: 3 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import (
44
"context"
55
"flag"
66
"fmt"
7-
"net"
8-
"sync/atomic"
97
"testing"
108
"time"
119

@@ -22,6 +20,7 @@ import (
2220
"github.com/lightningnetwork/lnd/build"
2321
"github.com/lightningnetwork/lnd/lntest"
2422
"github.com/lightningnetwork/lnd/lntest/node"
23+
"github.com/lightningnetwork/lnd/lntest/port"
2524
"github.com/lightningnetwork/lnd/lntest/wait"
2625
"github.com/lightningnetwork/lnd/signal"
2726
"github.com/stretchr/testify/require"
@@ -267,34 +266,6 @@ func (h *harnessTest) addFederationServer(host string, target *tapdHarness) {
267266
require.NoError(h.t, err)
268267
}
269268

270-
// nextAvailablePort returns the first port that is available for listening by
271-
// a new node. It panics if no port is found and the maximum available TCP port
272-
// is reached.
273-
func nextAvailablePort() int {
274-
port := atomic.AddUint32(&lastPort, 1)
275-
for port < 65535 {
276-
// If there are no errors while attempting to listen on this
277-
// port, close the socket and return it as available. While it
278-
// could be the case that some other process picks up this port
279-
// between the time the socket is closed and it's reopened in
280-
// the harness node, in practice in CI servers this seems much
281-
// less likely than simply some other process already being
282-
// bound at the start of the tests.
283-
addr := fmt.Sprintf("127.0.0.1:%d", port)
284-
l, err := net.Listen("tcp4", addr)
285-
if err == nil {
286-
err := l.Close()
287-
if err == nil {
288-
return int(port)
289-
}
290-
}
291-
port = atomic.AddUint32(&lastPort, 1)
292-
}
293-
294-
// No ports available? Must be a mistake.
295-
panic("no ports available for listening")
296-
}
297-
298269
// setupHarnesses creates new server and client harnesses that are connected
299270
// to each other through an in-memory gRPC connection.
300271
func setupHarnesses(t *testing.T, ht *harnessTest,
@@ -317,8 +288,8 @@ func setupHarnesses(t *testing.T, ht *harnessTest,
317288
var proofCourier proof.CourierHarness
318289
switch proofCourierType {
319290
case proof.HashmailCourierType:
320-
port := nextAvailablePort()
321-
apertureHarness := NewApertureHarness(ht.t, port)
291+
listenPort := port.NextAvailablePort()
292+
apertureHarness := NewApertureHarness(ht.t, listenPort)
322293
err := apertureHarness.Start(nil)
323294
require.NoError(t, err, "aperture proof courier harness")
324295

itest/test_list_on_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ package itest
44

55
import "github.com/lightninglabs/taproot-assets/proof"
66

7-
var testCases = []*testCase{
7+
var allTestCases = []*testCase{
88
{
99
name: "psbt stxo exclusion proofs",
1010
test: testPsbtSTXOExclusionProofs,

0 commit comments

Comments
 (0)