Merge pull request #9460 from lightningnetwork/0-18-5-branch-rc1

release: create branch for v0.18.5-beta.rc1

Author: Roasbeef

.github/workflows/main.yml

@@ -25,13 +25,9 @@ env:
 
   TRANCHES: 8
 
-  # If you change this value, please change it in the following files as well:
-  # /.travis.yml
-  # /Dockerfile
-  # /dev.Dockerfile
-  # /make/builder.Dockerfile
-  # /.github/workflows/release.yml
-  GO_VERSION: 1.22.6
+  # If you change this please also update GO_VERSION in Makefile (then run
+  # `make lint` to see where else it needs to be updated as well).
+  GO_VERSION: 1.22.11
 
 jobs:
   ########################
@@ -202,15 +198,6 @@ jobs:
         if: github.event_name == 'pull_request'
         uses: ./.github/actions/rebase
 
-      - name: git checkout fuzzing seeds
-        uses: actions/checkout@v3
-        with:
-          repository: lightninglabs/lnd-fuzz
-          path: lnd-fuzz
-
-      - name: rsync fuzzing seeds
-        run: rsync -a --ignore-existing lnd-fuzz/ ./
-
       - name: setup go ${{ env.GO_VERSION }}
         uses: ./.github/actions/setup-go
         with:
@@ -297,10 +284,10 @@ jobs:
       - name: Zip log files on failure
         if: ${{ failure() }}
         timeout-minutes: 5 # timeout after 5 minute
-        run: 7z a logs-itest-${{ matrix.name }}.zip itest/**/*.log
+        run: 7z a logs-itest-${{ matrix.name }}.zip itest/**/*.log itest/postgres.log
 
       - name: Upload log files on failure
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
           name: logs-itest-${{ matrix.name }}
@@ -345,7 +332,7 @@ jobs:
         run: 7z a logs-itest-windows.zip itest/**/*.log
 
       - name: Upload log files on failure
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
           name: logs-itest-windows
@@ -390,7 +377,7 @@ jobs:
         run: 7z a logs-itest-macos.zip itest/**/*.log
 
       - name: Upload log files on failure
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ failure() }}
         with:
           name: logs-itest-macos

.github/workflows/release.yaml

@@ -10,12 +10,9 @@ defaults:
     shell: bash
 
 env:
-  # If you change this value, please change it in the following files as well:
-  # /Dockerfile
-  # /dev.Dockerfile
-  # /make/builder.Dockerfile
-  # /.github/workflows/main.yml
-  GO_VERSION: 1.22.6
+  # If you change this please also update GO_VERSION in Makefile (then run
+  # `make lint` to see where else it needs to be updated as well).
+  GO_VERSION: 1.22.11
 
 jobs:
   main:

.golangci.yml

@@ -1,5 +1,7 @@
 run:
-  go: "1.22.6"
+  # If you change this please also update GO_VERSION in Makefile (then run
+  # `make lint` to see where else it needs to be updated as well).
+  go: "1.22.11"
 
   # Abort after 10 minutes.
   timeout: 10m

Dockerfile

@@ -1,9 +1,6 @@
-# If you change this value, please change it in the following files as well:
-# /dev.Dockerfile
-# /make/builder.Dockerfile
-# /.github/workflows/main.yml
-# /.github/workflows/release.yml
-FROM golang:1.22.6-alpine as builder
+# If you change this please also update GO_VERSION in Makefile (then run
+# `make lint` to see where else it needs to be updated as well).
+FROM golang:1.22.11-alpine as builder
 
 # Force Go to use the cgo based DNS resolver. This is required to ensure DNS
 # queries required to connect to linked containers succeed.

Makefile

@@ -35,7 +35,7 @@ endif
 # GO_VERSION is the Go version used for the release build, docker files, and
 # GitHub Actions. This is the reference version for the project. All other Go
 # versions are checked against this version.
-GO_VERSION = 1.22.6
+GO_VERSION = 1.22.11
 
 GOBUILD := $(LOOPVARFIX) go build -v
 GOINSTALL := $(LOOPVARFIX) go install -v
@@ -195,19 +195,31 @@ ifeq ($(dbbackend),postgres)
 	docker rm lnd-postgres --force || echo "Starting new postgres container"
 
 	# Start a fresh postgres instance. Allow a maximum of 500 connections so
-	# that multiple lnd instances with a maximum number of connections of 50
-	# each can run concurrently.
-	docker run --name lnd-postgres -e POSTGRES_PASSWORD=postgres -p 6432:5432 -d postgres:13-alpine -N 500
-	docker logs -f lnd-postgres &
+	# that multiple lnd instances with a maximum number of connections of 20
+	# each can run concurrently. Note that many of the settings here are
+	# specifically for integration testing and are not fit for running
+	# production nodes. The increase in max connections ensures that there
+	# are enough entries allocated for the RWConflictPool to allow multiple
+	# conflicting transactions to track serialization conflicts. The
+	# increase in predicate locks and locks per transaction is to allow the
+	# queries to lock individual rows instead of entire tables, helping
+	# reduce serialization conflicts. Disabling sequential scan for small
+	# tables also helps prevent serialization conflicts by ensuring lookups
+	# lock only relevant rows in the index rather than the entire table.
+	docker run --name lnd-postgres -e POSTGRES_PASSWORD=postgres -p 6432:5432 -d postgres:13-alpine -N 1500 -c max_pred_locks_per_transaction=1024 -c max_locks_per_transaction=128 -c enable_seqscan=off
+	docker logs -f lnd-postgres >itest/postgres.log 2>&1 &
 
 	# Wait for the instance to be started.
 	sleep $(POSTGRES_START_DELAY)
 endif
 
+clean-itest-logs:
+	rm -rf itest/*.log itest/.logs-*
+
 #? itest-only: Only run integration tests without re-building binaries
-itest-only: db-instance
+itest-only: clean-itest-logs db-instance
 	@$(call print, "Running integration tests with ${backend} backend.")
-	rm -rf itest/*.log itest/.logs-*; date
+	date
 	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_part.sh 0 1 $(TEST_FLAGS) $(ITEST_FLAGS)
 	$(COLLECT_ITEST_COVERAGE)
 
@@ -218,9 +230,9 @@ itest: build-itest itest-only
 itest-race: build-itest-race itest-only
 
 #? itest-parallel: Build and run integration tests in parallel mode, running up to ITEST_PARALLELISM test tranches in parallel (default 4)
-itest-parallel: build-itest db-instance
+itest-parallel: clean-itest-logs build-itest db-instance
 	@$(call print, "Running tests")
-	rm -rf itest/*.log itest/.logs-*; date
+	date
 	EXEC_SUFFIX=$(EXEC_SUFFIX) scripts/itest_parallel.sh $(ITEST_PARALLELISM) $(NUM_ITEST_TRANCHES) $(TEST_FLAGS) $(ITEST_FLAGS)
 	$(COLLECT_ITEST_COVERAGE)
 

batch/batch.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/sqldb"
 )
 
 // errSolo is a sentinel error indicating that the requester should re-run the
@@ -55,8 +56,20 @@ func (b *batch) run() {
 			for i, req := range b.reqs {
 				err := req.Update(tx)
 				if err != nil {
-					failIdx = i
-					return err
+					// If we get a serialization error, we
+					// want the underlying SQL retry
+					// mechanism to retry the entire batch.
+					// Otherwise, we can succeed in an
+					// sqldb retry and still re-execute the
+					// failing request individually.
+					dbErr := sqldb.MapSQLError(err)
+					if !sqldb.IsSerializationError(dbErr) {
+						failIdx = i
+
+						return err
+					}
+
+					return dbErr
 				}
 			}
 			return nil

batch/batch_test.go

@@ -0,0 +1,74 @@
+package batch
+
+import (
+	"errors"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/btcsuite/btcwallet/walletdb"
+	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRetry(t *testing.T) {
+	dbDir := t.TempDir()
+
+	dbName := filepath.Join(dbDir, "weks.db")
+	db, err := walletdb.Create(
+		"bdb", dbName, true, kvdb.DefaultDBTimeout,
+	)
+	if err != nil {
+		t.Fatalf("unable to create walletdb: %v", err)
+	}
+	t.Cleanup(func() {
+		db.Close()
+	})
+
+	var (
+		mu     sync.Mutex
+		called int
+	)
+	sched := NewTimeScheduler(db, &mu, time.Second)
+
+	// First, we construct a request that should retry individually and
+	// execute it non-lazily. It should still return the error the second
+	// time.
+	req := &Request{
+		Update: func(tx kvdb.RwTx) error {
+			called++
+
+			return errors.New("test")
+		},
+	}
+	err = sched.Execute(req)
+
+	// Check and reset the called counter.
+	mu.Lock()
+	require.Equal(t, 2, called)
+	called = 0
+	mu.Unlock()
+
+	require.ErrorContains(t, err, "test")
+
+	// Now, we construct a request that should NOT retry because it returns
+	// a serialization error, which should cause the underlying postgres
+	// transaction to retry. Since we aren't using postgres, this will
+	// cause the transaction to not be retried at all.
+	req = &Request{
+		Update: func(tx kvdb.RwTx) error {
+			called++
+
+			return errors.New("could not serialize access")
+		},
+	}
+	err = sched.Execute(req)
+
+	// Check the called counter.
+	mu.Lock()
+	require.Equal(t, 1, called)
+	mu.Unlock()
+
+	require.ErrorContains(t, err, "could not serialize access")
+}

build/version.go

@@ -43,11 +43,11 @@ const (
 	AppMinor uint = 18
 
 	// AppPatch defines the application patch for this binary.
-	AppPatch uint = 4
+	AppPatch uint = 5
 
 	// AppPreRelease MUST only contain characters from semanticAlphabet per
 	// the semantic versioning spec.
-	AppPreRelease = "beta"
+	AppPreRelease = "beta.rc1"
 )
 
 func init() {

channeldb/graph.go

@@ -1023,6 +1023,10 @@ func (c *ChannelGraph) AddChannelEdge(edge *models.ChannelEdgeInfo,
 	}
 
 	for _, f := range op {
+		if f == nil {
+			return fmt.Errorf("nil scheduler option was used")
+		}
+
 		f(r)
 	}
 
@@ -1079,7 +1083,7 @@ func (c *ChannelGraph) addChannelEdge(tx kvdb.RwTx,
 		err := addLightningNode(tx, &node1Shell)
 		if err != nil {
 			return fmt.Errorf("unable to create shell node "+
-				"for: %x", edge.NodeKey1Bytes)
+				"for: %x: %w", edge.NodeKey1Bytes, err)
 		}
 	case node1Err != nil:
 		return err
@@ -1095,7 +1099,7 @@ func (c *ChannelGraph) addChannelEdge(tx kvdb.RwTx,
 		err := addLightningNode(tx, &node2Shell)
 		if err != nil {
 			return fmt.Errorf("unable to create shell node "+
-				"for: %x", edge.NodeKey2Bytes)
+				"for: %x: %w", edge.NodeKey2Bytes, err)
 		}
 	case node2Err != nil:
 		return err
@@ -2622,8 +2626,14 @@ func (c *ChannelGraph) delChannelEdgeUnsafe(edges, edgeIndex, chanIndex,
 
 	// As part of deleting the edge we also remove all disabled entries
 	// from the edgePolicyDisabledIndex bucket. We do that for both directions.
-	updateEdgePolicyDisabledIndex(edges, cid, false, false)
-	updateEdgePolicyDisabledIndex(edges, cid, true, false)
+	err = updateEdgePolicyDisabledIndex(edges, cid, false, false)
+	if err != nil {
+		return err
+	}
+	err = updateEdgePolicyDisabledIndex(edges, cid, true, false)
+	if err != nil {
+		return err
+	}
 
 	// With the edge data deleted, we can purge the information from the two
 	// edge indexes.
@@ -4452,11 +4462,14 @@ func putChanEdgePolicy(edges kvdb.RwBucket, edge *models.ChannelEdgePolicy,
 		return err
 	}
 
-	updateEdgePolicyDisabledIndex(
+	err = updateEdgePolicyDisabledIndex(
 		edges, edge.ChannelID,
 		edge.ChannelFlags&lnwire.ChanUpdateDirection > 0,
 		edge.IsDisabled(),
 	)
+	if err != nil {
+		return err
+	}
 
 	return edges.Put(edgeKey[:], b.Bytes()[:])
 }