Merge pull request #432 from lightninglabs/accounts-prepare

litd: maintenance, code cleanup, middleware fixes

Author: guggero

.github/workflows/main.yml

@@ -171,12 +171,37 @@ jobs:
         working-directory: ./app
         run: yarn
 
-      - name: run check
-        run: make protos-check
+      - name: setup go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: '${{ env.GO_VERSION }}'
 
       - name: compile rpc for golang
         run: make rpc
 
+      - name: run check
+        run: make mod-check
+
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: git checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: setup go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: '${{ env.GO_VERSION }}'
+
+      - name: prepare dummy static data
+        run: mkdir -p app/build; touch app/build/index.html
+
+      - name: run check
+        run: make lint mod-check
+
   unit-race:
     name: unit-race
     runs-on: ubuntu-latest

.golangci.yml

@@ -0,0 +1,59 @@
+run:
+  # timeout for analysis
+  deadline: 4m
+  
+  build-tags:
+    - autopilotrpc
+    - signrpc
+    - walletrpc
+    - chainrpc
+    - invoicesrpc
+    - watchtowerrpc
+    - neutrinorpc
+    - peersrpc
+
+linters-settings:
+  govet:
+    # Don't report about shadowed variables
+    check-shadowing: false
+  gofmt:
+    # simplify code: gofmt with `-s` option, true by default
+    simplify: true
+  tagliatelle:
+    case:
+      rules:
+        json: snake
+  whitespace:
+    multi-func: true
+    multi-if: true
+  gosec:
+    excludes:
+      - G402 # Look for bad TLS connection settings.
+      - G306 # Poor file permissions used when writing to a new file.
+  staticcheck:
+    go: "1.18"
+    checks: ["-SA1019"]
+
+linters:
+  enable:
+    - gofmt
+    - tagliatelle
+    - whitespace
+    - gosec
+
+issues:
+  exclude-rules:
+    # Exclude gosec from running for tests so that tests with weak randomness
+    # (math/rand) will pass the linter.
+    - path: _test\.go
+      linters:
+        - gosec
+    - path: internal/test/.*
+      linters:
+        - gosec
+    # Fix false positives because of build flags in itest directory.
+    - path: itest/.*
+      linters:
+        - unused
+        - deadcode
+        - varcheck

Dockerfile

@@ -45,9 +45,6 @@ COPY --from=nodejsbuilder /go/src/github.com/lightninglabs/lightning-terminal /g
 # queries required to connect to linked containers succeed.
 ENV GODEBUG netdns=cgo
 
-# Explicitly turn on the use of modules (until this becomes the default).
-ENV GO111MODULE on
-
 # Install dependencies and install/build lightning-terminal.
 RUN apk add --no-cache --update alpine-sdk \
   make \

Makefile

@@ -5,14 +5,13 @@ LOOP_PKG := github.com/lightninglabs/loop
 POOL_PKG := github.com/lightninglabs/pool
 BTCD_PKG := github.com/btcsuite/btcd
 
-LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint
-GOVERALLS_PKG := github.com/mattn/goveralls
 GOACC_PKG := github.com/ory/go-acc
+GOIMPORTS_PKG := github.com/rinchsan/gosimports/cmd/gosimports
+TOOLS_DIR := tools
 
 GO_BIN := ${GOPATH}/bin
-GOVERALLS_BIN := $(GO_BIN)/goveralls
-LINT_BIN := $(GO_BIN)/golangci-lint
 GOACC_BIN := $(GO_BIN)/go-acc
+GOIMPORTS_BIN := $(GO_BIN)/gosimports
 
 COMMIT := $(shell git describe --abbrev=40 --dirty --tags)
 COMMIT_HASH := $(shell git rev-parse HEAD)
@@ -30,16 +29,12 @@ POOL_COMMIT := $(shell cat go.mod | \
 		awk -F " " '{ print $$2 }' | \
 		awk -F "/" '{ print $$1 }')
 
-LINT_COMMIT := v1.18.0
-GOACC_COMMIT := ddc355013f90fea78d83d3a6c71f1d37ac07ecd5
+GOBUILD := go build -v
+GOINSTALL := go install -v
+GOTEST := go test -v
+GOMOD := go mod
 
-DEPGET := cd /tmp && GO111MODULE=on go get -v
-GOBUILD := GO111MODULE=on go build -v
-GOINSTALL := GO111MODULE=on go install -v
-GOTEST := GO111MODULE=on go test -v
-GOMOD := GO111MODULE=on go mod
-
-GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
+GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name "*pb.go" -not -name "*pb.gw.go" -not -name "*.pb.json.go")
 GOLIST := go list -deps $(PKG)/... | grep '$(PKG)'| grep -v '/vendor/'
 GOLISTCOVER := $(shell go list -deps -f '{{.ImportPath}}' ./... | grep '$(PKG)' | sed -e 's/^$(ESCPKG)/./')
 
@@ -54,6 +49,7 @@ UNIT := $(GOLIST) | $(XARGS) env $(GOTEST)
 UNIT_RACE := $(UNIT) -race
 
 include make/release_flags.mk
+include make/testing_flags.mk
 
 # We only return the part inside the double quote here to avoid escape issues
 # when calling the external release script. The second parameter can be used to
@@ -73,6 +69,14 @@ LDFLAGS := $(call make_ldflags, $(LND_RELEASE_TAGS))
 # and omit the DWARF symbol table (-w). Also we clear the build ID.
 RELEASE_LDFLAGS := $(call make_ldflags, $(LND_RELEASE_TAGS), -s -w -buildid=)
 
+# Linting uses a lot of memory, so keep it under control by limiting the number
+# of workers if requested.
+ifneq ($(workers),)
+LINT_WORKERS = --concurrency=$(workers)
+endif
+
+DOCKER_TOOLS = docker run -v $$(pwd):/build litd-tools
+
 ITEST_TAGS := rpctest itest $(LND_RELEASE_TAGS)
 ITEST_LDFLAGS := $(call make_ldflags, $(ITEST_TAGS))
 
@@ -90,17 +94,13 @@ all: scratch check install
 # DEPENDENCIES
 # ============
 
-$(GOVERALLS_BIN):
-	@$(call print, "Fetching goveralls.")
-	go get -u $(GOVERALLS_PKG)
-
-$(LINT_BIN):
-	@$(call print, "Fetching linter")
-	$(DEPGET) $(LINT_PKG)@$(LINT_COMMIT)
-
 $(GOACC_BIN):
-	@$(call print, "Fetching go-acc")
-	$(DEPGET) $(GOACC_PKG)@$(GOACC_COMMIT)
+	@$(call print, "Installing go-acc.")
+	cd $(TOOLS_DIR); go install -trimpath -tags=tools $(GOACC_PKG)
+
+$(GOIMPORTS_BIN):
+	@$(call print, "Installing goimports.")
+	cd $(TOOLS_DIR); go install -trimpath $(GOIMPORTS_PKG)
 
 yarn-install:
 	@$(call print, "Installing app dependencies with yarn")
@@ -138,6 +138,10 @@ release: app-build
 	@$(call print, "Creating release of lightning-terminal.")
 	./release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(LND_RELEASE_TAGS)" "$(RELEASE_LDFLAGS)"
 
+docker-tools:
+	@$(call print, "Building tools docker image.")
+	docker build -q -t litd-tools $(TOOLS_DIR)
+
 scratch: build
 
 # =======
@@ -160,16 +164,6 @@ unit-race:
 	mkdir -p app/build && touch app/build/index.html
 	env CGO_ENABLED=1 GORACE="history_size=7 halt_on_errors=1" $(UNIT_RACE) -tags="$(LND_RELEASE_TAGS)"
 
-goveralls: $(GOVERALLS_BIN)
-	@$(call print, "Sending coverage report.")
-	$(GOVERALLS_BIN) -coverprofile=coverage.txt -service=travis-ci
-
-travis-race: lint unit-race
-
-travis-cover: lint unit-cover goveralls
-
-travis-itest: lint
-
 build-itest: app-build
 	@$(call print, "Building itest btcd and litd.")
 	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o itest/btcd-itest -ldflags "$(ITEST_LDFLAGS)" $(BTCD_PKG)
@@ -196,13 +190,15 @@ flake-unit:
 # =========
 # UTILITIES
 # =========
-fmt:
+fmt: $(GOIMPORTS_BIN)
+	@$(call print, "Fixing imports.")
+	gosimports -w $(GOFILES_NOVENDOR)
 	@$(call print, "Formatting source.")
 	gofmt -l -w -s $(GOFILES_NOVENDOR)
 
-lint: $(LINT_BIN)
+lint: docker-tools
 	@$(call print, "Linting source.")
-	$(LINT)
+	$(DOCKER_TOOLS) golangci-lint run -v $(LINT_WORKERS)
 
 mod:
 	@$(call print, "Tidying modules.")
@@ -230,9 +226,10 @@ protos:
 
 protos-check: protos
 	@$(call print, "Verifying compiled protos.")
-	if test -n "$$(git describe --dirty | grep dirty)"; then echo "Protos not properly formatted or not compiled with v3.4.0"; git status; git diff; exit 1; fi
+	if test -n "$$(git describe --dirty | grep dirty)"; then echo "Protos not properly formatted or not compiled with correct version"; git status; git diff; exit 1; fi
 
 clean:
 	@$(call print, "Cleaning source.$(NC)")
-	$(RM) ./lightning-terminal-debug
+	$(RM) ./litcli-debug
+	$(RM) ./litd-debug
 	$(RM) coverage.txt

cmd/litcli/main.go

@@ -8,7 +8,6 @@ import (
 	"strings"
 
 	terminal "github.com/lightninglabs/lightning-terminal"
-	"github.com/lightninglabs/lightning-terminal/litrpc"
 	"github.com/lightninglabs/lndclient"
 	"github.com/lightninglabs/protobuf-hex-display/jsonpb"
 	"github.com/lightninglabs/protobuf-hex-display/proto"
@@ -97,7 +96,7 @@ func fatal(err error) {
 	os.Exit(1)
 }
 
-func getClient(ctx *cli.Context) (litrpc.SessionsClient, func(), error) {
+func connectClient(ctx *cli.Context) (grpc.ClientConnInterface, func(), error) {
 	rpcServer := ctx.GlobalString("rpcserver")
 	tlsCertPath, macPath, err := extractPathArgs(ctx)
 	if err != nil {
@@ -109,8 +108,7 @@ func getClient(ctx *cli.Context) (litrpc.SessionsClient, func(), error) {
 	}
 	cleanup := func() { _ = conn.Close() }
 
-	sessionsClient := litrpc.NewSessionsClient(conn)
-	return sessionsClient, cleanup, nil
+	return conn, cleanup, nil
 }
 
 func getClientConn(address, tlsCertPath, macaroonPath string) (*grpc.ClientConn,

cmd/litcli/sessions.go

@@ -69,11 +69,12 @@ var addSessionCommand = cli.Command{
 }
 
 func addSession(ctx *cli.Context) error {
-	client, cleanup, err := getClient(ctx)
+	clientConn, cleanup, err := connectClient(ctx)
 	if err != nil {
 		return err
 	}
 	defer cleanup()
+	client := litrpc.NewSessionsClient(clientConn)
 
 	label := ctx.String("label")
 	if label == "" {
@@ -192,11 +193,12 @@ var sessionStateMap = map[litrpc.SessionState]sessionFilter{
 
 func listSessions(filter sessionFilter) func(ctx *cli.Context) error {
 	return func(ctx *cli.Context) error {
-		client, cleanup, err := getClient(ctx)
+		clientConn, cleanup, err := connectClient(ctx)
 		if err != nil {
 			return err
 		}
 		defer cleanup()
+		client := litrpc.NewSessionsClient(clientConn)
 
 		ctxb := context.Background()
 		resp, err := client.ListSessions(
@@ -240,11 +242,12 @@ var revokeSessionCommand = cli.Command{
 }
 
 func revokeSession(ctx *cli.Context) error {
-	client, cleanup, err := getClient(ctx)
+	clientConn, cleanup, err := connectClient(ctx)
 	if err != nil {
 		return err
 	}
 	defer cleanup()
+	client := litrpc.NewSessionsClient(clientConn)
 
 	pubkey, err := hex.DecodeString(ctx.String("localpubkey"))
 	if err != nil {

cmd/litd/main.go

@@ -6,7 +6,7 @@ import (
 	"os"
 
 	"github.com/jessevdk/go-flags"
-	"github.com/lightninglabs/lightning-terminal"
+	terminal "github.com/lightninglabs/lightning-terminal"
 )
 
 // main starts the lightning-terminal application.

config.go

@@ -758,10 +758,12 @@ func buildTLSConfigForHttp2(config *Config) (*tls.Config, error) {
 			log.Infof("Listening for Let's Encrypt challenges on "+
 				"%v", config.LetsEncryptListen)
 
-			err := http.ListenAndServe(
-				config.LetsEncryptListen,
-				manager.HTTPHandler(nil),
-			)
+			srv := &http.Server{
+				Addr:              config.LetsEncryptListen,
+				ReadHeaderTimeout: 3 * time.Second,
+				Handler:           manager.HTTPHandler(nil),
+			}
+			err := srv.ListenAndServe()
 			if err != nil {
 				log.Errorf("Error starting Let's Encrypt "+
 					"HTTP listener on port 80: %v", err)

deps.go

@@ -0,0 +1,14 @@
+//go:build deps
+// +build deps
+
+package terminal
+
+// This is a workaround to make sure go mod keeps around the dependencies in the
+// go.sum file that we only use during compilation of the CLI binaries that are
+// delivered together with LiT.
+import (
+	_ "github.com/lightninglabs/faraday/cmd/frcli"
+	_ "github.com/lightninglabs/loop/cmd/loop"
+	_ "github.com/lightninglabs/pool/cmd/pool"
+	_ "github.com/lightningnetwork/lnd/cmd/lncli"
+)

dev.Dockerfile

@@ -23,9 +23,6 @@ COPY --from=nodejsbuilder /go/src/github.com/lightninglabs/lightning-terminal /g
 # queries required to connect to linked containers succeed.
 ENV GODEBUG netdns=cgo
 
-# Explicitly turn on the use of modules (until this becomes the default).
-ENV GO111MODULE on
-
 # Install dependencies and install/build lightning-terminal.
 RUN apk add --no-cache --update alpine-sdk \
   make \