Merge pull request #27 from kaleido-io/kv3-wallet

Add filesystem listener interface to KeystoreV3 signer, and move to `pkg`

Author: nguyer

.vscode/settings.json

@@ -16,6 +16,8 @@
     "ffsigner",
     "fftypes",
     "filewallet",
+    "fsnotify",
+    "fswallet",
     "GJSON",
     "httpserver",
     "hyperledger",

Makefile

@@ -44,7 +44,7 @@ clean:
 		$(VGO) clean
 deps:
 		$(VGO) get ./ffsigner
-docs:
+reference:
 		$(VGO) test ./cmd -timeout=10s -tags docs
 docker:
 		docker build --build-arg BUILD_VERSION=${BUILD_VERSION} ${DOCKER_ARGS} -t hyperledger/firefly-signer .

README.md

@@ -19,10 +19,17 @@ A set of Ethereum transaction signing utilities designed for use across projects
   - EIP-155
   - EIP-1559
   - See `pkg/ethsigner` [go doc](https://pkg.go.dev/github.com/hyperledger/firefly-signer/pkg/ethsigner)
-- Keystore V3 wallet implementation
+- Keystore V3 key file implementation
   - Scrypt - read/write
   - pbkdf2 - read
   - See `pkg/keystorev3` [go doc](https://pkg.go.dev/github.com/hyperledger/firefly-signer/pkg/keystorev3)
+- Filesystem wallet
+  - Configurable caching for in-memory keys
+  - Files in directory with a given extension matching `{{ADDRESS}}.key`/`{{ADDRESS}}.toml` or arbitrary regex
+  - Files can be TOML/YAML/JSON metadata pointing to Keystore V3 files + password files
+  - Files can be Keystore V3 files directly, with accompanying `{{ADDRESS}}.pass` files
+  - Detects newly added files automatically
+  - See `pkg/fswallet` [go doc](https://pkg.go.dev/github.com/hyperledger/firefly-signer/pkg/fswallet)
 
 ## JSON/RPC proxy server
 
@@ -40,12 +47,6 @@ calls through unchanged.
   - Queries Chain ID via `net_version` on startup
   - `eth_accounts` JSON/RPC method support
   - Trivial nonce management built-in (calls `eth_getTransactionCount` for each request)
-- File based wallet
-  - Configurable caching for in-memory keys
-  - Files in directory with a given extension matching `{{ADDRESS}}.key`/`{{ADDRESS}}.toml`
-  - Customizable extension, and optional `0x` prefix to filename
-  - Files can be TOML/YAML/JSON metadata pointing to Keystore V3 files + password files
-  - Files can be Keystore V3 files directly, with accompanying `{{ADDRESS}}.pass` files
 
 ## JSON/RPC proxy server configuration
 

cmd/ffsigner.go

@@ -26,10 +26,10 @@ import (
 	"github.com/hyperledger/firefly-common/pkg/config"
 	"github.com/hyperledger/firefly-common/pkg/i18n"
 	"github.com/hyperledger/firefly-common/pkg/log"
-	"github.com/hyperledger/firefly-signer/internal/filewallet"
 	"github.com/hyperledger/firefly-signer/internal/rpcserver"
 	"github.com/hyperledger/firefly-signer/internal/signerconfig"
 	"github.com/hyperledger/firefly-signer/internal/signermsgs"
+	"github.com/hyperledger/firefly-signer/pkg/fswallet"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -92,7 +92,7 @@ func run() error {
 	if !config.GetBool(signerconfig.FileWalletEnabled) {
 		return i18n.NewError(ctx, signermsgs.MsgNoWalletEnabled)
 	}
-	fileWallet, err := filewallet.NewFileWallet(ctx)
+	fileWallet, err := fswallet.NewFilesystemWallet(ctx, fswallet.ReadConfig(signerconfig.FileWalletConfig))
 	if err != nil {
 		return err
 	}

config.md

@@ -21,7 +21,7 @@ nav_order: 2
 
 |Key|Description|Type|Default Value|
 |---|-----------|----|-------------|
-|chainId|Optionally set the Chain ID of the blockchain. Otherwise the Network ID will be queried, and used as the Chain ID in signind|number|`-1`
+|chainId|Optionally set the Chain ID of the blockchain. Otherwise the Network ID will be queried, and used as the Chain ID in signing|number|`-1`
 |connectionTimeout|The maximum amount of time that a connection is allowed to remain with no data transmitted|[`time.Duration`](https://pkg.go.dev/time#Duration)|`30s`
 |expectContinueTimeout|See [ExpectContinueTimeout in the Go docs](https://pkg.go.dev/net/http#Transport)|[`time.Duration`](https://pkg.go.dev/time#Duration)|`1s`
 |headers|Adds custom headers to HTTP requests|`map[string]string`|`<nil>`
@@ -80,6 +80,7 @@ nav_order: 2
 |Key|Description|Type|Default Value|
 |---|-----------|----|-------------|
 |defaultPasswordFile|Optional default password file to use, if one is not specified individually for the key (via metadata, or file extension)|string|`<nil>`
+|disableListener|Disable the filesystem listener that automatically detects the creation of new keystore files|boolean|`<nil>`
 |enabled|Whether the Keystore V3 filesystem wallet is enabled|boolean|`true`
 |path|Path on the filesystem where the metadata files (and/or key files) are located|string|`<nil>`
 |signerCacheSize|Maximum of signing keys to hold in memory|number|`250`
@@ -90,8 +91,11 @@ nav_order: 2
 |Key|Description|Type|Default Value|
 |---|-----------|----|-------------|
 |passwordExt|Optional to use to look up password files, that sit next to the key files directly. Alternative to metadata when you have a password per keystore|string|`<nil>`
-|primaryExt|Extension for the primary file to look up for an address string (can be key file directly, or metadata file)|string|`<nil>`
-|with0xPrefix|When true filenames will be resolved with an 0x prefix|boolean|`<nil>`
+|passwordPath|Optional directory in which to look for the password files, when passwordExt is configured. Default is the wallet directory|string|`<nil>`
+|passwordTrimSpace|Whether to trim leading/trailing whitespace (such as a newline) from the password when loaded from file|boolean|`true`
+|primaryExt|Extension for key/metadata files named by <ADDRESS>.<EXT>|string|`<nil>`
+|primaryMatchRegex|Regular expression run against key/metadata filenames to extract the address (takes precedence over primaryExt)|regexp|`<nil>`
+|with0xPrefix|When true and passwordExt is used, password filenames will be generated with an 0x prefix|boolean|`<nil>`
 
 ## fileWallet.metadata
 

go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	github.com/btcsuite/btcd/btcec/v2 v2.1.3
+	github.com/fsnotify/fsnotify v1.5.4
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/gorilla/mux v1.8.0
 	github.com/hyperledger/firefly-common v0.1.13
@@ -24,7 +25,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect

go.sum

@@ -78,6 +78,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/btcsuite/btcd/btcec/v2 v2.1.3 h1:xM/n3yIhHAhHy04z4i43C8p4ehixJZMsnrVJkgl+MTE=
 github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE=
+github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0 h1:MSskdM4/xJYcFzy0altH/C/xHopifpWzHUi1JeVI34Q=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -104,6 +105,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0=
 github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=