feat: standalone Redis client and HTTP did:web resolution

- Add standalone Redis client support for non-cluster deployments
- Add PEM file key loading via --key-file flag
- Add HTTP-based did:web resolution via --resolve-did-web flag
- Add --public-url and --insecure-did-resolution flags for configuration
- Add debug logging for claim URL fetching

Author: frrist

cmd/server.go

@@ -1,26 +1,36 @@
 package main
 
 import (
+	crypto_ed25519 "crypto/ed25519"
+	"crypto/x509"
 	"encoding/json"
+	"encoding/pem"
 	"fmt"
+	"io"
 	"net/url"
+	"os"
+	"time"
 
+	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipni/go-libipni/maurl"
 	"github.com/ipni/go-libipni/metadata"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multiaddr"
-	"github.com/redis/go-redis/v9"
+	goredis "github.com/redis/go-redis/v9"
+	"github.com/storacha/go-libstoracha/principalresolver"
 	"github.com/storacha/go-ucanto/did"
 	"github.com/storacha/go-ucanto/principal"
 	ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer"
 	"github.com/storacha/go-ucanto/principal/signer"
 	userver "github.com/storacha/go-ucanto/server"
+	"github.com/storacha/go-ucanto/validator"
+	"github.com/urfave/cli/v2"
+
 	"github.com/storacha/indexing-service/pkg/construct"
 	"github.com/storacha/indexing-service/pkg/presets"
-	"github.com/storacha/indexing-service/pkg/principalresolver"
+	"github.com/storacha/indexing-service/pkg/redis"
 	"github.com/storacha/indexing-service/pkg/server"
-	"github.com/urfave/cli/v2"
 )
 
 var serverCmd = &cli.Command{
@@ -42,6 +52,11 @@ var serverCmd = &cli.Command{
 					Aliases: []string{"pk"},
 					Usage:   "base64 encoded private key identity for the server",
 				},
+				&cli.StringFlag{
+					Name:    "key-file",
+					Aliases: []string{"kf"},
+					Usage:   "path to PEM-encoded Ed25519 private key file",
+				},
 				&cli.StringFlag{
 					Name:  "did",
 					Usage: "DID of the server (only needs to be set if different from what is derived from the private key i.e. a did:web DID)",
@@ -83,41 +98,99 @@ var serverCmd = &cli.Command{
 					Name:  "ipni-format-endpoint",
 					Usage: "HTTP endpoint of the IPNI node to use for format announcements (enables endpoint mimicking IPNI).",
 				},
+				&cli.StringSliceFlag{
+					Name:    "public-url",
+					EnvVars: []string{"PUBLIC_URL"},
+					Usage:   "Public URL(s) where the indexing service can be reached (for claim fetching). Can be specified multiple times or comma-separated in env var.",
+				},
+				&cli.StringSliceFlag{
+					Name:    "resolve-did-web",
+					EnvVars: []string{"RESOLVE_DID_WEB"},
+					Usage:   "did:web DIDs to resolve via HTTP (fetches /.well-known/did.json). Can be specified multiple times or comma-separated in env var.",
+				},
+				&cli.BoolFlag{
+					Name:    "insecure-did-resolution",
+					EnvVars: []string{"INSECURE_DID_RESOLUTION"},
+					Usage:   "Use HTTP instead of HTTPS for did:web resolution (for local development), used with --resolve-did-web",
+				},
 			},
 			Action: func(cCtx *cli.Context) error {
+				if cCtx.IsSet("private-key") && cCtx.IsSet("key-file") {
+					return fmt.Errorf("private-key and key-file are mutually exclusive")
+				}
+
 				addr := fmt.Sprintf(":%d", cCtx.Int("port"))
 				var id principal.Signer
 				var err error
 				var opts []server.Option
 
-				if !cCtx.IsSet("private-key") {
+				if cCtx.IsSet("key-file") {
+					// load from PEM file
+					id, err = signerFromPEMFile(cCtx.String("key-file"))
+					if err != nil {
+						return fmt.Errorf("loading key from PEM file: %w", err)
+					}
+				} else if cCtx.IsSet("private-key") {
+					id, err = ed25519.Parse(cCtx.String("private-key"))
+					if err != nil {
+						return fmt.Errorf("parsing server private key: %w", err)
+					}
+				} else {
 					// generate a new private key if one is not provided
 					id, err = ed25519.Generate()
 					if err != nil {
 						return fmt.Errorf("generating server private key: %w", err)
 					}
-				} else {
-					id, err = ed25519.Parse(cCtx.String("private-key"))
+				}
+
+				// wrap with custom DID if specified
+				if cCtx.String("did") != "" {
+					customDID, err := did.Parse(cCtx.String("did"))
 					if err != nil {
-						return fmt.Errorf("parsing server private key: %w", err)
+						return fmt.Errorf("parsing server DID: %w", err)
 					}
-					if cCtx.String("did") != "" {
-						did, err := did.Parse(cCtx.String("did"))
-						if err != nil {
-							return fmt.Errorf("parsing server DID: %w", err)
-						}
-						id, err = signer.Wrap(id, did)
-						if err != nil {
-							return fmt.Errorf("wrapping server DID: %w", err)
-						}
+					id, err = signer.Wrap(id, customDID)
+					if err != nil {
+						return fmt.Errorf("wrapping server DID: %w", err)
 					}
 				}
 
 				opts = append(opts, server.WithIdentity(id))
 
-				presolv, err := principalresolver.New(presets.PrincipalMapping)
-				if err != nil {
-					return fmt.Errorf("creating principal resolver: %w", err)
+				// Create principal resolver
+				var presolv validator.PrincipalResolver
+				resolveDIDs := cCtx.StringSlice("resolve-did-web")
+				if len(resolveDIDs) > 0 {
+					// Use HTTP-based resolution for specified DIDs
+					var webDIDs []did.DID
+					for _, d := range resolveDIDs {
+						parsed, err := did.Parse(d)
+						if err != nil {
+							return fmt.Errorf("parsing resolve-did-web DID %s: %w", d, err)
+						}
+						webDIDs = append(webDIDs, parsed)
+					}
+
+					var rslvOpts []principalresolver.Option
+					if cCtx.Bool("insecure-did-resolution") {
+						rslvOpts = append(rslvOpts, principalresolver.InsecureResolution())
+					}
+
+					httpResolver, err := principalresolver.NewHTTPResolver(webDIDs, rslvOpts...)
+					if err != nil {
+						return fmt.Errorf("creating HTTP principal resolver: %w", err)
+					}
+					presolv, err = principalresolver.NewCachedResolver(httpResolver, 24*time.Hour)
+					if err != nil {
+						return fmt.Errorf("creating cached resolver: %w", err)
+					}
+				} else {
+					// Fall back to static mapping from presets
+					staticResolver, err := principalresolver.NewMapResolver(presets.PrincipalMapping)
+					if err != nil {
+						return fmt.Errorf("creating principal resolver: %w", err)
+					}
+					presolv = staticResolver
 				}
 				opts = append(
 					opts,
@@ -133,19 +206,16 @@ var serverCmd = &cli.Command{
 				opts = append(opts, ipniSrvOpts...)
 				var sc construct.ServiceConfig
 				sc.ID = id
-				sc.ProvidersRedis = redis.ClusterOptions{
-					Addrs:    []string{cCtx.String("redis-url")},
-					Password: cCtx.String("redis-passwd"),
-				}
-				sc.ClaimsRedis = redis.ClusterOptions{
-					Addrs:    []string{cCtx.String("redis-url")},
-					Password: cCtx.String("redis-passwd"),
-				}
-				sc.IndexesRedis = redis.ClusterOptions{
-					Addrs:    []string{cCtx.String("redis-url")},
+				sc.IPNIFindURL = cCtx.String("ipni-endpoint")
+				sc.PublicURL = cCtx.StringSlice("public-url")
+
+				// Create standalone Redis client for local development
+				redisOpts := &goredis.Options{
+					Addr:     cCtx.String("redis-url"),
 					Password: cCtx.String("redis-passwd"),
 				}
-				sc.IPNIFindURL = cCtx.String("ipni-endpoint")
+				redisClient := goredis.NewClient(redisOpts)
+				clientAdapter := redis.NewClientAdapter(redisClient)
 
 				if cCtx.String("ipni-fallback-endpoints") != "" {
 					var urls []string
@@ -173,7 +243,13 @@ var serverCmd = &cli.Command{
 				}
 				sc.PrivateKey = privKey
 
-				indexer, err := construct.Construct(sc)
+				logging.SetAllLoggers(logging.LevelInfo)
+				indexer, err := construct.Construct(sc,
+					construct.WithProvidersClient(clientAdapter),
+					construct.WithNoProvidersClient(redisClient),
+					construct.WithClaimsClient(redisClient),
+					construct.WithIndexesClient(redisClient),
+				)
 				if err != nil {
 					return err
 				}
@@ -210,3 +286,48 @@ func ipniOpts(ipniFormatPeerID string, ipniFormatEndpoint string) ([]server.Opti
 		server.WithIPNI(peer.AddrInfo{ID: peerID, Addrs: []multiaddr.Multiaddr{ma}}, metadata.Default.New(metadata.IpfsGatewayHttp{})),
 	}, nil
 }
+
+// signerFromPEMFile loads an Ed25519 private key from a PEM file.
+func signerFromPEMFile(path string) (principal.Signer, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	pemData, err := io.ReadAll(f)
+	if err != nil {
+		return nil, fmt.Errorf("reading private key: %w", err)
+	}
+
+	var privateKey *crypto_ed25519.PrivateKey
+	rest := pemData
+
+	for {
+		block, remaining := pem.Decode(rest)
+		if block == nil {
+			break
+		}
+		rest = remaining
+
+		if block.Type == "PRIVATE KEY" {
+			parsedKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse PKCS#8 private key: %w", err)
+			}
+
+			key, ok := parsedKey.(crypto_ed25519.PrivateKey)
+			if !ok {
+				return nil, fmt.Errorf("the parsed key is not an ED25519 private key")
+			}
+			privateKey = &key
+			break
+		}
+	}
+
+	if privateKey == nil {
+		return nil, fmt.Errorf("could not find a PRIVATE KEY block in the PEM file")
+	}
+
+	return ed25519.FromRaw(*privateKey)
+}

go.mod

@@ -145,6 +145,7 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect

go.sum

@@ -550,6 +550,8 @@ github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgr
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=

pkg/service/service.go

@@ -160,8 +160,13 @@ func (is *IndexingService) jobHandler(mhCtx context.Context, j job, spawn func(j
 
 			// fetch (from cache or url) the actual content claim
 			claimCid := hasClaimCid.GetClaim()
+			log.Infow("query: fetching claim", "claimCid", claimCid, "providerId", result.Provider.ID, "numAddrs", len(result.Provider.Addrs))
+			for i, addr := range result.Provider.Addrs {
+				log.Infow("query: provider address", "index", i, "addr", addr.String())
+			}
 			url, err := fetchClaimURL(*result.Provider, claimCid)
 			if err != nil {
+				log.Errorw("query: failed to build claim URL", "error", err)
 				telemetry.Error(s, err, "building claim URL")
 				return err
 			}