diff --git a/cmd/query.go b/cmd/query.go index f16578d..73d03a4 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -40,6 +40,11 @@ var queryCmd = &cli.Command{ Usage: "type of query to perform ['standard' | 'location' | 'index_or_location']", Value: "standard", }, + &cli.StringFlag{ + Name: "delegation", + Aliases: []string{"d"}, + Usage: "a delegation allowing the indexer to fetch content from the space", + }, }, Action: func(cCtx *cli.Context) error { serviceURL, err := url.Parse(cCtx.String("url")) @@ -91,10 +96,20 @@ var queryCmd = &cli.Command{ } } + var delegations []delegation.Delegation + if cCtx.IsSet("delegation") { + d, err := delegation.Parse(cCtx.String("delegation")) + if err != nil { + return fmt.Errorf("parsing delegation: %w", err) + } + delegations = append(delegations, d) + } + qr, err := c.QueryClaims(cCtx.Context, types.Query{ - Type: queryType, - Hashes: digests, - Match: types.Match{Subject: spaces}, + Type: queryType, + Hashes: digests, + Match: types.Match{Subject: spaces}, + Delegations: delegations, }) if err != nil { return fmt.Errorf("querying service: %w", err) diff --git a/go.mod b/go.mod index 6b1ff1d..086a60b 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/redis/go-redis/extra/redisotel/v9 v9.10.0 github.com/redis/go-redis/v9 v9.10.0 - github.com/storacha/go-libstoracha v0.3.1 + github.com/storacha/go-libstoracha v0.3.2 github.com/storacha/go-ucanto v0.6.5 github.com/stretchr/testify v1.10.0 github.com/testcontainers/testcontainers-go v0.39.0 diff --git a/go.sum b/go.sum index fc717b0..e8eef91 100644 --- a/go.sum +++ b/go.sum @@ -676,8 +676,8 @@ github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t6 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/storacha/go-libstoracha v0.3.1 h1:cjXEVfEJHdBLBcs2cbk94Gf5xoL/eMxWtY6o6RxnY/0= -github.com/storacha/go-libstoracha v0.3.1/go.mod h1:UF4t2uPwq7vhqqoRWVPnsTsvnWghd8uTJ5WW6QekjVA= +github.com/storacha/go-libstoracha v0.3.2 h1:GH/6Q+lJ/HmjI3/T+3ECqOdblLTYLPtZWqLB5AG6Dus= +github.com/storacha/go-libstoracha v0.3.2/go.mod h1:UF4t2uPwq7vhqqoRWVPnsTsvnWghd8uTJ5WW6QekjVA= github.com/storacha/go-ucanto v0.6.5 h1:mxy1UkJDqszAGe6SkoT0N2SG9YJ62YX7fzU1Pg9lxnA= github.com/storacha/go-ucanto v0.6.5/go.mod h1:O35Ze4x18EWtz3ftRXXd/mTZ+b8OQVjYYrnadJ/xNjg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/pkg/construct/construct.go b/pkg/construct/construct.go index cfb9f60..8263b96 100644 --- a/pkg/construct/construct.go +++ b/pkg/construct/construct.go @@ -25,6 +25,7 @@ import ( "github.com/storacha/go-libstoracha/ipnipublisher/store" "github.com/storacha/go-libstoracha/jobqueue" "github.com/storacha/go-libstoracha/metadata" + ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer" "github.com/storacha/indexing-service/pkg/redis" "github.com/storacha/indexing-service/pkg/service" "github.com/storacha/indexing-service/pkg/service/blobindexlookup" @@ -498,10 +499,19 @@ func Construct(sc ServiceConfig, opts ...Option) (Service, error) { publicAddrInfo.Addrs = append(publicAddrInfo.Addrs, addr) } + skBytes, err := sc.PrivateKey.Raw() + if err != nil { + return nil, err + } + serviceID, err := ed25519.FromRaw(skBytes) + if err != nil { + return nil, err + } + // with concurrency will still get overridden if a different walker setting is used serviceOpts := append([]service.Option{service.WithConcurrency(15)}, cfg.opts...) - s.IndexingService = service.NewIndexingService(blobIndexLookup, claims, publicAddrInfo, providerIndex, serviceOpts...) + s.IndexingService = service.NewIndexingService(serviceID, blobIndexLookup, claims, publicAddrInfo, providerIndex, serviceOpts...) return s, nil } diff --git a/pkg/server/server.go b/pkg/server/server.go index 3ababc6..1949f8d 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -33,6 +33,7 @@ import ( ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer" "github.com/storacha/go-ucanto/principal/signer" "github.com/storacha/go-ucanto/server" + hcmsg "github.com/storacha/go-ucanto/transport/headercar/message" ucanhttp "github.com/storacha/go-ucanto/transport/http" "github.com/storacha/indexing-service/pkg/build" "github.com/storacha/indexing-service/pkg/service/contentclaims" @@ -291,12 +292,36 @@ func GetClaimsHandler(service types.Querier) http.HandlerFunc { spaces = append(spaces, space) } + var dlgs []delegation.Delegation + agentMsgHeader := r.Header.Get(hcmsg.HeaderName) + if agentMsgHeader != "" { + msg, err := hcmsg.DecodeHeader(agentMsgHeader) + if err != nil { + http.Error(w, fmt.Sprintf("decoding agent message: %s", err.Error()), http.StatusBadRequest) + return + } + + for _, root := range msg.Invocations() { + dlg, ok, err := msg.Invocation(root) + if err != nil { + log.Warnf("failed to extract delegation from agent message: %w", err) + continue + } + if !ok { + log.Warnf("delegation not found in agent message: %s", root.String()) + continue + } + dlgs = append(dlgs, dlg) + } + } + qr, err := service.Query(ctx, types.Query{ Type: queryType, Hashes: hashes, Match: types.Match{ Subject: spaces, }, + Delegations: dlgs, }) if err != nil { http.Error(w, fmt.Sprintf("processing query: %s", err.Error()), http.StatusInternalServerError) diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 11eac59..702d2ea 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -21,11 +21,16 @@ import ( "github.com/multiformats/go-multihash" "github.com/storacha/go-libstoracha/blobindex" "github.com/storacha/go-libstoracha/bytemap" + "github.com/storacha/go-libstoracha/capabilities/space/content" "github.com/storacha/go-libstoracha/digestutil" "github.com/storacha/go-libstoracha/testutil" "github.com/storacha/go-ucanto/core/delegation" + "github.com/storacha/go-ucanto/core/invocation" + "github.com/storacha/go-ucanto/core/message" "github.com/storacha/go-ucanto/did" "github.com/storacha/go-ucanto/principal/signer" + hcmsg "github.com/storacha/go-ucanto/transport/headercar/message" + "github.com/storacha/go-ucanto/ucan" "github.com/storacha/indexing-service/pkg/internal/link" "github.com/storacha/indexing-service/pkg/service/contentclaims" "github.com/storacha/indexing-service/pkg/service/queryresult" @@ -219,6 +224,63 @@ func TestGetClaimsHandler(t *testing.T) { require.Equal(t, http.StatusOK, res.StatusCode) }) + t.Run("authorized retrieval from space", func(t *testing.T) { + mockService := types.NewMockService(t) + + randomHash := testutil.RandomMultihash(t) + randomSubject := testutil.RandomPrincipal(t).DID() + + dlg, err := delegation.Delegate( + testutil.Alice, + testutil.Service, + []ucan.Capability[ucan.NoCaveats]{ + ucan.NewCapability(content.RetrieveAbility, randomSubject.String(), ucan.NoCaveats{}), + }, + ) + require.NoError(t, err) + + claims := map[cid.Cid]delegation.Delegation{} + indexes := bytemap.NewByteMap[types.EncodedContextID, blobindex.ShardedDagIndexView](-1) + queryResult := testutil.Must(queryresult.Build(claims, indexes))(t) + mockService.EXPECT().Query(mock.Anything, mock.AnythingOfType("Query")).Return(queryResult, nil) + + svr := httptest.NewServer(GetClaimsHandler(mockService)) + defer svr.Close() + + url := fmt.Sprintf("%s/claims?multihash=%s&spaces=%s", svr.URL, digestutil.Format(randomHash), randomSubject.String()) + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + + msg, err := message.Build([]invocation.Invocation{dlg}, nil) + require.NoError(t, err) + + headerValue, err := hcmsg.EncodeHeader(msg) + require.NoError(t, err) + + req.Header.Set(hcmsg.HeaderName, headerValue) + + res, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.Equal(t, http.StatusOK, res.StatusCode) + }) + + t.Run("invalid "+hcmsg.HeaderName, func(t *testing.T) { + mockService := types.NewMockService(t) + + svr := httptest.NewServer(GetClaimsHandler(mockService)) + defer svr.Close() + + url := fmt.Sprintf("%s/claims?multihash=%s", svr.URL, digestutil.Format(testutil.RandomMultihash(t))) + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + + req.Header.Set(hcmsg.HeaderName, "NOT AN AGENT MESSAGE") + + res, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.Equal(t, http.StatusBadRequest, res.StatusCode) + }) + t.Run("invalid space", func(t *testing.T) { mockService := types.NewMockService(t) diff --git a/pkg/service/blobindexlookup/cachinglookup.go b/pkg/service/blobindexlookup/cachinglookup.go index 2241340..8e6388b 100644 --- a/pkg/service/blobindexlookup/cachinglookup.go +++ b/pkg/service/blobindexlookup/cachinglookup.go @@ -4,11 +4,9 @@ import ( "context" "errors" "fmt" - "net/url" "github.com/ipni/go-libipni/find/model" "github.com/storacha/go-libstoracha/blobindex" - "github.com/storacha/go-libstoracha/metadata" "github.com/storacha/indexing-service/pkg/service/providercacher" "github.com/storacha/indexing-service/pkg/types" ) @@ -30,7 +28,7 @@ func WithCache(blobIndexLookup BlobIndexLookup, shardedDagIndexCache types.Shard } } -func (b *cachingLookup) Find(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, fetchURL *url.URL, rng *metadata.Range) (blobindex.ShardedDagIndexView, error) { +func (b *cachingLookup) Find(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, req types.RetrievalRequest) (blobindex.ShardedDagIndexView, error) { // attempt to read index from cache and return it if succesful index, err := b.shardDagIndexCache.Get(ctx, contextID) if err == nil { @@ -43,7 +41,7 @@ func (b *cachingLookup) Find(ctx context.Context, contextID types.EncodedContext } // attempt to fetch the index from the underlying blob index lookup - index, err = b.blobIndexLookup.Find(ctx, contextID, provider, fetchURL, rng) + index, err = b.blobIndexLookup.Find(ctx, contextID, provider, req) if err != nil { return nil, fmt.Errorf("fetching underlying index: %w", err) } diff --git a/pkg/service/blobindexlookup/cachinglookup_test.go b/pkg/service/blobindexlookup/cachinglookup_test.go index 5d8e678..32f83e6 100644 --- a/pkg/service/blobindexlookup/cachinglookup_test.go +++ b/pkg/service/blobindexlookup/cachinglookup_test.go @@ -4,12 +4,10 @@ import ( "context" "errors" "fmt" - "net/url" "testing" "github.com/ipni/go-libipni/find/model" "github.com/storacha/go-libstoracha/blobindex" - "github.com/storacha/go-libstoracha/metadata" "github.com/storacha/go-libstoracha/testutil" "github.com/storacha/indexing-service/pkg/service/blobindexlookup" "github.com/storacha/indexing-service/pkg/service/providercacher" @@ -124,7 +122,8 @@ func TestWithCache__Find(t *testing.T) { // Create ClaimLookup instance cl := blobindexlookup.WithCache(lookup, mockStore, providerCacher) - index, err := cl.Find(context.Background(), tc.contextID, provider, testutil.TestURL, nil) + req := types.NewRetrievalRequest(testutil.TestURL, nil, nil) + index, err := cl.Find(context.Background(), tc.contextID, provider, req) if tc.expectedErr != nil { require.EqualError(t, err, tc.expectedErr.Error()) } else { @@ -181,7 +180,7 @@ type mockBlobIndexLookup struct { err error } -func (m *mockBlobIndexLookup) Find(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, fetchURL *url.URL, rng *metadata.Range) (blobindex.ShardedDagIndexView, error) { +func (m *mockBlobIndexLookup) Find(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, req types.RetrievalRequest) (blobindex.ShardedDagIndexView, error) { return m.index, m.err } diff --git a/pkg/service/blobindexlookup/interface.go b/pkg/service/blobindexlookup/interface.go index 417c7d8..e3e9d15 100644 --- a/pkg/service/blobindexlookup/interface.go +++ b/pkg/service/blobindexlookup/interface.go @@ -2,11 +2,9 @@ package blobindexlookup import ( "context" - "net/url" "github.com/ipni/go-libipni/find/model" "github.com/storacha/go-libstoracha/blobindex" - "github.com/storacha/go-libstoracha/metadata" "github.com/storacha/indexing-service/pkg/types" ) @@ -18,5 +16,5 @@ type BlobIndexLookup interface { // 3. return the index // 4. asyncronously, add records to the ProviderStore from the parsed blob index so that we can avoid future queries to IPNI for // other multihashes in the index - Find(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, fetchURL *url.URL, rng *metadata.Range) (blobindex.ShardedDagIndexView, error) + Find(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, req types.RetrievalRequest) (blobindex.ShardedDagIndexView, error) } diff --git a/pkg/service/blobindexlookup/mock_BlobIndexLookup.go b/pkg/service/blobindexlookup/mock_BlobIndexLookup.go index df165ce..47518a5 100644 --- a/pkg/service/blobindexlookup/mock_BlobIndexLookup.go +++ b/pkg/service/blobindexlookup/mock_BlobIndexLookup.go @@ -6,11 +6,9 @@ package blobindexlookup import ( "context" - "net/url" "github.com/ipni/go-libipni/find/model" "github.com/storacha/go-libstoracha/blobindex" - "github.com/storacha/go-libstoracha/metadata" "github.com/storacha/indexing-service/pkg/types" mock "github.com/stretchr/testify/mock" ) @@ -43,8 +41,8 @@ func (_m *MockBlobIndexLookup) EXPECT() *MockBlobIndexLookup_Expecter { } // Find provides a mock function for the type MockBlobIndexLookup -func (_mock *MockBlobIndexLookup) Find(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, fetchURL *url.URL, rng *metadata.Range) (blobindex.ShardedDagIndexView, error) { - ret := _mock.Called(ctx, contextID, provider, fetchURL, rng) +func (_mock *MockBlobIndexLookup) Find(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, req types.RetrievalRequest) (blobindex.ShardedDagIndexView, error) { + ret := _mock.Called(ctx, contextID, provider, req) if len(ret) == 0 { panic("no return value specified for Find") @@ -52,18 +50,18 @@ func (_mock *MockBlobIndexLookup) Find(ctx context.Context, contextID types.Enco var r0 blobindex.ShardedDagIndexView var r1 error - if returnFunc, ok := ret.Get(0).(func(context.Context, types.EncodedContextID, model.ProviderResult, *url.URL, *metadata.Range) (blobindex.ShardedDagIndexView, error)); ok { - return returnFunc(ctx, contextID, provider, fetchURL, rng) + if returnFunc, ok := ret.Get(0).(func(context.Context, types.EncodedContextID, model.ProviderResult, types.RetrievalRequest) (blobindex.ShardedDagIndexView, error)); ok { + return returnFunc(ctx, contextID, provider, req) } - if returnFunc, ok := ret.Get(0).(func(context.Context, types.EncodedContextID, model.ProviderResult, *url.URL, *metadata.Range) blobindex.ShardedDagIndexView); ok { - r0 = returnFunc(ctx, contextID, provider, fetchURL, rng) + if returnFunc, ok := ret.Get(0).(func(context.Context, types.EncodedContextID, model.ProviderResult, types.RetrievalRequest) blobindex.ShardedDagIndexView); ok { + r0 = returnFunc(ctx, contextID, provider, req) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(blobindex.ShardedDagIndexView) } } - if returnFunc, ok := ret.Get(1).(func(context.Context, types.EncodedContextID, model.ProviderResult, *url.URL, *metadata.Range) error); ok { - r1 = returnFunc(ctx, contextID, provider, fetchURL, rng) + if returnFunc, ok := ret.Get(1).(func(context.Context, types.EncodedContextID, model.ProviderResult, types.RetrievalRequest) error); ok { + r1 = returnFunc(ctx, contextID, provider, req) } else { r1 = ret.Error(1) } @@ -79,13 +77,12 @@ type MockBlobIndexLookup_Find_Call struct { // - ctx context.Context // - contextID types.EncodedContextID // - provider model.ProviderResult -// - fetchURL *url.URL -// - rng *metadata.Range -func (_e *MockBlobIndexLookup_Expecter) Find(ctx interface{}, contextID interface{}, provider interface{}, fetchURL interface{}, rng interface{}) *MockBlobIndexLookup_Find_Call { - return &MockBlobIndexLookup_Find_Call{Call: _e.mock.On("Find", ctx, contextID, provider, fetchURL, rng)} +// - req types.RetrievalRequest +func (_e *MockBlobIndexLookup_Expecter) Find(ctx interface{}, contextID interface{}, provider interface{}, req interface{}) *MockBlobIndexLookup_Find_Call { + return &MockBlobIndexLookup_Find_Call{Call: _e.mock.On("Find", ctx, contextID, provider, req)} } -func (_c *MockBlobIndexLookup_Find_Call) Run(run func(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, fetchURL *url.URL, rng *metadata.Range)) *MockBlobIndexLookup_Find_Call { +func (_c *MockBlobIndexLookup_Find_Call) Run(run func(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, req types.RetrievalRequest)) *MockBlobIndexLookup_Find_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 context.Context if args[0] != nil { @@ -99,20 +96,15 @@ func (_c *MockBlobIndexLookup_Find_Call) Run(run func(ctx context.Context, conte if args[2] != nil { arg2 = args[2].(model.ProviderResult) } - var arg3 *url.URL + var arg3 types.RetrievalRequest if args[3] != nil { - arg3 = args[3].(*url.URL) - } - var arg4 *metadata.Range - if args[4] != nil { - arg4 = args[4].(*metadata.Range) + arg3 = args[3].(types.RetrievalRequest) } run( arg0, arg1, arg2, arg3, - arg4, ) }) return _c @@ -123,7 +115,7 @@ func (_c *MockBlobIndexLookup_Find_Call) Return(shardedDagIndexView blobindex.Sh return _c } -func (_c *MockBlobIndexLookup_Find_Call) RunAndReturn(run func(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, fetchURL *url.URL, rng *metadata.Range) (blobindex.ShardedDagIndexView, error)) *MockBlobIndexLookup_Find_Call { +func (_c *MockBlobIndexLookup_Find_Call) RunAndReturn(run func(ctx context.Context, contextID types.EncodedContextID, provider model.ProviderResult, req types.RetrievalRequest) (blobindex.ShardedDagIndexView, error)) *MockBlobIndexLookup_Find_Call { _c.Call.Return(run) return _c } diff --git a/pkg/service/blobindexlookup/simplelookup.go b/pkg/service/blobindexlookup/simplelookup.go index 622fbb2..46876ff 100644 --- a/pkg/service/blobindexlookup/simplelookup.go +++ b/pkg/service/blobindexlookup/simplelookup.go @@ -2,15 +2,21 @@ package blobindexlookup import ( "context" + "errors" "fmt" "io" "net/http" - "net/url" "strconv" "github.com/ipni/go-libipni/find/model" "github.com/storacha/go-libstoracha/blobindex" - "github.com/storacha/go-libstoracha/metadata" + rclient "github.com/storacha/go-ucanto/client/retrieval" + "github.com/storacha/go-ucanto/core/dag/blockstore" + "github.com/storacha/go-ucanto/core/delegation" + "github.com/storacha/go-ucanto/core/invocation" + "github.com/storacha/go-ucanto/core/receipt" + "github.com/storacha/go-ucanto/core/result" + fdm "github.com/storacha/go-ucanto/core/result/failure/datamodel" "github.com/storacha/indexing-service/pkg/types" ) @@ -25,27 +31,108 @@ func NewBlobIndexLookup(httpClient *http.Client) BlobIndexLookup { } // Find fetches the blob index from the given fetchURL -func (s *simpleLookup) Find(ctx context.Context, _ types.EncodedContextID, _ model.ProviderResult, fetchURL *url.URL, rng *metadata.Range) (blobindex.ShardedDagIndexView, error) { +func (s *simpleLookup) Find(ctx context.Context, _ types.EncodedContextID, result model.ProviderResult, request types.RetrievalRequest) (blobindex.ShardedDagIndexView, error) { + var body io.ReadCloser + if request.Auth != nil { + // If retrieval authorization details were provided, make a UCAN authorized + // retrieval request. + b, err := doAuthorizedRetrieval(ctx, s.httpClient, request) + if err != nil { + return nil, fmt.Errorf("executing authorized retrieval: %w", err) + } + body = b + } else { + // Otherwise, attempt a legacy public retrieval with no authorization. + b, err := doPublicRetrieval(ctx, s.httpClient, request) + if err != nil { + return nil, fmt.Errorf("executing public retrieval: %w", err) + } + body = b + } + defer body.Close() + return blobindex.Extract(body) +} + +func doAuthorizedRetrieval(ctx context.Context, httpClient *http.Client, request types.RetrievalRequest) (io.ReadCloser, error) { + headers := http.Header{} + if request.Range != nil { + if request.Range.Length != nil { + headers.Set("Range", fmt.Sprintf("bytes=%d-%d", request.Range.Offset, request.Range.Offset+*request.Range.Length-1)) + } else { + headers.Set("Range", fmt.Sprintf("bytes=%d-", request.Range.Offset)) + } + } + + conn, err := rclient.NewConnection( + request.Auth.Audience, + request.URL, + rclient.WithClient(httpClient), + rclient.WithHeaders(headers), + ) + if err != nil { + return nil, err + } + + iss, aud, cap := request.Auth.Issuer, request.Auth.Audience, request.Auth.Capability + inv, err := invocation.Invoke(iss, aud, cap, delegation.WithProof(request.Auth.Proofs...)) + if err != nil { + return nil, err + } + + xres, hres, err := rclient.Execute(ctx, inv, conn) + if err != nil { + return nil, fmt.Errorf("executing retrieval invocation: %w", err) + } + + rcptLink, ok := xres.Get(inv.Link()) + if !ok { + return nil, errors.New("execution response did not contain receipt for invocation") + } + + bs, err := blockstore.NewBlockReader(blockstore.WithBlocksIterator(xres.Blocks())) + if err != nil { + return nil, fmt.Errorf("adding blocks to reader: %w", err) + } + + rcpt, err := receipt.NewAnyReceipt(rcptLink, bs) + if err != nil { + return nil, fmt.Errorf("creating receipt: %w", err) + } + + _, x := result.Unwrap(rcpt.Out()) + if x != nil { + fail := fdm.Bind(x) + name := "Unnamed" + if fail.Name != nil { + name = *fail.Name + } + return nil, fmt.Errorf("execution resulted in failure: %s: %s", name, fail.Message) + } + + return hres.Body(), nil +} + +func doPublicRetrieval(ctx context.Context, httpClient *http.Client, request types.RetrievalRequest) (io.ReadCloser, error) { // attempt to fetch the index from provided url - req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL.String(), nil) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, request.URL.String(), nil) if err != nil { return nil, fmt.Errorf("constructing request: %w", err) } + rng := request.Range if rng != nil { rangeHeader := fmt.Sprintf("bytes=%d-", rng.Offset) if rng.Length != nil { rangeHeader += strconv.FormatUint(rng.Offset+*rng.Length-1, 10) } - req.Header.Set("Range", rangeHeader) + httpReq.Header.Set("Range", rangeHeader) } - resp, err := s.httpClient.Do(req) + resp, err := httpClient.Do(httpReq) if err != nil { return nil, fmt.Errorf("failed to fetch index: %w", err) } if resp.StatusCode < 200 || resp.StatusCode > 299 { body, _ := io.ReadAll(resp.Body) - - return nil, fmt.Errorf("failure response fetching index. status: %s, message: %s, url: %s", resp.Status, string(body), fetchURL.String()) + return nil, fmt.Errorf("failure response fetching index. status: %s, message: %s, url: %s", resp.Status, string(body), request.URL.String()) } - return blobindex.Extract(resp.Body) + return resp.Body, nil } diff --git a/pkg/service/blobindexlookup/simplelookup_test.go b/pkg/service/blobindexlookup/simplelookup_test.go index f887acb..ad09f1d 100644 --- a/pkg/service/blobindexlookup/simplelookup_test.go +++ b/pkg/service/blobindexlookup/simplelookup_test.go @@ -13,9 +13,19 @@ import ( cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/storacha/go-libstoracha/blobindex" + "github.com/storacha/go-libstoracha/capabilities/space/content" "github.com/storacha/go-libstoracha/metadata" "github.com/storacha/go-libstoracha/testutil" + "github.com/storacha/go-ucanto/core/invocation" + "github.com/storacha/go-ucanto/core/receipt/fx" + "github.com/storacha/go-ucanto/core/result" + "github.com/storacha/go-ucanto/core/result/failure" + "github.com/storacha/go-ucanto/server" + "github.com/storacha/go-ucanto/server/retrieval" + ucan_http "github.com/storacha/go-ucanto/transport/http" + "github.com/storacha/go-ucanto/ucan" "github.com/storacha/indexing-service/pkg/service/blobindexlookup" + "github.com/storacha/indexing-service/pkg/types" "github.com/stretchr/testify/require" ) @@ -25,11 +35,13 @@ func TestBlobIndexLookup__Find(t *testing.T) { _, index := testutil.RandomShardedDagIndexView(t, 32) indexBytes := testutil.Must(io.ReadAll(testutil.Must(index.Archive())(t)))(t) indexEncodedLength := uint64(len(indexBytes)) + // sample error testCases := []struct { name string handler http.HandlerFunc rngHeader *metadata.Range + auth *types.RetrievalAuth expectedErr error expectedIndex blobindex.ShardedDagIndexView }{ @@ -69,6 +81,57 @@ func TestBlobIndexLookup__Find(t *testing.T) { rngHeader: &metadata.Range{Offset: 10, Length: &indexEncodedLength}, expectedIndex: index, }, + { + name: "authorized retrieval", + handler: func(w http.ResponseWriter, r *http.Request) { + srv, err := retrieval.NewServer( + testutil.Service, + retrieval.WithServiceMethod( + content.RetrieveAbility, + retrieval.Provide( + content.Retrieve, + func(ctx context.Context, capability ucan.Capability[content.RetrieveCaveats], invocation invocation.Invocation, context server.InvocationContext, request retrieval.Request) (result.Result[content.RetrieveOk, failure.IPLDBuilderFailure], fx.Effects, retrieval.Response, error) { + res := result.Ok[content.RetrieveOk, failure.IPLDBuilderFailure](content.RetrieveOk{}) + resp := retrieval.NewResponse(http.StatusOK, nil, io.NopCloser(bytes.NewReader(indexBytes))) + return res, nil, resp, nil + }, + ), + ), + ) + require.NoError(t, err) + res, err := srv.Request(r.Context(), ucan_http.NewRequest(r.Body, r.Header)) + require.NoError(t, err) + for key, values := range res.Headers() { + for _, value := range values { + w.Header().Add(key, value) + } + } + if res.Status() > 0 { + w.WriteHeader(res.Status()) + } + _, err = io.Copy(w, res.Body()) + require.NoError(t, err) + }, + rngHeader: &metadata.Range{Offset: 10, Length: &indexEncodedLength}, + auth: &types.RetrievalAuth{ + Issuer: testutil.Service, + Audience: testutil.Service, + Capability: ucan.NewCapability[ucan.CaveatBuilder]( + content.RetrieveAbility, + testutil.Service.DID().String(), + content.RetrieveCaveats{ + Blob: content.BlobDigest{ + Digest: cid.Hash(), + }, + Range: content.Range{ + Start: 10, + End: indexEncodedLength - 1, + }, + }, + ), + }, + expectedIndex: index, + }, } // Run test cases for _, tc := range testCases { @@ -77,7 +140,8 @@ func TestBlobIndexLookup__Find(t *testing.T) { defer func() { testServer.Close() }() // Create BlobIndexLookup instance cl := blobindexlookup.NewBlobIndexLookup(testServer.Client()) - index, err := cl.Find(context.Background(), cid.Bytes(), provider, testutil.Must(url.Parse(testServer.URL))(t), tc.rngHeader) + req := types.NewRetrievalRequest(testutil.Must(url.Parse(testServer.URL))(t), tc.rngHeader, tc.auth) + index, err := cl.Find(context.Background(), cid.Bytes(), provider, req) if tc.expectedErr != nil { require.ErrorContains(t, err, tc.expectedErr.Error()) } else { diff --git a/pkg/service/service.go b/pkg/service/service.go index d5c8389..e95e824 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -10,6 +10,7 @@ import ( "sync" "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipni/go-libipni/find/model" @@ -19,11 +20,20 @@ import ( "github.com/multiformats/go-multicodec" "github.com/multiformats/go-multihash" "github.com/storacha/go-libstoracha/advertisement" + "github.com/storacha/go-libstoracha/capabilities/access" "github.com/storacha/go-libstoracha/capabilities/assert" + "github.com/storacha/go-libstoracha/capabilities/blob" + "github.com/storacha/go-libstoracha/capabilities/space/content" "github.com/storacha/go-libstoracha/metadata" + "github.com/storacha/go-ucanto/client" "github.com/storacha/go-ucanto/core/delegation" + "github.com/storacha/go-ucanto/core/invocation" "github.com/storacha/go-ucanto/core/iterable" + "github.com/storacha/go-ucanto/core/result" + "github.com/storacha/go-ucanto/did" "github.com/storacha/go-ucanto/principal/ed25519/verifier" + "github.com/storacha/go-ucanto/transport/http" + "github.com/storacha/go-ucanto/ucan" "github.com/storacha/go-ucanto/validator" "go.opentelemetry.io/otel/attribute" @@ -48,10 +58,13 @@ const ( blobCIDUrlPlaceholder = "{blobCID}" ) +var log = logging.Logger("service") + var ErrUnrecognizedClaim = errors.New("unrecognized claim type") // IndexingService implements read/write logic for indexing data with IPNI, content claims, sharded dag indexes, and a cache layer type IndexingService struct { + id ucan.Signer blobIndexLookup blobindexlookup.BlobIndexLookup claims contentclaims.Service providerIndex providerindex.ProviderIndex @@ -216,7 +229,39 @@ func (is *IndexingService) jobHandler(mhCtx context.Context, j job, spawn func(j } s.AddEvent("fetching index") - index, err := is.blobIndexLookup.Find(mhCtx, result.ContextID, *j.indexProviderRecord, url, typedProtocol.Range) + var auth *types.RetrievalAuth + match, err := assert.Location.Match(validator.NewSource(claim.Capabilities()[0], claim)) + if err != nil { + return fmt.Errorf("failed to match claim to location commitment: %w", err) + } + lcCaveats := match.Value().Nb() + space := lcCaveats.Space + dlgs := state.Access().q.Delegations + // Authorized retrieval requires a space in the location claim, a + // delegation for the retrieval, and an absolute byte range to extract. + if space != did.Undef && len(dlgs) > 0 && lcCaveats.Range != nil && lcCaveats.Range.Length != nil { + var proofs []delegation.Proof + for _, d := range dlgs { + for _, c := range d.Capabilities() { + if c.Can() == content.Retrieve.Can() && c.With() == space.String() { + proofs = append(proofs, delegation.FromDelegation(d)) + } + } + } + if len(proofs) > 0 { + cap := content.Retrieve.New(space.String(), content.RetrieveCaveats{ + Blob: content.BlobDigest{Digest: lcCaveats.Content.Hash()}, + Range: content.Range{ + Start: lcCaveats.Range.Offset, + End: lcCaveats.Range.Offset + *lcCaveats.Range.Length - 1, + }, + }) + a := types.NewRetrievalAuth(is.id, claim.Issuer(), cap, proofs) + auth = &a + } + } + req := types.NewRetrievalRequest(url, typedProtocol.Range, auth) + index, err := is.blobIndexLookup.Find(mhCtx, result.ContextID, *j.indexProviderRecord, req) if err != nil { telemetry.Error(s, err, "fetching index blob") return err @@ -343,7 +388,7 @@ func (is *IndexingService) Cache(ctx context.Context, provider peer.AddrInfo, cl // The service should lookup the index cid location claim, and fetch the ShardedDagIndexView, then use the hashes inside // to assemble all the multihashes in the index advertisement func (is *IndexingService) Publish(ctx context.Context, claim delegation.Delegation) error { - return Publish(ctx, is.blobIndexLookup, is.claims, is.providerIndex, is.provider, claim) + return Publish(ctx, is.id, is.blobIndexLookup, is.claims, is.providerIndex, is.provider, claim) } // Option configures an IndexingService @@ -357,13 +402,14 @@ func WithConcurrency(concurrency int) Option { } // NewIndexingService returns a new indexing service -func NewIndexingService(blobIndexLookup blobindexlookup.BlobIndexLookup, claims contentclaims.Service, publicAddrInfo peer.AddrInfo, providerIndex providerindex.ProviderIndex, options ...Option) *IndexingService { +func NewIndexingService(id ucan.Signer, blobIndexLookup blobindexlookup.BlobIndexLookup, claims contentclaims.Service, publicAddrInfo peer.AddrInfo, providerIndex providerindex.ProviderIndex, options ...Option) *IndexingService { provider := peer.AddrInfo{ID: publicAddrInfo.ID} for _, addr := range publicAddrInfo.Addrs { claimSuffix, _ := multiaddr.NewMultiaddr("/http-path/" + url.PathEscape("claim/"+ClaimUrlPlaceholder)) provider.Addrs = append(provider.Addrs, multiaddr.Join(addr, claimSuffix)) } is := &IndexingService{ + id: id, blobIndexLookup: blobIndexLookup, claims: claims, provider: provider, @@ -448,7 +494,7 @@ func cacheLocationCommitment(ctx context.Context, claims contentclaims.Service, return nil } -func Publish(ctx context.Context, blobIndex blobindexlookup.BlobIndexLookup, claims contentclaims.Service, provIndex providerindex.ProviderIndex, provider peer.AddrInfo, claim delegation.Delegation) error { +func Publish(ctx context.Context, id ucan.Signer, blobIndex blobindexlookup.BlobIndexLookup, claims contentclaims.Service, provIndex providerindex.ProviderIndex, provider peer.AddrInfo, claim delegation.Delegation) error { ctx, s := telemetry.StartSpan(ctx, "IndexingService.Publish") defer s.End() @@ -462,7 +508,7 @@ func Publish(ctx context.Context, blobIndex blobindexlookup.BlobIndexLookup, cla return publishEqualsClaim(ctx, claims, provIndex, provider, claim) case assert.IndexAbility: s.SetAttributes(attribute.KeyValue{Key: "claim", Value: attribute.StringValue("assert/index")}) - return publishIndexClaim(ctx, blobIndex, claims, provIndex, provider, claim) + return publishIndexClaim(ctx, id, blobIndex, claims, provIndex, provider, claim) default: return ErrUnrecognizedClaim } @@ -505,7 +551,7 @@ func publishEqualsClaim(ctx context.Context, claims contentclaims.Service, provI return nil } -func publishIndexClaim(ctx context.Context, blobIndex blobindexlookup.BlobIndexLookup, claims contentclaims.Service, provIndex providerindex.ProviderIndex, provider peer.AddrInfo, claim delegation.Delegation) error { +func publishIndexClaim(ctx context.Context, id ucan.Signer, blobIndex blobindexlookup.BlobIndexLookup, claims contentclaims.Service, provIndex providerindex.ProviderIndex, provider peer.AddrInfo, claim delegation.Delegation) error { capability := claim.Capabilities()[0] nb, rerr := assert.IndexCaveatsReader.Read(capability.Nb()) if rerr != nil { @@ -531,7 +577,7 @@ func publishIndexClaim(ctx context.Context, blobIndex blobindexlookup.BlobIndexL var idx blobindex.ShardedDagIndex var ferr error for _, r := range results { - idx, ferr = fetchBlobIndex(ctx, blobIndex, claims, nb.Index, r) + idx, ferr = fetchBlobIndex(ctx, id, blobIndex, claims, nb.Index, r, claim) if ferr != nil { continue } @@ -570,7 +616,15 @@ func publishIndexClaim(ctx context.Context, blobIndex blobindexlookup.BlobIndexL return nil } -func fetchBlobIndex(ctx context.Context, blobIndex blobindexlookup.BlobIndexLookup, claims contentclaims.Service, blob ipld.Link, result model.ProviderResult) (blobindex.ShardedDagIndex, error) { +func fetchBlobIndex( + ctx context.Context, + id ucan.Signer, + blobIndex blobindexlookup.BlobIndexLookup, + claims contentclaims.Service, + blobLink ipld.Link, + result model.ProviderResult, + cause invocation.Invocation, // supporting context (typically `assert/index`) +) (blobindex.ShardedDagIndex, error) { meta := metadata.MetadataContext.New() err := meta.UnmarshalBinary(result.Metadata) if err != nil { @@ -584,14 +638,19 @@ func fetchBlobIndex(ctx context.Context, blobIndex blobindexlookup.BlobIndexLook } if lcmeta.Shard != nil { - blob = cidlink.Link{Cid: *lcmeta.Shard} + blobLink = cidlink.Link{Cid: *lcmeta.Shard} } - blobURL, err := fetchRetrievalURL(*result.Provider, link.ToCID(blob)) + blobURL, err := fetchRetrievalURL(*result.Provider, link.ToCID(blobLink)) if err != nil { return nil, fmt.Errorf("building retrieval URL: %w", err) } + aud, err := peerToPrincipal(result.Provider.ID) + if err != nil { + return nil, fmt.Errorf("converting provider peer ID to UCAN principal: %w", err) + } + var wg sync.WaitGroup wg.Add(1) @@ -617,8 +676,30 @@ func fetchBlobIndex(ctx context.Context, blobIndex blobindexlookup.BlobIndexLook } }() + // Try to obtain a retrieval delegation. If this fails then fallback to trying + // to request without auth. Note: it'll fail for non-UCAN authorized retrieval + // nodes (legacy). + dlg, err := requestBlobRetrieveDelegation(ctx, blobURL, id, aud, cause) + if err != nil { + log.Debugw("requesting blob/retrieve delegation", "aud", aud.DID(), "endpoint", blobURL.String(), "err", err) + } + + var auth *types.RetrievalAuth + if dlg != nil { + cap := blob.Retrieve.New( + aud.DID().String(), + blob.RetrieveCaveats{ + Blob: blob.Blob{Digest: link.ToCID(blobLink).Hash()}, + }, + ) + prfs := []delegation.Proof{delegation.FromDelegation(dlg)} + a := types.NewRetrievalAuth(id, aud, cap, prfs) + auth = &a + } + + req := types.NewRetrievalRequest(blobURL, lcmeta.Range, auth) // Note: the ContextID here is of a location commitment provider - idx, err := blobIndex.Find(ctx, result.ContextID, result, blobURL, lcmeta.Range) + idx, err := blobIndex.Find(ctx, result.ContextID, result, req) if err != nil { return nil, fmt.Errorf("fetching index: %w", err) } @@ -660,3 +741,89 @@ func validateLocationCommitment(ctx context.Context, claim delegation.Delegation return auth, nil } + +// peerToPrincipal converts a peer ID into a UCAN principal object. Currently +// supports only ed25519 keys. +func peerToPrincipal(peer peer.ID) (ucan.Principal, error) { + pk, err := peer.ExtractPublicKey() + if err != nil { + return nil, fmt.Errorf("extracting public key from peer ID: %w", err) + } + pubBytes, err := pk.Raw() + if err != nil { + return nil, fmt.Errorf("extracting raw bytes of public key: %w", err) + } + v, err := verifier.FromRaw(pubBytes) + if err != nil { + return nil, fmt.Errorf("decoding raw ed25519 public key: %w", err) + } + return v, nil +} + +// requestBlobRetrieveDelegation obtains a delegation for `blob/retrieve` from a +// node by invoking `access/grant`, using the passed cause invocation as +// evidence that the delegation should be granted. +func requestBlobRetrieveDelegation( + ctx context.Context, + endpoint *url.URL, + issuer ucan.Signer, + audience ucan.Principal, + cause invocation.Invocation, +) (delegation.Delegation, error) { + inv, err := access.Grant.Invoke( + issuer, + audience, + issuer.DID().String(), + access.GrantCaveats{ + Att: []access.CapabilityRequest{{Can: blob.Retrieve.Can()}}, + Cause: cause.Link(), + }, + ) + if err != nil { + return nil, fmt.Errorf("creating %s invocation: %w", access.GrantAbility, err) + } + for b, err := range cause.Export() { + if err != nil { + return nil, fmt.Errorf("exporting blocks: %w", err) + } + if err = inv.Attach(b); err != nil { + return nil, fmt.Errorf("attaching blocks: %w", err) + } + } + + conn, err := client.NewConnection(audience, http.NewChannel(endpoint)) + if err != nil { + return nil, fmt.Errorf("creating connection to %s: %w", audience.DID(), err) + } + + resp, err := client.Execute(ctx, []invocation.Invocation{inv}, conn) + if err != nil { + return nil, fmt.Errorf("executing %s invocation: %w", access.GrantAbility, err) + } + + rcptLink, ok := resp.Get(inv.Link()) + if !ok { + return nil, fmt.Errorf("missing %s receipt: %s", access.GrantAbility, inv.Link()) + } + + rcptReader, err := access.NewGrantReceiptReader() + if err != nil { + return nil, err + } + + rcpt, err := rcptReader.Read(rcptLink, resp.Blocks()) + if err != nil { + return nil, fmt.Errorf("reading %s receipt: %w", access.GrantAbility, err) + } + + return result.MatchResultR2( + rcpt.Out(), + func(o access.GrantOk) (delegation.Delegation, error) { + dlgBytes := o.Delegations.Values[o.Delegations.Keys[0]] + return delegation.Extract(dlgBytes) + }, + func(x access.GrantError) (delegation.Delegation, error) { + return nil, x + }, + ) +} diff --git a/pkg/service/service_test.go b/pkg/service/service_test.go index b0965de..6472eab 100644 --- a/pkg/service/service_test.go +++ b/pkg/service/service_test.go @@ -1,10 +1,12 @@ package service import ( - "context" "errors" "fmt" "io" + "math/rand/v2" + "net/http" + "net/http/httptest" "net/url" "testing" "time" @@ -12,19 +14,34 @@ import ( "github.com/ipfs/go-cid" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipni/go-libipni/find/model" + "github.com/ipni/go-libipni/maurl" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" ma "github.com/multiformats/go-multiaddr" "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" "github.com/storacha/go-libstoracha/blobindex" "github.com/storacha/go-libstoracha/bytemap" + "github.com/storacha/go-libstoracha/capabilities/access" cassert "github.com/storacha/go-libstoracha/capabilities/assert" + "github.com/storacha/go-libstoracha/capabilities/space/content" ctypes "github.com/storacha/go-libstoracha/capabilities/types" "github.com/storacha/go-libstoracha/digestutil" "github.com/storacha/go-libstoracha/metadata" "github.com/storacha/go-libstoracha/testutil" + "github.com/storacha/go-ucanto/core/dag/blockstore" "github.com/storacha/go-ucanto/core/delegation" + "github.com/storacha/go-ucanto/core/invocation" + "github.com/storacha/go-ucanto/core/ipld" + "github.com/storacha/go-ucanto/core/message" + "github.com/storacha/go-ucanto/core/receipt" + "github.com/storacha/go-ucanto/core/receipt/ran" + "github.com/storacha/go-ucanto/core/result" "github.com/storacha/go-ucanto/core/result/ok" + "github.com/storacha/go-ucanto/did" + ed25519 "github.com/storacha/go-ucanto/principal/ed25519/signer" + ucan_car "github.com/storacha/go-ucanto/transport/car" + ucan_http "github.com/storacha/go-ucanto/transport/http" "github.com/storacha/go-ucanto/ucan" "github.com/storacha/indexing-service/pkg/internal/extmocks" "github.com/storacha/indexing-service/pkg/service/blobindexlookup" @@ -50,9 +67,10 @@ func TestQuery(t *testing.T) { contentLink := testutil.RandomCID(t) contentHash := contentLink.(cidlink.Link).Hash() + space := testutil.RandomDID(t) // content will have a location claim, an index claim and an equals claim - locationDelegationCid, locationDelegation, locationProviderResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + locationDelegationCid, locationDelegation, locationProviderResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) indexDelegationCid, indexDelegation, indexResult, indexCid, index := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) equalsDelegationCid, equalsDelegation, equalsResult, equivalentCid := buildTestEqualsClaim(t, contentLink.(cidlink.Link), providerAddr) @@ -73,7 +91,8 @@ func TestQuery(t *testing.T) { mockClaimsService.EXPECT().Find(extmocks.AnyContext, equalsDelegationCid, equalsClaimUrl).Return(equalsDelegation, nil) // then attempt to find records for the index referenced in the index claim - indexLocationDelegationCid, indexLocationDelegation, indexLocationProviderResult := buildTestLocationClaim(t, indexCid, providerAddr) + indexSize := rand.Uint64N(5000) + indexLocationDelegationCid, indexLocationDelegation, indexLocationProviderResult := buildTestLocationClaim(t, indexCid, providerAddr, space, indexSize) mockProviderIndex.EXPECT().Find(extmocks.AnyContext, providerindex.QueryKey{ Hash: indexCid.Hash(), @@ -86,10 +105,11 @@ func TestQuery(t *testing.T) { // and finally call the blob index lookup service to fetch the actual index indexBlobUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/blobs/%s", digestutil.Format(indexCid.Hash()))))(t) - mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, types.EncodedContextID(indexLocationProviderResult.ContextID), indexResult, indexBlobUrl, (*metadata.Range)(nil)).Return(index, nil) + retrievalReq := types.NewRetrievalRequest(indexBlobUrl, &metadata.Range{Length: &indexSize}, nil) + mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, types.EncodedContextID(indexLocationProviderResult.ContextID), indexResult, retrievalReq).Return(index, nil) // similarly, the equals claim should make the service ask for the location claim of the equivalent content - equalsLocationDelegationCid, equalsLocationDelegation, equalsLocationProviderResult := buildTestLocationClaim(t, equivalentCid, providerAddr) + equalsLocationDelegationCid, equalsLocationDelegation, equalsLocationProviderResult := buildTestLocationClaim(t, equivalentCid, providerAddr, space, rand.Uint64N(5000)) mockProviderIndex.EXPECT().Find(extmocks.AnyContext, providerindex.QueryKey{ Hash: equivalentCid.Hash(), @@ -100,9 +120,127 @@ func TestQuery(t *testing.T) { equalsLocationClaimUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/claims/%s", equalsLocationDelegationCid.String())))(t) mockClaimsService.EXPECT().Find(extmocks.AnyContext, equalsLocationDelegationCid, equalsLocationClaimUrl).Return(equalsLocationDelegation, nil) - service := NewIndexingService(mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) + service := NewIndexingService(testutil.Service, mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) - result, err := service.Query(context.Background(), types.Query{Hashes: []mh.Multihash{contentHash}}) + result, err := service.Query(t.Context(), types.Query{Hashes: []mh.Multihash{contentHash}}) + + require.NoError(t, err) + + expectedClaims := map[cid.Cid]delegation.Delegation{ + locationDelegationCid.Cid: locationDelegation, + indexDelegationCid.Cid: indexDelegation, + equalsDelegationCid.Cid: equalsDelegation, + indexLocationDelegationCid.Cid: indexLocationDelegation, + equalsLocationDelegationCid.Cid: equalsLocationDelegation, + } + expectedIndexes := bytemap.NewByteMap[types.EncodedContextID, blobindex.ShardedDagIndexView](1) + expectedIndexes.Set(types.EncodedContextID(indexLocationProviderResult.ContextID), index) + expectedResult := testutil.Must(queryresult.Build(expectedClaims, expectedIndexes))(t) + + require.ElementsMatch(t, expectedResult.Claims(), result.Claims()) + require.ElementsMatch(t, expectedResult.Indexes(), result.Indexes()) + }) + + t.Run("happy path with authorized index retrieval", func(t *testing.T) { + mockBlobIndexLookup := blobindexlookup.NewMockBlobIndexLookup(t) + mockClaimsService := contentclaims.NewMockContentClaimsService(t) + mockProviderIndex := providerindex.NewMockProviderIndex(t) + providerAddr := &peer.AddrInfo{ + Addrs: []ma.Multiaddr{ + testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fclaims%2F%7Bclaim%7D"))(t), + testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fblobs%2F%7Bblob%7D"))(t), + }, + } + + contentLink := testutil.RandomCID(t) + contentHash := contentLink.(cidlink.Link).Hash() + + space := testutil.Must(ed25519.Generate())(t) + dlg := testutil.Must(delegation.Delegate( + space, + testutil.Service, + []ucan.Capability[ucan.NoCaveats]{ + ucan.NewCapability(content.RetrieveAbility, space.DID().String(), ucan.NoCaveats{}), + }, + ))(t) + + // content will have a location claim, an index claim and an equals claim + locationDelegationCid, locationDelegation, locationProviderResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space.DID(), rand.Uint64N(5000)) + indexDelegationCid, indexDelegation, indexResult, indexCid, index := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) + equalsDelegationCid, equalsDelegation, equalsResult, equivalentCid := buildTestEqualsClaim(t, contentLink.(cidlink.Link), providerAddr) + + contentResults := []model.ProviderResult{locationProviderResult, indexResult, equalsResult} + + // expect a call to find records for content + mockProviderIndex.EXPECT().Find(extmocks.AnyContext, providerindex.QueryKey{ + Spaces: []did.DID{space.DID()}, + Hash: contentHash, + TargetClaims: []multicodec.Code{metadata.EqualsClaimID, metadata.IndexClaimID, metadata.LocationCommitmentID}, + }).Return(contentResults, nil) + + // the results for content should make the IndexingService ask for all claims + locationClaimUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/claims/%s", locationDelegationCid.String())))(t) + mockClaimsService.EXPECT().Find(extmocks.AnyContext, locationDelegationCid, locationClaimUrl).Return(locationDelegation, nil) + indexClaimUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/claims/%s", indexDelegationCid.String())))(t) + mockClaimsService.EXPECT().Find(extmocks.AnyContext, indexDelegationCid, indexClaimUrl).Return(indexDelegation, nil) + equalsClaimUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/claims/%s", equalsDelegationCid.String())))(t) + mockClaimsService.EXPECT().Find(extmocks.AnyContext, equalsDelegationCid, equalsClaimUrl).Return(equalsDelegation, nil) + + // then attempt to find records for the index referenced in the index claim + indexSize := rand.Uint64N(5000) + indexLocationDelegationCid, indexLocationDelegation, indexLocationProviderResult := buildTestLocationClaim(t, indexCid, providerAddr, space.DID(), indexSize) + + mockProviderIndex.EXPECT().Find(extmocks.AnyContext, providerindex.QueryKey{ + Spaces: []did.DID{space.DID()}, + Hash: indexCid.Hash(), + TargetClaims: []multicodec.Code{metadata.LocationCommitmentID}, + }).Return([]model.ProviderResult{indexLocationProviderResult}, nil) + + // fetch the index's location claim + indexLocationClaimUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/claims/%s", indexLocationDelegationCid.String())))(t) + mockClaimsService.EXPECT().Find(extmocks.AnyContext, indexLocationDelegationCid, indexLocationClaimUrl).Return(indexLocationDelegation, nil) + + // and finally call the blob index lookup service to fetch the actual index + indexBlobUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/blobs/%s", digestutil.Format(indexCid.Hash()))))(t) + retrievalAuth := types.NewRetrievalAuth( + testutil.Service, + locationDelegation.Issuer(), + content.Retrieve.New( + space.DID().String(), + content.RetrieveCaveats{ + Blob: content.BlobDigest{Digest: indexCid.Hash()}, + Range: content.Range{Start: 0, End: indexSize - 1}, + }, + ), + []delegation.Proof{delegation.FromDelegation(dlg)}, + ) + retrievalReq := types.NewRetrievalRequest( + indexBlobUrl, + &metadata.Range{Length: &indexSize}, + &retrievalAuth, + ) + mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, types.EncodedContextID(indexLocationProviderResult.ContextID), indexResult, retrievalReq).Return(index, nil) + + // similarly, the equals claim should make the service ask for the location claim of the equivalent content + equalsLocationDelegationCid, equalsLocationDelegation, equalsLocationProviderResult := buildTestLocationClaim(t, equivalentCid, providerAddr, space.DID(), rand.Uint64N(5000)) + + mockProviderIndex.EXPECT().Find(extmocks.AnyContext, providerindex.QueryKey{ + Spaces: []did.DID{space.DID()}, + Hash: equivalentCid.Hash(), + TargetClaims: []multicodec.Code{metadata.LocationCommitmentID}, + }).Return([]model.ProviderResult{equalsLocationProviderResult}, nil) + + // and fetch the equivalent content's location claim + equalsLocationClaimUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/claims/%s", equalsLocationDelegationCid.String())))(t) + mockClaimsService.EXPECT().Find(extmocks.AnyContext, equalsLocationDelegationCid, equalsLocationClaimUrl).Return(equalsLocationDelegation, nil) + + service := NewIndexingService(testutil.Service, mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) + + result, err := service.Query(t.Context(), types.Query{ + Hashes: []mh.Multihash{contentHash}, + Match: types.Match{Subject: []did.DID{space.DID()}}, + Delegations: []delegation.Delegation{dlg}, + }) require.NoError(t, err) @@ -126,7 +264,7 @@ func TestQuery(t *testing.T) { mockClaimsService := contentclaims.NewMockContentClaimsService(t) mockProviderIndex := providerindex.NewMockProviderIndex(t) - service := NewIndexingService(mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) + service := NewIndexingService(testutil.Service, mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) contentHash := testutil.RandomMultihash(t) @@ -143,7 +281,7 @@ func TestQuery(t *testing.T) { mockProviderIndex.EXPECT().Find(extmocks.AnyContext, expectedQueryKey).Return([]model.ProviderResult{}, nil) - _, err := service.Query(context.Background(), query) + _, err := service.Query(t.Context(), query) require.NoError(t, err) }) @@ -160,7 +298,7 @@ func TestQuery(t *testing.T) { mockProviderIndex.EXPECT().Find(extmocks.AnyContext, expectedQueryKey).Return([]model.ProviderResult{}, nil) - _, err := service.Query(context.Background(), query) + _, err := service.Query(t.Context(), query) require.NoError(t, err) }) @@ -177,7 +315,7 @@ func TestQuery(t *testing.T) { mockProviderIndex.EXPECT().Find(extmocks.AnyContext, expectedQueryKey).Return([]model.ProviderResult{}, nil) - _, err := service.Query(context.Background(), query) + _, err := service.Query(t.Context(), query) require.NoError(t, err) }) }) @@ -196,9 +334,9 @@ func TestQuery(t *testing.T) { TargetClaims: []multicodec.Code{metadata.EqualsClaimID, metadata.IndexClaimID, metadata.LocationCommitmentID}, }).Return([]model.ProviderResult{}, errors.New("provider index error")) - service := NewIndexingService(mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) + service := NewIndexingService(testutil.Service, mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) - _, err := service.Query(context.Background(), types.Query{Hashes: []mh.Multihash{contentHash}}) + _, err := service.Query(t.Context(), types.Query{Hashes: []mh.Multihash{contentHash}}) require.Error(t, err) }) @@ -215,9 +353,10 @@ func TestQuery(t *testing.T) { contentLink := testutil.RandomCID(t) contentHash := contentLink.(cidlink.Link).Hash() + space := testutil.RandomDID(t) // content will have a location claim - locationDelegationCid, _, locationProviderResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + locationDelegationCid, _, locationProviderResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) contentResults := []model.ProviderResult{locationProviderResult} @@ -230,9 +369,9 @@ func TestQuery(t *testing.T) { // the results for content should make the IndexingService ask for the location claim, but that will fail locationClaimUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/claims/%s", locationDelegationCid.String())))(t) mockClaimsService.EXPECT().Find(extmocks.AnyContext, locationDelegationCid, locationClaimUrl).Return(nil, errors.New("content claims service error")) - service := NewIndexingService(mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) + service := NewIndexingService(testutil.Service, mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) - _, err := service.Query(context.Background(), types.Query{Hashes: []mh.Multihash{contentHash}}) + _, err := service.Query(t.Context(), types.Query{Hashes: []mh.Multihash{contentHash}}) require.Error(t, err) }) @@ -250,9 +389,10 @@ func TestQuery(t *testing.T) { contentLink := testutil.RandomCID(t) contentHash := contentLink.(cidlink.Link).Hash() + space := testutil.RandomDID(t) // content will have a location claim and an index claim - locationDelegationCid, locationDelegation, locationProviderResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + locationDelegationCid, locationDelegation, locationProviderResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) indexDelegationCid, indexDelegation, indexResult, indexCid, _ := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) contentResults := []model.ProviderResult{locationProviderResult, indexResult} @@ -270,7 +410,8 @@ func TestQuery(t *testing.T) { mockClaimsService.EXPECT().Find(extmocks.AnyContext, indexDelegationCid, indexClaimUrl).Return(indexDelegation, nil) // then attempt to find records for the index referenced in the index claim - indexLocationDelegationCid, indexLocationDelegation, indexLocationProviderResult := buildTestLocationClaim(t, indexCid, providerAddr) + indexSize := rand.Uint64N(5000) + indexLocationDelegationCid, indexLocationDelegation, indexLocationProviderResult := buildTestLocationClaim(t, indexCid, providerAddr, space, indexSize) mockProviderIndex.EXPECT().Find(extmocks.AnyContext, providerindex.QueryKey{ Hash: indexCid.Hash(), @@ -283,23 +424,26 @@ func TestQuery(t *testing.T) { // and finally call the blob index lookup service to fetch the actual index, which will fail indexBlobUrl := testutil.Must(url.Parse(fmt.Sprintf("https://storacha.network/blobs/%s", digestutil.Format(indexCid.Hash()))))(t) - mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, types.EncodedContextID(indexLocationProviderResult.ContextID), indexResult, indexBlobUrl, (*metadata.Range)(nil)).Return(nil, errors.New("blob index lookup error")) + retrievalReq := types.NewRetrievalRequest(indexBlobUrl, &metadata.Range{Length: &indexSize}, nil) + mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, types.EncodedContextID(indexLocationProviderResult.ContextID), indexResult, retrievalReq).Return(nil, errors.New("blob index lookup error")) - service := NewIndexingService(mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) + service := NewIndexingService(testutil.Service, mockBlobIndexLookup, mockClaimsService, peer.AddrInfo{ID: testutil.RandomPeer(t)}, mockProviderIndex) - _, err := service.Query(context.Background(), types.Query{Hashes: []mh.Multihash{contentHash}}) + _, err := service.Query(t.Context(), types.Query{Hashes: []mh.Multihash{contentHash}}) require.Error(t, err) }) } -func buildTestLocationClaim(t *testing.T, contentLink cidlink.Link, providerAddr *peer.AddrInfo) (cidlink.Link, delegation.Delegation, model.ProviderResult) { - locationClaim := cassert.Location.New(testutil.Service.DID().String(), cassert.LocationCaveats{ +func buildTestLocationClaim(t *testing.T, contentLink cidlink.Link, providerAddr *peer.AddrInfo, space did.DID, size uint64) (cidlink.Link, delegation.Delegation, model.ProviderResult) { + locationClaim := cassert.Location.New(testutil.Alice.DID().String(), cassert.LocationCaveats{ Content: ctypes.FromHash(contentLink.Hash()), Location: []url.URL{*testutil.Must(url.Parse("https://storacha.network"))(t)}, + Space: space, + Range: &cassert.Range{Length: &size}, }) - locationDelegation := testutil.Must(delegation.Delegate(testutil.Service, testutil.Alice, []ucan.Capability[cassert.LocationCaveats]{locationClaim}))(t) + locationDelegation := testutil.Must(delegation.Delegate(testutil.Alice, space, []ucan.Capability[cassert.LocationCaveats]{locationClaim}))(t) locationDelegationCid := testutil.Must(cid.Prefix{ Version: 1, Codec: uint64(multicodec.Car), @@ -310,6 +454,7 @@ func buildTestLocationClaim(t *testing.T, contentLink cidlink.Link, providerAddr locationMetadata := metadata.LocationCommitmentMetadata{ Shard: &contentLink.Cid, Claim: locationDelegationCid, + Range: &metadata.Range{Length: &size}, Expiration: time.Now().Add(time.Hour).Unix(), } @@ -383,6 +528,58 @@ func buildTestEqualsClaim(t *testing.T, contentLink cidlink.Link, providerAddr * return cidlink.Link{Cid: equalsDelegationCid}, equalsDelegation, equalsProviderResults, equivalentCid.(cidlink.Link) } +// startMockStorageNode starts a mock storage node that accepts `access/grant` +// invocations. Actual `space/content/retrieve` and `blob/retrieve` invocations +// are mocked by the tests so are not supported here. +func startMockStorageNode(t *testing.T, id ucan.Signer) ma.Multiaddr { + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + req := ucan_http.NewRequest(r.Body, r.Header) + switch r.Method { + case http.MethodPost: // UCAN invocation for access/grant + codec := ucan_car.NewInboundCodec() + accept := testutil.Must(codec.Accept(req))(t) + msg := testutil.Must(accept.Decoder().Decode(req))(t) + bs := testutil.Must(blockstore.NewBlockReader(blockstore.WithBlocksIterator(msg.Blocks())))(t) + inv := testutil.Must(invocation.NewInvocationView(msg.Invocations()[0], bs))(t) + cap := inv.Capabilities()[0] + if cap.Can() != access.GrantAbility { + t.Fatal("unexpected invocation") + } + dlg := testutil.Must(delegation.Delegate( + id, + inv.Issuer(), + []ucan.Capability[ucan.NoCaveats]{ + ucan.NewCapability("*", id.DID().String(), ucan.NoCaveats{}), + }, + ))(t) + dlgsModel := access.DelegationsModel{ + Keys: []string{dlg.Link().String()}, + Values: map[string][]byte{ + dlg.Link().String(): testutil.Must(io.ReadAll(dlg.Archive()))(t), + }, + } + out := result.Ok[access.GrantOk, ipld.Builder](access.GrantOk{Delegations: dlgsModel}) + rcpt, err := receipt.Issue(id, out, ran.FromInvocation(inv)) + require.NoError(t, err) + msg, err = message.Build(nil, []receipt.AnyReceipt{rcpt}) + require.NoError(t, err) + resp, err := accept.Encoder().Encode(msg) + require.NoError(t, err) + for key, values := range resp.Headers() { + for _, val := range values { + w.Header().Add(key, val) + } + } + _ = testutil.Must(io.Copy(w, resp.Body()))(t) + default: + t.Fatal("unexpected invocation") + } + })) + t.Cleanup(httpServer.Close) + addr := testutil.Must(maurl.FromURL(testutil.Must(url.Parse(httpServer.URL))(t)))(t) + return ma.Join(addr, testutil.Must(ma.NewMultiaddr("/http-path/%2Fblobs%2F%7Bblob%7D"))(t)) +} + func TestPublishIndexClaim(t *testing.T) { t.Run("does not publish unknown claims", func(t *testing.T) { claim, err := delegation.Delegate( @@ -393,7 +590,7 @@ func TestPublishIndexClaim(t *testing.T) { }, ) require.NoError(t, err) - err = Publish(context.Background(), nil, nil, nil, peer.AddrInfo{}, claim) + err = Publish(t.Context(), testutil.Service, nil, nil, nil, peer.AddrInfo{}, claim) require.ErrorIs(t, err, ErrUnrecognizedClaim) }) @@ -402,15 +599,19 @@ func TestPublishIndexClaim(t *testing.T) { mockProviderIndex := providerindex.NewMockProviderIndex(t) mockBlobIndexLookup := blobindexlookup.NewMockBlobIndexLookup(t) contentLink := testutil.RandomCID(t) + space := testutil.RandomDID(t) + priv := testutil.Must(crypto.UnmarshalEd25519PrivateKey(testutil.Service.Raw()))(t) + peerID := testutil.Must(peer.IDFromPrivateKey(priv))(t) providerAddr := &peer.AddrInfo{ + ID: peerID, Addrs: []ma.Multiaddr{ testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fblobs%2F%7Bblob%7D"))(t), testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fclaims%2F%7Bclaim%7D"))(t), }, } // content will have a location claim, an index claim - locationDelegationCid, locationDelegation, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + locationDelegationCid, locationDelegation, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) _, indexDelegation, _, indexLink, shardIndex := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) // expect a call to cache the index claim using claims.Publish @@ -429,7 +630,7 @@ func TestPublishIndexClaim(t *testing.T) { // expect the blob index lookup service to be called once to fetch the shard index mockBlobIndexLookup.EXPECT().Find( - extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything, mock.Anything, + extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything, ).Return(shardIndex, nil) // expect the index claim to be published @@ -441,7 +642,61 @@ func TestPublishIndexClaim(t *testing.T) { } mockProviderIndex.EXPECT().Publish(extmocks.AnyContext, *providerAddr, mock.Anything, mock.Anything, mock.Anything).Return(nil) - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + require.NoError(t, err) + }) + + t.Run("publish index claim with service authorized retrieval", func(t *testing.T) { + mockClaimsService := contentclaims.NewMockContentClaimsService(t) + mockProviderIndex := providerindex.NewMockProviderIndex(t) + mockBlobIndexLookup := blobindexlookup.NewMockBlobIndexLookup(t) + contentLink := testutil.RandomCID(t) + space := testutil.RandomDID(t) + + priv := testutil.Must(crypto.UnmarshalEd25519PrivateKey(testutil.Service.Raw()))(t) + peerID := testutil.Must(peer.IDFromPrivateKey(priv))(t) + blobAddr := startMockStorageNode(t, testutil.Alice) + + providerAddr := &peer.AddrInfo{ + ID: peerID, + Addrs: []ma.Multiaddr{ + blobAddr, + testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fclaims%2F%7Bclaim%7D"))(t), + }, + } + // content will have a location claim, an index claim + locationDelegationCid, locationDelegation, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) + _, indexDelegation, _, indexLink, shardIndex := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) + + // expect a call to cache the index claim using claims.Publish + mockClaimsService.EXPECT().Publish(extmocks.AnyContext, indexDelegation).Return(nil) + + // expect a call to find records for index location commitment + mockProviderIndex.EXPECT().Find(extmocks.AnyContext, providerindex.QueryKey{ + Hash: indexLink.Hash(), + TargetClaims: []multicodec.Code{metadata.LocationCommitmentID}, + }).Return([]model.ProviderResult{locationResult}, nil) + + // expect the claim service to be called for each result from providerIndex.Find + mockClaimsService.EXPECT().Find( + extmocks.AnyContext, locationDelegationCid, mock.AnythingOfType("*url.URL"), + ).Return(locationDelegation, nil) + + // expect the blob index lookup service to be called once to fetch the shard index + mockBlobIndexLookup.EXPECT().Find( + extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything, + ).Return(shardIndex, nil) + + // expect the index claim to be published + digests := bytemap.NewByteMap[mh.Multihash, struct{}](-1) + for _, slices := range shardIndex.Shards().Iterator() { + for d := range slices.Iterator() { + digests.Set(d, struct{}{}) + } + } + mockProviderIndex.EXPECT().Publish(extmocks.AnyContext, *providerAddr, mock.Anything, mock.Anything, mock.Anything).Return(nil) + + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) require.NoError(t, err) }) @@ -465,7 +720,7 @@ func TestPublishIndexClaim(t *testing.T) { require.NoError(t, err) // Attempt to publish the claim - err = Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, claim) + err = Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, claim) // Expect an error indicating missing capabilities require.Error(t, err) @@ -489,7 +744,7 @@ func TestPublishIndexClaim(t *testing.T) { require.NoError(t, err) // Attempt to publish the claim - err = Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, faultyIndexClaim) + err = Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, faultyIndexClaim) // Expect an error indicating a problem with reading the index claim caveats require.Error(t, err) @@ -515,7 +770,7 @@ func TestPublishIndexClaim(t *testing.T) { mockClaimsService.EXPECT().Publish(extmocks.AnyContext, indexDelegation).Return(fmt.Errorf("failed to cache claim")) // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) // Expect an error indicating a problem with caching the claim require.Error(t, err) @@ -547,7 +802,7 @@ func TestPublishIndexClaim(t *testing.T) { }).Return([]model.ProviderResult{}, nil) // no location commitments found // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) // Expect an error indicating no location commitments found require.Error(t, err) @@ -579,7 +834,7 @@ func TestPublishIndexClaim(t *testing.T) { }).Return([]model.ProviderResult{}, fmt.Errorf("failed to find location commitments")) // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) // Expect an error indicating a problem with finding location commitments require.Error(t, err) @@ -611,7 +866,7 @@ func TestPublishIndexClaim(t *testing.T) { }).Return([]model.ProviderResult{{}}, nil) // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) // Expect an error indicating a problem with fetching the blob index require.Error(t, err) @@ -643,7 +898,7 @@ func TestPublishIndexClaim(t *testing.T) { }).Return([]model.ProviderResult{indexResult}, nil) // this is the wrong claim type // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) // Expect an error indicating a problem with the metadata type require.Error(t, err) @@ -655,6 +910,7 @@ func TestPublishIndexClaim(t *testing.T) { mockProviderIndex := providerindex.NewMockProviderIndex(t) mockBlobIndexLookup := blobindexlookup.NewMockBlobIndexLookup(t) contentLink := testutil.RandomCID(t) + space := testutil.RandomDID(t) providerAddr := &peer.AddrInfo{ Addrs: []ma.Multiaddr{ @@ -664,7 +920,7 @@ func TestPublishIndexClaim(t *testing.T) { } // content will have a location claim, an index claim - _, _, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + _, _, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) _, indexDelegation, _, indexLink, _ := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) // Simulate caching the claim in claims.Publish @@ -677,7 +933,7 @@ func TestPublishIndexClaim(t *testing.T) { }).Return([]model.ProviderResult{locationResult}, nil) // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) // Expect an error indicating a problem with building the retrieval URL require.Error(t, err) @@ -689,8 +945,12 @@ func TestPublishIndexClaim(t *testing.T) { mockProviderIndex := providerindex.NewMockProviderIndex(t) mockBlobIndexLookup := blobindexlookup.NewMockBlobIndexLookup(t) contentLink := testutil.RandomCID(t) + space := testutil.RandomDID(t) + priv := testutil.Must(crypto.UnmarshalEd25519PrivateKey(testutil.Service.Raw()))(t) + peerID := testutil.Must(peer.IDFromPrivateKey(priv))(t) providerAddr := &peer.AddrInfo{ + ID: peerID, Addrs: []ma.Multiaddr{ // Only the blob URL is provided, it is missing the claim URL which is used to build the claim URL testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fblobs%2F%7Bblob%7D"))(t), @@ -698,7 +958,7 @@ func TestPublishIndexClaim(t *testing.T) { } // content will have a location claim, an index claim - _, _, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + _, _, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) _, indexDelegation, _, indexLink, _ := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) // Simulate caching the claim in claims.Publish @@ -711,10 +971,10 @@ func TestPublishIndexClaim(t *testing.T) { }).Return([]model.ProviderResult{locationResult}, nil) // Simulate a successful result from blobIndexLookup.Find - mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) + mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) require.Error(t, err) require.Contains(t, err.Error(), "fetching blob index: verifying claim: building claim URL: no {claim} endpoint found") }) @@ -724,8 +984,12 @@ func TestPublishIndexClaim(t *testing.T) { mockProviderIndex := providerindex.NewMockProviderIndex(t) mockBlobIndexLookup := blobindexlookup.NewMockBlobIndexLookup(t) contentLink := testutil.RandomCID(t) + space := testutil.RandomDID(t) + priv := testutil.Must(crypto.UnmarshalEd25519PrivateKey(testutil.Service.Raw()))(t) + peerID := testutil.Must(peer.IDFromPrivateKey(priv))(t) providerAddr := &peer.AddrInfo{ + ID: peerID, Addrs: []ma.Multiaddr{ testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fclaims%2F%7Bclaim%7D"))(t), testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fblobs%2F%7Bblob%7D"))(t), @@ -733,7 +997,7 @@ func TestPublishIndexClaim(t *testing.T) { } // content will have a location claim, an index claim - _, _, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + _, _, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) _, indexDelegation, _, indexLink, _ := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) // Simulate caching the claim in claims.Publish @@ -746,13 +1010,13 @@ func TestPublishIndexClaim(t *testing.T) { }).Return([]model.ProviderResult{locationResult}, nil) // Simulate a successful result from blobIndexLookup.Find - mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) + mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) // Simulate a failure from claims.Find mockClaimsService.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("failed to find claim")) // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) require.Error(t, err) require.Contains(t, err.Error(), "fetching blob index: verifying claim: failed to find claim") }) @@ -762,8 +1026,12 @@ func TestPublishIndexClaim(t *testing.T) { mockProviderIndex := providerindex.NewMockProviderIndex(t) mockBlobIndexLookup := blobindexlookup.NewMockBlobIndexLookup(t) contentLink := testutil.RandomCID(t) + space := testutil.RandomDID(t) + priv := testutil.Must(crypto.UnmarshalEd25519PrivateKey(testutil.Service.Raw()))(t) + peerID := testutil.Must(peer.IDFromPrivateKey(priv))(t) providerAddr := &peer.AddrInfo{ + ID: peerID, Addrs: []ma.Multiaddr{ testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fclaims%2F%7Bclaim%7D"))(t), testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fblobs%2F%7Bblob%7D"))(t), @@ -771,7 +1039,7 @@ func TestPublishIndexClaim(t *testing.T) { } // content will have a location claim, an index claim - _, locationDelegation, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + _, locationDelegation, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) _, indexDelegation, _, indexLink, shardIndex := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) // Simulate caching the claim in claims.Publish @@ -784,7 +1052,7 @@ func TestPublishIndexClaim(t *testing.T) { }).Return([]model.ProviderResult{locationResult}, nil) // Simulate a successful result from blobIndexLookup.Find - mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(shardIndex, nil) + mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything).Return(shardIndex, nil) // Simulate a failure from claims.Find mockClaimsService.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything).Return(locationDelegation, nil) @@ -793,7 +1061,7 @@ func TestPublishIndexClaim(t *testing.T) { mockProviderIndex.EXPECT().Publish(extmocks.AnyContext, *providerAddr, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("failed to publish claim")) // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) require.Error(t, err) require.Contains(t, err.Error(), "publishing index claim: failed to publish claim") }) @@ -803,8 +1071,12 @@ func TestPublishIndexClaim(t *testing.T) { mockProviderIndex := providerindex.NewMockProviderIndex(t) mockBlobIndexLookup := blobindexlookup.NewMockBlobIndexLookup(t) contentLink := testutil.RandomCID(t) + space := testutil.RandomDID(t) + priv := testutil.Must(crypto.UnmarshalEd25519PrivateKey(testutil.Service.Raw()))(t) + peerID := testutil.Must(peer.IDFromPrivateKey(priv))(t) providerAddr := &peer.AddrInfo{ + ID: peerID, Addrs: []ma.Multiaddr{ testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fclaims%2F%7Bclaim%7D"))(t), testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fblobs%2F%7Bblob%7D"))(t), @@ -812,7 +1084,7 @@ func TestPublishIndexClaim(t *testing.T) { } // content will have a location claim, an index claim - _, locationDelegation, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + _, locationDelegation, locationResult := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) _, indexDelegation, _, indexLink, shardIndex := buildTestIndexClaim(t, contentLink.(cidlink.Link), providerAddr) // Simulate caching the claim in claims.Publish @@ -825,7 +1097,7 @@ func TestPublishIndexClaim(t *testing.T) { }).Return([]model.ProviderResult{locationResult}, nil) // Simulate a successful result from blobIndexLookup.Find - mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(shardIndex, nil) + mockBlobIndexLookup.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything, mock.Anything).Return(shardIndex, nil) // Simulate a failure from claims.Find mockClaimsService.EXPECT().Find(extmocks.AnyContext, mock.Anything, mock.Anything).Return(locationDelegation, nil) @@ -834,7 +1106,7 @@ func TestPublishIndexClaim(t *testing.T) { mockProviderIndex.EXPECT().Publish(extmocks.AnyContext, *providerAddr, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("failed to publish claim")) // Attempt to publish the claim - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, indexDelegation) require.Error(t, err) require.Contains(t, err.Error(), "publishing index claim: failed to publish claim") }) @@ -864,7 +1136,7 @@ func TestPublishEqualsClaim(t *testing.T) { // Simulate a successful result from provIndex.Publish mockProviderIndex.EXPECT().Publish(extmocks.AnyContext, *providerAddr, mock.Anything, mock.Anything, mock.Anything).Return(nil) - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, equalsDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, equalsDelegation) require.NoError(t, err) }) @@ -885,7 +1157,7 @@ func TestPublishEqualsClaim(t *testing.T) { require.NoError(t, err) // Attempt to publish the claim - err = Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, faultyEqualsClaim) + err = Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, faultyEqualsClaim) // Expect an error indicating a problem with reading the claim caveats require.Error(t, err) @@ -911,7 +1183,7 @@ func TestPublishEqualsClaim(t *testing.T) { // Simulate a failure from claims.Publish mockClaimsService.EXPECT().Publish(extmocks.AnyContext, equalsDelegation).Return(fmt.Errorf("failed to publish claim")) - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, equalsDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, equalsDelegation) require.Error(t, err) require.Contains(t, err.Error(), "caching equals claim with claim service: failed to publish claim") }) @@ -938,7 +1210,7 @@ func TestPublishEqualsClaim(t *testing.T) { // Simulate a failure from provIndex.Publish mockProviderIndex.EXPECT().Publish(extmocks.AnyContext, *providerAddr, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("failed to publish claim")) - err := Publish(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, equalsDelegation) + err := Publish(t.Context(), testutil.Service, mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, equalsDelegation) require.Error(t, err) require.Contains(t, err.Error(), "publishing equals claim: failed to publish claim") }) @@ -956,7 +1228,7 @@ func TestCacheClaim(t *testing.T) { }, ) require.NoError(t, err) - err = Cache(context.Background(), nil, nil, nil, peer.AddrInfo{}, claim) + err = Cache(t.Context(), nil, nil, nil, peer.AddrInfo{}, claim) require.ErrorIs(t, err, ErrUnrecognizedClaim) }) @@ -965,13 +1237,15 @@ func TestCacheClaim(t *testing.T) { mockProviderIndex := providerindex.NewMockProviderIndex(t) mockBlobIndexLookup := blobindexlookup.NewMockBlobIndexLookup(t) contentLink := testutil.RandomCID(t) + space := testutil.RandomDID(t) + providerAddr := &peer.AddrInfo{ Addrs: []ma.Multiaddr{ testutil.Must(ma.NewMultiaddr("/dns/storacha.network/tls/http/http-path/%2Fclaims%2F%7Bclaim%7D"))(t), }, } - _, locationDelegation, _ := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr) + _, locationDelegation, _ := buildTestLocationClaim(t, contentLink.(cidlink.Link), providerAddr, space, rand.Uint64N(5000)) mockClaimsService.EXPECT().Cache(extmocks.AnyContext, locationDelegation).Return(nil) anyContextID := mock.AnythingOfType("string") @@ -979,7 +1253,7 @@ func TestCacheClaim(t *testing.T) { anyMetadata := mock.AnythingOfType("metadata.Metadata") mockProviderIndex.EXPECT().Cache(extmocks.AnyContext, *providerAddr, anyContextID, anyMultihash, anyMetadata).Return(nil) - err := Cache(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) + err := Cache(t.Context(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) require.NoError(t, err) }) @@ -1001,7 +1275,7 @@ func TestCacheClaim(t *testing.T) { ) require.NoError(t, err) - err = Cache(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, claim) + err = Cache(t.Context(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, claim) require.Error(t, err) require.Contains(t, err.Error(), fmt.Sprintf("missing capabilities in claim: %s", claim.Link())) }) @@ -1022,7 +1296,7 @@ func TestCacheClaim(t *testing.T) { require.NoError(t, err) // Attempt to cache the claim, which will fail - err = Cache(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, faultyLocationClaim) + err = Cache(t.Context(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, faultyLocationClaim) require.Error(t, err) require.Contains(t, err.Error(), "reading index claim data") }) @@ -1056,7 +1330,7 @@ func TestCacheClaim(t *testing.T) { mockClaimsService.EXPECT().Cache(extmocks.AnyContext, locationDelegation).Return(nil) // Cache the claim with expiration - err := Cache(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) + err := Cache(t.Context(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) require.NoError(t, err) }) @@ -1090,7 +1364,7 @@ func TestCacheClaim(t *testing.T) { mockClaimsService.EXPECT().Cache(extmocks.AnyContext, locationDelegation).Return(nil) // Cache the claim with expiration - err := Cache(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) + err := Cache(t.Context(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) require.NoError(t, err) }) @@ -1120,7 +1394,7 @@ func TestCacheClaim(t *testing.T) { ) // Attempt to cache the claim - err := Cache(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) + err := Cache(t.Context(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) require.Error(t, err) require.Contains(t, err.Error(), "something went wrong while caching claim in claims.Cache") }) @@ -1155,7 +1429,7 @@ func TestCacheClaim(t *testing.T) { ) // Attempt to cache the claim - err := Cache(context.Background(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) + err := Cache(t.Context(), mockBlobIndexLookup, mockClaimsService, mockProviderIndex, *providerAddr, locationDelegation) require.Error(t, err) require.Contains(t, err.Error(), "something went wrong while caching claim in providerIndex.Cache") }) diff --git a/pkg/types/types.go b/pkg/types/types.go index 72addbc..91913d6 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "net/url" "github.com/ipfs/go-cid" "github.com/ipni/go-libipni/find/model" @@ -12,9 +13,11 @@ import ( "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" "github.com/storacha/go-libstoracha/blobindex" + "github.com/storacha/go-libstoracha/metadata" "github.com/storacha/go-ucanto/core/delegation" "github.com/storacha/go-ucanto/core/ipld" "github.com/storacha/go-ucanto/did" + "github.com/storacha/go-ucanto/ucan" ) // ContextID describes the data used to calculate a context id for IPNI @@ -188,3 +191,64 @@ type Service interface { Publisher Querier } + +// RetrievalRequest is all the details needed for retrieving data from the +// network. At minimum it contains the URL to retrieve a blob from. +// +// Legacy retrievals will not carry any retrieval authorization information i.e. +// the Auth field will be nil. +type RetrievalRequest struct { + // URL where the blob may be requested from. + URL *url.URL + // Optional byte range to request. + Range *metadata.Range + // Optional UCAN authorization parameters. + Auth *RetrievalAuth +} + +// NewRetrievalRequest creates a new [RetrievalRequest] object that contains all the +// details required to retrieve a blob from the network. +func NewRetrievalRequest( + url *url.URL, + byteRange *metadata.Range, + auth *RetrievalAuth, +) RetrievalRequest { + return RetrievalRequest{url, byteRange, auth} +} + +// RetrievalAuth are the details for a UCAN authorized content retrieval. +// +// The provided proofs are expected to contain the `space/content/retrieve` +// delegated capability allowing content to be retrieved using UCAN +// authorization. +type RetrievalAuth struct { + // The Indexing Service UCAN signing key. + Issuer ucan.Signer + // Identity of the storage node to retrieve data from. + Audience ucan.Principal + // Retrieval ability, resource (typically the space) and caveats. + Capability ucan.Capability[ucan.CaveatBuilder] + // Delegations from the client (e.g. `space/content/retrieve`) or a storage + // node (e.g. `blob/retrieve`) to the indexing service authorizing retrieval. + Proofs []delegation.Proof +} + +// NewRetrievalAuth creates a new [RetrievalAuth] object for UCAN authorizing +// blob retrievals. +func NewRetrievalAuth[C ucan.CaveatBuilder]( + issuer ucan.Signer, + audience ucan.Principal, + capability ucan.Capability[C], + proofs []delegation.Proof, +) RetrievalAuth { + return RetrievalAuth{ + issuer, + audience, + ucan.NewCapability[ucan.CaveatBuilder]( + capability.Can(), + capability.With(), + capability.Nb(), + ), + proofs, + } +}