Merge pull request #49 from filecoin-project/feat/support-r2-service

feat: support cloudflare r2 upload backend

Author: travisperson

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:buster as builder
+FROM golang:1.19.2-buster as builder
 
 RUN apt-get update && apt-get install -y ca-certificates
 
@@ -15,7 +15,7 @@ RUN go mod download
 
 COPY . .
 
-RUN go build -o filecoin-chain-archiver ./cmd/filecoin-chain-archiver
+RUN make all
 
 FROM debian:buster
 

cmd/filecoin-chain-archiver/cmds/create.go

@@ -72,6 +72,11 @@ var cmdCreate = &cli.Command{
 			Usage:   "bucket host and port for upload",
 			EnvVars: []string{"FCA_CREATE_BUCKET_ENDPOINT"},
 		},
+		&cli.StringFlag{
+			Name:    "retrieval-endpoint-prefix",
+			Usage:   "URL prefix where uploaded object can be retrieved from",
+			EnvVars: []string{"FCA_CREATE_RETRIEVAL_ENDPOINT_PREFIX"},
+		},
 		&cli.StringFlag{
 			Name:    "access-key",
 			Usage:   "access key for upload",
@@ -138,6 +143,7 @@ var cmdCreate = &cli.Command{
 		flagBucketAccessKey := cctx.String("access-key")
 		flagBucketSecretKey := cctx.String("secret-key")
 		flagNamePrefix := cctx.String("name-prefix")
+		flagRetrievalEndpointPrefix := cctx.String("retrieval-endpoint-prefix")
 		flagBucket := cctx.String("bucket")
 		flagDiscard := cctx.Bool("discard")
 		flagProgressUpdate := cctx.Duration("progress-update")
@@ -380,10 +386,10 @@ var cmdCreate = &cli.Command{
 				return err
 			}
 
-			latestLocation, err := url.QueryUnescape(info.Location)
+			latestLocation, err := url.JoinPath(flagRetrievalEndpointPrefix, info.Key)
 			if err != nil {
-				logger.Errorw("failed to decode location url", "location", info.Location, "err", err)
-				latestLocation = info.Location
+				logger.Errorw("failed to join request path", "request_prefix", flagRetrievalEndpointPrefix, "key", info.Key)
+				return fmt.Errorf("failed to join request path: %w", err)
 			}
 
 			sha256sum := fmt.Sprintf("%x *%s.car\n", h.Sum(nil), name)
@@ -397,8 +403,7 @@ var cmdCreate = &cli.Command{
 			}
 
 			info, err = minioClient.PutObject(ctx, flagBucket, fmt.Sprintf("%slatest", flagNamePrefix), strings.NewReader(latestLocation), -1, minio.PutObjectOptions{
-				WebsiteRedirectLocation: latestLocation,
-				ContentType:             "text/plain",
+				ContentType: "text/plain",
 			})
 			if err != nil {
 				return fmt.Errorf("failed to write latest", "object", fmt.Sprintf("%slatest", flagNamePrefix), "err", err)

pkg/index/index.go

@@ -2,13 +2,14 @@ package index
 
 import (
 	"context"
-	"fmt"
+	"io"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/minio/minio-go/v7"
-
 	"github.com/ipfs/go-log/v2"
+	"github.com/minio/minio-go/v7"
+	"golang.org/x/xerrors"
 )
 
 var (
@@ -23,7 +24,7 @@ type IndexS3Resolver struct {
 }
 
 type s3ClientInterface interface {
-	StatObject(ctx context.Context, bucketName, objectName string, opts minio.StatObjectOptions) (minio.ObjectInfo, error)
+	GetObject(ctx context.Context, bucketName, objectName string, opts minio.GetObjectOptions) (*minio.Object, error)
 }
 
 type Resolver interface {
@@ -38,16 +39,19 @@ func NewIndexS3Resolver(client s3ClientInterface, bucket string) *IndexS3Resolve
 }
 
 func (i *IndexS3Resolver) Resolve(ctx context.Context, obj string) (string, error) {
-	objInfo, err := i.client.StatObject(ctx, i.bucket, obj, minio.StatObjectOptions{})
+	object, err := i.client.GetObject(ctx, i.bucket, obj, minio.GetObjectOptions{})
 	if err != nil {
-		return "", err
+		return "", xerrors.Errorf("failed to resolve link: %w", err)
 	}
 
-	if v, ok := objInfo.Metadata["X-Amz-Website-Redirect-Location"]; ok {
-		return v[0], nil
+	data, err := io.ReadAll(object)
+	if err != nil {
+		return "", xerrors.Errorf("failed to resolve link: %w", err)
 	}
 
-	return "", fmt.Errorf("failed to resolve link")
+	logger.Infow("resolved", "link", string(data))
+
+	return strings.TrimSpace(string(data)), nil
 }
 
 type cacheMetadata struct {