feat: stand-alone port s3 server (#6242)

itsHenry35 · web-flow · commit 9c84b6596f4d · 2024-03-24T15:16:00.000+08:00
* feat: single port s3 server

* fix: unable to PUT files if not in root dir
diff --git a/cmd/server.go b/cmd/server.go
@@ -91,6 +91,27 @@ the address is defined in config file`,
 				}
 			}()
 		}
+		s3r := gin.New()
+		s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out))
+		server.InitS3(s3r)
+		if conf.Conf.S3.Port != -1 {
+			s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port)
+			utils.Log.Infof("start S3 server @ %s", s3Base)
+			go func() {
+				var err error
+				if conf.Conf.S3.SSL {
+					httpsSrv = &http.Server{Addr: s3Base, Handler: s3r}
+					err = httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile)
+				}
+				if !conf.Conf.S3.SSL {
+					httpSrv = &http.Server{Addr: s3Base, Handler: s3r}
+					err = httpSrv.ListenAndServe()
+				}
+				if err != nil && !errors.Is(err, http.ErrServerClosed) {
+					utils.Log.Fatalf("failed to start s3 server: %s", err.Error())
+				}
+			}()
+		}
 		// Wait for interrupt signal to gracefully shutdown the server with
 		// a timeout of 1 second.
 		quit := make(chan os.Signal, 1)
diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go
@@ -180,7 +180,6 @@ func InitialSettings() []model.SettingItem {
 		{Key: conf.LdapLoginTips, Value: "login with ldap", Type: conf.TypeString, Group: model.LDAP, Flag: model.PUBLIC},
 
 		//s3 settings
-		{Key: conf.S3Enabled, Value: "false", Type: conf.TypeBool, Group: model.S3, Flag: model.PRIVATE},
 		{Key: conf.S3AccessKeyId, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE},
 		{Key: conf.S3SecretAccessKey, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE},
 		{Key: conf.S3Buckets, Value: "[]", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE},
diff --git a/internal/conf/config.go b/internal/conf/config.go
@@ -1,9 +1,10 @@
 package conf
 
 import (
+	"path/filepath"
+
 	"github.com/alist-org/alist/v3/cmd/flags"
 	"github.com/alist-org/alist/v3/pkg/utils/random"
-	"path/filepath"
 )
 
 type Database struct {
@@ -63,6 +64,12 @@ type Cors struct {
 	AllowHeaders []string `json:"allow_headers" env:"ALLOW_HEADERS"`
 }
 
+type S3 struct {
+	Enable bool `json:"enable" env:"ENABLE"`
+	Port   int  `json:"port" env:"PORT"`
+	SSL    bool `json:"ssl" env:"SSL"`
+}
+
 type Config struct {
 	Force                 bool        `json:"force" env:"FORCE"`
 	SiteURL               string      `json:"site_url" env:"SITE_URL"`
@@ -81,6 +88,7 @@ type Config struct {
 	TlsInsecureSkipVerify bool        `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"`
 	Tasks                 TasksConfig `json:"tasks" envPrefix:"TASKS_"`
 	Cors                  Cors        `json:"cors" envPrefix:"CORS_"`
+	S3                    S3          `json:"s3" envPrefix:"S3_"`
 }
 
 func DefaultConfig() *Config {
@@ -142,5 +150,10 @@ func DefaultConfig() *Config {
 			AllowMethods: []string{"*"},
 			AllowHeaders: []string{"*"},
 		},
+		S3: S3{
+			Enable: false,
+			Port:   5246,
+			SSL:    false,
+		},
 	}
 }
diff --git a/internal/conf/const.go b/internal/conf/const.go
@@ -85,10 +85,9 @@ const (
 	LdapLoginTips         = "ldap_login_tips"
 
 	//s3
-	S3Enabled         = "s3_enabled"
+	S3Buckets         = "s3_buckets"
 	S3AccessKeyId     = "s3_access_key_id"
 	S3SecretAccessKey = "s3_secret_access_key"
-	S3Buckets         = "s3_buckets"
 
 	// qbittorrent
 	QbittorrentUrl      = "qbittorrent_url"
diff --git a/server/router.go b/server/router.go
@@ -171,3 +171,8 @@ func Cors(r *gin.Engine) {
 	config.AllowMethods = conf.Conf.Cors.AllowMethods
 	r.Use(cors.New(config))
 }
+
+func InitS3(e *gin.Engine) {
+	Cors(e)
+	S3Server(e.Group("/"))
+}
diff --git a/server/s3.go b/server/s3.go
@@ -6,24 +6,40 @@ import (
 	"strings"
 
 	"github.com/alist-org/alist/v3/internal/conf"
-	"github.com/alist-org/alist/v3/internal/setting"
 	"github.com/alist-org/alist/v3/server/common"
 	"github.com/alist-org/alist/v3/server/s3"
 	"github.com/gin-gonic/gin"
 )
 
 func S3(g *gin.RouterGroup) {
-	if !setting.GetBool(conf.S3Enabled) {
+	if !conf.Conf.S3.Enable {
 		g.Any("/*path", func(c *gin.Context) {
 			common.ErrorStrResp(c, "S3 server is not enabled", 403)
 		})
 		return
 	}
-	h, _ := s3.NewServer(context.Background(), []string{setting.GetStr(conf.S3AccessKeyId) + "," + setting.GetStr(conf.S3SecretAccessKey)})
+	if conf.Conf.S3.Port != -1 {
+		g.Any("/*path", func(c *gin.Context) {
+			common.ErrorStrResp(c, "S3 server bound to single port", 403)
+		})
+		return
+	}
+	h, _ := s3.NewServer(context.Background())
 
 	g.Any("/*path", func(c *gin.Context) {
 		adjustedPath := strings.TrimPrefix(c.Request.URL.Path, path.Join(conf.URL.Path, "/s3"))
 		c.Request.URL.Path = adjustedPath
 		gin.WrapH(h)(c)
 	})
 }
+
+func S3Server(g *gin.RouterGroup) {
+	if !conf.Conf.S3.Enable {
+		g.Any("/*path", func(c *gin.Context) {
+			common.ErrorStrResp(c, "S3 server is not enabled", 403)
+		})
+		return
+	}
+	h, _ := s3.NewServer(context.Background())
+	g.Any("/*path", gin.WrapH(h))
+}
diff --git a/server/s3/backend.go b/server/s3/backend.go
@@ -299,7 +299,7 @@ func (b *s3Backend) PutObject(
 		Mimetype: meta["Content-Type"],
 	}
 
-	err = fs.PutDirectly(ctx, path.Dir(reqPath), stream)
+	err = fs.PutDirectly(ctx, reqPath, stream)
 	if err != nil {
 		return result, err
 	}
diff --git a/server/s3/server.go b/server/s3/server.go
@@ -11,15 +11,15 @@ import (
 )
 
 // Make a new S3 Server to serve the remote
-func NewServer(ctx context.Context, authpair []string) (h http.Handler, err error) {
+func NewServer(ctx context.Context) (h http.Handler, err error) {
 	var newLogger logger
 	faker := gofakes3.New(
 		newBackend(),
 		// gofakes3.WithHostBucket(!opt.pathBucketMode),
 		gofakes3.WithLogger(newLogger),
 		gofakes3.WithRequestID(rand.Uint64()),
 		gofakes3.WithoutVersioning(),
-		gofakes3.WithV4Auth(authlistResolver(authpair)),
+		gofakes3.WithV4Auth(authlistResolver()),
 		gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied
 	)
 
diff --git a/server/s3/utils.go b/server/s3/utils.go
@@ -5,7 +5,6 @@ package s3
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"strings"
 
 	"github.com/Mikubill/gofakes3"
@@ -15,7 +14,6 @@ import (
 	"github.com/alist-org/alist/v3/internal/model"
 	"github.com/alist-org/alist/v3/internal/op"
 	"github.com/alist-org/alist/v3/internal/setting"
-	"github.com/alist-org/alist/v3/pkg/utils"
 )
 
 type Bucket struct {
@@ -150,15 +148,13 @@ func prefixParser(p *gofakes3.Prefix) (path, remaining string) {
 // 	}
 // }
 
-func authlistResolver(list []string) map[string]string {
-	authList := make(map[string]string)
-	for _, v := range list {
-		parts := strings.Split(v, ",")
-		if len(parts) != 2 {
-			utils.Log.Infof(fmt.Sprintf("Ignored: invalid auth pair %s", v))
-			continue
-		}
-		authList[parts[0]] = parts[1]
+func authlistResolver() map[string]string {
+	s3accesskeyid := setting.GetStr(conf.S3AccessKeyId)
+	s3secretaccesskey := setting.GetStr(conf.S3SecretAccessKey)
+	if s3accesskeyid == "" && s3secretaccesskey == "" {
+		return nil
 	}
+	authList := make(map[string]string)
+	authList[s3accesskeyid] = s3secretaccesskey
 	return authList
 }

Original file line number	Diff line number	Diff line change
`@@ -171,3 +171,8 @@ func Cors(r *gin.Engine) {`
`171`	`171`	`config.AllowMethods = conf.Conf.Cors.AllowMethods`
`172`	`172`	`r.Use(cors.New(config))`
`173`	`173`	`}`
	`174`	`+`
	`175`	`+func InitS3(e *gin.Engine) {`
	`176`	`+ Cors(e)`
	`177`	`+ S3Server(e.Group("/"))`
	`178`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -299,7 +299,7 @@ func (b *s3Backend) PutObject(`
`299`	`299`	`Mimetype: meta["Content-Type"],`
`300`	`300`	`}`
`301`	`301`
`302`		`- err = fs.PutDirectly(ctx, path.Dir(reqPath), stream)`
	`302`	`+ err = fs.PutDirectly(ctx, reqPath, stream)`
`303`	`303`	`if err != nil {`
`304`	`304`	`return result, err`
`305`	`305`	`}`