Add support for ali oss (#327)

hannatao · web-flow · commit efe124f7750a · 2021-06-18T16:25:08.000+08:00
diff --git a/api/object.go b/api/object.go
@@ -8,6 +8,7 @@ import (
 const (
 	CurrentAccount = "current"
 	OtherAccount   = "other"
+	PathStyle      = "pathStyle"
 )
 
 // ListObjectSourcesV2 ListObjectSourcesV2
@@ -62,7 +63,7 @@ func (api *API) ListBucketObjectsV2(c *common.Context) (interface{}, error) {
 
 func (api *API) parseObject(c *common.Context) (*models.ObjectRequestParams, error) {
 	params := &models.ObjectRequestParams{}
-	params.ExternalObjectInfo.PathStyle = true
+	params.ExternalObjectInfo.AddressFormat = PathStyle
 	if err := c.Bind(params); err != nil {
 		return nil, err
 	}
diff --git a/api/object_test.go b/api/object_test.go
@@ -83,10 +83,10 @@ func TestListBucketsV2(t *testing.T) {
 	assert.Equal(t, http.StatusOK, w.Code)
 
 	info := models.ExternalObjectInfo{
-		Endpoint:  "x",
-		Ak:        "xx",
-		Sk:        "xxx",
-		PathStyle: true,
+		Endpoint:      "x",
+		Ak:            "xx",
+		Sk:            "xxx",
+		AddressFormat: PathStyle,
 	}
 	mkObjectService.EXPECT().ListExternalBuckets(info, "baidubos").Return(buckets, nil).Times(1)
 	req, _ = http.NewRequest(http.MethodGet, "/v2/objects/baidubos/buckets?account=other&endpoint=x&ak=xx&sk=xxx", nil)
@@ -186,10 +186,10 @@ func TestListBucketObjectsV2(t *testing.T) {
 	assert.Equal(t, http.StatusNotFound, w4.Code)
 
 	info := models.ExternalObjectInfo{
-		Endpoint:  "x",
-		Ak:        "xx",
-		Sk:        "xxx",
-		PathStyle: true,
+		Endpoint:      "x",
+		Ak:            "xx",
+		Sk:            "xxx",
+		AddressFormat: PathStyle,
 	}
 	mkObjectService.EXPECT().ListExternalBucketObjects(info, "baetyl-test", "baidubos").Return(objectsResult, nil).Times(1)
 	req, _ = http.NewRequest(http.MethodGet, "/v2/objects/baidubos/buckets/baetyl-test/objects?account=other&endpoint=x&ak=xx&sk=xxx", nil)
diff --git a/models/configuration.go b/models/configuration.go
@@ -40,15 +40,16 @@ type ConfigFunctionItem struct {
 }
 
 type ConfigObjectItem struct {
-	Source   string `json:"source,omitempty"`
-	Account  string `json:"account,omitempty"`
-	Endpoint string `json:"endpoint,omitempty"`
-	Bucket   string `json:"bucket,omitempty"`
-	Object   string `json:"object,omitempty"`
-	Unpack   string `json:"unpack,omitempty"`
-	MD5      string `json:"md5,omitempty"`
-	Ak       string `json:"ak,omitempty"`
-	Sk       string `json:"sk,omitempty"`
+	Source        string `json:"source,omitempty"`
+	Account       string `json:"account,omitempty"`
+	Endpoint      string `json:"endpoint,omitempty"`
+	Bucket        string `json:"bucket,omitempty"`
+	Object        string `json:"object,omitempty"`
+	Unpack        string `json:"unpack,omitempty"`
+	MD5           string `json:"md5,omitempty"`
+	Ak            string `json:"ak,omitempty"`
+	Sk            string `json:"sk,omitempty"`
+	AddressFormat string `json:"addressFormat,omitempty" default:"pathStyle"`
 }
 
 func EqualConfig(config1, config2 *specV1.Configuration) bool {
diff --git a/models/object.go b/models/object.go
@@ -93,8 +93,8 @@ type ObjectRequestParams struct {
 }
 
 type ExternalObjectInfo struct {
-	Endpoint  string `form:"endpoint,omitempty"`
-	Ak        string `form:"ak,omitempty"`
-	Sk        string `form:"sk,omitempty"`
-	PathStyle bool   `form:"pathStyle,omitempty"`
+	Endpoint      string `form:"endpoint,omitempty"`
+	Ak            string `form:"ak,omitempty"`
+	Sk            string `form:"sk,omitempty"`
+	AddressFormat string `form:"addressFormat,omitempty"`
 }
diff --git a/plugin/awss3/awss3.go b/plugin/awss3/awss3.go
@@ -21,6 +21,10 @@ import (
 	"github.com/baetyl/baetyl-cloud/v2/plugin"
 )
 
+const (
+	VirtualHost = "virtualHost"
+)
+
 type awss3Storage struct {
 	s3Client *s3.S3
 	cfg      *S3Config
@@ -41,7 +45,7 @@ func New() (plugin.Plugin, error) {
 		return new(awss3Storage), nil
 	}
 
-	sessionProvider, err := newS3Session(cfg.AWSS3.Endpoint, cfg.AWSS3.Ak, cfg.AWSS3.Sk, cfg.AWSS3.Region, cfg.AWSS3.PathStyle)
+	sessionProvider, err := newS3Session(cfg.AWSS3.Endpoint, cfg.AWSS3.Ak, cfg.AWSS3.Sk, cfg.AWSS3.Region, cfg.AWSS3.AddressFormat)
 	if err != nil {
 		return nil, err
 	}
@@ -261,7 +265,7 @@ func (c *awss3Storage) GenInternalObjectURL(_, bucket, object string) (*models.O
 
 // ListExternalBuckets ListExternalBuckets
 func (c *awss3Storage) ListExternalBuckets(info models.ExternalObjectInfo) ([]models.Bucket, error) {
-	sessionProvider, err := newS3Session(info.Endpoint, info.Ak, info.Sk, "", info.PathStyle)
+	sessionProvider, err := newS3Session(info.Endpoint, info.Ak, info.Sk, "", info.AddressFormat)
 	if err != nil {
 		return nil, err
 	}
@@ -271,7 +275,7 @@ func (c *awss3Storage) ListExternalBuckets(info models.ExternalObjectInfo) ([]mo
 
 // HeadExternalBucket HeadExternalBucket
 func (c *awss3Storage) HeadExternalBucket(info models.ExternalObjectInfo, bucket string) error {
-	sessionProvider, err := newS3Session(info.Endpoint, info.Ak, info.Sk, "", info.PathStyle)
+	sessionProvider, err := newS3Session(info.Endpoint, info.Ak, info.Sk, "", info.AddressFormat)
 	if err != nil {
 		return err
 	}
@@ -281,7 +285,7 @@ func (c *awss3Storage) HeadExternalBucket(info models.ExternalObjectInfo, bucket
 
 // ListExternalBucketObjects ListExternalBucketObjects
 func (c *awss3Storage) ListExternalBucketObjects(info models.ExternalObjectInfo, bucket string, params *models.ObjectParams) (*models.ListObjectsResult, error) {
-	sessionProvider, err := newS3Session(info.Endpoint, info.Ak, info.Sk, "", info.PathStyle)
+	sessionProvider, err := newS3Session(info.Endpoint, info.Ak, info.Sk, "", info.AddressFormat)
 	if err != nil {
 		return nil, err
 	}
@@ -298,7 +302,7 @@ func (c *awss3Storage) ListExternalBucketObjects(info models.ExternalObjectInfo,
 
 // GenExternalObjectURL GenExternalObjectURL
 func (c *awss3Storage) GenExternalObjectURL(info models.ExternalObjectInfo, bucket, object string) (*models.ObjectURL, error) {
-	sessionProvider, err := newS3Session(info.Endpoint, info.Ak, info.Sk, "", info.PathStyle)
+	sessionProvider, err := newS3Session(info.Endpoint, info.Ak, info.Sk, "", info.AddressFormat)
 	if err != nil {
 		return nil, err
 	}
@@ -325,11 +329,15 @@ func (c *awss3Storage) checkInternalSupported() error {
 	return nil
 }
 
-func newS3Session(endpoint, ak, sk, region string, pathStyle bool) (*session.Session, error) {
+func newS3Session(endpoint, ak, sk, region, addressFormat string) (*session.Session, error) {
 	if region == "" {
 		region = "us-east-1"
 	}
 
+	pathStyle := true
+	if addressFormat == VirtualHost {
+		pathStyle = false
+	}
 	s3Config := &aws.Config{
 		Credentials:      credentials.NewStaticCredentials(ak, sk, ""),
 		Endpoint:         aws.String(endpoint),
diff --git a/plugin/awss3/awss3_config.go b/plugin/awss3/awss3_config.go
@@ -8,10 +8,10 @@ type CloudConfig struct {
 }
 
 type S3Config struct {
-	Endpoint   string        `yaml:"endpoint" json:"endpoint"`
-	Ak         string        `yaml:"ak" json:"ak" validate:"nonzero"`
-	Sk         string        `yaml:"sk" json:"sk" validate:"nonzero"`
-	Region     string        `yaml:"region" json:"region" default:"us-east-1"`
-	PathStyle  bool          `yaml:"pathStyle" json:"pathStyle" default:"true"`
-	Expiration time.Duration `yaml:"expiration" json:"expiration" default:"1h"`
+	Endpoint      string        `yaml:"endpoint" json:"endpoint"`
+	Ak            string        `yaml:"ak" json:"ak" validate:"nonzero"`
+	Sk            string        `yaml:"sk" json:"sk" validate:"nonzero"`
+	Region        string        `yaml:"region" json:"region" default:"us-east-1"`
+	AddressFormat string        `yaml:"addressFormat" json:"addressFormat" default:"pathStyle"`
+	Expiration    time.Duration `yaml:"expiration" json:"expiration" default:"1h"`
 }
diff --git a/service/sync.go b/service/sync.go
@@ -212,9 +212,10 @@ func (t *SyncServiceImpl) PopulateConfigObject(k, v string, cfg *specV1.Configur
 		res, err = t.ObjectService.GenInternalObjectURL(obj.Metadata["userID"], item.Bucket, item.Object, item.Source)
 	} else {
 		res, err = t.ObjectService.GenExternalObjectURL(models.ExternalObjectInfo{
-			Endpoint: item.Endpoint,
-			Ak:       item.Ak,
-			Sk:       item.Sk,
+			Endpoint:      item.Endpoint,
+			Ak:            item.Ak,
+			Sk:            item.Sk,
+			AddressFormat: item.AddressFormat,
 		}, item.Bucket, item.Object, item.Source)
 	}
 	if err != nil {

Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,7 @@ import (`
`8`	`8`	`const (`
`9`	`9`	`CurrentAccount = "current"`
`10`	`10`	`OtherAccount = "other"`
	`11`	`+ PathStyle = "pathStyle"`
`11`	`12`	`)`
`12`	`13`
`13`	`14`	`// ListObjectSourcesV2 ListObjectSourcesV2`
`@@ -62,7 +63,7 @@ func (api API) ListBucketObjectsV2(c common.Context) (interface{}, error) {`
`62`	`63`
`63`	`64`	`func (api API) parseObject(c common.Context) (*models.ObjectRequestParams, error) {`
`64`	`65`	`params := &models.ObjectRequestParams{}`
`65`		`- params.ExternalObjectInfo.PathStyle = true`
	`66`	`+ params.ExternalObjectInfo.AddressFormat = PathStyle`
`66`	`67`	`if err := c.Bind(params); err != nil {`
`67`	`68`	`return nil, err`
`68`	`69`	`}`
Original file line number	Diff line number	Diff line change
`@@ -93,8 +93,8 @@ type ObjectRequestParams struct {`
`93`	`93`	`}`
`94`	`94`
`95`	`95`	`type ExternalObjectInfo struct {`
`96`		- Endpoint string `form:"endpoint,omitempty"`
`97`		- Ak string `form:"ak,omitempty"`
`98`		- Sk string `form:"sk,omitempty"`
`99`		- PathStyle bool `form:"pathStyle,omitempty"`
	`96`	+ Endpoint string `form:"endpoint,omitempty"`
	`97`	+ Ak string `form:"ak,omitempty"`
	`98`	+ Sk string `form:"sk,omitempty"`
	`99`	+ AddressFormat string `form:"addressFormat,omitempty"`
`100`	`100`	`}`