diff --git a/_mocks/opencsg.com/csghub-server/component/mock_StorageGatewayComponent.go b/_mocks/opencsg.com/csghub-server/component/mock_StorageGatewayComponent.go new file mode 100644 index 00000000..b55fa22d --- /dev/null +++ b/_mocks/opencsg.com/csghub-server/component/mock_StorageGatewayComponent.go @@ -0,0 +1,376 @@ +// Code generated by mockery v2.53.5. DO NOT EDIT. + +package component + +import ( + context "context" + io "io" + + mock "github.com/stretchr/testify/mock" + + time "time" + + types "opencsg.com/csghub-server/common/types" +) + +// MockStorageGatewayComponent is an autogenerated mock type for the StorageGatewayComponent type +type MockStorageGatewayComponent struct { + mock.Mock +} + +type MockStorageGatewayComponent_Expecter struct { + mock *mock.Mock +} + +func (_m *MockStorageGatewayComponent) EXPECT() *MockStorageGatewayComponent_Expecter { + return &MockStorageGatewayComponent_Expecter{mock: &_m.Mock} +} + +// DeleteObject provides a mock function with given fields: ctx, bucket, key +func (_m *MockStorageGatewayComponent) DeleteObject(ctx context.Context, bucket string, key string) error { + ret := _m.Called(ctx, bucket, key) + + if len(ret) == 0 { + panic("no return value specified for DeleteObject") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { + r0 = rf(ctx, bucket, key) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStorageGatewayComponent_DeleteObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteObject' +type MockStorageGatewayComponent_DeleteObject_Call struct { + *mock.Call +} + +// DeleteObject is a helper method to define mock.On call +// - ctx context.Context +// - bucket string +// - key string +func (_e *MockStorageGatewayComponent_Expecter) DeleteObject(ctx interface{}, bucket interface{}, key interface{}) *MockStorageGatewayComponent_DeleteObject_Call { + return &MockStorageGatewayComponent_DeleteObject_Call{Call: _e.mock.On("DeleteObject", ctx, bucket, key)} +} + +func (_c *MockStorageGatewayComponent_DeleteObject_Call) Run(run func(ctx context.Context, bucket string, key string)) *MockStorageGatewayComponent_DeleteObject_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockStorageGatewayComponent_DeleteObject_Call) Return(_a0 error) *MockStorageGatewayComponent_DeleteObject_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockStorageGatewayComponent_DeleteObject_Call) RunAndReturn(run func(context.Context, string, string) error) *MockStorageGatewayComponent_DeleteObject_Call { + _c.Call.Return(run) + return _c +} + +// GetDirectURL provides a mock function with given fields: bucket, key +func (_m *MockStorageGatewayComponent) GetDirectURL(bucket string, key string) (string, error) { + ret := _m.Called(bucket, key) + + if len(ret) == 0 { + panic("no return value specified for GetDirectURL") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(string, string) (string, error)); ok { + return rf(bucket, key) + } + if rf, ok := ret.Get(0).(func(string, string) string); ok { + r0 = rf(bucket, key) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(bucket, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockStorageGatewayComponent_GetDirectURL_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDirectURL' +type MockStorageGatewayComponent_GetDirectURL_Call struct { + *mock.Call +} + +// GetDirectURL is a helper method to define mock.On call +// - bucket string +// - key string +func (_e *MockStorageGatewayComponent_Expecter) GetDirectURL(bucket interface{}, key interface{}) *MockStorageGatewayComponent_GetDirectURL_Call { + return &MockStorageGatewayComponent_GetDirectURL_Call{Call: _e.mock.On("GetDirectURL", bucket, key)} +} + +func (_c *MockStorageGatewayComponent_GetDirectURL_Call) Run(run func(bucket string, key string)) *MockStorageGatewayComponent_GetDirectURL_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string)) + }) + return _c +} + +func (_c *MockStorageGatewayComponent_GetDirectURL_Call) Return(_a0 string, _a1 error) *MockStorageGatewayComponent_GetDirectURL_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockStorageGatewayComponent_GetDirectURL_Call) RunAndReturn(run func(string, string) (string, error)) *MockStorageGatewayComponent_GetDirectURL_Call { + _c.Call.Return(run) + return _c +} + +// GetObject provides a mock function with given fields: ctx, bucket, key +func (_m *MockStorageGatewayComponent) GetObject(ctx context.Context, bucket string, key string) (*types.StorageObjectResponse, error) { + ret := _m.Called(ctx, bucket, key) + + if len(ret) == 0 { + panic("no return value specified for GetObject") + } + + var r0 *types.StorageObjectResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (*types.StorageObjectResponse, error)); ok { + return rf(ctx, bucket, key) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) *types.StorageObjectResponse); ok { + r0 = rf(ctx, bucket, key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.StorageObjectResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, bucket, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockStorageGatewayComponent_GetObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetObject' +type MockStorageGatewayComponent_GetObject_Call struct { + *mock.Call +} + +// GetObject is a helper method to define mock.On call +// - ctx context.Context +// - bucket string +// - key string +func (_e *MockStorageGatewayComponent_Expecter) GetObject(ctx interface{}, bucket interface{}, key interface{}) *MockStorageGatewayComponent_GetObject_Call { + return &MockStorageGatewayComponent_GetObject_Call{Call: _e.mock.On("GetObject", ctx, bucket, key)} +} + +func (_c *MockStorageGatewayComponent_GetObject_Call) Run(run func(ctx context.Context, bucket string, key string)) *MockStorageGatewayComponent_GetObject_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockStorageGatewayComponent_GetObject_Call) Return(_a0 *types.StorageObjectResponse, _a1 error) *MockStorageGatewayComponent_GetObject_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockStorageGatewayComponent_GetObject_Call) RunAndReturn(run func(context.Context, string, string) (*types.StorageObjectResponse, error)) *MockStorageGatewayComponent_GetObject_Call { + _c.Call.Return(run) + return _c +} + +// HeadObject provides a mock function with given fields: ctx, bucket, key +func (_m *MockStorageGatewayComponent) HeadObject(ctx context.Context, bucket string, key string) (*types.StorageObjectInfo, error) { + ret := _m.Called(ctx, bucket, key) + + if len(ret) == 0 { + panic("no return value specified for HeadObject") + } + + var r0 *types.StorageObjectInfo + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) (*types.StorageObjectInfo, error)); ok { + return rf(ctx, bucket, key) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) *types.StorageObjectInfo); ok { + r0 = rf(ctx, bucket, key) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.StorageObjectInfo) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, bucket, key) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockStorageGatewayComponent_HeadObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HeadObject' +type MockStorageGatewayComponent_HeadObject_Call struct { + *mock.Call +} + +// HeadObject is a helper method to define mock.On call +// - ctx context.Context +// - bucket string +// - key string +func (_e *MockStorageGatewayComponent_Expecter) HeadObject(ctx interface{}, bucket interface{}, key interface{}) *MockStorageGatewayComponent_HeadObject_Call { + return &MockStorageGatewayComponent_HeadObject_Call{Call: _e.mock.On("HeadObject", ctx, bucket, key)} +} + +func (_c *MockStorageGatewayComponent_HeadObject_Call) Run(run func(ctx context.Context, bucket string, key string)) *MockStorageGatewayComponent_HeadObject_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockStorageGatewayComponent_HeadObject_Call) Return(_a0 *types.StorageObjectInfo, _a1 error) *MockStorageGatewayComponent_HeadObject_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockStorageGatewayComponent_HeadObject_Call) RunAndReturn(run func(context.Context, string, string) (*types.StorageObjectInfo, error)) *MockStorageGatewayComponent_HeadObject_Call { + _c.Call.Return(run) + return _c +} + +// PresignURL provides a mock function with given fields: ctx, bucket, key, method, expiration +func (_m *MockStorageGatewayComponent) PresignURL(ctx context.Context, bucket string, key string, method string, expiration time.Duration) (string, error) { + ret := _m.Called(ctx, bucket, key, method, expiration) + + if len(ret) == 0 { + panic("no return value specified for PresignURL") + } + + var r0 string + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, time.Duration) (string, error)); ok { + return rf(ctx, bucket, key, method, expiration) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, time.Duration) string); ok { + r0 = rf(ctx, bucket, key, method, expiration) + } else { + r0 = ret.Get(0).(string) + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, string, time.Duration) error); ok { + r1 = rf(ctx, bucket, key, method, expiration) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockStorageGatewayComponent_PresignURL_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PresignURL' +type MockStorageGatewayComponent_PresignURL_Call struct { + *mock.Call +} + +// PresignURL is a helper method to define mock.On call +// - ctx context.Context +// - bucket string +// - key string +// - method string +// - expiration time.Duration +func (_e *MockStorageGatewayComponent_Expecter) PresignURL(ctx interface{}, bucket interface{}, key interface{}, method interface{}, expiration interface{}) *MockStorageGatewayComponent_PresignURL_Call { + return &MockStorageGatewayComponent_PresignURL_Call{Call: _e.mock.On("PresignURL", ctx, bucket, key, method, expiration)} +} + +func (_c *MockStorageGatewayComponent_PresignURL_Call) Run(run func(ctx context.Context, bucket string, key string, method string, expiration time.Duration)) *MockStorageGatewayComponent_PresignURL_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), args[4].(time.Duration)) + }) + return _c +} + +func (_c *MockStorageGatewayComponent_PresignURL_Call) Return(_a0 string, _a1 error) *MockStorageGatewayComponent_PresignURL_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockStorageGatewayComponent_PresignURL_Call) RunAndReturn(run func(context.Context, string, string, string, time.Duration) (string, error)) *MockStorageGatewayComponent_PresignURL_Call { + _c.Call.Return(run) + return _c +} + +// PutObject provides a mock function with given fields: ctx, bucket, key, reader, opts +func (_m *MockStorageGatewayComponent) PutObject(ctx context.Context, bucket string, key string, reader io.Reader, opts types.StoragePutObjectOptions) error { + ret := _m.Called(ctx, bucket, key, reader, opts) + + if len(ret) == 0 { + panic("no return value specified for PutObject") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, io.Reader, types.StoragePutObjectOptions) error); ok { + r0 = rf(ctx, bucket, key, reader, opts) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockStorageGatewayComponent_PutObject_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PutObject' +type MockStorageGatewayComponent_PutObject_Call struct { + *mock.Call +} + +// PutObject is a helper method to define mock.On call +// - ctx context.Context +// - bucket string +// - key string +// - reader io.Reader +// - opts types.StoragePutObjectOptions +func (_e *MockStorageGatewayComponent_Expecter) PutObject(ctx interface{}, bucket interface{}, key interface{}, reader interface{}, opts interface{}) *MockStorageGatewayComponent_PutObject_Call { + return &MockStorageGatewayComponent_PutObject_Call{Call: _e.mock.On("PutObject", ctx, bucket, key, reader, opts)} +} + +func (_c *MockStorageGatewayComponent_PutObject_Call) Run(run func(ctx context.Context, bucket string, key string, reader io.Reader, opts types.StoragePutObjectOptions)) *MockStorageGatewayComponent_PutObject_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(io.Reader), args[4].(types.StoragePutObjectOptions)) + }) + return _c +} + +func (_c *MockStorageGatewayComponent_PutObject_Call) Return(_a0 error) *MockStorageGatewayComponent_PutObject_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockStorageGatewayComponent_PutObject_Call) RunAndReturn(run func(context.Context, string, string, io.Reader, types.StoragePutObjectOptions) error) *MockStorageGatewayComponent_PutObject_Call { + _c.Call.Return(run) + return _c +} + +// NewMockStorageGatewayComponent creates a new instance of MockStorageGatewayComponent. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockStorageGatewayComponent(t interface { + mock.TestingT + Cleanup(func()) +}) *MockStorageGatewayComponent { + mock := &MockStorageGatewayComponent{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/cmd/csghub-server/cmd/root.go b/cmd/csghub-server/cmd/root.go index f7a7814a..22fca7a4 100644 --- a/cmd/csghub-server/cmd/root.go +++ b/cmd/csghub-server/cmd/root.go @@ -3,10 +3,11 @@ package cmd import ( "fmt" "log/slog" - "opencsg.com/csghub-server/cmd/csghub-server/cmd/temporal-worker" - "opencsg.com/csghub-server/common/log" "os" + temporal_worker "opencsg.com/csghub-server/cmd/csghub-server/cmd/temporal-worker" + "opencsg.com/csghub-server/common/log" + "github.com/spf13/cobra" "opencsg.com/csghub-server/cmd/csghub-server/cmd/accounting" "opencsg.com/csghub-server/cmd/csghub-server/cmd/aigateway" diff --git a/common/config/config.go b/common/config/config.go index 6feb5148..d3c3b5e6 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -535,6 +535,11 @@ type Config struct { Endpoint string `env:"STARHUB_SERVER_XNET_ENDPOINT" default:"http://localhost:8097"` ApiKey string `env:"STARHUB_SERVER_XNET_API_KEY" default:"f3a7b9c1d6e5f8e2a1b5d4f9e6a2b8d7c3a4e2b1d9f6e7a8d2c5a7b4c1e3f5b8a1d4f9b7d6e2f8a5d3b1e7f9c6a8b2d1e4f7d5b6e9f2a4b3c8e1d7f995hd82hf"` } + + StorageGateway struct { + PartSize int64 `env:"STARHUB_SERVER_STORAGE_GATEWAY_PART_SIZE" default:"67108864"` // 64MB + EnablePresignedURLProxy bool `env:"STARHUB_SERVER_STORAGE_GATEWAY_ENABLE_PRESIGNED_URL_PROXY" default:"true"` // Enable presigned URL proxy through gateway + } } func SetConfigFile(file string) { diff --git a/common/types/storagegateway.go b/common/types/storagegateway.go new file mode 100644 index 00000000..204565c3 --- /dev/null +++ b/common/types/storagegateway.go @@ -0,0 +1,80 @@ +package types + +import ( + "io" + "time" +) + +// AccessMode represents the access mode for object storage operations +type AccessMode string + +const ( + // ModeDirect allows direct access to OSS without gateway + ModeDirect AccessMode = "direct" + // ModeSigned generates presigned URLs for temporary access + ModeSigned AccessMode = "signed" + // ModeProxy streams objects through the gateway + ModeProxy AccessMode = "proxy" +) + +// PresignRequest represents a request to generate a presigned URL +type PresignRequest struct { + Bucket string `json:"bucket" binding:"required"` + Key string `json:"key" binding:"required"` + Expiration int `json:"expiration"` // Expiration time in seconds, default 3600 + Method string `json:"method"` // HTTP method: GET, PUT, default GET +} + +// PresignResponse represents the response containing a presigned URL +type PresignResponse struct { + URL string `json:"url"` + Expiration int `json:"expiration"` // Expiration time in seconds +} + +// DirectURLResponse represents the response for direct mode +type DirectURLResponse struct { + URL string `json:"url"` +} + +// BucketConfig represents per-bucket access mode configuration +type BucketConfig struct { + AccessMode AccessMode `json:"access_mode"` +} + +type AddressingStyle string + +const ( + AddressingVirtualHost AddressingStyle = "virtual-host" + AddressingPathStyle AddressingStyle = "path" +) + +// StorageObjectResponse represents the response for storage gateway object operations +type StorageObjectResponse struct { + Mode AccessMode + URL string + RedirectURL string + Stream io.ReadCloser + ContentType string + Size int64 + ETag string + LastModified time.Time + ContentRange string + CacheControl string + ContentDisposition string +} + +// StorageObjectInfo represents storage gateway object metadata +type StorageObjectInfo struct { + ContentType string + ContentLength int64 + LastModified time.Time + ETag string +} + +// StoragePutObjectOptions represents options for putting an object in storage gateway +type StoragePutObjectOptions struct { + Size int64 + ContentType string + ContentDisposition string + CacheControl string +} diff --git a/component/git_http.go b/component/git_http.go index 100a0916..ab94fd19 100644 --- a/component/git_http.go +++ b/component/git_http.go @@ -760,7 +760,7 @@ func (c *gitHTTPComponentImpl) buildUploadLink(ctx context.Context, req types.Ba reqHeader.Add("X-Amz-Checksum-SHA256", encodedOid) reqHeader.Add("X-Amz-Checksum-Algorithm", "SHA256") // u, err := c.s3Client.PresignedPutObject(ctx, c.config.S3.Bucket, common.BuildLfsPath(repo.ID, pointer.Oid, repo.Migrated), types.OssFileExpire) - u, err := c.s3Core.PresignHeader( + u, err := c.s3Client.PresignHeader( ctx, http.MethodPut, c.config.S3.Bucket,