feat(infra): add file listing support (#1836)

Author: fanlv

backend/infra/contract/storage/storage.go

@@ -19,6 +19,7 @@ package storage
 import (
 	"context"
 	"io"
+	"time"
 )
 
 //go:generate  mockgen -destination ../../../internal/mock/infra/contract/storage/storage_mock.go -package mock -source storage.go Factory
@@ -28,6 +29,13 @@ type Storage interface {
 	GetObject(ctx context.Context, objectKey string) ([]byte, error)
 	DeleteObject(ctx context.Context, objectKey string) error
 	GetObjectUrl(ctx context.Context, objectKey string, opts ...GetOptFn) (string, error)
+	// ListObjects returns all objects with the specified prefix.
+	// It may return a large number of objects, consider using ListObjectsPaginated for better performance.
+	ListObjects(ctx context.Context, prefix string) ([]*FileInfo, error)
+
+	// ListObjectsPaginated returns objects with pagination support.
+	// Use this method when dealing with large number of objects.
+	ListObjectsPaginated(ctx context.Context, input *ListObjectsPaginatedInput) (*ListObjectsPaginatedOutput, error)
 }
 
 type SecurityToken struct {
@@ -37,3 +45,23 @@ type SecurityToken struct {
 	ExpiredTime     string `thrift:"expired_time,4" frugal:"4,default,string" json:"expired_time"`
 	CurrentTime     string `thrift:"current_time,5" frugal:"5,default,string" json:"current_time"`
 }
+
+type ListObjectsPaginatedInput struct {
+	Prefix   string
+	PageSize int
+	Cursor   string
+}
+
+type ListObjectsPaginatedOutput struct {
+	Files  []*FileInfo
+	Cursor string
+	// false: All results have been returned
+	// true: There are more results to return
+	IsTruncated bool
+}
+type FileInfo struct {
+	Key          string
+	LastModified time.Time
+	ETag         string
+	Size         int64
+}

backend/infra/impl/storage/minio/minio.go

@@ -24,34 +24,30 @@ import (
 	"log"
 	"math/rand"
 	"net/url"
-	"os"
-	"strings"
 	"time"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 
-	"github.com/coze-dev/coze-studio/backend/infra/contract/imagex"
 	"github.com/coze-dev/coze-studio/backend/infra/contract/storage"
 	"github.com/coze-dev/coze-studio/backend/infra/impl/storage/proxy"
-	"github.com/coze-dev/coze-studio/backend/pkg/ctxcache"
-	"github.com/coze-dev/coze-studio/backend/types/consts"
+	"github.com/coze-dev/coze-studio/backend/pkg/logs"
 )
 
 type minioClient struct {
-	host            string
 	client          *minio.Client
 	accessKeyID     string
 	secretAccessKey string
 	bucketName      string
 	endpoint        string
 }
 
-func NewStorageImagex(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (imagex.ImageX, error) {
+func New(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (storage.Storage, error) {
 	m, err := getMinioClient(ctx, endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
 	if err != nil {
 		return nil, err
 	}
+
 	return m, nil
 }
 
@@ -76,14 +72,8 @@ func getMinioClient(_ context.Context, endpoint, accessKeyID, secretAccessKey, b
 	if err != nil {
 		return nil, fmt.Errorf("init minio client failed %v", err)
 	}
-	return m, nil
-}
 
-func New(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (storage.Storage, error) {
-	m, err := getMinioClient(ctx, endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
-	if err != nil {
-		return nil, err
-	}
+	// m.test()
 	return m, nil
 }
 
@@ -109,6 +99,8 @@ func (m *minioClient) test() {
 	ctx := context.Background()
 	objectName := fmt.Sprintf("test-file-%d.txt", rand.Int())
 
+	m.ListObjects(ctx, "")
+
 	err := m.PutObject(ctx, objectName, []byte("hello content"), storage.WithContentType("text/plain"))
 	if err != nil {
 		log.Fatalf("upload file failed: %v", err)
@@ -223,47 +215,48 @@ func (m *minioClient) GetObjectUrl(ctx context.Context, objectKey string, opts .
 	return presignedURL.String(), nil
 }
 
-func (m *minioClient) GetUploadHost(ctx context.Context) string {
-	currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx)
-	if !ok {
-		return ""
+func (m *minioClient) ListObjectsPaginated(ctx context.Context, input *storage.ListObjectsPaginatedInput) (*storage.ListObjectsPaginatedOutput, error) {
+	if input == nil {
+		return nil, fmt.Errorf("input cannot be nil")
 	}
-	return currentHost + consts.ApplyUploadActionURI
-}
-
-func (m *minioClient) GetServerID() string {
-	return ""
-}
-
-func (m *minioClient) GetUploadAuth(ctx context.Context, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) {
-	scheme := strings.ToLower(os.Getenv(consts.StorageUploadHTTPScheme))
-	if scheme == "" {
-		scheme = "http"
+	if input.PageSize <= 0 {
+		return nil, fmt.Errorf("page size must be positive")
 	}
-	return &imagex.SecurityToken{
-		AccessKeyID:     "",
-		SecretAccessKey: "",
-		SessionToken:    "",
-		ExpiredTime:     time.Now().Add(time.Hour).Format("2006-01-02 15:04:05"),
-		CurrentTime:     time.Now().Format("2006-01-02 15:04:05"),
-		HostScheme:      scheme,
-	}, nil
-}
 
-func (m *minioClient) GetResourceURL(ctx context.Context, uri string, opts ...imagex.GetResourceOpt) (*imagex.ResourceURL, error) {
-	url, err := m.GetObjectUrl(ctx, uri)
+	files, err := m.ListObjects(ctx, input.Prefix)
 	if err != nil {
 		return nil, err
 	}
-	return &imagex.ResourceURL{
-		URL: url,
+
+	return &storage.ListObjectsPaginatedOutput{
+		Files:       files,
+		IsTruncated: false,
+		Cursor:      "",
 	}, nil
 }
 
-func (m *minioClient) Upload(ctx context.Context, data []byte, opts ...imagex.UploadAuthOpt) (*imagex.UploadResult, error) {
-	return nil, nil
-}
+func (m *minioClient) ListObjects(ctx context.Context, prefix string) ([]*storage.FileInfo, error) {
+	opts := minio.ListObjectsOptions{
+		Prefix:    prefix,
+		Recursive: true,
+	}
+
+	objectCh := m.client.ListObjects(ctx, m.bucketName, opts)
+
+	var files []*storage.FileInfo
+	for object := range objectCh {
+		if object.Err != nil {
+			return nil, object.Err
+		}
+		files = append(files, &storage.FileInfo{
+			Key:          object.Key,
+			LastModified: object.LastModified,
+			ETag:         object.ETag,
+			Size:         object.Size,
+		})
+
+		logs.CtxDebugf(ctx, "key = %s, lastModified = %s, eTag = %s, size = %d", object.Key, object.LastModified, object.ETag, object.Size)
+	}
 
-func (m *minioClient) GetUploadAuthWithExpire(ctx context.Context, expire time.Duration, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) {
-	return nil, nil
+	return files, nil
 }

backend/infra/impl/storage/minio/minio_imagex.go

@@ -0,0 +1,81 @@
+/*
+ * Copyright 2025 coze-dev Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package minio
+
+import (
+	"context"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/coze-dev/coze-studio/backend/infra/contract/imagex"
+	"github.com/coze-dev/coze-studio/backend/pkg/ctxcache"
+	"github.com/coze-dev/coze-studio/backend/types/consts"
+)
+
+func NewStorageImagex(ctx context.Context, endpoint, accessKeyID, secretAccessKey, bucketName string, useSSL bool) (imagex.ImageX, error) {
+	m, err := getMinioClient(ctx, endpoint, accessKeyID, secretAccessKey, bucketName, useSSL)
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (m *minioClient) GetUploadHost(ctx context.Context) string {
+	currentHost, ok := ctxcache.Get[string](ctx, consts.HostKeyInCtx)
+	if !ok {
+		return ""
+	}
+	return currentHost + consts.ApplyUploadActionURI
+}
+
+func (m *minioClient) GetServerID() string {
+	return ""
+}
+
+func (m *minioClient) GetUploadAuth(ctx context.Context, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) {
+	scheme := strings.ToLower(os.Getenv(consts.StorageUploadHTTPScheme))
+	if scheme == "" {
+		scheme = "http"
+	}
+	return &imagex.SecurityToken{
+		AccessKeyID:     "",
+		SecretAccessKey: "",
+		SessionToken:    "",
+		ExpiredTime:     time.Now().Add(time.Hour).Format("2006-01-02 15:04:05"),
+		CurrentTime:     time.Now().Format("2006-01-02 15:04:05"),
+		HostScheme:      scheme,
+	}, nil
+}
+
+func (m *minioClient) GetResourceURL(ctx context.Context, uri string, opts ...imagex.GetResourceOpt) (*imagex.ResourceURL, error) {
+	url, err := m.GetObjectUrl(ctx, uri)
+	if err != nil {
+		return nil, err
+	}
+	return &imagex.ResourceURL{
+		URL: url,
+	}, nil
+}
+
+func (m *minioClient) Upload(ctx context.Context, data []byte, opts ...imagex.UploadAuthOpt) (*imagex.UploadResult, error) {
+	return nil, nil
+}
+
+func (m *minioClient) GetUploadAuthWithExpire(ctx context.Context, expire time.Duration, opt ...imagex.UploadAuthOpt) (*imagex.SecurityToken, error) {
+	return nil, nil
+}