[PBM-1239] use storage.Storage interface in tests

allows to perform checks for different storage types

Author: defbin

e2e-tests/docker/conf/minio.yaml

@@ -2,6 +2,7 @@ storage:
   type: s3
   s3:
     endpointUrl: http://minio:9000
+    region: us-east-1
     bucket: bcp
     prefix: pbme2etest
     credentials:

e2e-tests/pkg/tests/sharded/test_backup_cancellation.go

@@ -1,24 +1,18 @@
 package sharded
 
 import (
+	"bytes"
 	"context"
 	"log"
-	"net/url"
 	"os"
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	awsS3 "github.com/aws/aws-sdk-go/service/s3"
-
-	"gopkg.in/yaml.v2"
-
 	"github.com/percona/percona-backup-mongodb/pbm/config"
 	"github.com/percona/percona-backup-mongodb/pbm/defs"
 	"github.com/percona/percona-backup-mongodb/pbm/errors"
-	"github.com/percona/percona-backup-mongodb/pbm/storage/s3"
+	"github.com/percona/percona-backup-mongodb/pbm/storage"
+	"github.com/percona/percona-backup-mongodb/pbm/util"
 )
 
 func (c *Cluster) BackupCancellation(storage string) {
@@ -55,62 +49,39 @@ func (c *Cluster) BackupCancellation(storage string) {
 
 func checkNoBackupFiles(backupName, conf string) {
 	log.Println("check no artifacts left for backup", backupName)
-	buf, err := os.ReadFile(conf)
-	if err != nil {
-		log.Fatalln("Error: unable to read config file:", err)
-	}
 
-	var cfg config.Config
-	err = yaml.UnmarshalStrict(buf, &cfg)
+	files, err := listAllFiles(conf)
 	if err != nil {
-		log.Fatalln("Error: unmarshal yaml:", err)
+		log.Fatalln("ERROR: list files:", err)
 	}
 
-	stg := cfg.Storage
-
-	endopintURL := awsurl
-	if stg.S3.EndpointURL != "" {
-		eu, err := url.Parse(stg.S3.EndpointURL)
-		if err != nil {
-			log.Fatalln("Error: parse EndpointURL:", err)
+	for _, file := range files {
+		if strings.Contains(file.Name, backupName) {
+			log.Fatalln("ERROR: failed to delete lefover", file.Name)
 		}
-		endopintURL = eu.Host
 	}
+}
 
-	ss, err := newS3Client(endopintURL, stg.S3.Region, &stg.S3.Credentials)
+func listAllFiles(confFilepath string) ([]storage.FileInfo, error) {
+	buf, err := os.ReadFile(confFilepath)
 	if err != nil {
-		log.Fatalf("create S3 client: %v", err)
+		return nil, errors.Wrap(err, "read config file")
 	}
 
-	res, err := ss.ListObjectsV2(&awsS3.ListObjectsV2Input{
-		Bucket: &stg.S3.Bucket,
-		Prefix: &stg.S3.Prefix,
-	})
+	cfg, err := config.Parse(bytes.NewBuffer(buf))
 	if err != nil {
-		log.Fatalf("list files on S3: %v", err)
+		return nil, errors.Wrap(err, "parse config")
 	}
 
-	for _, object := range res.Contents {
-		s := object.String()
-		if strings.Contains(s, backupName) {
-			log.Fatalln("Error: failed to delete lefover", object.Key)
-		}
+	stg, err := util.StorageFromConfig(&cfg.Storage, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "storage from config")
 	}
-}
 
-func newS3Client(uri, region string, creds *s3.Credentials) (*awsS3.S3, error) {
-	sess, err := session.NewSession(&aws.Config{
-		Region:   &region,
-		Endpoint: &uri,
-		Credentials: credentials.NewStaticCredentials(
-			creds.AccessKeyID,
-			creds.SecretAccessKey,
-			creds.SessionToken,
-		),
-	})
+	files, err := stg.List("", "")
 	if err != nil {
-		return nil, errors.Wrap(err, "create AWS session")
+		return nil, errors.Wrap(err, "list files")
 	}
 
-	return awsS3.New(sess), nil
+	return files, nil
 }

e2e-tests/pkg/tests/sharded/test_delete_backup.go

@@ -4,15 +4,10 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"net/url"
 	"os"
 	"strings"
 	"time"
 
-	awsS3 "github.com/aws/aws-sdk-go/service/s3"
-	"gopkg.in/yaml.v2"
-
-	"github.com/percona/percona-backup-mongodb/pbm/config"
 	"github.com/percona/percona-backup-mongodb/pbm/ctrl"
 	"github.com/percona/percona-backup-mongodb/pbm/defs"
 	"github.com/percona/percona-backup-mongodb/pbm/lock"
@@ -165,66 +160,33 @@ func (c *Cluster) BackupDelete(storage string) {
 	checkData()
 }
 
-const awsurl = "s3.amazonaws.com"
-
 // checkArtefacts checks if all backups artifacts removed
 // except for the shouldStay
 func checkArtefacts(conf string, shouldStay map[string]struct{}) {
 	log.Println("check all artifacts deleted excepts backup's", shouldStay)
-	buf, err := os.ReadFile(conf)
-	if err != nil {
-		log.Fatalln("ERROR: unable to read config file:", err)
-	}
 
-	var cfg config.Config
-	err = yaml.UnmarshalStrict(buf, &cfg)
+	files, err := listAllFiles(conf)
 	if err != nil {
-		log.Fatalln("ERROR: unmarshal yaml:", err)
+		log.Fatalln("ERROR: list files:", err)
 	}
 
-	stg := cfg.Storage
-
-	if stg.Type == "azure" || stg.Type == "filesystem" {
-		return
-	}
-
-	endopintURL := awsurl
-	if stg.S3.EndpointURL != "" {
-		eu, err := url.Parse(stg.S3.EndpointURL)
-		if err != nil {
-			log.Fatalln("ERROR: parse EndpointURL:", err)
+	for _, file := range files {
+		if strings.Contains(file.Name, defs.StorInitFile) {
+			continue
 		}
-		endopintURL = eu.Host
-	}
-
-	ss, err := newS3Client(endopintURL, stg.S3.Region, &stg.S3.Credentials)
-	if err != nil {
-		log.Fatalf("create S3 client: %v", err)
-	}
-
-	res, err := ss.ListObjectsV2(&awsS3.ListObjectsV2Input{
-		Bucket: &stg.S3.Bucket,
-		Prefix: &stg.S3.Prefix,
-	})
-	if err != nil {
-		log.Fatalf("list files on S3: %v", err)
-	}
-
-	for _, object := range res.Contents {
-		objectKey := *object.Key
-		if strings.Contains(objectKey, defs.StorInitFile) || strings.Contains(objectKey, "/pbmPitr/") {
+		if strings.Contains(file.Name, defs.PITRfsPrefix) {
 			continue
 		}
 
 		var ok bool
 		for b := range shouldStay {
-			if strings.Contains(objectKey, b) {
+			if strings.Contains(file.Name, b) {
 				ok = true
 				break
 			}
 		}
 		if !ok {
-			log.Fatalln("ERROR: failed to delete lefover", objectKey)
+			log.Fatalln("ERROR: failed to delete lefover", file.Name)
 		}
 	}
 }