Integration testing with Testcontainers

CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- Various compliance fixes (to be discussed).
 
 ## [v7.13.0] - 2025-01-26
 ### Added

backend/azure/file.go

@@ -70,7 +70,7 @@ func (f *File) Close() error {
 // the file is created and read operations are performed against that.  The temp file is closed and flushed to Azure
 // when f.Close() is called.
 func (f *File) Read(p []byte) (n int, err error) {
-	if err := f.checkTempFile(); err != nil {
+	if err := f.checkTempFile(false); err != nil {
 		return 0, utils.WrapReadError(err)
 	}
 	read, err := f.tempFile.Read(p)
@@ -91,7 +91,7 @@ func (f *File) Read(p []byte) (n int, err error) {
 // the file is created and operations are performed against that.  The temp file is closed and flushed to Azure
 // when f.Close() is called.
 func (f *File) Seek(offset int64, whence int) (int64, error) {
-	if err := f.checkTempFile(); err != nil {
+	if err := f.checkTempFile(false); err != nil {
 		return 0, utils.WrapSeekError(err)
 	}
 	pos, err := f.tempFile.Seek(offset, whence)
@@ -104,7 +104,7 @@ func (f *File) Seek(offset int64, whence int) (int64, error) {
 // Write implements the io.Writer interface.  Writes are performed against a temporary local file.  The temp file is
 // closed and flushed to Azure with f.Close() is called.
 func (f *File) Write(p []byte) (int, error) {
-	if err := f.checkTempFile(); err != nil {
+	if err := f.checkTempFile(true); err != nil {
 		return 0, utils.WrapWriteError(err)
 	}
 
@@ -350,7 +350,7 @@ func (f *File) URI() string {
 	return utils.GetFileURI(f)
 }
 
-func (f *File) checkTempFile() error {
+func (f *File) checkTempFile(isWrite bool) error {
 	if f.tempFile == nil {
 		client, err := f.location.fileSystem.Client()
 		if err != nil {
@@ -361,23 +361,23 @@ func (f *File) checkTempFile() error {
 		if err != nil {
 			return err
 		}
-		if !exists {
-			tf, tfErr := os.CreateTemp("", fmt.Sprintf("%s.%d", path.Base(f.Name()), time.Now().UnixNano()))
-			if tfErr != nil {
-				return tfErr
+
+		tf, tfErr := os.CreateTemp("", fmt.Sprintf("%s.%d", path.Base(f.Name()), time.Now().UnixNano()))
+		if tfErr != nil {
+			return tfErr
+		}
+		f.tempFile = tf
+
+		if !isWrite {
+			if !exists {
+				return os.ErrNotExist
 			}
-			f.tempFile = tf
-		} else {
+
 			reader, dlErr := client.Download(f)
 			if dlErr != nil {
 				return dlErr
 			}
 
-			tf, tfErr := os.CreateTemp("", fmt.Sprintf("%s.%d", path.Base(f.Name()), time.Now().UnixNano()))
-			if tfErr != nil {
-				return tfErr
-			}
-
 			buffer := make([]byte, utils.TouchCopyMinBufferSize)
 			if _, err := io.CopyBuffer(tf, reader, buffer); err != nil {
 				return err
@@ -386,8 +386,6 @@ func (f *File) checkTempFile() error {
 			if _, err := tf.Seek(0, 0); err != nil {
 				return err
 			}
-
-			f.tempFile = tf
 		}
 	}
 	return nil

backend/azure/file_test.go

@@ -91,7 +91,6 @@ func (s *FileTestSuite) TestWrite() {
 	s.NotNil(f)
 	s.Require().NoError(err)
 	client.EXPECT().Properties("test-container", "/foo.txt").Return(&BlobProperties{}, nil)
-	client.EXPECT().Download(mock.Anything).Return(io.NopCloser(strings.NewReader("Hello World!")), nil)
 	n, err := f.Write([]byte(" Aaaaand, Goodbye!"))
 	s.Require().NoError(err)
 	s.Equal(18, n)
@@ -375,7 +374,7 @@ func (s *FileTestSuite) TestCheckTempFile() {
 	s.Nil(azureFile.tempFile, "No calls to checkTempFile have occurred so we expect tempFile to be nil")
 	client.EXPECT().Properties("test-container", "/foo.txt").Return(&BlobProperties{}, nil)
 	client.EXPECT().Download(mock.Anything).Return(io.NopCloser(strings.NewReader("Hello World!")), nil)
-	err = azureFile.checkTempFile()
+	err = azureFile.checkTempFile(false)
 	s.Require().NoError(err, "Check temp file should create a local temp file so no error is expected")
 	s.NotNil(azureFile.tempFile, "After the call to checkTempFile we should have a non-nil tempFile")
 
@@ -397,7 +396,7 @@ func (s *FileTestSuite) TestCheckTempFile_FileDoesNotExist() {
 
 	s.Nil(azureFile.tempFile, "No calls to checkTempFile have occurred so we expect tempFile to be nil")
 	client.EXPECT().Properties("test-container", "/foo.txt").Return(nil, errBlobNotFound)
-	err = azureFile.checkTempFile()
+	err = azureFile.checkTempFile(true)
 	s.Require().NoError(err, "Check temp file should create a local temp file so no error is expected")
 	s.NotNil(azureFile.tempFile, "After the call to checkTempFile we should have a non-nil tempFile")
 
@@ -420,7 +419,7 @@ func (s *FileTestSuite) TestCheckTempFile_DownloadError() {
 	s.Nil(azureFile.tempFile, "No calls to checkTempFile have occurred so we expect tempFile to be nil")
 	client.EXPECT().Properties("test-container", "/foo.txt").Return(&BlobProperties{}, nil)
 	client.EXPECT().Download(mock.Anything).Return(nil, errors.New("i always error"))
-	err = azureFile.checkTempFile()
+	err = azureFile.checkTempFile(false)
 	s.Require().Error(err, "The call to client.Download() errors so we expect to get an error")
 }
 

backend/ftp/location.go

@@ -152,6 +152,10 @@ func (l *Location) Path() string {
 
 // Exists returns true if the remote FTP directory exists.
 func (l *Location) Exists() (bool, error) {
+	if l.path == "/" {
+		return true, nil
+	}
+
 	dc, err := l.fileSystem.DataConn(context.TODO(), l.Authority(), types.SingleOp, nil)
 	if err != nil {
 		return false, err

backend/ftp/location_test.go

@@ -411,21 +411,6 @@ func (lt *locationTestSuite) TestExists() {
 
 	// location exists
 	locPath := "/"
-	entries := []*_ftp.Entry{
-		{
-			Name:   "file.txt",
-			Target: "",
-			Type:   _ftp.EntryTypeFile,
-			Time:   time.Now().UTC(),
-		},
-		{
-			Name:   locPath,
-			Target: "",
-			Type:   _ftp.EntryTypeFolder,
-			Time:   time.Now().UTC(),
-		},
-	}
-	lt.client.EXPECT().List(locPath).Return(entries, nil).Once()
 	loc, err := lt.ftpfs.NewLocation(authorityStr, locPath)
 	lt.Require().NoError(err)
 	exists, err := loc.Exists()
@@ -434,7 +419,7 @@ func (lt *locationTestSuite) TestExists() {
 
 	// locations does not exist
 	locPath = "/my/dir/"
-	entries = []*_ftp.Entry{
+	entries := []*_ftp.Entry{
 		{
 			Name:   "file.txt",
 			Target: "",

testcontainers/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]

testcontainers/atmoz.go

@@ -0,0 +1,52 @@
+package testcontainers
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+	"golang.org/x/crypto/ssh"
+
+	"github.com/c2fo/vfs/v7/backend"
+	"github.com/c2fo/vfs/v7/backend/sftp"
+)
+
+const (
+	atmozPort     = "22/tcp"
+	atmozUsername = "dummy"
+	atmozPassword = "dummy"
+)
+
+func registerAtmoz(t *testing.T) string {
+	ctx := context.Background()
+	is := require.New(t)
+
+	req := testcontainers.GenericContainerRequest{
+		ContainerRequest: testcontainers.ContainerRequest{
+			Name:       "vfs-atmoz-sftp",
+			Image:      "atmoz/sftp:alpine",
+			Env:        map[string]string{"SFTP_USERS": fmt.Sprintf("%s:%s:::upload", atmozUsername, atmozPassword)},
+			WaitingFor: wait.ForListeningPort(atmozPort),
+		},
+		Started: true,
+	}
+	ctr, err := testcontainers.GenericContainer(ctx, req)
+	testcontainers.CleanupContainer(t, ctr)
+	is.NoError(err)
+
+	host, err := ctr.Host(ctx)
+	is.NoError(err)
+
+	port, err := ctr.MappedPort(ctx, atmozPort)
+	is.NoError(err)
+
+	authority := fmt.Sprintf("sftp://%s@%s:%s/upload/", atmozUsername, host, port.Port())
+	backend.Register(authority, sftp.NewFileSystem(sftp.WithOptions(sftp.Options{
+		Password:           vsftpdPassword,
+		KnownHostsCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec
+	})))
+	return authority
+}

testcontainers/azurite.go

@@ -0,0 +1,53 @@
+package testcontainers
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/modules/azure/azurite"
+
+	"github.com/c2fo/vfs/v7/backend"
+	"github.com/c2fo/vfs/v7/backend/azure"
+)
+
+func registerAzurite(t *testing.T) string {
+	ctx := context.Background()
+	is := require.New(t)
+
+	ctr, err := azurite.Run(ctx, "mcr.microsoft.com/azure-storage/azurite:latest",
+		testcontainers.WithName("vfs-azurite"),
+		azurite.WithEnabledServices(azurite.BlobService),
+		testcontainers.WithCmdArgs("--skipApiVersionCheck"),
+	)
+	testcontainers.CleanupContainer(t, ctr)
+	is.NoError(err)
+
+	ep, err := ctr.BlobServiceURL(ctx)
+	is.NoError(err)
+
+	cred, err := azblob.NewSharedKeyCredential(azurite.AccountName, azurite.AccountKey)
+	is.NoError(err)
+
+	u, err := url.JoinPath(ep, azurite.AccountName)
+	is.NoError(err)
+
+	cli, err := azblob.NewClientWithSharedKeyCredential(u, cred, nil)
+	is.NoError(err)
+
+	_, err = cli.CreateContainer(ctx, "azurite", nil)
+	is.NoError(err)
+
+	c, err := azure.NewClient(&azure.Options{
+		ServiceURL:  u,
+		AccountName: azurite.AccountName,
+		AccountKey:  azurite.AccountKey,
+	})
+	is.NoError(err)
+
+	backend.Register("https://azurite/", azure.NewFileSystem(azure.WithClient(c)))
+	return "https://azurite/"
+}

testcontainers/backend_integration_test.go

@@ -0,0 +1,97 @@
+//go:build linux
+
+package testcontainers
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/c2fo/vfs/v7"
+	"github.com/c2fo/vfs/v7/vfssimple"
+)
+
+type vfsTestSuite struct {
+	suite.Suite
+	testLocations map[string]vfs.Location
+}
+
+func (s *vfsTestSuite) SetupSuite() {
+	registers := []func(*testing.T) string{
+		registerMem,
+		registerOS,
+		registerAtmoz,
+		registerAzurite,
+		registerGCSServer,
+		registerLocalStack,
+		registerMinio,
+		registerVSFTPD,
+	}
+	uris := make([]string, len(registers))
+	var wg sync.WaitGroup
+	wg.Add(len(registers))
+	for i := range registers {
+		go func() {
+			uris[i] = registers[i](s.T())
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+
+	s.testLocations = make(map[string]vfs.Location)
+	for _, loc := range uris {
+		l, err := vfssimple.NewLocation(loc)
+		s.Require().NoError(err)
+
+		// For file:// locations, ensure directory exists
+		if l.FileSystem().Scheme() == "file" {
+			exists, err := l.Exists()
+			if err != nil {
+				panic(err)
+			}
+			if !exists {
+				err := os.Mkdir(l.Path(), 0750)
+				if err != nil {
+					panic(err)
+				}
+			}
+		}
+
+		// Store location by scheme - no type assertion needed
+		s.testLocations[l.FileSystem().Scheme()] = l
+	}
+}
+
+func registerMem(*testing.T) string {
+	return "mem://test/"
+}
+
+func registerOS(t *testing.T) string {
+	return fmt.Sprintf("file://%s/", filepath.ToSlash(t.TempDir()))
+}
+
+// TestScheme runs conformance tests for each configured backend
+func (s *vfsTestSuite) TestScheme() {
+	for scheme, location := range s.testLocations {
+		fmt.Printf("************** TESTING scheme: %s **************\n", scheme)
+
+		// Determine conformance options based on scheme
+		opts := ConformanceOptions{
+			SkipFTPSpecificTests:   scheme == "ftp",
+			SkipTouchTimestampTest: scheme == "ftp",
+		}
+
+		// Run the exported conformance tests
+		s.Run(scheme, func() {
+			RunConformanceTests(s.T(), location, opts)
+		})
+	}
+}
+
+func TestVFS(t *testing.T) {
+	suite.Run(t, new(vfsTestSuite))
+}