Add storage migration support (#1174)

Author: ije

config.example.jsonc

@@ -54,13 +54,6 @@
     "secretAccessKey": ""
   },
 
-  // Cache package raw files in the storage, default is false.
-  // The server cleans up npm store periodically, to avoid unnecessary installation when accessing
-  // package raw files, you can enable this option to cache package raw files in the storage.
-  // Note: this option will increase the storage usage, we recommend to enable it if you are using
-  // a S3-compatible storage.
-  "cacheRawFile": false,
-
   // The custom landing page options, default is empty.
   // The server will proxy the `/` request to the `origin` server if it's provided.
   // If your custom landing page has own assets, you also need to provide those asset paths in the `assets` field.

internal/storage/migration.go

@@ -0,0 +1,62 @@
+package storage
+
+import (
+	"errors"
+	"io"
+)
+
+// NewMigrationStorage creates a new storage instance that migrates files from one storage to another.
+// It first tries to get files from the front storage, and if not found, it retrieves them from the back storage.
+func NewMigrationStorage(frontStorage Storage, backStorage Storage) (storage Storage) {
+	return &migrationStorage{
+		front: frontStorage,
+		back:  backStorage,
+	}
+}
+
+type migrationStorage struct {
+	front Storage
+	back  Storage
+}
+
+func (m *migrationStorage) Stat(key string) (stat Stat, err error) {
+	stat, err = m.front.Stat(key)
+	if err == ErrNotFound {
+		stat, err = m.back.Stat(key)
+	}
+	return
+}
+
+func (m *migrationStorage) Get(key string) (content io.ReadCloser, stat Stat, err error) {
+	content, stat, err = m.front.Get(key)
+	if err == ErrNotFound {
+		content, stat, err = m.back.Get(key)
+		if err == nil {
+			defer content.Close()
+			err = m.front.Put(key, content)
+			if err != nil {
+				return
+			}
+			content, stat, err = m.front.Get(key)
+		}
+	}
+	return
+}
+
+func (m *migrationStorage) List(prefix string) (keys []string, err error) {
+	return nil, errors.New("List operation is not supported in migration storage")
+}
+
+func (m *migrationStorage) Put(key string, content io.Reader) (err error) {
+	return m.front.Put(key, content)
+}
+
+func (m *migrationStorage) Delete(key string) (err error) {
+	m.back.Delete(key)
+	return m.front.Delete(key)
+}
+
+func (m *migrationStorage) DeleteAll(prefix string) (deletedKeys []string, err error) {
+	m.back.DeleteAll(prefix)
+	return m.front.DeleteAll(prefix)
+}

internal/storage/migration_test.go

@@ -0,0 +1,100 @@
+package storage
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/ije/gox/crypto/rand"
+)
+
+func TestMigrationStorage(t *testing.T) {
+	root1 := path.Join(os.TempDir(), "storage_test_"+rand.Hex.String(8))
+	back, err := NewFSStorage(&StorageOptions{Type: "fs", Endpoint: root1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(root1)
+
+	root2 := path.Join(os.TempDir(), "storage_test_"+rand.Hex.String(8))
+	front, err := NewFSStorage(&StorageOptions{Type: "fs", Endpoint: root2})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(root2)
+
+	migrationStorage := NewMigrationStorage(front, back)
+
+	err = back.Put("test.txt", bytes.NewBufferString("Hello World!"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = front.Stat("test.txt")
+	if err != ErrNotFound {
+		t.Fatal("Expected error, but got nil")
+	}
+
+	_, _, err = front.Get("test.txt")
+	if err != ErrNotFound {
+		t.Fatal("Expected error, but got nil")
+	}
+
+	fi, err := migrationStorage.Stat("test.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fi.Size() != 12 {
+		t.Fatalf("invalid file size(%d), shoud be 12", fi.Size())
+	}
+
+	f, fi, err := migrationStorage.Get("test.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	if fi.Size() != 12 {
+		t.Fatalf("invalid file size(%d), shoud be 12", fi.Size())
+	}
+
+	data, err := io.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(data) != "Hello World!" {
+		t.Fatalf("invalid file content('%s'), shoud be 'Hello World!'", string(data))
+	}
+
+	fi, err = front.Stat("test.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fi.Size() != 12 {
+		t.Fatalf("invalid file size(%d), shoud be 12", fi.Size())
+	}
+
+	f, fi, err = front.Get("test.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	if fi.Size() != 12 {
+		t.Fatalf("invalid file size(%d), shoud be 12", fi.Size())
+	}
+
+	data, err = io.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(data) != "Hello World!" {
+		t.Fatalf("invalid file content('%s'), shoud be 'Hello World!'", string(data))
+	}
+}

internal/storage/storage.go

@@ -20,10 +20,10 @@ type StorageOptions struct {
 
 type Storage interface {
 	Stat(key string) (stat Stat, err error)
-	List(prefix string) (keys []string, err error)
 	Get(key string) (content io.ReadCloser, stat Stat, err error)
+	List(prefix string) (keys []string, err error)
 	Put(key string, r io.Reader) error
-	Delete(keys ...string) error
+	Delete(key string) error
 	DeleteAll(prefix string) (deletedKeys []string, err error)
 }
 

internal/storage/storage_fs.go

@@ -42,11 +42,6 @@ func (fs *fsStorage) Stat(key string) (stat Stat, err error) {
 	return fi, nil
 }
 
-func (fs *fsStorage) List(prefix string) (keys []string, err error) {
-	dir := strings.TrimSuffix(utils.NormalizePathname(prefix)[1:], "/")
-	return findFiles(filepath.Join(fs.root, dir), dir)
-}
-
 func (fs *fsStorage) Get(key string) (content io.ReadCloser, stat Stat, err error) {
 	filename := filepath.Join(fs.root, key)
 	file, err := os.Open(filename)
@@ -63,6 +58,11 @@ func (fs *fsStorage) Get(key string) (content io.ReadCloser, stat Stat, err erro
 	return
 }
 
+func (fs *fsStorage) List(prefix string) (keys []string, err error) {
+	dir := strings.TrimSuffix(utils.NormalizePathname(prefix)[1:], "/")
+	return findFiles(filepath.Join(fs.root, dir), dir)
+}
+
 func (fs *fsStorage) Put(key string, content io.Reader) (err error) {
 	filename := filepath.Join(fs.root, key)
 	err = ensureDir(filepath.Dir(filename))
@@ -74,17 +74,17 @@ func (fs *fsStorage) Put(key string, content io.Reader) (err error) {
 	if err != nil {
 		return
 	}
-	defer file.Close()
 
 	_, err = io.Copy(file, content)
+	file.Close()
+	if err != nil {
+		os.Remove(filename) // clean up if error occurs
+	}
 	return
 }
 
-func (fs *fsStorage) Delete(keys ...string) (err error) {
-	for _, key := range keys {
-		os.Remove(filepath.Join(fs.root, key))
-	}
-	return
+func (fs *fsStorage) Delete(key string) (err error) {
+	return os.Remove(filepath.Join(fs.root, key))
 }
 
 func (fs *fsStorage) DeleteAll(prefix string) (deletedKeys []string, err error) {

internal/storage/storage_fs_test.go

@@ -18,42 +18,42 @@ func TestFSStorage(t *testing.T) {
 	}
 	defer os.RemoveAll(root)
 
-	err = fs.Put("foo.txt", bytes.NewBufferString("Hello, World!"))
+	err = fs.Put("test.txt", bytes.NewBufferString("Hello World!"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	err = fs.Put("foo/bar.txt", bytes.NewBufferString("Hello, World!"))
+	err = fs.Put("hello/world.txt", bytes.NewBufferString("Hello World!"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	fi, err := fs.Stat("foo.txt")
+	fi, err := fs.Stat("test.txt")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if fi.Size() != 13 {
-		t.Fatalf("invalid file size(%d), shoud be 13", fi.Size())
+	if fi.Size() != 12 {
+		t.Fatalf("invalid file size(%d), shoud be 12", fi.Size())
 	}
 
-	f, fi, err := fs.Get("foo.txt")
+	f, fi, err := fs.Get("test.txt")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer f.Close()
 
-	if fi.Size() != 13 {
-		t.Fatalf("invalid file size(%d), shoud be 13", fi.Size())
+	if fi.Size() != 12 {
+		t.Fatalf("invalid file size(%d), shoud be 12", fi.Size())
 	}
 
 	data, err := io.ReadAll(f)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if string(data) != "Hello, World!" {
-		t.Fatalf("invalid file content('%s'), shoud be 'Hello, World!'", string(data))
+	if string(data) != "Hello World!" {
+		t.Fatalf("invalid file content('%s'), shoud be 'Hello World!'", string(data))
 	}
 
 	keys, err := fs.List("")
@@ -65,7 +65,7 @@ func TestFSStorage(t *testing.T) {
 		t.Fatalf("invalid keys count(%d), shoud be 2", len(keys))
 	}
 
-	keys, err = fs.List("foo/")
+	keys, err = fs.List("hello/")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -74,26 +74,26 @@ func TestFSStorage(t *testing.T) {
 		t.Fatalf("invalid keys count(%d), shoud be 1", len(keys))
 	}
 
-	if keys[0] != "foo/bar.txt" {
-		t.Fatalf("invalid key('%s'), shoud be 'foo/bar.txt'", keys[0])
+	if keys[0] != "hello/world.txt" {
+		t.Fatalf("invalid key('%s'), shoud be 'hello/world.txt'", keys[0])
 	}
 
-	err = fs.Delete("foo.txt")
+	err = fs.Delete("test.txt")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	_, err = fs.Stat("foo.txt")
+	_, err = fs.Stat("test.txt")
 	if err != ErrNotFound {
 		t.Fatalf("File should be not existent")
 	}
 
-	_, _, err = fs.Get("foo.txt")
+	_, _, err = fs.Get("test.txt")
 	if err != ErrNotFound {
 		t.Fatalf("File should be not existent")
 	}
 
-	deletedKeys, err := fs.DeleteAll("foo/")
+	deletedKeys, err := fs.DeleteAll("hello/")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -102,8 +102,8 @@ func TestFSStorage(t *testing.T) {
 		t.Fatalf("invalid deleted keys count(%d), shoud be 1", len(deletedKeys))
 	}
 
-	if deletedKeys[0] != "foo/bar.txt" {
-		t.Fatalf("invalid deleted key('%s'), shoud be 'foo/bar.txt'", deletedKeys[0])
+	if deletedKeys[0] != "hello/world.txt" {
+		t.Fatalf("invalid deleted key('%s'), shoud be 'hello/world.txt'", deletedKeys[0])
 	}
 
 	keys, err = fs.List("")