feat(backup): restore secret storage by access key

Author: fiftin

db/BackupEntity.go

@@ -56,3 +56,11 @@ func (e Environment) GetID() int {
 func (e Environment) GetName() string {
 	return e.Name
 }
+
+func (e SecretStorage) GetID() int {
+	return e.ID
+}
+
+func (e SecretStorage) GetName() string {
+	return e.Name
+}

db/sql/SqlDb.go

@@ -111,6 +111,26 @@ func (d *SqlDbConnection) Close() {
 	}
 }
 
+func CreateTestStore() *SqlDb {
+	util.Config = &util.ConfigType{
+		SQLite: &util.DbConfig{
+			Hostname: ":memory:",
+		},
+		Dialect: "sqlite",
+		Log: &util.ConfigLog{
+			Events: &util.EventLogType{},
+			Tasks:  &util.TaskLogType{},
+		},
+	}
+	store := CreateDb(util.DbDriverSQLite)
+
+	store.Connect("")
+
+	db.Migrate(store, nil)
+
+	return store
+}
+
 func (d *SqlDbConnection) prepareQueryWithDialect(query string, dialect gorp.Dialect) string {
 	switch dialect.(type) {
 	case gorp.PostgresDialect:

services/project/backup.go

@@ -252,15 +252,11 @@ func (b *BackupDB) format() (*BackupFormat, error) {
 			SecretStorage: o,
 		}
 
-		for _, key := range keys {
-			if *key.StorageID == o.ID {
-				secretStorages[i].VaultTokenKeyName = key.Name
-				break
-			}
-		}
-
 		for k := range keys {
-			if *keys[k].SourceStorageID == o.ID {
+			if keys[k].StorageID != nil && *keys[k].StorageID == o.ID {
+				keys[k].Storage = &o.Name
+			}
+			if keys[k].SourceStorageID != nil && *keys[k].SourceStorageID == o.ID {
 				keys[k].SourceStorage = &o.Name
 			}
 		}

services/project/backup_test.go

@@ -1,10 +1,11 @@
 package project
 
 import (
+	"encoding/json"
+	"github.com/semaphoreui/semaphore/db/sql"
 	"testing"
 
 	"github.com/semaphoreui/semaphore/db"
-	"github.com/semaphoreui/semaphore/db/bolt"
 	"github.com/semaphoreui/semaphore/util"
 	"github.com/stretchr/testify/assert"
 )
@@ -18,7 +19,7 @@ func TestBackupProject(t *testing.T) {
 		TmpPath: "/tmp",
 	}
 
-	store := bolt.CreateTestStore()
+	store := sql.CreateTestStore()
 
 	proj, err := store.CreateProject(db.Project{
 		Name: "Test 123",
@@ -92,6 +93,106 @@ func TestBackupProject(t *testing.T) {
 	assert.Equal(t, proj.Name, restoredProj.Name)
 }
 
+func TestBackup_BackupSecretStorage(t *testing.T) {
+	util.Config = &util.ConfigType{
+		TmpPath: "/tmp",
+	}
+
+	store := sql.CreateTestStore()
+
+	proj, err := store.CreateProject(db.Project{
+		Name: "Test 123",
+	})
+	assert.NoError(t, err)
+
+	storage, err := store.CreateSecretStorage(db.SecretStorage{
+		ProjectID: proj.ID,
+		Type:      "vault",
+		Name:      "Test",
+	})
+	assert.NoError(t, err)
+
+	_, err = store.CreateAccessKey(db.AccessKey{
+		ProjectID: &proj.ID,
+		Type:      db.AccessKeyNone,
+		StorageID: &storage.ID,
+		Name:      "Test Key",
+	})
+	assert.NoError(t, err)
+
+	backup, err := GetBackup(proj.ID, store)
+	assert.NoError(t, err)
+	assert.Equal(t, proj.ID, backup.Meta.ID)
+
+	str, err := backup.Marshal()
+	assert.NoError(t, err)
+
+	var res map[string]any
+	json.Unmarshal([]byte(str), &res)
+
+	assert.Equal(t, `{
+  "environments": [],
+  "integration_aliases": [],
+  "integrations": [],
+  "inventories": [],
+  "keys": [
+    {
+      "name": "Test Key",
+      "owner": "",
+      "storage": "Test",
+      "type": "none"
+    }
+  ],
+  "meta": {
+    "alert": false,
+    "max_parallel_tasks": 0,
+    "name": "Test 123",
+    "type": ""
+  },
+  "repositories": [],
+  "schedules": [],
+  "secret_storages": [
+    {
+      "name": "Test",
+      "params": {},
+      "readonly": false,
+      "type": "vault"
+    }
+  ],
+  "templates": [],
+  "views": []
+}`, str)
+
+	restoredBackup := &BackupFormat{}
+	err = restoredBackup.Unmarshal(str)
+	assert.NoError(t, err)
+	assert.Equal(t, proj.Name, restoredBackup.Meta.Name)
+
+	user, err := store.CreateUser(db.UserWithPwd{
+		Pwd: "3412341234123",
+		User: db.User{
+			Username: "test",
+			Name:     "Test",
+			Email:    "[email protected]",
+			Admin:    true,
+		},
+	})
+	assert.NoError(t, err)
+
+	restoredProj, err := restoredBackup.Restore(user, store)
+	assert.Nil(t, err)
+
+	restoredStorages, err := store.GetSecretStorages(restoredProj.ID)
+	assert.NoError(t, err)
+	assert.Len(t, restoredStorages, 1)
+
+	restoredKeys, err := store.GetAccessKeys(restoredProj.ID, db.GetAccessKeyOptions{}, db.RetrieveQueryParams{})
+	assert.NoError(t, err)
+	assert.Len(t, restoredKeys, 1)
+
+	assert.Equal(t, *restoredKeys[0].StorageID, restoredStorages[0].ID)
+}
+
 func isUnique(items []testItem) bool {
 	for i, item := range items {
 		for k, other := range items {

services/project/restore.go

@@ -116,6 +116,22 @@ func (e BackupAccessKey) Restore(store db.Store, b *BackupDB) error {
 	key := e.AccessKey
 	key.ProjectID = &b.meta.ID
 
+	if e.Storage != nil {
+		storage := findEntityByName[db.SecretStorage](e.Storage, b.secretStorages)
+		if storage == nil {
+			return fmt.Errorf("secret storage does not exist in secret_storage[].name")
+		}
+		key.StorageID = &storage.ID
+	}
+
+	if e.SourceStorage != nil {
+		storage := findEntityByName[db.SecretStorage](e.SourceStorage, b.secretStorages)
+		if storage == nil {
+			return fmt.Errorf("secret storage does not exist in secret_storage[].name")
+		}
+		key.StorageID = &storage.ID
+	}
+
 	newKey, err := store.CreateAccessKey(key)
 
 	if err != nil {
@@ -461,6 +477,12 @@ func (backup *BackupFormat) Restore(user db.User, store db.Store) (*db.Project,
 
 	b.meta = newProject
 
+	for i, o := range backup.SecretStorages {
+		if err := o.Restore(store, &b); err != nil {
+			return nil, fmt.Errorf("error at secret storage[%d]: %s", i, err.Error())
+		}
+	}
+
 	for i, o := range backup.Environments {
 		if err := o.Restore(store, &b); err != nil {
 			return nil, fmt.Errorf("error at environments[%d]: %s", i, err.Error())
@@ -529,11 +551,5 @@ func (backup *BackupFormat) Restore(user db.User, store db.Store) (*db.Project,
 		}
 	}
 
-	for i, o := range backup.SecretStorages {
-		if err := o.Restore(store, &b); err != nil {
-			return nil, fmt.Errorf("error at secret storage[%d]: %s", i, err.Error())
-		}
-	}
-
 	return &newProject, nil
 }

services/project/types.go

@@ -48,6 +48,7 @@ type BackupEnvironment struct {
 type BackupAccessKey struct {
 	db.AccessKey
 	SourceStorage *string `backup:"source_storage"`
+	Storage       *string `backup:"storage"`
 }
 
 type BackupSchedule struct {
@@ -101,7 +102,6 @@ type BackupIntegration struct {
 
 type BackupSecretStorage struct {
 	db.SecretStorage
-	VaultTokenKeyName string `backup:"vault_token_key_name"`
 }
 
 type BackupEntry interface {