secret storage (#3128)

* feat(storage): add storage table

* feat(db): add secret storage

* feat(stoarge): add service

* feat(keys): add storage model

* feat(keys): model fields

* feat(keys): add storages page

* feat(keys): add secret storages controller

* refactor: use services for access key descryption

* feat(keys): add vault support

* feat(keys): add endpoints

* fix(keys): fill installer interface

* fix: null pointers of services

* fix(api): incorrect usage middleware

* fix(secrets): fix sql query

* feat(secrets): manipulation of storage

* feat(storage): works

* fix(storage): do not clear access key

* feat(storage): vault settings

* feat(storage): choose storage for access key

* refactor(secrets): copy method to serializer

* feat(secrets): add access key service

* test: fix tests

* test: fix tests

* fix: services

* feat(secrets): store keys to vault

* feat(secrets): delete storage endpoint

* feat(secrets): delete storage endpoint

* fix(secrets): correct condition

* feat: add icon

* feat(secrets): add menu to new storage button

* feat(secrets): add icon for strage

* fix(secrets): allow empty token field for existing storage

* feat(secrets): readonly secret storage

* feat(secrets): add readonly badge

* feat(secrets): delete secret from vault

* feat(secrets): delete remote secret when delete access key

* feat(secrets): use access key service

* security(secrets): upgrade dep

* refactor(secrets): split to foss and pro

* refactor(service): rename package name

* refactor(features): add GetFeatures function

* feat(secrets): update ui

* ci: dir to /tmp for deps

* refactor(pro): add mock pro dir

* chore: ingore go.work

* test(secrets): fix mock

* fix(secrets): sqlite migration

* ci: fix docker image build

Author: fiftin

.gitignore

@@ -38,4 +38,7 @@ __debug_bin*
 
 /events.log
 /tasks.log
-/task_results.log
+/task_results.log
+/pro_impl/
+/go.work
+/go.work.sum

Taskfile.yml

@@ -34,6 +34,7 @@ tasks:
 #      - go install github.com/go-swagger/go-swagger/cmd/swagger@{{ .SWAGGER_VERSION }}
       - go install github.com/goreleaser/goreleaser@{{ .GORELEASER_VERSION }}
 #      - go install github.com/golangci/golangci-lint/cmd/golangci-lint@{{ .GOLINTER_VERSION }}
+    dir: /tmp
 
   deps:be:
     desc: Vendor application dependencies

api/api_test.go

@@ -15,7 +15,16 @@ func TestApiPing(t *testing.T) {
 	req, _ := http.NewRequest("GET", "/api/ping", nil)
 	rr := httptest.NewRecorder()
 
-	r := Route(nil, nil)
+	r := Route(
+		nil,
+		nil,
+		nil,
+		nil,
+		nil,
+		nil,
+		nil,
+		nil,
+	)
 
 	r.ServeHTTP(rr, req)
 

api/apps.go

@@ -6,63 +6,13 @@ import (
 	"fmt"
 	"github.com/semaphoreui/semaphore/api/helpers"
 	"github.com/semaphoreui/semaphore/db"
+	"github.com/semaphoreui/semaphore/pkg/conv"
 	"github.com/semaphoreui/semaphore/util"
 	"net/http"
 	"reflect"
 	"sort"
-	"strings"
 )
 
-func structToFlatMap(obj any) map[string]any {
-	result := make(map[string]any)
-	val := reflect.ValueOf(obj)
-	typ := reflect.TypeOf(obj)
-
-	if typ.Kind() == reflect.Ptr {
-		val = val.Elem()
-		typ = typ.Elem()
-	}
-
-	if typ.Kind() != reflect.Struct {
-		return result
-	}
-
-	// Iterate over the struct fields
-	for i := 0; i < val.NumField(); i++ {
-		field := val.Field(i)
-		fieldType := typ.Field(i)
-		jsonTag := fieldType.Tag.Get("json")
-
-		// Use the json tag if it is set, otherwise use the field name
-		fieldName := jsonTag
-		if fieldName == "" || fieldName == "-" {
-			fieldName = fieldType.Name
-		} else {
-			// Handle the case where the json tag might have options like `json:"name,omitempty"`
-			fieldName = strings.Split(fieldName, ",")[0]
-		}
-
-		// Check if the field is a struct itself
-		if field.Kind() == reflect.Struct {
-			// Convert nested struct to map
-			nestedMap := structToFlatMap(field.Interface())
-			// Add nested map to result with a prefixed key
-			for k, v := range nestedMap {
-				result[fieldName+"."+k] = v
-			}
-		} else if (field.Kind() == reflect.Ptr ||
-			field.Kind() == reflect.Array ||
-			field.Kind() == reflect.Slice ||
-			field.Kind() == reflect.Map) && field.IsNil() {
-			result[fieldName] = nil
-		} else {
-			result[fieldName] = field.Interface()
-		}
-	}
-
-	return result
-}
-
 func validateAppID(str string) error {
 	return nil
 }
@@ -170,7 +120,7 @@ func setApp(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	options := structToFlatMap(app)
+	options := conv.StructToFlatMap(app)
 
 	for k, v := range options {
 		t := reflect.TypeOf(v)

api/apps_test.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"fmt"
+	"github.com/semaphoreui/semaphore/pkg/conv"
 	"testing"
 )
 
@@ -32,7 +33,7 @@ func TestStructToMap(t *testing.T) {
 	}
 
 	// Convert the struct to a flat map
-	flatMap := structToFlatMap(&p)
+	flatMap := conv.StructToFlatMap(&p)
 
 	if flatMap["address.city"] != "New York" {
 		t.Fail()

api/integration.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/semaphoreui/semaphore/pkg/conv"
+	"github.com/semaphoreui/semaphore/services/server"
 	task2 "github.com/semaphoreui/semaphore/services/tasks"
 	"io"
 	"net/http"
@@ -44,7 +45,17 @@ func hmacHashPayload(secret string, payloadBody []byte) string {
 	return fmt.Sprintf("%x", sum)
 }
 
-func ReceiveIntegration(w http.ResponseWriter, r *http.Request) {
+type IntegrationController struct {
+	integrationService server.IntegrationService
+}
+
+func NewIntegrationController(integrationService server.IntegrationService) *IntegrationController {
+	return &IntegrationController{
+		integrationService: integrationService,
+	}
+}
+
+func (c *IntegrationController) ReceiveIntegration(w http.ResponseWriter, r *http.Request) {
 
 	var err error
 
@@ -95,7 +106,7 @@ func ReceiveIntegration(w http.ResponseWriter, r *http.Request) {
 			panic("")
 		}
 
-		err = db.FillIntegration(store, &integration)
+		err = c.integrationService.FillIntegration(&integration)
 		if err != nil {
 			log.Error(err)
 			return
@@ -288,7 +299,7 @@ func RunIntegration(integration db.Integration, project db.Project, r *http.Requ
 		log.Error(err)
 		return
 	}
-	
+
 	pool := helpers.GetFromContext(r, "task_pool").(*task2.TaskPool)
 
 	_, err = pool.AddTask(taskDefinition, nil, "", integration.ProjectID, tpl.App.NeedTaskAlias())

api/projects/environment.go

@@ -4,10 +4,29 @@ import (
 	"fmt"
 	"github.com/semaphoreui/semaphore/api/helpers"
 	"github.com/semaphoreui/semaphore/db"
+	"github.com/semaphoreui/semaphore/services/server"
 	"net/http"
 )
 
-func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
+type EnvironmentController struct {
+	accessKeyRepo     db.AccessKeyManager
+	accessKeyService  server.AccessKeyService
+	encryptionService server.AccessKeyEncryptionService
+}
+
+func NewEnvironmentController(
+	accessKeyRepo db.AccessKeyManager,
+	encryptionService server.AccessKeyEncryptionService,
+	accessKeyService server.AccessKeyService,
+) *EnvironmentController {
+	return &EnvironmentController{
+		accessKeyRepo:     accessKeyRepo,
+		accessKeyService:  accessKeyService,
+		encryptionService: encryptionService,
+	}
+}
+
+func (c *EnvironmentController) updateEnvironmentSecrets(env db.Environment) error {
 	for _, secret := range env.Secrets {
 		err := secret.Validate()
 		if err != nil {
@@ -18,7 +37,7 @@ func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
 
 		switch secret.Operation {
 		case db.EnvironmentSecretCreate:
-			key, err = store.CreateAccessKey(db.AccessKey{
+			key, err = c.accessKeyService.CreateAccessKey(db.AccessKey{
 				Name:          secret.Name,
 				String:        secret.Secret,
 				EnvironmentID: &env.ID,
@@ -27,7 +46,7 @@ func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
 				Owner:         secret.Type.GetAccessKeyOwner(),
 			})
 		case db.EnvironmentSecretDelete:
-			key, err = store.GetAccessKey(env.ProjectID, secret.ID)
+			key, err = c.accessKeyRepo.GetAccessKey(env.ProjectID, secret.ID)
 
 			if err != nil {
 				continue
@@ -37,9 +56,9 @@ func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
 				continue
 			}
 
-			err = store.DeleteAccessKey(env.ProjectID, secret.ID)
+			err = c.accessKeyService.DeleteAccessKey(env.ProjectID, secret.ID)
 		case db.EnvironmentSecretUpdate:
-			key, err = store.GetAccessKey(env.ProjectID, secret.ID)
+			key, err = c.accessKeyRepo.GetAccessKey(env.ProjectID, secret.ID)
 
 			if err != nil {
 				continue
@@ -61,15 +80,15 @@ func updateEnvironmentSecrets(store db.Store, env db.Environment) error {
 				updateKey.OverrideSecret = true
 			}
 
-			err = store.UpdateAccessKey(updateKey)
+			err = c.accessKeyService.UpdateAccessKey(updateKey)
 		}
 	}
 
 	return nil
 }
 
 // EnvironmentMiddleware ensures an environment exists and loads it to the context
-func EnvironmentMiddleware(next http.Handler) http.Handler {
+func (c *EnvironmentController) EnvironmentMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		project := helpers.GetFromContext(r, "project").(db.Project)
 		envID, err := helpers.GetIntParam("environment_id", w, r)
@@ -85,7 +104,7 @@ func EnvironmentMiddleware(next http.Handler) http.Handler {
 			return
 		}
 
-		if err = db.FillEnvironmentSecrets(helpers.Store(r), &env, false); err != nil {
+		if err = c.encryptionService.FillEnvironmentSecrets(&env, false); err != nil {
 			helpers.WriteError(w, err)
 			return
 		}
@@ -128,7 +147,7 @@ func GetEnvironment(w http.ResponseWriter, r *http.Request) {
 }
 
 // UpdateEnvironment updates an existing environment in the database
-func UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
+func (c *EnvironmentController) UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
 	oldEnv := helpers.GetFromContext(r, "environment").(db.Environment)
 	var env db.Environment
 	if !helpers.Bind(w, r, &env) {
@@ -162,7 +181,7 @@ func UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
 		Description: fmt.Sprintf("Environment %s updated", env.Name),
 	})
 
-	if err := updateEnvironmentSecrets(helpers.Store(r), env); err != nil {
+	if err := c.updateEnvironmentSecrets(env); err != nil {
 		helpers.WriteError(w, err)
 		return
 	}
@@ -171,7 +190,7 @@ func UpdateEnvironment(w http.ResponseWriter, r *http.Request) {
 }
 
 // AddEnvironment creates an environment in the database
-func AddEnvironment(w http.ResponseWriter, r *http.Request) {
+func (c *EnvironmentController) AddEnvironment(w http.ResponseWriter, r *http.Request) {
 	project := helpers.GetFromContext(r, "project").(db.Project)
 	var env db.Environment
 
@@ -199,9 +218,9 @@ func AddEnvironment(w http.ResponseWriter, r *http.Request) {
 		Description: fmt.Sprintf("Environment %s created", newEnv.Name),
 	})
 
-	if err = updateEnvironmentSecrets(helpers.Store(r), newEnv); err != nil {
-		//helpers.WriteError(w, err)
-		//return
+	if err = c.updateEnvironmentSecrets(newEnv); err != nil {
+		helpers.WriteError(w, err)
+		return
 	}
 
 	// Reload env

api/projects/keys.go

@@ -1,13 +1,27 @@
 package projects
 
 import (
+	"errors"
 	"fmt"
+	"github.com/semaphoreui/semaphore/services/server"
 	"net/http"
 
 	"github.com/semaphoreui/semaphore/api/helpers"
 	"github.com/semaphoreui/semaphore/db"
 )
 
+type KeyController struct {
+	accessKeyService server.AccessKeyService
+}
+
+func NewKeyController(
+	accessKeyService server.AccessKeyService,
+) *KeyController {
+	return &KeyController{
+		accessKeyService: accessKeyService,
+	}
+}
+
 // KeyMiddleware ensures a key exists and loads it to the context
 func KeyMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -62,7 +76,7 @@ func GetKeys(w http.ResponseWriter, r *http.Request) {
 }
 
 // AddKey adds a new key to the database
-func AddKey(w http.ResponseWriter, r *http.Request) {
+func (c *KeyController) AddKey(w http.ResponseWriter, r *http.Request) {
 	project := helpers.GetFromContext(r, "project").(db.Project)
 	var key db.AccessKey
 
@@ -84,7 +98,7 @@ func AddKey(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	newKey, err := helpers.Store(r).CreateAccessKey(key)
+	newKey, err := c.accessKeyService.CreateAccessKey(key)
 
 	if err != nil {
 		helpers.WriteError(w, err)
@@ -111,7 +125,7 @@ func AddKey(w http.ResponseWriter, r *http.Request) {
 
 // UpdateKey updates key in database
 // nolint: gocyclo
-func UpdateKey(w http.ResponseWriter, r *http.Request) {
+func (c *KeyController) UpdateKey(w http.ResponseWriter, r *http.Request) {
 	var key db.AccessKey
 	oldKey := helpers.GetFromContext(r, "accessKey").(db.AccessKey)
 
@@ -136,7 +150,7 @@ func UpdateKey(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	err = helpers.Store(r).UpdateAccessKey(key)
+	err = c.accessKeyService.UpdateAccessKey(key)
 	if err != nil {
 		helpers.WriteError(w, err)
 		return
@@ -154,11 +168,11 @@ func UpdateKey(w http.ResponseWriter, r *http.Request) {
 }
 
 // RemoveKey deletes a key from the database
-func RemoveKey(w http.ResponseWriter, r *http.Request) {
+func (c *KeyController) RemoveKey(w http.ResponseWriter, r *http.Request) {
 	key := helpers.GetFromContext(r, "accessKey").(db.AccessKey)
 
-	err := helpers.Store(r).DeleteAccessKey(*key.ProjectID, key.ID)
-	if err == db.ErrInvalidOperation {
+	err := c.accessKeyService.DeleteAccessKey(*key.ProjectID, key.ID)
+	if errors.Is(err, db.ErrInvalidOperation) {
 		helpers.WriteJSON(w, http.StatusBadRequest, map[string]any{
 			"error": "Access Key is in use by one or more templates",
 			"inUse": true,

api/projects/project.go

@@ -4,7 +4,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/semaphoreui/semaphore/api/helpers"
 	"github.com/semaphoreui/semaphore/db"
-	"github.com/semaphoreui/semaphore/services"
+	"github.com/semaphoreui/semaphore/services/server"
 	"github.com/semaphoreui/semaphore/util"
 	log "github.com/sirupsen/logrus"
 	"net/http"
@@ -63,7 +63,7 @@ func GetMustCanMiddleware(permissions db.ProjectUserPermission) mux.MiddlewareFu
 }
 
 type ProjectController struct {
-	ProjectService services.ProjectService
+	ProjectService server.ProjectService
 }
 
 func (c *ProjectController) UpdateProject(w http.ResponseWriter, r *http.Request) {