Skip to content

Commit dc35533

Browse files
mmandrusmmandrus
authored
CloudMigrations: Store encryption key in unified secrets table (#90908)
* store encryption key in unified secrets table * fix local dev mode * make metadata more realistic * fix tests * fix sql queries against postgres * fix stats endpoint
1 parent 49c756d commit dc35533

File tree

11 files changed

+60
-35
lines changed

11 files changed

+60
-35
lines changed

‎pkg/services/cloudmigration/api/api.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,6 +445,7 @@ func (cma *CloudMigrationAPI) GetSnapshot(c *contextmodel.ReqContext) response.R
445445
dtoStats := SnapshotResourceStats{
446446
Types: make(map[MigrateDataType]int, len(snapshot.StatsRollup.CountsByStatus)),
447447
Statuses: make(map[ItemStatus]int, len(snapshot.StatsRollup.CountsByType)),
448+
Total: snapshot.StatsRollup.Total,
448449
}
449450
for s, c := range snapshot.StatsRollup.CountsByStatus {
450451
dtoStats.Statuses[ItemStatus(s)] = c

‎pkg/services/cloudmigration/api/api_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,7 @@ func TestCloudMigrationAPI_GetSnapshot(t *testing.T) {
471471
requestUrl: "/api/cloudmigration/migration/1234/snapshot/1",
472472
basicRole: org.RoleAdmin,
473473
expectedHttpResult: http.StatusOK,
474-
expectedBody: `{"uid":"fake_uid","status":"CREATING","sessionUid":"1234","created":"0001-01-01T00:00:00Z","finished":"0001-01-01T00:00:00Z","results":[],"stats":{"types":{},"statuses":{}}}`,
474+
expectedBody: `{"uid":"fake_uid","status":"CREATING","sessionUid":"1234","created":"0001-01-01T00:00:00Z","finished":"0001-01-01T00:00:00Z","results":[],"stats":{"types":{},"statuses":{},"total":0}}`,
475475
},
476476
{
477477
desc: "should return 403 if no used is not admin",

‎pkg/services/cloudmigration/api/dtos.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,7 @@ type GetSnapshotResponseDTO struct {
307307
type SnapshotResourceStats struct {
308308
Types map[MigrateDataType]int `json:"types"`
309309
Statuses map[ItemStatus]int `json:"statuses"`
310+
Total int `json:"total"`
310311
}
311312

312313
// swagger:parameters getShapshotList

‎pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"github.com/grafana/grafana/pkg/services/folder"
2929
"github.com/grafana/grafana/pkg/services/gcom"
3030
"github.com/grafana/grafana/pkg/services/secrets"
31+
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
3132
"github.com/grafana/grafana/pkg/services/user"
3233
"github.com/grafana/grafana/pkg/setting"
3334
"github.com/grafana/grafana/pkg/util"
@@ -82,6 +83,7 @@ func ProvideService(
8283
features featuremgmt.FeatureToggles,
8384
db db.DB,
8485
dsService datasources.DataSourceService,
86+
secretsStore secretskv.SecretsKVStore,
8587
secretsService secrets.Service,
8688
routeRegister routing.RouteRegister,
8789
prom prometheus.Registerer,
@@ -95,7 +97,7 @@ func ProvideService(
9597
}
9698

9799
s := &Service{
98-
store: &sqlStore{db: db, secretsService: secretsService},
100+
store: &sqlStore{db: db, secretsStore: secretsStore, secretsService: secretsService},
99101
log: log.New(LogPrefix),
100102
cfg: cfg,
101103
features: features,

‎pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/grafana/grafana/pkg/services/folder"
2525
"github.com/grafana/grafana/pkg/services/folder/foldertest"
2626
secretsfakes "github.com/grafana/grafana/pkg/services/secrets/fakes"
27+
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
2728
"github.com/grafana/grafana/pkg/services/user"
2829
"github.com/grafana/grafana/pkg/setting"
2930
"github.com/prometheus/client_golang/prometheus"
@@ -438,6 +439,7 @@ func setUpServiceTest(t *testing.T, withDashboardMock bool) cloudmigration.Servi
438439
featuremgmt.FlagDashboardRestore),
439440
sqlStore,
440441
dsService,
442+
secretskv.NewFakeSQLSecretsKVStore(t),
441443
secretsService,
442444
rr,
443445
prometheus.DefaultRegisterer,

‎pkg/services/cloudmigration/cloudmigrationimpl/xorm_store.go‎

Lines changed: 20 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/grafana/grafana/pkg/infra/db"
1010
"github.com/grafana/grafana/pkg/services/cloudmigration"
1111
"github.com/grafana/grafana/pkg/services/secrets"
12+
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
1213
"github.com/grafana/grafana/pkg/services/sqlstore"
1314
"github.com/grafana/grafana/pkg/util"
1415
)
@@ -17,11 +18,13 @@ var _ store = (*sqlStore)(nil)
1718

1819
type sqlStore struct {
1920
db db.DB
21+
secretsStore secretskv.SecretsKVStore
2022
secretsService secrets.Service
2123
}
2224

2325
const (
24-
tableName = "cloud_migration_resource"
26+
tableName = "cloud_migration_resource"
27+
secretType = "cloudmigration-snapshot-encryption-key"
2528
)
2629

2730
func (ss *sqlStore) GetMigrationSessionByUID(ctx context.Context, uid string) (*cloudmigration.CloudMigrationSession, error) {
@@ -157,14 +160,14 @@ func (ss *sqlStore) GetMigrationStatusList(ctx context.Context, migrationUID str
157160
}
158161

159162
func (ss *sqlStore) CreateSnapshot(ctx context.Context, snapshot cloudmigration.CloudMigrationSnapshot) (string, error) {
160-
if err := ss.encryptKey(ctx, &snapshot); err != nil {
161-
return "", err
162-
}
163-
164163
if snapshot.UID == "" {
165164
snapshot.UID = util.GenerateShortUID()
166165
}
167166

167+
if err := ss.secretsStore.Set(ctx, secretskv.AllOrganizations, snapshot.UID, secretType, string(snapshot.EncryptionKey)); err != nil {
168+
return "", err
169+
}
170+
168171
err := ss.db.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
169172
snapshot.Created = time.Now()
170173
snapshot.Updated = time.Now()
@@ -228,8 +231,12 @@ func (ss *sqlStore) GetSnapshotByUID(ctx context.Context, uid string, resultPage
228231
return nil, err
229232
}
230233

231-
if err := ss.decryptKey(ctx, &snapshot); err != nil {
234+
if secret, found, err := ss.secretsStore.Get(ctx, secretskv.AllOrganizations, snapshot.UID, secretType); err != nil {
232235
return &snapshot, err
236+
} else if !found {
237+
return &snapshot, fmt.Errorf("encryption key not found for snapshot with UID %s", snapshot.UID)
238+
} else {
239+
snapshot.EncryptionKey = []byte(secret)
233240
}
234241

235242
resources, err := ss.GetSnapshotResources(ctx, uid, resultPage, resultLimit)
@@ -259,8 +266,12 @@ func (ss *sqlStore) GetSnapshotList(ctx context.Context, query cloudmigration.Li
259266
return nil, err
260267
}
261268
for i, snapshot := range snapshots {
262-
if err := ss.decryptKey(ctx, &snapshot); err != nil {
269+
if secret, found, err := ss.secretsStore.Get(ctx, secretskv.AllOrganizations, snapshot.UID, secretType); err != nil {
263270
return nil, err
271+
} else if !found {
272+
return nil, fmt.Errorf("encryption key not found for snapshot with UID %s", snapshot.UID)
273+
} else {
274+
snapshot.EncryptionKey = []byte(secret)
264275
}
265276

266277
if stats, err := ss.GetSnapshotResourceStats(ctx, snapshot.UID); err != nil {
@@ -346,14 +357,14 @@ func (ss *sqlStore) GetSnapshotResourceStats(ctx context.Context, snapshotUid st
346357
} else {
347358
total = int(t)
348359
}
349-
sess.Select("count(uid) as 'count', resource_type as 'type'").
360+
sess.Select("count(uid) as \"count\", resource_type as \"type\"").
350361
Table(tableName).
351362
GroupBy("type").
352363
Where("snapshot_uid = ?", snapshotUid)
353364
if err := sess.Find(&typeCounts); err != nil {
354365
return err
355366
}
356-
sess.Select("count(uid) as 'count', status").
367+
sess.Select("count(uid) as \"count\", status").
357368
Table(tableName).
358369
GroupBy("status").
359370
Where("snapshot_uid = ?", snapshotUid)
@@ -411,24 +422,3 @@ func (ss *sqlStore) decryptToken(ctx context.Context, cm *cloudmigration.CloudMi
411422

412423
return nil
413424
}
414-
415-
func (ss *sqlStore) encryptKey(ctx context.Context, snapshot *cloudmigration.CloudMigrationSnapshot) error {
416-
s, err := ss.secretsService.Encrypt(ctx, snapshot.EncryptionKey, secrets.WithoutScope())
417-
if err != nil {
418-
return fmt.Errorf("encrypting key: %w", err)
419-
}
420-
421-
snapshot.EncryptionKey = s
422-
423-
return nil
424-
}
425-
426-
func (ss *sqlStore) decryptKey(ctx context.Context, snapshot *cloudmigration.CloudMigrationSnapshot) error {
427-
t, err := ss.secretsService.Decrypt(ctx, snapshot.EncryptionKey)
428-
if err != nil {
429-
return fmt.Errorf("decrypting key: %w", err)
430-
}
431-
snapshot.EncryptionKey = t
432-
433-
return nil
434-
}

‎pkg/services/cloudmigration/cloudmigrationimpl/xorm_store_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/grafana/grafana/pkg/infra/db"
1010
"github.com/grafana/grafana/pkg/services/cloudmigration"
1111
fakeSecrets "github.com/grafana/grafana/pkg/services/secrets/fakes"
12+
secretskv "github.com/grafana/grafana/pkg/services/secrets/kvstore"
1213
"github.com/grafana/grafana/pkg/services/sqlstore"
1314
"github.com/grafana/grafana/pkg/tests/testsuite"
1415
"github.com/grafana/grafana/pkg/util"
@@ -274,6 +275,7 @@ func setUpTest(t *testing.T) (*sqlstore.SQLStore, *sqlStore) {
274275
s := &sqlStore{
275276
db: testDB,
276277
secretsService: fakeSecrets.FakeSecretsService{},
278+
secretsStore: secretskv.NewFakeSQLSecretsKVStore(t),
277279
}
278280
ctx := context.Background()
279281

‎pkg/services/cloudmigration/gmsclient/inmemory_client.go‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package gmsclient
22

33
import (
44
"context"
5+
"encoding/json"
56
"fmt"
67
"math/rand"
78

@@ -51,16 +52,34 @@ func (c *memoryClientImpl) MigrateData(
5152
return &result, nil
5253
}
5354

54-
func (c *memoryClientImpl) StartSnapshot(context.Context, cloudmigration.CloudMigrationSession) (*cloudmigration.StartSnapshotResponse, error) {
55+
func (c *memoryClientImpl) StartSnapshot(_ context.Context, sess cloudmigration.CloudMigrationSession) (*cloudmigration.StartSnapshotResponse, error) {
5556
publicKey, _, err := box.GenerateKey(cryptoRand.Reader)
5657
if err != nil {
5758
return nil, fmt.Errorf("nacl: generating public and private key: %w", err)
5859
}
60+
61+
snapshotUid := uuid.NewString()
62+
63+
metadataBuffer, err := json.Marshal(struct {
64+
SnapshotID string `json:"snapshotID"`
65+
StackID string `json:"stackID"`
66+
Slug string `json:"slug"`
67+
}{
68+
SnapshotID: snapshotUid,
69+
StackID: fmt.Sprintf("%d", sess.StackID),
70+
Slug: sess.Slug,
71+
})
72+
73+
if err != nil {
74+
return nil, fmt.Errorf("marshalling metadata: %w", err)
75+
}
76+
5977
c.snapshot = &cloudmigration.StartSnapshotResponse{
6078
EncryptionKey: publicKey[:],
61-
SnapshotID: uuid.NewString(),
79+
SnapshotID: snapshotUid,
6280
MaxItemsPerPartition: 10,
6381
Algo: "nacl",
82+
Metadata: metadataBuffer,
6483
}
6584

6685
return c.snapshot, nil

‎pkg/services/cloudmigration/model.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ type CloudMigrationSnapshot struct {
3737
UID string `xorm:"uid"`
3838
SessionUID string `xorm:"session_uid"`
3939
Status SnapshotStatus
40-
EncryptionKey []byte `xorm:"encryption_key"` // stored in the unified secrets table
40+
EncryptionKey []byte `xorm:"-"` // stored in the unified secrets table
4141
LocalDir string `xorm:"local_directory"`
4242
GMSSnapshotUID string `xorm:"gms_snapshot_uid"`
4343
ErrorString string `xorm:"error_string"`

‎public/api-merged.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20421,6 +20421,10 @@
2042120421
"format": "int64"
2042220422
}
2042320423
},
20424+
"total": {
20425+
"type": "integer",
20426+
"format": "int64"
20427+
},
2042420428
"types": {
2042520429
"type": "object",
2042620430
"additionalProperties": {

0 commit comments

Comments
 (0)