Skip to content

Commit 0a74309

Browse files
authored
CLOUDP-153003: Encryption at Rest Tests for GCP & Azure (#1035)
1 parent 5701b50 commit 0a74309

File tree

11 files changed

+352
-407
lines changed

11 files changed

+352
-407
lines changed

.github/workflows/test-e2e.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -223,11 +223,11 @@ jobs:
223223
AWS_ACCOUNT_ARN_LIST: ${{ secrets.AWS_ACCOUNT_ARN_LIST }}
224224
AWS_KMS_KEY_ID: ${{ secrets.AWS_KMS_KEY_ID }}
225225
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
226-
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
226+
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID_NEW_TEST }}
227227
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
228-
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
228+
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET_NEW_TEST }}
229229
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
230-
GCP_SA_CRED: ${{ secrets.GCP_SA_CRED }}
230+
GCP_SA_CRED: ${{ secrets.GCP_SA_CRED_NEW_TEST }}
231231
DATADOG_KEY: ${{ secrets.DATADOG_KEY }}
232232
run: |
233233
helm version

go.mod

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@ module github.com/mongodb/mongodb-atlas-kubernetes
33
go 1.20
44

55
require (
6+
cloud.google.com/go/kms v1.12.1
67
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
78
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0
89
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
10+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0
911
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1
1012
github.com/Azure/go-autorest/autorest v0.11.29
1113
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
@@ -36,9 +38,11 @@ require (
3638
)
3739

3840
require (
41+
cloud.google.com/go/iam v1.1.0 // indirect
3942
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
4043
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 // indirect
4144
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
45+
github.com/benbjohnson/clock v1.3.0 // indirect
4246
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
4347
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
4448
github.com/google/s2a-go v0.1.4 // indirect

go.sum

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
2626
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
2727
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
2828
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
29+
cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94=
30+
cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk=
31+
cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U=
32+
cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM=
2933
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
3034
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
3135
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
@@ -45,6 +49,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9Orh
4549
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
4650
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
4751
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal v1.1.2 h1:mLY+pNLjCUeKhgnAJWAKhEUQM+RJQo2H1fuGSw1Ky1E=
52+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0 h1:8d4U82r7ItT1Es91x3eUcAQweih36KWvUha8AZ9X0Rs=
53+
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.2.0/go.mod h1:/1bkGperHinQbAHMWivoec/Ucu6//iXo6jn5mhmqCVU=
4854
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 h1:bWh0Z2rOEDfB/ywv/l0iHN1JgyazE6kW/aIA89+CEK0=
4955
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1/go.mod h1:Bzf34hhAE9NSxailk8xVeLEZbUjOXcC+GnU1mMKdhLw=
5056
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU=
@@ -90,6 +96,7 @@ github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzAR
9096
github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
9197
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
9298
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
99+
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
93100
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
94101
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
95102
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=

pkg/api/v1/encryption_at_rest.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,5 +94,6 @@ func (az AzureKeyVault) ToAtlas() mongodbatlas.AzureKeyVault {
9494
KeyVaultName: az.KeyVaultName,
9595
KeyIdentifier: az.KeyIdentifier,
9696
TenantID: az.TenantID,
97+
Secret: az.Secret,
9798
}
9899
}

pkg/controller/atlasproject/encryption_at_rest.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,14 +103,16 @@ func readAndFillGoogleSecret(kubeClient client.Client, parentNs string, gkms *md
103103
}
104104

105105
func readAndFillAzureSecret(kubeClient client.Client, parentNs string, azureVault *mdbv1.AzureKeyVault) (*watch.WatchedObject, error) {
106-
fieldData, watchObj, err := readSecretData(kubeClient, azureVault.SecretRef, parentNs, "ClientID", "AzureEnvironment", "SubscriptionID", "ResourceGroupName", "KeyVaultName", "KeyIdentifier")
106+
fieldData, watchObj, err := readSecretData(kubeClient, azureVault.SecretRef, parentNs, "ClientID", "Secret", "AzureEnvironment", "SubscriptionID", "ResourceGroupName", "KeyVaultName", "KeyIdentifier", "TenantID")
107107
if err != nil {
108108
return watchObj, err
109109
}
110110

111111
azureVault.ClientID = fieldData["ClientID"]
112+
azureVault.Secret = fieldData["Secret"]
112113
azureVault.AzureEnvironment = fieldData["AzureEnvironment"]
113114
azureVault.SubscriptionID = fieldData["SubscriptionID"]
115+
azureVault.TenantID = fieldData["TenantID"]
114116
azureVault.ResourceGroupName = fieldData["ResourceGroupName"]
115117
azureVault.KeyVaultName = fieldData["KeyVaultName"]
116118
azureVault.KeyIdentifier = fieldData["KeyIdentifier"]

pkg/controller/atlasproject/encryption_at_rest_test.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -211,8 +211,10 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
211211
t.Run("Azure with correct secret data", func(t *testing.T) {
212212
secretData := map[string][]byte{
213213
"ClientID": []byte("testClientID"),
214+
"Secret": []byte("testClientSecret"),
214215
"AzureEnvironment": []byte("testAzureEnvironment"),
215216
"SubscriptionID": []byte("testSubscriptionID"),
217+
"TenantID": []byte("testTenantID"),
216218
"ResourceGroupName": []byte("testResourceGroupName"),
217219
"KeyVaultName": []byte("testKeyVaultName"),
218220
"KeyIdentifier": []byte("testKeyIdentifier"),
@@ -226,7 +228,7 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
226228
APIVersion: "v1",
227229
},
228230
ObjectMeta: metav1.ObjectMeta{
229-
Name: "gcp-secret",
231+
Name: "azure-secret",
230232
Namespace: "test",
231233
},
232234
},
@@ -238,7 +240,7 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
238240
AzureKeyVault: mdbv1.AzureKeyVault{
239241
Enabled: toptr.MakePtr(true),
240242
SecretRef: common.ResourceRefNamespaced{
241-
Name: "gcp-secret",
243+
Name: "azure-secret",
242244
},
243245
},
244246
}
@@ -247,8 +249,10 @@ func TestReadEncryptionAtRestSecrets(t *testing.T) {
247249
assert.Nil(t, err)
248250

249251
assert.Equal(t, string(secretData["ClientID"]), encRest.AzureKeyVault.ClientID)
252+
assert.Equal(t, string(secretData["Secret"]), encRest.AzureKeyVault.Secret)
250253
assert.Equal(t, string(secretData["AzureEnvironment"]), encRest.AzureKeyVault.AzureEnvironment)
251254
assert.Equal(t, string(secretData["SubscriptionID"]), encRest.AzureKeyVault.SubscriptionID)
255+
assert.Equal(t, string(secretData["TenantID"]), encRest.AzureKeyVault.TenantID)
252256
assert.Equal(t, string(secretData["ResourceGroupName"]), encRest.AzureKeyVault.ResourceGroupName)
253257
assert.Equal(t, string(secretData["KeyVaultName"]), encRest.AzureKeyVault.KeyVaultName)
254258
assert.Equal(t, string(secretData["KeyIdentifier"]), encRest.AzureKeyVault.KeyIdentifier)

test/e2e/actions/cloud/aws.go

Lines changed: 135 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,22 @@
11
package cloud
22

33
import (
4+
"encoding/json"
5+
"errors"
46
"fmt"
7+
"os"
8+
"strings"
59

10+
"github.com/aws/aws-sdk-go/aws"
611
aws_sdk "github.com/aws/aws-sdk-go/aws"
712
"github.com/aws/aws-sdk-go/aws/session"
813
"github.com/aws/aws-sdk-go/service/ec2"
14+
"github.com/aws/aws-sdk-go/service/kms"
915
"github.com/aws/aws-sdk-go/service/sts"
1016
"github.com/onsi/ginkgo/v2/dsl/core"
1117

1218
"github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/toptr"
13-
"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/api/aws"
19+
"github.com/mongodb/mongodb-atlas-kubernetes/test/e2e/config"
1420
)
1521

1622
type AwsAction struct {
@@ -25,13 +31,136 @@ type awsNetwork struct {
2531
Subnets []*string
2632
}
2733

28-
func NewAwsAction() *AwsAction {
29-
return new(AwsAction)
34+
type principal struct {
35+
AWS []string `json:"AWS,omitempty"`
3036
}
3137

32-
func (awsAction *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (key string, err error) {
33-
session := aws.SessionAWS(region)
34-
return session.GetCustomerMasterKeyID(atlasAccountArn, assumedRoleArn)
38+
type kmsPolicy struct {
39+
Version string `json:"Version"`
40+
Statement []statement `json:"Statement"`
41+
}
42+
43+
type statement struct {
44+
Sid string `json:"Sid"`
45+
Effect string `json:"Effect"`
46+
Principal principal `json:"Principal"`
47+
Action []string `json:"Action"`
48+
Resource string `json:"Resource"`
49+
}
50+
51+
func (a *AwsAction) CreateKMS(region, atlasAccountArn, assumedRoleArn string) (key string, err error) {
52+
a.t.Helper()
53+
54+
kmsClient := kms.New(a.session, aws.NewConfig().WithRegion(config.AWSRegionUS))
55+
56+
keyId, adminARNs, err := getKeyIDAndAdminARNs()
57+
if err != nil {
58+
return "", err
59+
}
60+
61+
policyString, err := rolePolicyString(atlasAccountArn, assumedRoleArn, adminARNs)
62+
if err != nil {
63+
return "", err
64+
}
65+
66+
policyInput := &kms.PutKeyPolicyInput{
67+
KeyId: &keyId,
68+
PolicyName: aws_sdk.String("default"),
69+
Policy: aws_sdk.String(policyString),
70+
}
71+
72+
_, err = kmsClient.PutKeyPolicy(policyInput)
73+
if err != nil {
74+
return "", err
75+
}
76+
77+
return keyId, nil
78+
}
79+
80+
func getKeyIDAndAdminARNs() (keyID string, adminARNs []string, err error) {
81+
keyID = os.Getenv("AWS_KMS_KEY_ID")
82+
if keyID == "" {
83+
err = errors.New("AWS_KMS_KEY_ID secret is empty")
84+
return
85+
}
86+
adminArnString := os.Getenv("AWS_ACCOUNT_ARN_LIST")
87+
if adminArnString == "" {
88+
err = errors.New("AWS_ACCOUNT_ARN_LIST secret is empty")
89+
return
90+
}
91+
92+
adminARNs = strings.Split(adminArnString, ",")
93+
if len(adminARNs) == 0 {
94+
err = errors.New("AWS_ACCOUNT_ARN_LIST wasn't parsed properly, please separate accounts via a comma")
95+
return
96+
}
97+
98+
return keyID, adminARNs, nil
99+
}
100+
101+
func rolePolicyString(atlasAccountARN, assumedRoleARN string, adminARNs []string) (string, error) {
102+
policy := defaultKMSPolicy(atlasAccountARN, assumedRoleARN, adminARNs)
103+
byteStr, err := json.Marshal(policy)
104+
if err != nil {
105+
return "", err
106+
}
107+
return string(byteStr), nil
108+
}
109+
110+
func defaultKMSPolicy(atlasAccountArn, assumedRoleArn string, adminARNs []string) kmsPolicy {
111+
return kmsPolicy{
112+
Version: "2012-10-17",
113+
Statement: []statement{
114+
{
115+
Sid: "Enable IAM User Permissions",
116+
Effect: "Allow",
117+
Principal: principal{
118+
AWS: []string{atlasAccountArn},
119+
},
120+
Action: []string{"kms:*"},
121+
Resource: "*",
122+
},
123+
{
124+
Sid: "Allow access for Key Administrators",
125+
Effect: "Allow",
126+
Principal: principal{
127+
AWS: adminARNs,
128+
},
129+
Action: []string{
130+
"kms:Create*",
131+
"kms:Describe*",
132+
"kms:Enable*",
133+
"kms:List*",
134+
"kms:Put*",
135+
"kms:Update*",
136+
"kms:Revoke*",
137+
"kms:Disable*",
138+
"kms:Get*",
139+
"kms:Delete*",
140+
"kms:TagResource",
141+
"kms:UntagResource",
142+
"kms:ScheduleKeyDeletion",
143+
"kms:CancelKeyDeletion",
144+
},
145+
Resource: "*",
146+
},
147+
{
148+
Sid: "Allow use of the key",
149+
Effect: "Allow",
150+
Principal: principal{
151+
AWS: []string{assumedRoleArn},
152+
},
153+
Action: []string{
154+
"kms:Encrypt",
155+
"kms:Decrypt",
156+
"kms:ReEncrypt*",
157+
"kms:GenerateDataKey*",
158+
"kms:DescribeKey",
159+
},
160+
Resource: "*",
161+
},
162+
},
163+
}
35164
}
36165

37166
func (a *AwsAction) GetAccountID() (string, error) {

0 commit comments

Comments
 (0)