Skip to content

Commit a5a375c

Browse files
wallrjwallrj
committed
Simplify the conversion function and add a golden file test
Signed-off-by: Richard Wall <[email protected]>
1 parent 369494d commit a5a375c

File tree

4 files changed

+12558
-90
lines changed

4 files changed

+12558
-90
lines changed

‎pkg/internal/cyberark/dataupload/dataupload.go

Lines changed: 64 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"k8s.io/client-go/transport"
1616

1717
"github.com/jetstack/preflight/api"
18+
"github.com/jetstack/preflight/pkg/datagatherer/k8s"
1819
"github.com/jetstack/preflight/pkg/version"
1920
)
2021

@@ -29,6 +30,8 @@ const (
2930
apiPathSnapshotLinks = "/api/ingestions/kubernetes/snapshot-links"
3031
)
3132

33+
type ResourceData map[string][]interface{}
34+
3235
// Snapshot is the JSON that the CyberArk Discovery and Context API expects to
3336
// be uploaded to the AWS presigned URL.
3437
type Snapshot struct {
@@ -41,110 +44,81 @@ type Snapshot struct {
4144
RoleBindings []interface{} `json:"role_bindings"`
4245
}
4346

44-
// The names of Datagatherer configs which have the data to populate the Cyberark Snapshot
45-
const (
46-
Discovery = "k8s-discovery"
47-
SecretsGatherer = "k8s/secrets"
48-
ServiceAccountsGatherer = "k8s/serviceaccounts"
49-
RolesGatherer = "k8s/roles"
50-
RoleBindingsGatherer = "k8s/rolebindings"
51-
ClusterRolesGatherer = "k8s/clusterroles"
52-
ClusterRoleBindingsGatherer = "k8s/clusterrolebindings"
53-
)
47+
// The names of Datagatherers which have the data to populate the Cyberark Snapshot mapped to the key in the Cyberark snapshot.
48+
var gathererNameToresourceDataKeyMap = map[string]string{
49+
"k8s/secrets": "secrets",
50+
"k8s/serviceaccounts": "serviceaccounts",
51+
"k8s/roles": "roles",
52+
"k8s/clusterroles": "roles",
53+
"k8s/rolebindings": "rolebindings",
54+
"k8s/clusterrolebindings": "rolebindings",
55+
}
56+
57+
func extractResourceListFromReading(reading *api.DataReading) ([]interface{}, error) {
58+
data, ok := reading.Data.(*k8s.DynamicData)
59+
if !ok {
60+
return nil, fmt.Errorf("failed to convert data: %s", reading.DataGatherer)
61+
}
62+
items := data.Items
63+
resources := make([]interface{}, len(items))
64+
for i, resource := range items {
65+
resources[i] = resource.Resource
66+
}
67+
return resources, nil
68+
}
69+
70+
// TODO(wallj): Use k8s version.Info struct here
71+
func extractServerVersionFromReading(reading *api.DataReading) (string, error) {
72+
data, ok := reading.Data.(map[string]interface{})
73+
if !ok {
74+
return "", fmt.Errorf("failed to convert data: %s", reading.DataGatherer)
75+
}
76+
serverVersion, ok := data["server_version"]
77+
if !ok {
78+
return "", fmt.Errorf("server_version key not found in data: %v", data)
79+
}
80+
serverVersionBytes, err := json.Marshal(serverVersion)
81+
if err != nil {
82+
return "", fmt.Errorf("while marshalling server_version: %s", err)
83+
}
84+
var serverVersionInfo map[string]string
85+
if err := json.Unmarshal(serverVersionBytes, &serverVersionInfo); err != nil {
86+
return "", fmt.Errorf("while un-marshalling server_version bytes: %s", err)
87+
}
88+
return serverVersionInfo["gitVersion"], nil
89+
}
5490

5591
// ConvertDataReadingsToCyberarkSnapshot converts jetstack-secure DataReadings into Cyberark Snapshot format.
5692
func ConvertDataReadingsToCyberarkSnapshot(
5793
input api.DataReadingsPost,
58-
) (snapshot Snapshot, err error) {
59-
var (
60-
k8sVersion string
61-
secrets, serviceAccounts, roles, roleBindings []interface{}
62-
)
63-
94+
) (_ *Snapshot, err error) {
95+
k8sVersion := ""
96+
resourceData := ResourceData{}
6497
for _, reading := range input.DataReadings {
65-
switch reading.DataGatherer {
66-
case Discovery:
67-
data, ok := reading.Data.(map[string]interface{})
68-
if !ok {
69-
return snapshot, fmt.Errorf("failed to convert: %s", reading.DataGatherer)
70-
}
71-
serverVersion := data["server_version"]
72-
serverVersionBytes, err := json.Marshal(serverVersion)
98+
if reading.DataGatherer == "k8s/discovery" {
99+
k8sVersion, err = extractServerVersionFromReading(reading)
73100
if err != nil {
74-
return snapshot, fmt.Errorf("while marshalling server_version: %s", err)
75-
}
76-
var serverVersionInfo map[string]string
77-
if err := json.Unmarshal(serverVersionBytes, &serverVersionInfo); err != nil {
78-
return snapshot, fmt.Errorf("while un-marshalling server_version bytes: %s", err)
101+
return nil, fmt.Errorf("while extracting server version from data-reading: %s", err)
79102
}
80-
k8sVersion = serverVersionInfo["gitVersion"]
81-
case SecretsGatherer:
82-
if data, ok := reading.Data.(map[string]interface{}); ok {
83-
if items, ok := data["items"].([]*api.GatheredResource); ok {
84-
resources := make([]interface{}, len(items))
85-
for i, resource := range items {
86-
resources[i] = resource.Resource
87-
}
88-
secrets = append(secrets, resources...)
89-
} else {
90-
return snapshot, fmt.Errorf("failed to convert: %s", reading.DataGatherer)
91-
}
92-
} else {
93-
return snapshot, fmt.Errorf("failed to convert: %s", reading.DataGatherer)
94-
}
95-
case ServiceAccountsGatherer:
96-
if data, ok := reading.Data.(map[string]interface{}); ok {
97-
if items, ok := data["items"].([]*api.GatheredResource); ok {
98-
resources := make([]interface{}, len(items))
99-
for i, resource := range items {
100-
resources[i] = resource.Resource
101-
}
102-
serviceAccounts = append(serviceAccounts, resources...)
103-
} else {
104-
return snapshot, fmt.Errorf("failed to convert: %s", reading.DataGatherer)
105-
}
106-
} else {
107-
return snapshot, fmt.Errorf("failed to convert: %s", reading.DataGatherer)
108-
}
109-
case RolesGatherer, ClusterRoleBindingsGatherer:
110-
if data, ok := reading.Data.(map[string]interface{}); ok {
111-
if items, ok := data["items"].([]*api.GatheredResource); ok {
112-
resources := make([]interface{}, len(items))
113-
for i, resource := range items {
114-
resources[i] = resource.Resource
115-
}
116-
roles = append(roles, resources...)
117-
} else {
118-
return snapshot, fmt.Errorf("failed to convert: %s", reading.DataGatherer)
119-
}
120-
} else {
121-
return snapshot, fmt.Errorf("failed to convert: %s", reading.DataGatherer)
122-
}
123-
case RoleBindingsGatherer, ClusterRolesGatherer:
124-
if data, ok := reading.Data.(map[string]interface{}); ok {
125-
if items, ok := data["items"].([]*api.GatheredResource); ok {
126-
resources := make([]interface{}, len(items))
127-
for i, resource := range items {
128-
resources[i] = resource.Resource
129-
}
130-
roleBindings = append(roleBindings, resources...)
131-
} else {
132-
return snapshot, fmt.Errorf("failed to convert: %s", reading.DataGatherer)
133-
}
134-
} else {
135-
return snapshot, fmt.Errorf("failed to convert: %s", reading.DataGatherer)
103+
}
104+
if key, found := gathererNameToresourceDataKeyMap[reading.DataGatherer]; found {
105+
var resources []interface{}
106+
resources, err = extractResourceListFromReading(reading)
107+
if err != nil {
108+
return nil, fmt.Errorf("while extracting resource list from data-reading: %s", err)
136109
}
110+
resourceData[key] = append(resourceData[key], resources...)
137111
}
138112
}
139113

140-
return Snapshot{
114+
return &Snapshot{
141115
AgentVersion: input.AgentMetadata.Version,
142116
ClusterID: input.AgentMetadata.ClusterID,
143117
K8SVersion: k8sVersion,
144-
Secrets: secrets,
145-
ServiceAccounts: serviceAccounts,
146-
Roles: roles,
147-
RoleBindings: roleBindings,
118+
Secrets: resourceData["secrets"],
119+
ServiceAccounts: resourceData["serviceaccounts"],
120+
Roles: resourceData["roles"],
121+
RoleBindings: resourceData["rolebindings"],
148122
}, nil
149123
}
150124

‎pkg/internal/cyberark/dataupload/dataupload_test.go

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package dataupload_test
22

33
import (
4+
"bytes"
45
"crypto/x509"
6+
"encoding/json"
57
"encoding/pem"
68
"errors"
79
"fmt"
@@ -10,11 +12,13 @@ import (
1012
"testing"
1113
"time"
1214

15+
"github.com/stretchr/testify/assert"
1316
"github.com/stretchr/testify/require"
1417
"k8s.io/klog/v2"
1518
"k8s.io/klog/v2/ktesting"
1619

1720
"github.com/jetstack/preflight/api"
21+
"github.com/jetstack/preflight/pkg/datagatherer/k8s"
1822
"github.com/jetstack/preflight/pkg/internal/cyberark/dataupload"
1923
"github.com/jetstack/preflight/pkg/internal/cyberark/identity"
2024
"github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery"
@@ -187,16 +191,80 @@ func TestCyberArkClient_PostDataReadingsWithOptions_RealAPI(t *testing.T) {
187191
cyberArkClient, err := dataupload.NewCyberArkClient(nil, serviceURL, identityClient.AuthenticateRequest)
188192
require.NoError(t, err)
189193

194+
dataReadings := loadDataReadings(t, "testdata/example-1/datareadings.json")
190195
err = cyberArkClient.PostDataReadingsWithOptions(
191196
ctx,
192197
api.DataReadingsPost{
193198
AgentMetadata: &api.AgentMetadata{
194199
ClusterID: "bb068932-c80d-460d-88df-34bc7f3f3297",
195200
},
201+
DataReadings: dataReadings,
196202
},
197203
dataupload.Options{
198204
ClusterName: "bb068932-c80d-460d-88df-34bc7f3f3297",
199205
},
200206
)
201207
require.NoError(t, err)
202208
}
209+
210+
func loadDataReadings(t *testing.T, path string) []*api.DataReading {
211+
var dataReadings []*api.DataReading
212+
{
213+
f, err := os.Open(path)
214+
require.NoError(t, err)
215+
err = json.NewDecoder(f).Decode(&dataReadings)
216+
require.NoError(t, f.Close())
217+
require.NoError(t, err)
218+
}
219+
220+
for _, reading := range dataReadings {
221+
dataBytes, err := json.Marshal(reading.Data)
222+
require.NoError(t, err)
223+
in := bytes.NewReader(dataBytes)
224+
d := json.NewDecoder(in)
225+
d.DisallowUnknownFields()
226+
227+
var dynamicGatherData k8s.DynamicData
228+
if err := d.Decode(&dynamicGatherData); err == nil {
229+
reading.Data = &dynamicGatherData
230+
continue
231+
}
232+
233+
_, err = in.Seek(0, 0)
234+
require.NoError(t, err)
235+
236+
var discoveryData k8s.DiscoveryData
237+
if err = d.Decode(&discoveryData); err == nil {
238+
reading.Data = &discoveryData
239+
continue
240+
}
241+
242+
require.Failf(t, "failed to parse reading", "reading: %#v", reading)
243+
}
244+
return dataReadings
245+
}
246+
247+
func TestConvertDataReadingsToCyberarkSnapshot(t *testing.T) {
248+
dataReadings := loadDataReadings(t, "testdata/example-1/datareadings.json")
249+
snapshot, err := dataupload.ConvertDataReadingsToCyberarkSnapshot(api.DataReadingsPost{
250+
AgentMetadata: &api.AgentMetadata{
251+
Version: "test-version",
252+
ClusterID: "test-cluster-id",
253+
},
254+
DataReadings: dataReadings,
255+
})
256+
require.NoError(t, err)
257+
258+
actualSnapshotBytes, err := json.MarshalIndent(snapshot, "", " ")
259+
require.NoError(t, err)
260+
261+
goldenFilePath := "testdata/example-1/snapshot.json"
262+
if _, update := os.LookupEnv("UPDATE_GOLDEN_FILES"); update {
263+
err := os.WriteFile(goldenFilePath, actualSnapshotBytes, 0o0644)
264+
require.NoError(t, err)
265+
} else {
266+
expectedSnapshotBytes, err := os.ReadFile(goldenFilePath)
267+
require.NoError(t, err)
268+
assert.JSONEq(t, string(expectedSnapshotBytes), string(actualSnapshotBytes))
269+
}
270+
}

‎pkg/internal/cyberark/dataupload/testdata/example-1/README.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,9 @@ go run . agent \
1818
--output-path pkg/internal/cyberark/dataupload/testdata/example-1/datareadings.json
1919
```
2020

21+
22+
To recreate the golden output file:
23+
24+
```bash
25+
UPDATE_GOLDEN_FILES=true go test ./pkg/internal/cyberark/dataupload/... -run TestConvertDataReadingsToCyberarkSnapshot
26+
```

0 commit comments

Comments
 (0)