Skip to content

Commit 9d4a340

Browse files
charlieegan3William Squires
charlieegan3
and
William Squires
authored
Data gatherers file (#83)
* Add method for AKS data gatherer to load data from file rather than using AKS API Signed-off-by: wwwil <[email protected]> * Add local data gatherer for loading data from local files Signed-off-by: wwwil <[email protected]> * If any data gatherer specifies a data-path then use a LocalDataGatherer to impersonate it Signed-off-by: wwwil <[email protected]> * Remove data-path handling from AKS data gatherer Signed-off-by: wwwil <[email protected]> * Add an example fake data for AKS and GKE with config for using fake data Signed-off-by: wwwil <[email protected]> * Fake -> stubbed As discussed on standup this morning Signed-off-by: Charlie Egan <[email protected]> Co-authored-by: William Squires <[email protected]>
1 parent 33cfa96 commit 9d4a340

File tree

7 files changed

+261
-0
lines changed

7 files changed

+261
-0
lines changed

‎cmd/check.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/jetstack/preflight/pkg/datagatherer/eks"
1717
"github.com/jetstack/preflight/pkg/datagatherer/gke"
1818
"github.com/jetstack/preflight/pkg/datagatherer/k8s"
19+
localdatagatherer "github.com/jetstack/preflight/pkg/datagatherer/local"
1920
"github.com/jetstack/preflight/pkg/output"
2021
"github.com/jetstack/preflight/pkg/output/azblob"
2122
"github.com/jetstack/preflight/pkg/output/gcs"
@@ -130,6 +131,19 @@ func check() {
130131
for name, config := range gatherersConfig {
131132
// TODO: create gatherer from config in a more clever way. We need to read gatherer config from here and its schema depends on the gatherer itself.
132133
var dg datagatherer.DataGatherer
134+
dataGathererConfig, ok := config.(map[string]interface{})
135+
if !ok {
136+
log.Fatalf("Cannot parse %s data gatherer config.", name)
137+
}
138+
// Check if this data gatherer's config specifies a data-path.
139+
// If it does create a LocalDataGatherer to load this data but keep
140+
// the name of the data gatherer it is impersonating so it can
141+
// provide stubbed data.
142+
if dataPath, ok := dataGathererConfig["data-path"].(string); ok && dataPath != "" {
143+
dg = localdatagatherer.NewLocalDataGatherer(dataPath)
144+
gatherers[name] = dg
145+
continue
146+
}
133147
if name == "eks" {
134148
eksConfig, ok := config.(map[string]interface{})
135149
if !ok {
@@ -207,6 +221,13 @@ func check() {
207221
log.Fatalf("Cannot create k8s client: %+v", err)
208222
}
209223
dg = k8s.NewPodsDataGatherer(k8sClient)
224+
} else if name == "local" {
225+
localConfig, ok := config.(map[string]interface{})
226+
if !ok {
227+
log.Fatal("Cannot parse 'data-gatherers.local' in config.")
228+
}
229+
dataPath, ok := localConfig["data-path"].(string)
230+
dg = localdatagatherer.NewLocalDataGatherer(dataPath)
210231
} else {
211232
log.Fatalf("Found unsupported data-gatherer %q in config.", name)
212233
}

‎docs/configuration.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ The following data gatherers are available:
4141
- [Google Kubernetes Engine](docs/datagatherers/gke.md)
4242
- [Amazon Elastic Kubernetes Service](docs/datagatherers/eks.md)
4343
- [Microsoft Azure Kubernetes Service](docs/datagatherers/aks.md)
44+
- [Local](docs/datagatheres/local.md)
4445

4546
# Package Sources
4647

‎docs/datagatherers/local.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Local Data Gatherer
2+
3+
This is intended to be used for reading data for evaluation from the local file
4+
system. It can also be used for 'stubbing' remote data sources when testing
5+
other data gatherers. An example of this can be seen
6+
[here](../../examples/stubbed.preflight.yaml).

‎examples/data/aks.json‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"aks": {
3+
"Cluster": {
4+
"properties": {
5+
"networkProfile": {
6+
"dockerBridgeCidr": "172.17.0.1/16",
7+
"dnsServiceIP": "10.0.0.10",
8+
"serviceCidr": "10.0.0.0/16",
9+
"podCidr": "10.244.0.0/16",
10+
"networkPlugin": "kubenet"
11+
},
12+
"enableRBAC": true,
13+
"nodeResourceGroup": "MC_preflight_preflight-test_uksouth",
14+
"addonProfiles": {
15+
"httpApplicationRouting": {
16+
"config": null,
17+
"enabled": false
18+
}
19+
},
20+
"servicePrincipalProfile": {
21+
"clientId": "***********************"
22+
},
23+
"agentPoolProfiles": [
24+
{
25+
"osType": "Linux",
26+
"maxPods": 110,
27+
"storageProfile": "ManagedDisks",
28+
"osDiskSizeGB": 100,
29+
"vmSize": "Standard_B2s",
30+
"count": 1,
31+
"name": "agentpool"
32+
}
33+
],
34+
"fqdn": "preflight-test-dns-f2de96e5.hcp.uksouth.azmk8s.io",
35+
"dnsPrefix": "preflight-test-dns",
36+
"kubernetesVersion": "1.14.8",
37+
"provisioningState": "Succeeded"
38+
},
39+
"type": "Microsoft.ContainerService/ManagedClusters",
40+
"name": "preflight-test",
41+
"location": "uksouth",
42+
"id": "/subscriptions/*****************/resourcegroups/preflight/providers/Microsoft.ContainerService/managedClusters/preflight-test"
43+
}
44+
}
45+
}

‎examples/data/gke.json‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
{
2+
"addonsConfig": {
3+
"horizontalPodAutoscaling": {},
4+
"httpLoadBalancing": {},
5+
"istioConfig": {
6+
"disabled": true
7+
},
8+
"kubernetesDashboard": {
9+
"disabled": true
10+
},
11+
"networkPolicyConfig": {
12+
"disabled": true
13+
}
14+
},
15+
"authenticatorGroupsConfig": {},
16+
"clusterIpv4Cidr": "10.4.0.0/14",
17+
"clusterTelemetry": {
18+
"type": "ENABLED"
19+
},
20+
"createTime": "2020-01-15T18:55:55+00:00",
21+
"currentMasterVersion": "1.14.8-gke.12",
22+
"currentNodeCount": 3,
23+
"currentNodeVersion": "1.14.8-gke.12",
24+
"databaseEncryption": {
25+
"state": "DECRYPTED"
26+
},
27+
"defaultMaxPodsConstraint": {
28+
"maxPodsPerNode": "110"
29+
},
30+
"endpoint": "104.155.177.255",
31+
"initialClusterVersion": "1.14.8-gke.12",
32+
"instanceGroupUrls": [
33+
"https://www.googleapis.com/compute/v1/projects/jetstack-wil/zones/us-central1-a/instanceGroupManagers/gke-standard-cluster-2-default-pool-83d58bed-grp"
34+
],
35+
"ipAllocationPolicy": {
36+
"clusterIpv4Cidr": "10.4.0.0/14",
37+
"clusterIpv4CidrBlock": "10.4.0.0/14",
38+
"clusterSecondaryRangeName": "gke-standard-cluster-2-pods-fddd6f46",
39+
"servicesIpv4Cidr": "10.70.0.0/20",
40+
"servicesIpv4CidrBlock": "10.70.0.0/20",
41+
"servicesSecondaryRangeName": "gke-standard-cluster-2-services-fddd6f46",
42+
"useIpAliases": true
43+
},
44+
"labelFingerprint": "a9dc16a7",
45+
"legacyAbac": {},
46+
"location": "us-central1-a",
47+
"locations": [
48+
"us-central1-a"
49+
],
50+
"loggingService": "logging.googleapis.com/kubernetes",
51+
"maintenancePolicy": {
52+
"resourceVersion": "e3b0c442"
53+
},
54+
"masterAuth": {
55+
"clusterCaCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURERENDQWZTZ0F3SUJBZ0lSQUtQT2NXc0JGTzFiZTJrbWZxMFZZNzh3RFFZSktvWklodmNOQVFFTEJRQXcKTHpFdE1Dc0dBMVVFQXhNa1pXSmtPVGxqTlRNdE5XRTRZaTAwT1dRMUxUZzJOV1V0TUdVell6a3daakJsWldGaApNQjRYRFRJd01ERXhOVEUzTlRVMU5Wb1hEVEkxTURFeE16RTROVFUxTlZvd0x6RXRNQ3NHQTFVRUF4TWtaV0prCk9UbGpOVE10TldFNFlpMDBPV1ExTFRnMk5XVXRNR1V6WXprd1pqQmxaV0ZoTUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBM0xiSkJ0Vkoxa2gxNERXWmhKMlRZMkxtTzFXSk51NjNLZzhBazVEegpWb3hzTml4VXBZQkZwaUkxN0J1Qm12bXkvQmN1czIrcTFIWmxtLzZNc29wZ1JWaHFFaUdnaDhEanhsaHBhckkwCjNxekJjRTVkaXE1dGd3V3pGeDF0S20wRmdmUGt3ZzRUTHZ2d25VaERMU21ranhuS2RBRVdCbnR2WDZxNkJmZ3AKbEczazJtd0gyYUNiYVNMSm1kRXJUR3RNMVFWN1N0REVYR2N3TlVjOUZheDNQbzk1elFLNjQ4N1U3N291ZVExeQpTQU9JVWZ4aDU2MFpVU1E1OXRYdkVTTWN2SUdKaVUvSWNCVHAvZ1VIUm5xb0RRRitvQmVKRm1abENla2hPeWthCnVlK052a09GSnI1TW1Rdys0ck5uV2NvbTdlZzljRWh2cWhpSFBHYU1wem9xSndJREFRQUJveU13SVRBT0JnTlYKSFE4QkFmOEVCQU1DQWdRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQQoxRDJiT1p4b0lpMG9EOXVGQTY1WXowRFlyclMzWUo3K3lkWXNjVFR0VEg5Ry9PWkhZcU5sTTM1SXJmbE1SYUFaCmNFL3VBUThSaGorcWtuQ01OUHhLL3pqYVYyVGhxMzlYYzZlSTlqeWl5RDVRb3lmSHBPeWF1aGpFTXVoajU1L3oKVmlleHE2VGZ6cXhBakJhWTJhMmpQVk81c1ZyZVZqUnNzUFlncEh4YVk2a25WQlZ4Rzg0WHB6bWJiUFk2cjlBNwphZ2ROWldvQms4cUxtRmxxVnBoYVRsNnpoeXczS2dhL09VQUNlQkZZN0lXQzM2bTVkV2xOWlVXbHRWS3EvaE5NCkV1VlRKMXNUblhFVXplTlBnZ1FaRy9qbzA3bWI0SnFHWmZNaFBZTE9mZExhdUxPcTJhNjAwcnlkUTk4dFB5L0YKS2FSaG0xVWp3M1oxZTNZeno0cUQ4UT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
56+
},
57+
"masterAuthorizedNetworksConfig": {},
58+
"monitoringService": "monitoring.googleapis.com/kubernetes",
59+
"name": "standard-cluster-2",
60+
"network": "default",
61+
"networkConfig": {
62+
"network": "projects/jetstack-wil/global/networks/default",
63+
"subnetwork": "projects/jetstack-wil/regions/us-central1/subnetworks/default"
64+
},
65+
"networkPolicy": {},
66+
"nodeConfig": {
67+
"diskSizeGb": 100,
68+
"diskType": "pd-standard",
69+
"imageType": "COS",
70+
"machineType": "n1-standard-1",
71+
"metadata": {
72+
"disable-legacy-endpoints": "true"
73+
},
74+
"oauthScopes": [
75+
"https://www.googleapis.com/auth/devstorage.read_only",
76+
"https://www.googleapis.com/auth/logging.write",
77+
"https://www.googleapis.com/auth/monitoring",
78+
"https://www.googleapis.com/auth/servicecontrol",
79+
"https://www.googleapis.com/auth/service.management.readonly",
80+
"https://www.googleapis.com/auth/trace.append"
81+
],
82+
"serviceAccount": "default",
83+
"shieldedInstanceConfig": {
84+
"enableIntegrityMonitoring": true
85+
}
86+
},
87+
"nodePools": [
88+
{
89+
"autoscaling": {},
90+
"config": {
91+
"diskSizeGb": 100,
92+
"diskType": "pd-standard",
93+
"imageType": "COS",
94+
"machineType": "n1-standard-1",
95+
"metadata": {
96+
"disable-legacy-endpoints": "true"
97+
},
98+
"oauthScopes": [
99+
"https://www.googleapis.com/auth/devstorage.read_only",
100+
"https://www.googleapis.com/auth/logging.write",
101+
"https://www.googleapis.com/auth/monitoring",
102+
"https://www.googleapis.com/auth/servicecontrol",
103+
"https://www.googleapis.com/auth/service.management.readonly",
104+
"https://www.googleapis.com/auth/trace.append"
105+
],
106+
"serviceAccount": "default",
107+
"shieldedInstanceConfig": {
108+
"enableIntegrityMonitoring": true
109+
}
110+
},
111+
"initialNodeCount": 3,
112+
"instanceGroupUrls": [
113+
"https://www.googleapis.com/compute/v1/projects/jetstack-wil/zones/us-central1-a/instanceGroupManagers/gke-standard-cluster-2-default-pool-83d58bed-grp"
114+
],
115+
"locations": [
116+
"us-central1-a"
117+
],
118+
"management": {
119+
"autoRepair": true,
120+
"autoUpgrade": true
121+
},
122+
"maxPodsConstraint": {
123+
"maxPodsPerNode": "110"
124+
},
125+
"name": "default-pool",
126+
"podIpv4CidrSize": 24,
127+
"selfLink": "https://container.googleapis.com/v1beta1/projects/jetstack-wil/zones/us-central1-a/clusters/standard-cluster-2/nodePools/default-pool",
128+
"status": "RUNNING",
129+
"version": "1.14.8-gke.12"
130+
}
131+
],
132+
"releaseChannel": {
133+
"channel": "REGULAR"
134+
},
135+
"selfLink": "https://container.googleapis.com/v1beta1/projects/jetstack-wil/zones/us-central1-a/clusters/standard-cluster-2",
136+
"servicesIpv4Cidr": "10.70.0.0/20",
137+
"shieldedNodes": {},
138+
"status": "RUNNING",
139+
"subnetwork": "default",
140+
"tierSettings": {
141+
"tier": "STANDARD"
142+
},
143+
"zone": "us-central1-a"
144+
}

‎examples/stubbed.preflight.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
cluster-name: my-stubbed-cluster
2+
3+
data-gatherers:
4+
aks:
5+
# Stubbed data for AKS.
6+
data-path: ./examples/data/aks.json
7+
gke:
8+
# Stubbed data for GKE.
9+
data-path: ./examples/data/gke.json
10+
11+
package-sources:
12+
- type: local
13+
dir: preflight-packages/
14+
15+
enabled-packages:
16+
- "examples.jetstack.io/aks_basic"
17+
- "examples.jetstack.io/gke_basic"
18+
19+
outputs:
20+
# Write the report to the CLI.
21+
- type: cli

‎pkg/datagatherer/local/local.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package local
2+
3+
import "io/ioutil"
4+
5+
type LocalDataGatherer struct {
6+
dataPath string
7+
}
8+
9+
// NewLocalDataGatherer returns a LocalDatagatherer with the dataPath provided.
10+
func NewLocalDataGatherer(dataPath string) *LocalDataGatherer {
11+
return &LocalDataGatherer{
12+
dataPath: dataPath,
13+
}
14+
}
15+
16+
// Fetch loads and returns the data from the LocalDatagatherer's dataPath
17+
func (g *LocalDataGatherer) Fetch() (interface{}, error) {
18+
dataBytes, err := ioutil.ReadFile(g.dataPath)
19+
if err != nil {
20+
return nil, err
21+
}
22+
return dataBytes, nil
23+
}

0 commit comments

Comments
 (0)