Skip to content

Commit cb7733c

Browse files
j-fuentesj-fuentes
authored
Make agent support new endpoint for gathering data (#163)
Signed-off-by: Jose Fuentes <[email protected]>
1 parent 2482482 commit cb7733c

File tree

8 files changed

+235
-91
lines changed

8 files changed

+235
-91
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,12 @@ GO_INSTALL:=go install -ldflags '$(LDFLAGS)'
2626

2727
export GO111MODULE=on
2828

29-
.PHONY: build
30-
3129
clean:
3230
cd $(ROOT_DIR) && rm -rf ./builds ./bundles
3331

3432
# Golang cli
3533

34+
.PHONY: build
3635
build:
3736
cd $(ROOT_DIR) && $(GO_BUILD) -o builds/preflight .
3837

@@ -48,6 +47,7 @@ vet:
4847
lint: vet
4948
cd $(ROOT_DIR) && golint
5049

50+
.PHONY: ./builds/preflight-$(GOOS)-$(GOARCH)
5151
./builds/preflight-$(GOOS)-$(GOARCH):
5252
GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO_BUILD) -o ./builds/preflight-$(GOOS)-$(GOARCH) .
5353

‎api/agent.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package api
2+
3+
// AgentMetadata is metadata about the agent.
4+
type AgentMetadata struct {
5+
Version string `json:"version"`
6+
}

‎api/datareading.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,16 @@
11
package api
22

3+
// DataReadingsPost is the payload in the upload request.
4+
type DataReadingsPost struct {
5+
AgentMetadata *AgentMetadata `json:"agent_metadata"`
6+
DataReadings []*DataReading `json:"data_readings"`
7+
}
8+
39
// DataReading is the output of a DataGatherer.
410
type DataReading struct {
11+
// ClusterID is optional as it can be infered from the agent
12+
// token when using basic authentication.
13+
ClusterID string `json:"cluster_id,omitempty"`
514
DataGatherer string `json:"data-gatherer"`
615
Timestamp Time `json:"timestamp"`
716
Data interface{} `json:"data"`

‎pkg/agent/config.go‎

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package agent
22

33
import (
44
"fmt"
5-
"strings"
5+
"net/url"
66

77
"github.com/hashicorp/go-multierror"
88
"github.com/jetstack/preflight/pkg/datagatherer"
@@ -17,13 +17,22 @@ import (
1717

1818
// Config wraps the options for a run of the agent.
1919
type Config struct {
20-
Schedule string `yaml:"schedule"`
21-
Token string `yaml:"token"`
22-
Endpoint Endpoint `yaml:"endpoint"`
20+
Schedule string `yaml:"schedule"`
21+
// Token is the agent token if using basic authentication.
22+
// If not provided it will assume OAuth2 authentication.
23+
Token string `yaml:"token"`
24+
// Deprecated: Endpoint is being replaced with Server.
25+
Endpoint Endpoint `yaml:"endpoint"`
26+
// Server is the base url for the Preflight server.
27+
// It defaults to https://preflight.jetstack.io.
28+
Server string `yaml:"server"`
29+
// OrganizationID within Preflight that will receive the data.
30+
OrganizationID string `yaml:"organization_id"`
31+
// ClusterID is the cluster that the agent is scanning.
32+
ClusterID string `yaml:"cluster_id"`
2333
DataGatherers []dataGatherer `yaml:"data-gatherers"`
2434
}
2535

26-
// Endpoint is the configuration of the server where to post the data.
2736
type Endpoint struct {
2837
Protocol string `yaml:"protocol"`
2938
Host string `yaml:"host"`
@@ -114,19 +123,18 @@ func (c *Config) validate() error {
114123
var result *multierror.Error
115124

116125
if c.Token == "" {
117-
result = multierror.Append(result, fmt.Errorf("token is required"))
118-
}
119-
120-
if c.Schedule == "" {
121-
result = multierror.Append(result, fmt.Errorf("schedule is required"))
122-
}
123-
124-
if c.Endpoint.Host == "" {
125-
result = multierror.Append(result, fmt.Errorf("endpoint host is required"))
126+
if c.OrganizationID == "" {
127+
result = multierror.Append(result, fmt.Errorf("organization_id is required"))
128+
}
129+
if c.ClusterID == "" {
130+
result = multierror.Append(result, fmt.Errorf("cluster_id is required"))
131+
}
126132
}
127133

128-
if c.Endpoint.Path == "" {
129-
result = multierror.Append(result, fmt.Errorf("endpoint path is required"))
134+
if c.Server != "" {
135+
if url, err := url.Parse(c.Server); err != nil || url.Hostname() == "" {
136+
result = multierror.Append(result, fmt.Errorf("server is not a valid URL"))
137+
}
130138
}
131139

132140
for i, v := range c.DataGatherers {
@@ -150,17 +158,17 @@ func ParseConfig(data []byte) (Config, error) {
150158
return config, err
151159
}
152160

153-
if config.Endpoint.Protocol == "" {
161+
if config.Server == "" && config.Endpoint.Host == "" && config.Endpoint.Path == "" {
162+
config.Server = "https://preflight.jetstack.io"
163+
}
164+
165+
if config.Endpoint.Protocol == "" && config.Server == "" {
154166
config.Endpoint.Protocol = "http"
155167
}
156168

157169
if err = config.validate(); err != nil {
158170
return config, err
159171
}
160172

161-
if !strings.HasPrefix(config.Endpoint.Path, "/") {
162-
config.Endpoint.Path = fmt.Sprintf("/%s", config.Endpoint.Path)
163-
}
164-
165173
return config, nil
166174
}

‎pkg/agent/config_test.go‎

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,41 @@ import (
1111

1212
func TestValidConfigLoad(t *testing.T) {
1313
configFileContents := `
14+
token: "12345"
15+
server: "http://localhost:8080"
16+
data-gatherers:
17+
- name: d1
18+
kind: dummy
19+
config:
20+
param-1: "bar"
21+
`
22+
23+
loadedConfig, err := ParseConfig([]byte(configFileContents))
24+
if err != nil {
25+
t.Errorf("unexpected error: %v", err)
26+
}
27+
28+
expected := Config{
29+
Token: "12345",
30+
Server: "http://localhost:8080",
31+
DataGatherers: []dataGatherer{
32+
dataGatherer{
33+
Name: "d1",
34+
Kind: "dummy",
35+
Config: &dummyConfig{
36+
Param1: "bar",
37+
},
38+
},
39+
},
40+
}
41+
42+
if diff, equal := messagediff.PrettyDiff(expected, loadedConfig); !equal {
43+
t.Errorf("Diff %s", diff)
44+
}
45+
}
46+
47+
func TestValidConfigWithEndpointLoad(t *testing.T) {
48+
configFileContents := `
1449
endpoint:
1550
host: example.com
1651
path: api/v1/data
@@ -32,7 +67,7 @@ func TestValidConfigLoad(t *testing.T) {
3267
Endpoint: Endpoint{
3368
Protocol: "http",
3469
Host: "example.com",
35-
Path: "/api/v1/data",
70+
Path: "api/v1/data",
3671
},
3772
Schedule: "* * * * *",
3873
Token: "12345",
@@ -72,11 +107,9 @@ func TestMissingConfigError(t *testing.T) {
72107
}
73108

74109
expectedErrorLines := []string{
75-
"4 errors occurred:",
76-
"\t* token is required",
77-
"\t* schedule is required",
78-
"\t* endpoint host is required",
79-
"\t* endpoint path is required",
110+
"2 errors occurred:",
111+
"\t* organization_id is required",
112+
"\t* cluster_id is required",
80113
"\n",
81114
}
82115

@@ -118,6 +151,34 @@ func TestPartialMissingConfigError(t *testing.T) {
118151
}
119152
}
120153

154+
func TestInvalidServerError(t *testing.T) {
155+
_, parseError := ParseConfig([]byte(`
156+
server: "something not a URL"
157+
organization_id: "my_org"
158+
cluster_id: "my_cluster"
159+
data-gatherers:
160+
- kind: dummy
161+
name: dummy`))
162+
163+
if parseError == nil {
164+
t.Fatalf("expected error, got nil")
165+
}
166+
167+
expectedErrorLines := []string{
168+
"1 error occurred:",
169+
"\t* server is not a valid URL",
170+
"\n",
171+
}
172+
173+
expectedError := strings.Join(expectedErrorLines, "\n")
174+
175+
gotError := parseError.Error()
176+
177+
if gotError != expectedError {
178+
t.Errorf("\ngot=\n%v\nwant=\n%s\ndiff=\n%s", gotError, expectedError, diff.Diff(gotError, expectedError))
179+
}
180+
}
181+
121182
func TestInvalidDataGathered(t *testing.T) {
122183
_, parseError := ParseConfig([]byte(`
123184
endpoint:

‎pkg/agent/run.go‎

Lines changed: 44 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package agent
22

33
import (
4+
"bytes"
45
"context"
6+
"encoding/json"
57
"fmt"
68
"io/ioutil"
79
"log"
@@ -12,6 +14,7 @@ import (
1214
"github.com/jetstack/preflight/api"
1315
"github.com/jetstack/preflight/pkg/client"
1416
"github.com/jetstack/preflight/pkg/datagatherer"
17+
"github.com/jetstack/preflight/pkg/version"
1518
"github.com/spf13/cobra"
1619
)
1720

@@ -55,9 +58,14 @@ func Run(cmd *cobra.Command, args []string) {
5558
config.Token = "(redacted)"
5659
}
5760

58-
serverURL, err := url.Parse(fmt.Sprintf("%s://%s%s", config.Endpoint.Protocol, config.Endpoint.Host, config.Endpoint.Path))
59-
if err != nil {
60-
log.Fatalf("Failed to build URL: %s", err)
61+
baseURL := config.Server
62+
if baseURL == "" {
63+
log.Printf("Using deprecated Endpoint configuration. User Server instead.")
64+
baseURL = fmt.Sprintf("%s://%s", config.Endpoint.Protocol, config.Endpoint.Host)
65+
_, err = url.Parse(baseURL)
66+
if err != nil {
67+
log.Fatalf("Failed to build URL: %s", err)
68+
}
6169
}
6270

6371
dump, err := config.Dump()
@@ -83,10 +91,13 @@ func Run(cmd *cobra.Command, args []string) {
8391
}
8492
}
8593

94+
agentMetadata := &api.AgentMetadata{
95+
Version: version.PreflightVersion,
96+
}
8697
var preflightClient *client.PreflightClient
8798
if credentials != nil {
8899
log.Printf("A credentials file was specified. Using OAuth2 authentication...")
89-
preflightClient, err = client.New(credentials.UserID, credentials.UserSecret, serverURL.String())
100+
preflightClient, err = client.New(agentMetadata, credentials.UserID, credentials.UserSecret, baseURL)
90101
if err != nil {
91102
log.Fatalf("Error creating preflight client: %+v", err)
92103
}
@@ -95,7 +106,7 @@ func Run(cmd *cobra.Command, args []string) {
95106
log.Fatalf("Missing authorization token. Cannot continue.")
96107
}
97108

98-
preflightClient, err = client.NewWithBasicAuth(AuthToken, serverURL.String())
109+
preflightClient, err = client.NewWithBasicAuth(agentMetadata, AuthToken, baseURL)
99110
if err != nil {
100111
log.Fatalf("Error creating preflight client: %+v", err)
101112
}
@@ -130,6 +141,7 @@ func Run(cmd *cobra.Command, args []string) {
130141
log.Printf("Gathered data for %q:\n", k)
131142

132143
readings = append(readings, &api.DataReading{
144+
ClusterID: config.ClusterID,
133145
DataGatherer: k,
134146
Timestamp: api.Time{Time: now},
135147
Data: i,
@@ -138,14 +150,35 @@ func Run(cmd *cobra.Command, args []string) {
138150

139151
for {
140152
log.Println("Running Agent...")
141-
log.Println("Posting data to ", serverURL)
142-
err = preflightClient.PostDataReadings(readings)
143-
// TODO: handle errors gracefully: e.g. handle retries when it is possible
144-
if err != nil {
145-
log.Fatalf("Post to server failed: %+v", err)
153+
log.Println("Posting data to ", baseURL)
154+
if config.OrganizationID == "" {
155+
data, err := json.Marshal(readings)
156+
if err != nil {
157+
log.Fatalf("Cannot marshal readings: %+v", err)
158+
}
159+
path := config.Endpoint.Path
160+
if path == "" {
161+
path = "/api/v1/datareadings"
162+
}
163+
res, err := preflightClient.Post(path, bytes.NewBuffer(data))
164+
if code := res.StatusCode; code < 200 || code >= 300 {
165+
errorContent := ""
166+
body, _ := ioutil.ReadAll(res.Body)
167+
if err == nil {
168+
errorContent = string(body)
169+
}
170+
defer res.Body.Close()
171+
172+
log.Fatalf("Received response with status code %d. Body: %s", code, errorContent)
173+
}
146174
} else {
147-
log.Println("Data sent successfully.")
175+
err = preflightClient.PostDataReadings(config.OrganizationID, readings)
176+
// TODO: handle errors gracefully: e.g. handle retries when it is possible
177+
if err != nil {
178+
log.Fatalf("Post to server failed: %+v", err)
179+
}
148180
}
181+
log.Println("Data sent successfully.")
149182
time.Sleep(time.Duration(Period) * time.Second)
150183
}
151184
}

0 commit comments

Comments
 (0)