Skip to content

Commit b7cc812

Browse files
julienduchesnejulienduchesne
authored
Datasource: Arbitrary JSON Data (#607)
* Datasource: Arbitrary JSON Data Deprecates the current `json_data` and `secure_json_data` in favor of an arbitrary map format: `json_data_map` and `secure_json_data_map` Closes #63 #605 #582 #578 #568 #528 #516 #520 #493 #343 #236 #197 #188 #81 * Generate docs * Ready to merge! Added tests Updated client lib Added example
1 parent dee3b8e commit b7cc812

File tree

6 files changed

+153
-31
lines changed

6 files changed

+153
-31
lines changed

‎docs/resources/data_source.md‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,22 @@ source selected (via the 'type' argument).
1919
## Example Usage
2020

2121
```terraform
22+
resource "grafana_data_source" "arbitrary-data" {
23+
type = "stackdriver"
24+
name = "sd-arbitrary-data"
25+
26+
json_data_map = {
27+
"tokenUri" = "https://oauth2.googleapis.com/token"
28+
"authenticationType" = "jwt"
29+
"defaultProject" = "default-project"
30+
"clientEmail" = "[email protected]"
31+
}
32+
33+
secure_json_data_map = {
34+
"privateKey" = "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n"
35+
}
36+
}
37+
2238
resource "grafana_data_source" "influxdb" {
2339
type = "influxdb"
2440
name = "myapp-metrics"
@@ -90,9 +106,11 @@ resource "grafana_data_source" "stackdriver" {
90106
- `database_name` (String) (Required by some data source types) The name of the database to use on the selected data source server. Defaults to ``.
91107
- `http_headers` (Map of String, Sensitive) Custom HTTP headers
92108
- `is_default` (Boolean) Whether to set the data source as default. This should only be `true` to a single data source. Defaults to `false`.
93-
- `json_data` (Block List) (Required by some data source types) (see [below for nested schema](#nestedblock--json_data))
109+
- `json_data` (Block List, Deprecated) (Required by some data source types). Deprecated: Use json_data_map instead. It supports arbitrary JSON data, and therefore all attributes. (see [below for nested schema](#nestedblock--json_data))
110+
- `json_data_map` (Map of String) Replaces the json_data attribute, this attribute can be used to pass configuration options to the data source. To figure out what options a datasource has available, see its docs or inspect the network data when saving it from the Grafana UI.
94111
- `password` (String, Sensitive, Deprecated) (Required by some data source types) The password to use to authenticate to the data source. Deprecated: Use secure_json_data.password instead. This attribute is removed in Grafana 9.0+. Defaults to ``.
95-
- `secure_json_data` (Block List) (see [below for nested schema](#nestedblock--secure_json_data))
112+
- `secure_json_data` (Block List, Deprecated) Deprecated: Use secure_json_data instead. It supports arbitrary JSON data, and therefore all attributes. (see [below for nested schema](#nestedblock--secure_json_data))
113+
- `secure_json_data_map` (Map of String, Sensitive) Replaces the secure_json_data attribute, this attribute can be used to pass secure configuration options to the data source. To figure out what options a datasource has available, see its docs or inspect the network data when saving it from the Grafana UI.
96114
- `uid` (String) Unique identifier. If unset, this will be automatically generated.
97115
- `url` (String) The URL for the data source. The type of URL required varies depending on the chosen data source type.
98116
- `username` (String) (Required by some data source types) The username to use to authenticate to the data source. Defaults to ``.

‎examples/resources/grafana_data_source/resource.tf‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,19 @@
1+
resource "grafana_data_source" "arbitrary-data" {
2+
type = "stackdriver"
3+
name = "sd-arbitrary-data"
4+
5+
json_data_map = {
6+
"tokenUri" = "https://oauth2.googleapis.com/token"
7+
"authenticationType" = "jwt"
8+
"defaultProject" = "default-project"
9+
"clientEmail" = "[email protected]"
10+
}
11+
12+
secure_json_data_map = {
13+
"privateKey" = "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n"
14+
}
15+
}
16+
117
resource "grafana_data_source" "influxdb" {
218
type = "influxdb"
319
name = "myapp-metrics"

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.18
55
require (
66
github.com/Masterminds/semver/v3 v3.1.1
77
github.com/grafana/amixr-api-go-client v0.0.5
8-
github.com/grafana/grafana-api-golang-client v0.10.0
8+
github.com/grafana/grafana-api-golang-client v0.11.0
99
github.com/grafana/machine-learning-go-client v0.1.1
1010
github.com/grafana/synthetic-monitoring-agent v0.9.3
1111
github.com/grafana/synthetic-monitoring-api-go-client v0.6.0

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
114114
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
115115
github.com/grafana/amixr-api-go-client v0.0.5 h1:jqmljnd5FozuOsCNuyhZVpooxmj0BW9MmeLA7PaLK6U=
116116
github.com/grafana/amixr-api-go-client v0.0.5/go.mod h1:N6x26XUrM5zGtK5zL5vNJnAn2JFMxLFPPLTw/6pDkFE=
117-
github.com/grafana/grafana-api-golang-client v0.10.0 h1:R/dgpQOFynP/3Ivoh5SvEPKh4u9YoWuvoOKNe80wkH8=
118-
github.com/grafana/grafana-api-golang-client v0.10.0/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E=
117+
github.com/grafana/grafana-api-golang-client v0.11.0 h1:4KuqBJMTiuDcWMGa2MIGQOC1E0uzFJ7NcGHCBwN7ODs=
118+
github.com/grafana/grafana-api-golang-client v0.11.0/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E=
119119
github.com/grafana/machine-learning-go-client v0.1.1 h1:Gw6cX8xAd6IVF2LApkXOIdBK8Gzz07B3jQPukecw7fc=
120120
github.com/grafana/machine-learning-go-client v0.1.1/go.mod h1:QFfZz8NkqVF8++skjkKQXJEZfpCYd8S0yTWJUpsLLTA=
121121
github.com/grafana/synthetic-monitoring-agent v0.9.3 h1:1GAjyUMWPYndRF6tux8NWgk97bRFx9RVYMkEMerPNKU=

‎grafana/resource_data_source.go‎

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -105,10 +105,12 @@ source selected (via the 'type' argument).
105105
Computed: true,
106106
Description: "Unique identifier. If unset, this will be automatically generated.",
107107
},
108+
108109
"json_data": {
109110
Type: schema.TypeList,
110111
Optional: true,
111-
Description: "(Required by some data source types)",
112+
Description: "(Required by some data source types). Deprecated: Use json_data_map instead. It supports arbitrary JSON data, and therefore all attributes.",
113+
Deprecated: "Use json_data_map instead. It supports arbitrary JSON data, and therefore all attributes.",
112114
Elem: &schema.Resource{
113115
Schema: map[string]*schema.Schema{
114116
"assume_role_arn": {
@@ -454,7 +456,8 @@ source selected (via the 'type' argument).
454456
Type: schema.TypeList,
455457
Optional: true,
456458
Sensitive: true,
457-
Description: "",
459+
Description: "Deprecated: Use secure_json_data instead. It supports arbitrary JSON data, and therefore all attributes.",
460+
Deprecated: "Use secure_json_data instead. It supports arbitrary JSON data, and therefore all attributes.",
458461
Elem: &schema.Resource{
459462
Schema: map[string]*schema.Schema{
460463
"access_key": {
@@ -554,6 +557,19 @@ source selected (via the 'type' argument).
554557
Default: "",
555558
Description: "(Required by some data source types) The username to use to authenticate to the data source.",
556559
},
560+
"json_data_map": {
561+
Type: schema.TypeMap,
562+
Optional: true,
563+
ConflictsWith: []string{"json_data"},
564+
Description: "Replaces the json_data attribute, this attribute can be used to pass configuration options to the data source. To figure out what options a datasource has available, see its docs or inspect the network data when saving it from the Grafana UI.",
565+
},
566+
"secure_json_data_map": {
567+
Type: schema.TypeMap,
568+
Optional: true,
569+
Sensitive: true,
570+
ConflictsWith: []string{"secure_json_data"},
571+
Description: "Replaces the secure_json_data attribute, this attribute can be used to pass secure configuration options to the data source. To figure out what options a datasource has available, see its docs or inspect the network data when saving it from the Grafana UI.",
572+
},
557573
},
558574
}
559575
}
@@ -653,13 +669,27 @@ func makeDataSource(d *schema.ResourceData) (*gapi.DataSource, error) {
653669
var err error
654670
if idStr != "" {
655671
id, err = strconv.ParseInt(idStr, 10, 64)
672+
if err != nil {
673+
return nil, err
674+
}
656675
}
657676

658677
httpHeaders := make(map[string]string)
659678
for key, value := range d.Get("http_headers").(map[string]interface{}) {
660679
httpHeaders[key] = fmt.Sprintf("%v", value)
661680
}
662681

682+
jd, err := makeJSONData(d)
683+
if err != nil {
684+
return nil, err
685+
}
686+
sd, err := makeSecureJSONData(d)
687+
if err != nil {
688+
return nil, err
689+
}
690+
691+
jd, sd = gapi.JSONDataWithHeaders(jd, sd, httpHeaders)
692+
663693
return &gapi.DataSource{
664694
ID: id,
665695
Name: d.Get("name").(string),
@@ -674,13 +704,16 @@ func makeDataSource(d *schema.ResourceData) (*gapi.DataSource, error) {
674704
BasicAuthUser: d.Get("basic_auth_username").(string),
675705
BasicAuthPassword: d.Get("basic_auth_password").(string),
676706
UID: d.Get("uid").(string),
677-
HTTPHeaders: httpHeaders,
678-
JSONData: makeJSONData(d),
679-
SecureJSONData: makeSecureJSONData(d),
707+
JSONData: jd,
708+
SecureJSONData: sd,
680709
}, err
681710
}
682711

683-
func makeJSONData(d *schema.ResourceData) gapi.JSONData {
712+
func makeJSONData(d *schema.ResourceData) (map[string]interface{}, error) {
713+
if v, ok := d.GetOk("json_data_map"); ok {
714+
return v.(map[string]interface{}), nil
715+
}
716+
684717
var derivedFields []gapi.LokiDerivedField
685718
for _, field := range d.Get("json_data.0.derived_field").([]interface{}) {
686719
derivedField := field.(map[string]interface{})
@@ -751,10 +784,14 @@ func makeJSONData(d *schema.ResourceData) gapi.JSONData {
751784
Version: d.Get("json_data.0.version").(string),
752785
Workgroup: d.Get("json_data.0.workgroup").(string),
753786
XpackEnabled: d.Get("json_data.0.xpack_enabled").(bool),
754-
}
787+
}.Map()
755788
}
756789

757-
func makeSecureJSONData(d *schema.ResourceData) gapi.SecureJSONData {
790+
func makeSecureJSONData(d *schema.ResourceData) (map[string]interface{}, error) {
791+
if v, ok := d.GetOk("secure_json_data_map"); ok {
792+
return v.(map[string]interface{}), nil
793+
}
794+
758795
return gapi.SecureJSONData{
759796
AccessKey: d.Get("secure_json_data.0.access_key").(string),
760797
AccessToken: d.Get("secure_json_data.0.access_token").(string),
@@ -769,7 +806,7 @@ func makeSecureJSONData(d *schema.ResourceData) gapi.SecureJSONData {
769806
TLSCACert: d.Get("secure_json_data.0.tls_ca_cert").(string),
770807
TLSClientCert: d.Get("secure_json_data.0.tls_client_cert").(string),
771808
TLSClientKey: d.Get("secure_json_data.0.tls_client_key").(string),
772-
}
809+
}.Map()
773810
}
774811

775812
// TSDB Version and Resolution used to be strings, but now are integers.

‎grafana/resource_data_source_test.go‎

Lines changed: 68 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,16 @@ func TestAccDataSource_basic(t *testing.T) {
6969
additionalChecks: []resource.TestCheckFunc{
7070
func(s *terraform.State) error {
7171
// Check datasource IDs
72-
if dataSource.JSONData.DerivedFields[0].DatasourceUID != "" {
72+
derivedFields := dataSource.JSONData["derivedFields"].([]interface{})
73+
if len(derivedFields) != 2 {
74+
return fmt.Errorf("expected 2 derived fields, got %d", len(derivedFields))
75+
}
76+
firstDerivedField := derivedFields[0].(map[string]interface{})
77+
if _, ok := firstDerivedField["datasourceUid"]; ok {
7378
return fmt.Errorf("expected empty datasource_uid")
7479
}
75-
if !uidRegexp.MatchString(dataSource.JSONData.DerivedFields[1].DatasourceUID) {
80+
secondDerivedField := derivedFields[1].(map[string]interface{})
81+
if !uidRegexp.MatchString(secondDerivedField["datasourceUid"].(string)) {
7682
return fmt.Errorf("expected valid datasource_uid")
7783
}
7884
return nil
@@ -189,12 +195,58 @@ func TestAccDataSource_basic(t *testing.T) {
189195
if dataSource.Name != "influx" {
190196
return fmt.Errorf("bad name: %s", dataSource.Name)
191197
}
192-
if len(dataSource.HTTPHeaders) != 1 {
193-
return fmt.Errorf("expected 1 http header, got %d", len(dataSource.HTTPHeaders))
198+
if v, ok := dataSource.JSONData["httpHeaderName1"]; !ok && v != "Authorization" {
199+
return fmt.Errorf("http header Authorization not found")
194200
}
195-
196-
if _, ok := dataSource.HTTPHeaders["Authorization"]; !ok {
197-
return fmt.Errorf("http header header1 not found")
201+
return nil
202+
},
203+
},
204+
},
205+
{
206+
resource: "grafana_data_source.influx-arbitrary",
207+
config: `
208+
resource "grafana_data_source" "influx-arbitrary" {
209+
type = "influxdb"
210+
name = "influx"
211+
url = "http://acc-test.invalid/"
212+
http_headers = {
213+
Authorization = "Token sdkfjsdjflkdsjflksjdklfjslkdfjdksljfldksjsflkj"
214+
}
215+
json_data_map = {
216+
defaultBucket = "telegraf"
217+
organization = "organization"
218+
tlsAuth = false
219+
tlsAuthWithCACert = false
220+
version = "Flux"
221+
}
222+
}
223+
`,
224+
attrChecks: map[string]string{
225+
"type": "influxdb",
226+
"name": "influx",
227+
"url": "http://acc-test.invalid/",
228+
"json_data_map.defaultBucket": "telegraf",
229+
"json_data_map.organization": "organization",
230+
"json_data_map.tlsAuth": "false",
231+
"json_data_map.tlsAuthWithCACert": "false",
232+
"json_data_map.version": "Flux",
233+
"http_headers.Authorization": "Token sdkfjsdjflkdsjflksjdklfjslkdfjdksljfldksjsflkj",
234+
},
235+
additionalChecks: []resource.TestCheckFunc{
236+
func(s *terraform.State) error {
237+
if dataSource.Name != "influx" {
238+
return fmt.Errorf("bad name: %s", dataSource.Name)
239+
}
240+
expected := map[string]interface{}{
241+
"defaultBucket": "telegraf",
242+
"organization": "organization",
243+
"tlsAuth": "false",
244+
"tlsAuthWithCACert": "false",
245+
"version": "Flux",
246+
"httpHeaderName1": "Authorization",
247+
}
248+
if !reflect.DeepEqual(dataSource.JSONData, expected) {
249+
return fmt.Errorf("bad json_data: %#v. Expected: %+v", dataSource.JSONData, expected)
198250
}
199251
return nil
200252
},
@@ -236,7 +288,7 @@ func TestAccDataSource_basic(t *testing.T) {
236288
if dataSource.Name != "elasticsearch" {
237289
return fmt.Errorf("bad name: %s", dataSource.Name)
238290
}
239-
if dataSource.JSONData.XpackEnabled != true {
291+
if dataSource.JSONData["xpack"].(bool) != true {
240292
return errors.New("xpack_enabled should be true")
241293
}
242294
return nil
@@ -300,8 +352,9 @@ func TestAccDataSource_basic(t *testing.T) {
300352
if dataSource.Name != "cloudwatch" {
301353
return fmt.Errorf("bad name: %s", dataSource.Name)
302354
}
303-
if dataSource.JSONData.TracingDatasourceUID != "my-datasource-uid" {
304-
return fmt.Errorf("bad tracing_datasource_uid: %s", dataSource.JSONData.TracingDatasourceUID)
355+
datasourceUID := dataSource.JSONData["tracingDatasourceUid"].(string)
356+
if datasourceUID != "my-datasource-uid" {
357+
return fmt.Errorf("bad tracing_datasource_uid: %s", datasourceUID)
305358
}
306359
return nil
307360
},
@@ -409,13 +462,10 @@ func TestAccDataSource_basic(t *testing.T) {
409462
if dataSource.Name != "prometheus" {
410463
return fmt.Errorf("bad name: %s", dataSource.Name)
411464
}
412-
if len(dataSource.HTTPHeaders) != 1 {
413-
return fmt.Errorf("expected 1 http header, got %d", len(dataSource.HTTPHeaders))
414-
}
415-
if _, ok := dataSource.HTTPHeaders["header1"]; !ok {
465+
if v, ok := dataSource.JSONData["httpHeaderName1"]; !ok && v != "header1" {
416466
return fmt.Errorf("http header header1 not found")
417467
}
418-
if dataSource.JSONData.ManageAlerts != true {
468+
if dataSource.JSONData["manageAlerts"].(bool) != true {
419469
return errors.New("expected manage_alerts to be true")
420470
}
421471
return nil
@@ -494,8 +544,9 @@ func TestAccDataSource_basic(t *testing.T) {
494544
},
495545
additionalChecks: []resource.TestCheckFunc{
496546
func(s *terraform.State) error {
497-
if dataSource.JSONData.GitHubURL != "https://test-github.com" {
498-
return fmt.Errorf("bad github_url: %s. Expected: %s", dataSource.JSONData.GitHubURL, "https://test-github.com")
547+
githubURL := dataSource.JSONData["githubUrl"].(string)
548+
if githubURL != "https://test-github.com" {
549+
return fmt.Errorf("bad github_url: %s. Expected: %s", githubURL, "https://test-github.com")
499550
}
500551
return nil
501552
},

0 commit comments

Comments
 (0)