Skip to content

Commit 77fdccc

Browse files
authored
Resource to manage machine learning jobs (#330)
* Resource to manage machine learning jobs * Add checks for server ML resources Add tests to make sure the server has a Job, and make sure that after deletion the server no longer has a Job.
1 parent dddf535 commit 77fdccc

File tree

8 files changed

+364
-1
lines changed

8 files changed

+364
-1
lines changed
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
---
2+
# generated by https://github.com/hashicorp/terraform-plugin-docs
3+
page_title: "grafana_machine_learning_job Resource - terraform-provider-grafana"
4+
subcategory: ""
5+
description: |-
6+
A job defines the queries and model parameters for a machine learning task.
7+
---
8+
9+
# grafana_machine_learning_job (Resource)
10+
11+
A job defines the queries and model parameters for a machine learning task.
12+
13+
14+
15+
<!-- schema generated by tfplugindocs -->
16+
## Schema
17+
18+
### Required
19+
20+
- **datasource_id** (Number) The id of the datasource to query.
21+
- **datasource_type** (String) The type of datasource being queried. Currently allowed values are prometheus, graphite, loki, postgres, and datadog.
22+
- **metric** (String) The metric used to query the job results.
23+
- **name** (String) The name of the job.
24+
- **query_params** (Map of String) An object representing the query params to query Grafana with.
25+
26+
### Optional
27+
28+
- **description** (String) A description of the job.
29+
- **hyper_params** (Map of String) The hyperparameters used to fine tune the algorithm. See https://grafana.com/docs/grafana-cloud/machine-learning/models/ for the full list of available hyperparameters. Defaults to `map[]`.
30+
- **interval** (Number) The data interval in seconds to train the data on. Defaults to `300`.
31+
- **training_window** (Number) The data interval in seconds to train the data on. Defaults to `7776000`.
32+
33+
### Read-Only
34+
35+
- **id** (String) The ID of the job.
36+
37+
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
resource "grafana_machine_learning_job" "test_job" {
2+
name = "Test Job"
3+
metric = "tf_test_job"
4+
datasource_type = "prometheus"
5+
datasource_id = 10
6+
query_params = {
7+
expr = "grafanacloud_grafana_instance_active_user_count"
8+
}
9+
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
resource "grafana_machine_learning_job" "test_job" {
2+
name = "Test Job"
3+
metric = "tf_test_job"
4+
datasource_type = "prometheus"
5+
datasource_id = 10
6+
query_params = {
7+
expr = "grafanacloud_grafana_instance_active_user_count"
8+
}
9+
hyper_params = {
10+
daily_seasonality = 15
11+
weekly_seasonality = 10
12+
}
13+
}

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.16
44

55
require (
66
github.com/grafana/grafana-api-golang-client v0.2.1
7+
github.com/grafana/machine-learning-go-client v0.1.1
78
github.com/grafana/synthetic-monitoring-agent v0.4.1
89
github.com/grafana/synthetic-monitoring-api-go-client v0.3.0
910
github.com/hashicorp/go-cleanhttp v0.5.2

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -454,6 +454,8 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
454454
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
455455
github.com/grafana/grafana-api-golang-client v0.2.1 h1:5OizQpq+WP+A6t+wHoYCOrmbytWQMgnWwPfzXqSBGrI=
456456
github.com/grafana/grafana-api-golang-client v0.2.1/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E=
457+
github.com/grafana/machine-learning-go-client v0.1.1 h1:Gw6cX8xAd6IVF2LApkXOIdBK8Gzz07B3jQPukecw7fc=
458+
github.com/grafana/machine-learning-go-client v0.1.1/go.mod h1:QFfZz8NkqVF8++skjkKQXJEZfpCYd8S0yTWJUpsLLTA=
457459
github.com/grafana/synthetic-monitoring-agent v0.3.0/go.mod h1:P8WTBnw3SIZW5Nm5obOlSKvD887IxAfbrJkSnoZyIlA=
458460
github.com/grafana/synthetic-monitoring-agent v0.4.1 h1:RiVOHi059tYIDixXPxmFQ2vCWCY4Vksslpfm5gNkgnQ=
459461
github.com/grafana/synthetic-monitoring-agent v0.4.1/go.mod h1:hcx3Pe76ixYsbjtM0eFpyE1bzPxiFgJIyXtdqgPeIkk=

grafana/provider.go

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
1717

1818
gapi "github.com/grafana/grafana-api-golang-client"
19+
"github.com/grafana/machine-learning-go-client/mlapi"
1920
smapi "github.com/grafana/synthetic-monitoring-api-go-client"
2021
)
2122

@@ -125,6 +126,9 @@ func Provider(version string) func() *schema.Provider {
125126
// Synthetic Monitoring
126127
"grafana_synthetic_monitoring_check": resourceSyntheticMonitoringCheck(),
127128
"grafana_synthetic_monitoring_probe": resourceSyntheticMonitoringProbe(),
129+
130+
// Machine Learning
131+
"grafana_machine_learning_job": resourceMachineLearningJob(),
128132
},
129133

130134
DataSourcesMap: map[string]*schema.Resource{
@@ -147,6 +151,8 @@ func Provider(version string) func() *schema.Provider {
147151
type client struct {
148152
gapi *gapi.Client
149153
smapi *smapi.Client
154+
mlapi *mlapi.Client
155+
url string
150156
}
151157

152158
func configure(version string, p *schema.Provider) func(context.Context, *schema.ResourceData) (interface{}, diag.Diagnostics) {
@@ -186,6 +192,7 @@ func configure(version string, p *schema.Provider) func(context.Context, *schema
186192
transport.TLSClientConfig.InsecureSkipVerify = true
187193
}
188194

195+
c.url = d.Get("url").(string)
189196
cli.Transport = logging.NewTransport("Grafana", transport)
190197
cfg := gapi.Config{
191198
Client: cli,
@@ -197,13 +204,30 @@ func configure(version string, p *schema.Provider) func(context.Context, *schema
197204
} else {
198205
cfg.APIKey = auth[0]
199206
}
200-
gclient, err := gapi.New(d.Get("url").(string), cfg)
207+
gclient, err := gapi.New(c.url, cfg)
201208
if err != nil {
202209
return nil, diag.FromErr(err)
203210
}
204211

205212
c.gapi = gclient
206213

214+
mlcfg := mlapi.Config{
215+
BasicAuth: cfg.BasicAuth,
216+
BearerToken: cfg.APIKey,
217+
Client: cli,
218+
NumRetries: d.Get("retries").(int),
219+
}
220+
mlURL := c.url
221+
if !strings.HasSuffix(c.url, "/") {
222+
mlURL += "/"
223+
}
224+
mlURL += "api/plugins/grafana-ml-app/resources"
225+
mlclient, err := mlapi.New(mlURL, mlcfg)
226+
if err != nil {
227+
return nil, diag.FromErr(err)
228+
}
229+
c.mlapi = mlclient
230+
207231
smToken := d.Get("sm_access_token").(string)
208232
smURL := d.Get("sm_url").(string)
209233
c.smapi = smapi.NewClient(smURL, smToken, nil)
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
package grafana
2+
3+
import (
4+
"context"
5+
"time"
6+
7+
"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
8+
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
9+
10+
"github.com/grafana/machine-learning-go-client/mlapi"
11+
)
12+
13+
var (
14+
machineLearningJob = &schema.Resource{
15+
16+
Description: `
17+
A job defines the queries and model parameters for a machine learning task.
18+
`,
19+
20+
CreateContext: resourceMachineLearningJobCreate,
21+
ReadContext: resourceMachineLearningJobRead,
22+
UpdateContext: resourceMachineLearningJobUpdate,
23+
DeleteContext: resourceMachineLearningJobDelete,
24+
Importer: &schema.ResourceImporter{
25+
StateContext: schema.ImportStatePassthroughContext,
26+
},
27+
28+
Schema: map[string]*schema.Schema{
29+
"id": {
30+
Description: "The ID of the job.",
31+
Type: schema.TypeString,
32+
Computed: true,
33+
},
34+
"name": {
35+
Description: "The name of the job.",
36+
Type: schema.TypeString,
37+
Required: true,
38+
},
39+
"metric": {
40+
Description: "The metric used to query the job results.",
41+
Type: schema.TypeString,
42+
Required: true,
43+
},
44+
"description": {
45+
Description: "A description of the job.",
46+
Type: schema.TypeString,
47+
Optional: true,
48+
},
49+
"datasource_id": {
50+
Description: "The id of the datasource to query.",
51+
Type: schema.TypeInt,
52+
Required: true,
53+
},
54+
"datasource_type": {
55+
Description: "The type of datasource being queried. Currently allowed values are prometheus, graphite, loki, postgres, and datadog.",
56+
Type: schema.TypeString,
57+
Required: true,
58+
},
59+
"query_params": {
60+
Description: "An object representing the query params to query Grafana with.",
61+
Type: schema.TypeMap,
62+
Required: true,
63+
},
64+
"interval": {
65+
Description: "The data interval in seconds to train the data on.",
66+
Type: schema.TypeInt,
67+
Optional: true,
68+
Default: 300,
69+
},
70+
"hyper_params": {
71+
Description: "The hyperparameters used to fine tune the algorithm. See https://grafana.com/docs/grafana-cloud/machine-learning/models/ for the full list of available hyperparameters.",
72+
Type: schema.TypeMap,
73+
Optional: true,
74+
Default: map[string]interface{}{},
75+
},
76+
"training_window": {
77+
Description: "The data interval in seconds to train the data on.",
78+
Type: schema.TypeInt,
79+
Optional: true,
80+
Default: int(90 * 24 * time.Hour / time.Second),
81+
},
82+
},
83+
}
84+
)
85+
86+
func resourceMachineLearningJob() *schema.Resource {
87+
return machineLearningJob
88+
}
89+
90+
func resourceMachineLearningJobCreate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
91+
c := meta.(*client).mlapi
92+
job := makeMLJob(d, meta)
93+
job, err := c.NewJob(ctx, job)
94+
if err != nil {
95+
return diag.FromErr(err)
96+
}
97+
d.SetId(job.ID)
98+
return resourceMachineLearningJobRead(ctx, d, meta)
99+
}
100+
101+
func resourceMachineLearningJobRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
102+
c := meta.(*client).mlapi
103+
job, err := c.Job(ctx, d.Id())
104+
if err != nil {
105+
return diag.FromErr(err)
106+
}
107+
108+
d.Set("name", job.Name)
109+
d.Set("metric", job.Metric)
110+
d.Set("description", job.Description)
111+
d.Set("datasource_id", job.DatasourceID)
112+
d.Set("datasource_type", job.DatasourceType)
113+
d.Set("query_params", job.QueryParams)
114+
d.Set("interval", job.Interval)
115+
d.Set("hyper_params", job.HyperParams)
116+
d.Set("training_window", job.TrainingWindow)
117+
118+
return nil
119+
}
120+
121+
func resourceMachineLearningJobUpdate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
122+
c := meta.(*client).mlapi
123+
j := makeMLJob(d, meta)
124+
_, err := c.UpdateJob(ctx, j)
125+
if err != nil {
126+
return diag.FromErr(err)
127+
}
128+
return resourceMachineLearningJobRead(ctx, d, meta)
129+
}
130+
131+
func resourceMachineLearningJobDelete(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
132+
c := meta.(*client).mlapi
133+
err := c.DeleteJob(ctx, d.Id())
134+
if err != nil {
135+
return diag.FromErr(err)
136+
}
137+
d.SetId("")
138+
return nil
139+
}
140+
141+
func makeMLJob(d *schema.ResourceData, meta interface{}) mlapi.Job {
142+
return mlapi.Job{
143+
ID: d.Id(),
144+
Name: d.Get("name").(string),
145+
Metric: d.Get("metric").(string),
146+
Description: d.Get("description").(string),
147+
GrafanaURL: meta.(*client).url,
148+
DatasourceID: uint(d.Get("datasource_id").(int)),
149+
DatasourceType: d.Get("datasource_type").(string),
150+
QueryParams: d.Get("query_params").(map[string]interface{}),
151+
Interval: uint(d.Get("interval").(int)),
152+
Algorithm: "Prophet",
153+
HyperParams: d.Get("hyper_params").(map[string]interface{}),
154+
TrainingWindow: uint(d.Get("training_window").(int)),
155+
TrainingFrequency: uint(24 * time.Hour / time.Second),
156+
}
157+
}

0 commit comments

Comments
 (0)