Skip to content

Commit 8e1789b

Browse files
aahillaahill
authored
Merge pull request #117 from aahill/metrics-advisor-sample
adding metrics advisor sample
2 parents fb53a7b + 1ff71c1 commit 8e1789b

File tree

1 file changed

+387
-0
lines changed

1 file changed

+387
-0
lines changed

‎python/Decision/metrics_advisor.py‎

Lines changed: 387 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,387 @@
1+
import requests
2+
import json
3+
import time
4+
5+
6+
def add_data_feed(endpoint, subscription_key, api_key):
7+
url = endpoint + '/dataFeeds'
8+
data_feed_body = {
9+
"dataSourceType": "SqlServer",
10+
"dataFeedName": "test_data_feed_00000001",
11+
"dataFeedDescription": "",
12+
"dataSourceParameter": {
13+
"connectionString": "Server=ad-sample.database.windows.net,1433;Initial Catalog=ad-sample;Persist Security Info=False;User ID=adreadonly;Password=Readonly@2020;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;",
14+
"query": "select * from adsample3 where Timestamp = @StartTime"
15+
},
16+
"granularityName": "Daily",
17+
"granularityAmount": 0,
18+
"metrics": [
19+
{
20+
"metricName": "revenue",
21+
"metricDisplayName": "revenue",
22+
"metricDescription": ""
23+
},
24+
{
25+
"metricName": "cost",
26+
"metricDisplayName": "cost",
27+
"metricDescription": ""
28+
}
29+
],
30+
"dimension": [
31+
{
32+
"dimensionName": "city",
33+
"dimensionDisplayName": "city"
34+
},
35+
{
36+
"dimensionName": "category",
37+
"dimensionDisplayName": "category"
38+
}
39+
],
40+
"timestampColumn": "timestamp",
41+
"dataStartFrom": "2020-06-01T00:00:00.000Z",
42+
"startOffsetInSeconds": 0,
43+
"maxConcurrency": -1,
44+
"minRetryIntervalInSeconds": -1,
45+
"stopRetryAfterInSeconds": -1,
46+
"needRollup": "AlreadyRollup",
47+
"fillMissingPointType": "SmartFilling",
48+
"fillMissingPointValue": 0,
49+
"viewMode": "Private",
50+
"admins": [
51+
"[email protected]"
52+
],
53+
"viewers": [
54+
],
55+
"actionLinkTemplate": ""
56+
}
57+
res = requests.post(url, data=json.dumps(data_feed_body),
58+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
59+
'x-api-key': api_key})
60+
if res.status_code != 201:
61+
raise RuntimeError("add_data_feed failed " + res.text)
62+
else:
63+
print("add_data_feed success " + res.text)
64+
return res.headers['Location']
65+
66+
67+
def check_ingestion_latest_status(endpoint, subscription_key, api_key, datafeed_id):
68+
url = endpoint + '/dataFeeds/{}/ingestionProgress'.format(datafeed_id)
69+
res = requests.get(url, headers={'Ocp-Apim-Subscription-Key': subscription_key,
70+
'x-api-key': api_key})
71+
if res.status_code != 200:
72+
raise RuntimeError("check_ingestion_latest_status failed " + res.text)
73+
else:
74+
print("check_ingestion_latest_status success " + res.text)
75+
76+
77+
def check_ingestion_detail_status(endpoint, subscription_key, api_key, datafeed_id, start_time, end_time):
78+
url = endpoint + '/dataFeeds/{}/ingestionStatus/query'.format(datafeed_id)
79+
ingestion_detail_status_body = {
80+
"startTime": start_time,
81+
"endTime": end_time
82+
}
83+
res = requests.post(url, data=json.dumps(ingestion_detail_status_body),
84+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
85+
'x-api-key': api_key})
86+
if res.status_code != 200:
87+
raise RuntimeError("check_ingestion_detail_status failed " + res.text)
88+
else:
89+
print("check_ingestion_detail_status success " + res.text)
90+
91+
92+
def create_detection_config(endpoint, subscription_key, api_key, metric_id):
93+
url = endpoint + '/enrichment/anomalyDetection/configurations'
94+
detection_config_body = {
95+
"name": "test_detection_config0000000001",
96+
"description": "string",
97+
"metricId": metric_id,
98+
"wholeMetricConfiguration": {
99+
"smartDetectionCondition": {
100+
"sensitivity": 100,
101+
"anomalyDetectorDirection": "Both",
102+
"suppressCondition": {
103+
"minNumber": 1,
104+
"minRatio": 1
105+
}
106+
}
107+
},
108+
"dimensionGroupOverrideConfigurations": [
109+
],
110+
"seriesOverrideConfigurations": [
111+
]
112+
}
113+
res = requests.post(url, data=json.dumps(detection_config_body),
114+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
115+
'x-api-key': api_key})
116+
if res.status_code != 201:
117+
raise RuntimeError("create_detection_config failed " + res.text)
118+
else:
119+
120+
print("create_detection_config success " + res.text)
121+
return res.headers['Location']
122+
123+
124+
def create_web_hook(endpoint, subscription_key, api_key):
125+
url = endpoint + '/hooks'
126+
web_hook_body = {
127+
"hookType": "Webhook",
128+
"hookName": "test_web_hook000001",
129+
"description": "",
130+
"externalLink": "",
131+
"hookParameter": {
132+
"endpoint": "https://www.xxx.com",
133+
"username": "",
134+
"password": ""
135+
}
136+
}
137+
res = requests.post(url, data=json.dumps(web_hook_body),
138+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
139+
'x-api-key': api_key})
140+
if res.status_code != 201:
141+
raise RuntimeError("create_web_hook failed " + res.text)
142+
else:
143+
print("create_web_hook success " + res.text)
144+
return res.headers['Location']
145+
146+
147+
def create_alert_config(endpoint, subscription_key, api_key, anomaly_detection_configuration_id, hook_id):
148+
url = endpoint + '/alert/anomaly/configurations'
149+
web_hook_body = {
150+
"name": "test_alert_config00000001",
151+
"description": "",
152+
"crossMetricsOperator": "AND",
153+
"hookIds": [
154+
hook_id
155+
],
156+
"metricAlertingConfigurations": [
157+
{
158+
"anomalyDetectionConfigurationId": anomaly_detection_configuration_id,
159+
"anomalyScopeType": "All",
160+
"severityFilter": {
161+
"minAlertSeverity": "Low",
162+
"maxAlertSeverity": "High"
163+
},
164+
"snoozeFilter": {
165+
"autoSnooze": 0,
166+
"snoozeScope": "Metric",
167+
"onlyForSuccessive": True
168+
},
169+
}
170+
]
171+
}
172+
res = requests.post(url, data=json.dumps(web_hook_body),
173+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
174+
'x-api-key': api_key})
175+
if res.status_code != 201:
176+
raise RuntimeError("create_alert_config failed " + res.text)
177+
else:
178+
print("create_alert_config success " + res.text)
179+
return res.headers['Location']
180+
181+
182+
def query_alert_by_alert_config(endpoint, subscription_key, api_key, alert_config_id, start_time, end_time):
183+
url = endpoint + '/alert/anomaly/configurations/{}/alerts/query'.format(alert_config_id)
184+
alerts_body = {
185+
"startTime": start_time,
186+
"endTime": end_time,
187+
"timeMode": "AnomalyTime"
188+
}
189+
res = requests.post(url, data=json.dumps(alerts_body),
190+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
191+
'x-api-key': api_key})
192+
if res.status_code != 200:
193+
raise RuntimeError("query_alert_by_alert_config failed " + res.text)
194+
else:
195+
print("query_alert_by_alert_config success " + res.text)
196+
return [item['alertId'] for item in json.loads(res.content)['value']]
197+
198+
199+
def query_anomaly_by_alert(endpoint, subscription_key, api_key, alert_config_id, alert_id):
200+
url = endpoint + '/alert/anomaly/configurations/{}/alerts/{}/anomalies'.format(alert_config_id, alert_id)
201+
res = requests.get(url,
202+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
203+
'x-api-key': api_key})
204+
if res.status_code != 200:
205+
raise RuntimeError("query_anomaly_by_alert failed " + res.text)
206+
else:
207+
print("query_anomaly_by_alert success " + res.text)
208+
return json.loads(res.content)
209+
210+
211+
def query_incident_by_alert(endpoint, subscription_key, api_key, alert_config_id, alert_id):
212+
url = endpoint + '/alert/anomaly/configurations/{}/alerts/{}/incidents'.format(alert_config_id, alert_id)
213+
res = requests.get(url,
214+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
215+
'x-api-key': api_key})
216+
if res.status_code != 200:
217+
raise RuntimeError("query_incident_by_alert failed " + res.text)
218+
else:
219+
print("query_incident_by_alert success " + res.text)
220+
return json.loads(res.content)
221+
222+
223+
def query_root_cause_by_incident(endpoint, subscription_key, api_key, detection_config_id, incident_id):
224+
url = endpoint + '/enrichment/anomalyDetection/configurations/{}/incidents/{}/rootCause'.format(detection_config_id, incident_id)
225+
res = requests.get(url,
226+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
227+
'x-api-key': api_key})
228+
if res.status_code != 200:
229+
raise RuntimeError("query_root_cause_by_incident failed " + res.text)
230+
else:
231+
print("query_root_cause_by_incident success " + res.text)
232+
return json.loads(res.content)
233+
234+
235+
def query_anomaly_by_detection_config(endpoint, subscription_key, api_key, detection_config_id, start_time, end_time):
236+
url = endpoint + '/enrichment/anomalyDetection/configurations/{}/anomalies/query'.format(detection_config_id)
237+
body = {
238+
"startTime": start_time,
239+
"endTime": end_time,
240+
"filter": {
241+
"dimensionFilter": [
242+
],
243+
"severityFilter": {
244+
"min": "Low",
245+
"max": "High"
246+
}
247+
}
248+
}
249+
res = requests.post(url, data=json.dumps(body),
250+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
251+
'x-api-key': api_key})
252+
if res.status_code != 200:
253+
raise RuntimeError("query_anomaly_by_detection_config failed " + res.text)
254+
else:
255+
print("query_anomaly_by_detection_config success " + res.text)
256+
return json.loads(res.content)
257+
258+
259+
def query_incident_by_detection_config(endpoint, subscription_key, api_key, detection_config_id, start_time, end_time):
260+
url = endpoint + '/enrichment/anomalyDetection/configurations/{}/incidents/query'.format(detection_config_id)
261+
body = {
262+
"startTime": start_time,
263+
"endTime": end_time,
264+
"filter": {
265+
"dimensionFilter": [
266+
],
267+
}
268+
}
269+
res = requests.post(url, data=json.dumps(body),
270+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
271+
'x-api-key': api_key})
272+
if res.status_code != 200:
273+
raise RuntimeError("query_incident_by_detection_config failed " + res.text)
274+
else:
275+
print("query_incident_by_detection_config success " + res.text)
276+
return json.loads(res.content)
277+
278+
279+
if __name__ == '__main__':
280+
endpoint = "https://[placeholder].cognitiveservices.azure.com/anomalydetector-ee/v1.0"
281+
subscription_key = "[your subscription key]"
282+
api_key = "[your api key]"
283+
284+
'''
285+
First part
286+
1.onboard datafeed
287+
2.check datafeed latest status
288+
3.check datafeed status details
289+
4.create detection config
290+
5.create webhook
291+
6.create alert config
292+
'''
293+
datafeed_resource_url = add_data_feed(endpoint, subscription_key, api_key)
294+
print(datafeed_resource_url)
295+
296+
# datafeed_id and metrics_id can get from datafeed_resource_url
297+
datafeed_info = json.loads(requests.get(url=datafeed_resource_url,
298+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
299+
'x-api-key': api_key}).content)
300+
print(datafeed_info)
301+
datafeed_id = datafeed_info['dataFeedId']
302+
metrics_id = []
303+
for metrics in datafeed_info['metrics']:
304+
metrics_id.append(metrics['metricId'])
305+
time.sleep(60)
306+
307+
check_ingestion_latest_status(endpoint, subscription_key, api_key, datafeed_id)
308+
309+
check_ingestion_detail_status(endpoint, subscription_key, api_key, datafeed_id,
310+
"2020-06-01T00:00:00Z", "2020-07-01T00:00:00Z")
311+
312+
detection_config_resource_url = create_detection_config(endpoint, subscription_key, api_key, metrics_id[0])
313+
print(detection_config_resource_url)
314+
315+
# anomaly_detection_configuration_id can get from detection_config_resource_url
316+
detection_config = json.loads(requests.get(url=detection_config_resource_url,
317+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
318+
'x-api-key': api_key}).content)
319+
print(detection_config)
320+
anomaly_detection_configuration_id = detection_config['anomalyDetectionConfigurationId']
321+
322+
webhook_resource_url = create_web_hook(endpoint, subscription_key, api_key)
323+
print(webhook_resource_url)
324+
325+
# hook_id can get from webhook_resource_url
326+
webhook = json.loads(requests.get(url=webhook_resource_url,
327+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
328+
'x-api-key': api_key}).content)
329+
print(webhook)
330+
hook_id = webhook['hookId']
331+
332+
alert_config_resource_url = create_alert_config(endpoint, subscription_key, api_key,
333+
anomaly_detection_configuration_id, hook_id)
334+
335+
# anomaly_alerting_configuration_id can get from alert_config_resource_url
336+
alert_config = json.loads(requests.get(url=alert_config_resource_url,
337+
headers={'Ocp-Apim-Subscription-Key': subscription_key,
338+
'x-api-key': api_key}).content)
339+
print(alert_config)
340+
anomaly_alerting_configuration_id = alert_config['anomalyAlertingConfigurationId']
341+
342+
'''
343+
Second part
344+
after onboard everything, ingest data to pipeline and detect anomaly need a few minute, after that,
345+
you may get alert from web hook callback or you can also query alert under alert config.
346+
1.query alert under alert config
347+
2.query anomalys by alert id
348+
3.query incidents by alert id
349+
4.query root cause by incident id
350+
'''
351+
352+
# alert id can get from your webhook callback function
353+
# alert id also can get from query_alert_by_alert_config
354+
while True:
355+
alert_ids = query_alert_by_alert_config(endpoint, subscription_key, api_key, anomaly_alerting_configuration_id,
356+
"2020-08-01T00:00:00Z", "2020-09-01T00:00:00Z")
357+
if len(alert_ids) == 0:
358+
print("no alert, please wait for a minute.")
359+
time.sleep(60)
360+
else:
361+
break
362+
alert_id = alert_ids[0]
363+
anomalys = query_anomaly_by_alert(endpoint, subscription_key, api_key, anomaly_alerting_configuration_id,
364+
alert_id)
365+
366+
incidents = query_incident_by_alert(endpoint, subscription_key, api_key, anomaly_alerting_configuration_id,
367+
alert_id)
368+
369+
incident_ids = [item['incidentId'] for item in incidents['value']]
370+
371+
root_cause = query_root_cause_by_incident(endpoint, subscription_key, api_key, anomaly_detection_configuration_id,
372+
incident_ids[0])
373+
374+
'''
375+
Third part
376+
you can also query anomaly and incident under detection config
377+
1. query anomalys under anomaly detection config
378+
2. query incidents under anomaly detection config
379+
'''
380+
381+
anomalys = query_anomaly_by_detection_config(endpoint, subscription_key, api_key,
382+
anomaly_detection_configuration_id,
383+
"2020-06-01T00:00:00Z", "2020-09-01T00:00:00Z")
384+
385+
incidents = query_incident_by_detection_config(endpoint, subscription_key, api_key,
386+
anomaly_detection_configuration_id,
387+
"2020-06-01T00:00:00Z", "2020-09-01T00:00:00Z")

0 commit comments

Comments
 (0)