@@ -46,7 +46,7 @@ By default, the data on the page is shown for the last 30 days. But, you can sho
46
46
47
47
The information in the ** Defender for Office 365** summary at the top of the page is described in the following subsections.
48
48
49
- ### Efficacy card
49
+ ### Phish / Malware Efficacy card
50
50
51
51
<!-- - https://go.microsoft.com/fwlink/?linkid=2324012 --->
52
52
@@ -259,168 +259,86 @@ Organizations with Defender for Office 365 Plan 2 can use the following query in
259
259
260
260
``` kusto
261
261
// This query by default will take the last 30 days of data.
262
-
263
262
// The query and calculation can be tweaked to meet individual needs, and will update over time to get incrementally more accurate.
264
-
265
263
// Ben Harris - Microsoft Defender for Office 365 PM.
266
-
267
264
let _startTime = ago(30d);
268
-
269
265
let _endTime = now();
270
-
271
266
// Get all mailflow detected as clean at time of delivery
272
-
273
267
let EmailEventsClean = materialize(
274
-
275
268
EmailEvents
276
-
277
269
| where Timestamp between (_startTime .. _endTime) and EmailDirection == "Inbound"
278
-
279
270
| where ThreatTypes !contains "Phish" and ThreatTypes !contains "Malware"
280
-
281
271
| project NetworkMessageId,ThreatTypes
282
-
283
272
);
284
-
285
273
// Get all mailflow detected as phish or malware at time of delivery
286
-
287
274
let EmailEventsThreats = materialize(
288
-
289
275
EmailEvents
290
-
291
276
| where Timestamp between (_startTime .. _endTime) and EmailDirection == "Inbound"
292
-
293
277
| where ThreatTypes contains "Phish" or ThreatTypes contains "Malware"
294
-
295
278
| extend MDO_detection = parse_json(DetectionMethods)
296
-
297
279
| extend FirstDetection = iif(isempty(MDO_detection), "Clean", tostring(bag_keys(MDO_detection)[0]))
298
-
299
280
| extend FirstSubcategory = iif(FirstDetection != "Clean" and array_length(MDO_detection[FirstDetection]) > 0, strcat(FirstDetection, ": ", tostring(MDO_detection[FirstDetection][0])), "No Detection (clean)")
300
-
301
281
| project NetworkMessageId,FirstDetection,FirstSubcategory,MDO_detection,ThreatTypes
302
-
303
282
);
304
-
305
283
// Get all post delivery ZAP / Redelivery events, and arg_max them to ensure we have the latest verdict to work with for each
306
-
307
284
let EmailPostDeliveryFiltered = materialize(
308
-
309
285
EmailPostDeliveryEvents
310
-
311
286
| where Timestamp between (_startTime .. datetime_add('day', 7, _endTime))
312
-
313
287
| where ActionType in ("Malware ZAP","Phish ZAP","Redelivery")
314
-
315
288
| extend Key = strcat(NetworkMessageId , "-", RecipientEmailAddress)
316
-
317
289
| summarize arg_max(Timestamp, *) by Key
318
-
319
290
| project Action,ActionType,ActionResult,ThreatTypes,NetworkMessageId
320
-
321
291
);
322
-
323
292
// Optional - get all admin submissions for malware or phish, so we can also count these in the miss bucket.
324
-
325
293
let CloudAppEventsFiltered = materialize(
326
-
327
294
CloudAppEvents
328
-
329
295
| where Timestamp between (_startTime .. datetime_add('day', 7, _endTime))
330
-
331
296
| where ActionType == "AdminSubmissionSubmitted"
332
-
333
297
| extend SubmissionType = tostring(parse_json(RawEventData).SubmissionType)
334
-
335
298
| extend NetworkMessageId = tostring(parse_json(RawEventData).ObjectId)
336
-
337
299
| where SubmissionType in ("1", "2")
338
-
339
300
| project SubmissionType,NetworkMessageId
340
-
341
301
);
342
-
343
302
// get the number of threats caught in mailflow
344
-
345
303
let Mal_Phish_Mailflow = toscalar(
346
-
347
304
EmailEventsThreats
348
-
349
- | summarize count(NetworkMessageId)
350
-
305
+ | summarize count()
351
306
);
352
-
353
307
// get the number of threats caught in mailflow which turned out to be false positives (FPs) so we can correct the calculation
354
-
355
308
let FP_ZAP = toscalar(
356
-
357
309
EmailPostDeliveryFiltered
358
-
359
310
| where ThreatTypes !contains "Phish" and ThreatTypes !contains "Malware" and ActionType == "Redelivery"
360
-
361
311
| join kind=leftsemi (EmailEventsThreats) on NetworkMessageId
362
-
363
- | summarize count(NetworkMessageId)
364
-
312
+ | summarize count()
365
313
);
366
-
367
314
// get the number of threats successfully cleaned up post delivery, ignoring where administrative policy stopped action
368
-
369
315
let FN_ZAP_Successful = toscalar(
370
-
371
316
EmailPostDeliveryFiltered
372
-
373
317
| where ActionType in ("Malware ZAP","Phish ZAP") and ActionResult in ("Success","AdminPolicy")
374
-
375
318
| join kind=leftsemi (EmailEventsClean) on NetworkMessageId
376
-
377
- | summarize count(NetworkMessageId)
378
-
319
+ | summarize count()
379
320
);
380
-
381
321
// get the number of threats unsuccessfully cleaned up post delivery.
382
-
383
322
let FN_ZAP_Unsuccessful = toscalar(
384
-
385
323
EmailPostDeliveryFiltered
386
-
387
324
| where ActionType in ("Malware ZAP","Phish ZAP") and ActionResult !in ("Success","AdminPolicy")
388
-
389
325
| join kind=leftsemi (EmailEventsClean) on NetworkMessageId
390
-
391
- | summarize count(NetworkMessageId)
392
-
326
+ | summarize count()
393
327
);
394
-
395
328
// join the administrative submissions to clean mailflow to find the additional miss
396
-
397
329
let FN_Admin_Submissions = toscalar(
398
-
399
330
CloudAppEventsFiltered
400
-
401
331
| join kind=rightsemi (EmailEventsClean) on NetworkMessageId
402
-
403
- | summarize count(NetworkMessageId)
404
-
332
+ | summarize count()
405
333
);
406
-
407
334
// print each result, and run the calculation to work out effectiveness at time of delivery and post delivery.
408
-
409
335
union withsource=Table
410
-
411
336
(print StatisticName="Mal/Phish Mailflow totals - Minus FPs", Value=toreal(Mal_Phish_Mailflow) - toreal(FP_ZAP)),
412
-
413
337
(print StatisticName="Admin Mal/Phish FNs Submitted", Value=toreal(FN_Admin_Submissions)),
414
-
415
338
(print StatisticName="Mal/Phish FPs Reverse Zapped", Value=toreal(FP_ZAP)),
416
-
417
339
(print StatisticName="Mal / Phish Successfully Zapped", Value=toreal(FN_ZAP_Successful)),
418
-
419
340
(print StatisticName="Mal / Phish UN-Successfully Zapped", Value=toreal(FN_ZAP_Unsuccessful)),
420
-
421
341
(print StatisticName="Effectiveness Post Delivery", Value=abs(round(((toreal(FN_Admin_Submissions)+toreal(FN_ZAP_Unsuccessful))/(toreal(Mal_Phish_Mailflow)+toreal(FN_ZAP_Successful)+toreal(FN_ZAP_Unsuccessful)+toreal(FN_Admin_Submissions)-toreal(FP_ZAP))*100-100),2))),
422
-
423
342
(print StatisticName="Effectiveness Pre-Delivery", Value=abs(round(((toreal(FN_Admin_Submissions)+toreal(FN_ZAP_Unsuccessful)+toreal(FN_ZAP_Successful))/(toreal(Mal_Phish_Mailflow)+toreal(FN_ZAP_Successful)+toreal(FN_ZAP_Unsuccessful)+toreal(FN_Admin_Submissions)-toreal(FP_ZAP))*100-100),2)))
424
-
425
343
| project StatisticName, Value
426
344
```
0 commit comments