@@ -46,7 +46,7 @@ By default, the data on the page is shown for the last 30 days. But, you can sho
4646
4747The information in the ** Defender for Office 365** summary at the top of the page is described in the following subsections.
4848
49- ### Efficacy card
49+ ### Phish / Malware Efficacy card
5050
5151<!-- - https://go.microsoft.com/fwlink/?linkid=2324012 --->
5252
@@ -259,168 +259,86 @@ Organizations with Defender for Office 365 Plan 2 can use the following query in
259259
260260``` kusto
261261// This query by default will take the last 30 days of data.
262-
263262// The query and calculation can be tweaked to meet individual needs, and will update over time to get incrementally more accurate.
264-
265263// Ben Harris - Microsoft Defender for Office 365 PM.
266-
267264let _startTime = ago(30d);
268-
269265let _endTime = now();
270-
271266// Get all mailflow detected as clean at time of delivery
272-
273267let EmailEventsClean = materialize(
274-
275268 EmailEvents
276-
277269 | where Timestamp between (_startTime .. _endTime) and EmailDirection == "Inbound"
278-
279270 | where ThreatTypes !contains "Phish" and ThreatTypes !contains "Malware"
280-
281271 | project NetworkMessageId,ThreatTypes
282-
283272);
284-
285273// Get all mailflow detected as phish or malware at time of delivery
286-
287274let EmailEventsThreats = materialize(
288-
289275 EmailEvents
290-
291276 | where Timestamp between (_startTime .. _endTime) and EmailDirection == "Inbound"
292-
293277 | where ThreatTypes contains "Phish" or ThreatTypes contains "Malware"
294-
295278 | extend MDO_detection = parse_json(DetectionMethods)
296-
297279 | extend FirstDetection = iif(isempty(MDO_detection), "Clean", tostring(bag_keys(MDO_detection)[0]))
298-
299280 | extend FirstSubcategory = iif(FirstDetection != "Clean" and array_length(MDO_detection[FirstDetection]) > 0, strcat(FirstDetection, ": ", tostring(MDO_detection[FirstDetection][0])), "No Detection (clean)")
300-
301281 | project NetworkMessageId,FirstDetection,FirstSubcategory,MDO_detection,ThreatTypes
302-
303282);
304-
305283// Get all post delivery ZAP / Redelivery events, and arg_max them to ensure we have the latest verdict to work with for each
306-
307284let EmailPostDeliveryFiltered = materialize(
308-
309285 EmailPostDeliveryEvents
310-
311286 | where Timestamp between (_startTime .. datetime_add('day', 7, _endTime))
312-
313287 | where ActionType in ("Malware ZAP","Phish ZAP","Redelivery")
314-
315288 | extend Key = strcat(NetworkMessageId , "-", RecipientEmailAddress)
316-
317289 | summarize arg_max(Timestamp, *) by Key
318-
319290 | project Action,ActionType,ActionResult,ThreatTypes,NetworkMessageId
320-
321291);
322-
323292// Optional - get all admin submissions for malware or phish, so we can also count these in the miss bucket.
324-
325293let CloudAppEventsFiltered = materialize(
326-
327294 CloudAppEvents
328-
329295 | where Timestamp between (_startTime .. datetime_add('day', 7, _endTime))
330-
331296 | where ActionType == "AdminSubmissionSubmitted"
332-
333297 | extend SubmissionType = tostring(parse_json(RawEventData).SubmissionType)
334-
335298 | extend NetworkMessageId = tostring(parse_json(RawEventData).ObjectId)
336-
337299 | where SubmissionType in ("1", "2")
338-
339300 | project SubmissionType,NetworkMessageId
340-
341301);
342-
343302// get the number of threats caught in mailflow
344-
345303let Mal_Phish_Mailflow = toscalar(
346-
347304 EmailEventsThreats
348-
349- | summarize count(NetworkMessageId)
350-
305+ | summarize count()
351306);
352-
353307// get the number of threats caught in mailflow which turned out to be false positives (FPs) so we can correct the calculation
354-
355308let FP_ZAP = toscalar(
356-
357309 EmailPostDeliveryFiltered
358-
359310 | where ThreatTypes !contains "Phish" and ThreatTypes !contains "Malware" and ActionType == "Redelivery"
360-
361311 | join kind=leftsemi (EmailEventsThreats) on NetworkMessageId
362-
363- | summarize count(NetworkMessageId)
364-
312+ | summarize count()
365313);
366-
367314// get the number of threats successfully cleaned up post delivery, ignoring where administrative policy stopped action
368-
369315let FN_ZAP_Successful = toscalar(
370-
371316 EmailPostDeliveryFiltered
372-
373317 | where ActionType in ("Malware ZAP","Phish ZAP") and ActionResult in ("Success","AdminPolicy")
374-
375318 | join kind=leftsemi (EmailEventsClean) on NetworkMessageId
376-
377- | summarize count(NetworkMessageId)
378-
319+ | summarize count()
379320);
380-
381321// get the number of threats unsuccessfully cleaned up post delivery.
382-
383322let FN_ZAP_Unsuccessful = toscalar(
384-
385323 EmailPostDeliveryFiltered
386-
387324 | where ActionType in ("Malware ZAP","Phish ZAP") and ActionResult !in ("Success","AdminPolicy")
388-
389325 | join kind=leftsemi (EmailEventsClean) on NetworkMessageId
390-
391- | summarize count(NetworkMessageId)
392-
326+ | summarize count()
393327);
394-
395328// join the administrative submissions to clean mailflow to find the additional miss
396-
397329let FN_Admin_Submissions = toscalar(
398-
399330 CloudAppEventsFiltered
400-
401331 | join kind=rightsemi (EmailEventsClean) on NetworkMessageId
402-
403- | summarize count(NetworkMessageId)
404-
332+ | summarize count()
405333 );
406-
407334 // print each result, and run the calculation to work out effectiveness at time of delivery and post delivery.
408-
409335union withsource=Table
410-
411336 (print StatisticName="Mal/Phish Mailflow totals - Minus FPs", Value=toreal(Mal_Phish_Mailflow) - toreal(FP_ZAP)),
412-
413337 (print StatisticName="Admin Mal/Phish FNs Submitted", Value=toreal(FN_Admin_Submissions)),
414-
415338 (print StatisticName="Mal/Phish FPs Reverse Zapped", Value=toreal(FP_ZAP)),
416-
417339 (print StatisticName="Mal / Phish Successfully Zapped", Value=toreal(FN_ZAP_Successful)),
418-
419340 (print StatisticName="Mal / Phish UN-Successfully Zapped", Value=toreal(FN_ZAP_Unsuccessful)),
420-
421341 (print StatisticName="Effectiveness Post Delivery", Value=abs(round(((toreal(FN_Admin_Submissions)+toreal(FN_ZAP_Unsuccessful))/(toreal(Mal_Phish_Mailflow)+toreal(FN_ZAP_Successful)+toreal(FN_ZAP_Unsuccessful)+toreal(FN_Admin_Submissions)-toreal(FP_ZAP))*100-100),2))),
422-
423342 (print StatisticName="Effectiveness Pre-Delivery", Value=abs(round(((toreal(FN_Admin_Submissions)+toreal(FN_ZAP_Unsuccessful)+toreal(FN_ZAP_Successful))/(toreal(Mal_Phish_Mailflow)+toreal(FN_ZAP_Successful)+toreal(FN_ZAP_Unsuccessful)+toreal(FN_Admin_Submissions)-toreal(FP_ZAP))*100-100),2)))
424-
425343| project StatisticName, Value
426344```
0 commit comments