Merge pull request #276563 from yelevin/yelevin/analytics-rules-conceptual

Analytics rules conceptual document

Author: ttorble

.openpublishing.redirection.sentinel.json

@@ -1,5 +1,20 @@
 {
     "redirections": [
+        {
+            "source_path": "articles/sentinel/detect-threats-built-in.md#use-analytics-rule-templates",
+            "redirect_url": "/azure/sentinel/create-analytics-rule-from-template",
+            "redirect_document_id": true
+        },
+        {
+            "source_path": "articles/sentinel/detect-threats-built-in.md",
+            "redirect_url": "/azure/sentinel/threat-detection",
+            "redirect_document_id": true
+        },
+        {
+            "source_path": "articles/sentinel/detect-threats-custom.md",
+            "redirect_url": "/azure/sentinel/create-analytics-rules",
+            "redirect_document_id": true
+        },
         {
             "source_path": "articles/sentinel/automate-responses-with-playbooks.md#azure-logic-apps-basic-concepts",
             "redirect_url": "/azure/sentinel/playbooks/logic-apps-playbooks",

articles/sentinel/TOC.yml

@@ -944,28 +944,60 @@
       href: monitor-your-data.md
     - name: Create a Power BI report
       href: powerbi.md
+  - name: Threat detection (analytics) rules
+    items:
+    - name: Overview
+      href: threat-detection.md
+    - name: Scheduled analytics rules
+      items:
+      - name: Create a scheduled rule from a template
+        href: create-analytics-rule-from-template.md
+      - name: Create a scheduled rule from scratch
+        href: create-analytics-rules.md
+      - name: Enhance detections
+        items:
+        - name: Map data fields to entities
+          href: map-data-fields-to-entities.md
+        - name: Surface custom details in alerts
+          href: surface-custom-details-in-alerts.md
+        - name: Customize alert details
+          href: customize-alert-details.md
+    - name: Near-real-time (NRT) analytics rules
+      items:
+      - name: Overview
+        href: near-real-time-rules.md
+      - name: Create NRT analytics rules
+        href: create-nrt-rules.md
+    - name: Anomaly detection rules
+      items:
+      - name: Overview
+        href: soc-ml-anomalies.md
+      - name: Work with out-of-the-box anomaly rules
+        href: work-with-anomaly-rules.md
+    - name: Multistage attacks (Fusion)
+      items:
+      - name: Overview
+        href: fusion.md
+      - name: Configure multistage attack (Fusion) rules
+        href: configure-fusion-rules.md
+    - name: Create incidents from Microsoft Security alerts
+      href: create-incidents-from-alerts.md
+    - name: Export and import analytics rules
+      href: import-export-analytics-rules.md
+    - name: Manage template versions for analytics rules
+      href: manage-analytics-rule-templates.md
+    - name: Handle ingestion delay in analytics rules
+      href: ingestion-delay.md
+    - name: Get fine-tuning recommendations
+      href: detection-tuning.md
+    - name: Troubleshoot analytics rules
+      href: troubleshoot-analytics-rules.md
   - name: Tutorial - Detect threats using analytics rules
     href: tutorial-log4j-detection.md
   - name: MITRE ATT&CK coverage
     href: mitre-coverage.md
-  - name: Built-in threat detection rules
-    href: detect-threats-built-in.md
-  - name: Near-real-time (NRT) analytics rules
-    href: near-real-time-rules.md
   - name: User and entity behavior analytics (UEBA)
     href: identify-threats-with-entity-behavior-analytics.md
-  - name: Anomaly detection rules
-    items:
-    - name: Overview
-      href: soc-ml-anomalies.md
-    - name: Work with out-of-the-box anomaly rules
-      href: work-with-anomaly-rules.md
-  - name: Multistage attacks (Fusion)
-    items:
-    - name: Overview
-      href: fusion.md
-    - name: Configure multistage attack (Fusion) rules
-      href: configure-fusion-rules.md
   - name: Watchlists
     items:
     - name: Overview
@@ -976,34 +1008,10 @@
       href: watchlists-queries.md
     - name: Manage watchlists
       href: watchlists-manage.md
-  - name: Create threat detection rules
-    items:
-    - name: Create a scheduled query rule
-      href: detect-threats-custom.md
-    - name: Map data fields to entities
-      href: map-data-fields-to-entities.md
-    - name: Surface custom details in alerts
-      href: surface-custom-details-in-alerts.md
-    - name: Customize alert details
-      href: customize-alert-details.md
-    - name: Export and import analytics rules
-      href: import-export-analytics-rules.md
-    - name: Create near-real-time (NRT) analytics rules
-      href: create-nrt-rules.md
-    - name: Manage template versions for analytics rules
-      href: manage-analytics-rule-templates.md
-    - name: Handle ingestion delay in analytics rules
-      href: ingestion-delay.md
-    - name: Get fine-tuning recommendations
-      href: detection-tuning.md
-    - name: Troubleshoot analytics rules
-      href: troubleshoot-analytics-rules.md
   - name: Deploy and monitor decoy honeytokens
     href: monitor-key-vault-honeytokens.md
   - name: Handle false positives
     href: false-positives.md
-  - name: Create incidents from Microsoft Security alerts
-    href: create-incidents-from-alerts.md
 - name: Hunt for threats
   items:
   - name: Overview

articles/sentinel/create-analytics-rule-from-template.md

@@ -0,0 +1,99 @@
+---
+title: Create scheduled analytics rules from templates in Microsoft Sentinel | Microsoft Docs
+description: This article explains how to view and create scheduled analytics rules from templates in Microsoft Sentinel.
+author: yelevin
+ms.author: yelevin
+ms.topic: how-to
+ms.date: 05/27/2024
+appliesto:
+    - Microsoft Sentinel in the Azure portal
+    - Microsoft Sentinel in the Microsoft Defender portal
+ms.collection: usx-security
+---
+# Create scheduled analytics rules from templates
+
+By far the most common type of analytics rule, **Scheduled** rules are based on [Kusto queries](kusto-overview.md) that are configured to run at regular intervals and examine raw data from a defined "lookback" period. These queries can perform complex statistical operations on their target data, revealing baselines and outliers in groups of events. If the number of results captured by the query passes the threshold configured in the rule, the rule produces an alert.
+
+Microsoft makes a vast array of **analytics rule templates** available to you through the many [solutions provided in the Content hub](sentinel-solutions.md), and strongly encourages you to use them to create your rules. The queries in scheduled rule templates are written by security and data science experts, either from Microsoft or from the vendor of the solution providing the template.
+
+This article shows you how to create a scheduled analytics rule using a template.
+
+[!INCLUDE [unified-soc-preview](includes/unified-soc-preview.md)]
+
+## View existing analytics rules
+
+To view the installed analytics rules in Microsoft Sentinel, go to the **Analytics** page. The **Rule templates** tab displays all the installed rule templates. To find more rule templates, go to the **Content hub** in Microsoft Sentinel to install the related product solutions or standalone content.
+
+# [Azure portal](#tab/azure-portal)
+
+1. From the **Configuration** section of the Microsoft Sentinel navigation menu, select **Analytics**.
+
+1. On the **Analytics** screen, select the **Rule templates** tab.
+
+1. If you want to filter the list for **Scheduled** templates:
+
+    1. Select **Add filter** and choose **Rule type** from the list of filters.
+
+    1. From the resulting list, select **Scheduled**. Then select **Apply**.
+
+    :::image type="content" source="media/create-analytics-rule-from-template/view-detections.png" alt-text="Screenshot of scheduled analytics rule templates in Microsoft Azure portal." lightbox="media/create-analytics-rule-from-template/view-detections.png":::
+
+# [Defender portal](#tab/defender-portal)
+
+1. From the Microsoft Defender navigation menu, expand **Microsoft Sentinel**, then **Configuration**. Select **Analytics**.
+
+1. On the **Analytics** screen, select the **Rule templates** tab.
+
+1. If you want to filter the list for **Scheduled** templates:
+
+    1. Select **Add filter** and choose **Rule type** from the list of filters.
+
+    1. From the resulting list, select **Scheduled**. Then select **Apply**.
+
+    :::image type="content" source="media/create-analytics-rule-from-template/view-detections-defender.png" alt-text="Screenshot of scheduled analytics rule templates in Microsoft Defender portal." lightbox="media/create-analytics-rule-from-template/view-detections-defender.png":::
+
+---
+
+## Create a rule from a template
+
+This procedure describes how to create an analytics rule from a template.
+
+# [Azure portal](#tab/azure-portal)
+
+From the **Configuration** section of the Microsoft Sentinel navigation menu, select **Analytics**.
+
+# [Defender portal](#tab/defender-portal)
+
+From the Microsoft Defender navigation menu, expand **Microsoft Sentinel**, then **Configuration**. Select **Analytics**.
+
+---
+
+1. On the **Analytics** screen, select the **Rule templates** tab.
+
+1. Select a template name, and then select the **Create rule** button on the details pane to create a new active rule based on that template. 
+
+    Each template has a list of required data sources. When you open the template, the data sources are automatically checked for availability. If a data source isn't enabled, the **Create rule** button may be disabled, or you might see a message to that effect.
+
+    :::image type="content" source="media/create-analytics-rule-from-template/use-built-in-template.png" alt-text="Screenshot of analytics rule preview panel.":::
+
+1. The rule creation wizard opens. All the details are autofilled.
+
+1. Cycle through the tabs of the wizard, customizing the logic and other rule settings where possible to better suit your specific needs. 
+
+    When you get to the end of the rule creation wizard, Microsoft Sentinel creates the rule. The new rule appears in the **Active rules** tab.
+
+    Repeat the process to create more rules. For more details on how to customize your rules in the rule creation wizard, see [Create a custom analytics rule from scratch](create-analytics-rules.md).
+
+> [!TIP]
+> - Make sure that you **enable all rules associated with your connected data sources** in order to ensure full security coverage for your environment. The most efficient way to enable analytics rules is directly from the data connector page, which lists any related rules. For more information, see [Connect data sources](connect-data-sources.md).
+> 
+> - You can also **push rules to Microsoft Sentinel via [API](/rest/api/securityinsights/) and [PowerShell](https://www.powershellgallery.com/packages/Az.SecurityInsights/0.1.0)**, although doing so requires additional effort. 
+> 
+>     When using API or PowerShell, you must first export the rules to JSON before enabling the rules. API or PowerShell may be helpful when enabling rules in multiple instances of Microsoft Sentinel with identical settings in each instance.
+
+## Next steps
+
+In this document, you learned how to create scheduled analytics rules from templates in Microsoft Sentinel.
+
+- Learn more about [analytics rules](threat-detection.md).
+- Learn how to [create an analytics rule from scratch](create-analytics-rules.md).

articles/sentinel/detect-threats-custom.md renamed to articles/sentinel/create-analytics-rules.md

@@ -4,16 +4,23 @@ description: Learn how to create incidents from alerts in Microsoft Sentinel.
 author: yelevin
 ms.topic: how-to
 ms.custom: mvc
-ms.date: 11/09/2021
+ms.date: 05/29/2024
 ms.author: yelevin
 ---
 
 # Automatically create incidents from Microsoft security alerts
 
-Alerts triggered in Microsoft security solutions that are connected to Microsoft Sentinel, such as Microsoft Defender for Cloud Apps and Microsoft Defender for Identity, do not automatically create incidents in Microsoft Sentinel. By default, when you connect a Microsoft solution to Microsoft Sentinel, any alert generated in that service will be stored as raw data in Microsoft Sentinel, in the *SecurityAlert* table in your Microsoft Sentinel workspace. You can then use that data like any other raw data you ingest into Microsoft Sentinel.
+Alerts triggered in Microsoft security solutions that are connected to Microsoft Sentinel, such as Microsoft Defender for Cloud Apps and Microsoft Defender for Identity, do not automatically create incidents in Microsoft Sentinel. By default, when you connect a Microsoft solution to Microsoft Sentinel, any alert generated in that service will be ingested and stored in the *SecurityAlert* table in your Microsoft Sentinel workspace. You can then use that data like any other raw data you ingest into Microsoft Sentinel.
 
 You can easily configure Microsoft Sentinel to automatically create incidents every time an alert is triggered in a connected Microsoft security solution, by following the instructions in this article.
 
+> [!IMPORTANT]
+> **This article does not apply** if you have:
+> - Enabled [**Microsoft Defender XDR incident integration**](microsoft-365-defender-sentinel-integration.md), or 
+> - Onboarded Microsoft Sentinel to the [**unified security operations platform**](microsoft-sentinel-defender-portal.md).
+>
+> In these scenarios, Microsoft Defender XDR creates incidents from alerts generated in Microsoft services.
+
 ## Prerequisites
 
 Connect your security solution by installing the appropriate solution from the **Content Hub** in Microsoft Sentinel and setting up the data connector. For more information, see [Discover and manage Microsoft Sentinel out-of-the-box content](sentinel-solutions-deploy.md) and [Microsoft Sentinel data connectors](connect-data-sources.md).