Merge pull request #1 from databricks-solutions/latest_merge

Latest merge

Author: derekking-db

.python-version

@@ -0,0 +1 @@
+global

README.md

@@ -4,7 +4,7 @@ A collection of security detection notebooks for Databricks workspaces that anal
 
 ## Overview
 
-This detection app provides 25+ pre-built security detection notebooks designed for security operations teams to monitor Databricks workspace activities. The detections cover various security scenarios including:
+This detection app provides 30+ pre-built security detection notebooks designed for security operations teams to monitor Databricks workspace activities. The detections cover various security scenarios including:
 
 - **Authentication & Access Control**: Token creation/deletion, MFA changes, SSO configuration changes
 - **User Management**: Account creation/deletion, role modifications, group changes
@@ -14,11 +14,12 @@ This detection app provides 25+ pre-built security detection notebooks designed
 
 ## Features
 
-- **Coverage**: 25+ detection scenarios covering major security use cases
+- **Coverage**: 30+ detection scenarios covering major security use cases
 - **Production Ready**: Designed for batch execution via Databricks workflows
 - **Configurable**: Customizable time ranges and detection parameters
 - **Audit Table Focus**: Leverages Databricks `system.access.audit` table for comprehensive visibility
 - **Unity Catalog Compatible**: Designed for Unity Catalog enabled accounts
+- **MITRE ATT&CK Mapped**: Many detections include MITRE ATT&CK framework mappings for threat intelligence
 
 ## Detection Categories
 
@@ -50,6 +51,30 @@ This detection app provides 25+ pre-built security detection notebooks designed
 - Attempted Logon from Denied IP
 - Token Scanning Activity Detection
 
+### Data Exfiltration & Movement
+- Potential Data Movement via SQL Queries
+- Potential Data Movement via Workspace Downloads
+- Potential Data Movement via Explicit Credentials
+
+### Configuration & Policy Monitoring
+- High Priority Configuration Changes
+- Workspace-Level Configuration Changes
+- Account-Level Configuration Changes
+
+### Secrets & Credential Management
+- Secret Scanning Activity Detection
+- Admin User Account Changes
+
+## Enhanced Detection Capabilities
+
+The latest version includes advanced detection scenarios that go beyond basic audit monitoring:
+
+- **Data Exfiltration Detection**: Identifies potential data movement attempts using SQL queries, workspace downloads, and explicit credentials
+- **Configuration Tampering**: Monitors for unauthorized changes to security-critical workspace and account configurations
+- **Secret Enumeration**: Detects reconnaissance activities targeting secret scopes and credential harvesting
+- **Admin Privilege Escalation**: Tracks administrative privilege changes and group membership modifications
+- **Comprehensive Coverage**: Integrates both `system.access.audit` and `system.query.history` tables for complete visibility
+
 ## Installation
 
 ### Prerequisites

base/detections/access_token_created.py

@@ -6,24 +6,30 @@
 # MAGIC %md
 # MAGIC ```yaml
 # MAGIC dscc:
-# MAGIC   author: root
-# MAGIC   created: '2025-05-09T12:56:50'
-# MAGIC   modified: '2025-05-09T12:56:50'
-# MAGIC   uuid: 4e8de7fb-5fbe-424c-99be-22e6efbb5445
+# MAGIC   author: derek.king
+# MAGIC   created: '2025-06-17T11:59:15'
+# MAGIC   modified: '2025-06-17T11:59:15'
+# MAGIC   uuid: ea49708b-50cf-4926-b734-c8acef522744
 # MAGIC   content_type: detection
 # MAGIC   detection:
 # MAGIC     name: Access Token Created
 # MAGIC     description: Detects access tokens created from unknown IPs or user agents with
 # MAGIC       a non-zero lifetime.
+# MAGIC     fidelity: high
+# MAGIC     category: POLICY
 # MAGIC     objective: 'Identify potentially suspicious access token creation events by filtering
 # MAGIC       for: the `generateDbToken` action within the `accounts` service, tokens created
 # MAGIC       outside of known IP address and user agent baselines, and tokens with a measurable
 # MAGIC       lifespan (i.e., not ephemeral). This helps surface anomalous authentication
 # MAGIC       behavior that may signify credential abuse, script-based automation, or unauthorized
-# MAGIC       access attempts.
-# MAGIC 
-# MAGIC       '
-# MAGIC     taxonomy: []
+# MAGIC       access attempts.'
+# MAGIC     false_positives: unknown
+# MAGIC     severity: high
+# MAGIC     taxonomy:
+# MAGIC     - none
+# MAGIC     platform:
+# MAGIC     - databricks
+# MAGIC   version: 1.0.0
 # MAGIC dscc-tests:
 # MAGIC   tests:
 # MAGIC   - function: access_token_created
@@ -99,7 +105,12 @@ def access_token_created(earliest:str = None, latest: str = None):
 
 # COMMAND ----------
 
-display(access_token_created(earliest="2025-01-01", latest="2025-02-25"))
+if __name__ == "__main__" or dbutils.widgets.get("earliest"):
+    earliest, latest = get_time_range_from_widgets()
+    display(access_token_created(
+        earliest=dbutils.widgets.get("earliest"),
+        latest=dbutils.widgets.get("latest")
+    ))
 
 # COMMAND ----------
 

base/detections/access_token_deleted.py

@@ -6,23 +6,26 @@
 # MAGIC %md
 # MAGIC ```yaml
 # MAGIC dscc:
-# MAGIC   author: Derek King - Databricks
-# MAGIC   created: '2025-05-09T12:56:50'
-# MAGIC   modified: '2025-05-09T12:56:50'
-# MAGIC   uuid: 4e8de7fb-5fbe-424c-99be-22e6efbb5445
+# MAGIC   author: derek.king
+# MAGIC   created: '2025-06-17T12:21:38'
+# MAGIC   modified: '2025-06-17T12:21:38'
+# MAGIC   uuid: 479f8708-b446-4a2a-af32-0c14ee52a5d4
 # MAGIC   content_type: detection
 # MAGIC   detection:
 # MAGIC     name: Access Token Deleted
-# MAGIC     description: 'Detects access tokens being revoked via the accounts service.
-# MAGIC 
-# MAGIC       '
-# MAGIC     objective: 'Monitor for explicit revocation of access tokens to track potentially
-# MAGIC       suspicious cleanup activity,
-# MAGIC 
-# MAGIC       such as unauthorized credential invalidation or the concealment of prior access.
-# MAGIC 
-# MAGIC       '
-# MAGIC     taxonomy: []
+# MAGIC     description: Detects access tokens being revoked via the accounts service.
+# MAGIC     fidelity: high
+# MAGIC     category: POLICY
+# MAGIC     objective: Monitor for explicit revocation of access tokens to track potentially
+# MAGIC       suspicious cleanup activity, such as unauthorized credential invalidation or
+# MAGIC       the concealment of prior access.
+# MAGIC     false_positives: unknown
+# MAGIC     severity: low
+# MAGIC     taxonomy:
+# MAGIC     - none
+# MAGIC     platform:
+# MAGIC     - databricks
+# MAGIC   version: 1.0.0
 # MAGIC dscc-tests:
 # MAGIC   tests:
 # MAGIC   - function: access_token_deleted
@@ -77,4 +80,9 @@ def access_token_deleted(earliest:str = None, latest: str = None):
 
 # COMMAND ----------
 
-display(access_token_deleted(earliest="2020-01-01", latest="2025-02-25"))
+if __name__ == "__main__" or dbutils.widgets.get("earliest"):
+    earliest, latest = get_time_range_from_widgets()
+    display(access_token_deleted(
+        earliest=dbutils.widgets.get("earliest"),
+        latest=dbutils.widgets.get("latest")
+    ))

base/detections/attempted_logon_from_denied_ip.py

@@ -6,25 +6,27 @@
 # MAGIC %md
 # MAGIC ```yaml
 # MAGIC dscc:
-# MAGIC   author: Derek King - Databricks
-# MAGIC   created: '2025-05-09T12:56:50'
-# MAGIC   modified: '2025-05-09T12:56:50'
-# MAGIC   uuid: 4e8de7fb-5fbe-424c-99be-22e6efbb5445
+# MAGIC   author: derek.king
+# MAGIC   created: '2025-06-17T11:57:56'
+# MAGIC   modified: '2025-06-17T11:57:56'
+# MAGIC   uuid: 5e7e72b8-5174-447f-970d-3ba5772ca8e9
 # MAGIC   content_type: detection
 # MAGIC   detection:
-# MAGIC     name: Attempted Logon from Denied IP
-# MAGIC     description: 'Detects blocked login attempts from IP addresses denied by workspace
+# MAGIC     name: Attempted Logon From Denied Ip
+# MAGIC     description: Detects blocked login attempts from IP addresses denied by workspace
 # MAGIC       access control policies.
-# MAGIC 
-# MAGIC       '
-# MAGIC     objective: 'Identify logon attempts from explicitly denied IPs that bypass known
-# MAGIC       service agents and telemetry paths,
-# MAGIC 
-# MAGIC       which may indicate unauthorized scanning activity, policy testing, or brute-force
-# MAGIC       attempts from untrusted networks.
-# MAGIC 
-# MAGIC       '
-# MAGIC     taxonomy: []
+# MAGIC     fidelity: high
+# MAGIC     category: POLICY
+# MAGIC     objective: Identify logon attempts from explicitly denied IPs that bypass known
+# MAGIC       service agents and telemetry paths, which may indicate unauthorized scanning
+# MAGIC       activity, policy testing, or brute-force attempts from untrusted networks.
+# MAGIC     false_positives: unknown
+# MAGIC     severity: low
+# MAGIC     taxonomy:
+# MAGIC     - none
+# MAGIC     platform:
+# MAGIC     - databricks
+# MAGIC   version: 1.0.0
 # MAGIC dscc-tests:
 # MAGIC   tests:
 # MAGIC   - function: attempted_logon_from_denied_ip
@@ -55,7 +57,7 @@
 # COMMAND ----------
 
 @detect(output=Output.asDataFrame)
-def attempted_logon_from_denied_ip(earliest:str = None, latest: str = None, ignore_tokens: bool = False):
+def attempted_logon_from_denied_ip(earliest:str = None, latest: str = None, ignore_tokens: bool = True):
     from pyspark.sql.functions import (col, current_date, date_sub, to_date, current_timestamp, 
                                        expr, unix_timestamp, round, from_unixtime, when, to_timestamp)
 
@@ -111,5 +113,9 @@ def attempted_logon_from_denied_ip(earliest:str = None, latest: str = None, igno
 
 # COMMAND ----------
 
-df = attempted_logon_from_denied_ip(earliest="2025-03-13", latest="2025-03-14", ignore_tokens=True)
-display(df)
+if __name__ == "__main__" or dbutils.widgets.get("earliest"):
+    earliest, latest = get_time_range_from_widgets()
+    display(attempted_logon_from_denied_ip(
+        earliest=dbutils.widgets.get("earliest"),
+        latest=dbutils.widgets.get("latest")
+    ))

base/detections/configuration_changes_account_level.py

@@ -0,0 +1,107 @@
+# Databricks notebook source
+# MAGIC %run ../../lib/common
+
+# COMMAND ----------
+
+# MAGIC %md
+# MAGIC ```yaml
+# MAGIC dscc:
+# MAGIC   author: David Veuve - Databricks
+# MAGIC   created: '2025-06-04T12:00:00'
+# MAGIC   modified: '2025-06-04T12:00:00'
+# MAGIC   uuid: 1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d
+# MAGIC   content_type: detection
+# MAGIC   detection:
+# MAGIC     name: Account Level Configuration Changes
+# MAGIC     description: Detects when users modify account-level settings, which could
+# MAGIC       indicate privilege escalation attempts or unauthorized administrative changes
+# MAGIC       to tenant-wide configurations.
+# MAGIC     objective: 'Identify potentially suspicious account-level configuration changes
+# MAGIC       that could affect the entire Databricks tenant. These modifications might
+# MAGIC       indicate attempts to escalate privileges, weaken security controls, or
+# MAGIC       establish persistence at the tenant level. Maps to MITRE ATT&CK TA0004
+# MAGIC       - Privilege Escalation - T1484 - Domain or Tenant Policy Modification.'
+# MAGIC     taxonomy: 
+# MAGIC       - MITRE.TA0004.Privilege_Escalation
+# MAGIC       - MITRE.T1484.Domain_or_Tenant_Policy_Modification
+# MAGIC     fidelity: low
+# MAGIC     category: DETECTION
+# MAGIC     false_positives: unknown
+# MAGIC     severity: low
+# MAGIC     platform:
+# MAGIC     - databricks
+# MAGIC dscc-tests:
+# MAGIC   tests:
+# MAGIC   - function: configuration_changes_account_level
+# MAGIC     input:
+# MAGIC       earliest: '2025-01-01'
+# MAGIC       latest: '2025-02-25'
+# MAGIC     expect:
+# MAGIC       count: '>0'
+# MAGIC       schema: []
+# MAGIC       data: null
+# MAGIC     mocked_inputs:
+# MAGIC     - table: system.access.audit
+# MAGIC       path: None
+# MAGIC     required_columns:
+# MAGIC     - EVENT_TIME
+# MAGIC     - SERVICE_NAME
+# MAGIC     - ACTION_NAME
+# MAGIC     - ACTION_DESCRIPTION
+# MAGIC     - USER_EMAIL
+# MAGIC     - USER_AGENT
+# MAGIC     - SOURCE_IP_ADDRESS
+# MAGIC     - REQUEST_PARAMS
+# MAGIC     - event_time
+# MAGIC     - service_name
+# MAGIC     - action_name
+# MAGIC     - audit_level
+# MAGIC     - user_identity.email
+# MAGIC     - user_agent
+# MAGIC     - source_ip_address
+# MAGIC     - request_params
+# MAGIC ```
+
+# COMMAND ----------
+
+@detect(output=Output.asDataFrame)
+def configuration_changes_account_level(earliest: str = None, latest: str = None):
+    from pyspark.sql.functions import (col, current_date, date_sub, current_timestamp, 
+                                       expr, to_timestamp, concat, lit)
+
+    earliest = earliest or current_timestamp() - expr("INTERVAL 2 months")
+    latest = latest or current_timestamp()
+    
+    df = spark.table("system.access.audit")
+    
+    # Filter for account-level setting changes
+    df_filtered = df.filter(
+        (col("audit_level") == "ACCOUNT_LEVEL") &
+        (col("action_name") == "setSetting") &
+        (col("event_time").between(earliest, latest))
+    ).select(
+        to_timestamp(col("event_time")).alias("EVENT_TIME"),
+        col("service_name").alias("SERVICE_NAME"),
+        col("action_name").alias("ACTION_NAME"),
+        concat(
+            col("action_name"),
+            lit(" for "),
+            col("audit_level")
+        ).alias("ACTION_DESCRIPTION"),
+        col("user_identity.email").alias("USER_EMAIL"),
+        col("user_agent").alias("USER_AGENT"),
+        col("source_ip_address").alias("SOURCE_IP_ADDRESS"),
+        col("request_params").alias("REQUEST_PARAMS")
+    ).orderBy(col("EVENT_TIME").desc())
+    
+    return df_filtered
+
+# COMMAND ----------
+
+if __name__ == "__main__" or dbutils.widgets.get("earliest"):
+    earliest, latest = get_time_range_from_widgets()
+    display(configuration_changes_account_level(
+        earliest=dbutils.widgets.get("earliest"),
+        latest=dbutils.widgets.get("latest")
+    ))
+# COMMAND ----------