From 5dee690825186e03e5ca546c9045dfd8a2efe7c6 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Tue, 10 Jun 2025 19:45:39 +0200 Subject: [PATCH 01/20] Add ESQL for security section, threat hunt tutorial --- solutions/security/esql-for-security.md | 14 + .../esql-threat-hunting-tutorial.md | 509 ++++++++++++++++++ solutions/toc.yml | 4 +- 3 files changed, 526 insertions(+), 1 deletion(-) create mode 100644 solutions/security/esql-for-security.md create mode 100644 solutions/security/esql-for-security/esql-threat-hunting-tutorial.md diff --git a/solutions/security/esql-for-security.md b/solutions/security/esql-for-security.md new file mode 100644 index 0000000000..f0a72cf892 --- /dev/null +++ b/solutions/security/esql-for-security.md @@ -0,0 +1,14 @@ +--- +navigation_title: ES|QL for security +applies_to: + stack: all + serverless: +products: + - id: security +--- + +# {{esql}} for security use cases + +Use the following resources to get hands-on with [{{esql}}](elasticsearch://reference/query-languages/esql.md) for cybersecurity use cases: + +- [Tutorial: Threat hunting with {{esql}}](esql-for-security/esql-threat-hunting-tutorial.md): Learn how to use {{esql}} to hunt for threats in your data. \ No newline at end of file diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md new file mode 100644 index 0000000000..6b6068677d --- /dev/null +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -0,0 +1,509 @@ +--- +applies_to: + stack: all + serverless: all +products: + - id: security +--- + +# Tutorial: Threat hunting with {{esql}} + +This is a hands-on introduction to threat hunting using the [Elasticsearch Query language ({{esql}})](/explore-analyze/query-filter/languages/esql.md), following a realistic advanced persistent threat (APT) campaign scenario. We'll investigate a sophisticated attack involving initial compromise via a malicious email attachment. +The attack escalates through lateral movement, privilege escalation, and data exfiltration. + +Using ES|QL, you'll enrich raw security events with business context - asset criticality, user privileges, and threat intelligence - to build a complete picture of the attack and assess its impact on your organization. + +## Requirements + +You'll need a running {{es}} cluster, together with {{kib}}. Refer to [choose your deployment type](/deploy-manage/deploy.md#choosing-your-deployment-type) for deployment options. + +## How to run {{esql}} queries + +In this tutorial, you'll see {{esql}} examples in the following format: + +```esql +FROM windows-security-logs +| WHERE event.code == "4624" +| LIMIT 1000 +``` + +You can run these queries in [Discover](/explore-analyze/discover.md) or [Timeline](/solutions/security/investigate/timeline#esql-in-timeline) using the `ES|QL` query language. + +If you want to run these queries in the [Dev Tools Console](/explore-analyze/query-filter/languages/esql-rest.md#esql-kibana-console), you'll need to use the following syntax: + +```console +POST /_query?format=txt +{ + "query": """ + FROM windows-security-logs + | WHERE event.code == "4624" + | LIMIT 1000 + """ +} +``` + +If you'd prefer to use your favorite programming language, refer to [Client libraries](/solutions/search/site-or-app/clients.md) for a list of official and community-supported clients. + +## Step 0: Add sample data + +To follow along with this tutorial, you need to add sample data to your cluster, using the [Dev Tools Console](/explore-analyze/query-filter/languages/esql-rest.md#esql-kibana-console). + +Broadly there are two types of data: + +1. **Core indices**: These are the main security indices that contain the logs and events you want to analyze. In this tutorial, we'll create three core indices: `windows-security-logs`, `process-logs`, and `network-logs`. +2. **Lookup indices**: These are auxiliary indices that provide additional context to your core data. In this tutorial, we'll create three lookup indices: `asset-inventory`, `user-context`, and `threat-intel`. + +### Create core indices + +First, create the core security indices for our threat hunting scenario: + +```console +PUT /windows-security-logs +{ + "mappings": { + "properties": { + "@timestamp": {"type": "date"}, + "event": { + "properties": { + "code": {"type": "keyword"}, # Event codes like 4624 (successful logon) and 4625 (failed logon) are stored as keywords for exact matching. + "action": {"type": "keyword"} + } + }, + "user": { + "properties": { + "name": {"type": "keyword"}, + "domain": {"type": "keyword"} + } + }, + "host": { + "properties": { + "name": {"type": "keyword"}, + "ip": {"type": "ip"} + } + }, + "source": { + "properties": { + "ip": {"type": "ip"} + } + }, + "logon": { + "properties": { + "type": {"type": "keyword"} + } + } + } + } +} +``` + +Now let's add some sample data to the `windows-security-logs` index around authentication events i.e. failed and successful logins. + +```console +POST /_bulk?refresh=wait_for +{"index":{"_index":"windows-security-logs"}} +{"@timestamp":"2025-05-20T08:15:00Z","event":{"code":"4625","action":"logon_failed"},"user":{"name":"jsmith","domain":"corp"},"host":{"name":"WS-001","ip":"10.1.1.50"},"source":{"ip":"10.1.1.100"}} +{"index":{"_index":"windows-security-logs"}} +{"@timestamp":"2025-05-20T08:17:00Z","event":{"code":"4624","action":"logon_success"},"user":{"name":"jsmith","domain":"corp"},"host":{"name":"WS-001","ip":"10.1.1.50"},"source":{"ip":"10.1.1.100"},"logon":{"type":"3"}} +{"index":{"_index":"windows-security-logs"}} +{"@timestamp":"2025-05-20T09:30:00Z","event":{"code":"4624","action":"logon_success"},"user":{"name":"jsmith","domain":"corp"},"host":{"name":"SRV-001","ip":"10.1.2.10"},"source":{"ip":"10.1.1.50"},"logon":{"type":"3"}} +{"index":{"_index":"windows-security-logs"}} +{"@timestamp":"2025-05-20T10:45:00Z","event":{"code":"4624","action":"logon_success"},"user":{"name":"jsmith","domain":"corp"},"host":{"name":"DB-001","ip":"10.1.3.5"},"source":{"ip":"10.1.2.10"},"logon":{"type":"3"}} +{"index":{"_index":"windows-security-logs"}} +{"@timestamp":"2025-05-20T02:30:00Z","event":{"code":"4624","action":"logon_success"},"user":{"name":"admin","domain":"corp"},"host":{"name":"DC-001","ip":"10.1.4.10"},"source":{"ip":"10.1.3.5"},"logon":{"type":"3"}} +``` + +Next, create an index for process execution logs. + +```console +PUT /process-logs +{ + "mappings": { + "properties": { + "@timestamp": {"type": "date"}, + "process": { + "properties": { + "name": {"type": "keyword"}, + "command_line": {"type": "text"}, # Command lines are stored as text fields to enable full-text search for suspicious parameters and encoded commands. + "parent": { + "properties": { + "name": {"type": "keyword"} + } + } + } + }, + "user": { + "properties": { + "name": {"type": "keyword"} + } + }, + "host": { + "properties": { + "name": {"type": "keyword"} + } + } + } + } +} +``` + +Add some sample data to the `process-logs` index. + +```console +POST /_bulk?refresh=wait_for +{"index":{"_index":"process-logs"}} +{"@timestamp":"2025-05-20T08:20:00Z","process":{"name":"powershell.exe","command_line":"powershell.exe -enc JABzAD0ATgBlAHcALgBPAGIAagBlAGMAdAAgAFMAeQBzAHQAZQBtAC4ATgBlAHQALgBXAGUAYgBDAGwAaQBlAG4AdAA=","parent":{"name":"winword.exe"}},"user":{"name":"jsmith"},"host":{"name":"WS-001"}} +{"index":{"_index":"process-logs"}} +{"@timestamp":"2025-05-20T09:35:00Z","process":{"name":"net.exe","command_line":"net user /domain","parent":{"name":"cmd.exe"}},"user":{"name":"jsmith"},"host":{"name":"SRV-001"}} +{"index":{"_index":"process-logs"}} +{"@timestamp":"2025-05-20T10:50:00Z","process":{"name":"sqlcmd.exe","command_line":"sqlcmd -S localhost -Q \"SELECT * FROM customers\"","parent":{"name":"powershell.exe"}},"user":{"name":"jsmith"},"host":{"name":"DB-001"}} +{"index":{"_index":"process-logs"}} +{"@timestamp":"2025-05-20T02:35:00Z","process":{"name":"ntdsutil.exe","command_line":"ntdsutil \"ac i ntds\" \"ifm\" \"create full c:\\temp\\ntds\"","parent":{"name":"cmd.exe"}},"user":{"name":"admin"},"host":{"name":"DC-001"}} +{"index":{"_index":"process-logs"}} +{"@timestamp":"2025-05-20T12:15:00Z","process":{"name":"schtasks.exe","command_line":"schtasks.exe /create /tn UpdateCheck /tr c:\\windows\\temp\\update.exe /sc daily","parent":{"name":"cmd.exe"}},"user":{"name":"jsmith"},"host":{"name":"WS-001"}} +{"index":{"_index":"process-logs"}} +{"@timestamp":"2025-05-20T12:30:00Z","process":{"name":"schtasks.exe","command_line":"schtasks.exe /create /tn SystemManager /tr powershell.exe -enc ZQBjAGgAbwAgACIASABlAGwAbABvACIA /sc minute /mo 5","parent":{"name":"powershell.exe"}},"user":{"name":"jsmith"},"host":{"name":"SRV-001"}} +{"index":{"_index":"process-logs"}} +{"@timestamp":"2025-05-20T13:15:00Z","process":{"name":"sc.exe","command_line":"sc.exe create RemoteService binPath= c:\\windows\\temp\\remote.exe","parent":{"name":"cmd.exe"}},"user":{"name":"jsmith"},"host":{"name":"DB-001"}} +{"index":{"_index":"process-logs"}} +{"@timestamp":"2025-05-20T13:20:00Z","process":{"name":"sc.exe","command_line":"sc.exe create BackdoorService binPath= c:\\programdata\\svc.exe","parent":{"name":"powershell.exe"}},"user":{"name":"jsmith"},"host":{"name":"SRV-001"}} +{"index":{"_index":"process-logs"}} +{"@timestamp":"2025-05-20T13:25:00Z","process":{"name":"sc.exe","command_line":"sc.exe create PersistenceService binPath= c:\\windows\\system32\\malicious.exe","parent":{"name":"cmd.exe"}},"user":{"name":"admin"},"host":{"name":"DC-001"}} +``` + +Next, create an index for network traffic logs. + +```console +PUT /network-logs +{ + "mappings": { + "properties": { + "@timestamp": {"type": "date"}, + "source": { + "properties": { + "ip": {"type": "ip"}, + "port": {"type": "integer"} + } + }, + "destination": { + "properties": { + "ip": {"type": "ip"}, + "port": {"type": "integer"} + } + }, + "network": { + "properties": { + "bytes": {"type": "long"}, + "protocol": {"type": "keyword"} + } + }, + "host": { + "properties": { + "name": {"type": "keyword"} + } + } + } + } +} +``` + +Add some sample data to the `network-logs` index. + +```console +POST /_bulk?refresh=wait_for +{"index":{"_index":"network-logs"}} +{"@timestamp":"2025-05-20T08:25:00Z","source":{"ip":"10.1.1.50","port":52341},"destination":{"ip":"185.220.101.45","port":443},"network":{"bytes":2048,"protocol":"tcp"},"host":{"name":"WS-001"}} +{"index":{"_index":"network-logs"}} +{"@timestamp":"2025-05-20T11:15:00Z","source":{"ip":"10.1.3.5","port":54892},"destination":{"ip":"185.220.101.45","port":443},"network":{"bytes":50000000,"protocol":"tcp"},"host":{"name":"DB-001"}} +{"index":{"_index":"network-logs"}} +{"@timestamp":"2025-05-20T02:40:00Z","source":{"ip":"10.1.4.10","port":61234},"destination":{"ip":"185.220.101.45","port":443},"network":{"bytes":500000000,"protocol":"tcp"},"host":{"name":"DC-001"}} +``` + +### Create lookup indices + +The lookup mode enables these indices to be used with `LOOKUP JOIN` operations for enriching security events with asset context. + +Create the indices we need with the `lookup` index mode. + +```console +PUT /asset-inventory +{ + "mappings": { + "properties": { + "host.name": {"type": "keyword"}, + "asset.criticality": {"type": "keyword"}, + "asset.owner": {"type": "keyword"}, + "asset.department": {"type": "keyword"} + } + }, + "settings": { + "index.mode": "lookup" + } +} +``` + +```console +PUT /user-context +{ + "mappings": { + "properties": { + "user.name": {"type": "keyword"}, + "user.role": {"type": "keyword"}, + "user.department": {"type": "keyword"}, + "user.privileged": {"type": "boolean"} + } + }, + "settings": { + "index.mode": "lookup" + } +} +``` + +```console +PUT /threat-intel +{ + "mappings": { + "properties": { + "indicator.value": {"type": "keyword"}, + "indicator.type": {"type": "keyword"}, + "threat.name": {"type": "keyword"}, + "threat.severity": {"type": "keyword"} + } + }, + "settings": { + "index.mode": "lookup" + } +} +``` +```console +PUT /lolbins-lookup +{ + "mappings": { + "properties": { + "indicator.value": {"type": "keyword"}, + "lolbin": {"type": "boolean"}, + "description": {"type": "text"} + } + }, + "settings": { + "index.mode": "lookup" + } +} +``` + +Now we'll populate the lookup indices with contextual data. This single bulk operation indexes data into the `user-context`, `threat-intel`, `asset-inventory`, and `lolbins-lookup` indices with one request. + +```console +POST /_bulk?refresh=wait_for +{"index":{"_index":"asset-inventory"}} +{"host.name":"WS-001","asset.criticality":"medium","asset.owner":"IT","asset.department":"finance"} +{"index":{"_index":"asset-inventory"}} +{"host.name":"SRV-001","asset.criticality":"high","asset.owner":"IT","asset.department":"operations"} +{"index":{"_index":"asset-inventory"}} +{"host.name":"DB-001","asset.criticality":"critical","asset.owner":"DBA","asset.department":"finance"} +{"index":{"_index":"asset-inventory"}} +{"host.name":"DC-001","asset.criticality":"critical","asset.owner":"IT","asset.department":"infrastructure"} +{"index":{"_index":"user-context"}} +{"user.name":"jsmith","user.role":"analyst","user.department":"finance","user.privileged":false} +{"index":{"_index":"user-context"}} +{"user.name":"admin","user.role":"administrator","user.department":"IT","user.privileged":true} +{"index":{"_index":"threat-intel"}} +{"indicator.value":"185.220.101.45","indicator.type":"ip","threat.name":"APT-29","threat.severity":"high"} +{"index":{"_index":"threat-intel"}} +{"indicator.value":"powershell.exe","indicator.type":"process","threat.name":"Living off the Land","threat.severity":"medium"} +{"index":{"_index":"lolbins-lookup"}} +{"indicator.value":"powershell.exe","lolbin":true,"description":"Windows PowerShell execution engine"} +{"index":{"_index":"lolbins-lookup"}} +{"indicator.value":"cmd.exe","lolbin":true,"description":"Windows Command Processor"} +{"index":{"_index":"lolbins-lookup"}} +{"indicator.value":"wmic.exe","lolbin":true,"description":"Windows Management Instrumentation"} +``` + +## Step 1: Hunt for initial compromise indicators + +The first phase of our hunt focuses on identifying the initial compromise. We'll look for suspicious PowerShell execution from Office applications, which is a common initial attack vector. + +```esql +FROM process-logs +| WHERE process.name == "powershell.exe" AND process.parent.name LIKE "*word*" <1> +| LOOKUP JOIN asset-inventory ON host.name <2> +| LOOKUP JOIN user-context ON user.name <3> +| EVAL encoded_command = CASE(process.command_line LIKE "*-enc*", true, false) <4> +| WHERE encoded_command == true <5> +| STATS count = COUNT(*) BY host.name, user.name, asset.criticality <6> +| LIMIT 1000 +``` + +1. Uses `WHERE` with exact match (`==`) and pattern match (`LIKE`) operators to detect PowerShell processes spawned by Word +2. Enriches process data with asset inventory using `LOOKUP JOIN` to understand which business systems are affected +3. Adds user context to identify the compromised user account and their privileges +4. Creates a boolean field using `EVAL` and `CASE` to detect base64-encoded PowerShell commands (`-enc` parameter), which attackers use to obfuscate malicious code +5. Filters for only encoded commands using `WHERE` on the computed field - focusing on suspicious PowerShell usage +6. Aggregates results with `STATS` and `COUNT` grouped `BY` multiple fields + +## Step 2: Detect lateral movement patterns + +In this step, we will track user authentication across multiple systems. This is important for identifying lateral movement and potential privilege escalation. + +This query demonstrates how `DATE_TRUNC()` creates time windows for velocity analysis, combining `COUNT_DISTINCT()` aggregations with `DATE_DIFF()` calculations to measure both the scope and speed of user movement across network assets. + +```esql +FROM windows-security-logs +| WHERE event.code == "4624" AND logon.type == "3" <1> +| LOOKUP JOIN asset-inventory ON host.name +| EVAL time_bucket = DATE_TRUNC(30 minute, @timestamp) <2> +| STATS unique_hosts = COUNT_DISTINCT(host.name), + criticality_levels = COUNT_DISTINCT(asset.criticality), + active_periods = COUNT_DISTINCT(time_bucket), + first_login = MIN(@timestamp), + last_login = MAX(@timestamp) +BY user.name <3> +| WHERE unique_hosts > 2 <4> +| EVAL time_span_hours = DATE_DIFF("hour", first_login, last_login) <5> +| EVAL movement_velocity = ROUND(unique_hosts / (time_span_hours + 1), 2) +| EVAL lateral_movement_score = unique_hosts * criticality_levels <6> +| SORT lateral_movement_score DESC +| LIMIT 1000 +``` + +1. Filters for successful remote logons (type 3) - key indicator for lateral movement detection +2. Creates 30-minute time buckets to analyze temporal patterns in authentication activity +3. Complex aggregation combining host uniqueness, criticality levels, and timing metrics +4. Calculates attack duration using `DATE_DIFF` to measure campaign timespan +5. Custom scoring formula weighing both breadth (systems accessed) and depth (criticality) of compromise + +## Step 3: Identify data access and potential exfiltration + +Advanced attackers often target sensitive data. We'll hunt for database access and large data transfers to external systems. + +```esql +FROM network-logs +| EVAL dest_ip = TO_STRING(destination.ip) <1> +| WHERE dest_ip NOT LIKE "10.*" AND dest_ip NOT LIKE "192.168.*" +| EVAL indicator.value = TO_STRING(destination.ip) <2> +| LOOKUP JOIN threat-intel ON indicator.value +| LOOKUP JOIN asset-inventory ON host.name +| WHERE threat.name IS NOT NULL +| STATS total_bytes = SUM(network.bytes), + connection_count = COUNT(*), + time_span = DATE_DIFF("hour", MIN(@timestamp), MAX(@timestamp)) <3> +BY host.name, destination.ip, threat.name, asset.criticality +| EVAL mb_transferred = ROUND(total_bytes / 1048576, 2) <4> +| EVAL risk_score = CASE( + asset.criticality == "critical" AND mb_transferred > 100, 10, + asset.criticality == "high" AND mb_transferred > 100, 7, + mb_transferred > 50, 5, + 3 + ) <5> +| WHERE total_bytes > 1000000 +| SORT risk_score DESC, total_bytes DESC +| LIMIT 1000 +``` + +1. Converts IP addresses to strings to enable pattern matching with `LIKE` operator for identifying external destinations +2. Standardizes IP format for threat intel lookup join by casting to string type +3. Uses `DATE_DIFF` to calculate duration of data transfer activities in hours +4. Converts raw bytes to megabytes using division and `ROUND` for human-readable values +5. Assigns risk scores based on asset criticality and data volume using nested `CASE` conditions + +## Step 4: Build an attack timeline and assess impact + +To understand the attack progression, we need to build a timeline of events across multiple indices. This will help us correlate actions and identify the attacker's dwell time. + + +```esql +FROM windows-security-logs, process-logs, network-logs +| LOOKUP JOIN asset-inventory ON host.name +| LOOKUP JOIN user-context ON user.name +| WHERE user.name == "jsmith" OR user.name == "admin" +| EVAL event_type = CASE( +event.code IS NOT NULL, "Authentication", +process.name IS NOT NULL, "Process Execution", +destination.ip IS NOT NULL, "Network Activity", +"Unknown" +) +| EVAL dest_ip = TO_STRING(destination.ip) +| EVAL attack_stage = CASE( +process.parent.name LIKE "*word*", "Initial Compromise", +process.name IN ("net.exe", "nltest.exe"), "Reconnaissance", +event.code == "4624" AND logon.type == "3", "Lateral Movement", +process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", +dest_ip NOT LIKE "10.*", "Exfiltration", "Other") +| SORT @timestamp ASC +| KEEP @timestamp, event_type, attack_stage, host.name, asset.criticality, user.name, process.name, destination.ip +| LIMIT 1000 +``` + +1. Querying multiple indices simultaneously provides comprehensive event correlation +2. Nested `CASE` with `IS NOT NULL` checks categorizes events by their source data structure +3. Complex conditional logic maps technical events to attack framework stages (MITRE ATT&CK) +4. `SORT ASC` creates chronological ordering essential for timeline analysis + + +## Step 5: Hunt for unusual interpreter usage + +This query demonstrates how ESQL's COUNT_DISTINCT() and CASE statements can baseline interpreter usage patterns across users and departments, using aggregation functions to identify anomalous script execution that might indicate compromised accounts or insider threats. + +```esql +FROM process-logs +| WHERE process.name IN ("powershell.exe", "cmd.exe", "net.exe", "sqlcmd.exe", "schtasks.exe", "sc.exe") <1> +| LOOKUP JOIN asset-inventory ON host.name +| LOOKUP JOIN user-context ON user.name +| STATS executions = COUNT(*), + unique_hosts = COUNT_DISTINCT(host.name), + unique_commands = COUNT_DISTINCT(process.name) <3> +BY user.name, user.department +| WHERE executions > 1 +| EVAL usage_pattern = CASE( + executions > 5, "High Usage", + executions > 3, "Moderate Usage", + "Low Usage" + ) <4> +| SORT executions DESC +| LIMIT 1000 +``` + +1. Filters for common system administration and reconnaissance tools +2. `LOOKUP JOIN` enriches process data with user and asset context +3. `COUNT_DISTINCT` functions measure the breadth of tool usage across systems +4. `CASE` statement categorizes usage patterns for easier analysis +5 Groups by user and department to identify anomalous behavior patterns + +## Step 6: Hunt for persistence mechanisms + +This query showcases how `DATE_TRUNC` enables temporal analysis of persistence mechanisms, using time bucketing and `COUNT_DISTINCT` to identify suspicious patterns like rapid-fire task creation or persistence establishment across multiple time windows. + +```esql +FROM process-logs +| WHERE process.name == "schtasks.exe" AND process.command_line:"/create" <1> +| LOOKUP JOIN asset-inventory ON host.name +| LOOKUP JOIN user-context ON user.name +| EVAL time_bucket = DATE_TRUNC(1 hour, @timestamp) <2> +| STATS task_creations = COUNT(*), + creation_hours = COUNT_DISTINCT(time_bucket) <3> +BY user.name, host.name, asset.criticality +| WHERE task_creations > 0 +| EVAL persistence_pattern = CASE( + creation_hours > 1, "Multiple Hours", + task_creations > 1, "Burst Creation", + "Single Task" + ) <4> +| SORT task_creations DESC +| LIMIT 1000 +``` + +1. Uses match operator (`:`) to find scheduled task creation commands, providing more flexible matching than exact string comparison +2. Creates hourly time buckets with `DATE_TRUNC` to analyze temporal patterns in persistence activity +3. Measures temporal dispersion using `COUNT_DISTINCT` on time buckets to detect scheduled tasks created across multiple time periods +4. Creates meaningful categorization of persistence patterns using `CASE` to identify potentially malicious task scheduling sequences (burst creation vs spread across time) + +## Additional resources + +- Check a curated collection of threat hunting [queries](https://github.com/elastic/detection-rules/tree/main/hunting) in the `elastic/detection-rules` GitHub repo. + - The corresponding [blog](https://www.elastic.co/security-labs/elevate-your-threat-hunting) provides more information about how to use them in your threat hunting workflows. +- Explore more threat hunting examples in the following blogs: + - https://www.elastic.co/blog/security-exfiltration + - https://www.elastic.co/blog/detecting-command-scripting-interpreter + - https://www.elastic.co/blog/elastic-security-detecting-credential-dumping + - https://www.elastic.co/blog/elastic-security-detecting-covert-data-exfiltration + +- Learn more about the [{{esql}}](elasticsearch://reference/query-languages/esql.md) language in the reference documentation. diff --git a/solutions/toc.yml b/solutions/toc.yml index 40cb45e857..1e8b2a7666 100644 --- a/solutions/toc.yml +++ b/solutions/toc.yml @@ -461,7 +461,6 @@ toc: - file: security/get-started/elastic-security-requirements.md - file: security/get-started/create-security-project.md - file: security/get-started/elastic-security-ui.md - - file: security/get-started/ingest-data-to-elastic-security.md children: - file: security/get-started/enable-threat-intelligence-integrations.md - file: security/get-started/automatic-migration.md @@ -472,6 +471,9 @@ toc: - file: security/get-started/data-views-elastic-security.md - file: security/get-started/create-runtime-fields-in-elastic-security.md - file: security/get-started/configure-advanced-settings.md + - file: security/esql-for-security.md + children: + - file: security/esql-for-security/esql-threat-hunting-tutorial.md - file: security/ai.md children: - file: security/ai/ai-assistant.md From 123f3b00d74c7b6d27e2769ac4ab104c3d239124 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Tue, 10 Jun 2025 19:47:01 +0200 Subject: [PATCH 02/20] fix toc typo --- solutions/toc.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/solutions/toc.yml b/solutions/toc.yml index 1e8b2a7666..6b479a7cfd 100644 --- a/solutions/toc.yml +++ b/solutions/toc.yml @@ -461,6 +461,7 @@ toc: - file: security/get-started/elastic-security-requirements.md - file: security/get-started/create-security-project.md - file: security/get-started/elastic-security-ui.md + - file: security/get-started/ingest-data-to-elastic-security.md children: - file: security/get-started/enable-threat-intelligence-integrations.md - file: security/get-started/automatic-migration.md From 0406d49e70792651e6f5e47fd963a898d7b51305 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Tue, 10 Jun 2025 19:52:31 +0200 Subject: [PATCH 03/20] fix typos --- .../esql-for-security/esql-threat-hunting-tutorial.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index 6b6068677d..fe51820438 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -27,7 +27,7 @@ FROM windows-security-logs | LIMIT 1000 ``` -You can run these queries in [Discover](/explore-analyze/discover.md) or [Timeline](/solutions/security/investigate/timeline#esql-in-timeline) using the `ES|QL` query language. +You can run these queries in [Discover](/explore-analyze/discover.md) or [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline) using the `ES|QL` query language. If you want to run these queries in the [Dev Tools Console](/explore-analyze/query-filter/languages/esql-rest.md#esql-kibana-console), you'll need to use the following syntax: @@ -357,10 +357,10 @@ FROM windows-security-logs first_login = MIN(@timestamp), last_login = MAX(@timestamp) BY user.name <3> -| WHERE unique_hosts > 2 <4> -| EVAL time_span_hours = DATE_DIFF("hour", first_login, last_login) <5> +| WHERE unique_hosts > 2 +| EVAL time_span_hours = DATE_DIFF("hour", first_login, last_login) <4> | EVAL movement_velocity = ROUND(unique_hosts / (time_span_hours + 1), 2) -| EVAL lateral_movement_score = unique_hosts * criticality_levels <6> +| EVAL lateral_movement_score = unique_hosts * criticality_levels <5> | SORT lateral_movement_score DESC | LIMIT 1000 ``` @@ -498,12 +498,11 @@ BY user.name, host.name, asset.criticality ## Additional resources -- Check a curated collection of threat hunting [queries](https://github.com/elastic/detection-rules/tree/main/hunting) in the `elastic/detection-rules` GitHub repo. +- Explore a curated collection of threat hunting [queries](https://github.com/elastic/detection-rules/tree/main/hunting) in the `elastic/detection-rules` GitHub repo. - The corresponding [blog](https://www.elastic.co/security-labs/elevate-your-threat-hunting) provides more information about how to use them in your threat hunting workflows. - Explore more threat hunting examples in the following blogs: - https://www.elastic.co/blog/security-exfiltration - https://www.elastic.co/blog/detecting-command-scripting-interpreter - https://www.elastic.co/blog/elastic-security-detecting-credential-dumping - https://www.elastic.co/blog/elastic-security-detecting-covert-data-exfiltration - - Learn more about the [{{esql}}](elasticsearch://reference/query-languages/esql.md) language in the reference documentation. From bd8e877495b4dbde8507a0cdc98a7195f795eda0 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Thu, 12 Jun 2025 12:37:58 +0200 Subject: [PATCH 04/20] Emphasize Timeline/Discover for query exection, cleanup link formatting for blogs --- .../esql-threat-hunting-tutorial.md | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index fe51820438..201fa36066 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -27,22 +27,25 @@ FROM windows-security-logs | LIMIT 1000 ``` -You can run these queries in [Discover](/explore-analyze/discover.md) or [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline) using the `ES|QL` query language. - -If you want to run these queries in the [Dev Tools Console](/explore-analyze/query-filter/languages/esql-rest.md#esql-kibana-console), you'll need to use the following syntax: - -```console -POST /_query?format=txt -{ - "query": """ - FROM windows-security-logs - | WHERE event.code == "4624" - | LIMIT 1000 - """ -} -``` - -If you'd prefer to use your favorite programming language, refer to [Client libraries](/solutions/search/site-or-app/clients.md) for a list of official and community-supported clients. +You can run these queries using: + +- **Interactive interfaces**: + - [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline). Find **Timelines** in the main menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects). + - [Discover](/explore-analyze/discover.md#esql-in-discover). Find **Discover** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects). + +- **REST API** via [Dev Tools Console](/explore-analyze/query-filter/languages/esql-rest.md#esql-kibana-console). This requires additional formatting: + :::{dropdown} View Console syntax for {{esql}} + ```console + POST /_query?format=txt + { + "query": """ + FROM windows-security-logs + | WHERE event.code == "4624" + | LIMIT 1000 + """ + } + ``` + ::: ## Step 0: Add sample data @@ -501,8 +504,8 @@ BY user.name, host.name, asset.criticality - Explore a curated collection of threat hunting [queries](https://github.com/elastic/detection-rules/tree/main/hunting) in the `elastic/detection-rules` GitHub repo. - The corresponding [blog](https://www.elastic.co/security-labs/elevate-your-threat-hunting) provides more information about how to use them in your threat hunting workflows. - Explore more threat hunting examples in the following blogs: - - https://www.elastic.co/blog/security-exfiltration - - https://www.elastic.co/blog/detecting-command-scripting-interpreter - - https://www.elastic.co/blog/elastic-security-detecting-credential-dumping - - https://www.elastic.co/blog/elastic-security-detecting-covert-data-exfiltration + - [Detect and prevent data exfiltration with Elastic Security](https://www.elastic.co/blog/security-exfiltration) + - [Detecting command and scripting interpreter techniques](https://www.elastic.co/blog/detecting-command-scripting-interpreter) + - [Detecting credential dumping with Elastic Security](https://www.elastic.co/blog/elastic-security-detecting-credential-dumping) + - [Detecting covert data exfiltration techniques](https://www.elastic.co/blog/elastic-security-detecting-covert-data-exfiltration) - Learn more about the [{{esql}}](elasticsearch://reference/query-languages/esql.md) language in the reference documentation. From 125a28b6d8f37153e86ba20e2457fe3d6f2c6b48 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Thu, 12 Jun 2025 14:30:10 +0200 Subject: [PATCH 05/20] fix links --- .../esql-for-security/esql-threat-hunting-tutorial.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index 201fa36066..109ebde931 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -30,8 +30,8 @@ FROM windows-security-logs You can run these queries using: - **Interactive interfaces**: - - [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline). Find **Timelines** in the main menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects). - - [Discover](/explore-analyze/discover.md#esql-in-discover). Find **Discover** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects). + - [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline). Find **Timelines** in the main menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). + - [Discover](/discover/try-esql.md). Find **Discover** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). - **REST API** via [Dev Tools Console](/explore-analyze/query-filter/languages/esql-rest.md#esql-kibana-console). This requires additional formatting: :::{dropdown} View Console syntax for {{esql}} From bb06617acd7cd0e624e845e288fef59ebe244fe2 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Thu, 12 Jun 2025 14:52:25 +0200 Subject: [PATCH 06/20] fix link --- .../security/esql-for-security/esql-threat-hunting-tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index 109ebde931..6866a3c668 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -31,7 +31,7 @@ You can run these queries using: - **Interactive interfaces**: - [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline). Find **Timelines** in the main menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). - - [Discover](/discover/try-esql.md). Find **Discover** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). + - [Discover](/explore-analyze/discover/try-esql.md). Find **Discover** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). - **REST API** via [Dev Tools Console](/explore-analyze/query-filter/languages/esql-rest.md#esql-kibana-console). This requires additional formatting: :::{dropdown} View Console syntax for {{esql}} From 24605b95709092d2b9f69d9c8f3a150265034d4d Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Fri, 13 Jun 2025 15:11:25 +0200 Subject: [PATCH 07/20] Link all commands/funcs/operators, update queries --- .../esql-threat-hunting-tutorial.md | 54 ++++++++++--------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index 6866a3c668..31c0f551f0 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -223,7 +223,7 @@ POST /_bulk?refresh=wait_for ### Create lookup indices -The lookup mode enables these indices to be used with `LOOKUP JOIN` operations for enriching security events with asset context. +The lookup mode enables these indices to be used with [`LOOKUP JOIN`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join) operations for enriching security events with asset context. Create the indices we need with the `lookup` index mode. @@ -336,18 +336,20 @@ FROM process-logs | LIMIT 1000 ``` -1. Uses `WHERE` with exact match (`==`) and pattern match (`LIKE`) operators to detect PowerShell processes spawned by Word -2. Enriches process data with asset inventory using `LOOKUP JOIN` to understand which business systems are affected +1. Uses [`WHERE`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-where) with exact match (using the [equality operator `==`](elasticsearch://reference/query-languages/esql/functions-operators/operators.md#esql-equals) +) and pattern match ([`LIKE`](elasticsearch://reference/query-languages/esql/functions-operators/operators.md#esql-like)) operators to detect PowerShell processes spawned by Word +2. Enriches process data with asset inventory using [`LOOKUP JOIN`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join) to understand which business systems are affected 3. Adds user context to identify the compromised user account and their privileges -4. Creates a boolean field using `EVAL` and `CASE` to detect base64-encoded PowerShell commands (`-enc` parameter), which attackers use to obfuscate malicious code -5. Filters for only encoded commands using `WHERE` on the computed field - focusing on suspicious PowerShell usage -6. Aggregates results with `STATS` and `COUNT` grouped `BY` multiple fields +4. Creates a boolean field using [`EVAL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-eval) and [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) to detect base64-encoded PowerShell commands (`-enc` parameter), which attackers use to obfuscate malicious code +5. Filters for only encoded commands using [`WHERE`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-where) on the computed field - focusing on suspicious PowerShell usage +6. Aggregates results with [`STATS`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-stats-by) and [`COUNT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count) grouped `BY` multiple fields ## Step 2: Detect lateral movement patterns In this step, we will track user authentication across multiple systems. This is important for identifying lateral movement and potential privilege escalation. -This query demonstrates how `DATE_TRUNC()` creates time windows for velocity analysis, combining `COUNT_DISTINCT()` aggregations with `DATE_DIFF()` calculations to measure both the scope and speed of user movement across network assets. +This query demonstrates how [`DATE_TRUNC`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_trunc) creates time windows for velocity analysis, combining +[`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) aggregations with [`DATE_DIFF`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_diff) calculations to measure both the scope and speed of user movement across network assets. ```esql FROM windows-security-logs @@ -371,7 +373,7 @@ BY user.name <3> 1. Filters for successful remote logons (type 3) - key indicator for lateral movement detection 2. Creates 30-minute time buckets to analyze temporal patterns in authentication activity 3. Complex aggregation combining host uniqueness, criticality levels, and timing metrics -4. Calculates attack duration using `DATE_DIFF` to measure campaign timespan +4. Calculates attack duration using [`DATE_DIFF`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_diff) to measure campaign timespan 5. Custom scoring formula weighing both breadth (systems accessed) and depth (criticality) of compromise ## Step 3: Identify data access and potential exfiltration @@ -380,8 +382,7 @@ Advanced attackers often target sensitive data. We'll hunt for database access a ```esql FROM network-logs -| EVAL dest_ip = TO_STRING(destination.ip) <1> -| WHERE dest_ip NOT LIKE "10.*" AND dest_ip NOT LIKE "192.168.*" +| WHERE NOT CIDR_MATCH(destination.ip, "10.0.0.0/8", "192.168.0.0/16") <1> | EVAL indicator.value = TO_STRING(destination.ip) <2> | LOOKUP JOIN threat-intel ON indicator.value | LOOKUP JOIN asset-inventory ON host.name @@ -402,11 +403,11 @@ BY host.name, destination.ip, threat.name, asset.criticality | LIMIT 1000 ``` -1. Converts IP addresses to strings to enable pattern matching with `LIKE` operator for identifying external destinations +1. Uses [`CIDR_MATCH`](elasticsearch://reference/query-languages/esql/functions-operators/ip-functions.md#esql-cidr_match) function with multiple CIDR ranges to efficiently filter out internal IP ranges at the IP type level 2. Standardizes IP format for threat intel lookup join by casting to string type -3. Uses `DATE_DIFF` to calculate duration of data transfer activities in hours -4. Converts raw bytes to megabytes using division and `ROUND` for human-readable values -5. Assigns risk scores based on asset criticality and data volume using nested `CASE` conditions +3. Uses [`DATE_DIFF`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_diff) to calculate duration of data transfer activities in hours +4. Converts raw bytes to megabytes using division and [`ROUND`](elasticsearch://reference/query-languages/esql/functions-operators/math-functions.md#esql-round) for human-readable values +5. Assigns risk scores based on asset criticality and data volume using nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) conditions ## Step 4: Build an attack timeline and assess impact @@ -437,14 +438,14 @@ dest_ip NOT LIKE "10.*", "Exfiltration", "Other") ``` 1. Querying multiple indices simultaneously provides comprehensive event correlation -2. Nested `CASE` with `IS NOT NULL` checks categorizes events by their source data structure -3. Complex conditional logic maps technical events to attack framework stages (MITRE ATT&CK) -4. `SORT ASC` creates chronological ordering essential for timeline analysis +2. Nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) with [`IS NOT NULL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#null-predicates) checks categorizes events by their source data structure +3. Complex conditional logic maps technical events to attack framework stages ([MITRE ATT&CK)](https://attack.mitre.org/)) for better understanding of the attack lifecycle +4. `SORT ASC` uses the [`SORT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-sort) command to order events by timestamp, which creates chronological ordering essential for timeline analysis ## Step 5: Hunt for unusual interpreter usage -This query demonstrates how ESQL's COUNT_DISTINCT() and CASE statements can baseline interpreter usage patterns across users and departments, using aggregation functions to identify anomalous script execution that might indicate compromised accounts or insider threats. +This query demonstrates how ESQL's [COUNT_DISTINCT](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) function and conditional [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) statements can be used to baseline interpreter usage patterns across users and departments, using aggregation functions to identify anomalous script execution that might indicate compromised accounts or insider threats. ```esql FROM process-logs @@ -466,14 +467,14 @@ BY user.name, user.department ``` 1. Filters for common system administration and reconnaissance tools -2. `LOOKUP JOIN` enriches process data with user and asset context -3. `COUNT_DISTINCT` functions measure the breadth of tool usage across systems -4. `CASE` statement categorizes usage patterns for easier analysis +2. [`LOOKUP JOIN`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join) enriches process data with user and asset context +3. [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) functions measure the breadth of tool usage across systems +4. [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) statement categorizes usage patterns for easier analysis 5 Groups by user and department to identify anomalous behavior patterns ## Step 6: Hunt for persistence mechanisms -This query showcases how `DATE_TRUNC` enables temporal analysis of persistence mechanisms, using time bucketing and `COUNT_DISTINCT` to identify suspicious patterns like rapid-fire task creation or persistence establishment across multiple time windows. +This query showcases how [`DATE_TRUNC`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_trunc) enables temporal analysis of persistence mechanisms, using time bucketing and [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) to identify suspicious patterns like rapid-fire task creation or persistence establishment across multiple time windows. ```esql FROM process-logs @@ -494,10 +495,11 @@ BY user.name, host.name, asset.criticality | LIMIT 1000 ``` -1. Uses match operator (`:`) to find scheduled task creation commands, providing more flexible matching than exact string comparison -2. Creates hourly time buckets with `DATE_TRUNC` to analyze temporal patterns in persistence activity -3. Measures temporal dispersion using `COUNT_DISTINCT` on time buckets to detect scheduled tasks created across multiple time periods -4. Creates meaningful categorization of persistence patterns using `CASE` to identify potentially malicious task scheduling sequences (burst creation vs spread across time) +1. Uses [match operator (`:`)](elasticsearch://reference/query-languages/esql/functions-operators/operators.md#esql-match-operator) in the `WHERE` clause with a wildcard search on the `process.command_line` field +to find scheduled task creation commands, providing more flexible matching than exact string comparison +2. Creates hourly time buckets with [`DATE_TRUNC`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_trunc) to analyze temporal patterns in persistence activity +3. Measures temporal dispersion using [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) on time buckets to detect scheduled tasks created across multiple time periods +4. Creates meaningful categorization of persistence patterns using [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) to identify potentially malicious task scheduling sequences (burst creation vs spread across time) ## Additional resources From b6e5e027ffe69324c367682089cadf41c25560cb Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Fri, 13 Jun 2025 15:22:19 +0200 Subject: [PATCH 08/20] delete unused lolbins index/data from setup --- .../esql-threat-hunting-tutorial.md | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index 31c0f551f0..60494ee7a9 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -277,23 +277,8 @@ PUT /threat-intel } } ``` -```console -PUT /lolbins-lookup -{ - "mappings": { - "properties": { - "indicator.value": {"type": "keyword"}, - "lolbin": {"type": "boolean"}, - "description": {"type": "text"} - } - }, - "settings": { - "index.mode": "lookup" - } -} -``` -Now we'll populate the lookup indices with contextual data. This single bulk operation indexes data into the `user-context`, `threat-intel`, `asset-inventory`, and `lolbins-lookup` indices with one request. +Now we'll populate the lookup indices with contextual data. This single bulk operation indexes data into the `user-context`, `threat-intel` and `asset-inventory` indices with one request. ```console POST /_bulk?refresh=wait_for @@ -313,12 +298,6 @@ POST /_bulk?refresh=wait_for {"indicator.value":"185.220.101.45","indicator.type":"ip","threat.name":"APT-29","threat.severity":"high"} {"index":{"_index":"threat-intel"}} {"indicator.value":"powershell.exe","indicator.type":"process","threat.name":"Living off the Land","threat.severity":"medium"} -{"index":{"_index":"lolbins-lookup"}} -{"indicator.value":"powershell.exe","lolbin":true,"description":"Windows PowerShell execution engine"} -{"index":{"_index":"lolbins-lookup"}} -{"indicator.value":"cmd.exe","lolbin":true,"description":"Windows Command Processor"} -{"index":{"_index":"lolbins-lookup"}} -{"indicator.value":"wmic.exe","lolbin":true,"description":"Windows Management Instrumentation"} ``` ## Step 1: Hunt for initial compromise indicators From 1c311b312d799f5244d7a2d9ad3c5a93b96648fa Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Fri, 13 Jun 2025 15:32:10 +0200 Subject: [PATCH 09/20] update attack timeline query IP filtering to use CIDR_MATCH --- .../esql-threat-hunting-tutorial.md | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index 60494ee7a9..ca2a1ead50 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -399,28 +399,29 @@ FROM windows-security-logs, process-logs, network-logs | LOOKUP JOIN user-context ON user.name | WHERE user.name == "jsmith" OR user.name == "admin" | EVAL event_type = CASE( -event.code IS NOT NULL, "Authentication", -process.name IS NOT NULL, "Process Execution", -destination.ip IS NOT NULL, "Network Activity", -"Unknown" + event.code IS NOT NULL, "Authentication", + process.name IS NOT NULL, "Process Execution", + destination.ip IS NOT NULL, "Network Activity", + "Unknown" ) -| EVAL dest_ip = TO_STRING(destination.ip) | EVAL attack_stage = CASE( -process.parent.name LIKE "*word*", "Initial Compromise", -process.name IN ("net.exe", "nltest.exe"), "Reconnaissance", -event.code == "4624" AND logon.type == "3", "Lateral Movement", -process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", -dest_ip NOT LIKE "10.*", "Exfiltration", "Other") + process.parent.name LIKE "*word*", "Initial Compromise", + process.name IN ("net.exe", "nltest.exe"), "Reconnaissance", + event.code == "4624" AND logon.type == "3", "Lateral Movement", + process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", + NOT CIDR_MATCH(destination.ip, "10.0.0.0/8"), "Exfiltration", + "Other" +) | SORT @timestamp ASC | KEEP @timestamp, event_type, attack_stage, host.name, asset.criticality, user.name, process.name, destination.ip | LIMIT 1000 ``` 1. Querying multiple indices simultaneously provides comprehensive event correlation -2. Nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) with [`IS NOT NULL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#null-predicates) checks categorizes events by their source data structure -3. Complex conditional logic maps technical events to attack framework stages ([MITRE ATT&CK)](https://attack.mitre.org/)) for better understanding of the attack lifecycle -4. `SORT ASC` uses the [`SORT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-sort) command to order events by timestamp, which creates chronological ordering essential for timeline analysis - +2. Nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) statements efficiently categorize events by their data structure +3. Maps technical events to MITRE ATT&CK framework stages for attack lifecycle analysis +4. Uses native IP type handling with [`CIDR_MATCH`](elasticsearch://reference/query-languages/esql/functions-operators/ip-functions.md#esql-cidr_match) for efficient network segmentation checks +5. Creates chronological timeline using [`SORT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-sort) on timestamp field ## Step 5: Hunt for unusual interpreter usage From 7380aaf5b9676b41629ff89c2f75abf1b95eb589 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Fri, 13 Jun 2025 15:39:14 +0200 Subject: [PATCH 10/20] Revert "update attack timeline query IP filtering to use CIDR_MATCH" This was not correct This reverts commit 1c311b312d799f5244d7a2d9ad3c5a93b96648fa. --- .../esql-threat-hunting-tutorial.md | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index ca2a1ead50..60494ee7a9 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -399,29 +399,28 @@ FROM windows-security-logs, process-logs, network-logs | LOOKUP JOIN user-context ON user.name | WHERE user.name == "jsmith" OR user.name == "admin" | EVAL event_type = CASE( - event.code IS NOT NULL, "Authentication", - process.name IS NOT NULL, "Process Execution", - destination.ip IS NOT NULL, "Network Activity", - "Unknown" +event.code IS NOT NULL, "Authentication", +process.name IS NOT NULL, "Process Execution", +destination.ip IS NOT NULL, "Network Activity", +"Unknown" ) +| EVAL dest_ip = TO_STRING(destination.ip) | EVAL attack_stage = CASE( - process.parent.name LIKE "*word*", "Initial Compromise", - process.name IN ("net.exe", "nltest.exe"), "Reconnaissance", - event.code == "4624" AND logon.type == "3", "Lateral Movement", - process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", - NOT CIDR_MATCH(destination.ip, "10.0.0.0/8"), "Exfiltration", - "Other" -) +process.parent.name LIKE "*word*", "Initial Compromise", +process.name IN ("net.exe", "nltest.exe"), "Reconnaissance", +event.code == "4624" AND logon.type == "3", "Lateral Movement", +process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", +dest_ip NOT LIKE "10.*", "Exfiltration", "Other") | SORT @timestamp ASC | KEEP @timestamp, event_type, attack_stage, host.name, asset.criticality, user.name, process.name, destination.ip | LIMIT 1000 ``` 1. Querying multiple indices simultaneously provides comprehensive event correlation -2. Nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) statements efficiently categorize events by their data structure -3. Maps technical events to MITRE ATT&CK framework stages for attack lifecycle analysis -4. Uses native IP type handling with [`CIDR_MATCH`](elasticsearch://reference/query-languages/esql/functions-operators/ip-functions.md#esql-cidr_match) for efficient network segmentation checks -5. Creates chronological timeline using [`SORT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-sort) on timestamp field +2. Nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) with [`IS NOT NULL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#null-predicates) checks categorizes events by their source data structure +3. Complex conditional logic maps technical events to attack framework stages ([MITRE ATT&CK)](https://attack.mitre.org/)) for better understanding of the attack lifecycle +4. `SORT ASC` uses the [`SORT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-sort) command to order events by timestamp, which creates chronological ordering essential for timeline analysis + ## Step 5: Hunt for unusual interpreter usage From 1d9304ff20d5fa8c57e9a4dbcce33cf6a46a33e2 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Fri, 13 Jun 2025 15:40:31 +0200 Subject: [PATCH 11/20] nicer indent --- .../esql-threat-hunting-tutorial.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index 60494ee7a9..d1eb4e9c5d 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -399,18 +399,17 @@ FROM windows-security-logs, process-logs, network-logs | LOOKUP JOIN user-context ON user.name | WHERE user.name == "jsmith" OR user.name == "admin" | EVAL event_type = CASE( -event.code IS NOT NULL, "Authentication", -process.name IS NOT NULL, "Process Execution", -destination.ip IS NOT NULL, "Network Activity", -"Unknown" -) + event.code IS NOT NULL, "Authentication", + process.name IS NOT NULL, "Process Execution", + destination.ip IS NOT NULL, "Network Activity", + "Unknown") | EVAL dest_ip = TO_STRING(destination.ip) | EVAL attack_stage = CASE( -process.parent.name LIKE "*word*", "Initial Compromise", -process.name IN ("net.exe", "nltest.exe"), "Reconnaissance", -event.code == "4624" AND logon.type == "3", "Lateral Movement", -process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", -dest_ip NOT LIKE "10.*", "Exfiltration", "Other") + process.parent.name LIKE "*word*", "Initial Compromise", + process.name IN ("net.exe", "nltest.exe"), "Reconnaissance", + event.code == "4624" AND logon.type == "3", "Lateral Movement", + process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", + dest_ip NOT LIKE "10.*", "Exfiltration", "Other") | SORT @timestamp ASC | KEEP @timestamp, event_type, attack_stage, host.name, asset.criticality, user.name, process.name, destination.ip | LIMIT 1000 From 8977e1167c11ca292e5eb6750b71da0859bc8dad Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Mon, 16 Jun 2025 16:58:06 +0200 Subject: [PATCH 12/20] add responses --- .../esql-threat-hunting-tutorial.md | 88 +++++++++++++++++-- 1 file changed, 79 insertions(+), 9 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index d1eb4e9c5d..c2a33f8169 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -323,6 +323,16 @@ FROM process-logs 5. Filters for only encoded commands using [`WHERE`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-where) on the computed field - focusing on suspicious PowerShell usage 6. Aggregates results with [`STATS`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-stats-by) and [`COUNT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count) grouped `BY` multiple fields +**Response** + +The response will contain a summary of the suspicious PowerShell executions, including the host name, user name, and asset criticality. + + + count | host.name | user.name |asset.criticality +---------------|---------------|---------------|----------------- +1 |WS-001 |jsmith |medium + + ## Step 2: Detect lateral movement patterns In this step, we will track user authentication across multiple systems. This is important for identifying lateral movement and potential privilege escalation. @@ -355,6 +365,15 @@ BY user.name <3> 4. Calculates attack duration using [`DATE_DIFF`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_diff) to measure campaign timespan 5. Custom scoring formula weighing both breadth (systems accessed) and depth (criticality) of compromise +**Response** + +The response will show users who logged into multiple hosts, their criticality levels, and the velocity of their lateral movement. + +unique_hosts |criticality_levels|active_periods | first_login | last_login | user.name |time_span_hours|movement_velocity|lateral_movement_score +---------------|------------------|---------------|------------------------|------------------------|---------------|---------------|-----------------|---------------------- +3 |3 |3 |2025-05-20T08:17:00.000Z|2025-05-20T10:45:00.000Z|jsmith |2 |1 |9 + + ## Step 3: Identify data access and potential exfiltration Advanced attackers often target sensitive data. We'll hunt for database access and large data transfers to external systems. @@ -388,6 +407,16 @@ BY host.name, destination.ip, threat.name, asset.criticality 4. Converts raw bytes to megabytes using division and [`ROUND`](elasticsearch://reference/query-languages/esql/functions-operators/math-functions.md#esql-round) for human-readable values 5. Assigns risk scores based on asset criticality and data volume using nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) conditions +**Response** + +The response will show external data transfers, their risk scores, and the amount of data transferred. + +| total_bytes | connection_count | time_span | host.name | destination.ip | threat.name | asset.criticality | mb_transferred | risk_score | +|-------------|------------------|-----------|-----------|----------------|-------------|-------------------|----------------|------------| +| 500000000 | 1 | 0 | DC-001 | 185.220.101.45 | APT-29 | critical | 476 | 10 | +| 50000000 | 1 | 0 | DB-001 | 185.220.101.45 | APT-29 | critical | 47 | 3 | + + ## Step 4: Build an attack timeline and assess impact To understand the attack progression, we need to build a timeline of events across multiple indices. This will help us correlate actions and identify the attacker's dwell time. @@ -399,17 +428,18 @@ FROM windows-security-logs, process-logs, network-logs | LOOKUP JOIN user-context ON user.name | WHERE user.name == "jsmith" OR user.name == "admin" | EVAL event_type = CASE( - event.code IS NOT NULL, "Authentication", - process.name IS NOT NULL, "Process Execution", - destination.ip IS NOT NULL, "Network Activity", - "Unknown") + event.code IS NOT NULL, "Authentication", + process.name IS NOT NULL, "Process Execution", + destination.ip IS NOT NULL, "Network Activity", + "Unknown") | EVAL dest_ip = TO_STRING(destination.ip) | EVAL attack_stage = CASE( - process.parent.name LIKE "*word*", "Initial Compromise", - process.name IN ("net.exe", "nltest.exe"), "Reconnaissance", - event.code == "4624" AND logon.type == "3", "Lateral Movement", - process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", - dest_ip NOT LIKE "10.*", "Exfiltration", "Other") + process.parent.name LIKE "*word*", "Initial Compromise", + process.name IN ("net.exe", "nltest.exe"), "Reconnaissance", + event.code == "4624" AND logon.type == "3", "Lateral Movement", + process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", + dest_ip NOT LIKE "10.*", "Exfiltration", + "Other") | SORT @timestamp ASC | KEEP @timestamp, event_type, attack_stage, host.name, asset.criticality, user.name, process.name, destination.ip | LIMIT 1000 @@ -420,6 +450,29 @@ FROM windows-security-logs, process-logs, network-logs 3. Complex conditional logic maps technical events to attack framework stages ([MITRE ATT&CK)](https://attack.mitre.org/)) for better understanding of the attack lifecycle 4. `SORT ASC` uses the [`SORT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-sort) command to order events by timestamp, which creates chronological ordering essential for timeline analysis +**Response** + +The response will provide a chronological timeline of events, showing the attacker's actions and the impact on the organization. + +:::{dropdown} View response + +| @timestamp | event_type | attack_stage | host.name | asset.criticality | user.name | process.name | destination.ip | +|------------|------------|--------------|-----------|-------------------|-----------|--------------|----------------| +| 2025-05-20T02:30:00.000Z | Authentication | Lateral Movement | DC-001 | critical | admin | null | null | +| 2025-05-20T02:35:00.000Z | Process Execution | Data Access | DC-001 | critical | admin | ntdsutil.exe | null | +| 2025-05-20T08:15:00.000Z | Authentication | Other | WS-001 | medium | jsmith | null | null | +| 2025-05-20T08:17:00.000Z | Authentication | Lateral Movement | WS-001 | medium | jsmith | null | null | +| 2025-05-20T08:20:00.000Z | Process Execution | Initial Compromise | WS-001 | medium | jsmith | powershell.exe | null | +| 2025-05-20T09:30:00.000Z | Authentication | Lateral Movement | SRV-001 | high | jsmith | null | null | +| 2025-05-20T09:35:00.000Z | Process Execution | Reconnaissance | SRV-001 | high | jsmith | net.exe | null | +| 2025-05-20T10:45:00.000Z | Authentication | Lateral Movement | DB-001 | critical | jsmith | null | null | +| 2025-05-20T10:50:00.000Z | Process Execution | Data Access | DB-001 | critical | jsmith | sqlcmd.exe | null | +| 2025-05-20T12:15:00.000Z | Process Execution | Other | WS-001 | medium | jsmith | schtasks.exe | null | +| 2025-05-20T12:30:00.000Z | Process Execution | Other | SRV-001 | high | jsmith | schtasks.exe | null | +| 2025-05-20T13:15:00.000Z | Process Execution | Other | DB-001 | critical | jsmith | sc.exe | null | +| 2025-05-20T13:20:00.000Z | Process Execution | Other | SRV-001 | high | jsmith | sc.exe | null | +| 2025-05-20T13:25:00.000Z | Process Execution | Other | DC-001 | critical | admin | sc.exe | null | +::: ## Step 5: Hunt for unusual interpreter usage @@ -450,6 +503,14 @@ BY user.name, user.department 4. [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) statement categorizes usage patterns for easier analysis 5 Groups by user and department to identify anomalous behavior patterns +**Response** + +The response will show the number of executions, unique hosts, and usage patterns for each user and department. + +| executions | unique_hosts | unique_commands | user.name | user.department | usage_pattern | +|------------|--------------|-----------------|-----------|-----------------|---------------| +| 7 | 3 | 5 | jsmith | finance | High Usage | + ## Step 6: Hunt for persistence mechanisms This query showcases how [`DATE_TRUNC`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_trunc) enables temporal analysis of persistence mechanisms, using time bucketing and [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) to identify suspicious patterns like rapid-fire task creation or persistence establishment across multiple time windows. @@ -479,6 +540,15 @@ to find scheduled task creation commands, providing more flexible matching than 3. Measures temporal dispersion using [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) on time buckets to detect scheduled tasks created across multiple time periods 4. Creates meaningful categorization of persistence patterns using [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) to identify potentially malicious task scheduling sequences (burst creation vs spread across time) +**Response** + +The response will show the number of task creations, creation hours, and persistence patterns for each user and host. + +| task_creations | creation_hours | user.name | host.name | asset.criticality | persistence_pattern | +|----------------|----------------|-----------|-----------|-------------------|---------------------| +| 1 | 1 | jsmith | WS-001 | medium | Single Task | +| 1 | 1 | jsmith | SRV-001 | high | Single Task | + ## Additional resources - Explore a curated collection of threat hunting [queries](https://github.com/elastic/detection-rules/tree/main/hunting) in the `elastic/detection-rules` GitHub repo. From f3d253d94495abc285363a35ffa2e58cf6596127 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Mon, 16 Jun 2025 19:01:49 +0200 Subject: [PATCH 13/20] cleanup, use present tense consistently --- .../esql-threat-hunting-tutorial.md | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index c2a33f8169..b83bd4d549 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -8,18 +8,24 @@ products: # Tutorial: Threat hunting with {{esql}} -This is a hands-on introduction to threat hunting using the [Elasticsearch Query language ({{esql}})](/explore-analyze/query-filter/languages/esql.md), following a realistic advanced persistent threat (APT) campaign scenario. We'll investigate a sophisticated attack involving initial compromise via a malicious email attachment. -The attack escalates through lateral movement, privilege escalation, and data exfiltration. +This hands-on tutorial demonstrates advanced threat hunting techniques using the [Elasticsearch Query Language ({{esql}})](/explore-analyze/query-filter/languages/esql.md). -Using ES|QL, you'll enrich raw security events with business context - asset criticality, user privileges, and threat intelligence - to build a complete picture of the attack and assess its impact on your organization. +Following a simulated Advanced Persistent Threat (APT) campaign, we analyze security events across authentication, process execution, and network telemetry to detect: + +- Initial compromise via malicious email attachments +- Lateral movement through the network +- Privilege escalation attempts +- Data exfiltration activities + +{{esql}} enables powerful transformations, filtering, enrichment, and statistical analysis, making it ideal for complex security investigations. This tutorial provides practical examples of how to leverage {{esql}} for threat hunting, from identifying suspicious user behavior to building attack timelines. ## Requirements -You'll need a running {{es}} cluster, together with {{kib}}. Refer to [choose your deployment type](/deploy-manage/deploy.md#choosing-your-deployment-type) for deployment options. +You need a running {{es}} cluster, together with {{kib}}. Refer to [choose your deployment type](/deploy-manage/deploy.md#choosing-your-deployment-type) for deployment options. ## How to run {{esql}} queries -In this tutorial, you'll see {{esql}} examples in the following format: +In this tutorial, {{esql}} examples are displayed in the following format: ```esql FROM windows-security-logs @@ -53,8 +59,8 @@ To follow along with this tutorial, you need to add sample data to your cluster, Broadly there are two types of data: -1. **Core indices**: These are the main security indices that contain the logs and events you want to analyze. In this tutorial, we'll create three core indices: `windows-security-logs`, `process-logs`, and `network-logs`. -2. **Lookup indices**: These are auxiliary indices that provide additional context to your core data. In this tutorial, we'll create three lookup indices: `asset-inventory`, `user-context`, and `threat-intel`. +1. **Core indices**: These are the main security indices that contain the logs and events you want to analyze. We need three core indices: `windows-security-logs`, `process-logs`, and `network-logs`. +2. **Lookup indices**: These are auxiliary indices that provide additional context to your core data. We need three lookup indices: `asset-inventory`, `user-context`, and `threat-intel`. ### Create core indices @@ -99,7 +105,7 @@ PUT /windows-security-logs } ``` -Now let's add some sample data to the `windows-security-logs` index around authentication events i.e. failed and successful logins. +Now let's add some sample data to the `windows-security-logs` index around authentication events, namely failed and successful logins. ```console POST /_bulk?refresh=wait_for @@ -278,7 +284,7 @@ PUT /threat-intel } ``` -Now we'll populate the lookup indices with contextual data. This single bulk operation indexes data into the `user-context`, `threat-intel` and `asset-inventory` indices with one request. +Now we can populate the lookup indices with contextual data. This single bulk operation indexes data into the `user-context`, `threat-intel` and `asset-inventory` indices with one request. ```console POST /_bulk?refresh=wait_for @@ -302,7 +308,7 @@ POST /_bulk?refresh=wait_for ## Step 1: Hunt for initial compromise indicators -The first phase of our hunt focuses on identifying the initial compromise. We'll look for suspicious PowerShell execution from Office applications, which is a common initial attack vector. +The first phase of our hunt focuses on identifying the initial compromise. We want to search for suspicious PowerShell execution from Office applications, which is a common initial attack vector. ```esql FROM process-logs @@ -325,7 +331,7 @@ FROM process-logs **Response** -The response will contain a summary of the suspicious PowerShell executions, including the host name, user name, and asset criticality. +The response contains a summary of the suspicious PowerShell executions, including the host name, user name, and asset criticality. count | host.name | user.name |asset.criticality @@ -335,7 +341,7 @@ The response will contain a summary of the suspicious PowerShell executions, inc ## Step 2: Detect lateral movement patterns -In this step, we will track user authentication across multiple systems. This is important for identifying lateral movement and potential privilege escalation. +In this step, we track user authentication across multiple systems. This is important for identifying lateral movement and potential privilege escalation. This query demonstrates how [`DATE_TRUNC`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_trunc) creates time windows for velocity analysis, combining [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) aggregations with [`DATE_DIFF`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_diff) calculations to measure both the scope and speed of user movement across network assets. @@ -367,7 +373,7 @@ BY user.name <3> **Response** -The response will show users who logged into multiple hosts, their criticality levels, and the velocity of their lateral movement. +The response shows users who logged into multiple hosts, their criticality levels, and the velocity of their lateral movement. unique_hosts |criticality_levels|active_periods | first_login | last_login | user.name |time_span_hours|movement_velocity|lateral_movement_score ---------------|------------------|---------------|------------------------|------------------------|---------------|---------------|-----------------|---------------------- @@ -376,7 +382,7 @@ unique_hosts |criticality_levels|active_periods | first_login | ## Step 3: Identify data access and potential exfiltration -Advanced attackers often target sensitive data. We'll hunt for database access and large data transfers to external systems. +Advanced attackers often target sensitive data. We want to hunt for database access and large data transfers to external systems. ```esql FROM network-logs @@ -409,7 +415,7 @@ BY host.name, destination.ip, threat.name, asset.criticality **Response** -The response will show external data transfers, their risk scores, and the amount of data transferred. +The response shows external data transfers, their risk scores, and the amount of data transferred. | total_bytes | connection_count | time_span | host.name | destination.ip | threat.name | asset.criticality | mb_transferred | risk_score | |-------------|------------------|-----------|-----------|----------------|-------------|-------------------|----------------|------------| @@ -419,7 +425,7 @@ The response will show external data transfers, their risk scores, and the amoun ## Step 4: Build an attack timeline and assess impact -To understand the attack progression, we need to build a timeline of events across multiple indices. This will help us correlate actions and identify the attacker's dwell time. +To understand the attack progression, we need to build a timeline of events across multiple indices. This helps us correlate actions and identify the attacker's dwell time. ```esql @@ -452,7 +458,7 @@ FROM windows-security-logs, process-logs, network-logs **Response** -The response will provide a chronological timeline of events, showing the attacker's actions and the impact on the organization. +The response provides a chronological timeline of events, showing the attacker's actions and the impact on the organization. :::{dropdown} View response @@ -505,7 +511,7 @@ BY user.name, user.department **Response** -The response will show the number of executions, unique hosts, and usage patterns for each user and department. +The response shows the number of executions, unique hosts, and usage patterns for each user and department. | executions | unique_hosts | unique_commands | user.name | user.department | usage_pattern | |------------|--------------|-----------------|-----------|-----------------|---------------| @@ -542,7 +548,7 @@ to find scheduled task creation commands, providing more flexible matching than **Response** -The response will show the number of task creations, creation hours, and persistence patterns for each user and host. +The response shows the number of task creations, creation hours, and persistence patterns for each user and host. | task_creations | creation_hours | user.name | host.name | asset.criticality | persistence_pattern | |----------------|----------------|-----------|-----------|-------------------|---------------------| @@ -558,4 +564,4 @@ The response will show the number of task creations, creation hours, and persist - [Detecting command and scripting interpreter techniques](https://www.elastic.co/blog/detecting-command-scripting-interpreter) - [Detecting credential dumping with Elastic Security](https://www.elastic.co/blog/elastic-security-detecting-credential-dumping) - [Detecting covert data exfiltration techniques](https://www.elastic.co/blog/elastic-security-detecting-covert-data-exfiltration) -- Learn more about the [{{esql}}](elasticsearch://reference/query-languages/esql.md) language in the reference documentation. +- Learn more about the [{{esql}}](elasticsearch://reference/query-languages/esql.md) language in the reference documentation. \ No newline at end of file From 7a5d086e48075641b4098e58682a6f59a43e96f7 Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Thu, 19 Jun 2025 11:48:57 +0200 Subject: [PATCH 14/20] Apply suggestions Co-authored-by: natasha-moore-elastic <137783811+natasha-moore-elastic@users.noreply.github.com> --- solutions/security/esql-for-security.md | 2 +- .../esql-threat-hunting-tutorial.md | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/solutions/security/esql-for-security.md b/solutions/security/esql-for-security.md index f0a72cf892..77f82111f1 100644 --- a/solutions/security/esql-for-security.md +++ b/solutions/security/esql-for-security.md @@ -9,6 +9,6 @@ products: # {{esql}} for security use cases -Use the following resources to get hands-on with [{{esql}}](elasticsearch://reference/query-languages/esql.md) for cybersecurity use cases: +Use the following resources to get hands-on with the [Elasticsearch Query Language ({{esql}})](elasticsearch://reference/query-languages/esql.md) for cybersecurity use cases: - [Tutorial: Threat hunting with {{esql}}](esql-for-security/esql-threat-hunting-tutorial.md): Learn how to use {{esql}} to hunt for threats in your data. \ No newline at end of file diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index b83bd4d549..e504b2b832 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -36,7 +36,7 @@ FROM windows-security-logs You can run these queries using: - **Interactive interfaces**: - - [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline). Find **Timelines** in the main menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). + - [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline). Find **Timelines** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). - [Discover](/explore-analyze/discover/try-esql.md). Find **Discover** in the navigation menu or by using the [global search field](/explore-analyze/find-and-organize/find-apps-and-objects.md). - **REST API** via [Dev Tools Console](/explore-analyze/query-filter/languages/esql-rest.md#esql-kibana-console). This requires additional formatting: @@ -57,7 +57,7 @@ You can run these queries using: To follow along with this tutorial, you need to add sample data to your cluster, using the [Dev Tools Console](/explore-analyze/query-filter/languages/esql-rest.md#esql-kibana-console). -Broadly there are two types of data: +Broadly, there are two types of data: 1. **Core indices**: These are the main security indices that contain the logs and events you want to analyze. We need three core indices: `windows-security-logs`, `process-logs`, and `network-logs`. 2. **Lookup indices**: These are auxiliary indices that provide additional context to your core data. We need three lookup indices: `asset-inventory`, `user-context`, and `threat-intel`. @@ -453,7 +453,7 @@ FROM windows-security-logs, process-logs, network-logs 1. Querying multiple indices simultaneously provides comprehensive event correlation 2. Nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) with [`IS NOT NULL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#null-predicates) checks categorizes events by their source data structure -3. Complex conditional logic maps technical events to attack framework stages ([MITRE ATT&CK)](https://attack.mitre.org/)) for better understanding of the attack lifecycle +3. Complex conditional logic maps technical events to attack framework stages ([MITRE ATT&CK](https://attack.mitre.org/)) for better understanding of the attack lifecycle 4. `SORT ASC` uses the [`SORT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-sort) command to order events by timestamp, which creates chronological ordering essential for timeline analysis **Response** @@ -482,7 +482,7 @@ The response provides a chronological timeline of events, showing the attacker's ## Step 5: Hunt for unusual interpreter usage -This query demonstrates how ESQL's [COUNT_DISTINCT](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) function and conditional [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) statements can be used to baseline interpreter usage patterns across users and departments, using aggregation functions to identify anomalous script execution that might indicate compromised accounts or insider threats. +This query demonstrates how {{esql}}'s [COUNT_DISTINCT](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) function and conditional [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) statements can be used to baseline interpreter usage patterns across users and departments, using aggregation functions to identify anomalous script execution that might indicate compromised accounts or insider threats. ```esql FROM process-logs @@ -507,7 +507,7 @@ BY user.name, user.department 2. [`LOOKUP JOIN`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join) enriches process data with user and asset context 3. [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) functions measure the breadth of tool usage across systems 4. [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) statement categorizes usage patterns for easier analysis -5 Groups by user and department to identify anomalous behavior patterns +5. Groups by user and department to identify anomalous behavior patterns **Response** @@ -560,8 +560,8 @@ The response shows the number of task creations, creation hours, and persistence - Explore a curated collection of threat hunting [queries](https://github.com/elastic/detection-rules/tree/main/hunting) in the `elastic/detection-rules` GitHub repo. - The corresponding [blog](https://www.elastic.co/security-labs/elevate-your-threat-hunting) provides more information about how to use them in your threat hunting workflows. - Explore more threat hunting examples in the following blogs: - - [Detect and prevent data exfiltration with Elastic Security](https://www.elastic.co/blog/security-exfiltration) + - [Detect and prevent data exfiltration with {{elastic-sec}}](https://www.elastic.co/blog/security-exfiltration) - [Detecting command and scripting interpreter techniques](https://www.elastic.co/blog/detecting-command-scripting-interpreter) - - [Detecting credential dumping with Elastic Security](https://www.elastic.co/blog/elastic-security-detecting-credential-dumping) + - [Detecting credential dumping with {{elastic-sec}}](https://www.elastic.co/blog/elastic-security-detecting-credential-dumping) - [Detecting covert data exfiltration techniques](https://www.elastic.co/blog/elastic-security-detecting-covert-data-exfiltration) - Learn more about the [{{esql}}](elasticsearch://reference/query-languages/esql.md) language in the reference documentation. \ No newline at end of file From 3abf97dd23af92f71cb8f38aec328b4141223b54 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Thu, 19 Jun 2025 15:42:17 +0200 Subject: [PATCH 15/20] use requirements admonition --- .../esql-for-security/esql-threat-hunting-tutorial.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index e504b2b832..a785f1f657 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -19,9 +19,9 @@ Following a simulated Advanced Persistent Threat (APT) campaign, we analyze secu {{esql}} enables powerful transformations, filtering, enrichment, and statistical analysis, making it ideal for complex security investigations. This tutorial provides practical examples of how to leverage {{esql}} for threat hunting, from identifying suspicious user behavior to building attack timelines. -## Requirements - -You need a running {{es}} cluster, together with {{kib}}. Refer to [choose your deployment type](/deploy-manage/deploy.md#choosing-your-deployment-type) for deployment options. +::::{admonition} Requirements +You need a running {{es}} cluster, together with {{kib}} to run this tutorial. Refer to [choose your deployment type](/deploy-manage/deploy.md#choosing-your-deployment-type) for deployment options. +:::: ## How to run {{esql}} queries From 994da7a0cc1fd73a234b11460b119bad621f2b55 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Thu, 19 Jun 2025 16:27:56 +0200 Subject: [PATCH 16/20] standardize annotations --- .../esql-threat-hunting-tutorial.md | 68 +++++++++---------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index a785f1f657..e8937a8234 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -321,13 +321,12 @@ FROM process-logs | LIMIT 1000 ``` -1. Uses [`WHERE`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-where) with exact match (using the [equality operator `==`](elasticsearch://reference/query-languages/esql/functions-operators/operators.md#esql-equals) -) and pattern match ([`LIKE`](elasticsearch://reference/query-languages/esql/functions-operators/operators.md#esql-like)) operators to detect PowerShell processes spawned by Word -2. Enriches process data with asset inventory using [`LOOKUP JOIN`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join) to understand which business systems are affected -3. Adds user context to identify the compromised user account and their privileges -4. Creates a boolean field using [`EVAL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-eval) and [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) to detect base64-encoded PowerShell commands (`-enc` parameter), which attackers use to obfuscate malicious code -5. Filters for only encoded commands using [`WHERE`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-where) on the computed field - focusing on suspicious PowerShell usage -6. Aggregates results with [`STATS`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-stats-by) and [`COUNT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count) grouped `BY` multiple fields +1. Uses [`WHERE`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-where) with [`==`](elasticsearch://reference/query-languages/esql/functions-operators/operators.md#esql-equals) and [`LIKE`](elasticsearch://reference/query-languages/esql/functions-operators/operators.md#esql-like) operators to detect PowerShell processes +2. Enriches using [`LOOKUP JOIN`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join) with asset inventory +3. Enriches with user context using `LOOKUP JOIN` +4. Uses [`EVAL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-eval) and [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) to detect encoded commands +5. Additional filtering with [`WHERE`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-where) +6. Aggregates results with [`STATS`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-stats-by) and [`COUNT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count) grouped by multiple fields **Response** @@ -365,11 +364,11 @@ BY user.name <3> | LIMIT 1000 ``` -1. Filters for successful remote logons (type 3) - key indicator for lateral movement detection -2. Creates 30-minute time buckets to analyze temporal patterns in authentication activity -3. Complex aggregation combining host uniqueness, criticality levels, and timing metrics -4. Calculates attack duration using [`DATE_DIFF`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_diff) to measure campaign timespan -5. Custom scoring formula weighing both breadth (systems accessed) and depth (criticality) of compromise +1. Uses [`WHERE`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-where) for basic authentication filtering +2. Creates time buckets with [`DATE_TRUNC`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_trunc) for temporal analysis +3. Uses [`STATS`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-stats-by) with [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) for comprehensive access metrics +4. Uses [`DATE_DIFF`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_diff) for duration calculations +5. Uses [`EVAL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-eval) with [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case)for risk scoring **Response** @@ -407,11 +406,11 @@ BY host.name, destination.ip, threat.name, asset.criticality | LIMIT 1000 ``` -1. Uses [`CIDR_MATCH`](elasticsearch://reference/query-languages/esql/functions-operators/ip-functions.md#esql-cidr_match) function with multiple CIDR ranges to efficiently filter out internal IP ranges at the IP type level -2. Standardizes IP format for threat intel lookup join by casting to string type -3. Uses [`DATE_DIFF`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_diff) to calculate duration of data transfer activities in hours -4. Converts raw bytes to megabytes using division and [`ROUND`](elasticsearch://reference/query-languages/esql/functions-operators/math-functions.md#esql-round) for human-readable values -5. Assigns risk scores based on asset criticality and data volume using nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) conditions +1. Uses [`CIDR_MATCH`](elasticsearch://reference/query-languages/esql/functions-operators/ip-functions.md#esql-cidr_match) to filter internal IP ranges for external data transfer detection +2. Uses [`TO_STRING`](elasticsearch://reference/query-languages/esql/functions-operators/type-conversion-functions.md#esql-to_string) to standardize IP format for threat intel lookups +3. Uses [`DATE_DIFF`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_diff) with `SUM` and `COUNT` to measure data transfer volume over time +4. Uses [`ROUND`](elasticsearch://reference/query-languages/esql/functions-operators/math-functions.md#esql-round) for human-readable values +5. Uses [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) for risk scoring based on asset criticality and size of data transferred **Response** @@ -429,7 +428,7 @@ To understand the attack progression, we need to build a timeline of events acro ```esql -FROM windows-security-logs, process-logs, network-logs +FROM windows-security-logs, process-logs, network-logs <1> | LOOKUP JOIN asset-inventory ON host.name | LOOKUP JOIN user-context ON user.name | WHERE user.name == "jsmith" OR user.name == "admin" @@ -437,7 +436,7 @@ FROM windows-security-logs, process-logs, network-logs event.code IS NOT NULL, "Authentication", process.name IS NOT NULL, "Process Execution", destination.ip IS NOT NULL, "Network Activity", - "Unknown") + "Unknown") <2> | EVAL dest_ip = TO_STRING(destination.ip) | EVAL attack_stage = CASE( process.parent.name LIKE "*word*", "Initial Compromise", @@ -445,17 +444,16 @@ FROM windows-security-logs, process-logs, network-logs event.code == "4624" AND logon.type == "3", "Lateral Movement", process.name IN ("sqlcmd.exe", "ntdsutil.exe"), "Data Access", dest_ip NOT LIKE "10.*", "Exfiltration", - "Other") -| SORT @timestamp ASC + "Other") <3> +| SORT @timestamp ASC <4> | KEEP @timestamp, event_type, attack_stage, host.name, asset.criticality, user.name, process.name, destination.ip | LIMIT 1000 ``` -1. Querying multiple indices simultaneously provides comprehensive event correlation -2. Nested [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) with [`IS NOT NULL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#null-predicates) checks categorizes events by their source data structure -3. Complex conditional logic maps technical events to attack framework stages ([MITRE ATT&CK](https://attack.mitre.org/)) for better understanding of the attack lifecycle -4. `SORT ASC` uses the [`SORT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-sort) command to order events by timestamp, which creates chronological ordering essential for timeline analysis - +1. Uses `FROM` with multiple indices for comprehensive correlation +2. Uses [`IS NOT NULL`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#null-predicates) with [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) to classify event types from different data sources +3. Uses complex [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) logic to map events to MITRE ATT&CK stages +4. Uses [`SORT`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-sort) to build chronological attack timeline **Response** The response provides a chronological timeline of events, showing the attacker's actions and the impact on the organization. @@ -503,11 +501,10 @@ BY user.name, user.department | LIMIT 1000 ``` -1. Filters for common system administration and reconnaissance tools -2. [`LOOKUP JOIN`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join) enriches process data with user and asset context -3. [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) functions measure the breadth of tool usage across systems -4. [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) statement categorizes usage patterns for easier analysis -5. Groups by user and department to identify anomalous behavior patterns +1. Uses `WHERE...IN` to monitor high-risk system tools +2. Uses [`LOOKUP JOIN`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-lookup-join) with `asset-inventory` and `user-context` indices to enrich events with context +3. Uses [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) to measure breadth of suspicious tool usage +4. Uses [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case)to classify usage patterns for anomaly detection **Response** @@ -540,11 +537,10 @@ BY user.name, host.name, asset.criticality | LIMIT 1000 ``` -1. Uses [match operator (`:`)](elasticsearch://reference/query-languages/esql/functions-operators/operators.md#esql-match-operator) in the `WHERE` clause with a wildcard search on the `process.command_line` field -to find scheduled task creation commands, providing more flexible matching than exact string comparison -2. Creates hourly time buckets with [`DATE_TRUNC`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_trunc) to analyze temporal patterns in persistence activity -3. Measures temporal dispersion using [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) on time buckets to detect scheduled tasks created across multiple time periods -4. Creates meaningful categorization of persistence patterns using [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) to identify potentially malicious task scheduling sequences (burst creation vs spread across time) +1. Uses [`WHERE`](elasticsearch://reference/query-languages/esql/commands/processing-commands.md#esql-where) with [`:`](elasticsearch://reference/query-languages/esql/functions-operators/operators.md#esql-match-operator) match operator to detect scheduled task creation (a common persistence mechanism) +2. Uses [`DATE_TRUNC`](elasticsearch://reference/query-languages/esql/functions-operators/date-time-functions.md#esql-date_trunc) to group events into hourly time buckets for temporal analysis +3. Uses [`COUNT_DISTINCT`](elasticsearch://reference/query-languages/esql/functions-operators/aggregation-functions.md#esql-count_distinct) with `time_bucket` to measure task creation velocity +4. Uses [`CASE`](elasticsearch://reference/query-languages/esql/functions-operators/conditional-functions-and-expressions.md#esql-case) to classify suspicious patterns based on timing and frequency **Response** From bc9c716ca02e36d9a49fbee3b064b0c9644838dc Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Fri, 20 Jun 2025 11:54:25 +0200 Subject: [PATCH 17/20] added comprehensive esql for security docs links, improved navigation between pages --- solutions/security/esql-for-security.md | 16 ++++++++++++++-- .../esql-threat-hunting-tutorial.md | 6 +++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/solutions/security/esql-for-security.md b/solutions/security/esql-for-security.md index 77f82111f1..d395ced91d 100644 --- a/solutions/security/esql-for-security.md +++ b/solutions/security/esql-for-security.md @@ -9,6 +9,18 @@ products: # {{esql}} for security use cases -Use the following resources to get hands-on with the [Elasticsearch Query Language ({{esql}})](elasticsearch://reference/query-languages/esql.md) for cybersecurity use cases: +Use the following resources to get hands-on with the [Elasticsearch Query Language ({{esql}})](elasticsearch://reference/query-languages/esql.md) for cybersecurity use cases. -- [Tutorial: Threat hunting with {{esql}}](esql-for-security/esql-threat-hunting-tutorial.md): Learn how to use {{esql}} to hunt for threats in your data. \ No newline at end of file +## Tutorials + +- [Threat hunting with {{esql}}](esql-for-security/esql-threat-hunting-tutorial.md): Learn how to use {{esql}} to hunt for threats in your data. + +## Documentation + +To learn more about where you can use {{esql}} in {{elastic-sec}}, refer to the following pages: + +- Learn more about the [{{esql}}](elasticsearch://reference/query-languages/esql.md) language in the reference documentation. +- Learn how to use the AI Assistant to [generate, customize, and understand {{esql}} queries](/solutions/security/ai/generate-customize-learn-about-esorql-queries.md) +- Learn how to use {{esql}} to investigate events in [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline). +- Learn how to [create detection rules using {{esql}}](/solutions/security/detect-and-alert/create-detection-rule.md#create-esql-rule) +- Learn how to [convert Splunk SPL rules to {{esql}}](/solutions/security/get-started/automatic-migration.md) with Automatic Migration \ No newline at end of file diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index e8937a8234..964498be1b 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -560,4 +560,8 @@ The response shows the number of task creations, creation hours, and persistence - [Detecting command and scripting interpreter techniques](https://www.elastic.co/blog/detecting-command-scripting-interpreter) - [Detecting credential dumping with {{elastic-sec}}](https://www.elastic.co/blog/elastic-security-detecting-credential-dumping) - [Detecting covert data exfiltration techniques](https://www.elastic.co/blog/elastic-security-detecting-covert-data-exfiltration) -- Learn more about the [{{esql}}](elasticsearch://reference/query-languages/esql.md) language in the reference documentation. \ No newline at end of file + + +:::{tip} +To learn how you can use {{esql}} in various {{elastic-sec}} contexts, refer to [the overview](/solutions/security/esql-for-security#documentation). +::: \ No newline at end of file From 471ff5d5a0fc258adf0cee35d3601120e23163af Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Fri, 20 Jun 2025 12:06:55 +0200 Subject: [PATCH 18/20] i could spend all day tweaking this --- solutions/security/esql-for-security.md | 11 +++++------ .../esql-for-security/esql-threat-hunting-tutorial.md | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/solutions/security/esql-for-security.md b/solutions/security/esql-for-security.md index d395ced91d..3a258d457b 100644 --- a/solutions/security/esql-for-security.md +++ b/solutions/security/esql-for-security.md @@ -17,10 +17,9 @@ Use the following resources to get hands-on with the [Elasticsearch Query Langua ## Documentation -To learn more about where you can use {{esql}} in {{elastic-sec}}, refer to the following pages: +Learn how to: -- Learn more about the [{{esql}}](elasticsearch://reference/query-languages/esql.md) language in the reference documentation. -- Learn how to use the AI Assistant to [generate, customize, and understand {{esql}} queries](/solutions/security/ai/generate-customize-learn-about-esorql-queries.md) -- Learn how to use {{esql}} to investigate events in [Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline). -- Learn how to [create detection rules using {{esql}}](/solutions/security/detect-and-alert/create-detection-rule.md#create-esql-rule) -- Learn how to [convert Splunk SPL rules to {{esql}}](/solutions/security/get-started/automatic-migration.md) with Automatic Migration \ No newline at end of file +- [Generate and understand {{esql}} queries](/solutions/security/ai/generate-customize-learn-about-esorql-queries.md) using the AI Assistant +- [Investigate events in Timeline](/solutions/security/investigate/timeline.md#esql-in-timeline) using {{esql}} +- [Create detection rules](/solutions/security/detect-and-alert/create-detection-rule.md#create-esql-rule) using {{esql}} +- [Convert Splunk SPL rules to {{esql}}](/solutions/security/get-started/automatic-migration.md) with Automatic Migration \ No newline at end of file diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index 964498be1b..576db12424 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -563,5 +563,5 @@ The response shows the number of task creations, creation hours, and persistence :::{tip} -To learn how you can use {{esql}} in various {{elastic-sec}} contexts, refer to [the overview](/solutions/security/esql-for-security#documentation). +To learn where you can use {{esql}} in {{elastic-sec}} contexts, refer to [the overview](/solutions/security/esql-for-security#documentation). ::: \ No newline at end of file From ceb51d5bf82eb1037d0e572fa951563e1bcc871d Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Fri, 20 Jun 2025 12:07:20 +0200 Subject: [PATCH 19/20] fix link --- .../security/esql-for-security/esql-threat-hunting-tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md index 576db12424..49d49274aa 100644 --- a/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md +++ b/solutions/security/esql-for-security/esql-threat-hunting-tutorial.md @@ -563,5 +563,5 @@ The response shows the number of task creations, creation hours, and persistence :::{tip} -To learn where you can use {{esql}} in {{elastic-sec}} contexts, refer to [the overview](/solutions/security/esql-for-security#documentation). +To learn where you can use {{esql}} in {{elastic-sec}} contexts, refer to [the overview](/solutions/security/esql-for-security.md#documentation). ::: \ No newline at end of file From cf5600557151a5d5ddb1e2377e4c112932eae7c7 Mon Sep 17 00:00:00 2001 From: Liam Thompson Date: Fri, 20 Jun 2025 13:06:54 +0200 Subject: [PATCH 20/20] tweak overview sentence --- solutions/security/esql-for-security.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solutions/security/esql-for-security.md b/solutions/security/esql-for-security.md index 3a258d457b..3e2d0900df 100644 --- a/solutions/security/esql-for-security.md +++ b/solutions/security/esql-for-security.md @@ -9,7 +9,7 @@ products: # {{esql}} for security use cases -Use the following resources to get hands-on with the [Elasticsearch Query Language ({{esql}})](elasticsearch://reference/query-languages/esql.md) for cybersecurity use cases. +Use the following resources to learn about using the [Elasticsearch Query Language ({{esql}})](elasticsearch://reference/query-languages/esql.md) for cybersecurity use cases. ## Tutorials