diff --git a/packages/pps/_dev/deploy/docker/sample_logs/log.log b/packages/pps/_dev/deploy/docker/sample_logs/log.log index 1be8b1a3a2a..f4c72556396 100644 --- a/packages/pps/_dev/deploy/docker/sample_logs/log.log +++ b/packages/pps/_dev/deploy/docker/sample_logs/log.log @@ -7,4 +7,7 @@ <134>Jan 23 13:47:25 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Error - Identity Not Verified - User failed to verify themselves 127.0.0.1 23/01 13:47:25.593 <134>Jan 23 13:47:25 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Error - Sign-in Failed - User sign-in denied 127.0.0.1 23/01 13:47:25.641 <134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Created - User created entry as a duplicate 127.0.0.1 23/01 14:05:54.404 -<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Duplicated - User duplicated entry 127.0.0.1 23/01 14:05:54.450 +<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Duplicated - User duplicated entry 127.0.0.1 23/01 14:05:54.450 +<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Updated - User updated entry changing the name from to changing the notes +<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Updated - User updated entry changing the name from to changing the username from to changing the password changing the notes changing the expiry date from <> to <2027-09-23 00:00> + <134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Updated - User updated entry changing the username from to changing the password changing the expiry date from <> to <2027-10-02 08:00> \ No newline at end of file diff --git a/packages/pps/changelog.yml b/packages/pps/changelog.yml index 2d72899ce98..906cd54e973 100644 --- a/packages/pps/changelog.yml +++ b/packages/pps/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.1.0" + changes: + - description: Improved field extraction. + type: enhancement + link: https://github.com/elastic/integrations/pull/15666 - version: "1.0.1" changes: - description: Remove duplicated installation instructions from the documentation. diff --git a/packages/pps/data_stream/log/_dev/test/pipeline/test-log.log b/packages/pps/data_stream/log/_dev/test/pipeline/test-log.log index a0bb3c706ef..8384012ba51 100644 --- a/packages/pps/data_stream/log/_dev/test/pipeline/test-log.log +++ b/packages/pps/data_stream/log/_dev/test/pipeline/test-log.log @@ -7,4 +7,7 @@ <134>Jan 23 13:47:25 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Error - Identity Not Verified - User failed to verify themselves 127.0.0.1 23/01 13:47:25.593 <134>Jan 23 13:47:25 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Error - Sign-in Failed - User sign-in denied 127.0.0.1 23/01 13:47:25.641 <134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Created - User created entry as a duplicate 127.0.0.1 23/01 14:05:54.404 -<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Duplicated - User duplicated entry 127.0.0.1 23/01 14:05:54.450 \ No newline at end of file +<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Duplicated - User duplicated entry 127.0.0.1 23/01 14:05:54.450 +<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Updated - User updated entry changing the name from to changing the notes +<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Updated - User updated entry changing the name from to changing the username from to changing the password changing the notes changing the expiry date from <> to <2027-09-23 00:00> +<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Updated - User updated entry changing the username from to changing the password changing the expiry date from <> to <2027-10-02 08:00> \ No newline at end of file diff --git a/packages/pps/data_stream/log/_dev/test/pipeline/test-log.log-expected.json b/packages/pps/data_stream/log/_dev/test/pipeline/test-log.log-expected.json index 2c6d1d9a43e..a05ec9f3387 100644 --- a/packages/pps/data_stream/log/_dev/test/pipeline/test-log.log-expected.json +++ b/packages/pps/data_stream/log/_dev/test/pipeline/test-log.log-expected.json @@ -12,7 +12,8 @@ "created": "2025-01-23T09:49:10.000Z", "kind": "event", "original": "<134>Jan 23 09:49:10 SRV-PPS-001 Pleasant Password Server:192.168.1.2 - user@name.test - - Success - Syslog Settings Changed - User Syslogging setting updated changing the host from to <127.0.0.1> changing the port fr\t127.0.0.1\t23/01 09:49:10.894\t", - "outcome": "success" + "outcome": "success", + "reason": "Syslog Settings Changed" }, "host": { "hostname": "SRV-PPS-001" @@ -22,7 +23,7 @@ "priority": 134 } }, - "message": "Syslog Settings Changed - User Syslogging setting updated changing the host from to <127.0.0.1> changing the port fr\t127.0.0.1\t23/01 09:49:10.894\t", + "message": "User Syslogging setting updated changing the host from to <127.0.0.1> changing the port fr\t127.0.0.1\t23/01 09:49:10.894\t", "tags": [ "preserve_original_event" ], @@ -44,7 +45,8 @@ "created": "2025-01-23T11:32:57.000Z", "kind": "event", "original": "<134>Jan 23 11:32:57 SRV-PPS-001 Pleasant Password Server:192.168.1.2 - user@name.test - - Success - Password Fetched - User fetched the password for - test\t127.0.0.1\t23/01 11:32:57.857\t", - "outcome": "success" + "outcome": "success", + "reason": "Password Fetched" }, "host": { "hostname": "SRV-PPS-001" @@ -54,7 +56,13 @@ "priority": 134 } }, - "message": "Password Fetched - User fetched the password for - test\t127.0.0.1\t23/01 11:32:57.857\t", + "message": "User fetched the password for - test\t127.0.0.1\t23/01 11:32:57.857\t", + "pps": { + "entry": { + "path": "TOP/SECRET/PASSWORD", + "reason": "test\t127.0.0.1\t23/01 11:32:57.857\t" + } + }, "tags": [ "preserve_original_event" ], @@ -76,7 +84,8 @@ "created": "2025-01-23T12:20:07.000Z", "kind": "event", "original": "<134>Jan 23 12:20:07 SRV-PPS-001 Pleasant Password Server:0.0.0.0 - Backup Restore Service - - Success - Backup Occurred - User backing up database to backing up database to backing up database to Jan 23 12:37:37 SRV-PPS-001 Pleasant Password Server:192.168.1.1 - user@name.test - - Success - Session Log On - User logged on\t127.0.0.1\t23/01 12:37:37.346", - "outcome": "success" + "outcome": "success", + "reason": "Session Log On" }, "host": { "hostname": "SRV-PPS-001" @@ -113,7 +123,7 @@ "priority": 134 } }, - "message": "Session Log On - User logged on\t127.0.0.1\t23/01 12:37:37.346", + "message": "User logged on\t127.0.0.1\t23/01 12:37:37.346", "tags": [ "preserve_original_event" ], @@ -135,7 +145,8 @@ "created": "2025-01-23T12:38:07.000Z", "kind": "event", "original": "<134>Jan 23 12:38:07 SRV-PPS-001 Pleasant Password Server:192.168.1.1 - user@name.test - - Success - Entry Updated - User updated entry changing the password\t127.0.0.1\t23/01 12:38:07.629\t", - "outcome": "success" + "outcome": "success", + "reason": "Entry Updated" }, "host": { "hostname": "SRV-PPS-001" @@ -145,7 +156,12 @@ "priority": 134 } }, - "message": "Entry Updated - User updated entry changing the password\t127.0.0.1\t23/01 12:38:07.629\t", + "message": "User updated entry changing the password\t127.0.0.1\t23/01 12:38:07.629\t", + "pps": { + "entry": { + "path": "TOP/SECRET/PASSWORD" + } + }, "tags": [ "preserve_original_event" ], @@ -167,7 +183,8 @@ "created": "2025-01-23T13:43:47.000Z", "kind": "event", "original": "<134>Jan 23 13:43:47 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Identity Verified - User verified via ApplicationBasicOAuth\t127.0.0.1\t23/01 13:43:47.422\t", - "outcome": "success" + "outcome": "success", + "reason": "Identity Verified" }, "host": { "hostname": "SRV-PPS-001" @@ -177,7 +194,7 @@ "priority": 134 } }, - "message": "Identity Verified - User verified via ApplicationBasicOAuth\t127.0.0.1\t23/01 13:43:47.422\t", + "message": "User verified via ApplicationBasicOAuth\t127.0.0.1\t23/01 13:43:47.422\t", "tags": [ "preserve_original_event" ], @@ -199,7 +216,8 @@ "created": "2025-01-23T13:47:25.000Z", "kind": "event", "original": "<134>Jan 23 13:47:25 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Error - Identity Not Verified - User failed to verify themselves\t127.0.0.1\t23/01 13:47:25.593\t", - "outcome": "failure" + "outcome": "failure", + "reason": "Identity Not Verified" }, "host": { "hostname": "SRV-PPS-001" @@ -209,7 +227,7 @@ "priority": 134 } }, - "message": "Identity Not Verified - User failed to verify themselves\t127.0.0.1\t23/01 13:47:25.593\t", + "message": "User failed to verify themselves\t127.0.0.1\t23/01 13:47:25.593\t", "tags": [ "preserve_original_event" ], @@ -231,7 +249,8 @@ "created": "2025-01-23T13:47:25.000Z", "kind": "event", "original": "<134>Jan 23 13:47:25 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Error - Sign-in Failed - User sign-in denied\t127.0.0.1\t23/01 13:47:25.641\t", - "outcome": "failure" + "outcome": "failure", + "reason": "Sign-in Failed" }, "host": { "hostname": "SRV-PPS-001" @@ -241,7 +260,7 @@ "priority": 134 } }, - "message": "Sign-in Failed - User sign-in denied\t127.0.0.1\t23/01 13:47:25.641\t", + "message": "User sign-in denied\t127.0.0.1\t23/01 13:47:25.641\t", "tags": [ "preserve_original_event" ], @@ -263,7 +282,8 @@ "created": "2025-01-23T14:05:54.000Z", "kind": "event", "original": "<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Created - User created entry as a duplicate\t127.0.0.1\t23/01 14:05:54.404\t", - "outcome": "success" + "outcome": "success", + "reason": "Entry Created" }, "host": { "hostname": "SRV-PPS-001" @@ -273,7 +293,12 @@ "priority": 134 } }, - "message": "Entry Created - User created entry as a duplicate\t127.0.0.1\t23/01 14:05:54.404\t", + "message": "User created entry as a duplicate\t127.0.0.1\t23/01 14:05:54.404\t", + "pps": { + "entry": { + "path": "TOP/SECRET/PASSWORD" + } + }, "tags": [ "preserve_original_event" ], @@ -294,8 +319,9 @@ "event": { "created": "2025-01-23T14:05:54.000Z", "kind": "event", - "original": "<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Duplicated - User duplicated entry \t127.0.0.1\t23/01 14:05:54.450\t", - "outcome": "success" + "original": "<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Duplicated - User duplicated entry \t127.0.0.1\t23/01 14:05:54.450", + "outcome": "success", + "reason": "Entry Duplicated" }, "host": { "hostname": "SRV-PPS-001" @@ -305,7 +331,140 @@ "priority": 134 } }, - "message": "Entry Duplicated - User duplicated entry \t127.0.0.1\t23/01 14:05:54.450\t", + "message": "User duplicated entry \t127.0.0.1\t23/01 14:05:54.450", + "pps": { + "entry": { + "path": "TOP/SECRET/PASSWORD" + } + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "domain": "name.test", + "email": "user@name.test", + "name": "user" + } + }, + { + "@timestamp": "2025-01-23T14:05:54.000Z", + "client": { + "ip": "192.168.1.3" + }, + "ecs": { + "version": "8.17.0" + }, + "event": { + "created": "2025-01-23T14:05:54.000Z", + "kind": "event", + "original": "<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Updated - User updated entry changing the name from to changing the notes", + "outcome": "success", + "reason": "Entry Updated" + }, + "host": { + "hostname": "SRV-PPS-001" + }, + "log": { + "syslog": { + "priority": 134 + } + }, + "message": "User updated entry changing the name from to changing the notes", + "pps": { + "entry": { + "name": "PASSWORD", + "path": "TOP/SECRET/PASSWORD", + "target": { + "name": "PASSWORD2" + } + } + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "domain": "name.test", + "email": "user@name.test", + "name": "user" + } + }, + { + "@timestamp": "2025-01-23T14:05:54.000Z", + "client": { + "ip": "192.168.1.3" + }, + "ecs": { + "version": "8.17.0" + }, + "event": { + "created": "2025-01-23T14:05:54.000Z", + "kind": "event", + "original": "<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Updated - User updated entry changing the name from to changing the username from to changing the password changing the notes changing the expiry date from <> to <2027-09-23 00:00>", + "outcome": "success", + "reason": "Entry Updated" + }, + "host": { + "hostname": "SRV-PPS-001" + }, + "log": { + "syslog": { + "priority": 134 + } + }, + "message": "User updated entry changing the name from to changing the username from to changing the password changing the notes changing the expiry date from <> to <2027-09-23 00:00>", + "pps": { + "entry": { + "name": "PASSWORD", + "path": "TOP/SECRET/PASSWORD", + "target": { + "name": "PASSWORD2", + "username": "entry_username2" + }, + "username": "entry_username" + } + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "domain": "name.test", + "email": "user@name.test", + "name": "user" + } + }, + { + "@timestamp": "2025-01-23T14:05:54.000Z", + "client": { + "ip": "192.168.1.3" + }, + "ecs": { + "version": "8.17.0" + }, + "event": { + "created": "2025-01-23T14:05:54.000Z", + "kind": "event", + "original": "<134>Jan 23 14:05:54 SRV-PPS-001 Pleasant Password Server:192.168.1.3 - user@name.test - - Success - Entry Updated - User updated entry changing the username from to changing the password changing the expiry date from <> to <2027-10-02 08:00>", + "outcome": "success", + "reason": "Entry Updated" + }, + "host": { + "hostname": "SRV-PPS-001" + }, + "log": { + "syslog": { + "priority": 134 + } + }, + "message": "User updated entry changing the username from to changing the password changing the expiry date from <> to <2027-10-02 08:00>", + "pps": { + "entry": { + "path": "TOP/SECRET/PASSWORD", + "target": { + "username": "entry_username2" + }, + "username": "entry_username" + } + }, "tags": [ "preserve_original_event" ], diff --git a/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 6ae25cc5338..6095fe0d1b3 100644 --- a/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/pps/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -15,9 +15,30 @@ processors: - grok: field: event.original patterns: - - '^<%{NUMBER:log.syslog.priority:long}>%{SYSLOGTIMESTAMP:event.created}\s+%{NOTSPACE:host.hostname}\s%{DATA}:%{IP:client.ip}\s-\s%{USERNAME:user.name}@%{DATA:user.domain}\s%{DATA}(?(Success)|(Error))\s-\s%{GREEDYDATA:message}' - - '^<%{NUMBER:log.syslog.priority:long}>%{SYSLOGTIMESTAMP:event.created}\s+%{NOTSPACE:host.domain}\s%{DATA}:%{IP:client.ip}%{DATA}(?(Success)|(Error))\s-\s%{GREEDYDATA:message}' + - '^<%{NUMBER:log.syslog.priority:long}>%{SYSLOGTIMESTAMP:event.created}\s+%{NOTSPACE:host.hostname}\s%{DATA}:%{IP:client.ip}\s-\s%{USERNAME:user.name}@%{DATA:user.domain}\s%{DATA}(?(Success)|(Error))\s-\s%{DATA:event.reason}\s-\s%{GREEDYDATA:message}' + - '^<%{NUMBER:log.syslog.priority:long}>%{SYSLOGTIMESTAMP:event.created}\s+%{NOTSPACE:host.domain}\s%{DATA}:%{IP:client.ip}%{DATA}(?(Success)|(Error))\s-\s%{DATA:event.reason}\s-\s%{GREEDYDATA:message}' - '^%{GREEDYDATA:message}$' + - grok: + field: message + ignore_failure: true + ignore_missing: true + patterns: + - backing\sup\sdatabase\sto\s<%{DATA:pps.entry.path}> + - '[created|updated]\sentry\s<%{DATA:pps.entry.path}>' + - fetched\sthe\spassword\sfor\s<%{DATA:pps.entry.path}>$ + - fetched\sthe\spassword\sfor\s<%{DATA:pps.entry.path}>\s-\s%{DATA:pps.entry.reason}$ + - on\sentry\s<%{DATA:pps.entry.path}>\sfor\suser + - moved\sentry\s<%{DATA:pps.entry.path}>\sto\s<%{DATA:pps.entry.target.path}> + - created\sfolder\s<%{DATA:pps.entry.path}>$ + - comment\srequirement\s<.*>\s[from|to]\s<%{DATA:pps.entry.path}>$ + - notification\s.*>\s[from|to]\s<%{DATA:pps.entry.path}>$ + - updated\sentry\s<%{DATA:pps.entry.path}>\schanging\sthe\sname\sfrom\s<%{DATA:pps.entry.name}>\sto\s<%{DATA:pps.entry.target.name}> + - grok: + field: message + ignore_failure: true + ignore_missing: true + patterns: + - \schanging\sthe\susername\sfrom\s<%{DATA:pps.entry.username}>\sto\s<%{DATA:pps.entry.target.username}> # Set the Event Outcome to Lower Case to be ECS Compliant - lowercase: field: event.outcome diff --git a/packages/pps/data_stream/log/fields/fields.yml b/packages/pps/data_stream/log/fields/fields.yml new file mode 100644 index 00000000000..71294ad996e --- /dev/null +++ b/packages/pps/data_stream/log/fields/fields.yml @@ -0,0 +1,18 @@ +- name: pps.entry.path + type: keyword + description: Password Path +- name: pps.entry.reason + type: keyword + description: Reason the user interacted with a password +- name: pps.entry.name + type: keyword + description: The name of an entry in the password manager +- name: pps.entry.target.name + type: keyword + description: The new name of an entry in the password manager if it was changed +- name: pps.entry.username + type: keyword + description: The username of an entry in the password manager +- name: pps.entry.target.username + type: keyword + description: The new username of an entry in the password manager if it was changed diff --git a/packages/pps/docs/README.md b/packages/pps/docs/README.md index f83a5863bc3..2faacef44af 100644 --- a/packages/pps/docs/README.md +++ b/packages/pps/docs/README.md @@ -135,4 +135,10 @@ An example event for `log` looks as following: | input.type | Input type | keyword | | log.offset | Log offset | long | | log.source.address | Log source address | keyword | +| pps.entry.name | The name of an entry in the password manager | keyword | +| pps.entry.path | Password Path | keyword | +| pps.entry.reason | Reason the user interacted with a password | keyword | +| pps.entry.target.name | The new name of an entry in the password manager if it was changed | keyword | +| pps.entry.target.username | The new username of an entry in the password manager if it was changed | keyword | +| pps.entry.username | The username of an entry in the password manager | keyword |