[cisco_duo] Fix bad value in ip and date fields (#3700)

* Handle bad values in IP and Date fields

* Updated the changelog file

* Updated set to append processor in on_failure

Author: vinit-chauhan

packages/cisco_duo/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.4.0"
+  changes:
+    - description: Added support to handle bad values in ip and date fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/3700
 - version: "1.3.0"
   changes:
     - description: Update package to ECS 8.3.0.

packages/cisco_duo/data_stream/admin/elasticsearch/ingest_pipeline/default.yml

@@ -24,9 +24,13 @@ processors:
   - date:
       field: json.timestamp
       target_field: "@timestamp"
-      ignore_failure: true
+      if: ctx.json?.timestamp != null
       formats:
         - UNIX
+      on_failure:
+        - append:
+            field: error.message
+            value: "{{{_ingest.on_failure_message}}}"
   - set:
       field: event.category
       value: iam
@@ -150,6 +154,6 @@ processors:
       ignore_failure: true
       ignore_missing: true
 on_failure:
-  - set:
+  - append:
       field: error.message
       value: "{{{_ingest.on_failure_message}}}"

packages/cisco_duo/data_stream/admin/sample_event.json

@@ -1,11 +1,11 @@
 {
     "@timestamp": "2021-07-20T11:41:31.000Z",
     "agent": {
-        "ephemeral_id": "d5c469ec-2802-48c4-9828-95a1a38a3d57",
-        "id": "7cefd7f8-53e3-4884-ab65-da99d71b166f",
+        "ephemeral_id": "7c7523b2-666f-4792-812c-a5697a8929b9",
+        "id": "c934978b-c8c9-4484-8fbe-007cc0ace376",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.0.0-beta1"
+        "version": "8.2.0"
     },
     "cisco_duo": {
         "admin": {
@@ -24,16 +24,16 @@
         "version": "8.3.0"
     },
     "elastic_agent": {
-        "id": "7cefd7f8-53e3-4884-ab65-da99d71b166f",
+        "id": "c934978b-c8c9-4484-8fbe-007cc0ace376",
         "snapshot": false,
-        "version": "8.0.0-beta1"
+        "version": "8.2.0"
     },
     "event": {
         "action": "activation_begin",
         "agent_id_status": "verified",
-        "created": "2021-12-29T09:39:10.869Z",
+        "created": "2022-07-14T12:20:01.540Z",
         "dataset": "cisco_duo.admin",
-        "ingested": "2021-12-29T09:39:11Z",
+        "ingested": "2022-07-14T12:20:02Z",
         "kind": "event",
         "original": "{\"action\":\"activation_begin\",\"description\":\"Starting activation process\",\"isotimestamp\":\"2021-07-20T11: 41: 31+00: 00\",\"object\":null,\"timestamp\":1626781291,\"username\":\"narroway\"}",
         "outcome": "success",

packages/cisco_duo/data_stream/auth/elasticsearch/ingest_pipeline/default.yml

@@ -21,9 +21,13 @@ processors:
   - date:
       field: json.timestamp
       target_field: "@timestamp"
-      ignore_failure: true
+      if: ctx.json?.timestamp != null
       formats:
         - UNIX
+      on_failure:
+        - append:
+            field: error.message
+            value: "{{{_ingest.on_failure_message}}}"
   - set:
       field: event.category
       value: authentication
@@ -64,10 +68,22 @@ processors:
       field: json.access_device.ip
       type: ip
       ignore_missing: true
+      on_failure:
+        - remove:
+            field: json.access_device.ip
+        - append:
+            field: error.message
+            value: "{{{_ingest.on_failure_message}}}"
   - convert:
       field: json.access_device.port
       type: long
       ignore_missing: true
+      on_failure:
+        - remove:
+            field: json.access_device.port
+        - append:
+            field: error.message
+            value: "{{{_ingest.on_failure_message}}}"
   - set: 
       field: source.ip
       copy_from: json.access_device.ip
@@ -93,10 +109,22 @@ processors:
       field: json.auth_device.ip
       type: ip
       ignore_missing: true
+      on_failure:
+        - remove:
+            field: json.auth_device.ip
+        - append:
+            field: error.message
+            value: "{{{_ingest.on_failure_message}}}"
   - convert:
       field: json.auth_device.port
       type: long
       ignore_missing: true
+      on_failure:
+        - remove:
+            field: json.auth_device.port
+        - append:
+            field: error.message
+            value: "{{{_ingest.on_failure_message}}}"
   - set: 
       field: source.address
       copy_from: json.access_device.hostname
@@ -359,6 +387,6 @@ processors:
       ignore_failure: true
       ignore_missing: true
 on_failure:
-  - set:
+  - append:
       field: error.message
       value: "{{{_ingest.on_failure_message}}}"

packages/cisco_duo/data_stream/auth/sample_event.json

@@ -1,8 +1,8 @@
 {
     "@timestamp": "2020-02-13T18:56:20.000Z",
     "agent": {
-        "ephemeral_id": "0abea523-d85f-42ed-b1a3-8a4c84fa68e1",
-        "id": "9c2175d9-ba8c-4169-b98d-dfcbc2a7bda3",
+        "ephemeral_id": "ba2543c1-a84f-4f56-ae2f-cfecef7ef0e4",
+        "id": "c934978b-c8c9-4484-8fbe-007cc0ace376",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.2.0"
@@ -53,16 +53,16 @@
         "version": "8.3.0"
     },
     "elastic_agent": {
-        "id": "9c2175d9-ba8c-4169-b98d-dfcbc2a7bda3",
+        "id": "c934978b-c8c9-4484-8fbe-007cc0ace376",
         "snapshot": false,
         "version": "8.2.0"
     },
     "event": {
         "agent_id_status": "verified",
         "category": "authentication",
-        "created": "2022-05-31T11:02:38.919Z",
+        "created": "2022-07-14T12:20:57.028Z",
         "dataset": "cisco_duo.auth",
-        "ingested": "2022-05-31T11:02:40Z",
+        "ingested": "2022-07-14T12:20:58Z",
         "kind": "event",
         "original": "{\"access_device\":{\"browser\":\"Chrome\",\"browser_version\":\"67.0.3396.99\",\"flash_version\":\"uninstalled\",\"hostname\":null,\"ip\":\"89.160.20.156\",\"is_encryption_enabled\":true,\"is_firewall_enabled\":true,\"is_password_set\":true,\"java_version\":\"uninstalled\",\"location\":{\"city\":\"Ann Arbor\",\"country\":\"United States\",\"state\":\"Michigan\"},\"os\":\"Mac OS X\",\"os_version\":\"10.14.1\",\"security_agents\":null},\"alias\":\"\",\"application\":{\"key\":\"DIY231J8BR23QK4UKBY8\",\"name\":\"Microsoft Azure Active Directory\"},\"auth_device\":{\"ip\":\"192.168.225.254\",\"location\":{\"city\":\"Ann Arbor\",\"country\":\"United States\",\"state\":\"Michigan\"},\"name\":\"My iPhone X (734-555-2342)\"},\"email\":\"[email protected]\",\"event_type\":\"authentication\",\"factor\":\"duo_push\",\"isotimestamp\":\"2020-02-13T18:56:20.351346+00:00\",\"ood_software\":null,\"reason\":\"user_approved\",\"result\":\"success\",\"timestamp\":1581620180,\"trusted_endpoint_status\":\"not trusted\",\"txid\":\"340a23e3-23f3-23c1-87dc-1491a23dfdbb\",\"user\":{\"groups\":[\"Duo Users\",\"CorpHQ Users\"],\"key\":\"DU3KC77WJ06Y5HIV7XKQ\",\"name\":\"[email protected]\"}}",
         "outcome": "success",

packages/cisco_duo/data_stream/offline_enrollment/elasticsearch/ingest_pipeline/default.yml

@@ -24,9 +24,13 @@ processors:
   - date:
       field: json.timestamp
       target_field: "@timestamp"
-      ignore_failure: true
+      if: ctx.json?.timestamp != null
       formats:
         - UNIX
+      on_failure:
+        - append:
+            field: error.message
+            value: "{{{_ingest.on_failure_message}}}"
   - json: 
       field: json.description
       target_field: json_description
@@ -70,6 +74,6 @@ processors:
       ignore_failure: true
       ignore_missing: true
 on_failure:
-  - set:
+  - append:
       field: error.message
       value: "{{{_ingest.on_failure_message}}}"

packages/cisco_duo/data_stream/offline_enrollment/sample_event.json

@@ -1,11 +1,11 @@
 {
     "@timestamp": "2019-08-30T16:10:05.000Z",
     "agent": {
-        "ephemeral_id": "3470fbe5-8d73-49db-8555-7e5f4cfd8504",
-        "id": "7cefd7f8-53e3-4884-ab65-da99d71b166f",
+        "ephemeral_id": "0d45b27c-6405-44fa-beda-22c49ad27853",
+        "id": "c934978b-c8c9-4484-8fbe-007cc0ace376",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.0.0-beta1"
+        "version": "8.2.0"
     },
     "cisco_duo": {
         "offline_enrollment": {
@@ -30,15 +30,15 @@
         "version": "8.3.0"
     },
     "elastic_agent": {
-        "id": "7cefd7f8-53e3-4884-ab65-da99d71b166f",
+        "id": "c934978b-c8c9-4484-8fbe-007cc0ace376",
         "snapshot": false,
-        "version": "8.0.0-beta1"
+        "version": "8.2.0"
     },
     "event": {
         "agent_id_status": "verified",
-        "created": "2021-12-29T09:40:24.650Z",
+        "created": "2022-07-14T12:21:49.801Z",
         "dataset": "cisco_duo.offline_enrollment",
-        "ingested": "2021-12-29T09:40:25Z",
+        "ingested": "2022-07-14T12:21:53Z",
         "original": "{\"action\":\"o2fa_user_provisioned\",\"description\":\"{\\\"user_agent\\\": \\\"DuoCredProv/4.0.6.413 (Windows NT 6.3.9600; x64; Server)\\\", \\\"hostname\\\": \\\"WKSW10x64\\\", \\\"factor\\\": \\\"duo_otp\\\"}\",\"isotimestamp\":\"2019-08-30T16:10:05+00:00\",\"object\":\"Acme Laptop Windows Logon\",\"timestamp\":1567181405,\"username\":\"narroway\"}"
     },
     "input": {

packages/cisco_duo/data_stream/summary/_dev/test/pipeline/test-summary.log-expected.json

@@ -1,7 +1,7 @@
 {
     "expected": [
         {
-            "@timestamp": "2022-06-28T17:52:12.776176085Z",
+            "@timestamp": "2022-07-14T12:19:12.108699204Z",
             "cisco_duo": {
                 "summary": {
                     "admin_count": 6,
@@ -21,7 +21,7 @@
             ]
         },
         {
-            "@timestamp": "2022-06-28T17:52:12.776179085Z",
+            "@timestamp": "2022-07-14T12:19:12.108704244Z",
             "cisco_duo": {
                 "summary": {
                     "admin_count": 3,

packages/cisco_duo/data_stream/summary/elasticsearch/ingest_pipeline/default.yml

@@ -35,6 +35,6 @@ processors:
       ignore_failure: true
       ignore_missing: true
 on_failure:
-  - set:
+  - append:
       field: error.message
       value: "{{{_ingest.on_failure_message}}}"

packages/cisco_duo/data_stream/summary/sample_event.json

@@ -1,11 +1,11 @@
 {
-    "@timestamp": "2021-12-29T09:41:01.807330132Z",
+    "@timestamp": "2022-07-14T12:22:46.223536172Z",
     "agent": {
-        "ephemeral_id": "88177cd0-9798-45a3-86b1-48ab8de2fe35",
-        "id": "7cefd7f8-53e3-4884-ab65-da99d71b166f",
+        "ephemeral_id": "3cdfc076-fea6-4cfc-af6d-57f48e0517be",
+        "id": "c934978b-c8c9-4484-8fbe-007cc0ace376",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.0.0-beta1"
+        "version": "8.2.0"
     },
     "cisco_duo": {
         "summary": {
@@ -24,15 +24,15 @@
         "version": "8.3.0"
     },
     "elastic_agent": {
-        "id": "7cefd7f8-53e3-4884-ab65-da99d71b166f",
+        "id": "c934978b-c8c9-4484-8fbe-007cc0ace376",
         "snapshot": false,
-        "version": "8.0.0-beta1"
+        "version": "8.2.0"
     },
     "event": {
         "agent_id_status": "verified",
-        "created": "2021-12-29T09:41:00.695Z",
+        "created": "2022-07-14T12:22:42.739Z",
         "dataset": "cisco_duo.summary",
-        "ingested": "2021-12-29T09:41:01Z",
+        "ingested": "2022-07-14T12:22:46Z",
         "original": "{\"response\":{\"admin_count\":3,\"integration_count\":9,\"telephony_credits_remaining\":960,\"user_count\":8},\"stat\":\"OK\"}"
     },
     "input": {