[Servicenow] Add support to append sysparm query in CEL input (#13621)

Introduce an optional parameter to append additional query conditions
for advanced filtering needs. It is advised to avoid using timestamp
fields in the appended query to prevent potential issues with pagination and
cursor logic.

Author: moxarth-rathod

packages/servicenow/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.13.0"
+  changes:
+    - description: Add option to supply a value for the sysparm_query parameter.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13621
 - version: "0.12.1"
   changes:
     - description: Fix handling of SQS worker count configuration.

packages/servicenow/data_stream/event/_dev/test/system/test-default-config.yml

@@ -9,5 +9,6 @@ data_stream:
     interval: 5m
     table_name: incident
     batch_size: 1
+    query: active=true
 assert:
   hit_count: 3

packages/servicenow/data_stream/event/agent/stream/cel.yml.hbs

@@ -21,6 +21,7 @@ state:
   batch_size: {{batch_size}}
   initial_interval: {{initial_interval}}
   timestamp_field: {{timestamp_field}}
+  query: {{query}}
 regexp:
   next_link: '<([^,]*)>;rel="next"'
   content: 'text/html'
@@ -34,7 +35,7 @@ program: |
           "sysparm_display_value": ["all"],
           "sysparm_exclude_reference_link": ["true"],
           "sysparm_limit": [string(state.batch_size)],
-          "sysparm_query": ["ORDERBY"+state.timestamp_field+"^"+state.timestamp_field+">"+start],
+          "sysparm_query": ["ORDERBY"+state.timestamp_field+"^"+state.timestamp_field+">"+start+(state.?query.hasValue() ? "^"+state.?query.orValue("") : "")],
         }.format_query()
       )
     )).as(resp, resp.StatusCode == 200 && (resp.Header["Content-Type"][0].re_find_submatch("content") == []) ?

packages/servicenow/data_stream/event/manifest.yml

@@ -84,6 +84,15 @@ streams:
         default: America/Los_Angeles
         description: >-
           By default, datetimes in the logs without a time zone will be interpreted as relative to the time zone configured in the host where the agent is running. If ingesting logs from a different time zone, use this field to set the time zone offset so that datetimes are correctly parsed. Acceptable time zone formats are: a canonical ID (e.g. "Europe/Amsterdam"), or an HH:mm differential (e.g. "-05:00") from UTC.
+      - name: query
+        type: text
+        title: Sysparm Query
+        description: >-
+          Encoded query for filtering result sets with operators like `=`, `!=`, `^`, `^OR`, `LIKE`, `ORDERBY`, `ORDERBYDESC`
+          (e.g. `active=true^ORDERBYname`). Note: Avoid using the timestamp field in the query, as it may affect pagination and cursor logic.
+        multi: false
+        required: false
+        show_user: false
       - name: http_client_timeout
         type: text
         title: HTTP Client Timeout

packages/servicenow/data_stream/event/sample_event.json

@@ -1,24 +1,24 @@
 {
-    "@timestamp": "2024-09-24T05:39:40.000Z",
+    "@timestamp": "2024-09-23T22:39:40.000-07:00",
     "agent": {
-        "ephemeral_id": "121d1e8c-0c94-4812-a446-4e8c339cbf5e",
-        "id": "744b1c23-395c-4123-9b8a-7e975ed7b1f8",
-        "name": "docker-fleet-agent",
+        "ephemeral_id": "def1e9fc-c5bf-4313-aebd-00845c9b3d61",
+        "id": "fd63c8e0-f0fc-498e-baaa-319b3609c582",
+        "name": "elastic-agent-43184",
         "type": "filebeat",
-        "version": "8.14.0"
+        "version": "8.16.5"
     },
     "data_stream": {
         "dataset": "servicenow.event",
-        "namespace": "28538",
+        "namespace": "50362",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "744b1c23-395c-4123-9b8a-7e975ed7b1f8",
+        "id": "fd63c8e0-f0fc-498e-baaa-319b3609c582",
         "snapshot": false,
-        "version": "8.14.0"
+        "version": "8.16.5"
     },
     "event": {
         "agent_id_status": "verified",
@@ -29,7 +29,7 @@
         "created": "2016-12-12T15:19:57.000Z",
         "dataset": "servicenow.event",
         "id": "1c741bd70b2322007518478d83673af3",
-        "ingested": "2024-12-02T12:33:55Z",
+        "ingested": "2025-04-25T09:46:25Z",
         "kind": "event",
         "severity": 3,
         "timezone": "America/Los_Angeles",
@@ -118,4 +118,4 @@
     "user": {
         "name": "Joe Employee"
     }
-}
+}

packages/servicenow/docs/README.md

@@ -164,26 +164,26 @@ An example event for `event` looks as following:
 
 ```json
 {
-    "@timestamp": "2024-09-24T05:39:40.000Z",
+    "@timestamp": "2024-09-23T22:39:40.000-07:00",
     "agent": {
-        "ephemeral_id": "121d1e8c-0c94-4812-a446-4e8c339cbf5e",
-        "id": "744b1c23-395c-4123-9b8a-7e975ed7b1f8",
-        "name": "docker-fleet-agent",
+        "ephemeral_id": "def1e9fc-c5bf-4313-aebd-00845c9b3d61",
+        "id": "fd63c8e0-f0fc-498e-baaa-319b3609c582",
+        "name": "elastic-agent-43184",
         "type": "filebeat",
-        "version": "8.14.0"
+        "version": "8.16.5"
     },
     "data_stream": {
         "dataset": "servicenow.event",
-        "namespace": "28538",
+        "namespace": "50362",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "744b1c23-395c-4123-9b8a-7e975ed7b1f8",
+        "id": "fd63c8e0-f0fc-498e-baaa-319b3609c582",
         "snapshot": false,
-        "version": "8.14.0"
+        "version": "8.16.5"
     },
     "event": {
         "agent_id_status": "verified",
@@ -194,7 +194,7 @@ An example event for `event` looks as following:
         "created": "2016-12-12T15:19:57.000Z",
         "dataset": "servicenow.event",
         "id": "1c741bd70b2322007518478d83673af3",
-        "ingested": "2024-12-02T12:33:55Z",
+        "ingested": "2025-04-25T09:46:25Z",
         "kind": "event",
         "severity": 3,
         "timezone": "America/Los_Angeles",

packages/servicenow/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.2.1
 name: servicenow
 title: "ServiceNow"
-version: "0.12.1"
+version: "0.13.0"
 description: "Collect logs from ServiceNow with Elastic Agent."
 type: integration
 categories: