checkpoint_harmony_endpoint: improve handling of API errors (#13333)

chemamartinez · web-flow · commit 787d8963aca1 · 2025-03-28T11:58:21.000+01:00
In the generic error handling (including 404 error), clear task ID and page token if any, and reset the sequence for the same timeframe, to avoid getting stuck making requests with stale parameters.
diff --git a/packages/checkpoint_harmony_endpoint/changelog.yml b/packages/checkpoint_harmony_endpoint/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.6.0"
+  changes:
+    - description: Improve cleaning of query parameters on errors.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/13333
 - version: "0.5.1"
   changes:
     - description: Fix endTime query parameter assignment.
diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antibot/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/antibot/agent/stream/cel.yml.hbs
@@ -150,23 +150,6 @@ program: |
   					}
   				)
   			:
-  			(resp.StatusCode == 404) ?
-  				// 404 Not Found - Resubmit the task ID query for the same timeframe.
-  				state.with(
-  					{
-  						"events": [{"message": {"event": {"reason": "polling"}}.encode_json()}],
-  						"want_more": true,
-  						"cursor": state.cursor.with(
-  							{
-  								"auth_data": auth_data,
-  								"task_id": null,
-  								"next_startTime": state.cursor.current_startTime,
-  								"next_endTime": state.cursor.current_endTime,
-  							}
-  						),
-  					}
-  				)
-  			:
   			(resp.StatusCode == 200) ?
   				bytes(resp.Body).decode_json().as(body,
   					(body.data.state == "Ready") ?
@@ -212,6 +195,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -227,6 +211,14 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
@@ -315,6 +307,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID and page token, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -330,6 +323,15 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"page_token": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
diff --git a/packages/checkpoint_harmony_endpoint/data_stream/antimalware/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/antimalware/agent/stream/cel.yml.hbs
@@ -150,23 +150,6 @@ program: |
   					}
   				)
   			:
-  			(resp.StatusCode == 404) ?
-  				// 404 Not Found - Resubmit the task ID query for the same timeframe.
-  				state.with(
-  					{
-  						"events": [{"message": {"event": {"reason": "polling"}}.encode_json()}],
-  						"want_more": true,
-  						"cursor": state.cursor.with(
-  							{
-  								"auth_data": auth_data,
-  								"task_id": null,
-  								"next_startTime": state.cursor.current_startTime,
-  								"next_endTime": state.cursor.current_endTime,
-  							}
-  						),
-  					}
-  				)
-  			:
   			(resp.StatusCode == 200) ?
   				bytes(resp.Body).decode_json().as(body,
   					(body.data.state == "Ready") ?
@@ -212,6 +195,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -227,6 +211,14 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
@@ -315,6 +307,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID and page token, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -330,6 +323,15 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"page_token": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
diff --git a/packages/checkpoint_harmony_endpoint/data_stream/forensics/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/forensics/agent/stream/cel.yml.hbs
@@ -150,23 +150,6 @@ program: |
   					}
   				)
   			:
-  			(resp.StatusCode == 404) ?
-  				// 404 Not Found - Resubmit the task ID query for the same timeframe.
-  				state.with(
-  					{
-  						"events": [{"message": {"event": {"reason": "polling"}}.encode_json()}],
-  						"want_more": true,
-  						"cursor": state.cursor.with(
-  							{
-  								"auth_data": auth_data,
-  								"task_id": null,
-  								"next_startTime": state.cursor.current_startTime,
-  								"next_endTime": state.cursor.current_endTime,
-  							}
-  						),
-  					}
-  				)
-  			:
   			(resp.StatusCode == 200) ?
   				bytes(resp.Body).decode_json().as(body,
   					(body.data.state == "Ready") ?
@@ -212,6 +195,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -227,6 +211,14 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
@@ -315,6 +307,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID and page token, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -330,6 +323,15 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"page_token": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/threatemulation/agent/stream/cel.yml.hbs
@@ -150,23 +150,6 @@ program: |
   					}
   				)
   			:
-  			(resp.StatusCode == 404) ?
-  				// 404 Not Found - Resubmit the task ID query for the same timeframe.
-  				state.with(
-  					{
-  						"events": [{"message": {"event": {"reason": "polling"}}.encode_json()}],
-  						"want_more": true,
-  						"cursor": state.cursor.with(
-  							{
-  								"auth_data": auth_data,
-  								"task_id": null,
-  								"next_startTime": state.cursor.current_startTime,
-  								"next_endTime": state.cursor.current_endTime,
-  							}
-  						),
-  					}
-  				)
-  			:
   			(resp.StatusCode == 200) ?
   				bytes(resp.Body).decode_json().as(body,
   					(body.data.state == "Ready") ?
@@ -212,6 +195,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -227,6 +211,14 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
@@ -315,6 +307,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID and page token, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -330,6 +323,15 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"page_token": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
diff --git a/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/threatextraction/agent/stream/cel.yml.hbs
@@ -150,23 +150,6 @@ program: |
   					}
   				)
   			:
-  			(resp.StatusCode == 404) ?
-  				// 404 Not Found - Resubmit the task ID query for the same timeframe.
-  				state.with(
-  					{
-  						"events": [{"message": {"event": {"reason": "polling"}}.encode_json()}],
-  						"want_more": true,
-  						"cursor": state.cursor.with(
-  							{
-  								"auth_data": auth_data,
-  								"task_id": null,
-  								"next_startTime": state.cursor.current_startTime,
-  								"next_endTime": state.cursor.current_endTime,
-  							}
-  						),
-  					}
-  				)
-  			:
   			(resp.StatusCode == 200) ?
   				bytes(resp.Body).decode_json().as(body,
   					(body.data.state == "Ready") ?
@@ -212,6 +195,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -227,6 +211,14 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
@@ -315,6 +307,7 @@ program: |
   						)
   				)
   			:
+  				// Clear the task ID and page token, and reset the sequence for the same timeframe.
   				state.with(
   					{
   						"events": {
@@ -330,6 +323,15 @@ program: |
   							},
   						},
   						"want_more": false,
+  						"cursor": state.cursor.with(
+  							{
+  								"auth_data": auth_data,
+  								"task_id": null,
+  								"page_token": null,
+  								"next_startTime": state.cursor.current_startTime,
+  								"next_endTime": state.cursor.current_endTime,
+  							}
+  						),
   					}
   				)
   		)
diff --git a/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/urlfiltering/agent/stream/cel.yml.hbs
diff --git a/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/agent/stream/cel.yml.hbs b/packages/checkpoint_harmony_endpoint/data_stream/zerophishing/agent/stream/cel.yml.hbs
diff --git a/packages/checkpoint_harmony_endpoint/manifest.yml b/packages/checkpoint_harmony_endpoint/manifest.yml

Original file line number	Diff line number	Diff line change
`@@ -150,23 +150,6 @@ program: \|`
`150`	`150`	`}`
`151`	`151`	`)`
`152`	`152`	`:`
`153`		`- (resp.StatusCode == 404) ?`
`154`		`- // 404 Not Found - Resubmit the task ID query for the same timeframe.`
`155`		`- state.with(`
`156`		`- {`
`157`		`- "events": [{"message": {"event": {"reason": "polling"}}.encode_json()}],`
`158`		`- "want_more": true,`
`159`		`- "cursor": state.cursor.with(`
`160`		`- {`
`161`		`- "auth_data": auth_data,`
`162`		`- "task_id": null,`
`163`		`- "next_startTime": state.cursor.current_startTime,`
`164`		`- "next_endTime": state.cursor.current_endTime,`
`165`		`- }`
`166`		`- ),`
`167`		`- }`
`168`		`- )`
`169`		`- :`
`170`	`153`	`(resp.StatusCode == 200) ?`
`171`	`154`	`bytes(resp.Body).decode_json().as(body,`
`172`	`155`	`(body.data.state == "Ready") ?`
`@@ -212,6 +195,7 @@ program: \|`
`212`	`195`	`)`
`213`	`196`	`)`
`214`	`197`	`:`
	`198`	`+ // Clear the task ID, and reset the sequence for the same timeframe.`
`215`	`199`	`state.with(`
`216`	`200`	`{`
`217`	`201`	`"events": {`
`@@ -227,6 +211,14 @@ program: \|`
`227`	`211`	`},`
`228`	`212`	`},`
`229`	`213`	`"want_more": false,`
	`214`	`+ "cursor": state.cursor.with(`
	`215`	`+ {`
	`216`	`+ "auth_data": auth_data,`
	`217`	`+ "task_id": null,`
	`218`	`+ "next_startTime": state.cursor.current_startTime,`
	`219`	`+ "next_endTime": state.cursor.current_endTime,`
	`220`	`+ }`
	`221`	`+ ),`
`230`	`222`	`}`
`231`	`223`	`)`
`232`	`224`	`)`
`@@ -315,6 +307,7 @@ program: \|`
`315`	`307`	`)`
`316`	`308`	`)`
`317`	`309`	`:`
	`310`	`+ // Clear the task ID and page token, and reset the sequence for the same timeframe.`
`318`	`311`	`state.with(`
`319`	`312`	`{`
`320`	`313`	`"events": {`
`@@ -330,6 +323,15 @@ program: \|`
`330`	`323`	`},`
`331`	`324`	`},`
`332`	`325`	`"want_more": false,`
	`326`	`+ "cursor": state.cursor.with(`
	`327`	`+ {`
	`328`	`+ "auth_data": auth_data,`
	`329`	`+ "task_id": null,`
	`330`	`+ "page_token": null,`
	`331`	`+ "next_startTime": state.cursor.current_startTime,`
	`332`	`+ "next_endTime": state.cursor.current_endTime,`
	`333`	`+ }`
	`334`	`+ ),`
`333`	`335`	`}`
`334`	`336`	`)`
`335`	`337`	`)`