Skip to content

Commit f244061

Browse files
committed
CyrenThreatIntelligence v3.0.3: Fix duplicate data ingestion (follow-up to Azure#13603)
Changes in this PR: - Increased 'count' from 100 to 1000 in both IP Reputation and Malware URLs pollers (Cyren IP Rep feed has ~800 indicators, Malware URLs ~200 — all fit in one page) - Increased 'queryWindowInMin' from 15 to 360 minutes (6 hours) (Threat intelligence feeds are relatively static and do not require frequent polling) - Preserved PersistentToken paging from v3.0.2 - Added 3.0.3.zip package (all previous versions preserved: 3.0.0, 3.0.1, 3.0.2) - Updated ReleaseNotes.md Root cause of duplication: With count=100, the connector made 8+ page requests per poll cycle to fetch all ~800 indicators. Combined with 15-minute polling, this re-ingested the same data 96 times per day. Observed: 304,000 rows with only 198 unique IPs (1,535:1 duplicate ratio). Files changed: - Cyren_PollerConfig.json: count 100→1000, queryWindowInMin 15→360 - Package/mainTemplate.json: Same fixes + version bump to 3.0.3 - Package/3.0.3.zip: Updated package with all changes - ReleaseNotes.md: Added 3.0.3 entry
1 parent 02e9afb commit f244061

File tree

4 files changed

+107
-102
lines changed

4 files changed

+107
-102
lines changed
Lines changed: 98 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -1,98 +1,102 @@
11
[
2-
{
3-
"name": "CyrenIPReputation",
4-
"apiVersion": "2025-09-01",
5-
"type": "Microsoft.SecurityInsights/dataConnectors",
6-
"location": "{{location}}",
7-
"kind": "RestApiPoller",
8-
"properties": {
9-
"connectorDefinitionName": "CyrenThreatIntel",
10-
"dataType": "Cyren_Indicators_CL",
11-
"dcrConfig": {
12-
"streamName": "Custom-Cyren_Indicators_CL",
13-
"dataCollectionEndpoint": "{{dataCollectionEndpoint}}",
14-
"dataCollectionRuleImmutableId": "{{cyrenIPDcrImmutableId}}"
15-
},
16-
"auth": {
17-
"type": "APIKey",
18-
"ApiKeyName": "Authorization",
19-
"ApiKeyIdentifier": "Bearer",
20-
"ApiKey": "{{cyrenIpReputationToken}}"
21-
},
22-
"request": {
23-
"apiEndpoint": "https://api-feeds.cyren.com/v1/feed/data?feedId=ip_reputation",
24-
"httpMethod": "GET",
25-
"queryParameters": {
26-
"count": 100,
27-
"offset": 0,
28-
"format": "jsonl"
29-
},
30-
"queryWindowInMin": 15,
31-
"rateLimitQps": 1,
32-
"retryCount": 3,
33-
"timeoutInSeconds": 60,
34-
"headers": {
35-
"Accept": "application/json",
36-
"User-Agent": "Microsoft-Sentinel-Cyren/1.0"
2+
{
3+
"name": "CyrenIPReputation",
4+
"apiVersion": "2025-09-01",
5+
"type": "Microsoft.SecurityInsights/dataConnectors",
6+
"location": "{{location}}",
7+
"kind": "RestApiPoller",
8+
"properties": {
9+
"connectorDefinitionName": "CyrenThreatIntel",
10+
"dataType": "Cyren_Indicators_CL",
11+
"dcrConfig": {
12+
"streamName": "Custom-Cyren_Indicators_CL",
13+
"dataCollectionEndpoint": "{{dataCollectionEndpoint}}",
14+
"dataCollectionRuleImmutableId": "{{cyrenIPDcrImmutableId}}"
15+
},
16+
"auth": {
17+
"type": "APIKey",
18+
"ApiKeyName": "Authorization",
19+
"ApiKeyIdentifier": "Bearer",
20+
"ApiKey": "{{cyrenIpReputationToken}}"
21+
},
22+
"request": {
23+
"apiEndpoint": "https://api-feeds.cyren.com/v1/feed/data?feedId=ip_reputation",
24+
"httpMethod": "GET",
25+
"queryParameters": {
26+
"count": 1000,
27+
"offset": 0,
28+
"format": "jsonl"
29+
},
30+
"queryWindowInMin": 360,
31+
"rateLimitQps": 1,
32+
"retryCount": 3,
33+
"timeoutInSeconds": 60,
34+
"headers": {
35+
"Accept": "application/json",
36+
"User-Agent": "Microsoft-Sentinel-Cyren/1.0"
37+
}
38+
},
39+
"paging": {
40+
"pagingType": "PersistentToken",
41+
"nextPageTokenJsonPath": "$.[-1:].offset",
42+
"nextPageParaName": "offset"
43+
},
44+
"response": {
45+
"eventsJsonPaths": [
46+
"$"
47+
],
48+
"format": "jsonl"
49+
}
3750
}
38-
},
39-
"paging": {
40-
"pagingType": "PersistentToken",
41-
"nextPageTokenJsonPath": "$.[-1:].offset",
42-
"nextPageParaName": "offset"
43-
},
44-
"response": {
45-
"eventsJsonPaths": ["$"],
46-
"format": "jsonl"
47-
}
48-
}
49-
},
50-
{
51-
"name": "CyrenMalwareURLs",
52-
"apiVersion": "2025-09-01",
53-
"type": "Microsoft.SecurityInsights/dataConnectors",
54-
"location": "{{location}}",
55-
"kind": "RestApiPoller",
56-
"properties": {
57-
"connectorDefinitionName": "CyrenThreatIntel",
58-
"dataType": "Cyren_Indicators_CL",
59-
"dcrConfig": {
60-
"streamName": "Custom-Cyren_Indicators_CL",
61-
"dataCollectionEndpoint": "{{dataCollectionEndpoint}}",
62-
"dataCollectionRuleImmutableId": "{{cyrenMalwareDcrImmutableId}}"
63-
},
64-
"auth": {
65-
"type": "APIKey",
66-
"ApiKeyName": "Authorization",
67-
"ApiKeyIdentifier": "Bearer",
68-
"ApiKey": "{{cyrenMalwareUrlToken}}"
69-
},
70-
"request": {
71-
"apiEndpoint": "https://api-feeds.cyren.com/v1/feed/data?feedId=malware_urls",
72-
"httpMethod": "GET",
73-
"queryParameters": {
74-
"count": 100,
75-
"offset": 0,
76-
"format": "jsonl"
77-
},
78-
"queryWindowInMin": 15,
79-
"rateLimitQps": 1,
80-
"retryCount": 3,
81-
"timeoutInSeconds": 60,
82-
"headers": {
83-
"Accept": "application/json",
84-
"User-Agent": "Microsoft-Sentinel-Cyren/1.0"
51+
},
52+
{
53+
"name": "CyrenMalwareURLs",
54+
"apiVersion": "2025-09-01",
55+
"type": "Microsoft.SecurityInsights/dataConnectors",
56+
"location": "{{location}}",
57+
"kind": "RestApiPoller",
58+
"properties": {
59+
"connectorDefinitionName": "CyrenThreatIntel",
60+
"dataType": "Cyren_Indicators_CL",
61+
"dcrConfig": {
62+
"streamName": "Custom-Cyren_Indicators_CL",
63+
"dataCollectionEndpoint": "{{dataCollectionEndpoint}}",
64+
"dataCollectionRuleImmutableId": "{{cyrenMalwareDcrImmutableId}}"
65+
},
66+
"auth": {
67+
"type": "APIKey",
68+
"ApiKeyName": "Authorization",
69+
"ApiKeyIdentifier": "Bearer",
70+
"ApiKey": "{{cyrenMalwareUrlToken}}"
71+
},
72+
"request": {
73+
"apiEndpoint": "https://api-feeds.cyren.com/v1/feed/data?feedId=malware_urls",
74+
"httpMethod": "GET",
75+
"queryParameters": {
76+
"count": 1000,
77+
"offset": 0,
78+
"format": "jsonl"
79+
},
80+
"queryWindowInMin": 360,
81+
"rateLimitQps": 1,
82+
"retryCount": 3,
83+
"timeoutInSeconds": 60,
84+
"headers": {
85+
"Accept": "application/json",
86+
"User-Agent": "Microsoft-Sentinel-Cyren/1.0"
87+
}
88+
},
89+
"paging": {
90+
"pagingType": "PersistentToken",
91+
"nextPageTokenJsonPath": "$.[-1:].offset",
92+
"nextPageParaName": "offset"
93+
},
94+
"response": {
95+
"eventsJsonPaths": [
96+
"$"
97+
],
98+
"format": "jsonl"
99+
}
85100
}
86-
},
87-
"paging": {
88-
"pagingType": "PersistentToken",
89-
"nextPageTokenJsonPath": "$.[-1:].offset",
90-
"nextPageParaName": "offset"
91-
},
92-
"response": {
93-
"eventsJsonPaths": ["$"],
94-
"format": "jsonl"
95-
}
96101
}
97-
}
98-
]
102+
]
11.3 KB
Binary file not shown.

0 commit comments

Comments
 (0)