Skip to content

Commit db514a9

Browse files
timokoesslertimokoessler
committed
Add bot spoofing stats
1 parent cdc5aff commit db514a9

File tree

10 files changed

+306
-27
lines changed

10 files changed

+306
-27
lines changed

‎library/agent/Agent.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,7 @@ export class Agent {
319319
userAgents: stats.userAgents,
320320
ipAddresses: stats.ipAddresses,
321321
sqlTokenizationFailures: stats.sqlTokenizationFailures,
322+
botSpoofing: stats.botSpoofing,
322323
},
323324
packages,
324325
hostnames: outgoingDomains,

‎library/agent/InspectionStatistics.test.ts‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ t.test("it resets stats", async () => {
4848
ipAddresses: {
4949
breakdown: {},
5050
},
51+
botSpoofing: {
52+
breakdown: {},
53+
},
5154
sqlTokenizationFailures: 0,
5255
});
5356

@@ -70,6 +73,9 @@ t.test("it resets stats", async () => {
7073
ipAddresses: {
7174
breakdown: {},
7275
},
76+
botSpoofing: {
77+
breakdown: {},
78+
},
7379
sqlTokenizationFailures: 0,
7480
});
7581

@@ -103,6 +109,9 @@ t.test("it keeps track of amount of calls", async () => {
103109
ipAddresses: {
104110
breakdown: {},
105111
},
112+
botSpoofing: {
113+
breakdown: {},
114+
},
106115
sqlTokenizationFailures: 0,
107116
});
108117

@@ -144,6 +153,9 @@ t.test("it keeps track of amount of calls", async () => {
144153
ipAddresses: {
145154
breakdown: {},
146155
},
156+
botSpoofing: {
157+
breakdown: {},
158+
},
147159
sqlTokenizationFailures: 0,
148160
});
149161

@@ -185,6 +197,9 @@ t.test("it keeps track of amount of calls", async () => {
185197
ipAddresses: {
186198
breakdown: {},
187199
},
200+
botSpoofing: {
201+
breakdown: {},
202+
},
188203
sqlTokenizationFailures: 0,
189204
});
190205

@@ -219,6 +234,9 @@ t.test("it keeps track of amount of calls", async () => {
219234
ipAddresses: {
220235
breakdown: {},
221236
},
237+
botSpoofing: {
238+
breakdown: {},
239+
},
222240
sqlTokenizationFailures: 0,
223241
});
224242

@@ -260,6 +278,9 @@ t.test("it keeps track of amount of calls", async () => {
260278
ipAddresses: {
261279
breakdown: {},
262280
},
281+
botSpoofing: {
282+
breakdown: {},
283+
},
263284
sqlTokenizationFailures: 0,
264285
});
265286

@@ -301,6 +322,9 @@ t.test("it keeps track of amount of calls", async () => {
301322
ipAddresses: {
302323
breakdown: {},
303324
},
325+
botSpoofing: {
326+
breakdown: {},
327+
},
304328
sqlTokenizationFailures: 0,
305329
});
306330

@@ -361,6 +385,9 @@ t.test("it keeps track of amount of calls", async () => {
361385
ipAddresses: {
362386
breakdown: {},
363387
},
388+
botSpoofing: {
389+
breakdown: {},
390+
},
364391
sqlTokenizationFailures: 0,
365392
});
366393

@@ -418,6 +445,9 @@ t.test("it keeps track of requests", async () => {
418445
ipAddresses: {
419446
breakdown: {},
420447
},
448+
botSpoofing: {
449+
breakdown: {},
450+
},
421451
sqlTokenizationFailures: 0,
422452
});
423453

@@ -440,6 +470,9 @@ t.test("it keeps track of requests", async () => {
440470
ipAddresses: {
441471
breakdown: {},
442472
},
473+
botSpoofing: {
474+
breakdown: {},
475+
},
443476
sqlTokenizationFailures: 0,
444477
});
445478

@@ -463,6 +496,9 @@ t.test("it keeps track of requests", async () => {
463496
ipAddresses: {
464497
breakdown: {},
465498
},
499+
botSpoofing: {
500+
breakdown: {},
501+
},
466502
sqlTokenizationFailures: 0,
467503
});
468504

@@ -486,6 +522,9 @@ t.test("it keeps track of requests", async () => {
486522
ipAddresses: {
487523
breakdown: {},
488524
},
525+
botSpoofing: {
526+
breakdown: {},
527+
},
489528
sqlTokenizationFailures: 0,
490529
});
491530

@@ -510,6 +549,9 @@ t.test("it keeps track of requests", async () => {
510549
ipAddresses: {
511550
breakdown: {},
512551
},
552+
botSpoofing: {
553+
breakdown: {},
554+
},
513555
sqlTokenizationFailures: 0,
514556
});
515557

@@ -541,6 +583,9 @@ t.test("it force compresses stats", async () => {
541583
ipAddresses: {
542584
breakdown: {},
543585
},
586+
botSpoofing: {
587+
breakdown: {},
588+
},
544589
sqlTokenizationFailures: 0,
545590
});
546591

@@ -591,6 +636,9 @@ t.test("it keeps track of aborted requests", async () => {
591636
ipAddresses: {
592637
breakdown: {},
593638
},
639+
botSpoofing: {
640+
breakdown: {},
641+
},
594642
sqlTokenizationFailures: 0,
595643
});
596644

@@ -630,6 +678,9 @@ t.test("it keeps track of matched IPs and user agents", async () => {
630678
"known_threat_actors/public_scanners": 1,
631679
},
632680
},
681+
botSpoofing: {
682+
breakdown: {},
683+
},
633684
sqlTokenizationFailures: 0,
634685
});
635686

@@ -659,6 +710,9 @@ t.test("it keeps track of matched IPs and user agents", async () => {
659710
"known_threat_actors/public_scanners": 2,
660711
},
661712
},
713+
botSpoofing: {
714+
breakdown: {},
715+
},
662716
sqlTokenizationFailures: 0,
663717
});
664718

@@ -731,6 +785,9 @@ t.test("it keeps track of multiple operations of the same kind", async () => {
731785
ipAddresses: {
732786
breakdown: {},
733787
},
788+
botSpoofing: {
789+
breakdown: {},
790+
},
734791
sqlTokenizationFailures: 0,
735792
});
736793

@@ -793,6 +850,9 @@ t.test("it keeps track of multiple operations of the same kind", async () => {
793850
ipAddresses: {
794851
breakdown: {},
795852
},
853+
botSpoofing: {
854+
breakdown: {},
855+
},
796856
sqlTokenizationFailures: 0,
797857
});
798858

@@ -838,6 +898,9 @@ t.test("it handles empty operation strings", async () => {
838898
ipAddresses: {
839899
breakdown: {},
840900
},
901+
botSpoofing: {
902+
breakdown: {},
903+
},
841904
sqlTokenizationFailures: 0,
842905
});
843906

@@ -871,8 +934,85 @@ t.test("it increments sqlTokenizationFailures", async () => {
871934
ipAddresses: {
872935
breakdown: {},
873936
},
937+
botSpoofing: {
938+
breakdown: {},
939+
},
874940
sqlTokenizationFailures: 1,
875941
});
876942

877943
clock.uninstall();
878944
});
945+
946+
t.test("it keeps track of bot spoofing matches", async () => {
947+
const clock = FakeTimers.install();
948+
949+
const stats = new InspectionStatistics({
950+
maxPerfSamplesInMemory: 50,
951+
maxCompressedStatsInMemory: 5,
952+
});
953+
954+
stats.onBotSpoofingMatch("google_test");
955+
stats.onBotSpoofingMatch("bing_test");
956+
957+
t.same(stats.getStats(), {
958+
operations: {},
959+
startedAt: 0,
960+
requests: {
961+
total: 0,
962+
aborted: 0,
963+
attacksDetected: {
964+
total: 0,
965+
blocked: 0,
966+
},
967+
},
968+
userAgents: {
969+
breakdown: {},
970+
},
971+
ipAddresses: {
972+
breakdown: {},
973+
},
974+
botSpoofing: {
975+
breakdown: {
976+
// eslint-disable-next-line camelcase
977+
google_test: 1,
978+
// eslint-disable-next-line camelcase
979+
bing_test: 1,
980+
},
981+
},
982+
sqlTokenizationFailures: 0,
983+
});
984+
985+
// Test multiple occurrences
986+
stats.onBotSpoofingMatch("google_test");
987+
stats.onBotSpoofingMatch("bing_test");
988+
989+
t.same(stats.getStats(), {
990+
operations: {},
991+
startedAt: 0,
992+
requests: {
993+
total: 0,
994+
aborted: 0,
995+
attacksDetected: {
996+
total: 0,
997+
blocked: 0,
998+
},
999+
},
1000+
userAgents: {
1001+
breakdown: {},
1002+
},
1003+
ipAddresses: {
1004+
breakdown: {},
1005+
},
1006+
botSpoofing: {
1007+
breakdown: {
1008+
// eslint-disable-next-line camelcase
1009+
google_test: 2,
1010+
// eslint-disable-next-line camelcase
1011+
bing_test: 2,
1012+
},
1013+
},
1014+
sqlTokenizationFailures: 0,
1015+
});
1016+
1017+
clock.uninstall();
1018+
});

‎library/agent/InspectionStatistics.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ type IPAddressStats = {
3434
breakdown: Record<IPListKey, number>;
3535
};
3636

37+
type BotSpoofingStats = {
38+
breakdown: Record<string, number>;
39+
};
40+
3741
export class InspectionStatistics {
3842
private startedAt = Date.now();
3943
private operations: Record<string, OperationStats> = {};
@@ -58,6 +62,9 @@ export class InspectionStatistics {
5862
private ipAddresses: IPAddressStats = {
5963
breakdown: {},
6064
};
65+
private botSpoofing: BotSpoofingStats = {
66+
breakdown: {},
67+
};
6168

6269
constructor({
6370
maxPerfSamplesInMemory,
@@ -119,6 +126,9 @@ export class InspectionStatistics {
119126
ipAddresses: {
120127
breakdown: Record<string, number>;
121128
};
129+
botSpoofing: {
130+
breakdown: Record<string, number>;
131+
};
122132
} {
123133
const operations: Record<string, OperationStatsWithoutTimings> = {};
124134
for (const operation in this.operations) {
@@ -143,6 +153,7 @@ export class InspectionStatistics {
143153
requests: this.requests,
144154
userAgents: this.userAgents,
145155
ipAddresses: this.ipAddresses,
156+
botSpoofing: this.botSpoofing,
146157
};
147158
}
148159

@@ -246,6 +257,14 @@ export class InspectionStatistics {
246257
});
247258
}
248259

260+
onBotSpoofingMatch(key: string) {
261+
if (!this.botSpoofing.breakdown[key]) {
262+
this.botSpoofing.breakdown[key] = 0;
263+
}
264+
265+
this.botSpoofing.breakdown[key] += 1;
266+
}
267+
249268
onAbortedRequest() {
250269
this.requests.aborted += 1;
251270
}

‎library/agent/api/Event.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,9 @@ type Heartbeat = {
112112
ipAddresses: {
113113
breakdown: Record<string, number>;
114114
};
115+
botSpoofing: {
116+
breakdown: Record<string, number>;
117+
};
115118
};
116119
packages: {
117120
name: string;

‎library/agent/api/ReportingAPIRateLimitedClientSide.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,9 @@ function generateHeartbeatEvent(): Event {
161161
ipAddresses: {
162162
breakdown: {},
163163
},
164+
botSpoofing: {
165+
breakdown: {},
166+
},
164167
sqlTokenizationFailures: 0,
165168
},
166169
agent: {

0 commit comments

Comments
 (0)