diff --git a/library/agent/Agent.ts b/library/agent/Agent.ts index c31efcde7..0bf86f7d2 100644 --- a/library/agent/Agent.ts +++ b/library/agent/Agent.ts @@ -52,10 +52,7 @@ export class Agent { ); private routes: Routes = new Routes(200); private rateLimiter: RateLimiter = new RateLimiter(5000, 120 * 60 * 1000); - private statistics = new InspectionStatistics({ - maxPerfSamplesInMemory: 5000, - maxCompressedStatsInMemory: 100, - }); + private statistics = new InspectionStatistics(); private middlewareInstalled = false; private attackLogger = new AttackLogger(1000); @@ -352,12 +349,10 @@ export class Agent { const now = performance.now(); const diff = now - this.lastHeartbeat; const shouldSendHeartbeat = diff > this.sendHeartbeatEveryMS; - const hasCompressedStats = this.statistics.hasCompressedStats(); const canSendInitialStats = !this.serviceConfig.hasReceivedAnyStats() && !this.statistics.isEmpty(); const shouldReportInitialStats = - !this.reportedInitialStats && - (hasCompressedStats || canSendInitialStats); + !this.reportedInitialStats && canSendInitialStats; if (shouldSendHeartbeat || shouldReportInitialStats) { this.heartbeat(); @@ -555,7 +550,6 @@ export class Agent { } async flushStats(timeoutInMS: number) { - this.statistics.forceCompress(); await this.sendHeartbeat(timeoutInMS); } diff --git a/library/agent/InspectionStatistics.test.ts b/library/agent/InspectionStatistics.test.ts index fa3e92637..5cf513abf 100644 --- a/library/agent/InspectionStatistics.test.ts +++ b/library/agent/InspectionStatistics.test.ts @@ -5,10 +5,7 @@ import { InspectionStatistics } from "./InspectionStatistics"; t.test("it resets stats", async () => { const clock = FakeTimers.install(); - const stats = new InspectionStatistics({ - maxPerfSamplesInMemory: 50, - maxCompressedStatsInMemory: 5, - }); + const stats = new InspectionStatistics(); stats.onInspectedCall({ withoutContext: false, @@ -30,7 +27,6 @@ t.test("it resets stats", async () => { interceptorThrewError: 0, withoutContext: 0, total: 1, - compressedTimings: [], }, }, startedAt: 0, @@ -77,12 +73,7 @@ t.test("it resets stats", async () => { t.test("it keeps track of amount of calls", async () => { const clock = FakeTimers.install(); - const maxPerfSamplesInMemory = 50; - const maxCompressedStatsInMemory = 5; - const stats = new InspectionStatistics({ - maxPerfSamplesInMemory: maxPerfSamplesInMemory, - maxCompressedStatsInMemory: maxCompressedStatsInMemory, - }); + const stats = new InspectionStatistics(); t.same(stats.getStats(), { operations: {}, @@ -123,7 +114,6 @@ t.test("it keeps track of amount of calls", async () => { interceptorThrewError: 0, withoutContext: 0, total: 1, - compressedTimings: [], }, }, startedAt: 0, @@ -163,7 +153,6 @@ t.test("it keeps track of amount of calls", async () => { interceptorThrewError: 0, withoutContext: 1, total: 2, - compressedTimings: [], }, }, startedAt: 0, @@ -196,7 +185,6 @@ t.test("it keeps track of amount of calls", async () => { interceptorThrewError: 1, withoutContext: 1, total: 3, - compressedTimings: [], }, }, startedAt: 0, @@ -236,7 +224,6 @@ t.test("it keeps track of amount of calls", async () => { interceptorThrewError: 1, withoutContext: 1, total: 4, - compressedTimings: [], }, }, startedAt: 0, @@ -276,7 +263,6 @@ t.test("it keeps track of amount of calls", async () => { interceptorThrewError: 1, withoutContext: 1, total: 5, - compressedTimings: [], }, }, startedAt: 0, @@ -296,22 +282,8 @@ t.test("it keeps track of amount of calls", async () => { }, }); - t.same(stats.hasCompressedStats(), false); - clock.tick(1000); - for (let i = 0; i < maxPerfSamplesInMemory; i++) { - stats.onInspectedCall({ - withoutContext: false, - kind: "nosql_op", - operation: "mongodb.query", - blocked: false, - durationInMs: i * 0.1, - attackDetected: false, - }); - } - - t.same(stats.hasCompressedStats(), true); t.same(stats.getStats(), { operations: { "mongodb.query": { @@ -322,20 +294,7 @@ t.test("it keeps track of amount of calls", async () => { }, interceptorThrewError: 1, withoutContext: 1, - total: 55, - compressedTimings: [ - { - averageInMS: 2.1719999999999997, - percentiles: { - "50": 2.1, - "75": 3.4000000000000004, - "90": 4.1000000000000005, - "95": 4.4, - "99": 4.6000000000000005, - }, - compressedAt: 1000, - }, - ], + total: 5, }, }, startedAt: 0, @@ -355,42 +314,13 @@ t.test("it keeps track of amount of calls", async () => { }, }); - t.ok( - // @ts-expect-error Stats is private - stats.operations["mongodb.query"].durations.length < maxPerfSamplesInMemory - ); - - for ( - let i = 0; - i < maxPerfSamplesInMemory * maxCompressedStatsInMemory * 2; - i++ - ) { - stats.onInspectedCall({ - withoutContext: false, - kind: "nosql_op", - operation: "mongodb.query", - blocked: false, - durationInMs: i * 0.1, - attackDetected: false, - }); - } - - t.same( - // @ts-expect-error Stats is private - stats.operations["mongodb.query"].compressedTimings.length, - maxCompressedStatsInMemory - ); - clock.uninstall(); }); t.test("it keeps track of requests", async () => { const clock = FakeTimers.install(); - const stats = new InspectionStatistics({ - maxPerfSamplesInMemory: 50, - maxCompressedStatsInMemory: 5, - }); + const stats = new InspectionStatistics(); t.same(stats.getStats(), { operations: {}, @@ -502,60 +432,10 @@ t.test("it keeps track of requests", async () => { clock.uninstall(); }); -t.test("it force compresses stats", async () => { - const clock = FakeTimers.install(); - - const stats = new InspectionStatistics({ - maxPerfSamplesInMemory: 50, - maxCompressedStatsInMemory: 5, - }); - - t.same(stats.getStats(), { - operations: {}, - startedAt: 0, - requests: { - total: 0, - aborted: 0, - attacksDetected: { - total: 0, - blocked: 0, - }, - }, - userAgents: { - breakdown: {}, - }, - ipAddresses: { - breakdown: {}, - }, - }); - - stats.onRequest(); - - stats.onInspectedCall({ - withoutContext: false, - kind: "nosql_op", - operation: "mongodb.query", - blocked: false, - durationInMs: 0.1, - attackDetected: false, - }); - - t.same(stats.hasCompressedStats(), false); - - stats.forceCompress(); - - t.same(stats.hasCompressedStats(), true); - - clock.uninstall(); -}); - t.test("it keeps track of aborted requests", async () => { const clock = FakeTimers.install(); - const stats = new InspectionStatistics({ - maxPerfSamplesInMemory: 50, - maxCompressedStatsInMemory: 5, - }); + const stats = new InspectionStatistics(); stats.onAbortedRequest(); @@ -584,10 +464,7 @@ t.test("it keeps track of aborted requests", async () => { t.test("it keeps track of matched IPs and user agents", async () => { const clock = FakeTimers.install(); - const stats = new InspectionStatistics({ - maxPerfSamplesInMemory: 50, - maxCompressedStatsInMemory: 5, - }); + const stats = new InspectionStatistics(); stats.onIPAddressMatches(["known_threat_actors/public_scanners"]); stats.onUserAgentMatches(["ai_data_scrapers"]); @@ -650,10 +527,7 @@ t.test("it keeps track of matched IPs and user agents", async () => { t.test("it keeps track of multiple operations of the same kind", async () => { const clock = FakeTimers.install(); - const stats = new InspectionStatistics({ - maxPerfSamplesInMemory: 50, - maxCompressedStatsInMemory: 5, - }); + const stats = new InspectionStatistics(); stats.onInspectedCall({ withoutContext: false, @@ -684,7 +558,6 @@ t.test("it keeps track of multiple operations of the same kind", async () => { interceptorThrewError: 0, withoutContext: 0, total: 1, - compressedTimings: [], }, "mongodb.insert": { kind: "nosql_op", @@ -695,7 +568,6 @@ t.test("it keeps track of multiple operations of the same kind", async () => { interceptorThrewError: 0, withoutContext: 0, total: 1, - compressedTimings: [], }, }, startedAt: 0, @@ -745,7 +617,6 @@ t.test("it keeps track of multiple operations of the same kind", async () => { interceptorThrewError: 0, withoutContext: 1, total: 2, - compressedTimings: [], }, "mongodb.insert": { kind: "nosql_op", @@ -756,7 +627,6 @@ t.test("it keeps track of multiple operations of the same kind", async () => { interceptorThrewError: 0, withoutContext: 0, total: 2, - compressedTimings: [], }, }, startedAt: 0, @@ -782,10 +652,7 @@ t.test("it keeps track of multiple operations of the same kind", async () => { t.test("it handles empty operation strings", async () => { const clock = FakeTimers.install(); - const stats = new InspectionStatistics({ - maxPerfSamplesInMemory: 50, - maxCompressedStatsInMemory: 5, - }); + const stats = new InspectionStatistics(); // Test onInspectedCall with empty operation stats.onInspectedCall({ diff --git a/library/agent/InspectionStatistics.ts b/library/agent/InspectionStatistics.ts index 1f1660171..8789e949b 100644 --- a/library/agent/InspectionStatistics.ts +++ b/library/agent/InspectionStatistics.ts @@ -1,20 +1,9 @@ -import { percentiles } from "../helpers/percentiles"; import { OperationKind } from "./api/Event"; -type OperationCompressedTimings = { - averageInMS: number; - percentiles: Record; - compressedAt: number; -}; - type OperationStats = { kind: OperationKind; withoutContext: number; total: number; - // array where we accumulate durations for each sink-request (e.g. mysql.query) - durations: number[]; - // array where we put compressed blocks of stats - compressedTimings: OperationCompressedTimings[]; interceptorThrewError: number; attacksDetected: { total: number; @@ -37,8 +26,6 @@ type IPAddressStats = { export class InspectionStatistics { private startedAt = Date.now(); private operations: Record = {}; - private readonly maxPerfSamplesInMemory: number; - private readonly maxCompressedStatsInMemory: number; private requests: { total: number; aborted: number; @@ -58,23 +45,6 @@ export class InspectionStatistics { breakdown: {}, }; - constructor({ - maxPerfSamplesInMemory, - maxCompressedStatsInMemory, - }: { - maxPerfSamplesInMemory: number; - maxCompressedStatsInMemory: number; - }) { - this.maxPerfSamplesInMemory = maxPerfSamplesInMemory; - this.maxCompressedStatsInMemory = maxCompressedStatsInMemory; - } - - hasCompressedStats() { - return Object.values(this.operations).some( - (sinkStats) => sinkStats.compressedTimings.length > 0 - ); - } - isEmpty() { return ( this.requests.total === 0 && @@ -129,7 +99,6 @@ export class InspectionStatistics { }, interceptorThrewError: operationStats.interceptorThrewError, withoutContext: operationStats.withoutContext, - compressedTimings: operationStats.compressedTimings, }; } @@ -148,8 +117,6 @@ export class InspectionStatistics { withoutContext: 0, kind: kind, total: 0, - durations: [], - compressedTimings: [], interceptorThrewError: 0, attacksDetected: { total: 0, @@ -159,52 +126,6 @@ export class InspectionStatistics { } } - private compressPerfSamples(operation: string) { - if (operation.length === 0) { - return; - } - - /* c8 ignore start */ - if (!this.operations[operation]) { - return; - } - - if (this.operations[operation].durations.length === 0) { - return; - } - /* c8 ignore stop */ - - const timings = this.operations[operation].durations; - const averageInMS = - timings.reduce((acc, curr) => acc + curr, 0) / timings.length; - - const [p50, p75, p90, p95, p99] = percentiles( - [50, 75, 90, 95, 99], - timings - ); - - this.operations[operation].compressedTimings.push({ - averageInMS, - percentiles: { - "50": p50, - "75": p75, - "90": p90, - "95": p95, - "99": p99, - }, - compressedAt: Date.now(), - }); - - if ( - this.operations[operation].compressedTimings.length > - this.maxCompressedStatsInMemory - ) { - this.operations[operation].compressedTimings.shift(); - } - - this.operations[operation].durations = []; - } - interceptorThrewError(operation: string, kind: OperationKind) { if (operation.length === 0) { return; @@ -255,6 +176,8 @@ export class InspectionStatistics { kind, blocked, attackDetected, + // Let's remove later + // eslint-disable-next-line @typescript-eslint/no-unused-vars durationInMs, withoutContext, }: { @@ -278,14 +201,6 @@ export class InspectionStatistics { return; } - if ( - this.operations[operation].durations.length >= this.maxPerfSamplesInMemory - ) { - this.compressPerfSamples(operation); - } - - this.operations[operation].durations.push(durationInMs); - if (attackDetected) { this.operations[operation].attacksDetected.total += 1; if (blocked) { @@ -293,10 +208,4 @@ export class InspectionStatistics { } } } - - forceCompress() { - for (const kind in this.operations) { - this.compressPerfSamples(kind as OperationKind); - } - } } diff --git a/library/agent/api/Event.ts b/library/agent/api/Event.ts index 455512178..639755149 100644 --- a/library/agent/api/Event.ts +++ b/library/agent/api/Event.ts @@ -83,11 +83,6 @@ type OperationStats = { interceptorThrewError: number; withoutContext: number; total: number; - compressedTimings: { - averageInMS: number; - percentiles: Record; - compressedAt: number; - }[]; }; type Heartbeat = { diff --git a/library/helpers/percentiles.test.ts b/library/helpers/percentiles.test.ts deleted file mode 100644 index b30263b2d..000000000 --- a/library/helpers/percentiles.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import * as t from "tap"; -import { percentiles } from "./percentiles"; - -function generateArray( - length: number, - fn: (value: unknown, index: number) => number -) { - return Array.from({ length: length }).map(fn); -} - -function generateArraySimple(length: number) { - return generateArray(length, (v, i) => i + 1); -} - -function shuffleArray(arr: number[]) { - return arr.sort(() => 0.5 - Math.random()); -} - -const stubsSimple = [ - { percentile: 0, list: shuffleArray(generateArraySimple(100)), result: 1 }, - { percentile: 25, list: shuffleArray(generateArraySimple(100)), result: 25 }, - { percentile: 50, list: shuffleArray(generateArraySimple(100)), result: 50 }, - { percentile: 75, list: shuffleArray(generateArraySimple(100)), result: 75 }, - { - percentile: 100, - list: shuffleArray(generateArraySimple(100)), - result: 100, - }, - { - percentile: 75, - list: shuffleArray( - generateArraySimple(100).concat(generateArraySimple(30)) - ), - result: 68, - }, -]; - -t.test("percentile of simple values", async (t) => { - stubsSimple.forEach((stub) => { - t.same( - percentiles([stub.percentile], stub.list), - [stub.result], - JSON.stringify(stub) - ); - }); -}); - -t.test("percentile with negative values", async (t) => { - t.same(percentiles([50], shuffleArray([-1, -2, -3, -4, -5])), [-3]); - t.same(percentiles([50], shuffleArray([7, 6, -1, -2, -3, -4, -5])), [-2]); -}); - -t.test("array of percentiles", async (t) => { - t.same( - percentiles([0, 25, 50, 75, 100], shuffleArray(generateArraySimple(100))), - [1, 25, 50, 75, 100] - ); -}); - -t.test("throw an error if less than 0", async (t) => { - t.throws(() => percentiles([-1], [1])); -}); - -t.test("throw an error if grater than 100", async (t) => { - t.throws(() => percentiles([101], [1])); -}); - -t.test("empty list", async (t) => { - t.throws(() => percentiles([50], [])); -}); diff --git a/library/helpers/percentiles.ts b/library/helpers/percentiles.ts deleted file mode 100644 index 1c4a0107e..000000000 --- a/library/helpers/percentiles.ts +++ /dev/null @@ -1,35 +0,0 @@ -type NumberList = Array; - -export function percentiles(percentiles: number[], list: NumberList): number[] { - if (list.length === 0) { - throw new Error("List should not be empty"); - } - - percentiles.forEach((p) => { - if (p < 0) { - throw new Error( - `Expect percentile to be >= 0 but given "${p}" and its type is "${typeof p}".` - ); - } - - if (p > 100) { - throw new Error( - `Expect percentile to be <= 100 but given "${p}" and its type is "${typeof p}".` - ); - } - }); - - const sortedList: number[] = Array.from(list).sort((a, b) => a - b); - - return percentiles.map((p) => getPercentileValue(p, sortedList)); -} - -function getPercentileValue(p: number, list: number[]): number { - if (p === 0) { - return list[0]; - } - - const kIndex = Math.ceil(list.length * (p / 100)) - 1; - - return list[kIndex]; -} diff --git a/library/sources/Lambda.test.ts b/library/sources/Lambda.test.ts index b7bb2dd42..259ee1583 100644 --- a/library/sources/Lambda.test.ts +++ b/library/sources/Lambda.test.ts @@ -283,19 +283,6 @@ t.test("it sends heartbeat after first and every 10 minutes", async () => { }, interceptorThrewError: 0, withoutContext: 0, - compressedTimings: [ - { - averageInMS: 0.09999999999999981, - percentiles: { - 50: 0.1, - 75: 0.1, - 90: 0.1, - 95: 0.1, - 99: 0.1, - }, - compressedAt: 60 * 1000 * 10 + 1, - }, - ], }, }, startedAt: 0,