Skip to content

Commit 5e7d403

Browse files
ivanstanevivanstanev
committed
feat: improve cache control by skipping deleted workloads
This change has several goals: - remove cached keys of scanned workloads and images as soon as we receive an event about a workload being deleted - skip sending workload scan results if the workload was deleted while image scanning was in progress This change tries to prevent a series of race conditions and to hopefully improve memory management in the Snyk Monitor.
1 parent 9c3e96d commit 5e7d403

File tree

11 files changed

+317
-49
lines changed

11 files changed

+317
-49
lines changed

‎src/scanner/index.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
Telemetry,
1919
} from '../transmitter/types';
2020
import { IPullableImage, IScanImage } from './images/types';
21+
import { getWorkloadAlreadyScanned } from '../state';
2122

2223
export async function processWorkload(
2324
workloadMetadata: IWorkload[],
@@ -128,6 +129,17 @@ async function scanImagesAndSendResults(
128129
return;
129130
}
130131

132+
// All workloads are identical, pick the first one
133+
const workload = workloadMetadata[0];
134+
const state = await getWorkloadAlreadyScanned(workload);
135+
if (state === undefined) {
136+
logger.info(
137+
{ workloadName },
138+
'the workload has been deleted while scanning was in progress, skipping sending scan results',
139+
);
140+
return;
141+
}
142+
131143
telemetry.imageScanDurationMs = imageScanDurationMs;
132144

133145
logger.info(

‎src/state.ts‎

Lines changed: 94 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { V1Namespace } from '@kubernetes/client-node';
1+
import { KubernetesObject, V1Namespace } from '@kubernetes/client-node';
22
import * as LruCache from 'lru-cache';
33

44
import { config } from './common/config';
@@ -21,13 +21,103 @@ const workloadsLruCacheOptions = {
2121
updateAgeOnGet: false,
2222
};
2323

24-
const state = {
24+
interface WorkloadAlreadyScanned {
25+
namespace: string;
26+
type: string;
27+
uid: string;
28+
}
29+
30+
interface WorkloadImagesAlreadyScanned {
31+
namespace: string;
32+
type: string;
33+
uid: string;
34+
imageIds: string[];
35+
}
36+
37+
function getWorkloadAlreadyScannedKey(
38+
workload: WorkloadAlreadyScanned,
39+
): string {
40+
return `${workload.namespace}/${workload.type}/${workload.uid}`;
41+
}
42+
43+
function getWorkloadImageAlreadyScannedKey(
44+
workload: WorkloadAlreadyScanned,
45+
imageId: string,
46+
): string {
47+
return `${workload.namespace}/${workload.type}/${workload.uid}/${imageId}`;
48+
}
49+
50+
export async function getWorkloadAlreadyScanned(
51+
workload: WorkloadAlreadyScanned,
52+
): Promise<string | undefined> {
53+
const key = getWorkloadAlreadyScannedKey(workload);
54+
return state.workloadsAlreadyScanned.get(key);
55+
}
56+
57+
export async function setWorkloadAlreadyScanned(
58+
workload: WorkloadAlreadyScanned,
59+
value: string,
60+
): Promise<boolean> {
61+
const key = getWorkloadAlreadyScannedKey(workload);
62+
return state.workloadsAlreadyScanned.set(key, value);
63+
}
64+
65+
export async function deleteWorkloadAlreadyScanned(
66+
workload: WorkloadAlreadyScanned,
67+
): Promise<void> {
68+
const key = getWorkloadAlreadyScannedKey(workload);
69+
state.workloadsAlreadyScanned.del(key);
70+
}
71+
72+
export async function getWorkloadImageAlreadyScanned(
73+
workload: WorkloadAlreadyScanned,
74+
imageId: string,
75+
): Promise<string | undefined> {
76+
const key = getWorkloadImageAlreadyScannedKey(workload, imageId);
77+
return state.imagesAlreadyScanned.get(key);
78+
}
79+
80+
export async function setWorkloadImageAlreadyScanned(
81+
workload: WorkloadAlreadyScanned,
82+
imageId: string,
83+
value: string,
84+
): Promise<boolean> {
85+
const key = getWorkloadImageAlreadyScannedKey(workload, imageId);
86+
return state.imagesAlreadyScanned.set(key, value);
87+
}
88+
89+
export async function deleteWorkloadImagesAlreadyScanned(
90+
workload: WorkloadImagesAlreadyScanned,
91+
): Promise<void> {
92+
for (const imageId of workload.imageIds) {
93+
const key = getWorkloadImageAlreadyScannedKey(workload, imageId);
94+
state.imagesAlreadyScanned.del(key);
95+
}
96+
}
97+
98+
export function kubernetesObjectToWorkloadAlreadyScanned(
99+
workload: KubernetesObject,
100+
): WorkloadAlreadyScanned | undefined {
101+
if (
102+
workload.metadata &&
103+
workload.metadata.namespace &&
104+
workload.metadata.uid &&
105+
workload.kind
106+
) {
107+
return {
108+
namespace: workload.metadata.namespace,
109+
type: workload.kind,
110+
uid: workload.metadata.uid,
111+
};
112+
}
113+
return undefined;
114+
}
115+
116+
export const state = {
25117
shutdownInProgress: false,
26118
imagesAlreadyScanned: new LruCache<string, string>(imagesLruCacheOptions),
27119
workloadsAlreadyScanned: new LruCache<string, string>(
28120
workloadsLruCacheOptions,
29121
),
30122
watchedNamespaces: {} as Record<string, V1Namespace>,
31123
};
32-
33-
export { state };

‎src/supervisor/watchers/handlers/cron-job.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';
55
import { IncomingMessage } from 'http';
66
import { k8sApi } from '../../cluster';
77
import { paginatedList } from './pagination';
8+
import {
9+
deleteWorkloadAlreadyScanned,
10+
deleteWorkloadImagesAlreadyScanned,
11+
kubernetesObjectToWorkloadAlreadyScanned,
12+
} from '../../../state';
813

914
export async function paginatedCronJobList(namespace: string): Promise<{
1015
response: IncomingMessage;
@@ -37,6 +42,20 @@ export async function cronJobWatchHandler(
3742
return;
3843
}
3944

45+
const workloadAlreadyScanned =
46+
kubernetesObjectToWorkloadAlreadyScanned(cronJob);
47+
if (workloadAlreadyScanned !== undefined) {
48+
await Promise.all([
49+
deleteWorkloadAlreadyScanned(workloadAlreadyScanned),
50+
deleteWorkloadImagesAlreadyScanned({
51+
...workloadAlreadyScanned,
52+
imageIds: cronJob.spec.jobTemplate.spec.template.spec.containers
53+
.filter((container) => container.image !== undefined)
54+
.map((container) => container.image!),
55+
}),
56+
]);
57+
}
58+
4059
const workloadName = cronJob.metadata.name || FALSY_WORKLOAD_NAME_MARKER;
4160

4261
await deleteWorkload(

‎src/supervisor/watchers/handlers/daemon-set.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';
55
import { IncomingMessage } from 'http';
66
import { k8sApi } from '../../cluster';
77
import { paginatedList } from './pagination';
8+
import {
9+
deleteWorkloadAlreadyScanned,
10+
deleteWorkloadImagesAlreadyScanned,
11+
kubernetesObjectToWorkloadAlreadyScanned,
12+
} from '../../../state';
813

914
export async function paginatedDaemonSetList(namespace: string): Promise<{
1015
response: IncomingMessage;
@@ -35,6 +40,20 @@ export async function daemonSetWatchHandler(
3540
return;
3641
}
3742

43+
const workloadAlreadyScanned =
44+
kubernetesObjectToWorkloadAlreadyScanned(daemonSet);
45+
if (workloadAlreadyScanned !== undefined) {
46+
await Promise.all([
47+
deleteWorkloadAlreadyScanned(workloadAlreadyScanned),
48+
deleteWorkloadImagesAlreadyScanned({
49+
...workloadAlreadyScanned,
50+
imageIds: daemonSet.spec.template.spec.containers
51+
.filter((container) => container.image !== undefined)
52+
.map((container) => container.image!),
53+
}),
54+
]);
55+
}
56+
3857
const workloadName = daemonSet.metadata.name || FALSY_WORKLOAD_NAME_MARKER;
3958

4059
await deleteWorkload(

‎src/supervisor/watchers/handlers/deployment-config.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ import {
88
} from './types';
99
import { paginatedList } from './pagination';
1010
import { k8sApi } from '../../cluster';
11+
import {
12+
deleteWorkloadAlreadyScanned,
13+
deleteWorkloadImagesAlreadyScanned,
14+
kubernetesObjectToWorkloadAlreadyScanned,
15+
} from '../../../state';
1116

1217
export async function paginatedDeploymentConfigList(
1318
namespace: string,
@@ -59,6 +64,20 @@ export async function deploymentConfigWatchHandler(
5964
return;
6065
}
6166

67+
const workloadAlreadyScanned =
68+
kubernetesObjectToWorkloadAlreadyScanned(deploymentConfig);
69+
if (workloadAlreadyScanned !== undefined) {
70+
await Promise.all([
71+
deleteWorkloadAlreadyScanned(workloadAlreadyScanned),
72+
deleteWorkloadImagesAlreadyScanned({
73+
...workloadAlreadyScanned,
74+
imageIds: deploymentConfig.spec.template.spec.containers
75+
.filter((container) => container.image !== undefined)
76+
.map((container) => container.image!),
77+
}),
78+
]);
79+
}
80+
6281
const workloadName =
6382
deploymentConfig.metadata.name || FALSY_WORKLOAD_NAME_MARKER;
6483

‎src/supervisor/watchers/handlers/deployment.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';
55
import { IncomingMessage } from 'http';
66
import { k8sApi } from '../../cluster';
77
import { paginatedList } from './pagination';
8+
import {
9+
deleteWorkloadAlreadyScanned,
10+
deleteWorkloadImagesAlreadyScanned,
11+
kubernetesObjectToWorkloadAlreadyScanned,
12+
} from '../../../state';
813

914
export async function paginatedDeploymentList(namespace: string): Promise<{
1015
response: IncomingMessage;
@@ -35,6 +40,20 @@ export async function deploymentWatchHandler(
3540
return;
3641
}
3742

43+
const workloadAlreadyScanned =
44+
kubernetesObjectToWorkloadAlreadyScanned(deployment);
45+
if (workloadAlreadyScanned !== undefined) {
46+
await Promise.all([
47+
deleteWorkloadAlreadyScanned(workloadAlreadyScanned),
48+
deleteWorkloadImagesAlreadyScanned({
49+
...workloadAlreadyScanned,
50+
imageIds: deployment.spec.template.spec.containers
51+
.filter((container) => container.image !== undefined)
52+
.map((container) => container.image!),
53+
}),
54+
]);
55+
}
56+
3857
const workloadName = deployment.metadata.name || FALSY_WORKLOAD_NAME_MARKER;
3958

4059
await deleteWorkload(

‎src/supervisor/watchers/handlers/job.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,11 @@ import { FALSY_WORKLOAD_NAME_MARKER } from './types';
55
import { IncomingMessage } from 'http';
66
import { k8sApi } from '../../cluster';
77
import { paginatedList } from './pagination';
8+
import {
9+
deleteWorkloadAlreadyScanned,
10+
deleteWorkloadImagesAlreadyScanned,
11+
kubernetesObjectToWorkloadAlreadyScanned,
12+
} from '../../../state';
813

914
export async function paginatedJobList(namespace: string): Promise<{
1015
response: IncomingMessage;
@@ -32,6 +37,19 @@ export async function jobWatchHandler(job: V1Job): Promise<void> {
3237
return;
3338
}
3439

40+
const workloadAlreadyScanned = kubernetesObjectToWorkloadAlreadyScanned(job);
41+
if (workloadAlreadyScanned !== undefined) {
42+
await Promise.all([
43+
deleteWorkloadAlreadyScanned(workloadAlreadyScanned),
44+
deleteWorkloadImagesAlreadyScanned({
45+
...workloadAlreadyScanned,
46+
imageIds: job.spec.template.spec.containers
47+
.filter((container) => container.image !== undefined)
48+
.map((container) => container.image!),
49+
}),
50+
]);
51+
}
52+
3553
const workloadName = job.metadata.name || FALSY_WORKLOAD_NAME_MARKER;
3654

3755
await deleteWorkload(

0 commit comments

Comments
 (0)