Skip to content

Commit 8d28850

Browse files
agnushaagnusha
committed
test: improve integration test reliability for sqs and s3 to work with serverless v4
1 parent 640b2db commit 8d28850

File tree

5 files changed

+331
-123
lines changed

5 files changed

+331
-123
lines changed

‎tests/serverless-plugins-integration/lambda/handler.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
module.exports.promise = (event, context) => {
2-
console.log(JSON.stringify(event));
2+
console.log('lambda event promise', JSON.stringify(event));
33
return Promise.resolve();
44
};
55

66
module.exports.callback = (event, context, cb) => {
7-
console.log(JSON.stringify(event));
7+
console.log('lambda event callback', JSON.stringify(event));
88
cb();
99
};

‎tests/serverless-plugins-integration/serverless.sqs.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ functions:
4747

4848
resources:
4949
Resources:
50+
MyThirdQueue:
51+
Type: AWS::SQS::Queue
52+
Properties:
53+
QueueName: MyThirdQueue
5054
MyFourthQueue:
5155
Type: AWS::SQS::Queue
5256
Properties:

‎tests/serverless-plugins-integration/test-s3.js‎

Lines changed: 75 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ const onExit = require('signal-exit');
55
const pump = require('pump');
66
const {delay, getSplitLinesTransform} = require('./utils');
77

8+
const EXPECTED_LAMBDA_CALL = 7;
9+
const PATH = './files/test.txt';
10+
811
const client = new Minio.Client({
912
region: 'eu-west-1',
1013
endPoint: 'localhost',
@@ -14,45 +17,96 @@ const client = new Minio.Client({
1417
useSSL: false
1518
});
1619

17-
const path = './files/test.txt';
1820
const uploadFiles = async () => {
1921
await delay(1000);
2022

21-
await Promise.all([
22-
client.fPutObject('documents', 'first.txt', path),
23-
client.fPutObject('pictures', 'first.txt', path),
24-
client.fPutObject('files', 'first.txt', path),
25-
client.fPutObject('documents', 'second.txt', path),
26-
client.fPutObject('pictures', 'second.txt', path),
27-
client.fPutObject('files', 'second.txt', path),
28-
client.fPutObject('others', 'correct/test.txt', path),
29-
client.fPutObject('others', 'wrong/test.csv', path),
30-
client.fPutObject('others', 'correct/test.csv', path),
31-
client.fPutObject('others', 'wrong/test.txt', path)
32-
]);
23+
await client.fPutObject('documents', 'first.txt', PATH);
24+
await client.fPutObject('documents', 'second.txt', PATH);
25+
await delay(1000);
26+
27+
await client.fPutObject('pictures', 'first.txt', PATH);
28+
await client.fPutObject('pictures', 'second.txt', PATH);
29+
await delay(1000);
30+
31+
await client.fPutObject('files', 'first.txt', PATH);
32+
await client.fPutObject('files', 'second.txt', PATH);
33+
await delay(1000);
34+
35+
await client.fPutObject('others', 'correct/test.txt', PATH);
36+
await client.fPutObject('others', 'wrong/test.csv', PATH);
37+
await client.fPutObject('others', 'correct/test.csv', PATH);
38+
await client.fPutObject('others', 'wrong/test.txt', PATH);
3339
};
34-
const EXPECTED_LAMBDA_CALL = 9; // pictures files are consumed twice, by myPromiseHandler and myPythonHandler
3540

3641
const serverless = spawn('sls', ['offline', 'start', '--config', 'serverless.s3.yml'], {
3742
stdio: ['pipe', 'pipe', 'pipe'],
3843
cwd: __dirname
3944
});
4045

46+
let lambdaCallCounter = 0;
47+
const processedEvents = new Set();
48+
49+
function incrementlambdaCallCounter(eventId) {
50+
if (eventId && processedEvents.has(eventId)) {
51+
return;
52+
}
53+
54+
if (eventId) {
55+
processedEvents.add(eventId);
56+
}
57+
58+
lambdaCallCounter++;
59+
console.debug(`lambda call count for s3 test: ${lambdaCallCounter}/${EXPECTED_LAMBDA_CALL}`);
60+
61+
if (lambdaCallCounter >= EXPECTED_LAMBDA_CALL) {
62+
console.log(`${lambdaCallCounter}/${EXPECTED_LAMBDA_CALL} lambda calls reached`);
63+
serverless.kill();
64+
}
65+
}
66+
67+
function processS3Event(output) {
68+
if (!output.includes('Records') || !output.includes('eventSource":"minio:s3"')) return;
69+
70+
try {
71+
output
72+
.split('\n')
73+
.filter(line => line.includes('Records') && line.includes('eventSource":"minio:s3"'))
74+
.forEach(line => {
75+
const jsonStart = line.indexOf('{');
76+
if (jsonStart < 0) return;
77+
78+
const jsonEnd = line.lastIndexOf('}');
79+
if (jsonEnd <= jsonStart) return;
80+
81+
const jsonStr = line.slice(jsonStart, jsonEnd + 1);
82+
83+
const eventData = JSON.parse(jsonStr);
84+
if (!eventData || !eventData.Records || eventData.Records.length === 0) return;
85+
const eventId = `${eventData.Records[0].s3.bucket.name}-${eventData.Records[0].s3.object.key}`;
86+
incrementlambdaCallCounter(eventId);
87+
});
88+
} catch (err) {
89+
console.error('Error in processS3Event:', {err, output});
90+
}
91+
}
92+
93+
serverless.stdout.on('data', data => {
94+
processS3Event(data.toString());
95+
});
96+
97+
serverless.stderr.on('data', data => {
98+
console.log(`STDERR: ${data.toString().trim()}`);
99+
});
100+
41101
pump(
42102
serverless.stderr,
43103
getSplitLinesTransform(),
44104
new Writable({
45105
objectMode: true,
46-
write(line, enc, cb) {
106+
write(line, _enc, cb) {
47107
if (/Starting Offline S3/.test(line)) {
48108
uploadFiles();
49109
}
50-
51-
this.count =
52-
(this.count || 0) +
53-
(line.match(/\(λ: .*\) RequestId: .* Duration: .* ms {2}Billed Duration: .* ms/g) || [])
54-
.length;
55-
if (this.count === EXPECTED_LAMBDA_CALL) serverless.kill();
56110
cb();
57111
}
58112
})

‎tests/serverless-plugins-integration/test-sqs-autocreate.js‎

Lines changed: 84 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -3,64 +3,110 @@ const {spawn} = require('child_process');
33
const onExit = require('signal-exit');
44
const {SQS} = require('aws-sdk');
55
const pump = require('pump');
6-
const {getSplitLinesTransform} = require('./utils');
6+
const {delay, getSplitLinesTransform} = require('./utils');
7+
8+
const EXPECTED_LAMBDA_CALLS = 3;
79

810
const client = new SQS({
911
region: 'eu-west-1',
1012
accessKeyId: 'local',
1113
secretAccessKey: 'local',
12-
endpoint: 'http://localhost:9324'
14+
endpoint: 'http://localhost:9324',
15+
httpOptions: {
16+
connectTimeout: 4000,
17+
timeout: 8000
18+
}
1319
});
1420

15-
const sendMessages = () => {
16-
return Promise.all([
17-
client
18-
.sendMessage({
19-
QueueUrl: 'http://localhost:9324/queue/AutocreatedImplicitQueue',
20-
MessageBody: 'AutocreatedImplicitQueue'
21-
})
22-
.promise(),
23-
client
24-
.sendMessage({
25-
QueueUrl: 'http://localhost:9324/queue/AutocreatedQueue',
26-
MessageBody: 'AutocreatedQueue'
27-
})
28-
.promise(),
29-
client
30-
.sendMessage({
31-
QueueUrl: 'http://localhost:9324/queue/AutocreatedFifoQueue.fifo',
32-
MessageBody: 'AutocreatedFifoQueue',
33-
MessageGroupId: '1'
34-
})
35-
.promise()
36-
]);
21+
const sendMessages = async () => {
22+
await client
23+
.sendMessage({
24+
QueueUrl: 'http://localhost:9324/queue/AutocreatedImplicitQueue',
25+
MessageBody: 'AutocreatedImplicitQueue'
26+
})
27+
.promise();
28+
29+
await delay(1000);
30+
await client
31+
.sendMessage({
32+
QueueUrl: 'http://localhost:9324/queue/AutocreatedQueue',
33+
MessageBody: 'AutocreatedQueue'
34+
})
35+
.promise();
36+
await delay(1000);
37+
38+
await client
39+
.sendMessage({
40+
QueueUrl: 'http://localhost:9324/queue/AutocreatedFifoQueue.fifo',
41+
MessageBody: 'AutocreatedFifoQueue',
42+
MessageGroupId: '1'
43+
})
44+
.promise();
3745
};
3846

3947
const serverless = spawn('sls', ['offline', 'start', '--config', 'serverless.sqs.autocreate.yml'], {
4048
stdio: ['pipe', 'pipe', 'pipe'],
4149
cwd: __dirname
4250
});
4351

52+
let lambdaCallCount = 0;
53+
const processedEvents = new Set();
54+
55+
function incrementCounter(eventId) {
56+
if (eventId && processedEvents.has(eventId)) {
57+
return;
58+
}
59+
60+
if (eventId) {
61+
processedEvents.add(eventId);
62+
}
63+
64+
lambdaCallCount++;
65+
console.debug(`SQS-autocreate: Lambda call count: ${lambdaCallCount}/${EXPECTED_LAMBDA_CALLS}`);
66+
67+
if (lambdaCallCount >= EXPECTED_LAMBDA_CALLS) {
68+
console.log(
69+
`${lambdaCallCount}/${EXPECTED_LAMBDA_CALLS} lambda calls reached for sqs-autocreate test`
70+
);
71+
serverless.kill();
72+
}
73+
}
74+
75+
function processSqsEvent(output) {
76+
if (!output.includes('"eventSource":"aws:sqs"')) return;
77+
78+
try {
79+
const jsonStart = output.indexOf('{');
80+
if (jsonStart < 0) return;
81+
82+
const jsonStr = output.slice(Math.max(0, jsonStart));
83+
const eventData = JSON.parse(jsonStr);
84+
85+
if (!eventData || !eventData.Records || eventData.Records.length === 0) return;
86+
87+
const queueName = eventData.Records[0].eventSourceARN.split(':').pop();
88+
const eventId = `${queueName}-${eventData.Records[0].messageId}`;
89+
90+
incrementCounter(eventId);
91+
} catch (err) {
92+
console.error('Error in processSqsEvent:', err);
93+
}
94+
}
95+
96+
serverless.stdout.on('data', data => {
97+
const output = data.toString();
98+
processSqsEvent(output);
99+
});
100+
44101
pump(
45102
serverless.stderr,
46103
getSplitLinesTransform(),
47104
new Writable({
48105
objectMode: true,
49-
write(line, enc, cb) {
50-
console.log(line.toString());
106+
write(line, _enc, cb) {
51107
if (/Starting Offline SQS/.test(line)) {
52-
sendMessages()
53-
.then(() => console.log('sucessfully send messages'))
54-
.catch(err => {
55-
console.log('Some issue sending message:s', err.message);
56-
});
108+
sendMessages();
57109
}
58-
59-
this.count =
60-
(this.count || 0) +
61-
(line.match(/\(λ: .*\) RequestId: .* Duration: .* ms {2}Billed Duration: .* ms/g) || [])
62-
.length;
63-
if (this.count === 3) serverless.kill();
64110
cb();
65111
}
66112
})
@@ -96,6 +142,6 @@ serverless.on('close', code => {
96142
});
97143
});
98144

99-
onExit((code, signal) => {
145+
onExit((_code, signal) => {
100146
if (signal) serverless.kill(signal);
101147
});

0 commit comments

Comments
 (0)