Skip to content

change nodejs kafka client #141

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Leospard wants to merge 1 commit into
base: master
Choose a base branch
from
Open

change nodejs kafka client #141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
22 changes: 0 additions & 22 deletions ...sample-codes/quick-start-sample-codes-nodejs/kafka-producer-nodejs/README-en.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,20 +99,9 @@ c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13FC Initialize Start RequestId:
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13load code for handler:index.initialize
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:132022-08-08 10:34:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] Servers: alikafka-pre-cn-7mz2sr1xa00c-1-vpc.alikafka.aliyuncs.com:9092
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:132022-08-08 10:34:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] TopicName: HelloTopic
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:132022-08-08 10:34:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] connect ok
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13FC Initialize End RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13FC Invoke Start RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13load code for handler:index.handler
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:432022-08-08 10:34:43 1c233449-024d-4a67-8e7f-83fe3bab6bac [verbose] delivery-report err: null
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:432022-08-08 10:34:43 1c233449-024d-4a67-8e7f-83fe3bab6bac [verbose] delivery-report content: {
topic: 'HelloTopic',
partition: 0,
offset: 16,
key: null,
timestamp: 1659926083428,
value: <Buffer 7b 22 4b 65 79 22 3a 20 22 74 65 73 74 20 6e 6f 64 65 6a 73 31 34 20 6b 61 66 6b 61 20 70 72 6f 64 75 63 65 72 22 7d>,
size: 39
}
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:43FC Invoke End RequestId: 1c233449-024d-4a67-8e7f-83fe3bab6bac

Duration: 10006.53 ms, Billed Duration: 10007 ms, Memory Size: 128 MB, Max Memory Used: 52.28 MB
Expand Down Expand Up @@ -148,20 +137,9 @@ End of method: invoke
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13 load code for handler:index.initialize
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343fepcaa47e [verbose] Servers: alikafka-pre-cn-1-7mz2srikafkafka0-ada0-45d8-863f-a9343fepcaa47e .aliyuncs.com:9092
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] TopicName: HelloTopic
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] connect ok
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13FC Initialize End RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13FC Invoke Start RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13 load code for handler:index.handler
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] delivery-report err: null
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] delivery-report content: {
topic: 'HelloTopic',
partition: 8,
offset: 19,
key: null,
timestamp: 1659925933345,
value: <Buffer 7b 0a 20 20 20 20 22 6b 65 79 31 22 3a 20 22 4e 6f 64 65 6a 73 31 34 22 2c 0a 20 20 20 20 22 6b 65 79 32 22 3a 20 75 65 32 6 2c 0a 20 20 ... 20 more bytes>,
size: 70
}
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13FC Invoke End RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
````

Expand Down
22 changes: 0 additions & 22 deletions ...rt-sample-codes/quick-start-sample-codes-nodejs/kafka-producer-nodejs/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,20 +98,9 @@ c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13FC Initialize Start RequestId:
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13load code for handler:index.initialize
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:132022-08-08 10:34:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] Servers: alikafka-pre-cn-7mz2sr1xa00c-1-vpc.alikafka.aliyuncs.com:9092
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:132022-08-08 10:34:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] TopicName: HelloTopic
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:132022-08-08 10:34:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] connect ok
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13FC Initialize End RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13FC Invoke Start RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:13load code for handler:index.handler
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:432022-08-08 10:34:43 1c233449-024d-4a67-8e7f-83fe3bab6bac [verbose] delivery-report err: null
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:432022-08-08 10:34:43 1c233449-024d-4a67-8e7f-83fe3bab6bac [verbose] delivery-report content: {
topic: 'HelloTopic',
partition: 0,
offset: 16,
key: null,
timestamp: 1659926083428,
value: <Buffer 7b 22 4b 65 79 22 3a 20 22 74 65 73 74 20 6e 6f 64 65 6a 73 31 34 20 6b 61 66 6b 61 20 70 72 6f 64 75 63 65 72 22 7d>,
size: 39
}
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:34:43FC Invoke End RequestId: 1c233449-024d-4a67-8e7f-83fe3bab6bac

Duration: 10006.53 ms, Billed Duration: 10007 ms, Memory Size: 128 MB, Max Memory Used: 52.28 MB
Expand Down Expand Up @@ -148,20 +137,9 @@ End of method: invoke
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13load code for handler:index.initialize
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] Servers: alikafka-pre-cn-7mz2sr1xa00c-1-vpc.alikafka.aliyuncs.com:9092
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] TopicName: HelloTopic
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] connect ok
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13FC Initialize End RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13FC Invoke Start RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13load code for handler:index.handler
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] delivery-report err: null
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:132022-08-08 10:32:13 c47410a0-ada0-45d8-863f-a9343feaa47e [verbose] delivery-report content: {
topic: 'HelloTopic',
partition: 8,
offset: 19,
key: null,
timestamp: 1659925933345,
value: <Buffer 7b 0a 20 20 20 20 22 6b 65 79 31 22 3a 20 22 4e 6f 64 65 6a 73 31 34 22 2c 0a 20 20 20 20 22 6b 65 79 32 22 3a 20 22 76 61 6c 75 65 32 22 2c 0a 20 20 ... 20 more bytes>,
size: 70
}
c-62f075a3-58d34b2b8b444083bb972022-08-08 10:32:13FC Invoke End RequestId: c47410a0-ada0-45d8-863f-a9343feaa47e
```

Expand Down
4 changes: 0 additions & 4 deletions ...le-codes/quick-start-sample-codes-nodejs/kafka-producer-nodejs/src/README-en.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,6 @@ load code for handler: index.initialize
FC Initialize End RequestId: 9b230404-4e05-49ca-a3d6-3be4629b367f
FC Invoke Start RequestId: 3f19de00-ab31-48b4-a386-03cded9f97ee
load code for handler:index.handler
2022-07-31 14:30:31 3f19de00-ab31-48b4-a386-03cded9f97ee [verbose] connect ok
2022-07-31 14:30:36 3f19de00-ab31-48b4-a386-03cded9f97ee [verbose] delivery-report: producer ok
FC Invoke End RequestId: 3f19de00-ab31-48b4-a386-03cded9f97ee
````

Expand Down Expand Up @@ -149,8 +147,6 @@ load code for handler: index.initialize
FC Initialize End RequestId: f9f17f0b-62fb-4ffa-853a-67fb316e0bf1
FC Invoke Start RequestId: f9f17f0b-62fb-4ffa-853a-67fb316e0bf1
load code for handler:index.handler
2022-07-31T06:22:37.621Z f9f17f0b-62fb-4ffa-853a-67fb316e0bf1 [verbose] connect ok
2022-07-31T06:22:37.951Z f9f17f0b-62fb-4ffa-853a-67fb316e0bf1 [verbose] delivery-report: producer ok
FC Invoke End RequestId: f9f17f0b-62fb-4ffa-853a-67fb316e0bf1

Duration: 10006.34 ms, Billed Duration: 10007 ms, Memory Size: 128 MB, Max Memory Used: 53.24 MB
Expand Down
4 changes: 0 additions & 4 deletions ...ample-codes/quick-start-sample-codes-nodejs/kafka-producer-nodejs/src/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,6 @@ load code for handler:index.initialize
FC Initialize End RequestId: 9b230404-4e05-49ca-a3d6-3be4629b367f
FC Invoke Start RequestId: 3f19de00-ab31-48b4-a386-03cded9f97ee
load code for handler:index.handler
2022-07-31 14:30:31 3f19de00-ab31-48b4-a386-03cded9f97ee [verbose] connect ok
2022-07-31 14:30:36 3f19de00-ab31-48b4-a386-03cded9f97ee [verbose] delivery-report: producer ok
FC Invoke End RequestId: 3f19de00-ab31-48b4-a386-03cded9f97ee
```

Expand Down Expand Up @@ -149,8 +147,6 @@ load code for handler:index.initialize
FC Initialize End RequestId: f9f17f0b-62fb-4ffa-853a-67fb316e0bf1
FC Invoke Start RequestId: f9f17f0b-62fb-4ffa-853a-67fb316e0bf1
load code for handler:index.handler
2022-07-31T06:22:37.621Z f9f17f0b-62fb-4ffa-853a-67fb316e0bf1 [verbose] connect ok
2022-07-31T06:22:37.951Z f9f17f0b-62fb-4ffa-853a-67fb316e0bf1 [verbose] delivery-report: producer ok
FC Invoke End RequestId: f9f17f0b-62fb-4ffa-853a-67fb316e0bf1

Duration: 10006.34 ms, Billed Duration: 10007 ms, Memory Size: 128 MB, Max Memory Used: 53.24 MB
Expand Down
86 changes: 25 additions & 61 deletions ...tart-sample-codes/quick-start-sample-codes-nodejs/kafka-producer-nodejs/src/code/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,80 +1,44 @@
'use strict';

const Kafka = require('node-rdkafka');
const Kafka = require('kafkajs');

var BOOTSTRAP_SERVERS
var TOPIC_NAME
var producer

exports.initialize = (context, callback) => {
exports.initialize = async (context, callback) => {
BOOTSTRAP_SERVERS = process.env.BOOTSTRAP_SERVERS
TOPIC_NAME = process.env.TOPIC_NAME
console.log("Servers: ", BOOTSTRAP_SERVERS);
console.log("TopicName: ", TOPIC_NAME);
producer = new Kafka.Producer({
'api.version.request': 'true',
'bootstrap.servers': BOOTSTRAP_SERVERS,
'dr_cb': true,
'dr_msg_cb': true,
'message.send.max.retries': 10
});

producer.on('event.log', function(event) {
console.log("event.log", event);
callback(new Error(event.message), "");
});

producer.on("error", function(err) {
console.log("error:" + err);
callback(err, "");
});

// Any errors we encounter, including connection errors
producer.on('event.error', function(err) {
console.error('event.error:' + err);
callback(err, "");
var servers = BOOTSTRAP_SERVERS.split(",");
const kafka = new Kafka.Kafka({
clientId: 'testId',
brokers: servers
})
producer = kafka.producer()
await producer.connect()

// Poll for events every 10 ms
producer.setPollInterval(10);

producer.connect();

// Wait for the ready event before proceeding
producer.on('ready', function() {
console.log("connect ok")
callback(null, "");
});
};

exports.handler = async(event, context, callback) => {
// Wait for connection
producer.produce(
TOPIC_NAME,
null,
Buffer.from(event),
null,
Date.now()
);
producer.flush();

// waiting for sending
await producer.on('delivery-report', function(err, report) {
console.log("delivery-report err: ", err);
console.log("delivery-report content: ", report);
callback(null,"initialize");
}

if (err == null) {
callback(null, "Finish sending the message:" + event);
} else {
callback(err, "Send message fail!");
}
});
exports.handler = async (event, context, callback) => {
try {
await producer.send({
topic: 'TestTopic',
messages: [
{ value: event },
],
})
callback(null,"Finish sending the message:" + event);
} catch (e) {
console.log(e)
callback(e,"Send message fail!");
}
}

module.exports.preStop = function(context, callback) {
exports.preStop = async function(context, callback) {
console.log('preStop hook start');
if (producer != null) {
producer.disconnect();
await producer.disconnect();
}
console.log('preStop hook finish');
callback(null, "");
Expand Down
22 changes: 0 additions & 22 deletions ...le-codes-nodejs/kafka-producer-nodejs/src/code/node_modules/bindings/LICENSE.md
View file
Open in desktop

This file was deleted.

98 changes: 0 additions & 98 deletions ...ple-codes-nodejs/kafka-producer-nodejs/src/code/node_modules/bindings/README.md
View file
Open in desktop

This file was deleted.

Loading