Skip to content

Commit a34afb5

Browse files
mroiter-larusmroiter-larus
authored
Fixes #298: Add some notes to the documentation (#301)
1 parent 3a6d184 commit a34afb5

File tree

12 files changed

+470
-10
lines changed

12 files changed

+470
-10
lines changed

‎doc/asciidoc/consumer/index.adoc‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,20 @@ MERGE (n:Label {id: event.id})
7777
ON CREATE SET n += event.properties
7878
----
7979

80+
[NOTE]
81+
====
82+
When you decide to use Cypher template as Sink strategy to import data from Kafka into Neo4j, you have to be sure
83+
about the query correctness. If the query is not optimized, this could also results into possible performance issue or
84+
in situations where the plugin seems to be stuck, for example if the query loads a large amount of nodes and relationships
85+
into memory.
86+
We suggests the following:
87+
88+
* execute a query **EXPLAIN** in order to better analyze the query and avoid this kind of situations
89+
90+
* if Neo4j seems to be stuck then, from the Neo4j Browser, execute a `CALL dbms.listQueries()` to view all queries that
91+
are currently executing within the instance, and to be sure that there are no locked queries
92+
====
93+
8094
include::../sink-strategies/index.adoc[]
8195

8296
include::../cud-file-format/index.adoc[]

‎doc/asciidoc/docker/data/docker-compose-cluster.yml‎

Lines changed: 284 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,284 @@
1+
version: '3'
2+
3+
networks:
4+
kafka_cluster:
5+
driver: bridge
6+
7+
services:
8+
9+
core1:
10+
image: neo4j:4.0.3-enterprise
11+
hostname: core1
12+
container_name: core1
13+
ports:
14+
- 7474:7474
15+
- 6477:6477
16+
- 7687:7687
17+
volumes:
18+
- ./neo4j-cluster-40/core1/plugins:/plugins
19+
networks:
20+
- kafka_cluster
21+
environment:
22+
NEO4J_ACCEPT_LICENSE_AGREEMENT: "yes"
23+
NEO4J_AUTH: neo4j/streams
24+
NEO4J_dbms_mode: CORE
25+
NEO4J_causalClustering_expectedCoreClusterSize: 3
26+
NEO4J_causalClustering_initialDiscoveryMembers: core1:5000,core2:5000,core3:5000
27+
NEO4J_dbms_connector_http_listen__address: :7474
28+
NEO4J_dbms_connector_https_listen__address: :6477
29+
NEO4J_dbms_connector_bolt_listen__address: :7687
30+
NEO4J_dbms_logs_debug_level: DEBUG
31+
NEO4J_apoc_import_file_enabled: "true"
32+
NEO4J_kafka_auto_offset_reset: "latest"
33+
NEO4J_kafka_zookeeper_connect: zookeeper-1:22181,zookeeper-2:32181,zookeeper-3:42181
34+
NEO4J_kafka_bootstrap_servers: broker-1:29092,broker-2:39092,broker-3:49092
35+
NEO4J_kafka_group_id: "neo4j"
36+
NEO4J_kafka_client_id: "neo4j"
37+
NEO4J_kafka_enable_auto_commit: "false"
38+
NEO4J_kafka_key_deserializer: org.apache.kafka.common.serialization.ByteArrayDeserializer
39+
NEO4J_kafka_value_deserializer: org.apache.kafka.common.serialization.ByteArrayDeserializer
40+
NEO4J_streams_source_enabled: "false"
41+
NEO4J_streams_sink_enabled_to_dbtest: "true"
42+
NEO4J_streams_sink_topic_cypher_mytopic_to_dbtest: "CREATE (n:Person {id: event.id, name: event.name, surname: event.surname}) RETURN n"
43+
44+
core2:
45+
image: neo4j:4.0.3-enterprise
46+
hostname: core2
47+
container_name: core2
48+
ports:
49+
- 7475:7475
50+
- 6478:6478
51+
- 7688:7688
52+
volumes:
53+
- ./neo4j-cluster-40/core2/plugins:/plugins
54+
networks:
55+
- kafka_cluster
56+
environment:
57+
NEO4J_ACCEPT_LICENSE_AGREEMENT: "yes"
58+
NEO4J_AUTH: neo4j/streams
59+
NEO4J_dbms_mode: CORE
60+
NEO4J_causalClustering_expectedCoreClusterSize: 3
61+
NEO4J_causalClustering_initialDiscoveryMembers: core1:5000,core2:5000,core3:5000
62+
NEO4J_dbms_connector_http_listen__address: :7475
63+
NEO4J_dbms_connector_https_listen__address: :6478
64+
NEO4J_dbms_connector_bolt_listen__address: :7688
65+
NEO4J_dbms_logs_debug_level: DEBUG
66+
NEO4J_apoc_import_file_enabled: "true"
67+
NEO4J_kafka_auto_offset_reset: "latest"
68+
NEO4J_kafka_zookeeper_connect: zookeeper-1:22181,zookeeper-2:32181,zookeeper-3:42181
69+
NEO4J_kafka_bootstrap_servers: broker-1:29092,broker-2:39092,broker-3:49092
70+
NEO4J_kafka_group_id: "neo4j"
71+
NEO4J_kafka_client_id: "neo4j"
72+
NEO4J_kafka_enable_auto_commit: "false"
73+
NEO4J_kafka_key_deserializer: org.apache.kafka.common.serialization.ByteArrayDeserializer
74+
NEO4J_kafka_value_deserializer: org.apache.kafka.common.serialization.ByteArrayDeserializer
75+
NEO4J_streams_source_enabled: "false"
76+
NEO4J_streams_sink_enabled_to_dbtest: "true"
77+
NEO4J_streams_sink_topic_cypher_mytopic_to_dbtest: "CREATE (n:Person {id: event.id, name: event.name, surname: event.surname}) RETURN n"
78+
79+
core3:
80+
image: neo4j:4.0.3-enterprise
81+
hostname: core3
82+
container_name: core3
83+
ports:
84+
- 7476:7476
85+
- 6479:6479
86+
- 7689:7689
87+
volumes:
88+
- ./neo4j-cluster-40/core3/plugins:/plugins
89+
networks:
90+
- kafka_cluster
91+
environment:
92+
NEO4J_ACCEPT_LICENSE_AGREEMENT: "yes"
93+
NEO4J_AUTH: neo4j/streams
94+
NEO4J_dbms_mode: CORE
95+
NEO4J_causalClustering_expectedCoreClusterSize: 3
96+
NEO4J_causalClustering_initialDiscoveryMembers: core1:5000,core2:5000,core3:5000
97+
NEO4J_dbms_connector_http_listen__address: :7476
98+
NEO4J_dbms_connector_https_listen__address: :6479
99+
NEO4J_dbms_connector_bolt_listen__address: :7689
100+
NEO4J_dbms_logs_debug_level: DEBUG
101+
NEO4J_kafka_zookeeper_connect: zookeeper-1:22181,zookeeper-2:32181,zookeeper-3:42181
102+
NEO4J_kafka_bootstrap_servers: broker-1:29092,broker-2:39092,broker-3:49092
103+
NEO4J_kafka_group_id: "neo4j"
104+
NEO4J_kafka_client_id: "neo4j"
105+
NEO4J_kafka_enable_auto_commit: "false"
106+
NEO4J_kafka_key_deserializer: org.apache.kafka.common.serialization.ByteArrayDeserializer
107+
NEO4J_kafka_value_deserializer: org.apache.kafka.common.serialization.ByteArrayDeserializer
108+
NEO4J_streams_source_enabled: "false"
109+
NEO4J_streams_sink_enabled_to_dbtest: "true"
110+
NEO4J_streams_sink_topic_cypher_mytopic_to_dbtest: "CREATE (n:Person {id: event.id, name: event.name, surname: event.surname}) RETURN n"
111+
112+
read1:
113+
image: neo4j:4.0.3-enterprise
114+
hostname: read1
115+
container_name: read1
116+
ports:
117+
- 7477:7477
118+
- 6480:6480
119+
- 7690:7690
120+
volumes:
121+
- ./neo4j-cluster-40/read1/plugins:/plugins
122+
networks:
123+
- kafka_cluster
124+
environment:
125+
NEO4J_ACCEPT_LICENSE_AGREEMENT: "yes"
126+
NEO4J_AUTH: neo4j/streams
127+
NEO4J_dbms_mode: READ_REPLICA
128+
NEO4J_causalClustering_expectedCoreClusterSize: 3
129+
NEO4J_causalClustering_initialDiscoveryMembers: core1:5000,core2:5000,core3:5000
130+
NEO4J_dbms_connector_http_listen__address: :7477
131+
NEO4J_dbms_connector_https_listen__address: :6480
132+
NEO4J_dbms_connector_bolt_listen__address: :7690
133+
NEO4J_dbms_logs_debug_level: DEBUG
134+
NEO4J_kafka_zookeeper_connect: zookeeper-1:22181,zookeeper-2:32181,zookeeper-3:42181
135+
NEO4J_kafka_bootstrap_servers: broker-1:29092,broker-2:39092,broker-3:49092
136+
NEO4J_kafka_group_id: "neo4j"
137+
NEO4J_kafka_client_id: "neo4j"
138+
NEO4J_kafka_enable_auto_commit: "false"
139+
NEO4J_kafka_key_deserializer: org.apache.kafka.common.serialization.ByteArrayDeserializer
140+
NEO4J_kafka_value_deserializer: org.apache.kafka.common.serialization.ByteArrayDeserializer
141+
NEO4J_streams_source_enabled: "false"
142+
NEO4J_streams_sink_enabled_to_dbtest: "true"
143+
NEO4J_streams_sink_topic_cypher_mytopic_to_dbtest: "CREATE (n:Person {id: event.id, name: event.name, surname: event.surname}) RETURN n"
144+
145+
zookeeper-1:
146+
image: confluentinc/cp-zookeeper
147+
hostname: zookeeper-1
148+
container_name: zookeeper-1
149+
ports:
150+
- 22181:22181
151+
- 22888:22888
152+
- 23888:23888
153+
volumes:
154+
- ./zookeeper-1/data:/data
155+
environment:
156+
ZOOKEEPER_SERVER_ID: 1
157+
ZOOKEEPER_CLIENT_PORT: 22181
158+
ZOOKEEPER_TICK_TIME: 2000
159+
ZOOKEEPER_INIT_LIMIT: 5
160+
ZOOKEEPER_SYNC_LIMIT: 2
161+
ZOOKEEPER_SERVERS: zookeeper-1:22888:23888;zookeeper-2:32888:33888;zookeeper-3:42888:43888
162+
networks:
163+
- kafka_cluster
164+
165+
zookeeper-2:
166+
image: confluentinc/cp-zookeeper
167+
hostname: zookeeper-2
168+
container_name: zookeeper-2
169+
ports:
170+
- 32181:32181
171+
- 32888:32888
172+
- 33888:33888
173+
volumes:
174+
- ./zookeeper-2/data:/data
175+
environment:
176+
ZOOKEEPER_SERVER_ID: 2
177+
ZOOKEEPER_CLIENT_PORT: 32181
178+
ZOOKEEPER_TICK_TIME: 2000
179+
ZOOKEEPER_INIT_LIMIT: 5
180+
ZOOKEEPER_SYNC_LIMIT: 2
181+
ZOOKEEPER_SERVERS: zookeeper-1:22888:23888;zookeeper-2:32888:33888;zookeeper-3:42888:43888
182+
networks:
183+
- kafka_cluster
184+
185+
zookeeper-3:
186+
image: confluentinc/cp-zookeeper
187+
hostname: zookeeper-3
188+
container_name: zookeeper-3
189+
ports:
190+
- 42181:42181
191+
- 42888:42888
192+
- 43888:43888
193+
volumes:
194+
- ./zookeeper-3/data:/data
195+
environment:
196+
ZOOKEEPER_SERVER_ID: 3
197+
ZOOKEEPER_CLIENT_PORT: 42181
198+
ZOOKEEPER_TICK_TIME: 2000
199+
ZOOKEEPER_INIT_LIMIT: 5
200+
ZOOKEEPER_SYNC_LIMIT: 2
201+
ZOOKEEPER_SERVERS: zookeeper-1:22888:23888;zookeeper-2:32888:33888;zookeeper-3:42888:43888
202+
networks:
203+
- kafka_cluster
204+
205+
broker-1:
206+
image: confluentinc/cp-kafka
207+
hostname: broker-1
208+
container_name: broker-1
209+
ports:
210+
- 9092:9092
211+
- 29092:29092
212+
depends_on:
213+
- zookeeper-1
214+
- zookeeper-2
215+
- zookeeper-3
216+
environment:
217+
KAFKA_BROKER_ID: 1
218+
KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:22181,zookeeper-2:32181,zookeeper-3:42181
219+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
220+
KAFKA_LISTENERS: PLAINTEXT://:29092,PLAINTEXT_HOST://:9092
221+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-1:29092,PLAINTEXT_HOST://localhost:9092
222+
ALLOW_PLAINTEXT_LISTENER: 'yes'
223+
KAFKA_AUTO_OFFSET_RESET: "latest"
224+
KAFKA_MAX_POLL_INTERVAL_MS: 300000
225+
KAFKA_MAX_POLL_RECORDS: 20000
226+
KAFKA_MAX_PARTITION_FETCH_BYTES: 52428800
227+
KAFKA_NUM_PARTITIONS: 2
228+
KAFKA_MESSAGE_MAX_BYTES: 20220088
229+
networks:
230+
- kafka_cluster
231+
232+
broker-2:
233+
image: confluentinc/cp-kafka
234+
hostname: broker-2
235+
container_name: broker-2
236+
ports:
237+
- 9093:9093
238+
- 39092:39092
239+
depends_on:
240+
- zookeeper-1
241+
- zookeeper-2
242+
- zookeeper-3
243+
environment:
244+
KAFKA_BROKER_ID: 2
245+
KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:22181,zookeeper-2:32181,zookeeper-3:42181
246+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
247+
KAFKA_LISTENERS: PLAINTEXT://:39092,PLAINTEXT_HOST://:9093
248+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-2:39092,PLAINTEXT_HOST://localhost:9093
249+
ALLOW_PLAINTEXT_LISTENER: 'yes'
250+
KAFKA_AUTO_OFFSET_RESET: "latest"
251+
KAFKA_MAX_POLL_INTERVAL_MS: 300000
252+
KAFKA_MAX_POLL_RECORDS: 20000
253+
KAFKA_MAX_PARTITION_FETCH_BYTES: 52428800
254+
KAFKA_NUM_PARTITIONS: 2
255+
KAFKA_MESSAGE_MAX_BYTES: 20220088
256+
networks:
257+
- kafka_cluster
258+
259+
broker-3:
260+
image: confluentinc/cp-kafka
261+
hostname: broker-3
262+
container_name: broker-3
263+
ports:
264+
- 9094:9094
265+
- 49092:49092
266+
depends_on:
267+
- zookeeper-1
268+
- zookeeper-2
269+
- zookeeper-3
270+
environment:
271+
KAFKA_BROKER_ID: 3
272+
KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:22181,zookeeper-2:32181,zookeeper-3:42181
273+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
274+
KAFKA_LISTENERS: PLAINTEXT://:49092,PLAINTEXT_HOST://:9094
275+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-3:49092,PLAINTEXT_HOST://localhost:9094
276+
ALLOW_PLAINTEXT_LISTENER: 'yes'
277+
KAFKA_AUTO_OFFSET_RESET: "latest"
278+
KAFKA_MAX_POLL_INTERVAL_MS: 300000
279+
KAFKA_MAX_POLL_RECORDS: 20000
280+
KAFKA_MAX_PARTITION_FETCH_BYTES: 52428800
281+
KAFKA_NUM_PARTITIONS: 2
282+
KAFKA_MESSAGE_MAX_BYTES: 20220088
283+
networks:
284+
- kafka_cluster

‎doc/asciidoc/docker/data/docker-compose-source-sink.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ version: '3'
22

33
services:
44
neo4j-source:
5-
image: neo4j:3.5
5+
image: neo4j:4.0.3
66
hostname: neo4j-source
77
container_name: neo4j-source
88
depends_on:
@@ -24,7 +24,7 @@ services:
2424
NEO4J_streams_source_schema_polling_interval: 10000
2525

2626
neo4j-sink:
27-
image: neo4j:3.5
27+
image: neo4j:4.0.3
2828
hostname: neo4j-sink
2929
container_name: neo4j-sink
3030
depends_on:

‎doc/asciidoc/docker/data/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
version: '3'
22
services:
33
neo4j:
4-
image: neo4j:3.5
4+
image: neo4j:4.0.3
55
hostname: neo4j
66
container_name: neo4j
77
ports:

0 commit comments

Comments
 (0)