Skip to content

Commit 4ab3beb

Browse files
bgeorge111bgeorge
bgeorge111
and
bgeorge
authored
AVRO, JSON Schema and ProtoBuf support, ID Generation Strategies. (#36)
* Removing some erroneous comments in the example connect-standalone properties file. * Changes for additional Security options * Updated build.gradle to new kafka connect and datahub * Updated documentation for v1.2.2 * Updated Test Cases. * Some forgotten cleanup left over from building the Confluent archive. * Support for JSON Schema, ProtoBuf messages. * Also support of ID strategies - UUID, JSONPATH,HASH, KAFKA_META_WITH_SLASH, KAFKA_META_HASHED m> Co-authored-by: bgeorge <[email protected]>
1 parent 2401021 commit 4ab3beb

24 files changed

+939
-345
lines changed

‎MarkLogic_Kafka_Connector_v1.2.2.pdf‎

870 KB
Binary file not shown.

‎MarkLogic_Kafka_Connector_v1.2.3.pdf‎

1.14 MB
Binary file not shown.

‎README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
# v1.2.2 Changes
2+
1. Support of additional authentication options
3+
2. Documentation of how to update the connector for security options
4+
5+
Refer MarkLogic_Kafka_Connector_v1.2.2.pdf for details
6+
17
# kafka-connect-marklogic
28

39
This is a connector for subscribing to Kafka queues and pushing messages to MarkLogic

‎build.gradle‎

Lines changed: 151 additions & 146 deletions
Original file line numberDiff line numberDiff line change
@@ -1,146 +1,151 @@
1-
plugins {
2-
id 'java'
3-
id 'net.saliman.properties' version '1.4.6'
4-
id "com.github.jk1.dependency-license-report" version "1.3"
5-
}
6-
7-
sourceCompatibility = 1.8
8-
9-
repositories {
10-
mavenCentral()
11-
jcenter()
12-
}
13-
14-
configurations {
15-
documentation
16-
assets
17-
}
18-
19-
dependencies {
20-
compileOnly "org.apache.kafka:connect-api:2.3.0"
21-
22-
compile ("com.marklogic:marklogic-data-hub:5.2.0") {
23-
// Excluding these because there's no need for them
24-
exclude module: "spring-boot-autoconfigure"
25-
exclude module: "spring-integration-http"
26-
exclude module: "jaeger-core"
27-
exclude module: "jaeger-thrift"
28-
29-
// Excluding because it causes Kafka Connect to complain mightily if included
30-
exclude module: "logback-classic"
31-
}
32-
33-
testCompile "org.junit.jupiter:junit-jupiter-api:5.3.0"
34-
testCompile "org.apache.kafka:connect-api:2.3.0"
35-
36-
// Needed by Gradle 4.6+ - see https://www.petrikainulainen.net/programming/testing/junit-5-tutorial-running-unit-tests-with-gradle/
37-
testRuntime "org.junit.jupiter:junit-jupiter-engine:5.3.0"
38-
39-
// Forcing logback to be used for test logging
40-
testRuntime "ch.qos.logback:logback-classic:1.1.8"
41-
testRuntime group: "org.slf4j", name: "jcl-over-slf4j", version: "1.7.22"
42-
testRuntime group: "org.slf4j", name: "slf4j-api", version: "1.7.22"
43-
44-
documentation files('LICENSE.txt')
45-
documentation files('NOTICE.txt')
46-
documentation files('README.md')
47-
48-
assets files('MarkLogic_logo.png')
49-
assets files('apache_logo.png')
50-
}
51-
52-
// Needed by Gradle 4.6+ - see https://www.petrikainulainen.net/programming/testing/junit-5-tutorial-running-unit-tests-with-gradle/
53-
test {
54-
useJUnitPlatform()
55-
}
56-
57-
// Customize the Java plugin's jar task to produce a "fat" jar with all dependencies included
58-
jar {
59-
from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
60-
}
61-
62-
task copyJarToKafka(type: Copy) {
63-
description = "Used for local development and testing; copies the jar to your local Kafka install"
64-
from "build/libs"
65-
into "${kafkaHome}/libs"
66-
}
67-
copyJarToKafka.mustRunAfter(jar)
68-
69-
task copyPropertyFilesToKafka(type: Copy) {
70-
description = "Used for local development and testing; copies the properties files to your local Kafka install"
71-
from "config"
72-
into "${kafkaHome}/config"
73-
}
74-
75-
task deploy {
76-
description = "Used for local development and testing; builds the jar and copies it and the properties files to your local Kafka install"
77-
dependsOn = ["jar", "copyJarToKafka", "copyPropertyFilesToKafka"]
78-
}
79-
80-
// Tasks for building the archive required for submitting to the Confluence Connector Hub
81-
import org.apache.tools.ant.filters.*
82-
task connectorArchive_CopyManifestToBuildDirectory(type: Copy) {
83-
description = "Copy the project manifest into the root folder"
84-
group = 'connector archive'
85-
86-
from '.'
87-
include 'manifest.json'
88-
into 'build/connectorArchive'
89-
filter(ReplaceTokens, tokens:[CONFLUENT_USER:componentOwner, VERSION:version])
90-
}
91-
92-
task connectorArchive_CopyAssetsToBuildDirectory(type: Copy) {
93-
description = "Copy the project assets into the assets folder"
94-
group = 'connector archive'
95-
96-
from configurations.assets
97-
into 'build/connectorArchive/assets'
98-
}
99-
100-
task connectorArchive_CopyEtcToBuildDirectory(type: Copy) {
101-
description = "Copy the project support files into the etc folder"
102-
group = 'connector archive'
103-
104-
from 'config'
105-
include '*'
106-
into 'build/connectorArchive/etc'
107-
}
108-
109-
task connectorArchive_CopyDocumentationToBuildDirectory(type: Copy) {
110-
description = "Copy the project documentation into the doc folder"
111-
group = 'connector archive'
112-
113-
from configurations.documentation
114-
into 'build/connectorArchive/doc'
115-
}
116-
117-
task connectorArchive_CopyDependenciesToBuildDirectory(type: Copy) {
118-
description = "Copy the dependency jars into the lib folder"
119-
group = 'connector archive'
120-
dependsOn = [jar]
121-
122-
from jar
123-
from configurations.compile
124-
into 'build/connectorArchive/lib'
125-
}
126-
127-
task connectorArchive_BuildDirectory() {
128-
description = "Build the directory that will be used to create the Kafka Connector Archive"
129-
dependsOn = [connectorArchive_CopyManifestToBuildDirectory,
130-
connectorArchive_CopyDependenciesToBuildDirectory,
131-
connectorArchive_CopyDocumentationToBuildDirectory,
132-
connectorArchive_CopyEtcToBuildDirectory,
133-
connectorArchive_CopyAssetsToBuildDirectory]
134-
group = 'connector archive'
135-
}
136-
137-
task connectorArchive(type: Zip, dependsOn: connectorArchive_BuildDirectory) {
138-
description = 'Build a Connector Hub for the Confluent Connector Hub'
139-
group = 'connector archive'
140-
141-
from 'build/connectorArchive'
142-
include '*'
143-
include '*/*'
144-
archiveName "${componentOwner}-${componentName}-${version}.zip"
145-
destinationDir(file('build/distro'))
146-
}
1+
plugins {
2+
id 'java'
3+
id 'net.saliman.properties' version '1.4.6'
4+
id "com.github.jk1.dependency-license-report" version "1.3"
5+
}
6+
7+
sourceCompatibility = 1.8
8+
9+
repositories {
10+
mavenCentral()
11+
jcenter()
12+
}
13+
14+
configurations {
15+
documentation
16+
assets
17+
}
18+
19+
dependencies {
20+
compileOnly "org.apache.kafka:connect-api:2.5.0"
21+
compileOnly "org.apache.kafka:connect-json:2.5.0"
22+
23+
24+
compile ("com.marklogic:marklogic-data-hub:5.2.2") {
25+
// Excluding these because there's no need for them
26+
exclude module: "spring-boot-autoconfigure"
27+
exclude module: "spring-integration-http"
28+
exclude module: "jaeger-core"
29+
exclude module: "jaeger-thrift"
30+
31+
// Excluding because it causes Kafka Connect to complain mightily if included
32+
exclude module: "logback-classic"
33+
}
34+
35+
testCompile "org.junit.jupiter:junit-jupiter-api:5.3.0"
36+
testCompile "org.apache.kafka:connect-api:2.5.0"
37+
testCompile "org.apache.kafka:connect-json:2.5.0"
38+
testCompile "com.google.code.gson:gson:2.8.6"
39+
40+
// Needed by Gradle 4.6+ - see https://www.petrikainulainen.net/programming/testing/junit-5-tutorial-running-unit-tests-with-gradle/
41+
testRuntime "org.junit.jupiter:junit-jupiter-engine:5.3.0"
42+
43+
// Forcing logback to be used for test logging
44+
testRuntime "ch.qos.logback:logback-classic:1.1.8"
45+
testRuntime group: "org.slf4j", name: "jcl-over-slf4j", version: "1.7.22"
46+
testRuntime group: "org.slf4j", name: "slf4j-api", version: "1.7.22"
47+
48+
documentation files('LICENSE.txt')
49+
documentation files('NOTICE.txt')
50+
documentation files('README.md')
51+
52+
assets files('MarkLogic_logo.png')
53+
assets files('apache_logo.png')
54+
}
55+
56+
// Needed by Gradle 4.6+ - see https://www.petrikainulainen.net/programming/testing/junit-5-tutorial-running-unit-tests-with-gradle/
57+
test {
58+
useJUnitPlatform()
59+
}
60+
61+
// Customize the Java plugin's jar task to produce a "fat" jar with all dependencies included
62+
jar {
63+
from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
64+
}
65+
66+
task copyJarToKafka(type: Copy) {
67+
description = "Used for local development and testing; copies the jar to your local Kafka install"
68+
from "build/libs"
69+
into "${kafkaHome}/libs"
70+
}
71+
copyJarToKafka.mustRunAfter(jar)
72+
73+
task copyPropertyFilesToKafka(type: Copy) {
74+
description = "Used for local development and testing; copies the properties files to your local Kafka install"
75+
from "config"
76+
into "${kafkaHome}/config"
77+
}
78+
79+
task deploy {
80+
description = "Used for local development and testing; builds the jar and copies it and the properties files to your local Kafka install"
81+
dependsOn = ["jar", "copyJarToKafka", "copyPropertyFilesToKafka"]
82+
}
83+
84+
// Tasks for building the archive required for submitting to the Confluence Connector Hub
85+
import org.apache.tools.ant.filters.*
86+
def baseArchiveBuildDir = "build/connectorArchive"
87+
def baseArchiveName = "${componentOwner}-${componentName}-${version}"
88+
task connectorArchive_CopyManifestToBuildDirectory(type: Copy) {
89+
description = "Copy the project manifest into the root folder"
90+
group = 'connector archive'
91+
92+
from '.'
93+
include 'manifest.json'
94+
into "${baseArchiveBuildDir}/${baseArchiveName}"
95+
filter(ReplaceTokens, tokens:[CONFLUENT_USER:componentOwner, VERSION:version])
96+
}
97+
98+
task connectorArchive_CopyAssetsToBuildDirectory(type: Copy) {
99+
description = "Copy the project assets into the assets folder"
100+
group = 'connector archive'
101+
102+
from configurations.assets
103+
into "${baseArchiveBuildDir}/${baseArchiveName}/assets"
104+
}
105+
106+
task connectorArchive_CopyEtcToBuildDirectory(type: Copy) {
107+
description = "Copy the project support files into the etc folder"
108+
group = 'connector archive'
109+
110+
from 'config'
111+
include '*'
112+
into "${baseArchiveBuildDir}/${baseArchiveName}/etc"
113+
}
114+
115+
task connectorArchive_CopyDocumentationToBuildDirectory(type: Copy) {
116+
description = "Copy the project documentation into the doc folder"
117+
group = 'connector archive'
118+
119+
from configurations.documentation
120+
into "${baseArchiveBuildDir}/${baseArchiveName}/doc"
121+
}
122+
123+
task connectorArchive_CopyDependenciesToBuildDirectory(type: Copy) {
124+
description = "Copy the dependency jars into the lib folder"
125+
group = 'connector archive'
126+
dependsOn = [jar]
127+
128+
from jar
129+
from configurations.compile
130+
into "${baseArchiveBuildDir}/${baseArchiveName}/lib"
131+
}
132+
133+
task connectorArchive_BuildDirectory() {
134+
description = "Build the directory that will be used to create the Kafka Connector Archive"
135+
dependsOn = [connectorArchive_CopyManifestToBuildDirectory,
136+
connectorArchive_CopyDependenciesToBuildDirectory,
137+
connectorArchive_CopyDocumentationToBuildDirectory,
138+
connectorArchive_CopyEtcToBuildDirectory,
139+
connectorArchive_CopyAssetsToBuildDirectory]
140+
group = 'connector archive'
141+
}
142+
143+
task connectorArchive(type: Zip, dependsOn: connectorArchive_BuildDirectory) {
144+
description = 'Build a Connector Hub for the Confluent Connector Hub'
145+
group = 'connector archive'
146+
147+
from "${baseArchiveBuildDir}"
148+
include '**/*'
149+
archiveName "${baseArchiveName}.zip"
150+
destinationDir(file('build/distro'))
151+
}

‎gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
group=com.marklogic
2-
version=1.2.0
2+
version=1.3.0
33

44
# For the Confluent Connector Archive
55
componentOwner=marklogic

‎src/main/java/com/marklogic/client/ext/document/ContentIdExtractor.java‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎src/main/java/com/marklogic/client/ext/document/DefaultContentIdExtractor.java‎

Lines changed: 0 additions & 13 deletions
This file was deleted.

0 commit comments

Comments
 (0)