Merge pull request #176 from BillFarber/task/mergeTestExamples

Merging into a single test-app

Author: BillFarber

CONTRIBUTING.md

@@ -50,7 +50,7 @@ dependencies {
 
   // Note that in general, the version of the DHF jar must match that of the deployed DHF instance. Different versions
   // may work together, but that behavior is not guaranteed.
-  implementation("com.marklogic:marklogic-data-hub:5.8.0") {
+  implementation("com.marklogic:marklogic-data-hub:6.0.0") {
     exclude module: "marklogic-client-api"
     exclude module: "ml-javaclient-util"
     exclude module: "ml-app-deployer"
@@ -117,29 +117,6 @@ shadowJar {
   exclude "scaffolding/**"
 }
 
-task copyJarToKafka(type: Copy, dependsOn: shadowJar) {
-  description = "Used for local development and testing; copies the jar to your local Kafka install"
-  from "build/libs"
-  into "${kafkaHome}/libs"
-}
-
-task copyPropertyFilesToKafka(type: Copy) {
-  description = "Used for local development and testing; copies the properties files to your local Kafka install"
-  from "config"
-  into "${kafkaHome}/config"
-  filter { String line ->
-    line.startsWith('ml.connection.username=') ? 'ml.connection.username=' + kafkaMlUsername : line
-  }
-  filter { String line ->
-    line.startsWith('ml.connection.password=') ? 'ml.connection.password=' + kafkaMlPassword : line
-  }
-}
-
-task deploy {
-  description = "Used for local development and testing; builds the jar and copies it and the properties files to your local Kafka install"
-  dependsOn = ["copyJarToKafka", "copyPropertyFilesToKafka"]
-}
-
 ext {
   confluentArchiveGroup = "Confluent Connector Archive"
   confluentTestingGroup = "Confluent Platform Local Testing"
@@ -214,52 +191,10 @@ task connectorArchive(type: Zip, dependsOn: connectorArchive_BuildDirectory, gro
   destinationDirectory = file('build/distro')
 }
 
-// Tasks for working with Confluent Platform running locally.
-// See "Testing with Confluent Platform" in CONTRIBUTING.md
+// Tasks for using the connector with Confluent Platform on Docker
 
-task installConnectorInConfluent(type: Copy, dependsOn: connectorArchive, group: confluentTestingGroup) {
+task copyConnectorToDockerVolume(type: Copy, dependsOn: connectorArchive, group: confluentTestingGroup) {
   description = "Copies the connector's archive directory to the Docker volume shared with the Connect server"
   from "build/connectorArchive"
-  into "src/test/confluent-platform-example/docker/confluent-marklogic-components"
-}
-
-task loadDatagenPurchasesConnector(type: Exec, group: confluentTestingGroup) {
-  description = "Load an instance of the Datagen connector into Confluent Platform for sending JSON documents to " +
-    "the 'purchases' topic"
-  commandLine "curl", "-s", "-X", "POST", "-H", "Content-Type: application/json",
-    "--data", "@src/test/resources/confluent/datagen-purchases-source.json", "http://localhost:8083/connectors"
-}
-
-task loadMarkLogicPurchasesSinkConnector(type: Exec, group: confluentTestingGroup) {
-  description = "Load an instance of the MarkLogic Kafka connector into Confluent Platform for writing data to " +
-    "MarkLogic from the 'purchases' topic"
-  commandLine "curl", "-s", "-X", "POST", "-H", "Content-Type: application/json",
-    "--data", "@src/test/resources/confluent/marklogic-purchases-sink.json", "http://localhost:8083/connectors"
-}
-
-task loadMarkLogicPurchasesSourceConnector(type: Exec, group: confluentTestingGroup) {
-  description = "Load an instance of the MarkLogic Kafka connector into Confluent Platform for reading rows from " +
-    "the demo/purchases view"
-  commandLine "curl", "-s", "-X", "POST", "-H", "Content-Type: application/json",
-    "--data", "@src/test/resources/confluent/marklogic-purchases-source.json", "http://localhost:8083/connectors"
-}
-
-task loadMarkLogicAuthorsSourceConnector(type: Exec, group: confluentTestingGroup) {
-  description = "Loads a source connector that retrieves authors from the citations.xml file, which is also used for " +
-    "all the automated tests"
-  commandLine "curl", "-s", "-X", "POST", "-H", "Content-Type: application/json",
-    "--data", "@src/test/resources/confluent/marklogic-authors-source.json", "http://localhost:8083/connectors"
-}
-
-task loadMarkLogicEmployeesSourceConnector(type: Exec, group: confluentTestingGroup) {
-  commandLine "curl", "-s", "-X", "POST", "-H", "Content-Type: application/json",
-    "--data", "@src/test/resources/confluent/marklogic-employees-source.json", "http://localhost:8083/connectors"
-}
-
-task insertAuthors(type: Test) {
-  useJUnitPlatform()
-  systemProperty "AUTHOR_IDS", authorIds
-  description = "Insert a new author into the kafka-test-content database via a new citations XML document; " +
-    "use e.g. -PauthorIds=7,8,9 to insert 3 new authors with IDs of 7, 8, and 9"
-  include "com/marklogic/kafka/connect/source/debug/InsertAuthorsTest.class"
+  into "test-app/docker/confluent-marklogic-components"
 }

build.gradle

@@ -5,28 +5,12 @@ version=1.9-SNAPSHOT
 componentOwner=marklogic
 componentName=kafka-marklogic-connector
 
-# Override in gradle-local.properties if testing with regular Apache Kafka
-kafkaHome=
-kafkaMlUsername=
-kafkaMlPassword=
+systemProp.sonar.host.url=http://localhost:9000
+systemProp.sonar.token=<Define this property in gradle-local.properties>
+systemProp.sonar.projectKey=marklogic-kafka-connector
 
 # Only used for testing
-mlConfigPaths=src/test/ml-config
-mlModulePaths=src/test/ml-modules
-mlSchemaPaths=src/test/ml-schemas
-mlDataPaths=src/test/ml-data
-mlRestPort=8018
-mlTestRestPort=8019
-mlAppName=kafka-test
 mlUsername=admin
 mlPassword=admin
-mlContentForestsPerHost=2
-
-# For inserting new authors during manual testing; set this to a comma-delimited string of numbers
-# e.g. -PauthorIds=7,8,9
-authorIds=
-
-# For Sonar analysis; set systemProp.sonar.login in gradle-local.properties or via -D when running gradlew
-systemProp.sonar.host.url=http://localhost:9000
-systemProp.sonar.token=<Replace With Your Sonar Token>
-systemProp.sonar.projectKey=marklogic-kafka-connector
+mlRestPort=8011
+mlTestRestPort=8011

docker-compose.yml

@@ -84,7 +84,7 @@ void insertAuthors() {
         }
 
         DatabaseClientConfig config = testConfig.databaseClientConfig();
-        config.setPort(8018);
+        config.setPort(8010);
         DatabaseClient client = new DefaultConfiguredDatabaseClientFactory().newDatabaseClient(config);
 
         XMLDocumentManager mgr = client.newXMLDocumentManager();

gradle.properties

@@ -1,3 +1,2 @@
 src/main/hub-internal-config
-src/main/ml-config
 src/main/entity-config