Upgrade to Kafka 4.1.0 and Confluent Platform 8.0

Force upgrade to Spring 6
New files for the new CP stack and updated .gitignore for those files.

Author: BillFarber

.copyrightconfig

@@ -11,4 +11,4 @@ startyear: 2023
 #   - Dotfiles already skipped automatically
 # Enable by removing the leading '# ' from the next line and editing values.
 # filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js
-filesexcluded: .github/*, README.md, CONTRIBUTING.md, Jenkinsfile, gradle/*, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, docs/**, test-app/docker-compose.yml
+filesexcluded: .github/*, README.md, CONTRIBUTING.md, Jenkinsfile, gradle/*, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, docs/**, test-app/docker-compose.yml, docker/prometheus/*.yml

.gitignore

@@ -4,7 +4,8 @@
 build
 out
 gradle-local.properties
-docker
+docker/confluent-marklogic-components/marklogic-kafka-marklogic-connector*
+docker/marklogic
 
 bin
 .vscode

CONTRIBUTING.md

@@ -101,27 +101,30 @@ application. The instructions below describe how to get started.
 ## Docker Cluster Preparation for Manual Testing
 The docker-compose file in the test-app directory includes these services along with a MarkLogic server.
 ```
-docker-compose --env-file ./.env -f test-app/docker-compose.yml up -d --build
+docker-compose --env-file test-app/.env -f test-app/docker-compose.yml up -d --build
 ```
 
 When the setup is complete, you should be able to run
 ```
-docker-compose --env-file ./.env -f test-app/docker-compose.yml ps
+docker-compose --env-file test-app/.env -f test-app/docker-compose.yml ps
 ```
 and see results similar to the following.
 ```
-NAME                                    IMAGE                                             COMMAND                  SERVICE           CREATED          STATUS          PORTS
-broker                                  confluentinc/cp-kafka:7.6.1                       "/etc/confluent/dock…"   broker            14 minutes ago   Up 14 minutes   0.0.0.0:9092->9092/tcp, 0.0.0.0:9101->9101/tcp
-connect                                 cnfldemos/cp-server-connect-datagen:0.6.4-7.6.0   "/etc/confluent/dock…"   connect           14 minutes ago   Up 14 minutes   0.0.0.0:8083->8083/tcp, 9092/tcp
-control-center                          confluentinc/cp-enterprise-control-center:7.6.1   "/etc/confluent/dock…"   control-center    14 minutes ago   Up 14 minutes   0.0.0.0:9021->9021/tcp
-ksql-datagen                            confluentinc/ksqldb-examples:7.6.1                "bash -c 'echo Waiti…"   ksql-datagen      14 minutes ago   Up 14 minutes   
-ksqldb-cli                              confluentinc/cp-ksqldb-cli:7.6.1                  "/bin/sh"                ksqldb-cli        14 minutes ago   Up 14 minutes   
-ksqldb-server                           confluentinc/cp-ksqldb-server:7.6.1               "/etc/confluent/dock…"   ksqldb-server     14 minutes ago   Up 14 minutes   0.0.0.0:8088->8088/tcp
-marklogic                               marklogicdb/marklogic-db:11.2.0-centos-1.1.2      "/tini -- /usr/local…"   marklogic         14 minutes ago   Up 14 minutes   25/tcp, 7997-7999/tcp, 0.0.0.0:8000-8002->8000-8002/tcp, 0.0.0.0:8010-8013->8010-8013/tcp, 8003-8009/tcp, 0.0.0.0:8018-8019->8018-8019/tcp
-marklogic-kafka-confluent-postgres-1    postgres:15-alpine                                "docker-entrypoint.s…"   postgres          14 minutes ago   Up 14 minutes   5432/tcp
-marklogic-kafka-confluent-sonarqube-1   sonarqube:10.3.0-community                        "/opt/sonarqube/dock…"   sonarqube         14 minutes ago   Up 14 minutes   0.0.0.0:9000->9000/tcp
-rest-proxy                              confluentinc/cp-kafka-rest:7.6.1                  "/etc/confluent/dock…"   rest-proxy        14 minutes ago   Up 14 minutes   0.0.0.0:8082->8082/tcp
-schema-registry                         confluentinc/cp-schema-registry:7.6.1             "/etc/confluent/dock…"   schema-registry   14 minutes ago   Up 14 minutes   0.0.0.0:8081->8081/tcp
+NAME                                                 IMAGE                                                                                                      COMMAND                  SERVICE             CREATED          STATUS          PORTS
+alertmanager                                         confluentinc/cp-enterprise-alertmanager:2.2.0                                                              "alertmanager-start"     alertmanager        51 seconds ago   Up 50 seconds   0.0.0.0:9093->9093/tcp, [::]:9093->9093/tcp
+broker                                               confluentinc/cp-server:8.0.0                                                                               "/etc/confluent/dock…"   broker              51 seconds ago   Up 50 seconds   0.0.0.0:9092->9092/tcp, [::]:9092->9092/tcp, 0.0.0.0:9101->9101/tcp, [::]:9101->9101/tcp
+connect                                              cnfldemos/cp-server-connect-datagen:0.6.7-8.0.0                                                            "/etc/confluent/dock…"   connect             51 seconds ago   Up 49 seconds   0.0.0.0:8083->8083/tcp, [::]:8083->8083/tcp
+control-center                                       confluentinc/cp-enterprise-control-center-next-gen:2.2.0                                                   "/etc/confluent/dock…"   control-center      51 seconds ago   Up 49 seconds   0.0.0.0:9021->9021/tcp, [::]:9021->9021/tcp
+flink-jobmanager                                     cnfldemos/flink-kafka:1.19.1-scala_2.12-java17                                                             "/docker-entrypoint.…"   flink-jobmanager    51 seconds ago   Up 50 seconds   0.0.0.0:9081->9081/tcp, [::]:9081->9081/tcp
+flink-sql-client                                     cnfldemos/flink-sql-client-kafka:1.19.1-scala_2.12-java17                                                  "/docker-entrypoint.…"   flink-sql-client    51 seconds ago   Up 50 seconds   6123/tcp, 8081/tcp
+flink-taskmanager                                    cnfldemos/flink-kafka:1.19.1-scala_2.12-java17                                                             "/docker-entrypoint.…"   flink-taskmanager   51 seconds ago   Up 50 seconds   6123/tcp, 8081/tcp
+ksql-datagen                                         confluentinc/ksqldb-examples:8.0.0                                                                         "bash -c 'echo Waiti…"   ksql-datagen        51 seconds ago   Up 49 seconds   
+ksqldb-cli                                           confluentinc/cp-ksqldb-cli:8.0.0                                                                           "/bin/sh"                ksqldb-cli          51 seconds ago   Up 49 seconds   
+ksqldb-server                                        confluentinc/cp-ksqldb-server:8.0.0                                                                        "/etc/confluent/dock…"   ksqldb-server       51 seconds ago   Up 49 seconds   0.0.0.0:8088->8088/tcp, [::]:8088->8088/tcp
+manual-tests-marklogic-kafka-confluent-marklogic-1   ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12   "/tini -- /usr/local…"   marklogic           51 seconds ago   Up 50 seconds   0.0.0.0:8000-8002->8000-8002/tcp, [::]:8000-8002->8000-8002/tcp, 0.0.0.0:8010-8013->8010-8013/tcp, [::]:8010-8013->8010-8013/tcp, 0.0.0.0:8018-8019->8018-8019/tcp, [::]:8018-8019->8018-8019/tcp
+prometheus                                           confluentinc/cp-enterprise-prometheus:2.2.0                                                                "prometheus-start"       prometheus          51 seconds ago   Up 50 seconds   0.0.0.0:9090->9090/tcp, [::]:9090->9090/tcp
+rest-proxy                                           confluentinc/cp-kafka-rest:8.0.0                                                                           "/etc/confluent/dock…"   rest-proxy          51 seconds ago   Up 49 seconds   0.0.0.0:8082->8082/tcp, [::]:8082->8082/tcp
+schema-registry                                      confluentinc/cp-schema-registry:8.0.0                                                                      "/etc/confluent/dock…"   schema-registry     51 seconds ago   Up 50 seconds   0.0.0.0:8081->8081/tcp, [::]:8081->8081/tcp
 ```
 
 You can now visit several web applications:
@@ -139,6 +142,18 @@ The Confluent Platform servers in this docker-compose file are based on the Conf
 [Install a Confluent Platform cluster in Docker using a Confluent docker-compose file](https://docs.confluent.io/platform/current/platform-quickstart.html).
 
 
+### MarkLogic Preparation
+To prepare the MarkLogic server for automated testing as well as testing with the Confluent Platform, the Data Hub based
+application must be deployed. From the root directory, follow these steps:
+1. Run `./gradlew hubInit`
+2. Edit gradle-local.properties and set `mlUsername` and `mlPassword`
+3. Run `./gradlew -i mlDeploy`
+
+Note: If you change the version of Data Hub Framework used by this project, you should also delete the following directories:
+* 'test-app/src/main/entity-config'
+* 'test-app/src/main/hub-internal-config'
+
+
 ### Building and Sharing the Connector with the Docker Container
 Using gradle in the root directory, build the connector archive and copy it to a directory shared with the Confluent
 Platform Docker cluster built in the that section, using this gradle command in the root directory:
@@ -187,7 +202,7 @@ In the Control Center GUI, you can verify the MarkLogic Kafka connector instance
 3. Click on the "marklogic-purchases-sink" connector
 
 You can then verify that data is being written to MarkLogic by using MarkLogic's qconsole application to inspect the
-contents of the `data-hub-FINAL` database.
+contents of the `data-hub-FINAL` database. There should be documents with URIs that start with `/purchase/*`.
 
 ### Load a MarkLogic Kafka source connector instance
 You can also load an instance of the MarkLogic Kafka source connector that will read rows from the `demo/purchases`

build.gradle

@@ -27,10 +27,15 @@ plugins {
 }
 
 java {
-  sourceCompatibility = 1.8
-  targetCompatibility = 1.8
+  toolchain {
+    languageVersion = JavaLanguageVersion.of(17)
+  }
+  sourceCompatibility = JavaVersion.VERSION_17
+  targetCompatibility = JavaVersion.VERSION_17
 }
 
+
+
 repositories {
   mavenCentral()
 }
@@ -55,7 +60,7 @@ configurations {
 }
 
 ext {
-  kafkaVersion = "3.9.1"
+  kafkaVersion = "4.1.0"
 }
 
 dependencies {
@@ -66,7 +71,7 @@ dependencies {
   // Force DHF to use the latest version of ml-app-deployer, which minimizes security vulnerabilities
   implementation "com.marklogic:ml-app-deployer:5.0.0"
 
-  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.17.2"
+  implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.19.0"
 
   // Note that in general, the version of the DHF jar must match that of the deployed DHF instance. Different versions
   // may work together, but that behavior is not guaranteed.
@@ -81,7 +86,17 @@ dependencies {
     exclude module: "logback-classic"
   }
 
-  testImplementation 'com.marklogic:marklogic-junit5:1.5.0'
+  testImplementation('com.marklogic:marklogic-junit5:1.5.0') {
+    // Use the Java Client declared above.
+    exclude module: "marklogic-client-api"
+
+    // Use the Spring dependencies from ml-app-deployer 6 to avoid vulnerabilities in Spring 5.
+    exclude group: "org.springframework"
+  }
+
+  // Add back all required Spring 6 modules for tests, since junit5 and test code need more than just spring-test
+  testImplementation "org.springframework:spring-test:6.2.11"
+  testImplementation "org.springframework:spring-context:6.2.11"
 
   testImplementation "org.apache.kafka:connect-json:${kafkaVersion}"
   testImplementation kafkaConnectRuntime
@@ -223,5 +238,5 @@ task connectorArchive(type: Zip, dependsOn: connectorArchive_BuildDirectory, gro
 task copyConnectorToDockerVolume(type: Copy, dependsOn: connectorArchive, group: confluentTestingGroup) {
   description = "Copies the connector's archive directory to the Docker volume shared with the Connect server"
   from "build/connectorArchive"
-  into "test-app/docker/confluent-marklogic-components"
+  into "./docker/confluent-marklogic-components"
 }

docker-compose.yml

@@ -17,39 +17,3 @@ services:
       - "8018-8019:8018-8019"
     cap_drop:
       - NET_RAW
-
-  # Copied from https://docs.sonarsource.com/sonarqube/latest/setup-and-upgrade/install-the-server/#example-docker-compose-configuration .
-  sonarqube:
-    image: sonarqube:10.3.0-community
-    depends_on:
-      - postgres
-    environment:
-      SONAR_JDBC_URL: jdbc:postgresql://postgres:5432/sonar
-      SONAR_JDBC_USERNAME: sonar
-      SONAR_JDBC_PASSWORD: sonar
-    volumes:
-      - sonarqube_data:/opt/sonarqube/data
-      - sonarqube_extensions:/opt/sonarqube/extensions
-      - sonarqube_logs:/opt/sonarqube/logs
-    ports:
-      - "9000:9000"
-    cap_drop:
-      - NET_RAW
-
-  postgres:
-    image: postgres:15-alpine
-    environment:
-      POSTGRES_USER: sonar
-      POSTGRES_PASSWORD: sonar
-    volumes:
-      - postgresql:/var/lib/postgresql
-      - postgresql_data:/var/lib/postgresql/data
-    cap_drop:
-      - NET_RAW
-
-volumes:
-  sonarqube_data:
-  sonarqube_extensions:
-  sonarqube_logs:
-  postgresql:
-  postgresql_data:

docker/prometheus/config/alertmanager-generated.yml

@@ -0,0 +1,8 @@
+global:
+  resolve_timeout: 1m
+  smtp_require_tls: false
+receivers:
+- name: default
+route:
+  receiver: default
+  routes: []

docker/prometheus/config/prometheus-generated.yml

@@ -0,0 +1,2 @@
+MARKLOGIC_IMAGE=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12
+MARKLOGIC_LOGS_VOLUME=../docker/marklogic/logs