Merge branch 'develop' into feature/temp-master

Author: rjrudin

.copyrightconfig

@@ -0,0 +1,14 @@
+# COPYRIGHT VALIDATION CONFIG
+# ---------------------------------
+# Required start year (keep fixed; end year auto-updates in check output)
+startyear: 2023
+
+# Optional exclusions list (comma-separated). Leave commented if none.
+# Rules:
+#   - Relative paths (no leading ./)
+#   - Simple * wildcard only (no recursive **)
+#   - Use sparingly (third_party, generated, binary assets)
+#   - Dotfiles already skipped automatically
+# Enable by removing the leading '# ' from the next line and editing values.
+# filesexcluded: third_party/*, docs/generated/*.md, assets/*.png, scripts/temp_*.py, vendor/lib.js
+filesexcluded: .github/*, README.md, CONTRIBUTING.md, Jenkinsfile, gradle/*, docker-compose.yml, *.gradle, gradle.properties, gradlew, gradlew.bat, **/test/resources/**, docs/**, *.json, *.txt, CODEOWNERS

.github/workflows/pr-workflow.yaml

@@ -1,4 +1,4 @@
-name: 🏷️ JIRA ID Validator
+name: PR Workflow
 
 on:
   # Using pull_request_target instead of pull_request to handle PRs from forks
@@ -14,3 +14,10 @@ jobs:
     with:
       # Pass the PR title from the event context
       pr-title: ${{ github.event.pull_request.title }}
+  copyright-validation:
+    name: © Validate Copyright Headers
+    uses: marklogic/pr-workflows/.github/workflows/copyright-check.yml@main
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write

CODEOWNERS

@@ -2,4 +2,4 @@
 # Each line is a file pattern followed by one or more owners.
 
 # These owners will be the default owners for everything in the repo.
-* @anu3990 @billfarber @rjrudin @stevebio
+* @billfarber @rjrudin @stevebio

CONTRIBUTING.md

@@ -1,12 +1,7 @@
 This guide covers how to develop and test this project. It assumes that you have cloned this repository to your local
 workstation.
 
-**You must use Java 17 for developing, testing, and building this project**, even though the connector supports
-running on Java 11. For users, Java 17 is only required if using the splitting and embedding features, as those
-depend on a third party module that requires Java 17.
-
-**You also need Java 11 installed** so that the subprojects in this repository that require Java 11 have access to a 
-Java 11 SDK. [sdkman](https://sdkman.io/) is highly recommend for installing multiple JDKs.
+**You must use Java 17 for developing, testing, and building this project**.
 
 # Setup
 
@@ -44,6 +39,12 @@ To run the tests against the test application, run the following Gradle task:
 
     ./gradlew test
 
+To also get code coverage, run:
+
+    ./gradlew test jacocoTestReport
+
+The code coverage report will be in `marklogic-spark-connector/build/reports/jacoco/test/html/index.html`.
+
 **To run the tests in Intellij**, you must configure your JUnit template to include a few JVM args:
 
 1. Go to Run -> Edit Configurations.

Jenkinsfile

@@ -1,54 +1,74 @@
 @Library('shared-libraries') _
 
-def runtests(String javaVersion){
+def runtests(){
   // 'set -e' causes the script to fail if any command fails.
-  sh label:'test', script: '''#!/bin/bash
+  sh label:'deploy-test-app', script: '''#!/bin/bash
     set -e
-    export JAVA_HOME=$'''+javaVersion+'''
+    export JAVA_HOME=$JAVA17_HOME_DIR
     export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
-    export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
+    export PATH=$JAVA_HOME/bin:$PATH
     cd marklogic-spark-connector
-    echo "Waiting for MarkLogic server to initialize."
-    sleep 60s
-    ./gradlew clean
+    ./gradlew -i mlWaitTillReady
     ./gradlew mlTestConnections
     ./gradlew -i mlDeploy
     echo "Loading data a second time to try to avoid Optic bug with duplicate rows being returned."
     ./gradlew -i mlLoadData
-    ./gradlew clean testCodeCoverageReport || true
+  '''
+
+  sh label:'test', script: '''#!/bin/bash
+    set -e
+    export JAVA_HOME=$JAVA17_HOME_DIR
+    export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
+    export PATH=$JAVA_HOME/bin:$PATH
+    cd marklogic-spark-connector
+    ./gradlew clean test jacocoTestReport || true
   '''
   junit '**/build/**/*.xml'
 }
 
-def runSonarScan(String javaVersion){
-    sh label:'test', script: '''#!/bin/bash
-      export JAVA_HOME=$'''+javaVersion+'''
+def runSonarScan(){
+    sh label:'run-sonar', script: '''#!/bin/bash
+      export JAVA_HOME=$JAVA17_HOME_DIR
       export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
       export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
       cd marklogic-spark-connector
      ./gradlew sonar -Dsonar.projectKey='marklogic_marklogic-spark-connector_AY1bXn6J_50_odbCDKMX' -Dsonar.projectName='ML-DevExp-marklogic-spark-connector' || true
     '''
 }
 
+def tearDownDocker() {
+  updateWorkspacePermissions()
+  sh label:'mlcleanup', script: '''#!/bin/bash
+    cd marklogic-spark-connector
+    docker-compose down -v || true
+  '''
+  cleanupDocker()
+}
+
 pipeline{
   agent none
+  
   triggers{
     parameterizedCron(env.BRANCH_NAME == "develop" ? "00 02 * * * % regressions=true" : "")
   }
   parameters{
     booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions')
   }
+  
   options {
     checkoutToSubdirectory 'marklogic-spark-connector'
     buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '30', numToKeepStr: '')
   }
+  
   environment{
     JAVA17_HOME_DIR="/home/builder/java/jdk-17.0.2"
     GRADLE_DIR   =".gradle"
     DMC_USER     = credentials('MLBUILD_USER')
     DMC_PASSWORD = credentials('MLBUILD_PASSWORD')
   }
+
   stages{
+
     stage('tests'){
       environment{
         scannerHome = tool 'SONAR_Progress'
@@ -64,22 +84,18 @@ pipeline{
             cd marklogic-spark-connector
             MARKLOGIC_LOGS_VOLUME=/tmp docker-compose up -d --build
           '''
-        runtests('JAVA17_HOME_DIR')
+        runtests()
         withSonarQubeEnv('SONAR_Progress') {
-          runSonarScan('JAVA17_HOME_DIR')
+          runSonarScan()
         }
       }
       post{
         always{
-          updateWorkspacePermissions()
-          sh label:'mlcleanup', script: '''#!/bin/bash
-            cd marklogic-spark-connector
-            docker-compose down -v || true
-          '''
-          cleanupDocker()
+          tearDownDocker()
         }
       }
     }
+
     stage('publish'){
       agent {label 'devExpLinuxPool'}
       when {
@@ -89,14 +105,15 @@ pipeline{
       	sh label:'publish', script: '''#!/bin/bash
           export JAVA_HOME=$JAVA17_HOME_DIR
           export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
-          export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
+          export PATH=$JAVA_HOME/bin:$PATH
           cd marklogic-spark-connector
           ./gradlew clean
           cp ~/.gradle/gradle.properties $GRADLE_USER_HOME;
            ./gradlew publish
         '''
       }
     }
+
     stage('regressions'){
       agent {label 'devExpLinuxPool'}
       when{
@@ -116,19 +133,13 @@ pipeline{
                 docker-compose down -v || true
                 MARKLOGIC_LOGS_VOLUME=/tmp docker-compose up -d --build
             '''
-            runtests('JAVA17_HOME_DIR')
+            runtests()
       }
       post{
         always{
-          updateWorkspacePermissions()
-          sh label:'mlcleanup', script: '''#!/bin/bash
-            cd marklogic-spark-connector
-            docker-compose down -v || true
-          '''
-          cleanupDocker()
+          tearDownDocker()
         }
       }
-
     }
   }
 }

NOTICE.txt

@@ -10,53 +10,48 @@ product and version for which you are requesting source code.
 
 Third Party Notices
 
-jackson-dataformat-xml 2.15.2 (Apache-2.0)
+jackson-dataformat-xml 2.19.0 (Apache-2.0)
 jdom2 2.0.6.1 (Apache-2.0)
-jena-arq 4.10.0 (Apache-2.0)
-langchain4j 1.2.0 (Apache-2.0)
-marklogic-client-api 7.2.0 (Apache-2.0)
-okhttp 4.12.0 (Apache-2.0)
-Semaphore-CS-Client 5.6.1 (Apache-2.0)
-Semaphore-Cloud-Client 5.6.1 (Apache-2.0)
-tika-core 3.2.1 (Apache-2.0)
+jena-arq 5.5.0 (Apache-2.0)
+langchain4j 1.5.0 (Apache-2.0)
+marklogic-client-api 8.0.0 (Apache-2.0)
+Semaphore-CS-Client 5.10.0 (Apache-2.0)
+Semaphore-Cloud-Client 5.10.0 (Apache-2.0)
+tika-core 3.2.3 (Apache-2.0)
 
 Third-Party Components
 
 The following is a list of the third-party components used by the MarkLogic® Spark connector 2.7.0 (last updated July 31, 2025):
 
-jackson-dataformat-xml 2.15.2 (Apache-2.0)
+jackson-dataformat-xml 2.19.0 (Apache-2.0)
 https://repo1.maven.org/maven2/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
 jdom2 2.0.6.1 (Apache-2.0)
 https://repo1.maven.org/maven2/org/jdom/jdom2/
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
-jena-arq 4.10.0 (Apache-2.0)
+jena-arq 5.5.0 (Apache-2.0)
 https://repo1.maven.org/maven2/org/apache/jena/jena-arq/
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
-langchain4j 1.2.0 (Apache-2.0)
+langchain4j 1.5.0 (Apache-2.0)
 https://repo1.maven.org/maven2/dev/langchain4j/langchain4j/
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
 marklogic-client-api 7.2.0 (Apache-2.0)
 https://repo1.maven.org/maven2/com/marklogic/marklogic-client-api/
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
-okhttp 4.12.0 (Apache-2.0)
-https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/
-For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
-
-Semaphore-CS-Client 5.6.1 (Apache-2.0)
+Semaphore-CS-Client 5.10.0 (Apache-2.0)
 https://repo1.maven.org/maven2/com/smartlogic/csclient/Semaphore-CS-Client/
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
-Semaphore-CS-Client 5.6.1 (Apache-2.0)
+Semaphore-CS-Client 5.10.0 (Apache-2.0)
 https://repo1.maven.org/maven2/com/smartlogic/cloud/Semaphore-Cloud-Client/
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
-tika-core 3.2.1 (Apache-2.0)
+tika-core 3.2.3 (Apache-2.0)
 https://repo1.maven.org/maven2/org/apache/tika/tika-core/
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 

build.gradle

@@ -8,7 +8,7 @@ sonar {
   properties {
     property "sonar.projectKey", "marklogic-spark"
     property "sonar.host.url", "http://localhost:9000"
-    property "sonar.coverage.jacoco.xmlReportPaths", "code-coverage-report/build/reports/jacoco/testCodeCoverageReport/testCodeCoverageReport.xml"
+    property "sonar.coverage.jacoco.xmlReportPaths", "marklogic-spark-connector/build/reports/jacoco/test/jacocoTestReport.xml"
   }
 }
 
@@ -23,7 +23,7 @@ subprojects {
   // See https://docs.gradle.org/current/userguide/toolchains.html .
   java {
     toolchain {
-      languageVersion = JavaLanguageVersion.of(11)
+      languageVersion = JavaLanguageVersion.of(17)
     }
   }
 
@@ -35,53 +35,18 @@ subprojects {
 
   repositories {
     mavenCentral()
-    maven {
-      url "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
-    }
-  }
 
-  configurations.all {
-    resolutionStrategy.eachDependency { DependencyResolveDetails details ->
-      // Added after upgrading langchain4j to 1.0.0-beta2, which brought Jackson 2.18.2 in.
-      if (details.requested.group.startsWith('com.fasterxml.jackson')) {
-        details.useVersion '2.15.2'
-        details.because 'Need to match the version used by Spark.'
-      }
-      if (details.requested.group.equals("org.slf4j")) {
-        details.useVersion "2.0.16"
-        details.because "Ensures that slf4j-api 1.x does not appear on the Flux classpath in particular, which can " +
-          "lead to this issue - https://www.slf4j.org/codes.html#StaticLoggerBinder."
-      }
-      if (details.requested.group.equals("org.apache.logging.log4j")) {
-        details.useVersion "2.24.3"
-        details.because "Need to match the version used by Apache Tika. Spark uses 2.20.0 but automated tests confirm " +
-          "that Spark seems fine with 2.24.3."
-      }
-    }
-
-    resolutionStrategy {
-      // Avoids a classpath conflict between Spark and the tika-parser-microsoft-module. Tika needs a
-      // more recent version and Spark (and Jena as well) both seems fine with this (as they should be per semver).
-      force "org.apache.commons:commons-compress:1.27.1"
-
-      // Avoids CVEs in earlier minor versions.
-      force "org.apache.commons:commons-lang3:3.18.0"
+    maven {
+      url = "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
     }
-
-    // Excluded from Flux for size reasons, so excluded here as well to ensure we don't need it when running tests.
-    exclude module: "rocksdbjni"
-  }
-
-  task allDeps(type: DependencyReportTask) {
-    description = "Allows for generating dependency reports for every subproject in a single task."
   }
 
   test {
     useJUnitPlatform()
     finalizedBy jacocoTestReport
     testLogging {
-      events 'started', 'passed', 'skipped', 'failed'
-      exceptionFormat 'full'
+      events = ['started', 'passed', 'skipped', 'failed']
+      exceptionFormat = 'full'
     }
     environment "SEMAPHORE_API_KEY", semaphoreApiKey
     environment "SEMAPHORE_HOST", semaphoreHost