Skip to content

Commit 16ace44

Browse files
rjrudinrjrudin
committed
MLE-24826 Bumping Spring, Spark, and ml-gradle
And also trying out mlWaitTillReady, plus some general Jenkins cleanup.
1 parent d213dee commit 16ace44

File tree

8 files changed

+98
-60
lines changed

8 files changed

+98
-60
lines changed

‎Jenkinsfile‎

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,74 @@
11
@Library('shared-libraries') _
22

3-
def runtests(String javaVersion){
3+
def runtests(){
44
// 'set -e' causes the script to fail if any command fails.
5-
sh label:'test', script: '''#!/bin/bash
5+
sh label:'deploy-test-app', script: '''#!/bin/bash
66
set -e
7-
export JAVA_HOME=$'''+javaVersion+'''
7+
export JAVA_HOME=$JAVA17_HOME_DIR
88
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
9-
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
9+
export PATH=$JAVA_HOME/bin:$PATH
1010
cd marklogic-spark-connector
11-
echo "Waiting for MarkLogic server to initialize."
12-
sleep 60s
13-
./gradlew clean
11+
./gradlew -i mlWaitTillReady
1412
./gradlew mlTestConnections
1513
./gradlew -i mlDeploy
1614
echo "Loading data a second time to try to avoid Optic bug with duplicate rows being returned."
1715
./gradlew -i mlLoadData
16+
'''
17+
18+
sh label:'test', script: '''#!/bin/bash
19+
set -e
20+
export JAVA_HOME=$JAVA17_HOME_DIR
21+
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
22+
export PATH=$JAVA_HOME/bin:$PATH
23+
cd marklogic-spark-connector
1824
./gradlew clean test jacocoTestReport || true
1925
'''
2026
junit '**/build/**/*.xml'
2127
}
2228

23-
def runSonarScan(String javaVersion){
24-
sh label:'test', script: '''#!/bin/bash
25-
export JAVA_HOME=$'''+javaVersion+'''
29+
def runSonarScan(){
30+
sh label:'run-sonar', script: '''#!/bin/bash
31+
export JAVA_HOME=$JAVA17_HOME_DIR
2632
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
2733
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
2834
cd marklogic-spark-connector
2935
./gradlew sonar -Dsonar.projectKey='marklogic_marklogic-spark-connector_AY1bXn6J_50_odbCDKMX' -Dsonar.projectName='ML-DevExp-marklogic-spark-connector' || true
3036
'''
3137
}
3238

39+
def tearDownDocker() {
40+
updateWorkspacePermissions()
41+
sh label:'mlcleanup', script: '''#!/bin/bash
42+
cd marklogic-spark-connector
43+
docker-compose down -v || true
44+
'''
45+
cleanupDocker()
46+
}
47+
3348
pipeline{
3449
agent none
50+
3551
triggers{
3652
parameterizedCron(env.BRANCH_NAME == "develop" ? "00 02 * * * % regressions=true" : "")
3753
}
3854
parameters{
3955
booleanParam(name: 'regressions', defaultValue: false, description: 'indicator if build is for regressions')
4056
}
57+
4158
options {
4259
checkoutToSubdirectory 'marklogic-spark-connector'
4360
buildDiscarder logRotator(artifactDaysToKeepStr: '7', artifactNumToKeepStr: '', daysToKeepStr: '30', numToKeepStr: '')
4461
}
62+
4563
environment{
4664
JAVA17_HOME_DIR="/home/builder/java/jdk-17.0.2"
4765
GRADLE_DIR =".gradle"
4866
DMC_USER = credentials('MLBUILD_USER')
4967
DMC_PASSWORD = credentials('MLBUILD_PASSWORD')
5068
}
69+
5170
stages{
71+
5272
stage('tests'){
5373
environment{
5474
scannerHome = tool 'SONAR_Progress'
@@ -64,22 +84,18 @@ pipeline{
6484
cd marklogic-spark-connector
6585
MARKLOGIC_LOGS_VOLUME=/tmp docker-compose up -d --build
6686
'''
67-
runtests('JAVA17_HOME_DIR')
87+
runtests()
6888
withSonarQubeEnv('SONAR_Progress') {
69-
runSonarScan('JAVA17_HOME_DIR')
89+
runSonarScan()
7090
}
7191
}
7292
post{
7393
always{
74-
updateWorkspacePermissions()
75-
sh label:'mlcleanup', script: '''#!/bin/bash
76-
cd marklogic-spark-connector
77-
docker-compose down -v || true
78-
'''
79-
cleanupDocker()
94+
tearDownDocker()
8095
}
8196
}
8297
}
98+
8399
stage('publish'){
84100
agent {label 'devExpLinuxPool'}
85101
when {
@@ -89,14 +105,15 @@ pipeline{
89105
sh label:'publish', script: '''#!/bin/bash
90106
export JAVA_HOME=$JAVA17_HOME_DIR
91107
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
92-
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
108+
export PATH=$JAVA_HOME/bin:$PATH
93109
cd marklogic-spark-connector
94110
./gradlew clean
95111
cp ~/.gradle/gradle.properties $GRADLE_USER_HOME;
96112
./gradlew publish
97113
'''
98114
}
99115
}
116+
100117
stage('regressions'){
101118
agent {label 'devExpLinuxPool'}
102119
when{
@@ -116,19 +133,13 @@ pipeline{
116133
docker-compose down -v || true
117134
MARKLOGIC_LOGS_VOLUME=/tmp docker-compose up -d --build
118135
'''
119-
runtests('JAVA17_HOME_DIR')
136+
runtests()
120137
}
121138
post{
122139
always{
123-
updateWorkspacePermissions()
124-
sh label:'mlcleanup', script: '''#!/bin/bash
125-
cd marklogic-spark-connector
126-
docker-compose down -v || true
127-
'''
128-
cleanupDocker()
140+
tearDownDocker()
129141
}
130142
}
131-
132143
}
133144
}
134145
}

‎build.gradle‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ subprojects {
3535

3636
repositories {
3737
mavenCentral()
38+
39+
maven {
40+
url = "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
41+
}
3842
}
3943

4044
test {

‎gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
version=3.0-SNAPSHOT
2-
sparkVersion=4.1.0-preview1
2+
sparkVersion=4.1.0-preview2
33
tikaVersion=3.2.3
44
semaphoreVersion=5.10.0
55
langchain4jVersion=1.5.0

‎marklogic-spark-connector/build.gradle‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,13 @@ configurations {
88
resolutionStrategy.eachDependency { DependencyResolveDetails details ->
99
// These all impact Spark and its dependencies, but not the published connector as the connector does not contain
1010
// any Spark libraries.
11-
if (details.requested.group.equals("org.apache.hadoop") and details.requested.version.equals("3.4.1")) {
12-
details.useVersion "3.4.2"
13-
details.because "Using 3.4.2 to minimize CVEs and because Flux is doing the same thing."
14-
}
1511
if (details.requested.group.equals("org.codehaus.janino")) {
1612
details.useVersion "3.1.12"
1713
details.because "Bumping from 3.1.9 (what Spark SQL 4.0.1 depends on) to 3.1.12 to minimize CVEs."
1814
}
1915
if (details.requested.group.equals("io.netty") and details.requested.version.startsWith("4.1.1")) {
20-
details.useVersion "4.1.127.Final"
21-
details.because "Bumping from 4.1.118 (what Spark SQL 4.0.1 depends on) to 4.1.127 to minimize CVEs."
16+
details.useVersion "4.1.128.Final"
17+
details.because "Bumping from 4.1.127 (what Spark SQL 4.1.0-preview2 depends on) to minimize CVEs."
2218
}
2319
}
2420

@@ -92,7 +88,7 @@ dependencies {
9288
// https://docs.gradle.org/current/userguide/upgrading_version_8.html#test_framework_implementation_dependencies
9389
// Without this, once using JUnit 5.12 or higher, Gradle will not find any tests and report an error of:
9490
// org.junit.platform.commons.JUnitException: TestEngine with ID 'junit-jupiter' failed to discover tests
95-
testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.13.4"
91+
testRuntimeOnly "org.junit.platform:junit-platform-launcher:1.14.0"
9692

9793
testImplementation("org.apache.spark:spark-sql_2.13:${sparkVersion}") {
9894
exclude module: "rocksdbjni"
@@ -103,28 +99,16 @@ dependencies {
10399
exclude group: "com.fasterxml.jackson.core"
104100
}
105101

106-
testImplementation('com.marklogic:ml-app-deployer:6.0.1') {
102+
testImplementation('com.marklogic:ml-app-deployer:6.2-SNAPSHOT') {
107103
exclude group: "com.fasterxml.jackson.core"
108104
exclude group: "com.fasterxml.jackson.dataformat"
109-
110-
// Use the Java Client declared above.
111-
exclude module: "marklogic-client-api"
112105
}
113106

114-
testImplementation('com.marklogic:marklogic-junit5:1.5.0') {
107+
testImplementation('com.marklogic:marklogic-junit5:2.0-SNAPSHOT') {
115108
exclude group: "com.fasterxml.jackson.core"
116109
exclude group: "com.fasterxml.jackson.dataformat"
117-
118-
// Use the Java Client declared above.
119-
exclude module: "marklogic-client-api"
120-
121-
// Use the Spring dependencies from ml-app-deployer 6 to avoid vulnerabilities in Spring 5.
122-
exclude group: "org.springframework"
123110
}
124111

125-
// marklogic-junit5 still needs spring-test, but we want the 6 version to minimize vulnerabilities.
126-
testImplementation "org.springframework:spring-test:6.2.11"
127-
128112
testImplementation "ch.qos.logback:logback-classic:1.5.19"
129113
testImplementation "org.skyscreamer:jsonassert:1.5.3"
130114

‎marklogic-spark-connector/src/main/java/com/marklogic/spark/reader/CustomLongOffset.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright (c) 2023-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
3+
*/
4+
package com.marklogic.spark.reader;
5+
6+
import org.apache.spark.sql.connector.read.streaming.Offset;
7+
8+
// Added to avoid dependency on Spark's LongOffset class, which was removed in 4.1.0-preview2.
9+
public class CustomLongOffset extends Offset {
10+
11+
private final long value;
12+
13+
public CustomLongOffset(long value) {
14+
this.value = value;
15+
}
16+
17+
@Override
18+
public String json() {
19+
return String.valueOf(value);
20+
}
21+
22+
public long getValue() {
23+
return value;
24+
}
25+
}

‎marklogic-spark-connector/src/main/java/com/marklogic/spark/reader/customcode/CustomCodeMicroBatchStream.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@
44
package com.marklogic.spark.reader.customcode;
55

66
import com.marklogic.spark.Util;
7+
import com.marklogic.spark.reader.CustomLongOffset;
78
import org.apache.spark.sql.connector.read.InputPartition;
89
import org.apache.spark.sql.connector.read.PartitionReaderFactory;
910
import org.apache.spark.sql.connector.read.streaming.MicroBatchStream;
1011
import org.apache.spark.sql.connector.read.streaming.Offset;
11-
import org.apache.spark.sql.execution.streaming.LongOffset;
1212
import org.slf4j.Logger;
1313
import org.slf4j.LoggerFactory;
1414

@@ -35,7 +35,7 @@ class CustomCodeMicroBatchStream implements MicroBatchStream {
3535
*/
3636
@Override
3737
public Offset latestOffset() {
38-
Offset result = partitionIndex >= partitions.size() ? null : new LongOffset(partitionIndex);
38+
Offset result = partitionIndex >= partitions.size() ? null : new CustomLongOffset(partitionIndex);
3939
if (logger.isTraceEnabled()) {
4040
logger.trace("Returning latest offset: {}", partitionIndex);
4141
}
@@ -50,7 +50,7 @@ public Offset latestOffset() {
5050
*/
5151
@Override
5252
public InputPartition[] planInputPartitions(Offset start, Offset end) {
53-
long index = ((LongOffset) end).offset();
53+
long index = ((CustomLongOffset) end).getValue();
5454
return new InputPartition[]{new CustomCodePartition(partitions.get((int) index))};
5555
}
5656

@@ -61,12 +61,12 @@ public PartitionReaderFactory createReaderFactory() {
6161

6262
@Override
6363
public Offset initialOffset() {
64-
return new LongOffset(0);
64+
return new CustomLongOffset(0);
6565
}
6666

6767
@Override
6868
public Offset deserializeOffset(String json) {
69-
return new LongOffset(Long.parseLong(json));
69+
return new CustomLongOffset(Long.parseLong(json));
7070
}
7171

7272
@Override

‎marklogic-spark-connector/src/main/java/com/marklogic/spark/reader/optic/OpticMicroBatchStream.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@
44
package com.marklogic.spark.reader.optic;
55

66
import com.marklogic.spark.Util;
7+
import com.marklogic.spark.reader.CustomLongOffset;
78
import org.apache.spark.sql.connector.read.InputPartition;
89
import org.apache.spark.sql.connector.read.PartitionReaderFactory;
910
import org.apache.spark.sql.connector.read.streaming.MicroBatchStream;
1011
import org.apache.spark.sql.connector.read.streaming.Offset;
11-
import org.apache.spark.sql.execution.streaming.LongOffset;
1212
import org.slf4j.Logger;
1313
import org.slf4j.LoggerFactory;
1414

@@ -44,7 +44,7 @@ public Offset latestOffset() {
4444
if (logger.isTraceEnabled()) {
4545
logger.trace("Returning latest offset: {}", bucketIndex);
4646
}
47-
return new LongOffset(bucketIndex++);
47+
return new CustomLongOffset(bucketIndex++);
4848
}
4949

5050
/**
@@ -57,7 +57,7 @@ public Offset latestOffset() {
5757
*/
5858
@Override
5959
public InputPartition[] planInputPartitions(Offset start, Offset end) {
60-
int index = (int) ((LongOffset) end).offset();
60+
int index = (int) ((CustomLongOffset) end).getValue();
6161
return index >= allBuckets.size() ?
6262
null :
6363
new InputPartition[]{new PlanAnalysis.Partition(index + "", allBuckets.get(index))};
@@ -70,12 +70,12 @@ public PartitionReaderFactory createReaderFactory() {
7070

7171
@Override
7272
public Offset initialOffset() {
73-
return new LongOffset(0);
73+
return new CustomLongOffset(0);
7474
}
7575

7676
@Override
7777
public Offset deserializeOffset(String json) {
78-
return new LongOffset(Long.parseLong(json));
78+
return new CustomLongOffset(Long.parseLong(json));
7979
}
8080

8181
@Override

‎test-app/build.gradle‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,22 @@
1+
buildscript {
2+
repositories {
3+
mavenCentral()
4+
// Needed for ml-gradle 6.2-SNAPSHOT
5+
maven {
6+
url = "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
7+
}
8+
}
9+
dependencies {
10+
classpath "com.marklogic:ml-gradle:6.2-SNAPSHOT"
11+
}
12+
}
13+
114
plugins {
215
id "net.saliman.properties" version "1.5.2"
3-
id "com.marklogic.ml-gradle" version "6.0.1"
416
}
517

18+
apply plugin: "com.marklogic.ml-gradle"
19+
620
tasks.register("reloadTestData", com.marklogic.gradle.task.MarkLogicTask) {
721
description = "Convenience task for clearing the test database and reloading the test data; only intended for a connector developer to use."
822
doLast {

0 commit comments

Comments
 (0)