Merge pull request #267 from marklogic/develop

Merge develop into master

Author: rjrudin

CONTRIBUTING.md

@@ -256,15 +256,15 @@ are all synonyms):
 
     ./gradlew shadowJar
 
-This will produce an assembly jar at `./flux-cli/build/libs/marklogic-flux-1.1.0-all.jar`.
+This will produce an assembly jar at `./flux-cli/build/libs/marklogic-flux-1.1-SNAPSHOT-all.jar`.
 
 You can now run any CLI command via spark-submit. This is an example of previewing an import of files - change the value
 of `--path`, as an absolute path is needed, and of course change the value of `--master` to match that of your Spark
 cluster:
 
 ```
 $SPARK_HOME/bin/spark-submit --class com.marklogic.flux.spark.Submit \
---master spark://NYWHYC3G0W:7077 flux-cli/build/libs/marklogic-flux-1.1.0-all.jar \
+--master spark://NYWHYC3G0W:7077 flux-cli/build/libs/marklogic-flux-1.1-SNAPSHOT-all.jar \
 import-files --path /Users/rudin/workspace/flux/flux-cli/src/test/resources/mixed-files \
 --connection-string "admin:admin@localhost:8000" \
 --preview 5 --preview-drop content
@@ -281,7 +281,7 @@ to something you can access):
 $SPARK_HOME/bin/spark-submit --class com.marklogic.flux.spark.Submit \
 --packages org.apache.hadoop:hadoop-aws:3.3.4,org.apache.hadoop:hadoop-client:3.3.4 \
 --master spark://NYWHYC3G0W:7077 \
-flux-cli/build/libs/marklogic-flux-1.1.0-all.jar \
+flux-cli/build/libs/marklogic-flux-1.1-SNAPSHOT-all.jar \
 import-files --path "s3a://changeme/" \
 --connection-string "admin:admin@localhost:8000" \
 --s3-add-credentials \

Jenkinsfile

@@ -3,6 +3,7 @@ def runtests(){
           cd $WORKSPACE/flux;
           sudo /usr/local/sbin/mladmin stop;
           sudo /usr/local/sbin/mladmin remove;
+          mkdir -p $WORKSPACE/flux/docker/sonarqube;
           docker-compose up -d --build;
           sleep 30s;
         '''
@@ -38,10 +39,20 @@ def postCleanup(){
     cd $WORKSPACE/flux;
     sudo /usr/local/sbin/mladmin delete $WORKSPACE/flux/docker/marklogic/logs/;
     docker exec -i --privileged --user root flux-caddy-load-balancer-1 /bin/sh -c "chmod -R 777 /data" || true;
+    docker exec -i --privileged --user root flux-sonarqube-1 /bin/sh -c "chmod -R 777 /opt/sonarqube" || true;
     docker-compose rm -fsv || true;
     echo "y" | docker volume prune --filter all=1 || true;
   '''
 }
+def runSonarScan(String javaVersion){
+    sh label:'test', script: '''#!/bin/bash
+      export JAVA_HOME=$'''+javaVersion+'''
+      export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
+      export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
+      cd flux
+     ./gradlew sonar -Dsonar.projectKey='ML-DevExp-marklogic-flux' -Dsonar.projectName='ML-DevExp-marklogic-flux' || true
+    '''
+}
 pipeline{
   agent none
   options {
@@ -50,15 +61,22 @@ pipeline{
   }
   environment{
     JAVA_HOME_DIR="/home/builder/java/jdk-11.0.2"
+    JAVA17_HOME_DIR="/home/builder/java/jdk-17.0.2"
     GRADLE_DIR   =".gradle"
     DMC_USER     = credentials('MLBUILD_USER')
     DMC_PASSWORD = credentials('MLBUILD_PASSWORD')
   }
   stages{
     stage('tests'){
+      environment{
+        scannerHome = tool 'SONAR_Progress'
+      }
       agent{ label 'devExpLinuxPool'}
       steps{
         runtests()
+        withSonarQubeEnv('SONAR_Progress') {
+          runSonarScan('JAVA17_HOME_DIR')
+        }
       }
       post{
         always{

build.gradle

@@ -10,10 +10,10 @@ subprojects {
 
   repositories {
     mavenCentral()
-//    mavenLocal()
-//    maven {
-//      url "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
-//    }
+    mavenLocal()
+    maven {
+      url "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/"
+    }
   }
 
   test {

docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - 8007:8007
 
   marklogic:
-    image: "progressofficial/marklogic-db:latest"
+    image: "progressofficial/marklogic-db:11.3.0-ubi"
     platform: linux/amd64
     environment:
       - MARKLOGIC_INIT=true
@@ -63,13 +63,12 @@ services:
     volumes:
       - sonarqube_data:/opt/sonarqube/data
       - sonarqube_extensions:/opt/sonarqube/extensions
-      - sonarqube_logs:/opt/sonarqube/logs
+      - ./docker/sonarqube/logs:/opt/sonarqube/logs
     ports:
       - "9000:9000"
 
 volumes:
   sonarqube_data:
   sonarqube_extensions:
-  sonarqube_logs:
   postgresql:
   postgresql_data:

flux-cli/build.gradle

@@ -17,7 +17,7 @@ dependencies {
     // The rocksdbjni dependency weighs in at 50mb and so far does not appear necessary for our use of Spark.
     exclude module: "rocksdbjni"
   }
-  implementation "com.marklogic:marklogic-spark-connector:2.4.0"
+  implementation "com.marklogic:marklogic-spark-connector:2.4.1"
   implementation "info.picocli:picocli:4.7.6"
 
   // Spark 3.4.3 depends on Hadoop 3.3.4, which depends on AWS SDK 1.12.262. As of August 2024, all public releases of

gradle.properties

@@ -1,4 +1,4 @@
-version=1.1.0
+version=1.1.1
 
 # Define these on the command line to publish to OSSRH
 # See https://central.sonatype.org/publish/publish-gradle/#credentials for more information