Merge branch 'release-5.1.0' into 4.0-master

Author: itsshivaverma

CHANGELOG.md

@@ -1,5 +1,14 @@
 # CHANGELOG
 
+## 5.1.0
+#### New Functionality
+- New package to make it easy to implement connectors for dataflow frameworks. Refer to https://github.com/marklogic/java-client-api/wiki/Bulk-Data-Services page for details.
+- Two new splitters complementing JacksonCSVSplitter. LineSplitter for line-delimited payloads (JSON) and ZipSplitter splitter for entries in a zipfile
+
+#### Improvements and Bug Fixes
+- [#1163](https://github.com/marklogic/java-client-api/issues/1163) - Eliminated potential for deadlocks between batches with URI sort and support for multibyte characters within uris
+- [#1177](https://github.com/marklogic/java-client-api/issues/1177) - Addressed security vulnerability in Jackson by upgrading Jackson dependency
+
 ## 5.0.1
 #### New Functionality
 - [#636](https://github.com/marklogic/java-client-api/issues/636) - Support for debugging connection issues

NOTICE.txt

@@ -24,10 +24,19 @@ The Java API supports the following core features of the MarkLogic database:
 *  Query data structure trees, marked-up text, and all the hybrids in between those extremes.
 *  Project values, tuples, and triples from hierarchical documents and aggregate over them.
 *  Patch documents with partial updates.
-*  Match documents against alerting rules expressed as queries.
 *  Use Optimistic Locking to detect contention without creating locks on the server.
 *  Execute ACID modifications so the change either succeeds or throws an exception.
 *  Execute multi-statement transactions so changes to multiple documents succeed or fail together.
+*  Call Data Services by means of a Java interface on the client for data functionality 
+implemented by an endpoint on the server.
+
+### What's New in Java Client API 5
+
+*  Splitters for CSV records, for entries in a ZipInputStream, and for line-delimited JSON or XML
+for streaming to WriteBatcher.
+*  Support for Bulk IO Data Services to make it easy to implement connectors for dataflow
+frameworks - see https://github.com/marklogic/java-client-api/wiki/Bulk-Data-Services for more 
+detail.
 
 ### What's New in Java Client API 4
 

README.md

@@ -1,5 +1,5 @@
 group=com.marklogic
-version=5.0.1
-releaseVersion=5.0.1
+version=5.1.0
+releaseVersion=5.1.0
 describedName=MarkLogic Java Client API
 publishUrl=file:../marklogic-java/releases

gradle.properties

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2019 MarkLogic Corporation
+ * Copyright 2014-2020 MarkLogic Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -2221,4 +2221,109 @@ public void testQueryBatcherWithIterator() throws Exception {
 	  clearDB();
   }
   }
+
+  // Verify UTF-8 char in URI. Refer to Git Issue 1163.
+
+  @Test
+  public void testUTF8InUri() throws Exception
+  {
+        System.out.println("Running testUTF8InUri");
+        DatabaseClient clientTmp = null;
+        DataMovementManager dmManagerTmp = null;
+
+        try {
+          System.out.println("Running testUTF8InUri");
+
+          String[] filenames = { "constraint1.xml", "constraint2.xml", "constraint3.xml", "constraint4.xml", "constraint5.xml" };
+          String combinedQueryFileName = "combinedQueryOptionJSON.json";
+
+          clientTmp = getDatabaseClient("eval-user", "x", getConnType());
+          dmManagerTmp = clientTmp.newDataMovementManager();
+
+          QueryManager queryMgr = clientTmp.newQueryManager();
+          String dataFileDir = dataConfigDirPath + "/data/";
+          String combQueryFileDir = dataConfigDirPath + "/combined/";
+
+          // Use WriteBatcher to write the same files.
+          WriteBatcher wbatcher = dmManagerTmp.newWriteBatcher();
+
+          wbatcher.withBatchSize(2);
+          InputStreamHandle contentHandle1 = new InputStreamHandle();
+          contentHandle1.set(new FileInputStream(dataFileDir + filenames[0]));
+          InputStreamHandle contentHandle2 = new InputStreamHandle();
+          contentHandle2.set(new FileInputStream(dataFileDir + filenames[1]));
+          InputStreamHandle contentHandle3 = new InputStreamHandle();
+          contentHandle3.set(new FileInputStream(dataFileDir + filenames[2]));
+          InputStreamHandle contentHandle4 = new InputStreamHandle();
+          contentHandle4.set(new FileInputStream(dataFileDir + filenames[3]));
+          InputStreamHandle contentHandle5 = new InputStreamHandle();
+          contentHandle5.set(new FileInputStream(dataFileDir + filenames[4]));
+
+          wbatcher.add(filenames[0], contentHandle1);
+          wbatcher.add(filenames[1], contentHandle2);
+          wbatcher.add(filenames[2], contentHandle3);
+          wbatcher.add(filenames[3], contentHandle4);
+          wbatcher.add("/CXXXX_Ü_testqa.xml", contentHandle5);
+
+          wbatcher.flushAndWait();
+
+          // get the combined query
+          File file = new File(combQueryFileDir + combinedQueryFileName);
+
+          // create a handle for the search criteria
+          FileHandle rawHandle = (new FileHandle(file)).withFormat(Format.JSON);
+          // create a search definition based on the handle
+          RawCombinedQueryDefinition querydef = queryMgr.newRawCombinedQueryDefinition(rawHandle);
+
+          StringBuilder batchResults = new StringBuilder();
+          StringBuilder batchFailResults = new StringBuilder();
+
+          // Run a QueryBatcher on the new URIs.
+          QueryBatcher queryBatcher1 = dmManagerTmp.newQueryBatcher(querydef);
+
+          queryBatcher1.onUrisReady(batch -> {
+            for (String str : batch.getItems()) {
+              batchResults.append(str)
+                      .append('|');
+            }
+          });
+          queryBatcher1.onQueryFailure(throwable -> {
+            System.out.println("Exceptions thrown from callback onQueryFailure");
+            throwable.printStackTrace();
+            batchFailResults.append("Test has Exceptions");
+          });
+
+          dmManagerTmp.startJob(queryBatcher1);
+          boolean bJobFinished = queryBatcher1.awaitCompletion(3, TimeUnit.MINUTES);
+
+          if (bJobFinished) {
+
+            if (!batchFailResults.toString().isEmpty() && batchFailResults.toString().contains("Exceptions")) {
+              fail("Test failed due to exceptions");
+            }
+
+            // Verify the batch results now.
+            String[] res = batchResults.toString().split("\\|");
+            assertEquals("Number of reults returned is incorrect", 1, res.length);
+            assertTrue("URI returned not correct", res[0].trim().contains("/CXXXX_Ü_testqa.xml"));
+
+            // Read the document and assert on the value
+            DOMHandle contentHandle = new DOMHandle();
+            contentHandle = readDocumentUsingDOMHandle(clientTmp, "/CXXXX_Ü_testqa.xml", "XML");
+            Document readDoc = contentHandle.get();
+            System.out.println(convertXMLDocumentToString(readDoc));
+
+            assertTrue("Document content returned not correct", readDoc.getElementsByTagName("id").item(0).getTextContent().contains("0026"));
+            assertTrue("Document content returned not correct", readDoc.getElementsByTagName("title").item(0).getTextContent().contains("The memex"));
+            assertTrue("Document content returned not correct", readDoc.getElementsByTagName("date").item(0).getTextContent().contains("2009-05-05"));
+          }
+        } catch (Exception e) {
+          System.out.println("Exceptions thrown from testUTF8InUri");
+          System.out.println(e.getMessage());
+          fail("testUTF8InUri mathod failed");
+        } finally {
+          clientTmp.release();
+          clearDB();
+        }
+      }
 }

marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/StringQueryHostBatcherTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2019 MarkLogic Corporation
+ * Copyright 2014-2020 MarkLogic Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -246,14 +246,22 @@ private void replenishStream() throws Exception {
 			// the callback receives the output stream
 			public void write(OutputStream out) throws IOException {
 				// acquire the content
-				InputStream docStreamwrongjson = new FileInputStream(
-						WriteHostBatcherTest.class.getResource(TEST_DIR_PREFIX + "WrongFormat.json").getPath());
+				InputStream docStreamwrongjson = null;
+				try {
+					 docStreamwrongjson = new FileInputStream(
+							WriteHostBatcherTest.class.getResource(TEST_DIR_PREFIX + "WrongFormat.json").getPath());
 
-				// copy content to the output stream
-				byte[] buf = new byte[1024];
-				int byteCount = 0;
-				while ((byteCount = docStreamwrongjson.read(buf)) != -1) {
-					out.write(buf, 0, byteCount);
+					// copy content to the output stream
+					byte[] buf = new byte[1024];
+					int byteCount = 0;
+					while ((byteCount = docStreamwrongjson.read(buf)) != -1) {
+						out.write(buf, 0, byteCount);
+					}
+				} catch (Exception e) {
+					e.printStackTrace();
+				}
+				finally {
+					docStreamwrongjson.close();
 				}
 
 			}
@@ -1434,7 +1442,7 @@ public void run() {
 					// TODO Auto-generated catch block
 					e.printStackTrace();
 				}
-				Set threads = Thread.getAllStackTraces().keySet();
+				Set<Thread> threads = Thread.getAllStackTraces().keySet();
 				Iterator<Thread> iter = threads.iterator();
 				while (iter.hasNext()) {
 					Thread t = iter.next();
@@ -1519,7 +1527,7 @@ public void run() {
 					// TODO Auto-generated catch block
 					e.printStackTrace();
 				}
-				Set threads = Thread.getAllStackTraces().keySet();
+				Set<Thread> threads = Thread.getAllStackTraces().keySet();
 				Iterator<Thread> iter = threads.iterator();
 				while (iter.hasNext()) {
 					Thread t = iter.next();
@@ -1611,7 +1619,7 @@ public void run() {
 					// TODO Auto-generated catch block
 					e.printStackTrace();
 				}
-				Set threads = Thread.getAllStackTraces().keySet();
+				Set<Thread> threads = Thread.getAllStackTraces().keySet();
 				Iterator<Thread> iter = threads.iterator();
 				while (iter.hasNext()) {
 					Thread t = iter.next();
@@ -1762,7 +1770,7 @@ public void run() {
 					}
 					Iterator<Entry<String, Integer>> it = threadMap.entrySet().iterator();
 					while (it.hasNext()) {
-						Map.Entry<String, Integer> pair = (Map.Entry) it.next();
+						Map.Entry<String, Integer> pair = (Entry<String, Integer>) it.next();
 						System.out.println("Thread pool: " + pair.getKey() + " = " + pair.getValue() + " Threads");
 						if (pair.getValue() == 20) {
 							count.set(true);
@@ -1823,6 +1831,7 @@ public void testNPESuccessCallBack() throws Exception {
 			System.out.println("Job writes " + batch.getJobWritesSoFar());
 			successCount.getAndAdd(batch.getItems().length);
 			String s = null;
+			// s.length is called purposefully to set off NEE.
 			s.length();
 
 		}).onBatchFailure((batch, throwable) -> {
@@ -2364,8 +2373,6 @@ public void testNullConfig() throws Exception {
 
 		final String query1 = "fn:count(fn:doc())";
 
-		DocumentMetadataHandle meta6 = new DocumentMetadataHandle().withCollections("NoHost").withQuality(0);
-
 		Assert.assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 0);
 
 		WriteBatcher ihb2 = dmManager.newWriteBatcher();
@@ -2910,7 +2917,7 @@ public void testDocumentWriteOperationAdd() throws Exception {
 
 	    metadataHandle.setQuality(23);
 	    DocumentWriteOperationImpl docWriteOpsImp4 = new DocumentWriteOperationImpl(
-	    		OperationType.DISABLE_METADATA_DEFAULT,
+	    		OperationType.DOCUMENT_WRITE,
 	    		docId[4],
 	    		metadataHandle,
 	    		contHandle5

marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/WriteHostBatcherTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2014-2019 MarkLogic Corporation
+ * Copyright 2014-2020 MarkLogic Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -78,22 +78,22 @@ public static void setUpBeforeClass() throws Exception {
     configureRESTServer(dbName, fNames, false);
     TestEvalXquery.createUserRolesWithPrevilages("test-js-eval", "xdbc:eval", "xdbc:eval-in", "xdmp:eval-in", "xdmp:invoke-in", "xdmp:invoke", "xdbc:invoke-in", "any-uri",
         "xdbc:invoke");
-    TestEvalXquery.createRESTUser("eval-user", "x", "test-js-eval");
+    TestEvalXquery.createRESTUser("eval-userJS", "x", "test-js-eval");
     appServerHostname = getRestAppServerHostName();
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
     System.out.println("In tear down");
     cleanupRESTServer(dbName, fNames);
-    TestEvalXquery.deleteRESTUser("eval-user");
+    TestEvalXquery.deleteRESTUser("eval-userJS");
     TestEvalXquery.deleteUserRole("test-js-eval");
   }
 
   @Before
   public void setUp() throws KeyManagementException, NoSuchAlgorithmException, Exception {
     int restPort = getRestServerPort();
-    client = getDatabaseClientOnDatabase(appServerHostname, restPort, dbName, "eval-user", "x", getConnType());
+    client = getDatabaseClientOnDatabase(appServerHostname, restPort, dbName, "eval-userJS", "x", getConnType());
   }
 
   @After
@@ -579,8 +579,8 @@ public void testJSReturningDifferentTypesOrder3fromModules() throws Exception {
           Capability.UPDATE, Capability.READ, Capability.EXECUTE);
       DocumentManager dm = moduleClient.newDocumentManager();
       dm.write("/data/javascriptQueries.sjs", metadataHandle, ish);
-      DocumentBuilder db = DocumentBuilderFactory.newInstance()
-          .newDocumentBuilder();
+      /*DocumentBuilder db = DocumentBuilderFactory.newInstance()
+          .newDocumentBuilder();*/
       InputSource is = new InputSource();
       is.setCharacterStream(new StringReader(
           "<foo attr=\"attribute\"><?processing instruction?><!--comment-->test1</foo>"));

marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/functionaltest/TestEvalJavaScript.java

@@ -18,12 +18,12 @@ dependencies {
     compile group: 'com.sun.mail', name: 'javax.mail', version:'1.6.1'
     compile group: 'javax.ws.rs', name: 'javax.ws.rs-api', version:'2.1'
     compile group: 'org.slf4j', name: 'slf4j-api', version:'1.7.25'
-    compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version:'2.9.8'
-    compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version:'2.9.8'
-    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version:'2.9.8'
-    compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-csv', version:'2.9.8'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version:'2.10.1'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version:'2.10.1'
+    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version:'2.10.1'
+    compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-csv', version:'2.10.1'
     testCompile group: 'org.mockito', name: 'mockito-all', version:'1.10.19'
-    testCompile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-xml', version:'2.9.8'
+    testCompile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-xml', version:'2.10.1'
     testCompile group: 'ch.qos.logback', name: 'logback-classic', version:'1.2.3'
     testCompile group: 'org.hsqldb', name: 'hsqldb', version:'2.4.0'
     compileOnly group: 'org.jdom', name: 'jdom2', version:'2.0.6'

marklogic-client-api/build.gradle

@@ -158,14 +158,17 @@ public HostnameVerifierAdapter(SSLHostnameVerifier verifier) {
     	/**
     	 * verify method verifies the incoming hostname and SSLSession.
     	 * @param hostname denotes the hostname.
-    	 * @param session represents the SSLSession containing the pper certificates.
+    	 * @param session represents the SSLSession containing the peer certificates.
     	 * @return true if the hostname and peer certificates are valid and false if they are invalid.
     	 * */
     	@Override
     	public boolean verify(String hostname, SSLSession session) {
     		try {
     			Certificate[] certificates = session.getPeerCertificates();
-	            verify(hostname, (X509Certificate) certificates[0]);
+	            verify(
+	                hostname,
+                    (X509Certificate) ((certificates == null || certificates.length == 0) ? null : certificates[0])
+                );
 	            return true;
 	          } catch(SSLException e) {
 	            return false;
@@ -1496,11 +1499,11 @@ static public class Bean implements Serializable {
     private           String                password;
     private           Authentication        authentication;
     private           String                externalName;
-    private           SecurityContext       securityContext;
     private           DatabaseClient.ConnectionType connectionType;
-    private           HandleFactoryRegistry handleRegistry =
-      HandleFactoryRegistryImpl.newDefault();
 
+    transient private SecurityContext       securityContext;
+    transient private HandleFactoryRegistry handleRegistry =
+      HandleFactoryRegistryImpl.newDefault();
     transient private SSLContext            context;
     transient private SSLHostnameVerifier   verifier;
 

marklogic-client-api/src/main/java/com/marklogic/client/DatabaseClientFactory.java

@@ -21,10 +21,7 @@
 import javax.xml.bind.DatatypeConverter;
 import javax.xml.datatype.Duration;
 
-import com.marklogic.client.FailedRequestException;
-import com.marklogic.client.ForbiddenUserException;
-import com.marklogic.client.ResourceNotFoundException;
-import com.marklogic.client.Transaction;
+import com.marklogic.client.*;
 import com.marklogic.client.bitemporal.TemporalDescriptor;
 import com.marklogic.client.document.DocumentDescriptor;
 import com.marklogic.client.document.DocumentManager;
@@ -68,8 +65,9 @@ public String toString() {
           return "noWipe";
         case NOUPDATE:
           return "noUpdate";
+        default:
+          throw new MarkLogicInternalException("Unknown enumeration");
       }
-      return null;
     }
   }