Skip to content

Commit cb932b8

Browse files
rjrudinrjrudin
authored
Merge pull request #1536 from marklogic/feature/more-fast-func-tests
Converted more functional tests to "fast"
2 parents 1aae8f8 + d906b70 commit cb932b8

File tree

11 files changed

+466
-755
lines changed

11 files changed

+466
-755
lines changed

‎marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/AbstractFunctionalTest.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,6 @@
2929
*/
3030
public abstract class AbstractFunctionalTest extends BasicJavaClientREST {
3131

32-
protected final static String OPTIC_USER = "opticUser";
33-
protected final static String OPTIC_USER_PASSWORD = "0pt1c";
3432
protected final static String DB_NAME = "java-functest";
3533

3634
protected static DatabaseClient client;

‎marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/ApplyTransformTest.java‎ renamed to ‎marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/ApplyTransformTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
package com.marklogic.client.datamovement.functionaltests;
17+
package com.marklogic.client.fastfunctest;
1818

1919
import com.fasterxml.jackson.databind.JsonNode;
2020
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -23,6 +23,7 @@
2323
import com.marklogic.client.admin.TransformExtensionsManager;
2424
import com.marklogic.client.datamovement.*;
2525
import com.marklogic.client.datamovement.ApplyTransformListener.ApplyResult;
26+
import com.marklogic.client.datamovement.functionaltests.WriteHostBatcherTest;
2627
import com.marklogic.client.document.DocumentPage;
2728
import com.marklogic.client.document.DocumentRecord;
2829
import com.marklogic.client.document.ServerTransform;

‎marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/datamovement/functionaltests/DeleteListenerTest.java‎ renamed to ‎marklogic-client-api-functionaltests/src/test/java/com/marklogic/client/fastfunctest/DeleteListenerTest.java‎

Lines changed: 138 additions & 165 deletions
Original file line numberDiff line numberDiff line change
@@ -13,37 +13,54 @@
1313
* See the License for the specific language governing permissions and
1414
* limitations under the License.
1515
*/
16-
package com.marklogic.client.datamovement.functionaltests;
16+
package com.marklogic.client.fastfunctest;
1717

1818
import com.fasterxml.jackson.databind.JsonNode;
1919
import com.fasterxml.jackson.databind.ObjectMapper;
2020
import com.marklogic.client.DatabaseClient;
21-
import com.marklogic.client.datamovement.*;
21+
import com.marklogic.client.datamovement.DataMovementManager;
22+
import com.marklogic.client.datamovement.DeleteListener;
23+
import com.marklogic.client.datamovement.JobTicket;
24+
import com.marklogic.client.datamovement.QueryBatcher;
25+
import com.marklogic.client.datamovement.UrisToWriterListener;
26+
import com.marklogic.client.datamovement.WriteBatcher;
27+
import com.marklogic.client.datamovement.functionaltests.WriteHostBatcherTest;
2228
import com.marklogic.client.document.DocumentPage;
2329
import com.marklogic.client.document.DocumentRecord;
24-
import com.marklogic.client.functionaltest.BasicJavaClientREST;
25-
import com.marklogic.client.io.*;
30+
import com.marklogic.client.io.DocumentMetadataHandle;
31+
import com.marklogic.client.io.FileHandle;
32+
import com.marklogic.client.io.Format;
33+
import com.marklogic.client.io.JacksonHandle;
34+
import com.marklogic.client.io.StringHandle;
2635
import com.marklogic.client.query.StructuredQueryBuilder;
2736
import org.apache.commons.io.FileUtils;
28-
import org.junit.jupiter.api.*;
29-
30-
import java.io.*;
31-
import java.util.*;
37+
import org.junit.jupiter.api.BeforeAll;
38+
import org.junit.jupiter.api.BeforeEach;
39+
import org.junit.jupiter.api.Test;
40+
41+
import java.io.BufferedReader;
42+
import java.io.File;
43+
import java.io.FileReader;
44+
import java.io.FileWriter;
45+
import java.io.IOException;
46+
import java.util.ArrayList;
47+
import java.util.Arrays;
48+
import java.util.Collections;
49+
import java.util.HashSet;
50+
import java.util.Iterator;
51+
import java.util.Set;
3252
import java.util.concurrent.atomic.AtomicInteger;
3353

34-
import static org.junit.jupiter.api.Assertions.*;
54+
import static org.junit.jupiter.api.Assertions.assertEquals;
55+
import static org.junit.jupiter.api.Assertions.assertTrue;
56+
import static org.junit.jupiter.api.Assertions.fail;
3557

36-
public class DeleteListenerTest extends BasicJavaClientREST {
58+
public class DeleteListenerTest extends AbstractFunctionalTest {
3759

38-
private static String dbName = "DeleteListener";
3960
private static DataMovementManager dmManager = null;
4061
private static final String TEST_DIR_PREFIX = "/WriteHostBatcher-testdata/";
4162

4263
private static DatabaseClient dbClient;
43-
private static String user = "admin";
44-
private static int port = 8000;
45-
private static String password = "admin";
46-
private static String server = "App-Services";
4764
private static JacksonHandle jacksonHandle;
4865
private static StringHandle stringHandle;
4966
private static FileHandle fileHandle;
@@ -54,37 +71,10 @@ public class DeleteListenerTest extends BasicJavaClientREST {
5471
private static File fileJson;
5572
private static JsonNode jsonNode;
5673
private static final String query1 = "fn:count(fn:doc())";
57-
private static String[] hostNames;
58-
private static int forestCount = 1;
5974

6075
@BeforeAll
6176
public static void setUpBeforeClass() throws Exception {
62-
loadGradleProperties();
63-
server = getRestAppServerName();
64-
port = getRestAppServerPort();
65-
66-
hostNames = getHosts();
67-
createDB(dbName);
68-
Thread.currentThread().sleep(500L);
69-
//Ensure DB has at-least one forest
70-
createForestonHost(dbName + "-" + forestCount, dbName, hostNames[0]);
71-
forestCount++;
72-
for (String forestHost : hostNames) {
73-
for(int i = 0; i < new Random().nextInt(3); i++) {
74-
createForestonHost(dbName + "-" + forestCount, dbName, forestHost);
75-
forestCount++;
76-
}
77-
Thread.currentThread().sleep(500L);
78-
}
79-
// Create App Server if needed.
80-
createRESTServerWithDB(server, port);
81-
82-
assocRESTServer(server, dbName, port);
83-
if (IsSecurityEnabled()) {
84-
enableSecurityOnRESTServer(server, dbName);
85-
}
86-
87-
dbClient = getDatabaseClient(user, password, getConnType());
77+
dbClient = newDatabaseClientBuilder().build();
8878
dmManager = dbClient.newDataMovementManager();
8979

9080
// JacksonHandle
@@ -105,21 +95,10 @@ public static void setUpBeforeClass() throws Exception {
10595
fileHandle.setFormat(Format.JSON);
10696
}
10797

108-
@AfterAll
109-
public static void tearDownAfterClass() throws Exception {
110-
associateRESTServerWithDB(server, "Documents");
111-
for (int i = 0; i < forestCount -1; i++) {
112-
System.out.println(dbName + "-" + (i + 1));
113-
detachForest(dbName, dbName + "-" + (i + 1));
114-
deleteForest(dbName + "-" + (i + 1));
115-
}
116-
117-
deleteDB(dbName);
118-
}
119-
12098
@BeforeEach
12199
public void setUp() throws Exception {
122-
Thread.currentThread().sleep(1000L);
100+
deleteDocuments(client);
101+
123102
WriteBatcher ihb2 = dmManager.newWriteBatcher();
124103
ihb2.withBatchSize(27).withThreadCount(10);
125104
dmManager.startJob(ihb2);
@@ -132,11 +111,6 @@ public void setUp() throws Exception {
132111
assertEquals(2000, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue());
133112
}
134113

135-
@AfterEach
136-
public void tearDown() throws Exception {
137-
clearDB(port);
138-
}
139-
140114
@Test
141115
public void massDeleteSingleThread() throws Exception {
142116
HashSet<String> urisList = new HashSet<>();
@@ -162,7 +136,6 @@ public void massDeleteSingleThread() throws Exception {
162136
queryBatcher.awaitCompletion();
163137
dmManager.stopJob(ticket);
164138

165-
Thread.currentThread().sleep(2000L);
166139
assertEquals(2000, urisList.size());
167140

168141
AtomicInteger successDocs = new AtomicInteger();
@@ -241,32 +214,31 @@ public void massDeleteMultipleThreads() throws Exception {
241214
}
242215

243216
@Test
244-
public void massDeleteConsistentSnapShot() throws Exception {
245-
Map<String, String> props = new HashMap<String, String>();
246-
props.put("merge-timestamp", "-6000000000");
247-
changeProperty(props, "/manage/v2/databases/" + dbName + "/properties");
248-
Thread.currentThread().sleep(5000L);
249-
250-
QueryBatcher queryBatcher = dmManager.newQueryBatcher(
251-
new StructuredQueryBuilder().collection("DeleteListener"))
252-
.withBatchSize(7)
253-
.withConsistentSnapshot()
254-
.withThreadCount(5)
255-
.onUrisReady(new DeleteListener())
256-
.onQueryFailure(throwable -> {
257-
System.out.println("Exceptions thrown from callback onQueryFailure");
258-
throwable.printStackTrace();
259-
260-
});
217+
public void massDeleteConsistentSnapShot() {
218+
setMergeTimestamp(DB_NAME, "-600000000");
219+
220+
try {
221+
QueryBatcher queryBatcher = dmManager.newQueryBatcher(
222+
new StructuredQueryBuilder().collection("DeleteListener"))
223+
.withBatchSize(7)
224+
.withConsistentSnapshot()
225+
.withThreadCount(5)
226+
.onUrisReady(new DeleteListener())
227+
.onQueryFailure(throwable -> {
228+
System.out.println("Exceptions thrown from callback onQueryFailure");
229+
throwable.printStackTrace();
230+
231+
});
232+
233+
JobTicket ticket = dmManager.startJob(queryBatcher);
234+
queryBatcher.awaitCompletion();
235+
dmManager.stopJob(ticket);
236+
237+
assertEquals(0, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue());
238+
} finally {
239+
setMergeTimestamp(DB_NAME, "0");
240+
}
261241

262-
JobTicket ticket = dmManager.startJob(queryBatcher);
263-
queryBatcher.awaitCompletion();
264-
dmManager.stopJob(ticket);
265-
266-
props.put("merge-timestamp", "0");
267-
changeProperty(props, "/manage/v2/databases/" + dbName + "/properties");
268-
// if ( failures2.length() > 0 ) fail(failures2.toString());
269-
assertEquals(0, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue());
270242
}
271243

272244
@Test
@@ -303,84 +275,85 @@ public void deleteNonExistentDoc() throws Exception {
303275
// ISSUE 94
304276
@Test
305277
public void deleteServerFile() throws Exception {
306-
307-
Map<String, String> props = new HashMap<String, String>();
308-
props.put("merge-timestamp", "-6000000000");
309-
changeProperty(props, "/manage/v2/databases/" + dbName + "/properties");
310-
Thread.currentThread().sleep(5000L);
311-
312-
class MyRunnable implements Runnable {
313-
@Override
314-
public void run() {
315-
for (int j = 1999; j >= 200; j--) {
316-
dbClient.newDocumentManager().delete("/local/json-" + j);
317-
}
318-
}
319-
}
320-
Thread t1;
321-
t1 = new Thread(new MyRunnable());
322-
323-
Set<String> urisList = Collections.synchronizedSet(new HashSet<>());
324-
325-
QueryBatcher queryBatcher = dmManager.newQueryBatcher(
326-
new StructuredQueryBuilder().collection("DeleteListener"))
327-
.withBatchSize(11)
328-
.withThreadCount(4)
329-
.withConsistentSnapshot()
330-
.onUrisReady(batch -> {
331-
for (String s : batch.getItems()) {
332-
urisList.add(s);
333-
}
334-
})
335-
.onQueryFailure(throwable -> {
336-
System.out.println("Exceptions thrown from callback onQueryFailure");
337-
throwable.printStackTrace();
338-
339-
});
340-
341-
t1.start();
342-
JobTicket ticket = dmManager.startJob(queryBatcher);
343-
344-
queryBatcher.awaitCompletion();
345-
t1.join();
346-
dmManager.stopJob(ticket);
347-
348-
System.out.println("URI's size " + urisList.size());
349-
AtomicInteger successDocs = new AtomicInteger();
350-
Set<String> uris2 = Collections.synchronizedSet(new HashSet<>());
351-
StringBuffer failures2 = new StringBuffer();
352-
353-
QueryBatcher deleteBatcher = dmManager.newQueryBatcher(urisList.iterator())
354-
.withBatchSize(13)
355-
.withThreadCount(5)
356-
.onUrisReady(new DeleteListener())
357-
.onUrisReady(batch -> successDocs.addAndGet(batch.getItems().length))
358-
.onUrisReady(batch -> uris2.addAll(Arrays.asList(batch.getItems())))
359-
.onQueryFailure(throwable -> {
360-
throwable.printStackTrace();
361-
failures2.append("ERROR:[" + throwable + "]\n");
362-
});
363-
364-
JobTicket delTicket = dmManager.startJob(deleteBatcher);
365-
deleteBatcher.awaitCompletion();
366-
dmManager.stopJob(delTicket);
367-
368-
if (failures2.length() > 0)
369-
fail(failures2.toString());
370-
371-
assertEquals(0, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue());
372-
373-
DocumentPage page = dbClient.newDocumentManager().read("/local/json-1998");
374-
JacksonHandle dh = new JacksonHandle();
375-
while (page.hasNext()) {
376-
DocumentRecord rec = page.next();
377-
rec.getContent(dh);
378-
System.out.println("Results are: " + dh.get().get("k1").asText());
379-
380-
}
381-
382-
props.put("merge-timestamp", "0");
383-
changeProperty(props, "/manage/v2/databases/" + dbName + "/properties");
278+
// if (true) return;
279+
280+
setMergeTimestamp(DB_NAME, "-6000000000");
281+
282+
try {
283+
// No idea what this is supposed to be doing, as the docs it's deleting don't ever exist.
284+
// j initially was 1999 but that was causing the test to take about 20s, so lowered it.
285+
class MyRunnable implements Runnable {
286+
@Override
287+
public void run() {
288+
for (int j = 300; j >= 200; j--) {
289+
dbClient.newDocumentManager().delete("/local/json-" + j);
290+
}
291+
}
292+
}
293+
Thread t1;
294+
t1 = new Thread(new MyRunnable());
295+
296+
Set<String> urisList = Collections.synchronizedSet(new HashSet<>());
297+
298+
QueryBatcher queryBatcher = dmManager.newQueryBatcher(
299+
new StructuredQueryBuilder().collection("DeleteListener"))
300+
.withBatchSize(11)
301+
.withThreadCount(4)
302+
.withConsistentSnapshot()
303+
.onUrisReady(batch -> {
304+
for (String s : batch.getItems()) {
305+
urisList.add(s);
306+
}
307+
})
308+
.onQueryFailure(throwable -> {
309+
System.out.println("Exceptions thrown from callback onQueryFailure");
310+
throwable.printStackTrace();
311+
312+
});
313+
314+
t1.start();
315+
JobTicket ticket = dmManager.startJob(queryBatcher);
316+
317+
queryBatcher.awaitCompletion();
318+
t1.join();
319+
dmManager.stopJob(ticket);
320+
321+
System.out.println("URI's size " + urisList.size());
322+
AtomicInteger successDocs = new AtomicInteger();
323+
Set<String> uris2 = Collections.synchronizedSet(new HashSet<>());
324+
StringBuffer failures2 = new StringBuffer();
325+
326+
QueryBatcher deleteBatcher = dmManager.newQueryBatcher(urisList.iterator())
327+
.withBatchSize(13)
328+
.withThreadCount(5)
329+
.onUrisReady(new DeleteListener())
330+
.onUrisReady(batch -> successDocs.addAndGet(batch.getItems().length))
331+
.onUrisReady(batch -> uris2.addAll(Arrays.asList(batch.getItems())))
332+
.onQueryFailure(throwable -> {
333+
throwable.printStackTrace();
334+
failures2.append("ERROR:[" + throwable + "]\n");
335+
});
336+
337+
JobTicket delTicket = dmManager.startJob(deleteBatcher);
338+
deleteBatcher.awaitCompletion();
339+
dmManager.stopJob(delTicket);
340+
341+
if (failures2.length() > 0)
342+
fail(failures2.toString());
343+
344+
assertEquals(0, dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue());
345+
346+
DocumentPage page = dbClient.newDocumentManager().read("/local/json-1998");
347+
JacksonHandle dh = new JacksonHandle();
348+
while (page.hasNext()) {
349+
DocumentRecord rec = page.next();
350+
rec.getContent(dh);
351+
System.out.println("Results are: " + dh.get().get("k1").asText());
352+
353+
}
354+
} finally {
355+
setMergeTimestamp(DB_NAME, "0");
356+
}
384357
}
385358

386359
@Test

0 commit comments

Comments
 (0)