Skip to content

MLE-23146 Trying another approach with clearDB #1811

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 25, 2025
Merged

MLE-23146 Trying another approach with clearDB #1811

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 25 additions & 48 deletions ...st/java/com/marklogic/client/datamovement/functionaltests/StringQueryHostBatcherTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1220,7 +1220,7 @@ public void testQueryBatcherFailures() throws IOException, InterruptedException
}

@Test
public void testServerXQueryTransform() throws IOException, ParserConfigurationException, SAXException, TransformerException, InterruptedException, XPathExpressionException
public void testServerXQueryTransform() throws InterruptedException, XPathExpressionException
{
System.out.println("Running testServerXQueryTransform");
try {
Expand Down Expand Up @@ -1272,8 +1272,7 @@ public void testServerXQueryTransform() throws IOException, ParserConfigurationE
batcher.flushAndWait();
dmManager.stopJob(batcher);

StringBuffer batchResults = new StringBuffer();
StringBuffer batchFailResults = new StringBuffer();
List<String> uris = new ArrayList<>();

// create query def
QueryManager queryMgr = client.newQueryManager();
Expand All @@ -1285,56 +1284,34 @@ public void testServerXQueryTransform() throws IOException, ParserConfigurationE
queryBatcher1.withBatchSize(5);

queryBatcher1.onUrisReady(batch -> {
for (String str : batch.getItems()) {
batchResults.append(str);
batchResults.append("|");
for (String item : batch.getItems()) {
uris.add(item);
}
});
queryBatcher1.onQueryFailure(throwable -> {
System.out.println("Exceptions thrown from callback onQueryFailure");
throwable.printStackTrace();
batchFailResults.append("Test has Exceptions");
});
dmManager.startJob(queryBatcher1);
queryBatcher1.awaitCompletion(3, TimeUnit.MINUTES);
while (!queryBatcher1.isStopped()) {
// Do nothing. Wait for batcher to complete.
}

if (queryBatcher1.isStopped()) {
// Verify the batch results now.
String[] res = batchResults.toString().split("\\|");
assertEquals(res.length, 20);

// Get a random URI, since the URIs returned are not ordered. Get the 3rd
// URI.
assertTrue( res[2].contains("foo") || res[2].contains("bar"));

// do a lookup with the first URI using the client to verify transforms
// are done.
DOMHandle readHandle = readDocumentUsingDOMHandle(client, res[0], "XML");
String contents = readHandle.evaluateXPath("/foo/text()", String.class);
String attribute = readHandle.evaluateXPath("/foo/@Lang", String.class);
// Verify that the contents are of xmlStr1 or xmlStr2.

System.out.println("Contents are : " + contents);
System.out.println("Contents are : " + attribute);
assertTrue( xmlStr1.contains(contents) || xmlStr2.contains(contents));
assertTrue( attribute.equalsIgnoreCase("English"));
}
else {
fail("testServerXQueryTransform method failed");
}
} catch (Exception e) {
e.printStackTrace();
fail("testServerXQueryTransform method failed");
queryBatcher1.awaitCompletion();

// Verify the batch results now.
assertEquals(20, uris.size());

// Get a random URI, since the URIs returned are not ordered. Get the 3rd
// URI.
String thirdUri = uris.get(2);
assertTrue( thirdUri.contains("foo") || thirdUri.contains("bar"), "Unexpected URI: " + thirdUri);

// do a lookup with the first URI using the client to verify transforms
// are done.
DOMHandle readHandle = readDocumentUsingDOMHandle(client, uris.get(0), "XML");
String contents = readHandle.evaluateXPath("/foo/text()", String.class);
String attribute = readHandle.evaluateXPath("/foo/@Lang", String.class);
// Verify that the contents are of xmlStr1 or xmlStr2.
System.out.println("Contents are : " + contents);
System.out.println("Contents are : " + attribute);
assertTrue( xmlStr1.contains(contents) || xmlStr2.contains(contents));
assertTrue( attribute.equalsIgnoreCase("English"));
}
finally {
try {
clearDB();
} catch (Exception e) {
e.printStackTrace();
}
clearDB();
}
}

Expand Down
16 changes: 11 additions & 5 deletions ...pi-functionaltests/src/test/java/com/marklogic/client/functionaltest/ConnectedRESTQA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.marklogic.client.DatabaseClient.ConnectionType;
import com.marklogic.client.DatabaseClientBuilder;
import com.marklogic.client.DatabaseClientFactory;
import com.marklogic.client.FailedRequestException;
import com.marklogic.client.admin.ServerConfigurationManager;
import com.marklogic.client.impl.SSLUtil;
import com.marklogic.client.io.DocumentMetadataHandle;
Expand Down Expand Up @@ -317,10 +318,15 @@ public static void clearDB(int port) {
try (DatabaseClient client = newDatabaseClientBuilder().withPort(port).build()) {
QueryManager mgr = client.newQueryManager();
mgr.delete(mgr.newDeleteDefinition());
// Clearing the database occasionally causes a forest to not be available for a moment or two when the tests
// are running on Jenkins. This leads to intermittent failures. Waiting is not guaranteed to avoid the
// error but simply hopes to minimize the chance of an intermittent failure.
waitFor(2000);
} catch (FailedRequestException ex) {
LoggerFactory.getLogger(ConnectedRESTQA.class).warn("Unable to clear database. This intermittently " +
"happens while running tests on Jenkins, typically with a server error message such as: " +
"XDMP-FORESTNOT: Forest StringQueryHostBatcherDB-1 not available: XDMP-FORESTERR: " +
"Error in clear of forest StringQueryHostBatcherDB-1: SVC-FILREN: File rename error: " +
"rename '/var/opt/MarkLogic/TmpForests/StringQueryHostBatcherDB-1/Journals to " +
"/var/opt/MarkLogic/Forests/StringQueryHostBatcherDB-1/Journals': No such file or directory. " +
"This error is caught and logged in the hopes that proceeding tests will succeed even though " +
"this clearDB call failed.", ex);
Comment on lines +322 to +329
Copy link

Copilot AI Jul 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider catching a more specific exception type or adding additional catch blocks for different failure scenarios. FailedRequestException is quite broad and may catch unrelated database errors that should be handled differently than forest availability issues.

Suggested change
LoggerFactory.getLogger(ConnectedRESTQA.class).warn("Unable to clear database. This intermittently " +
"happens while running tests on Jenkins, typically with a server error message such as: " +
"XDMP-FORESTNOT: Forest StringQueryHostBatcherDB-1 not available: XDMP-FORESTERR: " +
"Error in clear of forest StringQueryHostBatcherDB-1: SVC-FILREN: File rename error: " +
"rename '/var/opt/MarkLogic/TmpForests/StringQueryHostBatcherDB-1/Journals to " +
"/var/opt/MarkLogic/Forests/StringQueryHostBatcherDB-1/Journals': No such file or directory. " +
"This error is caught and logged in the hopes that proceeding tests will succeed even though " +
"this clearDB call failed.", ex);
String errorMessage = ex.getMessage();
if (errorMessage != null && (errorMessage.contains("XDMP-FORESTNOT") || errorMessage.contains("XDMP-FORESTERR"))) {
LoggerFactory.getLogger(ConnectedRESTQA.class).warn("Unable to clear database due to forest-related error. " +
"This intermittently happens while running tests on Jenkins, typically with a server error message such as: " +
"XDMP-FORESTNOT: Forest StringQueryHostBatcherDB-1 not available: XDMP-FORESTERR: " +
"Error in clear of forest StringQueryHostBatcherDB-1: SVC-FILREN: File rename error: " +
"rename '/var/opt/MarkLogic/TmpForests/StringQueryHostBatcherDB-1/Journals to " +
"/var/opt/MarkLogic/Forests/StringQueryHostBatcherDB-1/Journals': No such file or directory. " +
"This error is caught and logged in the hopes that proceeding tests will succeed even though " +
"this clearDB call failed.", ex);
} else {
LoggerFactory.getLogger(ConnectedRESTQA.class).error("Failed to clear database due to an unexpected error.", ex);
throw ex; // Rethrow the exception to ensure proper handling.
}

Copilot uses AI. Check for mistakes.
}
}

Expand Down Expand Up @@ -814,7 +820,7 @@ public X509Certificate[] getAcceptedIssuers() {
* Server.
* @throws Exception
*/
public static void clearDB() throws Exception {
public static void clearDB() {
clearDB(getRestServerPort());
}

Expand Down