DHFPROD-8258: Reduce index sleep escalation and look for SVC-EXTIME

Author: ryanjdew

marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.smart-mastering/impl/util.xqy

@@ -283,12 +283,13 @@ declare function util-impl:process-items-in-set-time(
     for $item in $items
     return $fun($item)
   } catch ($e) {
-    if ($e/error:code eq "XDMP-EXTIME") then (
+    if ($e/error:code = ("XDMP-EXTIME", "SVC-EXTIME")) then (
       let $items-cost := $items ! $item-cost-fun(.)
       let $high-outliers-cost := util-impl:determine-high-outliers($items-cost)
       let $high-outlier-indexes := $high-outliers-cost ! fn:index-of($items-cost, .)
-      let $high-outliers := $items[fn:position() = $high-outlier-indexes]
-      let $others := $items[fn:not(fn:position() = $high-outlier-indexes)]
+      let $others := if (fn:count($high-outlier-indexes) eq 0) then () else $items[fn:not(fn:position() = $high-outlier-indexes)]
+      (: If there are no high outliers, than all items have a high cost and should be handled by our high outlier function :)
+      let $high-outliers := if (fn:empty($others)) then $items else $items[fn:position() = $high-outlier-indexes]
       return (
         util-impl:process-items-in-set-time(
           $fun,

marklogic-data-hub/src/test/ml-modules/root/test/data-hub-test-helper.xqy

@@ -374,20 +374,20 @@ declare function run-with-roles-and-privileges($roles, $privileges, $func-or-mod
 declare function wait-for-indexes()
 {
   try {
-    wait-for-indexes(1)
+    run-with-roles-and-privileges("admin", (), function() {
+      wait-for-indexes(1)
+    }, ())
   } catch ($e) {
-    if ($e/error:code = "XDMP-EXTIME") then ()
+    if ($e/error:code = ("XDMP-EXTIME", "SVC-EXTIME")) then ()
     else xdmp:rethrow()
   }
 };
 
 declare function wait-for-indexes($count as xs:unsignedLong) {
-  let $is-indexing := run-with-roles-and-privileges("admin", (), function() {
-    0 lt fn:sum(xdmp:forest-counts(xdmp:database-forests(xdmp:database()), (), "preview-reindexer")/*:reindex-refragment-fragment-count, 0)
-  }, ())
+  let $is-indexing := 0 lt fn:sum(xdmp:forest-counts(xdmp:database-forests(xdmp:database()), (), "preview-reindexer")/*:reindex-refragment-fragment-count, 0)
   return
-    if ($is-indexing) then
-      let $_sleep := xdmp:sleep(5 * $count)
+    if ($is-indexing and $count lt 250) then
+      let $_sleep := xdmp:sleep(250 * (($count idiv 100) + 1))
       return wait-for-indexes($count + 1)
     else
       ()