DHFPROD-3068: Take ...n on ml: prefix handling

Author: ryanjdew

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/HubRESTDocumentFileProcessor.java

@@ -101,7 +101,6 @@ protected void prepareAssetFileLoader(AssetFileLoader loader, CommandContext con
         jarDocumentFileReader.addDocumentFileProcessor(new TokenReplacerDocumentFileProcessor(buildModuleTokenReplacer(appConfig)));
         jarDocumentFileReader.addDocumentFileProcessor(new CollectionsDocumentFileProcessor("hub-core-module"));
         jarDocumentFileReader.addDocumentFileProcessor(new PermissionsDocumentFileProcessor(appConfig.getModulePermissions()));
-        jarDocumentFileReader.addDocumentFileProcessor(new HubRESTDocumentFileProcessor());
         loader.setDocumentFileReader(jarDocumentFileReader);
     }
 

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubModulesCommand.java

@@ -346,13 +346,13 @@ public void clearUserModules() {
             }
 
             HashSet<String> services = new HashSet<>();
-            for (Resource r : resolver.getResources("classpath*:/ml-modules/root/marklogic.rest.resource/*")) {
-                services.add(r.getFilename());
+            for (Resource r : resolver.getResources("classpath*:/ml-modules/services/*")) {
+                services.add(r.getFilename().replaceAll("\\.(sjs|xqy)$",""));
             }
 
             HashSet<String> transforms = new HashSet<>();
-            for (Resource r : resolver.getResources("classpath*:/ml-modules/root/marklogic.rest.transform/*")) {
-                transforms.add(r.getFilename());
+            for (Resource r : resolver.getResources("classpath*:/ml-modules/transforms/*")) {
+                transforms.add(r.getFilename().replaceAll("\\.(sjs|xqy)$",""));
             }
 
             ServerConfigurationManager configMgr = hubConfig.newStagingClient().newServerConfigManager();
@@ -388,8 +388,7 @@ public void clearUserModules() {
             JsonNode transformsList = transformExtensionsManager.listTransforms(new JacksonHandle()).get();
             transformsList.findValuesAsText("name").forEach(
                 x -> {
-                    // may be missing ml: prefix
-                    if (!(transforms.contains(x) || transforms.contains(x.substring(3)) || x.startsWith("ml:"))) {
+                    if (!(transforms.contains(x) || x.startsWith("ml"))) {
                         transformExtensionsManager.deleteTransform(x);
                     }
                 }
@@ -400,8 +399,7 @@ public void clearUserModules() {
             JsonNode resourceExtensions = resourceExtensionsManager.listServices(new JacksonHandle()).get();
             resourceExtensions.findValuesAsText("name").forEach(
                 x -> {
-                    // may be missing ml: prefix
-                    if (!(services.contains(x) || services.contains(x.substring(3)) || x.startsWith("ml:"))) {
+                    if (!(services.contains(x) || x.startsWith("ml"))) {
                         resourceExtensionsManager.deleteServices(x);
                     }
                 }

marklogic-data-hub/src/main/java/com/marklogic/hub/impl/DataHubImpl.java

@@ -30,61 +30,50 @@ parameters.queryParameter(requestParams, "options",fn.false(),fn.false())
 parameters.queryParameter(requestParams, "step",fn.false(),fn.false())
 parameters.queryParameter(requestParams, "database",fn.true(),fn.false())
 
-if(method === 'GET') {
-  const flowName = requestParams["flow-name"];
-  let step = requestParams.step;
-  if (!step) {
-    step = 1;
-  }
-  let options = requestParams.options ? JSON.parse(requestParams.options) : {};
+// Refactored to not set xdmp.eval results to variable for efficiency reasons
+if (method !== 'GET') {
+  fn.error(null, 'RESTAPI-INVALIDREQ', 'unsupported method: '+method);
+}
+const flowName = requestParams["flow-name"];
+let step = requestParams.step;
+if (!step) {
+  step = 1;
+}
+let options = requestParams.options ? JSON.parse(requestParams.options) : {};
 
-  let flowDoc= datahub.flow.getFlow(flowName);
-  let resp;
-  if (!fn.exists(flowDoc)) {
-    resp = fn.error(null, "RESTAPI-SRVEXERR", Sequence.from([404, "Not Found", "The requested flow was not found"]));
+let flowDoc= datahub.flow.getFlow(flowName);
+if (!fn.exists(flowDoc)) {
+  fn.error(null, "RESTAPI-SRVEXERR", Sequence.from([404, "Not Found", "The requested flow was not found"]));
+}
+let stepDoc = flowDoc.steps[step];
+if (!stepDoc) {
+  fn.error(null, "RESTAPI-SRVEXERR", Sequence.from([404, "Not Found", `The step number "${step}" of the flow was not found`]));
+}
+let baseStep = datahub.flow.step.getStepByNameAndType(stepDoc.stepDefinitionName, stepDoc.stepDefinitionType);
+if (!baseStep) {
+  fn.error(null, "RESTAPI-SRVEXERR", Sequence.from([404, "Not Found", `A step with name "${stepDoc.stepDefinitionName}" and type of "${stepDoc.stepDefinitionType}" was not found`]));
+}
+let combinedOptions = Object.assign({}, baseStep.options, flowDoc.options, stepDoc.options, options);
+const database = combinedOptions.sourceDatabase || requestParams.database;
+if(!combinedOptions.sourceQuery && flowDoc.sourceQuery) {
+  combinedOptions.sourceQuery = flowDoc.sourceQuery;
+}
+let query = combinedOptions.sourceQuery;
+if (!query) {
+  datahub.debug.log("The collector query was empty");
+  fn.error(null, "RESTAPI-SRVEXERR", Sequence.from([404, "Not Found", "The collector query was empty"]));
+}
+try {
+  let urisEval;
+  if (/^\s*cts\.uris\(.*\)\s*$/.test(query)) {
+    urisEval = query;
   } else {
-    let stepDoc = flowDoc.steps[step];
-    if (!stepDoc) {
-      resp = fn.error(null, "RESTAPI-SRVEXERR", Sequence.from([404, "Not Found", `The step number "${step}" of the flow was not found`]));
-    }
-    let baseStep = datahub.flow.step.getStepByNameAndType(stepDoc.stepDefinitionName, stepDoc.stepDefinitionType);
-    if (!baseStep) {
-      resp = fn.error(null, "RESTAPI-SRVEXERR", Sequence.from([404, "Not Found", `A step with name "${stepDoc.stepDefinitionName}" and type of "${stepDoc.stepDefinitionType}" was not found`]));
-    }
-    let combinedOptions = Object.assign({}, baseStep.options, flowDoc.options, stepDoc.options, options);
-    const database = combinedOptions.sourceDatabase || requestParams.database;
-    if (stepDoc) {
-      if(!combinedOptions.sourceQuery && flowDoc.sourceQuery) {
-        combinedOptions.sourceQuery = flowDoc.sourceQuery;
-      }
-      let query = combinedOptions.sourceQuery;
-      if (query) {
-        try {
-          let urisEval;
-          if (/^\s*cts\.uris\(.*\)\s*$/.test(query)) {
-            urisEval = query;
-          } else {
-            urisEval = "cts.uris(null, null, " + query + ")";
-          }
-          resp = xdmp.eval(urisEval, {options: options}, {database: xdmp.database(database)});
-        } catch (err) {
-          //TODO log error message from 'err'
-
-          datahub.debug.log(err);
-          resp = fn.error(null, 'RESTAPI-INVALIDREQ', err);
-        }
-      } else {
-        resp = fn.error(null, "RESTAPI-SRVEXERR", Sequence.from([404, "Not Found", "The collector query was empty"]));
-        datahub.debug.log("The collector query was empty");
-      }
-    } else {
-      resp = fn.error(null, "RESTAPI-SRVEXERR", Sequence.from([404, "Not Found", "The requested step was not found"]));
-      datahub.debug.log("The requested step was not found");
-
-    }
+    urisEval = "cts.uris(null, null, " + query + ")";
   }
-  resp
-} else {
-  fn.error(null, 'RESTAPI-INVALIDREQ', 'unsupported method: '+method);
-}
+  xdmp.eval(urisEval, {options: options}, {database: xdmp.database(database)});
+} catch (err) {
+  //TODO log error message from 'err'
 
+  datahub.debug.log(err);
+  fn.error(null, 'RESTAPI-INVALIDREQ', err);
+}

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/endpoints/collector.sjs

@@ -473,6 +473,7 @@ declare private function extut:get-extension-function(
         if (exists($fcached))
         then $fcached
         else
+            let $extension-name := extut:translate-ml-prefix($extension-name)
             let $function :=
               xdmp:function(
                 if ($source-format eq "xquery")
@@ -593,7 +594,7 @@ declare private function extut:make-base-uri(
     $extension-name as xs:string
 ) as xs:string
 {
-    concat("/marklogic.rest.",$extension-type,"/",$extension-name,"/")
+    concat("/marklogic.rest.",$extension-type,"/",extut:translate-ml-prefix($extension-name),"/")
 };
 
 declare private function extut:make-metadata-uri(
@@ -1385,3 +1386,16 @@ declare function {$extension-name}:transform(
 </case>)
     return document { $module-body }
 };
+
+(: DH 5 addition for handling the ml: prefix :)
+declare private function extut:translate-ml-prefix(
+  $extension-name as xs:string
+) as xs:string
+{
+  if (starts-with($extension-name,"ml:")) then
+    let $post-fix := substring-after($extension-name, "ml:")
+    return
+      concat("ml", upper-case(substring($post-fix,1,1)), substring($post-fix,2))
+  else
+    $extension-name
+};