DHFPROD-7234: No caching or double counting calculated weights

Author: ryanjdew

marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.smart-mastering/matcher-impl/matcher-impl.xqy

@@ -542,7 +542,7 @@ declare function match-impl:search(
         )
       else ()
     where $contains
-    return $query-map => map:with("weight", $weight)
+    return map:new(($query-map, map:entry("weight", $weight)))
   let $score :=
       fn:sum(
           $matching-query-maps ! map:get(., "weight")
@@ -660,12 +660,20 @@ declare function match-impl:instance-query-wrapper(
 };
 
 declare function match-impl:score-from-cts-query($result as node(), $query as cts:query) as xs:double {
-  fn:sum(
+  (: We don't want to double count for the same query/value pair hit :)
+  let $queries-and-values-hit := map:map()
+  return
+    fn:sum(
       cts:walk(
           $result,
           $query,
-          document{$cts:queries}
-          //schema-element(cts:query)[fn:node-name(.) = $QUERIES_WITH_WEIGHT] ! fn:number(fn:head((./@weight, 1)))
+          let $key := xdmp:hash64(xdmp:describe($cts:queries, (), ())) ||  ":" || $cts:text
+          where fn:not(map:contains($queries-and-values-hit, $key))
+          return (
+            map:put($queries-and-values-hit, $key, fn:true()),
+            document{$cts:queries}
+              //schema-element(cts:query)[fn:node-name(.) = $QUERIES_WITH_WEIGHT] ! fn:number(fn:head((./@weight, 1)))
+          )
       )
-  )
+    )
 };

marklogic-data-hub/src/test/ml-modules/root/test/suites/data-hub/5/smart-mastering/matching/custom-controlled-weight.xqy

@@ -16,13 +16,18 @@ declare option xdmp:mapping "false";
   test:assert-equal(2.5, match-impl:score-from-cts-query(document { element test{"specific score"} }, cts:word-query("specific score", (), 2.5)), "The match score should be 0")
 ),
 (: test that score is used in matching :)
-let $doc := fn:doc($lib:URI-DOB1)
+let $jsonDoc3 := fn:doc($lib:URI6)
 let $options := test:get-test-file($lib:MATCH-OPTIONS-CUST-ALG-CUST-WEIGHT || ".json")
-let $actual := matcher:find-document-matches-by-options($doc, $options, 1, 10, 0, fn:true(), cts:true-query())
+let $actual := matcher:find-document-matches-by-options($jsonDoc3, $options, 1, 10, 0, fn:true(), cts:true-query())
+let $jsonDoc1Match := $actual/result[@uri = $lib:URI4]
+let $jsonDoc2Match := $actual/result[@uri = $lib:URI5]
 return
   (
     test:assert-exists($actual, "There should be a result returned"),
-    test:assert-equal("1", fn:string($actual/@total), "The match count should be 1"),
-    test:assert-equal("/source/5/dob2.json", fn:string($actual/result/@uri), "The matching document should be dob2.json")
+    test:assert-equal("2", fn:string($actual/@total), "The match count should be 2"),
+    test:assert-exists($jsonDoc1Match, "There should be a match for " || $lib:URI4),
+    test:assert-equal("5", fn:string($jsonDoc1Match/@score), "The match score for "|| $lib:URI4 ||" should be 5"),
+    test:assert-exists($jsonDoc2Match,  "There should be a match for " || $lib:URI5),
+    test:assert-equal("10", fn:string($jsonDoc2Match/@score), "The match score for "|| $lib:URI5 ||" should be 10")
   )
 

marklogic-data-hub/src/test/ml-modules/root/test/suites/data-hub/5/smart-mastering/matching/test-data/custom-matching-with-custom-weight-options.json

@@ -12,18 +12,18 @@
     "propertyDefs": {
       "property": [
         {
-          "localname": "DateOfBirth",
-          "name": "DateOfBirth"
+          "localname": "id",
+          "name": "id"
         }
       ]
     },
     "algorithms": {
       "algorithm": [
         {
-          "name": "dob-match",
-          "function": "dob-match",
+          "name": "custom-id",
+          "function": "custom-id",
           "namespace": "http://marklogic.com/smart-mastering/algorithms",
-          "at": "/custom-xqy-matching-algo-dob.xqy"
+          "at": "/test/suites/data-hub/5/smart-mastering/matching/test-data/custom-matching-with-custom-weight.xqy"
         }
       ]
     },
@@ -33,9 +33,8 @@
     "scoring": {
       "expand": [
         {
-          "propertyName": "DateOfBirth",
-          "algorithmRef": "dob-match",
-          "customWeight": "10"
+          "propertyName": "id",
+          "algorithmRef": "custom-id"
         }
       ]
     },

marklogic-data-hub/src/test/ml-modules/root/test/suites/data-hub/5/smart-mastering/matching/test-data/custom-matching-with-custom-weight.xqy

@@ -0,0 +1,28 @@
+xquery version "1.0-ml";
+
+module namespace algorithms = "http://marklogic.com/smart-mastering/algorithms";
+
+import module namespace helper-impl = "http://marklogic.com/smart-mastering/helper-impl"
+at "/com.marklogic.smart-mastering/matcher-impl/helper-impl.xqy";
+
+declare namespace matcher = "http://marklogic.com/smart-mastering/matcher";
+
+declare option xdmp:mapping "false";
+
+
+
+(:
+Example of custom weights
+:)
+declare function algorithms:custom-id(
+    $expand-values as xs:string*,
+    $expand-xml as element(matcher:expand),
+    $options-xml as element(matcher:options)
+) as cts:query*
+{
+  cts:or-query((
+    cts:word-query("6986792174", (), 5),
+    (: different scores for the different documents to ensure weight isn't cached :)
+    cts:word-query($expand-values, (), 10)
+  ))
+};

marklogic-data-hub/src/test/ml-modules/root/test/suites/data-hub/5/smart-mastering/matching/test-data/doc1.json

@@ -38,7 +38,7 @@
               }
             },
             "IncidentCategoryCodeDate": null,
-            "id": "6986792174",
+            "id": ["6986792174", "repeating ID to ensure score is only counted once","6986792174"],
             "PersonBirthDate": "19801001",
             "PersonSex": "F",
             "CaseAmount": 1287.9,