Alternate queries (#142) (#143)

* Bump the production-dependencies group with 3 updates (#120)
Bumps the production-dependencies group with 3 updates: [@natlibfi/marc-record](https://github.com/natlibfi/marc-record-js), [@natlibfi/melinda-commons](https://github.com/natlibfi/melinda-commons-js) and [natural](https://github.com/NaturalNode/natural).

* Bump the development-dependencies group with 7 updates (#119)

| Package | From | To |
| --- | --- | --- |
| [@babel/cli](https://github.com/babel/babel/tree/HEAD/packages/babel-cli) | `7.23.0` | `7.23.4` |
| [@babel/core](https://github.com/babel/babel/tree/HEAD/packages/babel-core) | `7.23.3` | `7.23.6` |
| [@babel/preset-env](https://github.com/babel/babel/tree/HEAD/packages/babel-preset-env) | `7.23.3` | `7.23.6` |
| [@natlibfi/fixugen](https://github.com/natlibfi/fixugen-js) | `2.0.2` | `2.0.3` |
| [@natlibfi/fixugen-http-client](https://github.com/natlibfi/fixugen-http-client-js) | `3.0.1` | `3.0.2` |
| [eslint](https://github.com/eslint/eslint) | `8.53.0` | `8.55.0` |
| [nodemon](https://github.com/remy/nodemon) | `3.0.1` | `3.0.2` |

* Simpler scripts (#122)

* Add feature alternateQueries
alternates -type candidateSearch creates several search queries, gets search result amounts for all of those queries and actually uses only the queries that get less than maxCandidates search results

* Add new alternates-type candidateSearchType for bib: titleAuthorYearAlternates

* Fix using word search for queries starting with a boolean [MRA-762]
Use word search for queries only when the first word is a boolean, not when the first word starts with a boolean
Add new type of candidateSearch queries: 'alternates'

* Update copyright year
* Update deps

* 4.3.2-alpha.7

Author: ammsalme

README.md

@@ -2,6 +2,6 @@
 
 ## License and copyright
 
-Copyright (c) 2020-2023 **University Of Helsinki (The National Library Of Finland)**
+Copyright (c) 2020-2024 **University Of Helsinki (The National Library Of Finland)**
 
 This project's source code is licensed under the terms of **GNU Lesser General Public License Version 3** or any later version.

package-lock.json

@@ -14,7 +14,7 @@
 		"url": "git@github.com:natlibfi/melinda-record-matching-js.git"
 	},
 	"license": "LGPL-3.0+",
-	"version": "4.3.1",
+	"version": "4.3.2-alpha.7",
 	"main": "./dist/index.js",
 	"engines": {
 		"node": ">=18"
@@ -43,7 +43,7 @@
 		"@natlibfi/melinda-commons": "^13.0.12",
 		"@natlibfi/sru-client": "^6.0.8",
 		"debug": "^4.3.4",
-		"isbn3": "^1.1.44",
+		"isbn3": "^1.1.45",
 		"moment": "^2.30.1",
 		"natural": "^6.10.4",
 		"uuid": "^9.0.1",

package.json

@@ -0,0 +1,123 @@
+/**
+*
+* @licstart  The following is the entire license notice for the JavaScript code in this file.
+*
+* Melinda record matching modules for Javascript
+*
+* Copyright (C) 2023 University Of Helsinki (The National Library Of Finland)
+*
+* This file is part of melinda-record-matching-js
+*
+* melinda-record-matching-js program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU Lesser General Public License as
+* published by the Free Software Foundation, either version 3 of the
+* License, or (at your option) any later version.
+*
+* melinda-record-matching-js is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+* @licend  The above is the entire license notice
+* for the JavaScript code in this file.
+*
+*/
+
+import createDebugLogger from 'debug';
+import createClient, {SruSearchError} from '@natlibfi/sru-client';
+
+export class CandidateSearchError extends Error {}
+
+export default async function ({url, queryList, queryListType, maxCandidates = 50}) {
+
+  const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search:choose-queries');
+  const debugData = debug.extend('data');
+  const debugDev = debug.extend('dev');
+
+  debugData(`Url: ${url}`);
+  debugData(`QueryList: ${queryList}`);
+  debugData(`queryListType: ${queryListType}`);
+
+  const client = createClient({
+    url,
+    maxRecordsPerRequest: 0,
+    version: '2.0',
+    retrieveAll: false
+  });
+
+  debugDev(`QueryList (type: ${queryListType}) ${JSON.stringify(queryList)}`);
+  try {
+    const {queriesWithTotals} = await getQueryTotals({queryList, queryOffset: 0, queriesWithTotals: []});
+    debugDev(`QueryResult: ${JSON.stringify(queriesWithTotals)}`);
+    const filteredQueryResult = filterQueryResult({queriesWithTotals, maxCandidates});
+    debugDev(`filteredQueryResult: ${JSON.stringify(filteredQueryResult)}`);
+    return filteredQueryResult;
+  } catch (err) {
+    throw new CandidateSearchError(err);
+  }
+
+  async function getQueryTotals({queryList, queryOffset = 0, queriesWithTotals = []}) {
+
+    const query = queryList[queryOffset];
+    debug(`Running query ${JSON.stringify(query)} (${queryOffset}) for total`);
+
+    if (query) {
+      const {total} = await retrieveTotal();
+
+      const newQueriesWithTotals = [...queriesWithTotals, {query, total}];
+      debug(`Query ${queryOffset} ${query} done.`);
+      debug(`There are (${queryList.length - (queryOffset + 1)} queries left)`);
+      return getQueryTotals({queryList, queryOffset: queryOffset + 1, queriesWithTotals: newQueriesWithTotals});
+    }
+
+    debug(`All ${queryList.length} queries done, there's no query for ${queryOffset}`);
+    return {queriesWithTotals};
+
+    function retrieveTotal() {
+      return new Promise((resolve, reject) => {
+        // eslint-disable-next-line functional/no-let
+        let totalRecords = 0;
+
+        debug(`Searching total amount of candidates for query: ${query}`);
+
+        client.searchRetrieve(query)
+          .on('error', err => {
+            // eslint-disable-next-line functional/no-conditional-statements
+            if (err instanceof SruSearchError) {
+              debug(`SRU SruSearchError for query: ${query}: ${err}`);
+              reject(new CandidateSearchError(`SRU SruSearchError for query: ${query}: ${err}`));
+            }
+            debug(`SRU error for query: ${query}: ${err}`);
+            reject(new CandidateSearchError(`SRU error for query: ${query}: ${err}`));
+          })
+          .on('total', total => {
+            debug(`Got total: ${total}`);
+            totalRecords += total;
+          })
+          .on('end', () => {
+            try {
+              resolve({total: totalRecords});
+            } catch (err) {
+              debug(`Error caught on END`);
+              reject(err);
+            }
+          })
+          .on('record', () => {
+            debugDev(`RECORD: We should no get records here`);
+          });
+      });
+    }
+  }
+  function filterQueryResult({queriesWithTotals, maxCandidates}) {
+    debug(`Filtering queries (${queriesWithTotals.length}), maxCandidates: ${maxCandidates}`);
+    debugData(`${JSON.stringify(queriesWithTotals)}`);
+    // Drop queries where total result is 0 or greater than given maxCandidates
+    const filteredQueryResult = queriesWithTotals.filter((queryWithTotal) => queryWithTotal.total !== 0 && queryWithTotal.total < maxCandidates);
+    debugData(`${JSON.stringify(filteredQueryResult)}`);
+    return filteredQueryResult;
+  }
+
+}

src/candidate-search/choose-queries.js

@@ -4,7 +4,7 @@
 *
 * Melinda record matching modules for Javascript
 *
-* Copyright (C) 2020-2022 University Of Helsinki (The National Library Of Finland)
+* Copyright (C) 2020-2023 University Of Helsinki (The National Library Of Finland)
 *
 * This file is part of melinda-record-matching-js
 *
@@ -32,14 +32,15 @@ import {MarcRecord} from '@natlibfi/marc-record';
 import {MARCXML} from '@natlibfi/marc-record-serializers';
 import generateQueryList from './query-list';
 import {Error as MatchingError} from '@natlibfi/melinda-commons';
+import chooseQueries from './choose-queries';
 
 export {searchTypes} from './query-list';
 
 export class CandidateSearchError extends Error {}
 
 // serverMaxResults : maximum size of total search result available from the server, defaults to Aleph's 20000
 
-export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {
+export default async ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest = 50, serverMaxResult = 20000}) => {
   MarcRecord.setValidationOptions({subfieldValues: false});
 
   const debug = createDebugLogger('@natlibfi/melinda-record-matching:candidate-search');
@@ -55,7 +56,20 @@ export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest =
   const adjustedMaxRecordsPerRequest = maxRecordsPerRequest >= maxCandidates ? maxCandidates : maxRecordsPerRequest;
 
   const inputRecordId = getRecordId(record);
-  const queryList = generateQueryList(record, searchSpec);
+  const queryListResult = generateQueryList(record, searchSpec);
+  const queryList = queryListResult[0]?.queryList ? queryListResult[0].queryList : queryListResult;
+  const queryListType = queryListResult[0]?.queryListType ? queryListResult[0].queryListType : undefined;
+
+  // if generateQueryList errored we should throw 422
+  if (queryList.length === 0) {
+    debug(`Empty list`);
+    throw new CandidateSearchError(`Generated query list contains no queries`);
+  }
+  if (queryListType && queryListType !== 'alternates') {
+    debug(`Unknown queryListType`);
+    throw new CandidateSearchError(`Generated query list has invalid type`);
+  }
+
   const client = createClient({
     url,
     maxRecordsPerRequest: adjustedMaxRecordsPerRequest,
@@ -64,14 +78,19 @@ export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest =
   });
 
   debug(`Searching matches for ${inputRecordId}`);
-  debug(`Generated queryList ${JSON.stringify(queryList)}`);
+  const chosenQueryList = await filterQueryList({queryList, queryListType});
+  debug(`Chosen queries: ${JSON.stringify(chosenQueryList)}`);
 
-  // if generateQueryList errored we should throw 422
+  async function filterQueryList({queryList, queryListType, maxCandidates}) {
+    debug(`Generated queryList (type: ${queryListType}) ${JSON.stringify(queryList)}`);
 
-  if (queryList.length === 0) {
-    throw new CandidateSearchError(`Generated query list contains no queries`);
+    if (queryListType === 'alternates' && queryList.length > 1) {
+      const queryListResult = await chooseQueries({url, queryList, queryListType, maxCandidates});
+      debug(`queryListResult: ${JSON.stringify(queryListResult)}`);
+      return queryListResult.map(elem => elem.query);
+    }
+    return queryList;
   }
-
   // state.totalRecords : amount of candidate records available to the current query (undefined, if there was no queries left)
   // state.query : current query (undefined if there was no queries left)
   // state.searchCounter : sequence for current search for current query (undefined, if there we no queries left)
@@ -80,9 +99,13 @@ export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest =
   // state.queryCounter : sequence for current query
   // state.maxedQueries : queries that resulted in more than serverMaxResults hits
 
+  return {search};
 
-  return async ({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) => {
-    const query = queryList[queryOffset];
+  // eslint-disable-next-line max-statements
+  async function search({queryOffset = 0, resultSetOffset = 1, totalRecords = 0, searchCounter = 0, queryCandidateCounter = 0, queryCounter = 0, maxedQueries = []}) {
+
+    const query = chosenQueryList[queryOffset];
+    debug(`Running query ${JSON.stringify(query)} (${queryOffset})`);
 
     if (query) {
       const {records, failures, nextOffset, total} = await retrieveRecords();
@@ -171,7 +194,7 @@ export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest =
           });
       });
     }
-  };
+  }
 
   function checkMaxedQuery(query, total, serverMaxResult) {
     if (total >= serverMaxResult) {
@@ -180,7 +203,6 @@ export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest =
     }
   }
 
-
   function getRecordId(record) {
     const [field] = record.get(/^001$/u);
     return field ? field.value : '';
@@ -191,5 +213,4 @@ export default ({record, searchSpec, url, maxCandidates, maxRecordsPerRequest =
     debug(`Cannot yet find possible database record id from recordXML (length ${recordXML.length})`);
     return undefined;
   }
-
 };

src/candidate-search/index.js

@@ -46,6 +46,7 @@ describe('candidate-search', () => {
     }
   });
 
+  // eslint-disable-next-line max-statements
   async function callback({getFixture, factoryOptions, searchOptions, expectedFactoryError = false, expectedSearchError = false, enabled = true}) {
     const url = 'http://foo.bar';
 
@@ -54,16 +55,29 @@ describe('candidate-search', () => {
     }
 
     if (expectedFactoryError) {
+      debug(`We're expecting an error`);
       if (expectedFactoryError.isCandidateSearchError) {
-        expect(() => createSearchInterface({...formatFactoryOptions(), url})).to.throw(CandidateSearchError, new RegExp(expectedFactoryError, 'u'));
+        try {
+          const result = createSearchInterface({...formatFactoryOptions(), url});
+          debug(result);
+        } catch (err) {
+          expect(err).to.equal(new CandidateSearchError(expectedFactoryError));
+        }
         return;
       }
 
-      expect(() => createSearchInterface({...formatFactoryOptions(), url})).to.throw(new RegExp(expectedFactoryError, 'u'));
+      try {
+        const result = createSearchInterface({...formatFactoryOptions(), url});
+        debug(result);
+      } catch (err) {
+        expect(err).to.equal(new Error(expectedFactoryError));
+      }
       return;
     }
 
-    const search = createSearchInterface({...formatFactoryOptions(), url});
+    const {search} = await createSearchInterface({...formatFactoryOptions(), url});
+    // eslint-disable-next-line no-console
+    console.log(search);
     await iterate({searchOptions, expectedSearchError});
 
     function formatFactoryOptions() {