fix(uid-search): ZMSA-67: improve UID SEARCH performance by reducing calls to imap-compiler (#955)

* ZMSA-67: improve UID SEARCH performance by reducing calls to imap-compiler

* mongodb, when fetching contiguous uid then use range query

Author: NickOvt

imap-core/lib/commands/search.js

@@ -33,7 +33,7 @@ module.exports = {
         let parsed;
 
         try {
-            parsed = parseQueryTerms(terms, this.selected.uidList);
+            parsed = parseQueryTerms(terms, this.selected.uidList, isUid);
         } catch (E) {
             return callback(E);
         }
@@ -90,24 +90,34 @@ module.exports = {
                 if (Array.isArray(matches) && matches.length) {
                     matches.sort((a, b) => a - b);
 
-                    matches.forEach(nr => {
-                        let seq;
+                    if (isUid) {
+                        response.attributes.push({
+                            type: 'TEXT',
+                            value: matches.join(' ')
+                        });
+                    } else {
+                        let uidList = this.selected.uidList || [];
+                        let uidIndex = new Map();
+                        let seqList = [];
+
+                        for (let i = 0; i < uidList.length; i++) {
+                            uidIndex.set(uidList[i], i + 1);
+                        }
 
-                        if (!isUid) {
-                            seq = this.selected.uidList.indexOf(nr) + 1;
+                        for (let i = 0; i < matches.length; i++) {
+                            let seq = uidIndex.get(matches[i]);
                             if (seq) {
-                                response.attributes.push({
-                                    type: 'atom',
-                                    value: String(seq)
-                                });
+                                seqList.push(seq);
                             }
-                        } else {
+                        }
+
+                        if (seqList.length) {
                             response.attributes.push({
-                                type: 'atom',
-                                value: String(nr)
+                                type: 'TEXT',
+                                value: seqList.join(' ')
                             });
                         }
-                    });
+                    }
                 }
 
                 // append (MODSEQ 123) for queries that include MODSEQ criteria
@@ -136,8 +146,14 @@ module.exports = {
     parseQueryTerms // expose for testing
 };
 
-function parseQueryTerms(terms, uidList) {
+function isFixedRange(value) {
+    value = (value || '').toString();
+    return value.indexOf(':') >= 0 && value.indexOf(',') < 0;
+}
+
+function parseQueryTerms(terms, uidList, isUidSearch) {
     terms = [].concat(terms || []);
+    isUidSearch = !!isUidSearch;
 
     let pos = 0;
     let term;
@@ -168,8 +184,12 @@ function parseQueryTerms(terms, uidList) {
         if (!termType) {
             // try if it is a sequence set
             if (imapTools.validateSequence(term)) {
+                let messageRange = imapTools.getMessageRange(uidList, term, isUidSearch);
+                if (isUidSearch && isFixedRange(term)) {
+                    messageRange.isContiguous = true;
+                }
                 // resolve sequence list to an array of UID values
-                curTerm = ['uid', imapTools.getMessageRange(uidList, term, false)];
+                curTerm = ['uid', messageRange];
             } else {
                 // no idea what the term is for
                 throw new Error('Unknown search term ' + term.toUpperCase());
@@ -183,7 +203,14 @@ function parseQueryTerms(terms, uidList) {
                         throw new Error('Invalid sequence set for ' + term.toUpperCase());
                     }
                     // resolve sequence list to an array of UID values
-                    curTerm.push(imapTools.getMessageRange(uidList, terms[pos++], true));
+                    {
+                        let sequence = terms[pos++];
+                        let messageRange = imapTools.getMessageRange(uidList, sequence, true);
+                        if (isFixedRange(sequence)) {
+                            messageRange.isContiguous = true;
+                        }
+                        curTerm.push(messageRange);
+                    }
                 } else {
                     curTerm.push(terms[pos++]);
                 }

lib/handlers/on-search.js

@@ -105,7 +105,7 @@ module.exports = server => (mailbox, options, session, callback) => {
                                 if (term.value.length !== session.selected.uidList.length) {
                                     // not 1:*
                                     parent.push({
-                                        uid: tools.checkRangeQuery(term.value, ne)
+                                        uid: tools.checkRangeQuery(term.value, ne, term.value?.isContiguous)
                                     });
                                 } else if (ne) {
                                     parent.push({

lib/tools.js

@@ -20,22 +20,23 @@ const https = require('https');
 
 let templates = false;
 
-function checkRangeQuery(uids, ne) {
-    // check if uids is a straight continuous array and if such then return a range query,
-    // otherwise return a $in query
+function checkRangeQuery(uids, ne, isContiguous = false) {
+    // If isContiguous is true, assume the original criteria was a fixed range and skip gap checks.
 
     if (uids.length === 1) {
         return {
             [!ne ? '$eq' : '$ne']: uids[0]
         };
     }
 
-    for (let i = 1, len = uids.length; i < len; i++) {
-        if (uids[i] !== uids[i - 1] + 1) {
-            // TODO: group into AND conditions, otherwise expands too much!
-            return {
-                [!ne ? '$in' : '$nin']: uids
-            };
+    if (!isContiguous) {
+        for (let i = 1, len = uids.length; i < len; i++) {
+            if (uids[i] !== uids[i - 1] + 1) {
+                // TODO: group into AND conditions, otherwise expands too much!
+                return {
+                    [!ne ? '$in' : '$nin']: uids
+                };
+            }
         }
     }