Fix for at-least-once guarantee not ensured in case a seek happens on one partition and there are messages being fetched for other partitions

Author: emasab

lib/kafkajs/_consumer.js

@@ -1402,21 +1402,21 @@ class Consumer {
     return ppc;
   }
 
-  #discardMessages(ms, ppc) {
-    if (ms) {
-      let m = ms[0];
-      if (m.constructor === Array) {
-        m = m[0];
-      }
-      ppc = ms[1];
-      if (m && !this.#lastConsumedOffsets.has(ppc.key)) {
+  #returnMessages(ms) {
+    let m = ms[0];
+    // ppc could have been change we must return it as well.
+    let ppc = ms[1];
+    const messagesToReturn = m.constructor === Array ? m : [m];
+    const firstMessage = messagesToReturn[0];
+    if (firstMessage && !this.#lastConsumedOffsets.has(ppc.key)) {
         this.#lastConsumedOffsets.set(ppc.key, {
-          topic: m.topic,
-          partition: m.partition,
-          offset: m.offset - 1,
+          topic: firstMessage.topic,
+          partition: firstMessage.partition,
+          offset: firstMessage.offset - 1,
         });
-      }
     }
+ 
+    this.#messageCache.returnMessages(messagesToReturn);
     return ppc;
   }
 
@@ -1463,11 +1463,15 @@ class Consumer {
           continue;
 
         if (this.#pendingOperations.length) {
-          ppc  = this.#discardMessages(ms, ppc);
-          break;
+          /* 
+           * Don't process messages anymore, execute the operations first.
+           * Return the messages to the cache that will be cleared if needed.
+           * `ppc` could have been changed, we must return it as well.
+           */
+          ppc  = this.#returnMessages(ms);
+        } else {
+          ppc = await perMessageProcessor(ms, config);
         }
-
-        ppc = await perMessageProcessor(ms, config);
       } catch (e) {
         /* Since this error cannot be exposed to the user in the current situation, just log and retry.
           * This is due to restartOnFailure being set to always true. */
@@ -1548,7 +1552,9 @@ class Consumer {
    * @private
    */
   async #executePendingOperations() {
-    for (const op of this.#pendingOperations) {
+    // Execute all pending operations, they could add more operations.
+    while (this.#pendingOperations.length > 0) {
+      const op = this.#pendingOperations.shift();
       await op();
     }
     this.#pendingOperations = [];

lib/kafkajs/_consumer_cache.js

@@ -28,12 +28,19 @@ class PerPartitionMessageCache {
     }
 
     /**
-     * Adds a message to the cache.
+     * Adds a message to the cache as last one.
      */
-    _add(message) {
+    _addLast(message) {
         this.#cache.addLast(message);
     }
 
+    /**
+     * Adds a message to the cache as first one.
+     */
+    _addFirst(message) {
+        this.#cache.addFirst(message);
+    }
+
     get key() {
         return this.#key;
     }
@@ -126,7 +133,7 @@ class MessageCache {
      *
      * @param {Object} message - the message to add to the cache.
      */
-    #add(message) {
+    #add(message, head = false) {
         const key = partitionKey(message);
         let cache = this.#tpToPpc.get(key);
         if (!cache) {
@@ -135,7 +142,11 @@ class MessageCache {
             cache._node = this.#availablePartitions.addLast(cache);
             this.notifyAvailablePartitions();
         }
-        cache._add(message);
+        if (head) {
+            cache._addFirst(message);
+        } else {
+            cache._addLast(message);
+        }
     }
 
     get availableSize() {
@@ -183,7 +194,21 @@ class MessageCache {
      */
     addMessages(messages) {
         for (const message of messages)
-            this.#add(message);
+            this.#add(message, false);
+        this.#size += messages.length;
+    }
+
+    /**
+     * Return messages to the cache, to be read again.
+     *
+     * @param {Array} messages - the messages to return to the cache.
+     */
+    returnMessages(messages) {
+        let i = messages.length - 1;
+        while (i >= 0) {
+            this.#add(messages[i], true);
+            i--;
+        }
         this.#size += messages.length;
     }