Merge pull request #32 from gsnewmark/fix-unicode-decoding

Fix issue #30 Unicode decoding in conversion to CharSequence

Author: ztellman

src/byte_streams/char_sequence.clj

@@ -33,10 +33,22 @@
   (parse-result (.decode decoder in out false)))
 
 (defn flush
-  [^CharsetDecoder decoder ^CharBuffer out]
-  (and
-    (parse-result (.decode decoder (ByteBuffer/allocate 0) out true))
-    (parse-result (.flush decoder out))))
+  ([decoder out] (flush decoder (ByteBuffer/allocate 0) out))
+  ([^CharsetDecoder decoder ^ByteBuffer in ^CharBuffer out]
+   (and
+    (parse-result (.decode decoder in out true))
+    (parse-result (.flush decoder out)))))
+
+(defn has-remaining-bytes? [^ByteBuffer byte-buffer]
+  {:pre [(some? byte-buffer)]}
+  (.hasRemaining byte-buffer))
+
+(defn merge-byte-buffers [^ByteBuffer l ^ByteBuffer r]
+  {:pre [(some? l) (some? r)]}
+  (-> (ByteBuffer/allocate (+ (.remaining l) (.remaining r)))
+      (.put l)
+      (.put r)
+      .flip))
 
 (defn lazy-char-buffer-sequence
   [^CharsetDecoder decoder
@@ -64,17 +76,26 @@
           (lazy-char-buffer-sequence decoder chunk-size extra-bytes close-fn byte-source))
 
         (if-let [in (byte-source chunk-size)]
-          (let [result (decode decoder in out)]
+          (let [expanded-in (if (some-> extra-bytes has-remaining-bytes?)
+                              ;; in case of underflow we need to pass new buffer
+                              ;; containing remaining bytes from the initial input
+                              ;; along with some new bytes to the CharsetDecoder
+                              (merge-byte-buffers extra-bytes in)
+                              in)
+                result (decode decoder expanded-in out)]
             (cons
               (.flip out)
               (lazy-char-buffer-sequence
                 decoder
                 chunk-size
-                (when (= :overflow result) in)
+                (when (has-remaining-bytes? expanded-in)
+                  expanded-in)
                 close-fn
                 byte-source)))
           (do
-            (flush decoder out)
+            (if (some? extra-bytes)
+              (flush decoder extra-bytes out)
+              (flush decoder out))
             (when close-fn (close-fn))
             (.flip out)))))))