added error handling to UTF8 streams

Author: halgari

benchmarks/deftype_fields.pxi

@@ -11,6 +11,9 @@
     b))
 
 
-(def adder (->Adder 1 0))
-(dotimes [x (* 1024 1024 1024)]
+
+(def adder (->Adder 1.0 0))
+(println "Starting....")
+(dotimes [x (* 1024 1024 1024 20)]
   (assert (= (inc x) (add-them adder))))
+(println "Ending....")

pixie/math.pxi

@@ -23,4 +23,8 @@
   (i/defcfn ceil)
   (i/defcfn fabs)
   (i/defcfn floor)
-  (i/defcfn fmod))
+  (i/defcfn fmod)
+
+  (i/defconst M_PI))
+
+(def pi M_PI)

pixie/streams/utf8.pxi

@@ -2,10 +2,10 @@
   (require pixie.streams :refer :all))
 
 (defprotocol IUTF8OutputStream
-  (write-char [this char]))
+  (write-char [this char] "Write a single character to the UTF8 stream"))
 
 (defprotocol IUTF8InputStream
-  (read-char [this]))
+  (read-char [this] "Read a single character from the UTF8 stream"))
 
 (deftype UTF8OutputStream [out]
   IUTF8OutputStream
@@ -21,35 +21,49 @@
        (<= ch 0x1FFFFF) (do (write-byte out (bit-or 0xE0 (bit-shift-right ch 18)))
                             (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 12) 0x3F)))
                             (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 6) 0x3F)))
-                            (write-byte out (bit-or 0x80 (bit-and ch 0x3F))) ))))
+                            (write-byte out (bit-or 0x80 (bit-and ch 0x3F))))
+       :else (assert false (str "Cannot encode a UTF8 character of code " ch)))))
   IDisposable
   (-dispose! [this]
     (dispose! out)))
 
 
-(deftype UTF8InputStream [in]
+(deftype UTF8InputStream [in bad-char]
   IUTF8InputStream
   (read-char [this]
     (let [ch (int (read-byte in))
-          [n bytes] (cond
-                      (>= 0x7F ch) [ch 1]
-                      (= 0xC0 (bit-and ch 0xE0)) [(bit-and ch 31) 2]
-                      (= 0xE0 (bit-and ch 0xF0)) [(bit-and ch 15) 3]
-                      (= 0xF0 (bit-and ch 0xF8)) [(bit-and ch 7) 4]
-                      :else (assert false (str "Got bad code " ch)))]
+          [n bytes error?] (cond
+                            (>= 0x7F ch) [ch 1]
+                            (= 0xC0 (bit-and ch 0xE0)) [(bit-and ch 31) 2 false]
+                            (= 0xE0 (bit-and ch 0xF0)) [(bit-and ch 15) 3 false]
+                            (= 0xF0 (bit-and ch 0xF8)) [(bit-and ch 7) 4 false]
+                            (= 0xF8 (bit-and ch 0xF8)) [(bit-and ch 3) 5 true]
+                            (= 0xFC (bit-and ch 0xFE)) [(bit-and ch 1) 6 true]
+                            :else [n 1 true])]
       (loop [i (dec bytes)
              n n]
         (if (pos? i)
           (recur (dec i)
                  (bit-or (bit-shift-left n 6)
                          (bit-and (read-byte in) 0x3F)))
-          (char n)))))
+          (if error?
+            (if bad-char
+              bad-char
+              (throw (str "Invalid UTF8 character decoded: " n)))
+            (char n))))))
   IDisposable
   (-dispose! [this]
     (dispose! in)))
 
-(defn utf8-input-stream [i]
-  (->UTF8InputStream i))
+(defn utf8-input-stream
+  "Creates a UTF8 decoder that reads characters from the given IByteInputStream. If a bad character is found
+   an error will be thrown, unless an optional bad-character marker character is provided."
+  ([i]
+   (->UTF8InputStream i nil))
+  ([i bad-char]
+   (->UTF8InputStream i bad-char)))
 
-(defn utf8-output-stream [o]
+(defn utf8-output-stream
+  "Creates a UTF8 encoder that writes characters to the given IByteOutputStream."
+  [o]
   (->UTF8OutputStream o))