Add Utf8 recognizer

Author: saintfish

detector.go

@@ -11,11 +11,14 @@ type Result struct {
 }
 
 type Detector struct {
-	recognizers []recognizer
+}
+
+// List of charset recognizers
+var recognizers = []recognizer {
+    new(recognizerUtf8),
 }
 
 func NewDetector() *Detector {
-    // Init recognizer
     return &Detector{}
 }
 
@@ -31,7 +34,19 @@ func (d *Detector) DetectBest(b []byte, stripTag bool, declaredCharset string) (
 	return
 }
 
+func matchHelper(r recognizer, input *recognizerInput, outputChan chan<- recognizerOutput) {
+    outputChan <- r.Match(input)
+}
+
 func (d *Detector) DetectAll(b []byte, stripTag bool, declaredCharset string) ([]Result, error) {
-	_ = newRecognizerInput(b, stripTag, declaredCharset)
+    input := newRecognizerInput(b, stripTag, declaredCharset)
+    outputChan := make(chan recognizerOutput)
+    for _, r := range recognizers {
+        go matchHelper(r, input, outputChan)
+    }
+    outputs := make([]recognizerOutput, 0, len(recognizers))
+    for i := 0; i < len(recognizers); i++ {
+        outputs = append(outputs, <-outputChan)
+    }
 	return nil, NotDetectedError
 }

recognizer.go

@@ -1,7 +1,7 @@
 package chardet
 
 type recognizer interface {
-    Match(recognizerInput) recognizerOutput
+    Match(*recognizerInput) recognizerOutput
 }
 
 type recognizerOutput struct {

utf8.go

@@ -0,0 +1,67 @@
+package chardet
+
+import (
+    "bytes"
+)
+
+var utf8Bom = []byte { 0xEF, 0xBB, 0xBF }
+
+type recognizerUtf8 struct {
+}
+
+func (*recognizerUtf8) Match(input *recognizerInput) (output recognizerOutput) {
+    output = recognizerOutput {
+        Charset: "UTF-8",
+    }
+    hasBom := bytes.HasPrefix(input.raw, utf8Bom)
+    inputLen := len(input.raw)
+    var numValid, numInvalid uint32
+    var trailBytes uint8
+    for i := 0; i < inputLen; i++ {
+        c := input.raw[i]
+        if c & 0x80 == 0 {
+            continue
+        }
+        if c & 0xE0 == 0xC0 {
+            trailBytes = 1
+        } else if c & 0xF0 == 0xE0 {
+            trailBytes = 2
+        } else if c & 0xF8 == 0xF0 {
+            trailBytes = 3
+        } else {
+            numInvalid++
+            if numInvalid > 5 {
+                break
+            }
+            trailBytes = 0
+        }
+
+        for i++; i < inputLen; i++ {
+            c = input.raw[i]
+            if c & 0xC0 != 0x80 {
+                numInvalid++
+                break
+            }
+            if trailBytes--; trailBytes == 0 {
+                numValid++
+                break
+            }
+        }
+    }
+
+    if hasBom && numInvalid == 0 {
+        output.Confidence = 100
+    } else if hasBom && numValid > numInvalid * 10 {
+        output.Confidence = 80
+    } else if numValid > 3 && numInvalid == 0 {
+        output.Confidence = 100
+    } else if numValid > 0 && numInvalid == 0 {
+        output.Confidence = 80
+    } else if numValid == 0 && numInvalid == 0 {
+        // Plain ASCII
+        output.Confidence = 10
+    } else if numValid > numInvalid * 10 {
+        output.Confidence = 25
+    }
+    return
+}