Define interface of chardet.Detector. Also implement the input processing logic.

Author: saintfish

detector.go

@@ -0,0 +1,37 @@
+package chardet
+
+import (
+	"errors"
+)
+
+type Result struct {
+	Charset    string
+	Language   string
+	Confidence int
+}
+
+type Detector struct {
+	recognizers []recognizer
+}
+
+func NewDetector() *Detector {
+    // Init recognizer
+    return &Detector{}
+}
+
+var (
+    NotDetectedError = errors.New("Charset not detected.")
+)
+
+func (d *Detector) DetectBest(b []byte, stripTag bool, declaredCharset string) (r *Result, err error) {
+	var all []Result
+	if all, err = d.DetectAll(b, stripTag, declaredCharset); err != nil {
+		r = &all[0]
+	}
+	return
+}
+
+func (d *Detector) DetectAll(b []byte, stripTag bool, declaredCharset string) ([]Result, error) {
+	_ = newRecognizerInput(b, stripTag, declaredCharset)
+	return nil, NotDetectedError
+}

input.go

@@ -0,0 +1,79 @@
+package chardet
+
+type recognizerInput struct {
+	raw             []byte
+	input           []byte
+	tagStripped     bool
+	declaredCharset string
+	byteStats       []int
+	hasC1Bytes      bool
+}
+
+func newRecognizerInput(raw []byte, stripTag bool, declaredCharset string) *recognizerInput {
+	input, stripped := mayStripInput(raw, stripTag)
+	byteStats := computeByteStats(input)
+	return &recognizerInput{
+		raw:             raw,
+		input:           input,
+		tagStripped:     stripped,
+		declaredCharset: declaredCharset,
+		byteStats:       byteStats,
+		hasC1Bytes:      computeHasC1Bytes(byteStats),
+	}
+}
+
+func mayStripInput(raw []byte, stripTag bool) (out []byte, stripped bool) {
+	const inputBufferSize = 8192
+	out = make([]byte, 0, inputBufferSize)
+	var badTags, openTags int32
+	var inMarkup bool = false
+	stripped = false
+	if stripTag {
+		stripped = true
+		for _, c := range raw {
+			if c == '<' {
+				if inMarkup {
+					badTags += 1
+				}
+				inMarkup = true
+				openTags += 1
+			}
+			if !inMarkup {
+				out = append(out, c)
+				if len(out) >= inputBufferSize {
+					break
+				}
+			}
+			if c == '>' {
+				inMarkup = false
+			}
+		}
+	}
+	if openTags < 5 || openTags/5 < badTags || (len(out) < 100 && len(raw) > 600) {
+		limit := len(raw)
+		if limit > inputBufferSize {
+			limit = inputBufferSize
+		}
+		out = make([]byte, limit)
+		copy(out, raw[:limit])
+		stripped = false
+	}
+	return
+}
+
+func computeByteStats(input []byte) []int {
+	r := make([]int, 256)
+	for _, c := range input {
+		r[c] += 1
+	}
+	return r
+}
+
+func computeHasC1Bytes(byteStats []int) bool {
+	for _, count := range byteStats[0x80 : 0x9F+1] {
+		if count > 0 {
+			return true
+		}
+	}
+	return false
+}

recognizer.go

@@ -0,0 +1,4 @@
+package chardet
+
+type recognizer interface {
+}