refactor XML decoding to support charset handling

Author: aafeher

go.mod

@@ -1,3 +1,7 @@
 module github.com/aafeher/go-sitemap-parser
 
 go 1.21
+
+require golang.org/x/net v0.34.0
+
+require golang.org/x/text v0.21.0 // indirect

go.sum

@@ -0,0 +1,4 @@
+golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=

sitemap.go

@@ -6,6 +6,7 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"golang.org/x/net/html/charset"
 	"io"
 	"math/rand"
 	"net/http"
@@ -546,7 +547,10 @@ func (s *S) parseSitemapIndex(data string) (sitemapIndex, error) {
 		return smIndex, fmt.Errorf("sitemapindex is empty")
 	}
 
-	err := xml.Unmarshal([]byte(data), &smIndex)
+	decoder := xml.NewDecoder(bytes.NewReader([]byte(data)))
+	decoder.CharsetReader = charset.NewReaderLabel
+
+	err := decoder.Decode(&smIndex)
 	return smIndex, err
 
 }
@@ -562,11 +566,10 @@ func (s *S) parseURLSet(data string) (URLSet, error) {
 		return urlSet, fmt.Errorf("sitemap is empty")
 	}
 
-	err := xml.Unmarshal([]byte(data), &urlSet)
-	if err != nil {
-		return urlSet, err
-	}
+	decoder := xml.NewDecoder(bytes.NewReader([]byte(data)))
+	decoder.CharsetReader = charset.NewReaderLabel
 
+	err := decoder.Decode(&urlSet)
 	return urlSet, err
 }