Fix data validation issues (#975)

* Fix `SetDropList` to allow XML special characters

* This closes #971, allow quotation marks in SetDropList()

This patch included a XML entity mapping table instead of
xml.EscapeText() to be fully compatible with Microsoft Excel.

* This closes #972, allow more than 255 bytes of validation formulas

This patch changed the string length calculation unit of data
validation formulas from UTF-8 bytes to UTF-16 code units.

* Add unit tests for SetDropList()

* Fix: allow MaxFloat64 to be used in validation range

17 decimal significant digits should be more than enough to represent
every IEEE-754 double-precision float number without losing precision,
and numbers in this form will never reach the Excel limitation of 255
UTF-16 code units.

Author: Arnie97

datavalidation.go

@@ -13,6 +13,7 @@ package excelize
 
 import (
 	"fmt"
+	"math"
 	"strings"
 	"unicode/utf16"
 )
@@ -35,10 +36,8 @@ const (
 )
 
 const (
-	// dataValidationFormulaStrLen 255 characters+ 2 quotes
-	dataValidationFormulaStrLen = 257
-	// dataValidationFormulaStrLenErr
-	dataValidationFormulaStrLenErr = "data validation must be 0-255 characters"
+	// dataValidationFormulaStrLen 255 characters
+	dataValidationFormulaStrLen = 255
 )
 
 // DataValidationErrorStyle defined the style of data validation error alert.
@@ -75,6 +74,15 @@ const (
 	DataValidationOperatorNotEqual
 )
 
+// formulaEscaper mimics the Excel escaping rules for data validation,
+// which converts `"` to `""` instead of `"`.
+var formulaEscaper = strings.NewReplacer(
+	`&`, `&`,
+	`<`, `&lt;`,
+	`>`, `&gt;`,
+	`"`, `""`,
+)
+
 // NewDataValidation return data validation struct.
 func NewDataValidation(allowBlank bool) *DataValidation {
 	return &DataValidation{
@@ -111,25 +119,22 @@ func (dd *DataValidation) SetInput(title, msg string) {
 
 // SetDropList data validation list.
 func (dd *DataValidation) SetDropList(keys []string) error {
-	formula := "\"" + strings.Join(keys, ",") + "\""
+	formula := strings.Join(keys, ",")
 	if dataValidationFormulaStrLen < len(utf16.Encode([]rune(formula))) {
-		return fmt.Errorf(dataValidationFormulaStrLenErr)
+		return ErrDataValidationFormulaLenth
 	}
-	dd.Formula1 = formula
+	dd.Formula1 = fmt.Sprintf(`<formula1>"%s"</formula1>`, formulaEscaper.Replace(formula))
 	dd.Type = convDataValidationType(typeList)
 	return nil
 }
 
 // SetRange provides function to set data validation range in drop list.
 func (dd *DataValidation) SetRange(f1, f2 float64, t DataValidationType, o DataValidationOperator) error {
-	formula1 := fmt.Sprintf("%f", f1)
-	formula2 := fmt.Sprintf("%f", f2)
-	if dataValidationFormulaStrLen < len(utf16.Encode([]rune(dd.Formula1))) || dataValidationFormulaStrLen < len(utf16.Encode([]rune(dd.Formula2))) {
-		return fmt.Errorf(dataValidationFormulaStrLenErr)
+	if math.Abs(f1) > math.MaxFloat32 || math.Abs(f2) > math.MaxFloat32 {
+		return ErrDataValidationRange
 	}
-
-	dd.Formula1 = formula1
-	dd.Formula2 = formula2
+	dd.Formula1 = fmt.Sprintf("<formula1>%.17g</formula1>", f1)
+	dd.Formula2 = fmt.Sprintf("<formula2>%.17g</formula2>", f2)
 	dd.Type = convDataValidationType(t)
 	dd.Operator = convDataValidationOperatior(o)
 	return nil

datavalidation_test.go

@@ -10,6 +10,7 @@
 package excelize
 
 import (
+	"math"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -40,7 +41,20 @@ func TestDataValidation(t *testing.T) {
 
 	dvRange = NewDataValidation(true)
 	dvRange.Sqref = "A5:B6"
-	assert.NoError(t, dvRange.SetDropList([]string{"1", "2", "3"}))
+	for _, listValid := range [][]string{
+		{"1", "2", "3"},
+		{strings.Repeat("&", 255)},
+		{strings.Repeat("\u4E00", 255)},
+		{strings.Repeat("\U0001F600", 100), strings.Repeat("\u4E01", 50), "<&>"},
+		{`A<`, `B>`, `C"`, "D\t", `E'`, `F`},
+	} {
+		dvRange.Formula1 = ""
+		assert.NoError(t, dvRange.SetDropList(listValid),
+			"SetDropList failed for valid input %v", listValid)
+		assert.NotEqual(t, "", dvRange.Formula1,
+			"Formula1 should not be empty for valid input %v", listValid)
+	}
+	assert.Equal(t, `<formula1>"A&lt;,B&gt;,C"",D	,E',F"</formula1>`, dvRange.Formula1)
 	assert.NoError(t, f.AddDataValidation("Sheet1", dvRange))
 	assert.NoError(t, f.SaveAs(resultFile))
 }
@@ -62,24 +76,44 @@ func TestDataValidationError(t *testing.T) {
 	assert.EqualError(t, err, "cross-sheet sqref cell are not supported")
 
 	assert.NoError(t, f.AddDataValidation("Sheet1", dvRange))
-	assert.NoError(t, f.SaveAs(resultFile))
 
 	dvRange = NewDataValidation(true)
 	err = dvRange.SetDropList(make([]string, 258))
 	if dvRange.Formula1 != "" {
 		t.Errorf("data validation error. Formula1 must be empty!")
 		return
 	}
-	assert.EqualError(t, err, "data validation must be 0-255 characters")
+	assert.EqualError(t, err, ErrDataValidationFormulaLenth.Error())
 	assert.NoError(t, dvRange.SetRange(10, 20, DataValidationTypeWhole, DataValidationOperatorGreaterThan))
 	dvRange.SetSqref("A9:B10")
 
 	assert.NoError(t, f.AddDataValidation("Sheet1", dvRange))
-	assert.NoError(t, f.SaveAs(resultFile))
 
 	// Test width invalid data validation formula.
-	dvRange.Formula1 = strings.Repeat("s", dataValidationFormulaStrLen+22)
-	assert.EqualError(t, dvRange.SetRange(10, 20, DataValidationTypeWhole, DataValidationOperatorGreaterThan), "data validation must be 0-255 characters")
+	prevFormula1 := dvRange.Formula1
+	for _, keys := range [][]string{
+		make([]string, 257),
+		{strings.Repeat("s", 256)},
+		{strings.Repeat("\u4E00", 256)},
+		{strings.Repeat("\U0001F600", 128)},
+		{strings.Repeat("\U0001F600", 127), "s"},
+	} {
+		err = dvRange.SetDropList(keys)
+		assert.Equal(t, prevFormula1, dvRange.Formula1,
+			"Formula1 should be unchanged for invalid input %v", keys)
+		assert.EqualError(t, err, ErrDataValidationFormulaLenth.Error())
+	}
+	assert.NoError(t, f.AddDataValidation("Sheet1", dvRange))
+	assert.NoError(t, dvRange.SetRange(
+		-math.MaxFloat32, math.MaxFloat32,
+		DataValidationTypeWhole, DataValidationOperatorGreaterThan))
+	assert.EqualError(t, dvRange.SetRange(
+		-math.MaxFloat64, math.MaxFloat32,
+		DataValidationTypeWhole, DataValidationOperatorGreaterThan), ErrDataValidationRange.Error())
+	assert.EqualError(t, dvRange.SetRange(
+		math.SmallestNonzeroFloat64, math.MaxFloat64,
+		DataValidationTypeWhole, DataValidationOperatorGreaterThan), ErrDataValidationRange.Error())
+	assert.NoError(t, f.SaveAs(resultFile))
 
 	// Test add data validation on no exists worksheet.
 	f = NewFile()

errors.go

@@ -105,4 +105,10 @@ var (
 	ErrSheetIdx = errors.New("invalid worksheet index")
 	// ErrGroupSheets defined the error message on group sheets.
 	ErrGroupSheets = errors.New("group worksheet must contain an active worksheet")
+	// ErrDataValidationFormulaLenth defined the error message for receiving a
+	// data validation formula length that exceeds the limit.
+	ErrDataValidationFormulaLenth = errors.New("data validation must be 0-255 characters")
+	// ErrDataValidationRange defined the error message on set decimal range
+	// exceeds limit.
+	ErrDataValidationRange = errors.New("data validation range exceeds limit")
 )

xmlWorksheet.go

@@ -436,8 +436,8 @@ type DataValidation struct {
 	ShowInputMessage bool    `xml:"showInputMessage,attr,omitempty"`
 	Sqref            string  `xml:"sqref,attr"`
 	Type             string  `xml:"type,attr,omitempty"`
-	Formula1         string  `xml:"formula1,omitempty"`
-	Formula2         string  `xml:"formula2,omitempty"`
+	Formula1         string  `xml:",innerxml"`
+	Formula2         string  `xml:",innerxml"`
 }
 
 // xlsxC collection represents a cell in the worksheet. Information about the