From 6bb25b53e3c94c2c5646da86a19ee8027d92dcd7 Mon Sep 17 00:00:00 2001 From: Steven Tran Date: Thu, 12 Nov 2020 17:18:03 -0800 Subject: [PATCH 1/7] Add setting for unmarshalling JSON with UseNumber --- orderedmap.go | 33 ++++++++++++++++++++++++++++++--- orderedmap_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/orderedmap.go b/orderedmap.go index cf8db2a..98b48d3 100644 --- a/orderedmap.go +++ b/orderedmap.go @@ -46,6 +46,7 @@ type OrderedMap struct { keys []string values map[string]interface{} escapeHTML bool + useNumber bool } func New() *OrderedMap { @@ -60,6 +61,10 @@ func (o *OrderedMap) SetEscapeHTML(on bool) { o.escapeHTML = on } +func (o *OrderedMap) SetUseNumber(on bool) { + o.useNumber = on +} + func (o *OrderedMap) Get(key string) (interface{}, bool) { val, exists := o.values[key] return val, exists @@ -125,13 +130,34 @@ func (o *OrderedMap) UnmarshalJSON(b []byte) error { return nil } +func unmarshalJSONUseNumber(s string, v interface{}) error { + r := strings.NewReader(s) + decoder := json.NewDecoder(r) + decoder.UseNumber() + err := decoder.Decode(v) + if err != nil { + return err + } + + return nil +} + func mapStringToOrderedMap(s string, o *OrderedMap) error { // parse string into map m := map[string]interface{}{} - err := json.Unmarshal([]byte(s), &m) - if err != nil { - return err + + if o.useNumber { + err := unmarshalJSONUseNumber(s, &m) + if err != nil { + return err + } + } else { + err := json.Unmarshal([]byte(s), &m) + if err != nil { + return err + } } + // Get the order of the keys orderedKeys := []KeyIndex{} for k, _ := range m { @@ -194,6 +220,7 @@ func mapStringToOrderedMap(s string, o *OrderedMap) error { // this may be recursive it values in the map are also maps if hasValidJson { newMap := New() + newMap.SetUseNumber(o.useNumber) // Preserve number setting err := mapStringToOrderedMap(valueStr, newMap) if err != nil { return err diff --git a/orderedmap_test.go b/orderedmap_test.go index 6a06899..a091327 100644 --- a/orderedmap_test.go +++ b/orderedmap_test.go @@ -400,6 +400,50 @@ func TestUnmarshalJSONStruct(t *testing.T) { } } +func TestRemarshalJSONWithoutUseNumber(t *testing.T) { + // 9007199254740993 is the smallest integer which cannot be represented exactly as a float64 + const input = `{"data":{"x":9007199254740993}}` + const expected = `{"data":{"x":9007199254740992}}` + + o := New() + o.SetUseNumber(false) + + err := json.Unmarshal([]byte(input), &o) + if err != nil { + t.Fatalf("JSON unmarshal error: %v", err) + } + + marshalled, err := json.Marshal(o) + if err != nil { + t.Errorf("Marshal failed: %v", err) + } + + if string(marshalled) != expected { + t.Errorf("unexpected value: %s instead of %s", marshalled, expected) + } +} + +func TestRemarshalJSONUseNumber(t *testing.T) { + const input = `{"data":{"x":9007199254740993}}` + + o := New() + o.SetUseNumber(true) + + err := json.Unmarshal([]byte(input), &o) + if err != nil { + t.Fatalf("JSON unmarshal error: %v", err) + } + + marshalled, err := json.Marshal(o) + if err != nil { + t.Errorf("Marshal failed: %v", err) + } + + if string(marshalled) != input { + t.Errorf("unexpected value: %s instead of %s", marshalled, input) + } +} + func TestOrderedMap_SortKeys(t *testing.T) { s := ` { From 13e64a866d1e5cb3f071a095fb3152f28c69ca88 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 16 Dec 2021 15:05:55 -0800 Subject: [PATCH 2/7] Added Go module so `go test` would work. --- go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d08a464 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/Pexeso/orderedmap + +go 1.17 From d8e346d795d5e0b2546b564fece3e099dc20ed36 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 16 Dec 2021 15:06:10 -0800 Subject: [PATCH 3/7] Added breaking test for slice problem. --- orderedmap_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/orderedmap_test.go b/orderedmap_test.go index a091327..15c1569 100644 --- a/orderedmap_test.go +++ b/orderedmap_test.go @@ -1,8 +1,10 @@ package orderedmap import ( + "bytes" "encoding/json" "fmt" + "math" "sort" "strings" "testing" @@ -530,3 +532,23 @@ func TestOrderedMap_empty_map(t *testing.T) { t.Error("Got", marshalledStr) } } + +func TestOrderedMap_uint64_slice_serialization(t *testing.T) { + uints := []uint64{math.MaxUint64, math.MaxUint64-100, math.MaxUint64-1000} + + str1 := []byte(fmt.Sprintf(`{"key":[%d,%d,%d]}`, uints[0], uints[1], uints[2])) + om1 := New() + + if err := json.Unmarshal(str1, om1); err != nil { + t.Fatal(err) + } + + str2, err := json.Marshal(om1) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(str1, str2) { + t.Fatalf("%s != %s", str1, str2) + } +} \ No newline at end of file From b32cdfb989dfb546e10e4724cc42bb56ddfb8b1c Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 16 Dec 2021 15:07:25 -0800 Subject: [PATCH 4/7] Added interface checks & whitespace. --- orderedmap.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/orderedmap.go b/orderedmap.go index 98b48d3..30674aa 100644 --- a/orderedmap.go +++ b/orderedmap.go @@ -3,19 +3,19 @@ package orderedmap import ( "bytes" "encoding/json" - "errors" "sort" "strings" ) -var NoValueError = errors.New("No value for this key") - type KeyIndex struct { Key string Index int } + type ByIndex []KeyIndex +var _ sort.Interface = &ByIndex{} + func (a ByIndex) Len() int { return len(a) } func (a ByIndex) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByIndex) Less(i, j int) bool { return a[i].Index < a[j].Index } @@ -38,6 +38,8 @@ type ByPair struct { LessFunc func(a *Pair, j *Pair) bool } +var _ sort.Interface = &ByPair{} + func (a ByPair) Len() int { return len(a.Pairs) } func (a ByPair) Swap(i, j int) { a.Pairs[i], a.Pairs[j] = a.Pairs[j], a.Pairs[i] } func (a ByPair) Less(i, j int) bool { return a.LessFunc(a.Pairs[i], a.Pairs[j]) } @@ -99,12 +101,12 @@ func (o *OrderedMap) Keys() []string { return o.keys } -// SortKeys Sort the map keys using your sort func +// SortKeys sorts the map keys using your sort func func (o *OrderedMap) SortKeys(sortFunc func(keys []string)) { sortFunc(o.keys) } -// Sort Sort the map using your sort func +// Sort sorts the map using your sort func func (o *OrderedMap) Sort(lessFunc func(a *Pair, b *Pair) bool) { pairs := make([]*Pair, len(o.keys)) for i, key := range o.keys { @@ -118,28 +120,21 @@ func (o *OrderedMap) Sort(lessFunc func(a *Pair, b *Pair) bool) { } } +var _ json.Unmarshaler = &OrderedMap{} + +// UnmarshalJSON implements the json.Unmarshaler interface for OrderedMap. func (o *OrderedMap) UnmarshalJSON(b []byte) error { if o.values == nil { o.values = map[string]interface{}{} } - var err error - err = mapStringToOrderedMap(string(b), o) - if err != nil { - return err - } - return nil + return mapStringToOrderedMap(string(b), o) } func unmarshalJSONUseNumber(s string, v interface{}) error { r := strings.NewReader(s) decoder := json.NewDecoder(r) decoder.UseNumber() - err := decoder.Decode(v) - if err != nil { - return err - } - - return nil + return decoder.Decode(v) } func mapStringToOrderedMap(s string, o *OrderedMap) error { @@ -341,6 +336,9 @@ func sliceStringToSliceWithOrderedMaps(valueStr string, newSlice *[]interface{}) return nil } +var _ json.Marshaler = &OrderedMap{} + +// MarshalJSON implements the json.Marshaler interface for OrderedMap. func (o OrderedMap) MarshalJSON() ([]byte, error) { s := "{" for _, k := range o.keys { From 00f886ea4c249aff10285c690e6ba4e73ccb2c8c Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 16 Dec 2021 15:14:27 -0800 Subject: [PATCH 5/7] Fixed bug. --- orderedmap.go | 15 +++++++++++---- orderedmap_test.go | 7 ++++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/orderedmap.go b/orderedmap.go index 30674aa..5d96364 100644 --- a/orderedmap.go +++ b/orderedmap.go @@ -248,7 +248,7 @@ func mapStringToOrderedMap(s string, o *OrderedMap) error { // this may be recursive if values in the slice are slices if hasValidJson { newSlice := []interface{}{} - err := sliceStringToSliceWithOrderedMaps(valueStr, &newSlice) + err := sliceStringToSliceWithOrderedMaps(valueStr, &newSlice, o.useNumber) if err != nil { return err } @@ -274,7 +274,7 @@ func mapStringToOrderedMap(s string, o *OrderedMap) error { return nil } -func sliceStringToSliceWithOrderedMaps(valueStr string, newSlice *[]interface{}) error { +func sliceStringToSliceWithOrderedMaps(valueStr string, newSlice *[]interface{}, useNumber bool) error { // if the value for this key is a []interface, convert any map items to an orderedmap. // find end of valueStr by removing everything after last ] // until it forms valid json @@ -292,14 +292,21 @@ func sliceStringToSliceWithOrderedMaps(valueStr string, newSlice *[]interface{}) endItem = endItem + 1 continue } + // if this substring compiles to json, it's the next item possibleItemStr := strings.TrimSpace(itemsStr[startItem:endItem]) var possibleItem interface{} - err := json.Unmarshal([]byte(possibleItemStr), &possibleItem) + var err error + if useNumber { + err = unmarshalJSONUseNumber(possibleItemStr, &possibleItem) + } else { + err = json.Unmarshal([]byte(possibleItemStr), &possibleItem) + } if err != nil { endItem = endItem + 1 continue } + if possibleItemStr[0] == '{' { // if item is map, convert to orderedmap oo := New() @@ -314,7 +321,7 @@ func sliceStringToSliceWithOrderedMaps(valueStr string, newSlice *[]interface{}) } else if possibleItemStr[0] == '[' { // if item is slice, convert to slice with orderedmaps newItem := []interface{}{} - err := sliceStringToSliceWithOrderedMaps(possibleItemStr, &newItem) + err := sliceStringToSliceWithOrderedMaps(possibleItemStr, &newItem, useNumber) if err != nil { return err } diff --git a/orderedmap_test.go b/orderedmap_test.go index 15c1569..6eaf67c 100644 --- a/orderedmap_test.go +++ b/orderedmap_test.go @@ -537,13 +537,14 @@ func TestOrderedMap_uint64_slice_serialization(t *testing.T) { uints := []uint64{math.MaxUint64, math.MaxUint64-100, math.MaxUint64-1000} str1 := []byte(fmt.Sprintf(`{"key":[%d,%d,%d]}`, uints[0], uints[1], uints[2])) - om1 := New() + om := New() + om.SetUseNumber(true) - if err := json.Unmarshal(str1, om1); err != nil { + if err := json.Unmarshal(str1, om); err != nil { t.Fatal(err) } - str2, err := json.Marshal(om1) + str2, err := json.Marshal(om) if err != nil { t.Fatal(err) } From 6141a72d3ef9cfaadf3b5afe666fcbbeb7c68f5f Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 16 Dec 2021 15:44:47 -0800 Subject: [PATCH 6/7] Added missing newline. --- orderedmap_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orderedmap_test.go b/orderedmap_test.go index 6eaf67c..c72169a 100644 --- a/orderedmap_test.go +++ b/orderedmap_test.go @@ -552,4 +552,4 @@ func TestOrderedMap_uint64_slice_serialization(t *testing.T) { if !bytes.Equal(str1, str2) { t.Fatalf("%s != %s", str1, str2) } -} \ No newline at end of file +} From bdabbd04f13dba5ae678f0f36bf01c59a35d3de6 Mon Sep 17 00:00:00 2001 From: Jon Anderson Date: Thu, 16 Dec 2021 17:12:11 -0800 Subject: [PATCH 7/7] Removed set number flag. --- orderedmap.go | 36 +++++++++--------------------------- orderedmap_test.go | 39 +++++++++++---------------------------- 2 files changed, 20 insertions(+), 55 deletions(-) diff --git a/orderedmap.go b/orderedmap.go index 5d96364..3ce83a2 100644 --- a/orderedmap.go +++ b/orderedmap.go @@ -48,7 +48,6 @@ type OrderedMap struct { keys []string values map[string]interface{} escapeHTML bool - useNumber bool } func New() *OrderedMap { @@ -63,10 +62,6 @@ func (o *OrderedMap) SetEscapeHTML(on bool) { o.escapeHTML = on } -func (o *OrderedMap) SetUseNumber(on bool) { - o.useNumber = on -} - func (o *OrderedMap) Get(key string) (interface{}, bool) { val, exists := o.values[key] return val, exists @@ -130,9 +125,10 @@ func (o *OrderedMap) UnmarshalJSON(b []byte) error { return mapStringToOrderedMap(string(b), o) } -func unmarshalJSONUseNumber(s string, v interface{}) error { +func unmarshalJSON(s string, v interface{}) error { r := strings.NewReader(s) decoder := json.NewDecoder(r) + // This option ensures that decoder.UseNumber() return decoder.Decode(v) } @@ -140,17 +136,9 @@ func unmarshalJSONUseNumber(s string, v interface{}) error { func mapStringToOrderedMap(s string, o *OrderedMap) error { // parse string into map m := map[string]interface{}{} - - if o.useNumber { - err := unmarshalJSONUseNumber(s, &m) - if err != nil { - return err - } - } else { - err := json.Unmarshal([]byte(s), &m) - if err != nil { - return err - } + err := unmarshalJSON(s, &m) + if err != nil { + return err } // Get the order of the keys @@ -215,7 +203,6 @@ func mapStringToOrderedMap(s string, o *OrderedMap) error { // this may be recursive it values in the map are also maps if hasValidJson { newMap := New() - newMap.SetUseNumber(o.useNumber) // Preserve number setting err := mapStringToOrderedMap(valueStr, newMap) if err != nil { return err @@ -248,7 +235,7 @@ func mapStringToOrderedMap(s string, o *OrderedMap) error { // this may be recursive if values in the slice are slices if hasValidJson { newSlice := []interface{}{} - err := sliceStringToSliceWithOrderedMaps(valueStr, &newSlice, o.useNumber) + err := sliceStringToSliceWithOrderedMaps(valueStr, &newSlice) if err != nil { return err } @@ -274,7 +261,7 @@ func mapStringToOrderedMap(s string, o *OrderedMap) error { return nil } -func sliceStringToSliceWithOrderedMaps(valueStr string, newSlice *[]interface{}, useNumber bool) error { +func sliceStringToSliceWithOrderedMaps(valueStr string, newSlice *[]interface{}) error { // if the value for this key is a []interface, convert any map items to an orderedmap. // find end of valueStr by removing everything after last ] // until it forms valid json @@ -296,12 +283,7 @@ func sliceStringToSliceWithOrderedMaps(valueStr string, newSlice *[]interface{}, // if this substring compiles to json, it's the next item possibleItemStr := strings.TrimSpace(itemsStr[startItem:endItem]) var possibleItem interface{} - var err error - if useNumber { - err = unmarshalJSONUseNumber(possibleItemStr, &possibleItem) - } else { - err = json.Unmarshal([]byte(possibleItemStr), &possibleItem) - } + err := unmarshalJSON(possibleItemStr, &possibleItem) if err != nil { endItem = endItem + 1 continue @@ -321,7 +303,7 @@ func sliceStringToSliceWithOrderedMaps(valueStr string, newSlice *[]interface{}, } else if possibleItemStr[0] == '[' { // if item is slice, convert to slice with orderedmaps newItem := []interface{}{} - err := sliceStringToSliceWithOrderedMaps(possibleItemStr, &newItem, useNumber) + err := sliceStringToSliceWithOrderedMaps(possibleItemStr, &newItem) if err != nil { return err } diff --git a/orderedmap_test.go b/orderedmap_test.go index c72169a..07b469d 100644 --- a/orderedmap_test.go +++ b/orderedmap_test.go @@ -397,39 +397,15 @@ func TestUnmarshalJSONStruct(t *testing.T) { x, ok := v.Data.Get("x") if !ok { t.Errorf("missing expected key") - } else if x != float64(1) { + } else if x != json.Number("1") { t.Errorf("unexpected value: %#v", x) } } -func TestRemarshalJSONWithoutUseNumber(t *testing.T) { - // 9007199254740993 is the smallest integer which cannot be represented exactly as a float64 +func TestRemarshalJSON(t *testing.T) { const input = `{"data":{"x":9007199254740993}}` - const expected = `{"data":{"x":9007199254740992}}` o := New() - o.SetUseNumber(false) - - err := json.Unmarshal([]byte(input), &o) - if err != nil { - t.Fatalf("JSON unmarshal error: %v", err) - } - - marshalled, err := json.Marshal(o) - if err != nil { - t.Errorf("Marshal failed: %v", err) - } - - if string(marshalled) != expected { - t.Errorf("unexpected value: %s instead of %s", marshalled, expected) - } -} - -func TestRemarshalJSONUseNumber(t *testing.T) { - const input = `{"data":{"x":9007199254740993}}` - - o := New() - o.SetUseNumber(true) err := json.Unmarshal([]byte(input), &o) if err != nil { @@ -484,7 +460,15 @@ func TestOrderedMap_Sort(t *testing.T) { o := New() json.Unmarshal([]byte(s), &o) o.Sort(func(a *Pair, b *Pair) bool { - return a.value.(float64) > b.value.(float64) + na, err := a.value.(json.Number).Float64() + if err != nil { + panic(err) + } + nb, err := b.value.(json.Number).Float64() + if err != nil { + panic(err) + } + return na > nb }) // Check the root keys @@ -538,7 +522,6 @@ func TestOrderedMap_uint64_slice_serialization(t *testing.T) { str1 := []byte(fmt.Sprintf(`{"key":[%d,%d,%d]}`, uints[0], uints[1], uints[2])) om := New() - om.SetUseNumber(true) if err := json.Unmarshal(str1, om); err != nil { t.Fatal(err)