From 529ddaa02dee2d1de5ed31b58c55c428ede9fbb7 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Sun, 14 Sep 2025 14:52:19 +0300 Subject: [PATCH 01/19] feat: added encoding and decoding functions for bool, uint8, int8, float32 --- cmd/main.go | 23 +++++++++ pkg/msgpack.go | 138 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 cmd/main.go create mode 100644 pkg/msgpack.go diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 0000000..bf7aa29 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + msgpack "messagepack/pkg" +) + +func main() { + object := msgpack.Object{ + IsObject: true, + Flag: false, + Unum: 0, + Snum: -3, + Fnum: 3.14, + } + bytes, err := msgpack.Pack(object) + if err != nil { + fmt.Println(err) + } + + fmt.Println(bytes) +} + diff --git a/pkg/msgpack.go b/pkg/msgpack.go new file mode 100644 index 0000000..5af030a --- /dev/null +++ b/pkg/msgpack.go @@ -0,0 +1,138 @@ +package msgpack + +import ( + "encoding/binary" + "fmt" + "math" + "reflect" +) + +const Msgpack_Nil = 0xc0 +const Msgpack_False = 0xc2 +const Msgpack_True = 0xc3 +const Msgpack_Uint_8 = 0xcc +const Msgpack_Int_8 = 0xd0 +const msgpack_Float_32 = 0xca + +type Uint_8 struct { + typeByte byte + value byte +} + +type Int_8 struct { + typeByte byte + value byte +} + +type Float_32 struct { + typeByte byte + value []byte +} + +type Object struct { + IsObject bool + Flag bool + Unum uint8 + Snum int8 + Fnum float32 +} + +func encodeBool(value bool) byte { + if value { + return byte(Msgpack_True) + } else { + return byte(Msgpack_False) + } +} +func encodeUint(value uint8) Uint_8 { + return Uint_8{ + typeByte: Msgpack_Uint_8, + value: value, + } +} + +func encodeFloat(value float32) Float_32{ + bin := math.Float32bits(value) + bytes := make([]byte, 4) + binary.BigEndian.PutUint32(bytes,bin) + return Float_32{ + typeByte: msgpack_Float_32, + value: bytes, + } +} + +func encodeInt(value int8) Int_8 { + return Int_8{ + typeByte: Msgpack_Uint_8, + value: byte(value), + } +} + +func decodeBool(value byte) bool { + if value == byte(Msgpack_True) { + return true + } else { + return false + } +} + +func decodeUint(value byte) uint8 { + return uint8(value) +} + +func decodeInt(value byte) int8 { + return int8(value) +} + +func decodeFloat(value []byte) float32 { + bin := binary.BigEndian.Uint32(value) + return math.Float32frombits(bin) +} + +func Pack(obj interface{}) ([]byte, error) { + t := reflect.TypeOf(obj) + v := reflect.ValueOf(obj) + bytes := make([]byte, 0) + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + memberType := field.Type.Kind() + memberName := field.Name + memberValue := v.Field(i) + fmt.Println(memberType, memberName, memberValue) + objectBytes := make([]byte, 0) + + switch memberType { + case reflect.Bool: + objectBytes = append(objectBytes, encodeBool(memberValue.Bool())) + fmt.Println(decodeBool(objectBytes[0])) + case reflect.Uint8: + encodedUint := encodeUint(uint8(memberValue.Uint())) + objectBytes = append(objectBytes, encodedUint.typeByte, encodedUint.value) + fmt.Println(decodeUint(encodedUint.value)) + case reflect.Int8: + encodedInt := encodeInt(int8(memberValue.Int())) + objectBytes = append(objectBytes, encodedInt.typeByte, encodedInt.value) + fmt.Println(decodeInt(encodedInt.value)) + case reflect.Float32: + encodedFloat := encodeFloat(float32(memberValue.Float())) + objectBytes = append(objectBytes, encodedFloat.typeByte) + objectBytes = append(objectBytes, encodedFloat.value...) + fmt.Println(decodeFloat(encodedFloat.value)) + default: + } + + objectBytes = append(objectBytes, memberName...) + fmt.Println(objectBytes) + bytes = append(bytes, objectBytes...) // to store the name + bytes = append(bytes, byte('\n')) // to store the separator == 10 + + } + + return bytes, nil +} + +func Unpack(data []byte) (interface{}, error) { + return nil, nil +} + +// encoder, decoder From e845095e77fc40b37a5f4032c333d1391a0a59c9 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Sun, 14 Sep 2025 16:28:35 +0300 Subject: [PATCH 02/19] feat: added encoding and decoding functions for string8, string16, string32 --- cmd/main.go | 3 +- go.mod | 3 ++ pkg/msgpack.go | 74 +++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 go.mod diff --git a/cmd/main.go b/cmd/main.go index bf7aa29..ab1bd1f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,7 +2,7 @@ package main import ( "fmt" - msgpack "messagepack/pkg" + msgpack "github.com/codescalersinternships/Msgpack-Fatma-Ebrahim/pkg" ) func main() { @@ -12,6 +12,7 @@ func main() { Unum: 0, Snum: -3, Fnum: 3.14, + Str: "Fatma", } bytes, err := msgpack.Pack(object) if err != nil { diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..691f6b6 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/codescalersinternships/Msgpack-Fatma-Ebrahim + +go 1.25.1 diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 5af030a..cba36dd 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -12,7 +12,11 @@ const Msgpack_False = 0xc2 const Msgpack_True = 0xc3 const Msgpack_Uint_8 = 0xcc const Msgpack_Int_8 = 0xd0 -const msgpack_Float_32 = 0xca +const Msgpack_Float_32 = 0xca + +const Msgpack_String_8 = 0xd9 +const Msgpack_String_16 = 0xda +const Msgpack_String_32 = 0xdb type Uint_8 struct { typeByte byte @@ -29,12 +33,19 @@ type Float_32 struct { value []byte } +type String struct { + typeByte byte + value []byte + size []byte +} + type Object struct { IsObject bool Flag bool Unum uint8 Snum int8 Fnum float32 + Str string } func encodeBool(value bool) byte { @@ -51,20 +62,54 @@ func encodeUint(value uint8) Uint_8 { } } -func encodeFloat(value float32) Float_32{ +func encodeInt(value int8) Int_8 { + return Int_8{ + typeByte: Msgpack_Uint_8, + value: byte(value), + } +} + +func encodeFloat(value float32) Float_32 { bin := math.Float32bits(value) bytes := make([]byte, 4) - binary.BigEndian.PutUint32(bytes,bin) + binary.BigEndian.PutUint32(bytes, bin) return Float_32{ - typeByte: msgpack_Float_32, + typeByte: Msgpack_Float_32, value: bytes, } } -func encodeInt(value int8) Int_8 { - return Int_8{ - typeByte: Msgpack_Uint_8, - value: byte(value), +func encodeString(value string) String { + string_8 := int(math.Pow(2, 8) - 1) + string_16 := int(math.Pow(2, 16) - 1) + string_32 := int(math.Pow(2, 32) - 1) + var size []byte + numBytes := 8 + var t byte + switch { + case len(value) < string_8: + numBytes = 8 + size = make([]byte, 0) + size = append(size, byte(len(value))) + t = Msgpack_String_8 + case len(value) < string_16: + numBytes = 16 + size = make([]byte, 2) + binary.BigEndian.PutUint16(size, uint16(len(value))) + t = Msgpack_String_16 + case len(value) < string_32: + numBytes = 32 + size = make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(value))) + t = Msgpack_String_32 + } + + bytes := make([]byte, int(math.Pow(2, float64(numBytes))-1)) + copy(bytes, value) + return String{ + typeByte: t, + value: bytes, + size: size, } } @@ -89,6 +134,10 @@ func decodeFloat(value []byte) float32 { return math.Float32frombits(bin) } +func decodeString(value []byte) string { + return string(value) +} + func Pack(obj interface{}) ([]byte, error) { t := reflect.TypeOf(obj) v := reflect.ValueOf(obj) @@ -118,6 +167,13 @@ func Pack(obj interface{}) ([]byte, error) { objectBytes = append(objectBytes, encodedFloat.typeByte) objectBytes = append(objectBytes, encodedFloat.value...) fmt.Println(decodeFloat(encodedFloat.value)) + case reflect.String: + encodedString := encodeString(memberValue.String()) + objectBytes = append(objectBytes, encodedString.typeByte) + objectBytes = append(objectBytes, encodedString.size...) + objectBytes = append(objectBytes, encodedString.value...) + fmt.Println(decodeString(encodedString.value)) + default: } @@ -134,5 +190,3 @@ func Pack(obj interface{}) ([]byte, error) { func Unpack(data []byte) (interface{}, error) { return nil, nil } - -// encoder, decoder From 79862d0c90db1462e8854d82bdc3b8abd095acad Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Sun, 14 Sep 2025 17:32:26 +0300 Subject: [PATCH 03/19] feat: added nil and 16,32,64 int and uint --- cmd/main.go | 2 +- pkg/msgpack.go | 165 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 154 insertions(+), 13 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index ab1bd1f..44e143f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -10,7 +10,7 @@ func main() { IsObject: true, Flag: false, Unum: 0, - Snum: -3, + Snum: -1000, Fnum: 3.14, Str: "Fatma", } diff --git a/pkg/msgpack.go b/pkg/msgpack.go index cba36dd..5263de6 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -10,9 +10,19 @@ import ( const Msgpack_Nil = 0xc0 const Msgpack_False = 0xc2 const Msgpack_True = 0xc3 + const Msgpack_Uint_8 = 0xcc +const Msgpack_Uint_16 = 0xcd +const Msgpack_Uint_32 = 0xce +const Msgpack_Uint_64 = 0xcf + const Msgpack_Int_8 = 0xd0 +const Msgpack_Int_16 = 0xd1 +const Msgpack_Int_32 = 0xd2 +const Msgpack_Int_64 = 0xd3 + const Msgpack_Float_32 = 0xca +const Msgpack_Float_64 = 0xcb const Msgpack_String_8 = 0xd9 const Msgpack_String_16 = 0xda @@ -23,12 +33,22 @@ type Uint_8 struct { value byte } +type Uint struct { + typeByte byte + value []byte +} + type Int_8 struct { typeByte byte value byte } -type Float_32 struct { +type Int struct { + typeByte byte + value []byte +} + +type Float struct { typeByte byte value []byte } @@ -43,7 +63,7 @@ type Object struct { IsObject bool Flag bool Unum uint8 - Snum int8 + Snum int16 Fnum float32 Str string } @@ -62,6 +82,33 @@ func encodeUint(value uint8) Uint_8 { } } +func encodeUint16(value uint16) Uint { + bytes := make([]byte, 2) + binary.BigEndian.PutUint16(bytes, value) + return Uint{ + typeByte: Msgpack_Uint_16, + value: bytes, + } +} + +func encodeUint32(value uint32) Uint { + bytes := make([]byte, 4) + binary.BigEndian.PutUint32(bytes, value) + return Uint{ + typeByte: Msgpack_Uint_32, + value: bytes, + } +} + +func encodeUint64(value uint64) Uint { + bytes := make([]byte, 4) + binary.BigEndian.PutUint64(bytes, value) + return Uint{ + typeByte: Msgpack_Uint_64, + value: bytes, + } +} + func encodeInt(value int8) Int_8 { return Int_8{ typeByte: Msgpack_Uint_8, @@ -69,42 +116,74 @@ func encodeInt(value int8) Int_8 { } } -func encodeFloat(value float32) Float_32 { +func encodeInt16(value int16) Int { + bytes := make([]byte, 2) + binary.BigEndian.PutUint16(bytes, uint16(value)) + return Int{ + typeByte: Msgpack_Uint_16, + value: bytes, + } +} + +func encodeInt32(value int32) Int { + bytes := make([]byte, 4) + binary.BigEndian.PutUint32(bytes, uint32(value)) + return Int{ + typeByte: Msgpack_Uint_32, + value: bytes, + } +} + +func encodeInt64(value int64) Int { + bytes := make([]byte, 8) + binary.BigEndian.PutUint64(bytes, uint64(value)) + return Int{ + typeByte: Msgpack_Uint_64, + value: bytes, + } +} + +func encodeFloat32(value float32) Float { bin := math.Float32bits(value) bytes := make([]byte, 4) binary.BigEndian.PutUint32(bytes, bin) - return Float_32{ + return Float{ typeByte: Msgpack_Float_32, value: bytes, } } +func encodeFloat64(value float64) Float { + bin := math.Float64bits(value) + bytes := make([]byte, 8) + binary.BigEndian.PutUint64(bytes, bin) + return Float{ + typeByte: Msgpack_Float_64, + value: bytes, + } +} func encodeString(value string) String { string_8 := int(math.Pow(2, 8) - 1) string_16 := int(math.Pow(2, 16) - 1) string_32 := int(math.Pow(2, 32) - 1) var size []byte - numBytes := 8 var t byte switch { case len(value) < string_8: - numBytes = 8 size = make([]byte, 0) size = append(size, byte(len(value))) t = Msgpack_String_8 case len(value) < string_16: - numBytes = 16 size = make([]byte, 2) binary.BigEndian.PutUint16(size, uint16(len(value))) t = Msgpack_String_16 case len(value) < string_32: - numBytes = 32 size = make([]byte, 4) binary.BigEndian.PutUint32(size, uint32(len(value))) t = Msgpack_String_32 } - bytes := make([]byte, int(math.Pow(2, float64(numBytes))-1)) + bytes := []byte(value) copy(bytes, value) return String{ typeByte: t, @@ -121,14 +200,36 @@ func decodeBool(value byte) bool { } } -func decodeUint(value byte) uint8 { +func decodeUint8(value byte) uint8 { return uint8(value) } +func decodeUint16(value []byte) uint16 { + return binary.BigEndian.Uint16(value) +} + +func decodeUint32(value []byte) uint32 { + return binary.BigEndian.Uint32(value) +} +func decodeUint64(value []byte) uint64 { + return binary.BigEndian.Uint64(value) +} + func decodeInt(value byte) int8 { return int8(value) } +func decodeInt16(value []byte) int16 { + return int16(binary.BigEndian.Uint16(value)) +} +func decodeInt32(value []byte) int32 { + return int32(binary.BigEndian.Uint32(value)) +} + +func decodeInt64(value []byte) int64 { + return int64(binary.BigEndian.Uint64(value)) +} + func decodeFloat(value []byte) float32 { bin := binary.BigEndian.Uint32(value) return math.Float32frombits(bin) @@ -154,19 +255,58 @@ func Pack(obj interface{}) ([]byte, error) { case reflect.Bool: objectBytes = append(objectBytes, encodeBool(memberValue.Bool())) fmt.Println(decodeBool(objectBytes[0])) + case reflect.Uint8: encodedUint := encodeUint(uint8(memberValue.Uint())) objectBytes = append(objectBytes, encodedUint.typeByte, encodedUint.value) - fmt.Println(decodeUint(encodedUint.value)) + fmt.Println(decodeUint8(encodedUint.value)) + case reflect.Uint16: + encodedUint := encodeUint16(uint16(memberValue.Uint())) + objectBytes = append(objectBytes, encodedUint.typeByte) + objectBytes = append(objectBytes, encodedUint.value...) + fmt.Println(decodeUint16(encodedUint.value)) + case reflect.Uint32: + encodedUint := encodeUint32(uint32(memberValue.Uint())) + objectBytes = append(objectBytes, encodedUint.typeByte) + objectBytes = append(objectBytes, encodedUint.value...) + fmt.Println(decodeUint32(encodedUint.value)) + case reflect.Uint64: + encodedUint := encodeUint64(uint64(memberValue.Uint())) + objectBytes = append(objectBytes, encodedUint.typeByte) + objectBytes = append(objectBytes, encodedUint.value...) + fmt.Println(decodeUint64(encodedUint.value)) + case reflect.Int8: encodedInt := encodeInt(int8(memberValue.Int())) objectBytes = append(objectBytes, encodedInt.typeByte, encodedInt.value) fmt.Println(decodeInt(encodedInt.value)) + case reflect.Int16: + encodedInt := encodeInt16(int16(memberValue.Int())) + objectBytes = append(objectBytes, encodedInt.typeByte) + objectBytes = append(objectBytes, encodedInt.value...) + fmt.Println(decodeInt16(encodedInt.value)) + case reflect.Int32: + encodedInt := encodeInt32(int32(memberValue.Int())) + objectBytes = append(objectBytes, encodedInt.typeByte) + objectBytes = append(objectBytes, encodedInt.value...) + fmt.Println(decodeInt16(encodedInt.value)) + case reflect.Int64: + encodedInt := encodeInt64(int64(memberValue.Int())) + objectBytes = append(objectBytes, encodedInt.typeByte) + objectBytes = append(objectBytes, encodedInt.value...) + fmt.Println(decodeInt16(encodedInt.value)) + case reflect.Float32: - encodedFloat := encodeFloat(float32(memberValue.Float())) + encodedFloat := encodeFloat32(float32(memberValue.Float())) objectBytes = append(objectBytes, encodedFloat.typeByte) objectBytes = append(objectBytes, encodedFloat.value...) fmt.Println(decodeFloat(encodedFloat.value)) + case reflect.Float64: + encodedFloat := encodeFloat64(memberValue.Float()) + objectBytes = append(objectBytes, encodedFloat.typeByte) + objectBytes = append(objectBytes, encodedFloat.value...) + fmt.Println(decodeFloat(encodedFloat.value)) + case reflect.String: encodedString := encodeString(memberValue.String()) objectBytes = append(objectBytes, encodedString.typeByte) @@ -175,6 +315,7 @@ func Pack(obj interface{}) ([]byte, error) { fmt.Println(decodeString(encodedString.value)) default: + objectBytes = append(objectBytes, Msgpack_Nil) } objectBytes = append(objectBytes, memberName...) From 38d695fd0cc1fcc238626fe1507b35e6c0c12d02 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Mon, 15 Sep 2025 12:50:49 +0300 Subject: [PATCH 04/19] feat: added array encoding function --- cmd/main.go | 6 ++ pkg/msgpack.go | 238 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 166 insertions(+), 78 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 44e143f..87ad6e8 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,6 +6,11 @@ import ( ) func main() { + arr := make([]any, 3) + arr[0] = "hello" + arr[1] = 7 + arr[2] = 9.8 + object := msgpack.Object{ IsObject: true, Flag: false, @@ -13,6 +18,7 @@ func main() { Snum: -1000, Fnum: 3.14, Str: "Fatma", + Arr: arr, } bytes, err := msgpack.Pack(object) if err != nil { diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 5263de6..bd4bb40 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -28,6 +28,9 @@ const Msgpack_String_8 = 0xd9 const Msgpack_String_16 = 0xda const Msgpack_String_32 = 0xdb +const Msgpack_Array_16 = 0xdc +const Msgpack_Array_32 = 0xdd + type Uint_8 struct { typeByte byte value byte @@ -59,6 +62,12 @@ type String struct { size []byte } +type Array struct { + typeByte byte + value []byte + size []byte +} + type Object struct { IsObject bool Flag bool @@ -66,6 +75,7 @@ type Object struct { Snum int16 Fnum float32 Str string + Arr []any } func encodeBool(value bool) byte { @@ -101,7 +111,7 @@ func encodeUint32(value uint32) Uint { } func encodeUint64(value uint64) Uint { - bytes := make([]byte, 4) + bytes := make([]byte, 8) binary.BigEndian.PutUint64(bytes, value) return Uint{ typeByte: Msgpack_Uint_64, @@ -109,7 +119,16 @@ func encodeUint64(value uint64) Uint { } } -func encodeInt(value int8) Int_8 { +func encodeInt(value int) Int { + bytes := make([]byte, 8) + binary.BigEndian.PutUint64(bytes, uint64(value)) + return Int{ + typeByte: Msgpack_Int_64, + value: bytes, + } +} + +func encodeInt8(value int8) Int_8 { return Int_8{ typeByte: Msgpack_Uint_8, value: byte(value), @@ -120,7 +139,7 @@ func encodeInt16(value int16) Int { bytes := make([]byte, 2) binary.BigEndian.PutUint16(bytes, uint16(value)) return Int{ - typeByte: Msgpack_Uint_16, + typeByte: Msgpack_Int_16, value: bytes, } } @@ -129,7 +148,7 @@ func encodeInt32(value int32) Int { bytes := make([]byte, 4) binary.BigEndian.PutUint32(bytes, uint32(value)) return Int{ - typeByte: Msgpack_Uint_32, + typeByte: Msgpack_Int_32, value: bytes, } } @@ -138,7 +157,7 @@ func encodeInt64(value int64) Int { bytes := make([]byte, 8) binary.BigEndian.PutUint64(bytes, uint64(value)) return Int{ - typeByte: Msgpack_Uint_64, + typeByte: Msgpack_Int_64, value: bytes, } } @@ -192,6 +211,41 @@ func encodeString(value string) String { } } +func encodeArray(arr []any) Array { + array_16 := int(math.Pow(2, 16) - 1) + array_32 := int(math.Pow(2, 32) - 1) + bytes := make([]byte, 0) + var size []byte + var t byte + + if len(arr) <= array_16 { + size = make([]byte, 2) + binary.BigEndian.PutUint16(size, uint16(len(arr))) + t = Msgpack_Array_16 + } else if len(arr) <= array_32 { + size = make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(arr))) + t = Msgpack_Array_32 + } + fmt.Println("array start") + for _, element := range arr { + elementType := reflect.TypeOf(element).Kind() + elementValue := reflect.ValueOf(element) + fmt.Println(elementType, elementValue) + objectBytes := make([]byte, 0) + serializeElement(&objectBytes, elementType, elementValue) + fmt.Println(objectBytes) + bytes = append(bytes, objectBytes...) + } + fmt.Println("array data",size, t) + + return Array{ + typeByte: t, + value: bytes, + size: size, + } +} + func decodeBool(value byte) bool { if value == byte(Msgpack_True) { return true @@ -215,7 +269,11 @@ func decodeUint64(value []byte) uint64 { return binary.BigEndian.Uint64(value) } -func decodeInt(value byte) int8 { +func decodeInt(value []byte) int { + return int(binary.BigEndian.Uint64(value)) +} + +func decodeInt8(value byte) int8 { return int8(value) } @@ -230,15 +288,102 @@ func decodeInt64(value []byte) int64 { return int64(binary.BigEndian.Uint64(value)) } -func decodeFloat(value []byte) float32 { +func decodeFloat32(value []byte) float32 { bin := binary.BigEndian.Uint32(value) return math.Float32frombits(bin) } +func decodeFloat64(value []byte) float64 { + bin := binary.BigEndian.Uint64(value) + return math.Float64frombits(bin) +} + func decodeString(value []byte) string { return string(value) } +func serializeElement(objectBytes *[]byte, elementType reflect.Kind, elementValue reflect.Value) { + switch elementType { + case reflect.Bool: + *objectBytes = append(*objectBytes, encodeBool(elementValue.Bool())) + fmt.Println(decodeBool((*objectBytes)[0])) + + case reflect.Uint8: + encodedUint := encodeUint(uint8(elementValue.Uint())) + *objectBytes = append(*objectBytes, encodedUint.typeByte, encodedUint.value) + fmt.Println(decodeUint8(encodedUint.value)) + case reflect.Uint16: + encodedUint := encodeUint16(uint16(elementValue.Uint())) + *objectBytes = append(*objectBytes, encodedUint.typeByte) + *objectBytes = append(*objectBytes, encodedUint.value...) + fmt.Println(decodeUint16(encodedUint.value)) + case reflect.Uint32: + encodedUint := encodeUint32(uint32(elementValue.Uint())) + *objectBytes = append(*objectBytes, encodedUint.typeByte) + *objectBytes = append(*objectBytes, encodedUint.value...) + fmt.Println(decodeUint32(encodedUint.value)) + case reflect.Uint64: + encodedUint := encodeUint64(uint64(elementValue.Uint())) + *objectBytes = append(*objectBytes, encodedUint.typeByte) + *objectBytes = append(*objectBytes, encodedUint.value...) + fmt.Println(decodeUint64(encodedUint.value)) + + case reflect.Int8: + encodedInt := encodeInt8(int8(elementValue.Int())) + *objectBytes = append(*objectBytes, encodedInt.typeByte, encodedInt.value) + fmt.Println(decodeInt8(encodedInt.value)) + case reflect.Int16: + encodedInt := encodeInt16(int16(elementValue.Int())) + *objectBytes = append(*objectBytes, encodedInt.typeByte) + *objectBytes = append(*objectBytes, encodedInt.value...) + fmt.Println(decodeInt16(encodedInt.value)) + case reflect.Int32: + encodedInt := encodeInt32(int32(elementValue.Int())) + *objectBytes = append(*objectBytes, encodedInt.typeByte) + *objectBytes = append(*objectBytes, encodedInt.value...) + fmt.Println(decodeInt32(encodedInt.value)) + case reflect.Int64: + encodedInt := encodeInt64(int64(elementValue.Int())) + *objectBytes = append(*objectBytes, encodedInt.typeByte) + *objectBytes = append(*objectBytes, encodedInt.value...) + fmt.Println(decodeInt64(encodedInt.value)) + case reflect.Int: + encodedInt := encodeInt(int(elementValue.Int())) + *objectBytes = append(*objectBytes, encodedInt.typeByte) + *objectBytes = append(*objectBytes, encodedInt.value...) + fmt.Println(decodeInt(encodedInt.value)) + + case reflect.Float32: + encodedFloat := encodeFloat32(float32(elementValue.Float())) + *objectBytes = append(*objectBytes, encodedFloat.typeByte) + *objectBytes = append(*objectBytes, encodedFloat.value...) + fmt.Println(decodeFloat32(encodedFloat.value)) + case reflect.Float64: + encodedFloat := encodeFloat64(elementValue.Float()) + *objectBytes = append(*objectBytes, encodedFloat.typeByte) + *objectBytes = append(*objectBytes, encodedFloat.value...) + fmt.Println(decodeFloat64(encodedFloat.value)) + + case reflect.String: + encodedString := encodeString(elementValue.String()) + *objectBytes = append(*objectBytes, encodedString.typeByte) + *objectBytes = append(*objectBytes, encodedString.size...) + *objectBytes = append(*objectBytes, encodedString.value...) + fmt.Println(decodeString(encodedString.value)) + + case reflect.Slice: + encodedArray := encodeArray(elementValue.Interface().([]any)) + *objectBytes = append(*objectBytes, encodedArray.typeByte) + *objectBytes = append(*objectBytes, encodedArray.size...) + *objectBytes = append(*objectBytes, encodedArray.value...) + case reflect.Map: + + default: + *objectBytes = append(*objectBytes, Msgpack_Nil) + } + +} + func Pack(obj interface{}) ([]byte, error) { t := reflect.TypeOf(obj) v := reflect.ValueOf(obj) @@ -249,79 +394,16 @@ func Pack(obj interface{}) ([]byte, error) { memberName := field.Name memberValue := v.Field(i) fmt.Println(memberType, memberName, memberValue) - objectBytes := make([]byte, 0) - switch memberType { - case reflect.Bool: - objectBytes = append(objectBytes, encodeBool(memberValue.Bool())) - fmt.Println(decodeBool(objectBytes[0])) - - case reflect.Uint8: - encodedUint := encodeUint(uint8(memberValue.Uint())) - objectBytes = append(objectBytes, encodedUint.typeByte, encodedUint.value) - fmt.Println(decodeUint8(encodedUint.value)) - case reflect.Uint16: - encodedUint := encodeUint16(uint16(memberValue.Uint())) - objectBytes = append(objectBytes, encodedUint.typeByte) - objectBytes = append(objectBytes, encodedUint.value...) - fmt.Println(decodeUint16(encodedUint.value)) - case reflect.Uint32: - encodedUint := encodeUint32(uint32(memberValue.Uint())) - objectBytes = append(objectBytes, encodedUint.typeByte) - objectBytes = append(objectBytes, encodedUint.value...) - fmt.Println(decodeUint32(encodedUint.value)) - case reflect.Uint64: - encodedUint := encodeUint64(uint64(memberValue.Uint())) - objectBytes = append(objectBytes, encodedUint.typeByte) - objectBytes = append(objectBytes, encodedUint.value...) - fmt.Println(decodeUint64(encodedUint.value)) - - case reflect.Int8: - encodedInt := encodeInt(int8(memberValue.Int())) - objectBytes = append(objectBytes, encodedInt.typeByte, encodedInt.value) - fmt.Println(decodeInt(encodedInt.value)) - case reflect.Int16: - encodedInt := encodeInt16(int16(memberValue.Int())) - objectBytes = append(objectBytes, encodedInt.typeByte) - objectBytes = append(objectBytes, encodedInt.value...) - fmt.Println(decodeInt16(encodedInt.value)) - case reflect.Int32: - encodedInt := encodeInt32(int32(memberValue.Int())) - objectBytes = append(objectBytes, encodedInt.typeByte) - objectBytes = append(objectBytes, encodedInt.value...) - fmt.Println(decodeInt16(encodedInt.value)) - case reflect.Int64: - encodedInt := encodeInt64(int64(memberValue.Int())) - objectBytes = append(objectBytes, encodedInt.typeByte) - objectBytes = append(objectBytes, encodedInt.value...) - fmt.Println(decodeInt16(encodedInt.value)) - - case reflect.Float32: - encodedFloat := encodeFloat32(float32(memberValue.Float())) - objectBytes = append(objectBytes, encodedFloat.typeByte) - objectBytes = append(objectBytes, encodedFloat.value...) - fmt.Println(decodeFloat(encodedFloat.value)) - case reflect.Float64: - encodedFloat := encodeFloat64(memberValue.Float()) - objectBytes = append(objectBytes, encodedFloat.typeByte) - objectBytes = append(objectBytes, encodedFloat.value...) - fmt.Println(decodeFloat(encodedFloat.value)) - - case reflect.String: - encodedString := encodeString(memberValue.String()) - objectBytes = append(objectBytes, encodedString.typeByte) - objectBytes = append(objectBytes, encodedString.size...) - objectBytes = append(objectBytes, encodedString.value...) - fmt.Println(decodeString(encodedString.value)) - - default: - objectBytes = append(objectBytes, Msgpack_Nil) - } - - objectBytes = append(objectBytes, memberName...) + objectBytes := make([]byte, 0) + serializeElement(&objectBytes, memberType, memberValue) fmt.Println(objectBytes) - bytes = append(bytes, objectBytes...) // to store the name - bytes = append(bytes, byte('\n')) // to store the separator == 10 + + // objectBytes = append(objectBytes, memberName...) + // fmt.Println(objectBytes) + + bytes = append(bytes, objectBytes...) + bytes = append(bytes, byte('\n')) // to store the separator == 10 } From ff14fa7f3ed140c717587e46066416b53305b3e9 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Mon, 15 Sep 2025 15:16:56 +0300 Subject: [PATCH 05/19] feat: added deserializeElement function for decoding different formats --- cmd/main.go | 3 +- pkg/msgpack.go | 182 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 139 insertions(+), 46 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 87ad6e8..205eeba 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,10 +6,11 @@ import ( ) func main() { - arr := make([]any, 3) + arr := make([]any, 4) arr[0] = "hello" arr[1] = 7 arr[2] = 9.8 + arr[3] = "world" object := msgpack.Object{ IsObject: true, diff --git a/pkg/msgpack.go b/pkg/msgpack.go index bd4bb40..70f7413 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -237,7 +237,7 @@ func encodeArray(arr []any) Array { fmt.Println(objectBytes) bytes = append(bytes, objectBytes...) } - fmt.Println("array data",size, t) + fmt.Println("array data", size, t) return Array{ typeByte: t, @@ -246,136 +246,225 @@ func encodeArray(arr []any) Array { } } -func decodeBool(value byte) bool { +func decodeBool(value byte) (bool, int) { if value == byte(Msgpack_True) { - return true + return true, 1 } else { - return false + return false, 1 } } -func decodeUint8(value byte) uint8 { - return uint8(value) +func decodeUint8(value byte) (uint8, int) { + return uint8(value), 2 } -func decodeUint16(value []byte) uint16 { - return binary.BigEndian.Uint16(value) +func decodeUint16(value []byte) (uint16, int) { + return binary.BigEndian.Uint16(value[:2]), 3 } -func decodeUint32(value []byte) uint32 { - return binary.BigEndian.Uint32(value) +func decodeUint32(value []byte) (uint32, int) { + return binary.BigEndian.Uint32(value[:4]), 5 } -func decodeUint64(value []byte) uint64 { - return binary.BigEndian.Uint64(value) +func decodeUint64(value []byte) (uint64, int) { + return binary.BigEndian.Uint64(value[:8]), 9 } -func decodeInt(value []byte) int { - return int(binary.BigEndian.Uint64(value)) +func decodeInt(value []byte) (int, int) { + return int(binary.BigEndian.Uint64(value[:8])), 9 } -func decodeInt8(value byte) int8 { - return int8(value) +func decodeInt8(value byte) (int8, int) { + return int8(value), 2 } -func decodeInt16(value []byte) int16 { - return int16(binary.BigEndian.Uint16(value)) +func decodeInt16(value []byte) (int16, int) { + return int16(binary.BigEndian.Uint16(value[:2])), 3 } -func decodeInt32(value []byte) int32 { - return int32(binary.BigEndian.Uint32(value)) +func decodeInt32(value []byte) (int32, int) { + return int32(binary.BigEndian.Uint32(value[:4])), 5 } -func decodeInt64(value []byte) int64 { - return int64(binary.BigEndian.Uint64(value)) +func decodeInt64(value []byte) (int64, int) { + return int64(binary.BigEndian.Uint64(value[:8])), 9 } -func decodeFloat32(value []byte) float32 { - bin := binary.BigEndian.Uint32(value) - return math.Float32frombits(bin) +func decodeFloat32(value []byte) (float32, int) { + bin := binary.BigEndian.Uint32(value[:4]) + return math.Float32frombits(bin), 5 } -func decodeFloat64(value []byte) float64 { - bin := binary.BigEndian.Uint64(value) - return math.Float64frombits(bin) +func decodeFloat64(value []byte) (float64, int) { + bin := binary.BigEndian.Uint64(value[:8]) + return math.Float64frombits(bin), 9 } -func decodeString(value []byte) string { - return string(value) +func decodeString8(value []byte) (string, int) { + size := int(value[0]) + return string(value[1 : size+1]), 1 + 1 + size +} + +func decodeString16(value []byte) (string, int) { + size := decodeSize(value[:2]) + return string(value[2 : size+2]), 1 + 2 + size +} + +func decodeString32(value []byte) (string, int) { + size := decodeSize(value[:4]) + return string(value[4 : size+4]), 1 + 4 + size +} + +func decodeSize(value []byte) int { + if len(value) == 2 { + return int(binary.BigEndian.Uint16(value)) + } else if len(value) == 4 { + return int(binary.BigEndian.Uint32(value)) + } else if len(value) == 8 { + return int(binary.BigEndian.Uint64(value)) + } + return 0 +} + +func decodeArray16(value []byte) ([]any, int) { + size := decodeSize(value[:2]) + data := value[2:] + array := make([]any, size) + var offset int + + for i := 0; i < size; i++ { + element, offset := deserializeElement(data) + array[i] = element + data = data[offset:] + fmt.Println("decode array:", offset, element, data) + } + return array, offset +} + +func decodeArray32(value []byte) ([]any, int) { + size := decodeSize(value[1:5]) + data := value[5:] + array := make([]any, size) + var offset int + + for i := 0; i < size; i++ { + element, offset := deserializeElement(data) + array[i] = element + data = data[offset:] + fmt.Println("decode array:", offset, element, data) + } + return array, offset +} + +func deserializeElement(value []byte) (any, int) { + elementType := value[0] + switch elementType { + case Msgpack_True: + return decodeBool(value[0]) + case Msgpack_False: + return decodeBool(value[0]) + case Msgpack_Uint_8: + return decodeUint8(value[1]) + case Msgpack_Uint_16: + return decodeUint16(value[1:]) + case Msgpack_Uint_32: + return decodeUint32(value[1:]) + case Msgpack_Uint_64: + return decodeUint64(value[1:]) + case Msgpack_Int_8: + return decodeInt8(value[1]) + case Msgpack_Int_16: + return decodeInt16(value[1:]) + case Msgpack_Int_32: + return decodeInt32(value[1:]) + case Msgpack_Int_64: + return decodeInt64(value[1:]) + case Msgpack_Float_32: + return decodeFloat32(value[1:]) + case Msgpack_Float_64: + return decodeFloat64(value[1:]) + case Msgpack_String_8: + return decodeString8(value[1:]) + case Msgpack_String_16: + return decodeString16(value[1:]) + case Msgpack_String_32: + return decodeString32(value[1:]) + case Msgpack_Array_16: + return decodeArray16(value[1:]) + case Msgpack_Array_32: + return decodeArray32(value[1:]) + } + return nil, 0 } func serializeElement(objectBytes *[]byte, elementType reflect.Kind, elementValue reflect.Value) { switch elementType { case reflect.Bool: *objectBytes = append(*objectBytes, encodeBool(elementValue.Bool())) - fmt.Println(decodeBool((*objectBytes)[0])) case reflect.Uint8: encodedUint := encodeUint(uint8(elementValue.Uint())) *objectBytes = append(*objectBytes, encodedUint.typeByte, encodedUint.value) - fmt.Println(decodeUint8(encodedUint.value)) + case reflect.Uint16: encodedUint := encodeUint16(uint16(elementValue.Uint())) *objectBytes = append(*objectBytes, encodedUint.typeByte) *objectBytes = append(*objectBytes, encodedUint.value...) - fmt.Println(decodeUint16(encodedUint.value)) + case reflect.Uint32: encodedUint := encodeUint32(uint32(elementValue.Uint())) *objectBytes = append(*objectBytes, encodedUint.typeByte) *objectBytes = append(*objectBytes, encodedUint.value...) - fmt.Println(decodeUint32(encodedUint.value)) + case reflect.Uint64: encodedUint := encodeUint64(uint64(elementValue.Uint())) *objectBytes = append(*objectBytes, encodedUint.typeByte) *objectBytes = append(*objectBytes, encodedUint.value...) - fmt.Println(decodeUint64(encodedUint.value)) case reflect.Int8: encodedInt := encodeInt8(int8(elementValue.Int())) *objectBytes = append(*objectBytes, encodedInt.typeByte, encodedInt.value) - fmt.Println(decodeInt8(encodedInt.value)) + case reflect.Int16: encodedInt := encodeInt16(int16(elementValue.Int())) *objectBytes = append(*objectBytes, encodedInt.typeByte) *objectBytes = append(*objectBytes, encodedInt.value...) - fmt.Println(decodeInt16(encodedInt.value)) + case reflect.Int32: encodedInt := encodeInt32(int32(elementValue.Int())) *objectBytes = append(*objectBytes, encodedInt.typeByte) *objectBytes = append(*objectBytes, encodedInt.value...) - fmt.Println(decodeInt32(encodedInt.value)) + case reflect.Int64: encodedInt := encodeInt64(int64(elementValue.Int())) *objectBytes = append(*objectBytes, encodedInt.typeByte) *objectBytes = append(*objectBytes, encodedInt.value...) - fmt.Println(decodeInt64(encodedInt.value)) + case reflect.Int: encodedInt := encodeInt(int(elementValue.Int())) *objectBytes = append(*objectBytes, encodedInt.typeByte) *objectBytes = append(*objectBytes, encodedInt.value...) - fmt.Println(decodeInt(encodedInt.value)) case reflect.Float32: encodedFloat := encodeFloat32(float32(elementValue.Float())) *objectBytes = append(*objectBytes, encodedFloat.typeByte) *objectBytes = append(*objectBytes, encodedFloat.value...) - fmt.Println(decodeFloat32(encodedFloat.value)) + case reflect.Float64: encodedFloat := encodeFloat64(elementValue.Float()) *objectBytes = append(*objectBytes, encodedFloat.typeByte) *objectBytes = append(*objectBytes, encodedFloat.value...) - fmt.Println(decodeFloat64(encodedFloat.value)) case reflect.String: encodedString := encodeString(elementValue.String()) *objectBytes = append(*objectBytes, encodedString.typeByte) *objectBytes = append(*objectBytes, encodedString.size...) *objectBytes = append(*objectBytes, encodedString.value...) - fmt.Println(decodeString(encodedString.value)) case reflect.Slice: encodedArray := encodeArray(elementValue.Interface().([]any)) *objectBytes = append(*objectBytes, encodedArray.typeByte) *objectBytes = append(*objectBytes, encodedArray.size...) *objectBytes = append(*objectBytes, encodedArray.value...) + case reflect.Map: default: @@ -393,11 +482,14 @@ func Pack(obj interface{}) ([]byte, error) { memberType := field.Type.Kind() memberName := field.Name memberValue := v.Field(i) - fmt.Println(memberType, memberName, memberValue) + fmt.Printf("struct member %d: %s %s %v \n", i, memberType, memberName, memberValue) objectBytes := make([]byte, 0) serializeElement(&objectBytes, memberType, memberValue) - fmt.Println(objectBytes) + fmt.Println("serialized:", objectBytes) + + data, offset := deserializeElement(objectBytes) + fmt.Println("deserialized:", data, offset) // objectBytes = append(objectBytes, memberName...) // fmt.Println(objectBytes) From 3da2fa41d0303d561b4f36d63618903772197b42 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Mon, 15 Sep 2025 16:07:26 +0300 Subject: [PATCH 06/19] feat: added encoding and decoding functions for map --- cmd/main.go | 18 +++++---- pkg/msgpack.go | 103 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 106 insertions(+), 15 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 205eeba..f2db351 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -12,14 +12,19 @@ func main() { arr[2] = 9.8 arr[3] = "world" + m := make(map[any]any) + m["hello"] = 10 + m["world"] = -20 + object := msgpack.Object{ IsObject: true, - Flag: false, - Unum: 0, - Snum: -1000, - Fnum: 3.14, - Str: "Fatma", - Arr: arr, + Flag: false, + Unum: 0, + Snum: -1000, + Fnum: 3.14, + Str: "Fatma", + Arr: arr, + Map: m, } bytes, err := msgpack.Pack(object) if err != nil { @@ -28,4 +33,3 @@ func main() { fmt.Println(bytes) } - diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 70f7413..751c9f5 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -30,6 +30,8 @@ const Msgpack_String_32 = 0xdb const Msgpack_Array_16 = 0xdc const Msgpack_Array_32 = 0xdd +const Msgpack_Map_16 = 0xde +const Msgpack_Map_32 = 0xdf type Uint_8 struct { typeByte byte @@ -68,6 +70,12 @@ type Array struct { size []byte } +type Map struct { + typeByte byte + value []byte + size []byte +} + type Object struct { IsObject bool Flag bool @@ -76,6 +84,7 @@ type Object struct { Fnum float32 Str string Arr []any + Map map[any]any } func encodeBool(value bool) byte { @@ -227,17 +236,14 @@ func encodeArray(arr []any) Array { binary.BigEndian.PutUint32(size, uint32(len(arr))) t = Msgpack_Array_32 } - fmt.Println("array start") + for _, element := range arr { elementType := reflect.TypeOf(element).Kind() elementValue := reflect.ValueOf(element) - fmt.Println(elementType, elementValue) objectBytes := make([]byte, 0) serializeElement(&objectBytes, elementType, elementValue) - fmt.Println(objectBytes) bytes = append(bytes, objectBytes...) } - fmt.Println("array data", size, t) return Array{ typeByte: t, @@ -246,6 +252,46 @@ func encodeArray(arr []any) Array { } } +func encodeMap(m map[any]any) Map { + map_16 := int(math.Pow(2, 16) - 1) + map_32 := int(math.Pow(2, 32) - 1) + bytes := make([]byte, 0) + var size []byte + var t byte + + if len(m) <= map_16 { + size = make([]byte, 2) + binary.BigEndian.PutUint16(size, uint16(len(m))) + t = Msgpack_Map_16 + } else if len(m) <= map_32 { + size = make([]byte, 4) + binary.BigEndian.PutUint32(size, uint32(len(m))) + t = Msgpack_Map_32 + } + + for key, val := range m { + keyType := reflect.TypeOf(key).Kind() + keyValue := reflect.ValueOf(key) + objectBytes := make([]byte, 0) + serializeElement(&objectBytes, keyType, keyValue) + // fmt.Println("key bytes:",key,objectBytes) + bytes = append(bytes, objectBytes...) + + valueType := reflect.TypeOf(val).Kind() + valueValue := reflect.ValueOf(val) + objectBytes = make([]byte, 0) + serializeElement(&objectBytes, valueType, valueValue) + // fmt.Println("value bytes:",val,objectBytes) + bytes = append(bytes, objectBytes...) + } + + return Map{ + typeByte: t, + value: bytes, + size: size, + } +} + func decodeBool(value byte) (bool, int) { if value == byte(Msgpack_True) { return true, 1 @@ -265,6 +311,7 @@ func decodeUint16(value []byte) (uint16, int) { func decodeUint32(value []byte) (uint32, int) { return binary.BigEndian.Uint32(value[:4]), 5 } + func decodeUint64(value []byte) (uint64, int) { return binary.BigEndian.Uint64(value[:8]), 9 } @@ -280,6 +327,7 @@ func decodeInt8(value byte) (int8, int) { func decodeInt16(value []byte) (int16, int) { return int16(binary.BigEndian.Uint16(value[:2])), 3 } + func decodeInt32(value []byte) (int32, int) { return int32(binary.BigEndian.Uint32(value[:4])), 5 } @@ -334,14 +382,13 @@ func decodeArray16(value []byte) ([]any, int) { element, offset := deserializeElement(data) array[i] = element data = data[offset:] - fmt.Println("decode array:", offset, element, data) } return array, offset } func decodeArray32(value []byte) ([]any, int) { - size := decodeSize(value[1:5]) - data := value[5:] + size := decodeSize(value[:4]) + data := value[4:] array := make([]any, size) var offset int @@ -349,11 +396,42 @@ func decodeArray32(value []byte) ([]any, int) { element, offset := deserializeElement(data) array[i] = element data = data[offset:] - fmt.Println("decode array:", offset, element, data) } return array, offset } +func decodeMap16(value []byte) (map[any]any, int) { + size := decodeSize(value[:2]) + data := value[2:] + m := make(map[any]any) + var offset int + + for i := 0; i < size; i++ { + key, offset := deserializeElement(data) + data = data[offset:] + value, offset := deserializeElement(data) + data = data[offset:] + m[key] = value + } + return m, offset +} + +func decodeMap32(value []byte) (map[any]any, int) { + size := decodeSize(value[:4]) + data := value[4:] + m := make(map[any]any) + var offset int + + for i := 0; i < size; i++ { + key, offset := deserializeElement(data) + data = data[offset:] + value, offset := deserializeElement(data) + data = data[offset:] + m[key] = value + } + return m, offset +} + func deserializeElement(value []byte) (any, int) { elementType := value[0] switch elementType { @@ -391,7 +469,12 @@ func deserializeElement(value []byte) (any, int) { return decodeArray16(value[1:]) case Msgpack_Array_32: return decodeArray32(value[1:]) + case Msgpack_Map_16: + return decodeMap16(value[1:]) + case Msgpack_Map_32: + return decodeMap32(value[1:]) } + return nil, 0 } @@ -466,6 +549,10 @@ func serializeElement(objectBytes *[]byte, elementType reflect.Kind, elementValu *objectBytes = append(*objectBytes, encodedArray.value...) case reflect.Map: + encodedMap := encodeMap(elementValue.Interface().(map[any]any)) + *objectBytes = append(*objectBytes, encodedMap.typeByte) + *objectBytes = append(*objectBytes, encodedMap.size...) + *objectBytes = append(*objectBytes, encodedMap.value...) default: *objectBytes = append(*objectBytes, Msgpack_Nil) From 3622316832e48e2c0394d2f131ed97ab20f2d0da Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Mon, 15 Sep 2025 19:09:52 +0300 Subject: [PATCH 07/19] feat: added pack and unpack functions --- cmd/main.go | 14 +++++-- pkg/msgpack.go | 104 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 80 insertions(+), 38 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index f2db351..10c5d5f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -24,12 +24,20 @@ func main() { Fnum: 3.14, Str: "Fatma", Arr: arr, - Map: m, + Mapp: m, } - bytes, err := msgpack.Pack(object) + packed, err := msgpack.Pack(object) if err != nil { fmt.Println(err) } - fmt.Println(bytes) + fmt.Printf("Packed: %+v\n", packed) + + unpacked, err := msgpack.Unpack(packed) + if err != nil { + fmt.Println(err) + } + + fmt.Printf("Unpacked: %+v\n", unpacked) + } diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 751c9f5..8846896 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -84,7 +84,7 @@ type Object struct { Fnum float32 Str string Arr []any - Map map[any]any + Mapp map[any]any } func encodeBool(value bool) byte { @@ -200,14 +200,17 @@ func encodeString(value string) String { case len(value) < string_8: size = make([]byte, 0) size = append(size, byte(len(value))) + // size = encodeSize(len(value)) t = Msgpack_String_8 case len(value) < string_16: - size = make([]byte, 2) - binary.BigEndian.PutUint16(size, uint16(len(value))) + // size = make([]byte, 2) + // binary.BigEndian.PutUint16(size, uint16(len(value))) + size = encodeSize(len(value)) t = Msgpack_String_16 case len(value) < string_32: - size = make([]byte, 4) - binary.BigEndian.PutUint32(size, uint32(len(value))) + // size = make([]byte, 4) + // binary.BigEndian.PutUint32(size, uint32(len(value))) + size = encodeSize(len(value)) t = Msgpack_String_32 } @@ -228,12 +231,14 @@ func encodeArray(arr []any) Array { var t byte if len(arr) <= array_16 { - size = make([]byte, 2) - binary.BigEndian.PutUint16(size, uint16(len(arr))) + // size = make([]byte, 2) + // binary.BigEndian.PutUint16(size, uint16(len(arr))) + size = encodeSize(len(arr)) t = Msgpack_Array_16 } else if len(arr) <= array_32 { - size = make([]byte, 4) - binary.BigEndian.PutUint32(size, uint32(len(arr))) + // size = make([]byte, 4) + // binary.BigEndian.PutUint32(size, uint32(len(arr))) + size = encodeSize(len(arr)) t = Msgpack_Array_32 } @@ -260,12 +265,14 @@ func encodeMap(m map[any]any) Map { var t byte if len(m) <= map_16 { - size = make([]byte, 2) - binary.BigEndian.PutUint16(size, uint16(len(m))) + // size = make([]byte, 2) + // binary.BigEndian.PutUint16(size, uint16(len(m))) + size = encodeSize(len(m)) t = Msgpack_Map_16 } else if len(m) <= map_32 { - size = make([]byte, 4) - binary.BigEndian.PutUint32(size, uint32(len(m))) + // size = make([]byte, 4) + // binary.BigEndian.PutUint32(size, uint32(len(m))) + size = encodeSize(len(m)) t = Msgpack_Map_32 } @@ -274,14 +281,12 @@ func encodeMap(m map[any]any) Map { keyValue := reflect.ValueOf(key) objectBytes := make([]byte, 0) serializeElement(&objectBytes, keyType, keyValue) - // fmt.Println("key bytes:",key,objectBytes) bytes = append(bytes, objectBytes...) valueType := reflect.TypeOf(val).Kind() valueValue := reflect.ValueOf(val) objectBytes = make([]byte, 0) serializeElement(&objectBytes, valueType, valueValue) - // fmt.Println("value bytes:",val,objectBytes) bytes = append(bytes, objectBytes...) } @@ -316,10 +321,6 @@ func decodeUint64(value []byte) (uint64, int) { return binary.BigEndian.Uint64(value[:8]), 9 } -func decodeInt(value []byte) (int, int) { - return int(binary.BigEndian.Uint64(value[:8])), 9 -} - func decodeInt8(value byte) (int8, int) { return int8(value), 2 } @@ -372,39 +373,58 @@ func decodeSize(value []byte) int { return 0 } +func encodeSize(size int) []byte { + size_16 := int(math.Pow(2, 16) - 1) + size_32 := int(math.Pow(2, 32) - 1) + var sizeBytes []byte + if size <= size_16 { + sizeBytes = make([]byte, 2) + binary.BigEndian.PutUint16(sizeBytes, uint16(size)) + } else if size <= size_32 { + sizeBytes = make([]byte, 4) + binary.BigEndian.PutUint32(sizeBytes, uint32(size)) + } else { + sizeBytes = make([]byte, 8) + binary.BigEndian.PutUint64(sizeBytes, uint64(size)) + } + return sizeBytes +} + func decodeArray16(value []byte) ([]any, int) { size := decodeSize(value[:2]) data := value[2:] array := make([]any, size) - var offset int - + overallOffset := 0 for i := 0; i < size; i++ { element, offset := deserializeElement(data) array[i] = element data = data[offset:] + overallOffset += offset } - return array, offset + + return array, overallOffset + 3 } func decodeArray32(value []byte) ([]any, int) { size := decodeSize(value[:4]) data := value[4:] array := make([]any, size) - var offset int + overallOffset := 0 for i := 0; i < size; i++ { element, offset := deserializeElement(data) array[i] = element data = data[offset:] + overallOffset += offset } - return array, offset + return array, overallOffset + 5 } func decodeMap16(value []byte) (map[any]any, int) { size := decodeSize(value[:2]) data := value[2:] m := make(map[any]any) - var offset int + overallOffset := 0 for i := 0; i < size; i++ { key, offset := deserializeElement(data) @@ -412,15 +432,16 @@ func decodeMap16(value []byte) (map[any]any, int) { value, offset := deserializeElement(data) data = data[offset:] m[key] = value + overallOffset += offset } - return m, offset + return m, overallOffset + 3 } func decodeMap32(value []byte) (map[any]any, int) { size := decodeSize(value[:4]) data := value[4:] m := make(map[any]any) - var offset int + overallOffset := 0 for i := 0; i < size; i++ { key, offset := deserializeElement(data) @@ -428,8 +449,9 @@ func decodeMap32(value []byte) (map[any]any, int) { value, offset := deserializeElement(data) data = data[offset:] m[key] = value + overallOffset += offset } - return m, offset + return m, overallOffset + 5 } func deserializeElement(value []byte) (any, int) { @@ -564,6 +586,18 @@ func Pack(obj interface{}) ([]byte, error) { t := reflect.TypeOf(obj) v := reflect.ValueOf(obj) bytes := make([]byte, 0) + size := make([]byte, 0) + var mapType byte + if t.NumField() <= 16 { + size = encodeSize(t.NumField()) + mapType = Msgpack_Map_16 + } else if t.NumField() <= 32 { + size = encodeSize(t.NumField()) + mapType = Msgpack_Map_32 + } + bytes = append(bytes, mapType) + bytes = append(bytes, size...) + for i := 0; i < t.NumField(); i++ { field := t.Field(i) memberType := field.Type.Kind() @@ -578,17 +612,17 @@ func Pack(obj interface{}) ([]byte, error) { data, offset := deserializeElement(objectBytes) fmt.Println("deserialized:", data, offset) - // objectBytes = append(objectBytes, memberName...) - // fmt.Println(objectBytes) + nameBytes := make([]byte, 0) + serializeElement(&nameBytes, reflect.String, reflect.ValueOf(memberName)) + fmt.Println("name bytes:", nameBytes, memberName) + bytes = append(bytes, nameBytes...) bytes = append(bytes, objectBytes...) - bytes = append(bytes, byte('\n')) // to store the separator == 10 - } - return bytes, nil } -func Unpack(data []byte) (interface{}, error) { - return nil, nil +func Unpack(bytes []byte) (any, error) { + unpacked, _ := deserializeElement(bytes) + return unpacked, nil } From 6af3bba2258441f0c800fbe5c48814f9a1dc202c Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Tue, 16 Sep 2025 12:39:14 +0300 Subject: [PATCH 08/19] fix: handled objects in unpack --- cmd/main.go | 7 +++---- pkg/msgpack.go | 9 ++++++++- pkg/msgpack_test.go | 49 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 pkg/msgpack_test.go diff --git a/cmd/main.go b/cmd/main.go index 10c5d5f..efb5311 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -24,20 +24,19 @@ func main() { Fnum: 3.14, Str: "Fatma", Arr: arr, - Mapp: m, } packed, err := msgpack.Pack(object) if err != nil { fmt.Println(err) } - fmt.Printf("Packed: %+v\n", packed) - unpacked, err := msgpack.Unpack(packed) + var obj msgpack.Object + unpacked, err := msgpack.Unpack(packed, &obj) if err != nil { fmt.Println(err) } - fmt.Printf("Unpacked: %+v\n", unpacked) + fmt.Printf("Object: %+v\n", obj) } diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 8846896..b5fa658 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -622,7 +622,14 @@ func Pack(obj interface{}) ([]byte, error) { return bytes, nil } -func Unpack(bytes []byte) (any, error) { +func Unpack(bytes []byte, obj interface{}) (any, error) { unpacked, _ := deserializeElement(bytes) + + v := reflect.ValueOf(obj).Elem() + for key, val := range unpacked.(map[any]any) { + field := v.FieldByName(key.(string)) + field.Set(reflect.ValueOf(val)) + } + return unpacked, nil } diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go new file mode 100644 index 0000000..9f6dcc3 --- /dev/null +++ b/pkg/msgpack_test.go @@ -0,0 +1,49 @@ +package msgpack + +import ( + "strings" + "testing" +) + +type user struct { + Name string + Id string +} + +func TestPack(t *testing.T) { + t.Run("test pack with string8", func(t *testing.T) { + user := user{ + Name: "Fatma", + Id: "id_1", + } + packed, err := Pack(user) + expected := []byte{222, 0, 2, + 217, 4, 78, 97, 109, 101, 217, 5, 70, 97, 116, 109, 97, + 217, 2, 73, 100, 217, 4, 105, 100, 95, 49} + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + if string(packed) != string(expected) { + t.Errorf("Expected %v, got %v", expected, packed) + } + + }) + + t.Run("test pack with string16", func(t *testing.T) { + user := user{ + Id: strings.Repeat("x", 260), + } + packed, err := Pack(user) + expected := []byte{222, 0, 2, + 217, 2, 73, 100, 217, 4, 105, 100, 95, 49} + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + if string(packed) != string(expected) { + t.Errorf("Expected %v, got %v", expected, packed) + } + + }) +} From e620aca2f539dad789ee403b2f2f1a9351d2b02f Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Tue, 16 Sep 2025 13:22:44 +0300 Subject: [PATCH 09/19] test: added unit tests for string8/16/32 --- pkg/msgpack_test.go | 74 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go index 9f6dcc3..7171829 100644 --- a/pkg/msgpack_test.go +++ b/pkg/msgpack_test.go @@ -1,49 +1,99 @@ package msgpack import ( + "reflect" "strings" "testing" ) -type user struct { +type User struct { Name string Id string } -func TestPack(t *testing.T) { - t.Run("test pack with string8", func(t *testing.T) { - user := user{ +func TestPackString(t *testing.T) { + t.Run("test pack and unpack with string8", func(t *testing.T) { + user_to_pack := User{ Name: "Fatma", Id: "id_1", } - packed, err := Pack(user) + packed, err := Pack(user_to_pack) expected := []byte{222, 0, 2, 217, 4, 78, 97, 109, 101, 217, 5, 70, 97, 116, 109, 97, 217, 2, 73, 100, 217, 4, 105, 100, 95, 49} - if err != nil { t.Errorf("Expected nil, got %v", err) } + if string(packed) != string(expected) { t.Errorf("Expected %v, got %v", expected, packed) } + var user_unpacked User + _, err = Unpack(packed, &user_unpacked) + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + if !reflect.DeepEqual(user_to_pack, user_unpacked) { + t.Errorf("Expected %v, got %v", user_to_pack, user_unpacked) + } + }) t.Run("test pack with string16", func(t *testing.T) { - user := user{ - Id: strings.Repeat("x", 260), + user := User{ + Id: strings.Repeat("x", 260), } packed, err := Pack(user) - expected := []byte{222, 0, 2, - 217, 2, 73, 100, 217, 4, 105, 100, 95, 49} + data_bytes := packed[15:] if err != nil { t.Errorf("Expected nil, got %v", err) } - if string(packed) != string(expected) { - t.Errorf("Expected %v, got %v", expected, packed) + + if len(data_bytes) != 263 { + t.Errorf("Expected size of data to be 263, got %d", len(data_bytes)) + } + + str_type := data_bytes[0] + if str_type != 218 { + t.Errorf("Expected type of string to be 218, got %d", str_type) + } + + str_size := decodeSize(data_bytes[1:3]) + if str_size != 260 { + t.Errorf("Expected size of string to be 260, got %d", str_size) } }) + + t.Run("test pack with string32", func(t *testing.T) { + user := User{ + Id: strings.Repeat("x", 70000), + } + packed, err := Pack(user) + data_bytes := packed[15:] + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + if len(data_bytes) != 70005 { + t.Errorf("Expected size of data to be 70005, got %d", len(data_bytes)) + } + + str_type := data_bytes[0] + if str_type != 219 { + t.Errorf("Expected type of string to be 219 (str32), got %d", str_type) + } + + str_size := decodeSize(data_bytes[1:5]) + if str_size != 70000 { + t.Errorf("Expected size of string to be 70000, got %d", str_size) + } +}) + } + From a52f7fd9f8e05de47d4bf3312753430fe723bf9d Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Tue, 16 Sep 2025 14:34:02 +0300 Subject: [PATCH 10/19] test: added unit tests for uint8/16/32/64 int8/16/32/64 float32/64 --- pkg/msgpack.go | 43 +++--- pkg/msgpack_test.go | 369 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 357 insertions(+), 55 deletions(-) diff --git a/pkg/msgpack.go b/pkg/msgpack.go index b5fa658..687097d 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -7,31 +7,32 @@ import ( "reflect" ) -const Msgpack_Nil = 0xc0 -const Msgpack_False = 0xc2 -const Msgpack_True = 0xc3 +const Msgpack_Nil = 0xc0 //192 +const Msgpack_False = 0xc2 //194 +const Msgpack_True = 0xc3 //195 -const Msgpack_Uint_8 = 0xcc -const Msgpack_Uint_16 = 0xcd -const Msgpack_Uint_32 = 0xce -const Msgpack_Uint_64 = 0xcf +const Msgpack_Uint_8 = 0xcc //204 +const Msgpack_Uint_16 = 0xcd //205 +const Msgpack_Uint_32 = 0xce //206 +const Msgpack_Uint_64 = 0xcf //207 -const Msgpack_Int_8 = 0xd0 -const Msgpack_Int_16 = 0xd1 -const Msgpack_Int_32 = 0xd2 -const Msgpack_Int_64 = 0xd3 +const Msgpack_Int_8 = 0xd0 //208 +const Msgpack_Int_16 = 0xd1 //209 +const Msgpack_Int_32 = 0xd2 //210 +const Msgpack_Int_64 = 0xd3 //211 -const Msgpack_Float_32 = 0xca -const Msgpack_Float_64 = 0xcb +const Msgpack_Float_32 = 0xca //202 +const Msgpack_Float_64 = 0xcb //203 -const Msgpack_String_8 = 0xd9 -const Msgpack_String_16 = 0xda -const Msgpack_String_32 = 0xdb +const Msgpack_String_8 = 0xd9 //217 +const Msgpack_String_16 = 0xda //218 +const Msgpack_String_32 = 0xdb //219 -const Msgpack_Array_16 = 0xdc -const Msgpack_Array_32 = 0xdd -const Msgpack_Map_16 = 0xde -const Msgpack_Map_32 = 0xdf +const Msgpack_Array_16 = 0xdc //220 +const Msgpack_Array_32 = 0xdd //221 + +const Msgpack_Map_16 = 0xde //222 +const Msgpack_Map_32 = 0xdf //223 type Uint_8 struct { typeByte byte @@ -139,7 +140,7 @@ func encodeInt(value int) Int { func encodeInt8(value int8) Int_8 { return Int_8{ - typeByte: Msgpack_Uint_8, + typeByte: Msgpack_Int_8, value: byte(value), } } diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go index 7171829..758caab 100644 --- a/pkg/msgpack_test.go +++ b/pkg/msgpack_test.go @@ -1,6 +1,7 @@ package msgpack import ( + "fmt" "reflect" "strings" "testing" @@ -12,34 +13,35 @@ type User struct { } func TestPackString(t *testing.T) { - t.Run("test pack and unpack with string8", func(t *testing.T) { - user_to_pack := User{ - Name: "Fatma", - Id: "id_1", + t.Run("test pack with string8", func(t *testing.T) { + user := User{ + Id: strings.Repeat("x", 200), } - packed, err := Pack(user_to_pack) - expected := []byte{222, 0, 2, - 217, 4, 78, 97, 109, 101, 217, 5, 70, 97, 116, 109, 97, - 217, 2, 73, 100, 217, 4, 105, 100, 95, 49} + packed, err := Pack(user) + data_bytes := packed[15:] + if err != nil { t.Errorf("Expected nil, got %v", err) } - if string(packed) != string(expected) { - t.Errorf("Expected %v, got %v", expected, packed) + if len(data_bytes) != 202 { + t.Errorf("Expected size of data to be 202, got %d", len(data_bytes)) } - var user_unpacked User - _, err = Unpack(packed, &user_unpacked) - - if err != nil { - t.Errorf("Expected nil, got %v", err) + str_type := data_bytes[0] + if str_type != 217 { + t.Errorf("Expected type of string to be 217 (str8), got %d", str_type) } - if !reflect.DeepEqual(user_to_pack, user_unpacked) { - t.Errorf("Expected %v, got %v", user_to_pack, user_unpacked) + str_size := int(data_bytes[1]) + if str_size != 200 { + t.Errorf("Expected size of string to be 200, got %d", str_size) } + str_value, _ := deserializeElement(data_bytes) + if str_value != strings.Repeat("x", 200) { + t.Errorf("Expected value of string to be %s, got %s", strings.Repeat("x", 200), str_value) + } }) t.Run("test pack with string16", func(t *testing.T) { @@ -67,33 +69,332 @@ func TestPackString(t *testing.T) { t.Errorf("Expected size of string to be 260, got %d", str_size) } + str_value, _ := deserializeElement(data_bytes) + if str_value != strings.Repeat("x", 260) { + t.Errorf("Expected value of string to be %s, got %s", strings.Repeat("x", 260), str_value) + } + }) t.Run("test pack with string32", func(t *testing.T) { - user := User{ - Id: strings.Repeat("x", 70000), - } - packed, err := Pack(user) - data_bytes := packed[15:] + user := User{ + Id: strings.Repeat("x", 70000), + } + packed, err := Pack(user) + data_bytes := packed[15:] - if err != nil { - t.Errorf("Expected nil, got %v", err) - } + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + if len(data_bytes) != 70005 { + t.Errorf("Expected size of data to be 70005, got %d", len(data_bytes)) + } + + str_type := data_bytes[0] + if str_type != 219 { + t.Errorf("Expected type of string to be 219 (str32), got %d", str_type) + } + + str_size := decodeSize(data_bytes[1:5]) + if str_size != 70000 { + t.Errorf("Expected size of string to be 70000, got %d", str_size) + } + + str_value, _ := deserializeElement(data_bytes) + if str_value != strings.Repeat("x", 70000) { + t.Errorf("Expected value of string to be %s, got %s", strings.Repeat("x", 70000), str_value) + } + }) +} - if len(data_bytes) != 70005 { - t.Errorf("Expected size of data to be 70005, got %d", len(data_bytes)) +func TestPackFloat(t *testing.T) { + type Object struct { + Fnum32 float32 + Fnum64 float64 } + t.Run("test pack with float32", func(t *testing.T) { + + obj := Object{ + Fnum32: 3.14, + } + + packed, err := Pack(obj) + data_bytes := packed[3+8 : 16] + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + fmt.Println(data_bytes) + float_type := data_bytes[0] + if float_type != 202 { + t.Errorf("Expected type of float to be 203 (float32), got %d", float_type) + } - str_type := data_bytes[0] - if str_type != 219 { - t.Errorf("Expected type of string to be 219 (str32), got %d", str_type) + float_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(float_value).Kind() != reflect.Float32 { + t.Errorf("Expected type of float to be float32, got %v", reflect.TypeOf(float_value).Kind()) + } + + }) + + t.Run("test pack with float64", func(t *testing.T) { + obj := Object{ + Fnum64: 3.14, + } + + packed, err := Pack(obj) + fmt.Println(packed) + data_bytes := packed[16+8:] + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + fmt.Println(data_bytes) + + float_type := data_bytes[0] + if float_type != 203 { + t.Errorf("Expected type of float to be 203 (float64), got %d", float_type) + } + + float_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(float_value).Kind() != reflect.Float64 { + t.Errorf("Expected type of float to be float64, got %v", reflect.TypeOf(float_value).Kind()) + } + }) +} + +func TestPackInt(t *testing.T) { + type Object struct { + Snum8 int8 + Snum16 int16 + Snum32 int32 + Snum64 int64 } + t.Run("test pack with int8", func(t *testing.T) { + + obj := Object{ + Snum8: 3, + } + + packed, err := Pack(obj) + data_bytes := packed[3+2+5 : 10+2] + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + int_type := data_bytes[0] + if int_type != 208 { + t.Errorf("Expected type of int to be 208 (int8), got %d", int_type) + } + + int_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(int_value).Kind() != reflect.Int8 { + t.Errorf("Expected type of int to be int8, got %v", reflect.TypeOf(int_value).Kind()) + } + }) + + t.Run("test pack with int16", func(t *testing.T) { + obj := Object{ + Snum16: 300, + } + + packed, err := Pack(obj) + data_bytes := packed[12+2+6 : 20+3] + fmt.Println(data_bytes) + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + int_type := data_bytes[0] + if int_type != 209 { + t.Errorf("Expected type of int to be 209 (int16), got %d", int_type) + } + + int_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(int_value).Kind() != reflect.Int16 { + t.Errorf("Expected type of int to be int16, got %v", reflect.TypeOf(int_value).Kind()) + } + }) + + t.Run("test pack with int32", func(t *testing.T) { + obj := Object{ + Snum32: 70000, + } + + packed, err := Pack(obj) + data_bytes := packed[23+2+6 : 31+5] + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + int_type := data_bytes[0] + if int_type != 210 { + t.Errorf("Expected type of int to be 210 (int32), got %d", int_type) + } + + int_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(int_value).Kind() != reflect.Int32 { + t.Errorf("Expected type of int to be int32, got %v", reflect.TypeOf(int_value).Kind()) + } + }) - str_size := decodeSize(data_bytes[1:5]) - if str_size != 70000 { - t.Errorf("Expected size of string to be 70000, got %d", str_size) + t.Run("test pack with int64", func(t *testing.T) { + obj := Object{ + Snum64: 5000000000, + } + + packed, err := Pack(obj) + data_bytes := packed[36+2+6:] + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + int_type := data_bytes[0] + if int_type != 211 { + t.Errorf("Expected type of int to be 211 (int64), got %d", int_type) + } + + int_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(int_value).Kind() != reflect.Int64 { + t.Errorf("Expected type of int to be int64, got %v", reflect.TypeOf(int_value).Kind()) + } + }) + +} + +func TestPackUint(t *testing.T) { + type Object struct { + Unum8 uint8 + Unum16 uint16 + Unum32 uint32 + Unum64 uint64 } -}) + t.Run("test pack with uint8", func(t *testing.T) { + obj := Object{ + Unum8: 250, + } + + packed, err := Pack(obj) + data_bytes := packed[3+2+5 : 10+2] + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + uint_type := data_bytes[0] + if uint_type != 204 { + t.Errorf("Expected type of uint to be 204 (uint8), got %d", uint_type) + } + + uint_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(uint_value).Kind() != reflect.Uint8 { + t.Errorf("Expected type of uint to be uint8, got %v", reflect.TypeOf(uint_value).Kind()) + } + }) + + t.Run("test pack with uint16", func(t *testing.T) { + obj := Object{ + Unum16: 700, + } + + packed, err := Pack(obj) + data_bytes := packed[12+2+6 : 20+3] + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + uint_type := data_bytes[0] + if uint_type != 205 { + t.Errorf("Expected type of uint to be 205 (uint16), got %d", uint_type) + } + + uint_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(uint_value).Kind() != reflect.Uint16 { + t.Errorf("Expected type of uint to be uint16, got %v", reflect.TypeOf(uint_value).Kind()) + } + }) + + t.Run("test pack with uint32", func(t *testing.T) { + obj := Object{ + Unum32: 100000, + } + + packed, err := Pack(obj) + data_bytes := packed[23+2+6 : 31+5] + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + uint_type := data_bytes[0] + if uint_type != 206 { + t.Errorf("Expected type of uint to be 206 (uint32), got %d", uint_type) + } + + uint_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(uint_value).Kind() != reflect.Uint32 { + t.Errorf("Expected type of uint to be uint32, got %v", reflect.TypeOf(uint_value).Kind()) + } + }) + + t.Run("test pack with uint64", func(t *testing.T) { + obj := Object{ + Unum64: 5000000000, + } + + packed, err := Pack(obj) + data_bytes := packed[36+2+6:] + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + uint_type := data_bytes[0] + if uint_type != 207 { + t.Errorf("Expected type of uint to be 207 (uint64), got %d", uint_type) + } + + uint_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(uint_value).Kind() != reflect.Uint64 { + t.Errorf("Expected type of uint to be uint64, got %v", reflect.TypeOf(uint_value).Kind()) + } + }) } + +func TestAll(t *testing.T) { + t.Run("test pack and unpack with string8", func(t *testing.T) { + user_to_pack := User{ + Name: "Fatma", + Id: "id_1", + } + packed, err := Pack(user_to_pack) + expected := []byte{222, 0, 2, + 217, 4, 78, 97, 109, 101, 217, 5, 70, 97, 116, 109, 97, + 217, 2, 73, 100, 217, 4, 105, 100, 95, 49} + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + if string(packed) != string(expected) { + t.Errorf("Expected %v, got %v", expected, packed) + } + + var user_unpacked User + _, err = Unpack(packed, &user_unpacked) + + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + if !reflect.DeepEqual(user_to_pack, user_unpacked) { + t.Errorf("Expected %v, got %v", user_to_pack, user_unpacked) + } + + }) +} From a658010a1c0c676d91f1677ac8594dda65c3bdbd Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Tue, 16 Sep 2025 15:39:21 +0300 Subject: [PATCH 11/19] test: added unit tests for array16/32 map/16/32 --- cmd/main.go | 24 ++--- pkg/msgpack.go | 34 ++++--- pkg/msgpack_test.go | 230 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 252 insertions(+), 36 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index efb5311..ec263a7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,24 +6,24 @@ import ( ) func main() { - arr := make([]any, 4) - arr[0] = "hello" - arr[1] = 7 - arr[2] = 9.8 - arr[3] = "world" + arr := make([]any, 3) + // arr[0] = "hello" + // arr[1] = 7 + // arr[2] = 9.8 m := make(map[any]any) m["hello"] = 10 m["world"] = -20 object := msgpack.Object{ - IsObject: true, - Flag: false, - Unum: 0, - Snum: -1000, - Fnum: 3.14, - Str: "Fatma", - Arr: arr, + // IsObject: true, + // Flag: false, + // Unum: 0, + // Snum: -1000, + // Fnum: 3.14, + // Str: "Fatma", + Arr: arr, + Mapp: m, } packed, err := msgpack.Pack(object) if err != nil { diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 687097d..c33e1c1 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -2,7 +2,7 @@ package msgpack import ( "encoding/binary" - "fmt" + // "fmt" "math" "reflect" ) @@ -78,14 +78,14 @@ type Map struct { } type Object struct { - IsObject bool - Flag bool - Unum uint8 - Snum int16 - Fnum float32 - Str string - Arr []any - Mapp map[any]any + // IsObject bool + // Flag bool + // Unum uint8 + // Snum int16 + // Fnum float32 + // Str string + Arr []any + Mapp map[any]any } func encodeBool(value bool) byte { @@ -244,6 +244,10 @@ func encodeArray(arr []any) Array { } for _, element := range arr { + if element == nil { + bytes = append(bytes, Msgpack_Nil) + continue + } elementType := reflect.TypeOf(element).Kind() elementValue := reflect.ValueOf(element) objectBytes := make([]byte, 0) @@ -496,6 +500,8 @@ func deserializeElement(value []byte) (any, int) { return decodeMap16(value[1:]) case Msgpack_Map_32: return decodeMap32(value[1:]) + case Msgpack_Nil: + return nil, 1 } return nil, 0 @@ -604,18 +610,18 @@ func Pack(obj interface{}) ([]byte, error) { memberType := field.Type.Kind() memberName := field.Name memberValue := v.Field(i) - fmt.Printf("struct member %d: %s %s %v \n", i, memberType, memberName, memberValue) + // fmt.Printf("struct member %d: %s %s %v \n", i, memberType, memberName, memberValue) objectBytes := make([]byte, 0) serializeElement(&objectBytes, memberType, memberValue) - fmt.Println("serialized:", objectBytes) + // fmt.Println("serialized:", objectBytes) - data, offset := deserializeElement(objectBytes) - fmt.Println("deserialized:", data, offset) + // data, offset := deserializeElement(objectBytes) + // fmt.Println("deserialized:", data, offset) nameBytes := make([]byte, 0) serializeElement(&nameBytes, reflect.String, reflect.ValueOf(memberName)) - fmt.Println("name bytes:", nameBytes, memberName) + // fmt.Println("name bytes:", nameBytes, memberName) bytes = append(bytes, nameBytes...) bytes = append(bytes, objectBytes...) diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go index 758caab..5c034b9 100644 --- a/pkg/msgpack_test.go +++ b/pkg/msgpack_test.go @@ -125,7 +125,7 @@ func TestPackFloat(t *testing.T) { if err != nil { t.Errorf("Expected nil, got %v", err) } - fmt.Println(data_bytes) + float_type := data_bytes[0] if float_type != 202 { t.Errorf("Expected type of float to be 203 (float32), got %d", float_type) @@ -144,13 +144,11 @@ func TestPackFloat(t *testing.T) { } packed, err := Pack(obj) - fmt.Println(packed) data_bytes := packed[16+8:] if err != nil { t.Errorf("Expected nil, got %v", err) } - fmt.Println(data_bytes) float_type := data_bytes[0] if float_type != 203 { @@ -197,12 +195,11 @@ func TestPackInt(t *testing.T) { t.Run("test pack with int16", func(t *testing.T) { obj := Object{ - Snum16: 300, + Snum16: -300, } packed, err := Pack(obj) data_bytes := packed[12+2+6 : 20+3] - fmt.Println(data_bytes) if err != nil { t.Errorf("Expected nil, got %v", err) @@ -280,7 +277,7 @@ func TestPackUint(t *testing.T) { } packed, err := Pack(obj) - data_bytes := packed[3+2+5 : 10+2] + data_bytes := packed[3+2+5 : 10+2] if err != nil { t.Errorf("Expected nil, got %v", err) @@ -303,7 +300,7 @@ func TestPackUint(t *testing.T) { } packed, err := Pack(obj) - data_bytes := packed[12+2+6 : 20+3] + data_bytes := packed[12+2+6 : 20+3] if err != nil { t.Errorf("Expected nil, got %v", err) @@ -322,11 +319,11 @@ func TestPackUint(t *testing.T) { t.Run("test pack with uint32", func(t *testing.T) { obj := Object{ - Unum32: 100000, + Unum32: 70000, } packed, err := Pack(obj) - data_bytes := packed[23+2+6 : 31+5] + data_bytes := packed[23+2+6 : 31+5] if err != nil { t.Errorf("Expected nil, got %v", err) @@ -349,7 +346,7 @@ func TestPackUint(t *testing.T) { } packed, err := Pack(obj) - data_bytes := packed[36+2+6:] + data_bytes := packed[36+2+6:] if err != nil { t.Errorf("Expected nil, got %v", err) } @@ -366,6 +363,219 @@ func TestPackUint(t *testing.T) { }) } +func TestPackArray(t *testing.T) { + type Object struct { + Arr []any + } + t.Run("test pack with array16", func(t *testing.T) { + arr := []any{1.1, 2.2, 3.3} + obj := Object{ + Arr: arr, + } + packed, err := Pack(obj) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + data_bytes := packed[3+5:] + + array_type := data_bytes[0] + + if array_type != 220 { + t.Errorf("Expected type of array to be 220 (array), got %d", array_type) + } + + array_size := decodeSize(data_bytes[1:3]) + if array_size != 3 { + t.Errorf("Expected size of array to be 3, got %d", array_size) + } + + array_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(array_value).Kind() != reflect.Slice { + t.Errorf("Expected type of array to be slice, got %v", reflect.TypeOf(array_value).Kind()) + } + if !reflect.DeepEqual(array_value, arr) { + t.Errorf("Expected array to be %v, got %v", arr, array_value) + } + + }) + + t.Run("test pack with array32", func(t *testing.T) { + arr := make([]any, 70000) + for i := range arr { + arr[i] = true + } + obj := Object{ + Arr: arr, + } + packed, err := Pack(obj) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + data_bytes := packed[3+5:] + + array_type := data_bytes[0] + + if array_type != 221 { + t.Errorf("Expected type of array to be 221 (array32), got %d", array_type) + } + + array_size := decodeSize(data_bytes[1:5]) + if array_size != 70000 { + t.Errorf("Expected size of array to be 70000, got %d", array_size) + } + + array_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(array_value).Kind() != reflect.Slice { + t.Errorf("Expected type of array to be slice, got %v", reflect.TypeOf(array_value).Kind()) + } + if !reflect.DeepEqual(array_value, arr) { + t.Errorf("Expected array to be %v, got %v", arr[:5], array_value.([]any)[:5]) // print first few elements + } + }) + + t.Run("test pack with empty array", func(t *testing.T) { + arr := make([]any, 3) + obj := Object{ + Arr: arr, + } + packed, err := Pack(obj) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + data_bytes := packed[3+5:] + + array_type := data_bytes[0] + + if array_type != 220 { + t.Errorf("Expected type of array to be 220 (array), got %d", array_type) + } + + array_size := decodeSize(data_bytes[1:3]) + if array_size != 3 { + t.Errorf("Expected size of array to be 3, got %d", array_size) + } + + array_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(array_value).Kind() != reflect.Slice { + t.Errorf("Expected type of array to be slice, got %v", reflect.TypeOf(array_value).Kind()) + } + + array := array_value.([]any) + if array[0] != nil { + t.Errorf("Expected array element to be nil, got %v", array[0]) + } + + }) + +} + +func TestPackMap(t *testing.T) { + type Object struct { + Dict map[any]any + } + + t.Run("test pack with map16", func(t *testing.T) { + m := map[any]any{ + "Lang": "Go", + "Ver": "1.19", + } + obj := Object{ + Dict: m, + } + packed, err := Pack(obj) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + data_bytes := packed[3+6:] + + map_type := data_bytes[0] + if map_type != 222 { + t.Errorf("Expected type of map to be 222 (map16), got %d", map_type) + } + + map_size := decodeSize(data_bytes[1:3]) + if map_size != 2 { + t.Errorf("Expected size of map to be 2, got %d", map_size) + } + + map_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(map_value).Kind() != reflect.Map { + t.Errorf("Expected type of map to be map, got %v", reflect.TypeOf(map_value).Kind()) + } + + if !reflect.DeepEqual(map_value, m) { + t.Errorf("Expected map to be %v, got %v", m, map_value) + } + }) + + t.Run("test pack with map32", func(t *testing.T) { + m := make(map[any]any, 70000) + for i := 0; i < 70000; i++ { + m[fmt.Sprintf("k%d", i)] = fmt.Sprintf("v%d", i) + } + obj := Object{ + Dict: m, + } + packed, err := Pack(obj) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + data_bytes := packed[3+6:] + + map_type := data_bytes[0] + if map_type != 223 { + t.Errorf("Expected type of map to be 223 (map32), got %d", map_type) + } + + map_size := decodeSize(data_bytes[1:5]) + if map_size != 70000 { + t.Errorf("Expected size of map to be 70000, got %d", map_size) + } + + map_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(map_value).Kind() != reflect.Map { + t.Errorf("Expected type of map to be map, got %v", reflect.TypeOf(map_value).Kind()) + } + + if !reflect.DeepEqual(map_value, m) { + t.Errorf("Expected map to be %v, got %v", m, map_value) + } + }) + + t.Run("test pack with empty map", func(t *testing.T) { + m := make(map[any]any) + obj := Object{ + Dict: m, + } + packed, err := Pack(obj) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + data_bytes := packed[3+6:] + + map_type := data_bytes[0] + if map_type != 222 { + t.Errorf("Expected type of map to be 222 (map16), got %d", map_type) + } + + map_size := decodeSize(data_bytes[1:3]) + if map_size != 0 { + t.Errorf("Expected size of map to be 0, got %d", map_size) + } + + map_value, _ := deserializeElement(data_bytes) + if reflect.TypeOf(map_value).Kind() != reflect.Map { + t.Errorf("Expected type of map to be map, got %v", reflect.TypeOf(map_value).Kind()) + } + + if !reflect.DeepEqual(map_value, m) { + t.Errorf("Expected map to be %v, got %v", m, map_value) + } + }) +} func TestAll(t *testing.T) { t.Run("test pack and unpack with string8", func(t *testing.T) { From d82826b990d8faf52d91e851dc23e3db19b8dc90 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Tue, 16 Sep 2025 16:11:12 +0300 Subject: [PATCH 12/19] test: added unit tests for complex struct --- cmd/main.go | 16 ++++---- pkg/msgpack.go | 16 ++++---- pkg/msgpack_test.go | 95 +++++++++++++++++++++++++++++++++++++++------ 3 files changed, 99 insertions(+), 28 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index ec263a7..c43b623 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -16,14 +16,14 @@ func main() { m["world"] = -20 object := msgpack.Object{ - // IsObject: true, - // Flag: false, - // Unum: 0, - // Snum: -1000, - // Fnum: 3.14, - // Str: "Fatma", - Arr: arr, - Mapp: m, + IsObject: true, + Flag: false, + Unum: 0, + Snum: -1000, + Fnum: 3.14, + Str: "Fatma", + Arr: arr, + Mapp: m, } packed, err := msgpack.Pack(object) if err != nil { diff --git a/pkg/msgpack.go b/pkg/msgpack.go index c33e1c1..5577151 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -78,14 +78,14 @@ type Map struct { } type Object struct { - // IsObject bool - // Flag bool - // Unum uint8 - // Snum int16 - // Fnum float32 - // Str string - Arr []any - Mapp map[any]any + IsObject bool + Flag bool + Unum uint8 + Snum int16 + Fnum float32 + Str string + Arr []any + Mapp map[any]any } func encodeBool(value bool) byte { diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go index 5c034b9..be19c69 100644 --- a/pkg/msgpack_test.go +++ b/pkg/msgpack_test.go @@ -476,10 +476,10 @@ func TestPackMap(t *testing.T) { } t.Run("test pack with map16", func(t *testing.T) { - m := map[any]any{ - "Lang": "Go", - "Ver": "1.19", - } + m := make(map[any]any) + m["Lang"] = "Go" + m["Ver"] = "1.19" + obj := Object{ Dict: m, } @@ -578,23 +578,17 @@ func TestPackMap(t *testing.T) { } func TestAll(t *testing.T) { - t.Run("test pack and unpack with string8", func(t *testing.T) { + t.Run("test pack and unpack with simple struct", func(t *testing.T) { user_to_pack := User{ Name: "Fatma", Id: "id_1", } packed, err := Pack(user_to_pack) - expected := []byte{222, 0, 2, - 217, 4, 78, 97, 109, 101, 217, 5, 70, 97, 116, 109, 97, - 217, 2, 73, 100, 217, 4, 105, 100, 95, 49} + if err != nil { t.Errorf("Expected nil, got %v", err) } - if string(packed) != string(expected) { - t.Errorf("Expected %v, got %v", expected, packed) - } - var user_unpacked User _, err = Unpack(packed, &user_unpacked) @@ -607,4 +601,81 @@ func TestAll(t *testing.T) { } }) + + t.Run("test pack and unpack with complex struct", func(t *testing.T) { + arr := make([]any, 3) + arr[0] = "hello" + arr[1] = -7.7 + arr[2] = 9.8 + + m := make(map[any]any) + m["hello"] = 10.0 + m["world"] = -20.0 + + obj_to_pack := Object{ + IsObject: true, + Flag: false, + Unum: 0, + Snum: -1000, + Fnum: 3.14, + Str: "Fatma", + Arr: arr, + Mapp: m, + } + + packed, err := Pack(obj_to_pack) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + var obj_unpacked Object + _, err = Unpack(packed, &obj_unpacked) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + if !reflect.DeepEqual(obj_to_pack, obj_unpacked) { + t.Errorf("Expected %+v, got %+v", obj_to_pack, obj_unpacked) + } + }) + + type ComplexUser struct { + ID uint32 + Name string + Age int8 + Balance float64 + IsActive bool + Tags []any + Metadata map[any]any + } + + t.Run("test pack and unpack with more complex struct", func(t *testing.T) { + user_to_pack := ComplexUser{ + ID: 123456, + Name: "Fatma Ebrahim", + Age: 27, + Balance: 1050.75, + IsActive: true, + Tags: []any{"golang", "backend", "testing"}, + Metadata: map[any]any{ + "country": "Egypt", + "city": "Cairo", + }, + } + + packed, err := Pack(user_to_pack) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + var user_unpacked ComplexUser + _, err = Unpack(packed, &user_unpacked) + if err != nil { + t.Errorf("Expected nil, got %v", err) + } + + if !reflect.DeepEqual(user_to_pack, user_unpacked) { + t.Errorf("Expected %+v, got %+v", user_to_pack, user_unpacked) + } + }) } From 74f218a5bdaf5d3f94af21420fc226897e560493 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Tue, 16 Sep 2025 17:23:34 +0300 Subject: [PATCH 13/19] feat: added serialize and deserialize functions --- cmd/main.go | 18 ++++- pkg/msgpack.go | 170 ++++++++++++++++++++------------------------ pkg/msgpack_test.go | 38 +++++----- 3 files changed, 110 insertions(+), 116 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index c43b623..5bf1ac7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -7,9 +7,9 @@ import ( func main() { arr := make([]any, 3) - // arr[0] = "hello" - // arr[1] = 7 - // arr[2] = 9.8 + arr[0] = "hello" + arr[1] = 7 + arr[2] = 9.8 m := make(map[any]any) m["hello"] = 10 @@ -39,4 +39,16 @@ func main() { fmt.Printf("Unpacked: %+v\n", unpacked) fmt.Printf("Object: %+v\n", obj) + ser := msgpack.Serialize(m) + if err != nil { + fmt.Println(err) + } + fmt.Printf("Serialized: %+v\n", ser) + + des := msgpack.Deserialize(ser) + if err != nil { + fmt.Println(err) + } + fmt.Printf("Deserialized: %+v\n", des) + } diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 5577151..1be7819 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -2,7 +2,6 @@ package msgpack import ( "encoding/binary" - // "fmt" "math" "reflect" ) @@ -201,16 +200,11 @@ func encodeString(value string) String { case len(value) < string_8: size = make([]byte, 0) size = append(size, byte(len(value))) - // size = encodeSize(len(value)) t = Msgpack_String_8 case len(value) < string_16: - // size = make([]byte, 2) - // binary.BigEndian.PutUint16(size, uint16(len(value))) size = encodeSize(len(value)) t = Msgpack_String_16 case len(value) < string_32: - // size = make([]byte, 4) - // binary.BigEndian.PutUint32(size, uint32(len(value))) size = encodeSize(len(value)) t = Msgpack_String_32 } @@ -232,13 +226,9 @@ func encodeArray(arr []any) Array { var t byte if len(arr) <= array_16 { - // size = make([]byte, 2) - // binary.BigEndian.PutUint16(size, uint16(len(arr))) size = encodeSize(len(arr)) t = Msgpack_Array_16 } else if len(arr) <= array_32 { - // size = make([]byte, 4) - // binary.BigEndian.PutUint32(size, uint32(len(arr))) size = encodeSize(len(arr)) t = Msgpack_Array_32 } @@ -248,10 +238,7 @@ func encodeArray(arr []any) Array { bytes = append(bytes, Msgpack_Nil) continue } - elementType := reflect.TypeOf(element).Kind() - elementValue := reflect.ValueOf(element) - objectBytes := make([]byte, 0) - serializeElement(&objectBytes, elementType, elementValue) + objectBytes := Serialize(element) bytes = append(bytes, objectBytes...) } @@ -270,28 +257,18 @@ func encodeMap(m map[any]any) Map { var t byte if len(m) <= map_16 { - // size = make([]byte, 2) - // binary.BigEndian.PutUint16(size, uint16(len(m))) size = encodeSize(len(m)) t = Msgpack_Map_16 } else if len(m) <= map_32 { - // size = make([]byte, 4) - // binary.BigEndian.PutUint32(size, uint32(len(m))) size = encodeSize(len(m)) t = Msgpack_Map_32 } for key, val := range m { - keyType := reflect.TypeOf(key).Kind() - keyValue := reflect.ValueOf(key) - objectBytes := make([]byte, 0) - serializeElement(&objectBytes, keyType, keyValue) + objectBytes := Serialize(key) bytes = append(bytes, objectBytes...) - valueType := reflect.TypeOf(val).Kind() - valueValue := reflect.ValueOf(val) - objectBytes = make([]byte, 0) - serializeElement(&objectBytes, valueType, valueValue) + objectBytes = Serialize(val) bytes = append(bytes, objectBytes...) } @@ -302,6 +279,23 @@ func encodeMap(m map[any]any) Map { } } +func encodeSize(size int) []byte { + size_16 := int(math.Pow(2, 16) - 1) + size_32 := int(math.Pow(2, 32) - 1) + var sizeBytes []byte + if size <= size_16 { + sizeBytes = make([]byte, 2) + binary.BigEndian.PutUint16(sizeBytes, uint16(size)) + } else if size <= size_32 { + sizeBytes = make([]byte, 4) + binary.BigEndian.PutUint32(sizeBytes, uint32(size)) + } else { + sizeBytes = make([]byte, 8) + binary.BigEndian.PutUint64(sizeBytes, uint64(size)) + } + return sizeBytes +} + func decodeBool(value byte) (bool, int) { if value == byte(Msgpack_True) { return true, 1 @@ -378,22 +372,7 @@ func decodeSize(value []byte) int { return 0 } -func encodeSize(size int) []byte { - size_16 := int(math.Pow(2, 16) - 1) - size_32 := int(math.Pow(2, 32) - 1) - var sizeBytes []byte - if size <= size_16 { - sizeBytes = make([]byte, 2) - binary.BigEndian.PutUint16(sizeBytes, uint16(size)) - } else if size <= size_32 { - sizeBytes = make([]byte, 4) - binary.BigEndian.PutUint32(sizeBytes, uint32(size)) - } else { - sizeBytes = make([]byte, 8) - binary.BigEndian.PutUint64(sizeBytes, uint64(size)) - } - return sizeBytes -} + func decodeArray16(value []byte) ([]any, int) { size := decodeSize(value[:2]) @@ -401,7 +380,7 @@ func decodeArray16(value []byte) ([]any, int) { array := make([]any, size) overallOffset := 0 for i := 0; i < size; i++ { - element, offset := deserializeElement(data) + element, offset := deserialize_helper(data) array[i] = element data = data[offset:] overallOffset += offset @@ -417,7 +396,7 @@ func decodeArray32(value []byte) ([]any, int) { overallOffset := 0 for i := 0; i < size; i++ { - element, offset := deserializeElement(data) + element, offset := deserialize_helper(data) array[i] = element data = data[offset:] overallOffset += offset @@ -432,9 +411,9 @@ func decodeMap16(value []byte) (map[any]any, int) { overallOffset := 0 for i := 0; i < size; i++ { - key, offset := deserializeElement(data) + key, offset := deserialize_helper(data) data = data[offset:] - value, offset := deserializeElement(data) + value, offset := deserialize_helper(data) data = data[offset:] m[key] = value overallOffset += offset @@ -449,9 +428,9 @@ func decodeMap32(value []byte) (map[any]any, int) { overallOffset := 0 for i := 0; i < size; i++ { - key, offset := deserializeElement(data) + key, offset := deserialize_helper(data) data = data[offset:] - value, offset := deserializeElement(data) + value, offset := deserialize_helper(data) data = data[offset:] m[key] = value overallOffset += offset @@ -459,7 +438,8 @@ func decodeMap32(value []byte) (map[any]any, int) { return m, overallOffset + 5 } -func deserializeElement(value []byte) (any, int) { + +func deserialize_helper(value []byte) (any, int) { elementType := value[0] switch elementType { case Msgpack_True: @@ -507,88 +487,99 @@ func deserializeElement(value []byte) (any, int) { return nil, 0 } -func serializeElement(objectBytes *[]byte, elementType reflect.Kind, elementValue reflect.Value) { +func Serialize(elementVal any) []byte { + elementValue := reflect.ValueOf(elementVal) + objectBytes := make([]byte, 0) + elementType := reflect.TypeOf(elementVal).Kind() switch elementType { case reflect.Bool: - *objectBytes = append(*objectBytes, encodeBool(elementValue.Bool())) + objectBytes = append(objectBytes, encodeBool(elementValue.Bool())) case reflect.Uint8: encodedUint := encodeUint(uint8(elementValue.Uint())) - *objectBytes = append(*objectBytes, encodedUint.typeByte, encodedUint.value) + objectBytes = append(objectBytes, encodedUint.typeByte, encodedUint.value) case reflect.Uint16: encodedUint := encodeUint16(uint16(elementValue.Uint())) - *objectBytes = append(*objectBytes, encodedUint.typeByte) - *objectBytes = append(*objectBytes, encodedUint.value...) + objectBytes = append(objectBytes, encodedUint.typeByte) + objectBytes = append(objectBytes, encodedUint.value...) case reflect.Uint32: encodedUint := encodeUint32(uint32(elementValue.Uint())) - *objectBytes = append(*objectBytes, encodedUint.typeByte) - *objectBytes = append(*objectBytes, encodedUint.value...) + objectBytes = append(objectBytes, encodedUint.typeByte) + objectBytes = append(objectBytes, encodedUint.value...) case reflect.Uint64: encodedUint := encodeUint64(uint64(elementValue.Uint())) - *objectBytes = append(*objectBytes, encodedUint.typeByte) - *objectBytes = append(*objectBytes, encodedUint.value...) + objectBytes = append(objectBytes, encodedUint.typeByte) + objectBytes = append(objectBytes, encodedUint.value...) case reflect.Int8: encodedInt := encodeInt8(int8(elementValue.Int())) - *objectBytes = append(*objectBytes, encodedInt.typeByte, encodedInt.value) + objectBytes = append(objectBytes, encodedInt.typeByte, encodedInt.value) case reflect.Int16: encodedInt := encodeInt16(int16(elementValue.Int())) - *objectBytes = append(*objectBytes, encodedInt.typeByte) - *objectBytes = append(*objectBytes, encodedInt.value...) + objectBytes = append(objectBytes, encodedInt.typeByte) + objectBytes = append(objectBytes, encodedInt.value...) case reflect.Int32: encodedInt := encodeInt32(int32(elementValue.Int())) - *objectBytes = append(*objectBytes, encodedInt.typeByte) - *objectBytes = append(*objectBytes, encodedInt.value...) + objectBytes = append(objectBytes, encodedInt.typeByte) + objectBytes = append(objectBytes, encodedInt.value...) case reflect.Int64: encodedInt := encodeInt64(int64(elementValue.Int())) - *objectBytes = append(*objectBytes, encodedInt.typeByte) - *objectBytes = append(*objectBytes, encodedInt.value...) + objectBytes = append(objectBytes, encodedInt.typeByte) + objectBytes = append(objectBytes, encodedInt.value...) case reflect.Int: encodedInt := encodeInt(int(elementValue.Int())) - *objectBytes = append(*objectBytes, encodedInt.typeByte) - *objectBytes = append(*objectBytes, encodedInt.value...) + objectBytes = append(objectBytes, encodedInt.typeByte) + objectBytes = append(objectBytes, encodedInt.value...) case reflect.Float32: encodedFloat := encodeFloat32(float32(elementValue.Float())) - *objectBytes = append(*objectBytes, encodedFloat.typeByte) - *objectBytes = append(*objectBytes, encodedFloat.value...) + objectBytes = append(objectBytes, encodedFloat.typeByte) + objectBytes = append(objectBytes, encodedFloat.value...) case reflect.Float64: encodedFloat := encodeFloat64(elementValue.Float()) - *objectBytes = append(*objectBytes, encodedFloat.typeByte) - *objectBytes = append(*objectBytes, encodedFloat.value...) + objectBytes = append(objectBytes, encodedFloat.typeByte) + objectBytes = append(objectBytes, encodedFloat.value...) case reflect.String: encodedString := encodeString(elementValue.String()) - *objectBytes = append(*objectBytes, encodedString.typeByte) - *objectBytes = append(*objectBytes, encodedString.size...) - *objectBytes = append(*objectBytes, encodedString.value...) + objectBytes = append(objectBytes, encodedString.typeByte) + objectBytes = append(objectBytes, encodedString.size...) + objectBytes = append(objectBytes, encodedString.value...) case reflect.Slice: encodedArray := encodeArray(elementValue.Interface().([]any)) - *objectBytes = append(*objectBytes, encodedArray.typeByte) - *objectBytes = append(*objectBytes, encodedArray.size...) - *objectBytes = append(*objectBytes, encodedArray.value...) + objectBytes = append(objectBytes, encodedArray.typeByte) + objectBytes = append(objectBytes, encodedArray.size...) + objectBytes = append(objectBytes, encodedArray.value...) case reflect.Map: encodedMap := encodeMap(elementValue.Interface().(map[any]any)) - *objectBytes = append(*objectBytes, encodedMap.typeByte) - *objectBytes = append(*objectBytes, encodedMap.size...) - *objectBytes = append(*objectBytes, encodedMap.value...) + objectBytes = append(objectBytes, encodedMap.typeByte) + objectBytes = append(objectBytes, encodedMap.size...) + objectBytes = append(objectBytes, encodedMap.value...) default: - *objectBytes = append(*objectBytes, Msgpack_Nil) + objectBytes = append(objectBytes, Msgpack_Nil) } + return objectBytes + +} + +func Deserialize(value []byte) any { + res, _ := deserialize_helper(value) + return res } + func Pack(obj interface{}) ([]byte, error) { t := reflect.TypeOf(obj) v := reflect.ValueOf(obj) @@ -607,21 +598,12 @@ func Pack(obj interface{}) ([]byte, error) { for i := 0; i < t.NumField(); i++ { field := t.Field(i) - memberType := field.Type.Kind() memberName := field.Name - memberValue := v.Field(i) - // fmt.Printf("struct member %d: %s %s %v \n", i, memberType, memberName, memberValue) - - objectBytes := make([]byte, 0) - serializeElement(&objectBytes, memberType, memberValue) - // fmt.Println("serialized:", objectBytes) + memberValue := v.Field(i).Interface() - // data, offset := deserializeElement(objectBytes) - // fmt.Println("deserialized:", data, offset) + objectBytes := Serialize(memberValue) - nameBytes := make([]byte, 0) - serializeElement(&nameBytes, reflect.String, reflect.ValueOf(memberName)) - // fmt.Println("name bytes:", nameBytes, memberName) + nameBytes := Serialize(memberName) bytes = append(bytes, nameBytes...) bytes = append(bytes, objectBytes...) @@ -630,7 +612,7 @@ func Pack(obj interface{}) ([]byte, error) { } func Unpack(bytes []byte, obj interface{}) (any, error) { - unpacked, _ := deserializeElement(bytes) + unpacked := Deserialize(bytes) v := reflect.ValueOf(obj).Elem() for key, val := range unpacked.(map[any]any) { diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go index be19c69..7d1a0eb 100644 --- a/pkg/msgpack_test.go +++ b/pkg/msgpack_test.go @@ -38,7 +38,7 @@ func TestPackString(t *testing.T) { t.Errorf("Expected size of string to be 200, got %d", str_size) } - str_value, _ := deserializeElement(data_bytes) + str_value:= Deserialize(data_bytes) if str_value != strings.Repeat("x", 200) { t.Errorf("Expected value of string to be %s, got %s", strings.Repeat("x", 200), str_value) } @@ -69,7 +69,7 @@ func TestPackString(t *testing.T) { t.Errorf("Expected size of string to be 260, got %d", str_size) } - str_value, _ := deserializeElement(data_bytes) + str_value:= Deserialize(data_bytes) if str_value != strings.Repeat("x", 260) { t.Errorf("Expected value of string to be %s, got %s", strings.Repeat("x", 260), str_value) } @@ -101,7 +101,7 @@ func TestPackString(t *testing.T) { t.Errorf("Expected size of string to be 70000, got %d", str_size) } - str_value, _ := deserializeElement(data_bytes) + str_value:= Deserialize(data_bytes) if str_value != strings.Repeat("x", 70000) { t.Errorf("Expected value of string to be %s, got %s", strings.Repeat("x", 70000), str_value) } @@ -131,7 +131,7 @@ func TestPackFloat(t *testing.T) { t.Errorf("Expected type of float to be 203 (float32), got %d", float_type) } - float_value, _ := deserializeElement(data_bytes) + float_value:= Deserialize(data_bytes) if reflect.TypeOf(float_value).Kind() != reflect.Float32 { t.Errorf("Expected type of float to be float32, got %v", reflect.TypeOf(float_value).Kind()) } @@ -155,7 +155,7 @@ func TestPackFloat(t *testing.T) { t.Errorf("Expected type of float to be 203 (float64), got %d", float_type) } - float_value, _ := deserializeElement(data_bytes) + float_value:= Deserialize(data_bytes) if reflect.TypeOf(float_value).Kind() != reflect.Float64 { t.Errorf("Expected type of float to be float64, got %v", reflect.TypeOf(float_value).Kind()) } @@ -187,7 +187,7 @@ func TestPackInt(t *testing.T) { t.Errorf("Expected type of int to be 208 (int8), got %d", int_type) } - int_value, _ := deserializeElement(data_bytes) + int_value := Deserialize(data_bytes) if reflect.TypeOf(int_value).Kind() != reflect.Int8 { t.Errorf("Expected type of int to be int8, got %v", reflect.TypeOf(int_value).Kind()) } @@ -210,7 +210,7 @@ func TestPackInt(t *testing.T) { t.Errorf("Expected type of int to be 209 (int16), got %d", int_type) } - int_value, _ := deserializeElement(data_bytes) + int_value := Deserialize(data_bytes) if reflect.TypeOf(int_value).Kind() != reflect.Int16 { t.Errorf("Expected type of int to be int16, got %v", reflect.TypeOf(int_value).Kind()) } @@ -232,7 +232,7 @@ func TestPackInt(t *testing.T) { t.Errorf("Expected type of int to be 210 (int32), got %d", int_type) } - int_value, _ := deserializeElement(data_bytes) + int_value := Deserialize(data_bytes) if reflect.TypeOf(int_value).Kind() != reflect.Int32 { t.Errorf("Expected type of int to be int32, got %v", reflect.TypeOf(int_value).Kind()) } @@ -255,7 +255,7 @@ func TestPackInt(t *testing.T) { t.Errorf("Expected type of int to be 211 (int64), got %d", int_type) } - int_value, _ := deserializeElement(data_bytes) + int_value:= Deserialize(data_bytes) if reflect.TypeOf(int_value).Kind() != reflect.Int64 { t.Errorf("Expected type of int to be int64, got %v", reflect.TypeOf(int_value).Kind()) } @@ -288,7 +288,7 @@ func TestPackUint(t *testing.T) { t.Errorf("Expected type of uint to be 204 (uint8), got %d", uint_type) } - uint_value, _ := deserializeElement(data_bytes) + uint_value:= Deserialize(data_bytes) if reflect.TypeOf(uint_value).Kind() != reflect.Uint8 { t.Errorf("Expected type of uint to be uint8, got %v", reflect.TypeOf(uint_value).Kind()) } @@ -311,7 +311,7 @@ func TestPackUint(t *testing.T) { t.Errorf("Expected type of uint to be 205 (uint16), got %d", uint_type) } - uint_value, _ := deserializeElement(data_bytes) + uint_value:= Deserialize(data_bytes) if reflect.TypeOf(uint_value).Kind() != reflect.Uint16 { t.Errorf("Expected type of uint to be uint16, got %v", reflect.TypeOf(uint_value).Kind()) } @@ -334,7 +334,7 @@ func TestPackUint(t *testing.T) { t.Errorf("Expected type of uint to be 206 (uint32), got %d", uint_type) } - uint_value, _ := deserializeElement(data_bytes) + uint_value:= Deserialize(data_bytes) if reflect.TypeOf(uint_value).Kind() != reflect.Uint32 { t.Errorf("Expected type of uint to be uint32, got %v", reflect.TypeOf(uint_value).Kind()) } @@ -356,7 +356,7 @@ func TestPackUint(t *testing.T) { t.Errorf("Expected type of uint to be 207 (uint64), got %d", uint_type) } - uint_value, _ := deserializeElement(data_bytes) + uint_value:= Deserialize(data_bytes) if reflect.TypeOf(uint_value).Kind() != reflect.Uint64 { t.Errorf("Expected type of uint to be uint64, got %v", reflect.TypeOf(uint_value).Kind()) } @@ -389,7 +389,7 @@ func TestPackArray(t *testing.T) { t.Errorf("Expected size of array to be 3, got %d", array_size) } - array_value, _ := deserializeElement(data_bytes) + array_value:= Deserialize(data_bytes) if reflect.TypeOf(array_value).Kind() != reflect.Slice { t.Errorf("Expected type of array to be slice, got %v", reflect.TypeOf(array_value).Kind()) } @@ -425,7 +425,7 @@ func TestPackArray(t *testing.T) { t.Errorf("Expected size of array to be 70000, got %d", array_size) } - array_value, _ := deserializeElement(data_bytes) + array_value:= Deserialize(data_bytes) if reflect.TypeOf(array_value).Kind() != reflect.Slice { t.Errorf("Expected type of array to be slice, got %v", reflect.TypeOf(array_value).Kind()) } @@ -456,7 +456,7 @@ func TestPackArray(t *testing.T) { t.Errorf("Expected size of array to be 3, got %d", array_size) } - array_value, _ := deserializeElement(data_bytes) + array_value:= Deserialize(data_bytes) if reflect.TypeOf(array_value).Kind() != reflect.Slice { t.Errorf("Expected type of array to be slice, got %v", reflect.TypeOf(array_value).Kind()) } @@ -500,7 +500,7 @@ func TestPackMap(t *testing.T) { t.Errorf("Expected size of map to be 2, got %d", map_size) } - map_value, _ := deserializeElement(data_bytes) + map_value := Deserialize(data_bytes) if reflect.TypeOf(map_value).Kind() != reflect.Map { t.Errorf("Expected type of map to be map, got %v", reflect.TypeOf(map_value).Kind()) } @@ -535,7 +535,7 @@ func TestPackMap(t *testing.T) { t.Errorf("Expected size of map to be 70000, got %d", map_size) } - map_value, _ := deserializeElement(data_bytes) + map_value:= Deserialize(data_bytes) if reflect.TypeOf(map_value).Kind() != reflect.Map { t.Errorf("Expected type of map to be map, got %v", reflect.TypeOf(map_value).Kind()) } @@ -566,7 +566,7 @@ func TestPackMap(t *testing.T) { t.Errorf("Expected size of map to be 0, got %d", map_size) } - map_value, _ := deserializeElement(data_bytes) + map_value := Deserialize(data_bytes) if reflect.TypeOf(map_value).Kind() != reflect.Map { t.Errorf("Expected type of map to be map, got %v", reflect.TypeOf(map_value).Kind()) } From a73c5c721200a4c6426a305e9af395e7ef6ad390 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Tue, 16 Sep 2025 17:49:46 +0300 Subject: [PATCH 14/19] refactor: used serialize instead of pack for unit tests --- cmd/main.go | 1 + pkg/msgpack_test.go | 342 ++++++++++++-------------------------------- 2 files changed, 92 insertions(+), 251 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5bf1ac7..1322df7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -25,6 +25,7 @@ func main() { Arr: arr, Mapp: m, } + packed, err := msgpack.Pack(object) if err != nil { fmt.Println(err) diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go index 7d1a0eb..b6f2b66 100644 --- a/pkg/msgpack_test.go +++ b/pkg/msgpack_test.go @@ -12,21 +12,11 @@ type User struct { Id string } -func TestPackString(t *testing.T) { - t.Run("test pack with string8", func(t *testing.T) { - user := User{ - Id: strings.Repeat("x", 200), - } - packed, err := Pack(user) - data_bytes := packed[15:] - - if err != nil { - t.Errorf("Expected nil, got %v", err) - } +func TestString(t *testing.T) { + t.Run("test serialize and deserialize with string8", func(t *testing.T) { + str := strings.Repeat("x", 200) - if len(data_bytes) != 202 { - t.Errorf("Expected size of data to be 202, got %d", len(data_bytes)) - } + data_bytes := Serialize(str) str_type := data_bytes[0] if str_type != 217 { @@ -38,26 +28,15 @@ func TestPackString(t *testing.T) { t.Errorf("Expected size of string to be 200, got %d", str_size) } - str_value:= Deserialize(data_bytes) + str_value := Deserialize(data_bytes) if str_value != strings.Repeat("x", 200) { t.Errorf("Expected value of string to be %s, got %s", strings.Repeat("x", 200), str_value) } }) - t.Run("test pack with string16", func(t *testing.T) { - user := User{ - Id: strings.Repeat("x", 260), - } - packed, err := Pack(user) - data_bytes := packed[15:] - - if err != nil { - t.Errorf("Expected nil, got %v", err) - } - - if len(data_bytes) != 263 { - t.Errorf("Expected size of data to be 263, got %d", len(data_bytes)) - } + t.Run("test serialize and deserialize with string16", func(t *testing.T) { + str := strings.Repeat("x", 260) + data_bytes := Serialize(str) str_type := data_bytes[0] if str_type != 218 { @@ -69,23 +48,16 @@ func TestPackString(t *testing.T) { t.Errorf("Expected size of string to be 260, got %d", str_size) } - str_value:= Deserialize(data_bytes) + str_value := Deserialize(data_bytes) if str_value != strings.Repeat("x", 260) { t.Errorf("Expected value of string to be %s, got %s", strings.Repeat("x", 260), str_value) } }) - t.Run("test pack with string32", func(t *testing.T) { - user := User{ - Id: strings.Repeat("x", 70000), - } - packed, err := Pack(user) - data_bytes := packed[15:] - - if err != nil { - t.Errorf("Expected nil, got %v", err) - } + t.Run("test serialize and deserialize with string32", func(t *testing.T) { + str := strings.Repeat("x", 70000) + data_bytes := Serialize(str) if len(data_bytes) != 70005 { t.Errorf("Expected size of data to be 70005, got %d", len(data_bytes)) @@ -101,86 +73,58 @@ func TestPackString(t *testing.T) { t.Errorf("Expected size of string to be 70000, got %d", str_size) } - str_value:= Deserialize(data_bytes) + str_value := Deserialize(data_bytes) if str_value != strings.Repeat("x", 70000) { t.Errorf("Expected value of string to be %s, got %s", strings.Repeat("x", 70000), str_value) } }) } -func TestPackFloat(t *testing.T) { - type Object struct { - Fnum32 float32 - Fnum64 float64 - } - t.Run("test pack with float32", func(t *testing.T) { - - obj := Object{ - Fnum32: 3.14, - } - - packed, err := Pack(obj) - data_bytes := packed[3+8 : 16] - - if err != nil { - t.Errorf("Expected nil, got %v", err) - } +func TestFloat(t *testing.T) { + t.Run("test serialize and deserialize with float32", func(t *testing.T) { + fnum32 := float32(3.14) + data_bytes := Serialize(fnum32) float_type := data_bytes[0] if float_type != 202 { t.Errorf("Expected type of float to be 203 (float32), got %d", float_type) } - float_value:= Deserialize(data_bytes) + float_value := Deserialize(data_bytes) if reflect.TypeOf(float_value).Kind() != reflect.Float32 { t.Errorf("Expected type of float to be float32, got %v", reflect.TypeOf(float_value).Kind()) } - }) - - t.Run("test pack with float64", func(t *testing.T) { - obj := Object{ - Fnum64: 3.14, + if float_value != fnum32 { + t.Errorf("Expected value of float to be %f, got %f", fnum32, float_value) } - packed, err := Pack(obj) - data_bytes := packed[16+8:] + }) - if err != nil { - t.Errorf("Expected nil, got %v", err) - } + t.Run("test serialize and deserialize with float64", func(t *testing.T) { + fnum64 := float64(3.14) + data_bytes := Serialize(fnum64) float_type := data_bytes[0] if float_type != 203 { t.Errorf("Expected type of float to be 203 (float64), got %d", float_type) } - float_value:= Deserialize(data_bytes) + float_value := Deserialize(data_bytes) if reflect.TypeOf(float_value).Kind() != reflect.Float64 { t.Errorf("Expected type of float to be float64, got %v", reflect.TypeOf(float_value).Kind()) } - }) -} - -func TestPackInt(t *testing.T) { - type Object struct { - Snum8 int8 - Snum16 int16 - Snum32 int32 - Snum64 int64 - } - t.Run("test pack with int8", func(t *testing.T) { - obj := Object{ - Snum8: 3, + if float_value != fnum64 { + t.Errorf("Expected value of float to be %f, got %f", fnum64, float_value) } + }) +} - packed, err := Pack(obj) - data_bytes := packed[3+2+5 : 10+2] - - if err != nil { - t.Errorf("Expected nil, got %v", err) - } +func TestInt(t *testing.T) { + t.Run("test serialize and deserialize with int8", func(t *testing.T) { + snum8 := int8(-100) + data_bytes := Serialize(snum8) int_type := data_bytes[0] if int_type != 208 { @@ -191,19 +135,14 @@ func TestPackInt(t *testing.T) { if reflect.TypeOf(int_value).Kind() != reflect.Int8 { t.Errorf("Expected type of int to be int8, got %v", reflect.TypeOf(int_value).Kind()) } - }) - - t.Run("test pack with int16", func(t *testing.T) { - obj := Object{ - Snum16: -300, + if int_value != snum8 { + t.Errorf("Expected value of int to be %d, got %d", snum8, int_value) } + }) - packed, err := Pack(obj) - data_bytes := packed[12+2+6 : 20+3] - - if err != nil { - t.Errorf("Expected nil, got %v", err) - } + t.Run("test serialize and deserialize with int16", func(t *testing.T) { + snum16 := int16(300) + data_bytes := Serialize(snum16) int_type := data_bytes[0] if int_type != 209 { @@ -214,19 +153,15 @@ func TestPackInt(t *testing.T) { if reflect.TypeOf(int_value).Kind() != reflect.Int16 { t.Errorf("Expected type of int to be int16, got %v", reflect.TypeOf(int_value).Kind()) } - }) - - t.Run("test pack with int32", func(t *testing.T) { - obj := Object{ - Snum32: 70000, - } - packed, err := Pack(obj) - data_bytes := packed[23+2+6 : 31+5] - if err != nil { - t.Errorf("Expected nil, got %v", err) + if int_value != snum16 { + t.Errorf("Expected value of int to be %d, got %d", snum16, int_value) } + }) + t.Run("test serialize and deserialize with int32", func(t *testing.T) { + snum32 := int32(70000) + data_bytes := Serialize(snum32) int_type := data_bytes[0] if int_type != 210 { t.Errorf("Expected type of int to be 210 (int32), got %d", int_type) @@ -236,148 +171,97 @@ func TestPackInt(t *testing.T) { if reflect.TypeOf(int_value).Kind() != reflect.Int32 { t.Errorf("Expected type of int to be int32, got %v", reflect.TypeOf(int_value).Kind()) } - }) - t.Run("test pack with int64", func(t *testing.T) { - obj := Object{ - Snum64: 5000000000, - } - - packed, err := Pack(obj) - data_bytes := packed[36+2+6:] - - if err != nil { - t.Errorf("Expected nil, got %v", err) + if int_value != snum32 { + t.Errorf("Expected value of int to be %d, got %d", snum32, int_value) } + }) + t.Run("test serialize and deserialize with int64", func(t *testing.T) { + snum64 := int64(100) + data_bytes := Serialize(snum64) int_type := data_bytes[0] if int_type != 211 { t.Errorf("Expected type of int to be 211 (int64), got %d", int_type) } - int_value:= Deserialize(data_bytes) + int_value := Deserialize(data_bytes) if reflect.TypeOf(int_value).Kind() != reflect.Int64 { t.Errorf("Expected type of int to be int64, got %v", reflect.TypeOf(int_value).Kind()) } + + if int_value != snum64 { + t.Errorf("Expected value of int to be %d, got %d", snum64, int_value) + } }) } -func TestPackUint(t *testing.T) { - type Object struct { - Unum8 uint8 - Unum16 uint16 - Unum32 uint32 - Unum64 uint64 - } - - t.Run("test pack with uint8", func(t *testing.T) { - obj := Object{ - Unum8: 250, - } - - packed, err := Pack(obj) - data_bytes := packed[3+2+5 : 10+2] +func TestUint(t *testing.T) { - if err != nil { - t.Errorf("Expected nil, got %v", err) - } + t.Run("test serialize and deserialize with uint8", func(t *testing.T) { + unum8 := uint8(100) + data_bytes := Serialize(unum8) uint_type := data_bytes[0] if uint_type != 204 { t.Errorf("Expected type of uint to be 204 (uint8), got %d", uint_type) } - uint_value:= Deserialize(data_bytes) + uint_value := Deserialize(data_bytes) if reflect.TypeOf(uint_value).Kind() != reflect.Uint8 { t.Errorf("Expected type of uint to be uint8, got %v", reflect.TypeOf(uint_value).Kind()) } }) - t.Run("test pack with uint16", func(t *testing.T) { - obj := Object{ - Unum16: 700, - } - - packed, err := Pack(obj) - data_bytes := packed[12+2+6 : 20+3] - - if err != nil { - t.Errorf("Expected nil, got %v", err) - } + t.Run("test serialize and deserialize with uint16", func(t *testing.T) { + unum16 := uint16(300) + data_bytes := Serialize(unum16) uint_type := data_bytes[0] if uint_type != 205 { t.Errorf("Expected type of uint to be 205 (uint16), got %d", uint_type) } - uint_value:= Deserialize(data_bytes) + uint_value := Deserialize(data_bytes) if reflect.TypeOf(uint_value).Kind() != reflect.Uint16 { t.Errorf("Expected type of uint to be uint16, got %v", reflect.TypeOf(uint_value).Kind()) } }) - t.Run("test pack with uint32", func(t *testing.T) { - obj := Object{ - Unum32: 70000, - } - - packed, err := Pack(obj) - data_bytes := packed[23+2+6 : 31+5] - - if err != nil { - t.Errorf("Expected nil, got %v", err) - } - + t.Run("test serialize and deserialize with uint32", func(t *testing.T) { + unum32 := uint32(70000) + data_bytes := Serialize(unum32) uint_type := data_bytes[0] if uint_type != 206 { t.Errorf("Expected type of uint to be 206 (uint32), got %d", uint_type) } - uint_value:= Deserialize(data_bytes) + uint_value := Deserialize(data_bytes) if reflect.TypeOf(uint_value).Kind() != reflect.Uint32 { t.Errorf("Expected type of uint to be uint32, got %v", reflect.TypeOf(uint_value).Kind()) } }) - t.Run("test pack with uint64", func(t *testing.T) { - obj := Object{ - Unum64: 5000000000, - } - - packed, err := Pack(obj) - data_bytes := packed[36+2+6:] - if err != nil { - t.Errorf("Expected nil, got %v", err) - } - + t.Run("test serialize and deserialize with uint64", func(t *testing.T) { + unum64 := uint64(100) + data_bytes := Serialize(unum64) uint_type := data_bytes[0] if uint_type != 207 { t.Errorf("Expected type of uint to be 207 (uint64), got %d", uint_type) } - uint_value:= Deserialize(data_bytes) + uint_value := Deserialize(data_bytes) if reflect.TypeOf(uint_value).Kind() != reflect.Uint64 { t.Errorf("Expected type of uint to be uint64, got %v", reflect.TypeOf(uint_value).Kind()) } }) } -func TestPackArray(t *testing.T) { - type Object struct { - Arr []any - } - t.Run("test pack with array16", func(t *testing.T) { +func TestArray(t *testing.T) { + t.Run("test serialize and deserialize with array16", func(t *testing.T) { arr := []any{1.1, 2.2, 3.3} - obj := Object{ - Arr: arr, - } - packed, err := Pack(obj) - if err != nil { - t.Errorf("Expected nil, got %v", err) - } - data_bytes := packed[3+5:] - + data_bytes := Serialize(arr) array_type := data_bytes[0] if array_type != 220 { @@ -389,7 +273,7 @@ func TestPackArray(t *testing.T) { t.Errorf("Expected size of array to be 3, got %d", array_size) } - array_value:= Deserialize(data_bytes) + array_value := Deserialize(data_bytes) if reflect.TypeOf(array_value).Kind() != reflect.Slice { t.Errorf("Expected type of array to be slice, got %v", reflect.TypeOf(array_value).Kind()) } @@ -399,21 +283,13 @@ func TestPackArray(t *testing.T) { }) - t.Run("test pack with array32", func(t *testing.T) { + t.Run("test serialize and deserialize with array32", func(t *testing.T) { arr := make([]any, 70000) for i := range arr { arr[i] = true } - obj := Object{ - Arr: arr, - } - packed, err := Pack(obj) - if err != nil { - t.Errorf("Expected nil, got %v", err) - } - - data_bytes := packed[3+5:] + data_bytes := Serialize(arr) array_type := data_bytes[0] if array_type != 221 { @@ -425,7 +301,7 @@ func TestPackArray(t *testing.T) { t.Errorf("Expected size of array to be 70000, got %d", array_size) } - array_value:= Deserialize(data_bytes) + array_value := Deserialize(data_bytes) if reflect.TypeOf(array_value).Kind() != reflect.Slice { t.Errorf("Expected type of array to be slice, got %v", reflect.TypeOf(array_value).Kind()) } @@ -434,17 +310,9 @@ func TestPackArray(t *testing.T) { } }) - t.Run("test pack with empty array", func(t *testing.T) { + t.Run("test serialize and deserialize with empty array", func(t *testing.T) { arr := make([]any, 3) - obj := Object{ - Arr: arr, - } - packed, err := Pack(obj) - if err != nil { - t.Errorf("Expected nil, got %v", err) - } - data_bytes := packed[3+5:] - + data_bytes := Serialize(arr) array_type := data_bytes[0] if array_type != 220 { @@ -456,7 +324,7 @@ func TestPackArray(t *testing.T) { t.Errorf("Expected size of array to be 3, got %d", array_size) } - array_value:= Deserialize(data_bytes) + array_value := Deserialize(data_bytes) if reflect.TypeOf(array_value).Kind() != reflect.Slice { t.Errorf("Expected type of array to be slice, got %v", reflect.TypeOf(array_value).Kind()) } @@ -470,25 +338,12 @@ func TestPackArray(t *testing.T) { } -func TestPackMap(t *testing.T) { - type Object struct { - Dict map[any]any - } - - t.Run("test pack with map16", func(t *testing.T) { +func TestMap(t *testing.T) { + t.Run("test serialize and deserialize with map16", func(t *testing.T) { m := make(map[any]any) m["Lang"] = "Go" m["Ver"] = "1.19" - - obj := Object{ - Dict: m, - } - packed, err := Pack(obj) - if err != nil { - t.Errorf("Expected nil, got %v", err) - } - - data_bytes := packed[3+6:] + data_bytes := Serialize(m) map_type := data_bytes[0] if map_type != 222 { @@ -510,20 +365,12 @@ func TestPackMap(t *testing.T) { } }) - t.Run("test pack with map32", func(t *testing.T) { + t.Run("test serialize and deserialize with map32", func(t *testing.T) { m := make(map[any]any, 70000) for i := 0; i < 70000; i++ { m[fmt.Sprintf("k%d", i)] = fmt.Sprintf("v%d", i) } - obj := Object{ - Dict: m, - } - packed, err := Pack(obj) - if err != nil { - t.Errorf("Expected nil, got %v", err) - } - - data_bytes := packed[3+6:] + data_bytes := Serialize(m) map_type := data_bytes[0] if map_type != 223 { @@ -535,7 +382,7 @@ func TestPackMap(t *testing.T) { t.Errorf("Expected size of map to be 70000, got %d", map_size) } - map_value:= Deserialize(data_bytes) + map_value := Deserialize(data_bytes) if reflect.TypeOf(map_value).Kind() != reflect.Map { t.Errorf("Expected type of map to be map, got %v", reflect.TypeOf(map_value).Kind()) } @@ -545,16 +392,9 @@ func TestPackMap(t *testing.T) { } }) - t.Run("test pack with empty map", func(t *testing.T) { + t.Run("test serialize and deserialize with empty map", func(t *testing.T) { m := make(map[any]any) - obj := Object{ - Dict: m, - } - packed, err := Pack(obj) - if err != nil { - t.Errorf("Expected nil, got %v", err) - } - data_bytes := packed[3+6:] + data_bytes := Serialize(m) map_type := data_bytes[0] if map_type != 222 { From 0e22e50c421cf68f5e64dcfaa0790308e2262593 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Tue, 16 Sep 2025 18:17:07 +0300 Subject: [PATCH 15/19] docs: added functions documentation --- README.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++++- pkg/msgpack.go | 8 +++--- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ff3a23b..bf58611 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,66 @@ # Msgpack-Fatma-Ebrahim -a Go package that implements MessagePack serialization and deserialization. +a Go library that implements MessagePack serialization and deserialization. + +Check this link to learn more: https://github.com/msgpack/msgpack/blob/master/spec.md + +## Functions: +### `Serialize(elementVal any) []byte` +serializes a single element into a byte array + +### `Deserialize(value []byte) any` +deserializes a byte array into a single element + +### `Pack(obj interface{}) ([]byte, error)` +packs an object into a byte array + +### `Unpack(bytes []byte, obj interface{}) (any, error)` +unpacks a byte array into an object + + +## How to Use: +### Step 1: Install the library using `go get` + + ```bash + go get github.com/codescalersinternships/Msgpack-Fatma-Ebrahim + ``` + +This command fetches the library and adds it to your project's `go.mod` file. + +### Step 2: Import and use the library in your code + + After running `go get`, you can import the library into your project and use the functions as described: + + +``` +package main + +import ( + "fmt" + msgpack "github.com/codescalersinternships/Msgpack-Fatma-Ebrahim/pkg" +) + +type Object struct { + IsObject bool + Flag bool +} + +func main() { + object := Object{ + IsObject: true, + Flag: false, + } + + packed, err := msgpack.Pack(object) + if err != nil { + fmt.Println(err) + } + + var obj Object + _, err = msgpack.Unpack(packed, &obj) + if err != nil { + fmt.Println(err) + } + + fmt.Printf("Object: %+v\n", obj) +} +``` diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 1be7819..6a05c44 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -372,8 +372,6 @@ func decodeSize(value []byte) int { return 0 } - - func decodeArray16(value []byte) ([]any, int) { size := decodeSize(value[:2]) data := value[2:] @@ -438,7 +436,6 @@ func decodeMap32(value []byte) (map[any]any, int) { return m, overallOffset + 5 } - func deserialize_helper(value []byte) (any, int) { elementType := value[0] switch elementType { @@ -487,6 +484,7 @@ func deserialize_helper(value []byte) (any, int) { return nil, 0 } +// a function that serializes an element into a byte array func Serialize(elementVal any) []byte { elementValue := reflect.ValueOf(elementVal) objectBytes := make([]byte, 0) @@ -574,12 +572,13 @@ func Serialize(elementVal any) []byte { } +// a function that deserializes a byte array into an element func Deserialize(value []byte) any { res, _ := deserialize_helper(value) return res } - +// a function that packs an object into a byte array func Pack(obj interface{}) ([]byte, error) { t := reflect.TypeOf(obj) v := reflect.ValueOf(obj) @@ -611,6 +610,7 @@ func Pack(obj interface{}) ([]byte, error) { return bytes, nil } +// a function that unpacks a byte array into an object func Unpack(bytes []byte, obj interface{}) (any, error) { unpacked := Deserialize(bytes) From 11ac409435b781cf4b1d24ccbacdfd6bd7a7c944 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Wed, 17 Sep 2025 12:57:38 +0300 Subject: [PATCH 16/19] fix: handled typed mapped in serialize and deserialize --- cmd/main.go | 19 +++++++++---------- pkg/msgpack.go | 28 +++++++++++++++++++--------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 1322df7..1ea9aef 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,10 @@ func main() { m["hello"] = 10 m["world"] = -20 + typed_map := make(map[float32]int16) + typed_map[1.1] = 1 + typed_map[2.2] = 2 + object := msgpack.Object{ IsObject: true, Flag: false, @@ -24,8 +28,9 @@ func main() { Str: "Fatma", Arr: arr, Mapp: m, + Typed: typed_map, } - + packed, err := msgpack.Pack(object) if err != nil { fmt.Println(err) @@ -37,19 +42,13 @@ func main() { if err != nil { fmt.Println(err) } - fmt.Printf("Unpacked: %+v\n", unpacked) - fmt.Printf("Object: %+v\n", obj) + unpacked_val := unpacked.(map[any]any) + fmt.Printf("Unpacked: %+v\n", unpacked_val["Typed"]) - ser := msgpack.Serialize(m) - if err != nil { - fmt.Println(err) - } + ser := msgpack.Serialize(typed_map) fmt.Printf("Serialized: %+v\n", ser) des := msgpack.Deserialize(ser) - if err != nil { - fmt.Println(err) - } fmt.Printf("Deserialized: %+v\n", des) } diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 6a05c44..20eb602 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -85,6 +85,7 @@ type Object struct { Str string Arr []any Mapp map[any]any + Typed map[float32]int16 } func encodeBool(value bool) byte { @@ -249,22 +250,29 @@ func encodeArray(arr []any) Array { } } -func encodeMap(m map[any]any) Map { +func encodeMap(m any) Map { + map_value := reflect.ValueOf(m) + map_size := map_value.Len() map_16 := int(math.Pow(2, 16) - 1) map_32 := int(math.Pow(2, 32) - 1) + bytes := make([]byte, 0) var size []byte var t byte - if len(m) <= map_16 { - size = encodeSize(len(m)) + if map_size <= map_16 { + size = encodeSize(map_size) t = Msgpack_Map_16 - } else if len(m) <= map_32 { - size = encodeSize(len(m)) + } else if map_size <= map_32 { + size = encodeSize(map_size) t = Msgpack_Map_32 } - for key, val := range m { + iter := map_value.MapRange() + for iter.Next() { + key := iter.Key().Interface() + val := iter.Value().Interface() + objectBytes := Serialize(key) bytes = append(bytes, objectBytes...) @@ -402,7 +410,7 @@ func decodeArray32(value []byte) ([]any, int) { return array, overallOffset + 5 } -func decodeMap16(value []byte) (map[any]any, int) { +func decodeMap16(value []byte) (any, int) { size := decodeSize(value[:2]) data := value[2:] m := make(map[any]any) @@ -411,6 +419,7 @@ func decodeMap16(value []byte) (map[any]any, int) { for i := 0; i < size; i++ { key, offset := deserialize_helper(data) data = data[offset:] + overallOffset += offset value, offset := deserialize_helper(data) data = data[offset:] m[key] = value @@ -419,7 +428,7 @@ func decodeMap16(value []byte) (map[any]any, int) { return m, overallOffset + 3 } -func decodeMap32(value []byte) (map[any]any, int) { +func decodeMap32(value []byte) (any, int) { size := decodeSize(value[:4]) data := value[4:] m := make(map[any]any) @@ -428,6 +437,7 @@ func decodeMap32(value []byte) (map[any]any, int) { for i := 0; i < size; i++ { key, offset := deserialize_helper(data) data = data[offset:] + overallOffset += offset value, offset := deserialize_helper(data) data = data[offset:] m[key] = value @@ -559,7 +569,7 @@ func Serialize(elementVal any) []byte { objectBytes = append(objectBytes, encodedArray.value...) case reflect.Map: - encodedMap := encodeMap(elementValue.Interface().(map[any]any)) + encodedMap := encodeMap(elementValue.Interface()) objectBytes = append(objectBytes, encodedMap.typeByte) objectBytes = append(objectBytes, encodedMap.size...) objectBytes = append(objectBytes, encodedMap.value...) From 4f9e2e27d7ffa8218d74c371e33dda902494ba96 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Wed, 17 Sep 2025 13:52:50 +0300 Subject: [PATCH 17/19] fix: handled typed map in unpack --- cmd/main.go | 2 ++ pkg/msgpack.go | 30 ++++++++++++++++++++++++++---- pkg/msgpack_test.go | 13 +++++++++---- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 1ea9aef..97da8a3 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -45,6 +45,8 @@ func main() { unpacked_val := unpacked.(map[any]any) fmt.Printf("Unpacked: %+v\n", unpacked_val["Typed"]) + fmt.Printf("object: %+v\n", obj) + ser := msgpack.Serialize(typed_map) fmt.Printf("Serialized: %+v\n", ser) diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 20eb602..110bd13 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -420,9 +420,9 @@ func decodeMap16(value []byte) (any, int) { key, offset := deserialize_helper(data) data = data[offset:] overallOffset += offset - value, offset := deserialize_helper(data) + val, offset := deserialize_helper(data) data = data[offset:] - m[key] = value + m[key] = val overallOffset += offset } return m, overallOffset + 3 @@ -438,9 +438,9 @@ func decodeMap32(value []byte) (any, int) { key, offset := deserialize_helper(data) data = data[offset:] overallOffset += offset - value, offset := deserialize_helper(data) + val, offset := deserialize_helper(data) data = data[offset:] - m[key] = value + m[key] = val overallOffset += offset } return m, overallOffset + 5 @@ -620,14 +620,36 @@ func Pack(obj interface{}) ([]byte, error) { return bytes, nil } +func convertMap(m map[any]any, key_type reflect.Type, value_type reflect.Type) any { + new_map := reflect.MakeMap(reflect.MapOf(key_type, value_type)) + + for key, val := range m { + new_map.SetMapIndex(reflect.ValueOf(key), reflect.ValueOf(val)) + } + return new_map.Interface() +} + // a function that unpacks a byte array into an object func Unpack(bytes []byte, obj interface{}) (any, error) { unpacked := Deserialize(bytes) v := reflect.ValueOf(obj).Elem() + for key, val := range unpacked.(map[any]any) { field := v.FieldByName(key.(string)) + packed_map_type := reflect.TypeOf(val) + object_map_type := reflect.TypeOf(field.Interface()) + + if field.Kind() == reflect.Map && object_map_type != packed_map_type { + object_map_key_type := object_map_type.Key() + object_map_val_type := object_map_type.Elem() + new_map := convertMap(val.(map[any]any), object_map_key_type, object_map_val_type) + field.Set(reflect.ValueOf(new_map)) + continue + } + field.Set(reflect.ValueOf(val)) + } return unpacked, nil diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go index b6f2b66..e9f3af2 100644 --- a/pkg/msgpack_test.go +++ b/pkg/msgpack_test.go @@ -474,9 +474,14 @@ func TestAll(t *testing.T) { t.Errorf("Expected nil, got %v", err) } - if !reflect.DeepEqual(obj_to_pack, obj_unpacked) { - t.Errorf("Expected %+v, got %+v", obj_to_pack, obj_unpacked) + packed_unpacked, err := Pack(obj_unpacked) + if err != nil { + t.Errorf("Expected nil, got %v", err) } + if !reflect.DeepEqual(packed, packed_unpacked) { + t.Errorf("Expected %v, got %v", packed, packed_unpacked) + } + }) type ComplexUser struct { @@ -486,7 +491,7 @@ func TestAll(t *testing.T) { Balance float64 IsActive bool Tags []any - Metadata map[any]any + Metadata map[string]any } t.Run("test pack and unpack with more complex struct", func(t *testing.T) { @@ -497,7 +502,7 @@ func TestAll(t *testing.T) { Balance: 1050.75, IsActive: true, Tags: []any{"golang", "backend", "testing"}, - Metadata: map[any]any{ + Metadata: map[string]any{ "country": "Egypt", "city": "Cairo", }, From a11bb3bc601b72ff5a0bdb394a4aeef2a5d05138 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Wed, 17 Sep 2025 18:10:38 +0300 Subject: [PATCH 18/19] feat: handled binary types --- cmd/main.go | 10 ++-- pkg/msgpack.go | 101 ++++++++++++++++++++++++++++++++++----- pkg/msgpack_test.go | 112 +++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 202 insertions(+), 21 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 97da8a3..f47a3be 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,10 +6,10 @@ import ( ) func main() { - arr := make([]any, 3) - arr[0] = "hello" + arr := make([]byte, 3) + arr[0] = 0 arr[1] = 7 - arr[2] = 9.8 + arr[2] = 9 m := make(map[any]any) m["hello"] = 10 @@ -47,7 +47,9 @@ func main() { fmt.Printf("object: %+v\n", obj) - ser := msgpack.Serialize(typed_map) + bytes := make([]byte, 3) + + ser := msgpack.Serialize(bytes) fmt.Printf("Serialized: %+v\n", ser) des := msgpack.Deserialize(ser) diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 110bd13..04f27e0 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -10,6 +10,10 @@ const Msgpack_Nil = 0xc0 //192 const Msgpack_False = 0xc2 //194 const Msgpack_True = 0xc3 //195 +const Msgpack_Bin_8 = 0xc4 //196 +const Msgpack_Bin_16 = 0xc5 //197 +const Msgpack_Bin_32 = 0xc6 //198 + const Msgpack_Uint_8 = 0xcc //204 const Msgpack_Uint_16 = 0xcd //205 const Msgpack_Uint_32 = 0xce //206 @@ -83,7 +87,7 @@ type Object struct { Snum int16 Fnum float32 Str string - Arr []any + Arr []byte Mapp map[any]any Typed map[float32]int16 } @@ -219,22 +223,46 @@ func encodeString(value string) String { } } -func encodeArray(arr []any) Array { +func encodeArray(arr any) Array { + array_value := reflect.ValueOf(arr) + array_size := array_value.Len() + + array_8 := int(math.Pow(2, 8) - 1) array_16 := int(math.Pow(2, 16) - 1) array_32 := int(math.Pow(2, 32) - 1) bytes := make([]byte, 0) var size []byte var t byte - - if len(arr) <= array_16 { - size = encodeSize(len(arr)) + bin_flag := false + + if array_size != 0 { + element := array_value.Index(0).Interface() + if element != nil { + if reflect.TypeOf(element).Kind() == reflect.Uint8 { + bin_flag = true + } + } + } + if array_size <= array_16 && !bin_flag { + size = encodeSize(array_size) t = Msgpack_Array_16 - } else if len(arr) <= array_32 { - size = encodeSize(len(arr)) + } else if array_size <= array_32 && !bin_flag { + size = encodeSize(array_size) t = Msgpack_Array_32 - } - - for _, element := range arr { + } else if array_size <= array_8 && bin_flag { + size = make([]byte, 0) + size = append(size, byte(array_size)) + t = Msgpack_Bin_8 + } else if array_size <= array_16 && bin_flag { + size = encodeSize(array_size) + t = Msgpack_Bin_16 + } else if array_size <= array_32 && bin_flag { + size = encodeSize(array_size) + t = Msgpack_Bin_32 + } + + for i := 0; i < array_size; i++ { + element := array_value.Index(i).Interface() if element == nil { bytes = append(bytes, Msgpack_Nil) continue @@ -446,6 +474,51 @@ func decodeMap32(value []byte) (any, int) { return m, overallOffset + 5 } +func decodeBin8(value []byte) ([]byte, int) { + size := int(value[0]) + data := value[1:] + array := make([]byte, size) + overallOffset := 0 + for i := 0; i < size; i++ { + element, offset := deserialize_helper(data) + array[i] = element.(byte) + data = data[offset:] + overallOffset += offset + } + + return array, overallOffset + 2 +} + +func decodeBin16(value []byte) ([]byte, int) { + size := decodeSize(value[:2]) + data := value[2:] + array := make([]byte, size) + overallOffset := 0 + for i := 0; i < size; i++ { + element, offset := deserialize_helper(data) + array[i] = element.(byte) + data = data[offset:] + overallOffset += offset + } + + return array, overallOffset + 3 +} + +func decodeBin32(value []byte) ([]byte, int) { + size := decodeSize(value[:4]) + data := value[4:] + array := make([]byte, size) + overallOffset := 0 + for i := 0; i < size; i++ { + element, offset := deserialize_helper(data) + array[i] = element.(byte) + data = data[offset:] + overallOffset += offset + } + + return array, overallOffset + 5 +} + func deserialize_helper(value []byte) (any, int) { elementType := value[0] switch elementType { @@ -487,6 +560,12 @@ func deserialize_helper(value []byte) (any, int) { return decodeMap16(value[1:]) case Msgpack_Map_32: return decodeMap32(value[1:]) + case Msgpack_Bin_8: + return decodeBin8(value[1:]) + case Msgpack_Bin_16: + return decodeBin16(value[1:]) + case Msgpack_Bin_32: + return decodeBin32(value[1:]) case Msgpack_Nil: return nil, 1 } @@ -563,7 +642,7 @@ func Serialize(elementVal any) []byte { objectBytes = append(objectBytes, encodedString.value...) case reflect.Slice: - encodedArray := encodeArray(elementValue.Interface().([]any)) + encodedArray := encodeArray(elementValue.Interface()) objectBytes = append(objectBytes, encodedArray.typeByte) objectBytes = append(objectBytes, encodedArray.size...) objectBytes = append(objectBytes, encodedArray.value...) diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go index e9f3af2..f406ea2 100644 --- a/pkg/msgpack_test.go +++ b/pkg/msgpack_test.go @@ -265,7 +265,7 @@ func TestArray(t *testing.T) { array_type := data_bytes[0] if array_type != 220 { - t.Errorf("Expected type of array to be 220 (array), got %d", array_type) + t.Errorf("Expected type of array to be 220 (array16), got %d", array_type) } array_size := decodeSize(data_bytes[1:3]) @@ -316,7 +316,7 @@ func TestArray(t *testing.T) { array_type := data_bytes[0] if array_type != 220 { - t.Errorf("Expected type of array to be 220 (array), got %d", array_type) + t.Errorf("Expected type of array to be 220 (array16), got %d", array_type) } array_size := decodeSize(data_bytes[1:3]) @@ -338,6 +338,106 @@ func TestArray(t *testing.T) { } +func TestBinary(t *testing.T) { + t.Run("test serialize and deserialize with binary8", func(t *testing.T) { + arr := []byte{1, 2, 3} + data_bytes := Serialize(arr) + bin_type := data_bytes[0] + + if bin_type != 196 { + t.Errorf("Expected type of binary to be 196 (binary8), got %d", bin_type) + } + + bin_size := int(data_bytes[1]) + if bin_size != 3 { + t.Errorf("Expected size of binary to be 3, got %d", bin_size) + } + + bin_value := Deserialize(data_bytes) + if reflect.TypeOf(bin_value).Kind() != reflect.Slice { + t.Errorf("Expected type of binary to be slice, got %v", reflect.TypeOf(bin_value).Kind()) + } + if !reflect.DeepEqual(bin_value, arr) { + t.Errorf("Expected binary to be %v, got %v", arr, bin_value) + } + + }) + + t.Run("test serialize and deserialize with binary16", func(t *testing.T) { + arr := make([]byte, 500) + data_bytes := Serialize(arr) + bin_type := data_bytes[0] + + if bin_type != 197 { + t.Errorf("Expected type of binary to be 197 (binary16), got %d", bin_type) + } + + bin_size := decodeSize(data_bytes[1:3]) + if bin_size != 500 { + t.Errorf("Expected size of binary to be 500, got %d", bin_size) + } + + bin_value := Deserialize(data_bytes) + if reflect.TypeOf(bin_value).Kind() != reflect.Slice { + t.Errorf("Expected type of binary to be slice, got %v", reflect.TypeOf(bin_value).Kind()) + } + if !reflect.DeepEqual(bin_value, arr) { + t.Errorf("Expected binary to be %v, got %v", arr, bin_value) + } + + }) + + t.Run("test serialize and deserialize with binary32", func(t *testing.T) { + arr := make([]byte, 70000) + data_bytes := Serialize(arr) + bin_type := data_bytes[0] + + if bin_type != 198 { + t.Errorf("Expected type of binary to be 198 (binary32), got %d", bin_type) + } + + bin_size := decodeSize(data_bytes[1:5]) + if bin_size != 70000 { + t.Errorf("Expected size of binary to be 70000, got %d", bin_size) + } + + bin_value := Deserialize(data_bytes) + if reflect.TypeOf(bin_value).Kind() != reflect.Slice { + t.Errorf("Expected type of binary to be slice, got %v", reflect.TypeOf(bin_value).Kind()) + } + if !reflect.DeepEqual(bin_value, arr) { + t.Errorf("Expected binary to be %v, got %v", arr, bin_value) + } + + }) + + t.Run("test serialize and deserialize with empty binary", func(t *testing.T) { + arr := make([]byte, 1) + data_bytes := Serialize(arr) + bin_type := data_bytes[0] + + if bin_type != 196 { + t.Errorf("Expected type of binary to be 196 (binary8), got %d", bin_type) + } + + bin_size := int(data_bytes[1]) + if bin_size != 1 { + t.Errorf("Expected size of binary to be 1, got %d", bin_size) + } + + bin_value := Deserialize(data_bytes) + if reflect.TypeOf(bin_value).Kind() != reflect.Slice { + t.Errorf("Expected type of binary to be slice, got %v", reflect.TypeOf(bin_value).Kind()) + } + + bin := bin_value.([]byte) + if bin[0] != 0 { + t.Errorf("Expected binary element to be 0, got %v", bin[0]) + } + + }) +} + func TestMap(t *testing.T) { t.Run("test serialize and deserialize with map16", func(t *testing.T) { m := make(map[any]any) @@ -443,10 +543,10 @@ func TestAll(t *testing.T) { }) t.Run("test pack and unpack with complex struct", func(t *testing.T) { - arr := make([]any, 3) - arr[0] = "hello" - arr[1] = -7.7 - arr[2] = 9.8 + arr := make([]byte, 3) + arr[0] = 1 + arr[1] = 2 + arr[2] = 3 m := make(map[any]any) m["hello"] = 10.0 From b426b9e2db7f149f8c1f5f2e2560ab9239f45ff3 Mon Sep 17 00:00:00 2001 From: fatmaebrahim Date: Thu, 18 Sep 2025 10:47:11 +0300 Subject: [PATCH 19/19] fix: handled typed array in unpack --- cmd/main.go | 11 +++++++---- pkg/msgpack.go | 19 ++++++++++++++++--- pkg/msgpack_test.go | 8 ++++---- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index f47a3be..06dec22 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -6,10 +6,10 @@ import ( ) func main() { - arr := make([]byte, 3) + arr := make([]float32, 3) arr[0] = 0 - arr[1] = 7 - arr[2] = 9 + arr[1] = 7.1 + arr[2] = 9.8 m := make(map[any]any) m["hello"] = 10 @@ -47,7 +47,10 @@ func main() { fmt.Printf("object: %+v\n", obj) - bytes := make([]byte, 3) + bytes := make([]float32, 3) + bytes[0] = 1.1 + bytes[1] = 2.2 + bytes[2] = 3.3 ser := msgpack.Serialize(bytes) fmt.Printf("Serialized: %+v\n", ser) diff --git a/pkg/msgpack.go b/pkg/msgpack.go index 04f27e0..12f2c1a 100644 --- a/pkg/msgpack.go +++ b/pkg/msgpack.go @@ -87,7 +87,7 @@ type Object struct { Snum int16 Fnum float32 Str string - Arr []byte + Arr []float32 Mapp map[any]any Typed map[float32]int16 } @@ -408,7 +408,7 @@ func decodeSize(value []byte) int { return 0 } -func decodeArray16(value []byte) ([]any, int) { +func decodeArray16(value []byte) (any, int) { size := decodeSize(value[:2]) data := value[2:] array := make([]any, size) @@ -423,7 +423,7 @@ func decodeArray16(value []byte) ([]any, int) { return array, overallOffset + 3 } -func decodeArray32(value []byte) ([]any, int) { +func decodeArray32(value []byte) (any, int) { size := decodeSize(value[:4]) data := value[4:] array := make([]any, size) @@ -708,6 +708,14 @@ func convertMap(m map[any]any, key_type reflect.Type, value_type reflect.Type) a return new_map.Interface() } +func convertArray(arr []any, value_type reflect.Type) any { + new_slice := reflect.MakeSlice(reflect.SliceOf(value_type), 0, len(arr)) + for _, val := range arr { + new_slice = reflect.Append(new_slice, reflect.ValueOf(val)) + } + return new_slice.Interface() +} + // a function that unpacks a byte array into an object func Unpack(bytes []byte, obj interface{}) (any, error) { unpacked := Deserialize(bytes) @@ -725,6 +733,11 @@ func Unpack(bytes []byte, obj interface{}) (any, error) { new_map := convertMap(val.(map[any]any), object_map_key_type, object_map_val_type) field.Set(reflect.ValueOf(new_map)) continue + } else if field.Kind() == reflect.Slice && object_map_type != packed_map_type { + object_slice_type := object_map_type.Elem() + new_slice := convertArray(val.([]any), object_slice_type) + field.Set(reflect.ValueOf(new_slice)) + continue } field.Set(reflect.ValueOf(val)) diff --git a/pkg/msgpack_test.go b/pkg/msgpack_test.go index f406ea2..6fcd269 100644 --- a/pkg/msgpack_test.go +++ b/pkg/msgpack_test.go @@ -543,10 +543,10 @@ func TestAll(t *testing.T) { }) t.Run("test pack and unpack with complex struct", func(t *testing.T) { - arr := make([]byte, 3) - arr[0] = 1 - arr[1] = 2 - arr[2] = 3 + arr := make([]float32, 3) + arr[0] = 1.1 + arr[1] = 2.2 + arr[2] = 3.3 m := make(map[any]any) m["hello"] = 10.0