Skip to content

Commit fbe38e2

Browse files
plopezlpzskmcgrail
plopezlpz
and
skmcgrail
authored
Support for custom time formats for encoding and decoding DynamoDB AttributeValues (#1627)
Co-authored-by: Sean McGrail <[email protected]>
1 parent 4811576 commit fbe38e2

File tree

11 files changed

+865
-34
lines changed

11 files changed

+865
-34
lines changed

‎.changelog/d10ed36c72e84ccaa5befe7975f10af6.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"id": "d10ed36c-72e8-4cca-a5be-fe7975f10af6",
3+
"type": "feature",
4+
"description": "Support has been added for specifying a custom time format when encoding and decoding DynamoDB AttributeValues. Use `EncoderOptions.EncodeTime` to specify a custom time encoding function, and use `DecoderOptions.DecodeTime` for specifying how to handle the corresponding AttributeValues using the format. Thank you [Pablo Lopez](https://github.com/plopezlpz) for this contribution.",
5+
"modules": [
6+
"feature/dynamodb/attributevalue",
7+
"feature/dynamodbstreams/attributevalue"
8+
]
9+
}

‎feature/dynamodb/attributevalue/decode.go‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,18 @@ func UnmarshalListOfMapsWithOptions(l []map[string]types.AttributeValue, out int
197197
return UnmarshalListWithOptions(items, out, optFns...)
198198
}
199199

200+
// DecodeTimeAttributes is the set of time decoding functions for different AttributeValues.
201+
type DecodeTimeAttributes struct {
202+
// Will decode S attribute values and SS attribute value elements into time.Time
203+
//
204+
// Default string parsing format is time.RFC3339
205+
S func(string) (time.Time, error)
206+
// Will decode N attribute values and NS attribute value elements into time.Time
207+
//
208+
// Default number parsing format is seconds since January 1, 1970 UTC
209+
N func(string) (time.Time, error)
210+
}
211+
200212
// DecoderOptions is a collection of options to configure how the decoder
201213
// unmarshals the value.
202214
type DecoderOptions struct {
@@ -212,6 +224,12 @@ type DecoderOptions struct {
212224
// Number type instead of float64 when the destination type
213225
// is interface{}. Similar to encoding/json.Number
214226
UseNumber bool
227+
228+
// Contains the time decoding functions for different AttributeValues
229+
//
230+
// Default string parsing format is time.RFC3339
231+
// Default number parsing format is seconds since January 1, 1970 UTC
232+
DecodeTime DecodeTimeAttributes
215233
}
216234

217235
// A Decoder provides unmarshaling AttributeValues to Go value types.
@@ -222,11 +240,25 @@ type Decoder struct {
222240
// NewDecoder creates a new Decoder with default configuration. Use
223241
// the `opts` functional options to override the default configuration.
224242
func NewDecoder(optFns ...func(*DecoderOptions)) *Decoder {
225-
options := DecoderOptions{TagKey: defaultTagKey}
243+
options := DecoderOptions{
244+
TagKey: defaultTagKey,
245+
DecodeTime: DecodeTimeAttributes{
246+
S: defaultDecodeTimeS,
247+
N: defaultDecodeTimeN,
248+
},
249+
}
226250
for _, fn := range optFns {
227251
fn(&options)
228252
}
229253

254+
if options.DecodeTime.S == nil {
255+
options.DecodeTime.S = defaultDecodeTimeS
256+
}
257+
258+
if options.DecodeTime.N == nil {
259+
options.DecodeTime.N = defaultDecodeTimeN
260+
}
261+
230262
return &Decoder{
231263
options: options,
232264
}
@@ -459,6 +491,14 @@ func (d *Decoder) decodeNumber(n string, v reflect.Value, fieldTag tag) error {
459491
v.Set(reflect.ValueOf(t).Convert(v.Type()))
460492
return nil
461493
}
494+
if v.Type().ConvertibleTo(timeType) {
495+
t, err := d.options.DecodeTime.N(n)
496+
if err != nil {
497+
return err
498+
}
499+
v.Set(reflect.ValueOf(t).Convert(v.Type()))
500+
return nil
501+
}
462502
return &UnmarshalTypeError{Value: "number", Type: v.Type()}
463503
}
464504

@@ -686,7 +726,7 @@ func (d *Decoder) decodeString(s string, v reflect.Value, fieldTag tag) error {
686726
// To maintain backwards compatibility with ConvertFrom family of methods which
687727
// converted strings to time.Time structs
688728
if v.Type().ConvertibleTo(timeType) {
689-
t, err := time.Parse(time.RFC3339, s)
729+
t, err := d.options.DecodeTime.S(s)
690730
if err != nil {
691731
return err
692732
}
@@ -940,3 +980,15 @@ func (e *UnmarshalError) Error() string {
940980
return fmt.Sprintf("unmarshal failed, cannot unmarshal %q into %s, %v",
941981
e.Value, e.Type.String(), e.Err)
942982
}
983+
984+
func defaultDecodeTimeS(v string) (time.Time, error) {
985+
t, err := time.Parse(time.RFC3339, v)
986+
if err != nil {
987+
return time.Time{}, &UnmarshalError{Err: err, Value: v, Type: timeType}
988+
}
989+
return t, nil
990+
}
991+
992+
func defaultDecodeTimeN(v string) (time.Time, error) {
993+
return decodeUnixTime(v)
994+
}

‎feature/dynamodb/attributevalue/decode_test.go‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -797,6 +797,218 @@ func (t *testUnmarshalMapKeyComplex) UnmarshalDynamoDBAttributeValue(av types.At
797797
return nil
798798
}
799799

800+
func TestUnmarshalTime_S_SS(t *testing.T) {
801+
type A struct {
802+
TimeField time.Time
803+
TimeFields []time.Time
804+
TimeFieldsL []time.Time
805+
}
806+
cases := map[string]struct {
807+
input string
808+
expect time.Time
809+
decodeTimeS func(string) (time.Time, error)
810+
}{
811+
"String RFC3339Nano (Default)": {
812+
input: "1970-01-01T00:02:03.01Z",
813+
expect: time.Unix(123, 10000000).UTC(),
814+
},
815+
"String UnixDate": {
816+
input: "Thu Jan 1 00:02:03 UTC 1970",
817+
expect: time.Unix(123, 0).UTC(),
818+
decodeTimeS: func(v string) (time.Time, error) {
819+
t, err := time.Parse(time.UnixDate, v)
820+
if err != nil {
821+
return time.Time{}, &UnmarshalError{Err: err, Value: v, Type: timeType}
822+
}
823+
return t, nil
824+
},
825+
},
826+
"String RFC3339 millis keeping zeroes": {
827+
input: "1970-01-01T00:02:03.010Z",
828+
expect: time.Unix(123, 10000000).UTC(),
829+
decodeTimeS: func(v string) (time.Time, error) {
830+
t, err := time.Parse("2006-01-02T15:04:05.000Z07:00", v)
831+
if err != nil {
832+
return time.Time{}, &UnmarshalError{Err: err, Value: v, Type: timeType}
833+
}
834+
return t, nil
835+
},
836+
},
837+
"String RFC822": {
838+
input: "01 Jan 70 00:02 UTC",
839+
expect: time.Unix(120, 0).UTC(),
840+
decodeTimeS: func(v string) (time.Time, error) {
841+
t, err := time.Parse(time.RFC822, v)
842+
if err != nil {
843+
return time.Time{}, &UnmarshalError{Err: err, Value: v, Type: timeType}
844+
}
845+
return t, nil
846+
},
847+
},
848+
}
849+
for name, c := range cases {
850+
t.Run(name, func(t *testing.T) {
851+
inputMap := &types.AttributeValueMemberM{
852+
Value: map[string]types.AttributeValue{
853+
"TimeField": &types.AttributeValueMemberS{Value: c.input},
854+
"TimeFields": &types.AttributeValueMemberSS{Value: []string{c.input}},
855+
"TimeFieldsL": &types.AttributeValueMemberL{Value: []types.AttributeValue{
856+
&types.AttributeValueMemberS{Value: c.input},
857+
}},
858+
},
859+
}
860+
expectedValue := A{
861+
TimeField: c.expect,
862+
TimeFields: []time.Time{c.expect},
863+
TimeFieldsL: []time.Time{c.expect},
864+
}
865+
866+
var actualValue A
867+
if err := UnmarshalWithOptions(inputMap, &actualValue, func(options *DecoderOptions) {
868+
if c.decodeTimeS != nil {
869+
options.DecodeTime.S = c.decodeTimeS
870+
}
871+
}); err != nil {
872+
t.Errorf("expect no error, got %v", err)
873+
}
874+
if diff := cmp.Diff(expectedValue, actualValue, getIgnoreAVUnexportedOptions()...); diff != "" {
875+
t.Errorf("expect attribute value match\n%s", diff)
876+
}
877+
})
878+
}
879+
}
880+
881+
func TestUnmarshalTime_N_NS(t *testing.T) {
882+
type A struct {
883+
TimeField time.Time
884+
TimeFields []time.Time
885+
TimeFieldsL []time.Time
886+
}
887+
cases := map[string]struct {
888+
input string
889+
expect time.Time
890+
decodeTimeN func(string) (time.Time, error)
891+
}{
892+
"Number Unix seconds (Default)": {
893+
input: "123",
894+
expect: time.Unix(123, 0),
895+
},
896+
"Number Unix milli": {
897+
input: "123010",
898+
expect: time.Unix(123, 10000000),
899+
decodeTimeN: func(v string) (time.Time, error) {
900+
n, err := strconv.ParseInt(v, 10, 64)
901+
if err != nil {
902+
return time.Time{}, &UnmarshalError{
903+
Err: err, Value: v, Type: timeType,
904+
}
905+
}
906+
return time.Unix(0, n*int64(time.Millisecond)), nil
907+
},
908+
},
909+
}
910+
for name, c := range cases {
911+
t.Run(name, func(t *testing.T) {
912+
inputMap := &types.AttributeValueMemberM{
913+
Value: map[string]types.AttributeValue{
914+
"TimeField": &types.AttributeValueMemberN{Value: c.input},
915+
"TimeFields": &types.AttributeValueMemberNS{Value: []string{c.input}},
916+
"TimeFieldsL": &types.AttributeValueMemberL{Value: []types.AttributeValue{
917+
&types.AttributeValueMemberN{Value: c.input},
918+
}},
919+
},
920+
}
921+
expectedValue := A{
922+
TimeField: c.expect,
923+
TimeFields: []time.Time{c.expect},
924+
TimeFieldsL: []time.Time{c.expect},
925+
}
926+
927+
var actualValue A
928+
if err := UnmarshalWithOptions(inputMap, &actualValue, func(options *DecoderOptions) {
929+
if c.decodeTimeN != nil {
930+
options.DecodeTime.N = c.decodeTimeN
931+
}
932+
}); err != nil {
933+
t.Errorf("expect no error, got %v", err)
934+
}
935+
if diff := cmp.Diff(expectedValue, actualValue, getIgnoreAVUnexportedOptions()...); diff != "" {
936+
t.Errorf("expect attribute value match\n%s", diff)
937+
}
938+
})
939+
}
940+
}
941+
942+
func TestCustomDecodeSAndDefaultDecodeN(t *testing.T) {
943+
type A struct {
944+
TimeFieldS time.Time
945+
TimeFieldN time.Time
946+
}
947+
inputMap := &types.AttributeValueMemberM{
948+
Value: map[string]types.AttributeValue{
949+
"TimeFieldS": &types.AttributeValueMemberS{Value: "01 Jan 70 00:02 UTC"},
950+
"TimeFieldN": &types.AttributeValueMemberN{Value: "123"},
951+
},
952+
}
953+
expectedValue := A{
954+
TimeFieldS: time.Unix(120, 0).UTC(),
955+
TimeFieldN: time.Unix(123, 0).UTC(),
956+
}
957+
958+
var actualValue A
959+
if err := UnmarshalWithOptions(inputMap, &actualValue, func(options *DecoderOptions) {
960+
// overriding only the S time decoder will keep the default N time decoder
961+
options.DecodeTime.S = func(v string) (time.Time, error) {
962+
t, err := time.Parse(time.RFC822, v)
963+
if err != nil {
964+
return time.Time{}, &UnmarshalError{Err: err, Value: v, Type: timeType}
965+
}
966+
return t, nil
967+
}
968+
}); err != nil {
969+
t.Errorf("expect no error, got %v", err)
970+
}
971+
if diff := cmp.Diff(expectedValue, actualValue, getIgnoreAVUnexportedOptions()...); diff != "" {
972+
t.Errorf("expect attribute value match\n%s", diff)
973+
}
974+
}
975+
976+
func TestCustomDecodeNAndDefaultDecodeS(t *testing.T) {
977+
type A struct {
978+
TimeFieldS time.Time
979+
TimeFieldN time.Time
980+
}
981+
inputMap := &types.AttributeValueMemberM{
982+
Value: map[string]types.AttributeValue{
983+
"TimeFieldS": &types.AttributeValueMemberS{Value: "1970-01-01T00:02:03.01Z"},
984+
"TimeFieldN": &types.AttributeValueMemberN{Value: "123010"},
985+
},
986+
}
987+
expectedValue := A{
988+
TimeFieldS: time.Unix(123, 10000000).UTC(),
989+
TimeFieldN: time.Unix(123, 10000000).UTC(),
990+
}
991+
992+
var actualValue A
993+
if err := UnmarshalWithOptions(inputMap, &actualValue, func(options *DecoderOptions) {
994+
// overriding only the N time decoder will keep the default S time decoder
995+
options.DecodeTime.N = func(v string) (time.Time, error) {
996+
n, err := strconv.ParseInt(v, 10, 64)
997+
if err != nil {
998+
return time.Time{}, &UnmarshalError{
999+
Err: err, Value: v, Type: timeType,
1000+
}
1001+
}
1002+
return time.Unix(0, n*int64(time.Millisecond)), nil
1003+
}
1004+
}); err != nil {
1005+
t.Errorf("expect no error, got %v", err)
1006+
}
1007+
if diff := cmp.Diff(expectedValue, actualValue, getIgnoreAVUnexportedOptions()...); diff != "" {
1008+
t.Errorf("expect attribute value match\n%s", diff)
1009+
}
1010+
}
1011+
8001012
func TestUnmarshalMap_keyTypes(t *testing.T) {
8011013
type StrAlias string
8021014
type IntAlias int

‎feature/dynamodb/attributevalue/encode.go‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,11 @@ type EncoderOptions struct {
370370
// Defaults to enabled, because AttributeValue sets cannot currently be
371371
// empty lists.
372372
NullEmptySets bool
373+
374+
// Will encode time.Time fields
375+
//
376+
// Default encoding is time.RFC3339Nano in a DynamoDB String (S) data type.
377+
EncodeTime func(time.Time) (types.AttributeValue, error)
373378
}
374379

375380
// An Encoder provides marshaling Go value types to AttributeValues.
@@ -383,11 +388,16 @@ func NewEncoder(optFns ...func(*EncoderOptions)) *Encoder {
383388
options := EncoderOptions{
384389
TagKey: defaultTagKey,
385390
NullEmptySets: true,
391+
EncodeTime: defaultEncodeTime,
386392
}
387393
for _, fn := range optFns {
388394
fn(&options)
389395
}
390396

397+
if options.EncodeTime == nil {
398+
options.EncodeTime = defaultEncodeTime
399+
}
400+
391401
return &Encoder{
392402
options: options,
393403
}
@@ -466,7 +476,7 @@ func (e *Encoder) encodeStruct(v reflect.Value, fieldTag tag) (types.AttributeVa
466476
if fieldTag.AsUnixTime {
467477
return UnixTime(t).MarshalDynamoDBAttributeValue()
468478
}
469-
return &types.AttributeValueMemberS{Value: t.Format(time.RFC3339Nano)}, nil
479+
return e.options.EncodeTime(t)
470480
}
471481

472482
m := &types.AttributeValueMemberM{Value: map[string]types.AttributeValue{}}
@@ -844,3 +854,9 @@ type InvalidMarshalError struct {
844854
func (e *InvalidMarshalError) Error() string {
845855
return fmt.Sprintf("marshal failed, %s", e.msg)
846856
}
857+
858+
func defaultEncodeTime(t time.Time) (types.AttributeValue, error) {
859+
return &types.AttributeValueMemberS{
860+
Value: t.Format(time.RFC3339Nano),
861+
}, nil
862+
}

0 commit comments

Comments
 (0)