Range from to for date type (#117)

* TDD config range from/to for date type

* config range from/to for date type

* TDD generator range from/to for date type

* generator range from/to for date type

* update docs

* fix milliseconds resolution in custom template tests

* cr fixes

* fix tests

* fix tests

* fix docs

* fix for real

Author: Andrea Spacca

docs/fields-configuration.md

@@ -12,8 +12,9 @@ For each config entry the following fields are available:
 - `name` *mandatory*: dotted path field, matching an entry in [Fields definition](./glossary.md#fields-definition)
 - `fuzziness` *optional (`long` and `double` type only)*: when generating data you could want generated values to change in a known interval. Fuzziness allow to specify the maximum delta a generated value can have from the previous value (for the same field), as a delta percentage; value must be between 0.0 and 1.0, where 0 is 0% and 1 is 100%. When not specified there is no constraint on the generated values, boundaries will be defined by the underlying field type
 - `range` *optional (`long` and `double` type only)*: value will be generated between `min` and `max`
+- `range` *optional (`date` type only)*: value will be generated between `from` and `to`. Only one between `from` and `to` can be set, in this case the dates will be generated between `from`/`to` and `time.Now()`. Progressive order of the generated dates is always assured regardless the interval involving `from`, `to` and `time.Now()` is positive or negative. If both at least one of `from` or `to` and `period` settings are defined an error will be returned and the generator will stop. The format of the date must be parsable by the following golang date format: `2006-01-02T15:04:05.999999999-07:00`. 
 - `cardinality` *optional*: number of different values for the field; note that this value may not be respected if not enough events are generated. Es `cardinality: 1000` with `100` generated events would produce `100` different values, not `1000`.
-- `period` *optional (`date` type only)*: values will be evenly generated between `time.Now()` and `time.Now().Add(period)`, where period is expressed as `time.Duration`. It accepts also a negative duration: in this case  values will be evenly generated between `time.Now().Add(period)` and `time.Now()`.
+- `period` *optional (`date` type only)*: values will be evenly generated between `time.Now()` and `time.Now().Add(period)`, where period is expressed as `time.Duration`. It accepts also a negative duration: in this case  values will be evenly generated between `time.Now().Add(period)` and `time.Now()`. If both `period` and at least one of `from` or `to` settings are defined an error will be returned and the generator will stop.
 - `object_keys` *optional (`object` type only)*: list of field names to generate in a object field type; if not specified a random number of field names will be generated in the object filed type
 - `value` *optional*: hardcoded value to set for the field (any `cardinality` will be ignored)
 - `enum` *optional (`keyword` type only)*: list of strings to randomly chose from a value to set for the field (any `cardinality` will be applied limited to the size of the `enum` values)
@@ -26,6 +27,10 @@ If you have an `object` type field that you defined one or multiple `object_keys
 fields:
   - name: timestamp
     period: "1h"
+  - name: lastSnapshot
+    range:
+      from: "2023-11-23T11:29:48-00:00"
+      to: "2023-12-13T01:39:58-00:00"
   - name: aws.dynamodb.metrics.AccountMaxReads.max
     fuzziness: 0.1
     range:
@@ -61,6 +66,8 @@ Related [fields definition](./writing-templates.md#fieldsyml---fields-definition
 ```yaml
 - name: timestamp
   type: date
+- name: lastSnapshot
+  type: date
 - name: data_stream.type
   type: constant_keyword
 - name: data_stream.dataset

pkg/genlib/config/config.go

@@ -12,11 +12,25 @@ import (
 )
 
 var rangeBoundNotSet = errors.New("range bound not set")
+var rangeTimeNotSet = errors.New("range time not set")
+var rangeInvalidConfig = errors.New("range defining both `period` and `from`/`to`")
+
+type TimeRange struct {
+	time.Time
+}
+
+func (ct *TimeRange) Unpack(t string) error {
+	var err error
+	ct.Time, err = time.Parse("2006-01-02T15:04:05.999999999-07:00", t)
+	return err
+}
 
 type Range struct {
-	// NOTE: we want to distinguish when Min/Max are explicitly set to zero value or are not set at all. We use a pointer, such that when not set will be `nil`.
-	Min *float64 `config:"min"`
-	Max *float64 `config:"max"`
+	// NOTE: we want to distinguish when Min/Max/From/To are explicitly set to zero value or are not set at all. We use a pointer, such that when not set will be `nil`.
+	Min  *float64   `config:"min"`
+	Max  *float64   `config:"max"`
+	From *TimeRange `config:"from"`
+	To   *TimeRange `config:"to"`
 }
 
 type Config struct {
@@ -34,6 +48,30 @@ type ConfigField struct {
 	Value       any           `config:"value"`
 }
 
+func (cf ConfigField) ValidForDateField() error {
+	if cf.Period.Abs() > 0 && (cf.Range.From != nil || cf.Range.To != nil) {
+		return rangeInvalidConfig
+	}
+
+	return nil
+}
+
+func (r Range) FromAsTime() (time.Time, error) {
+	if r.From == nil {
+		return time.Time{}, rangeTimeNotSet
+	}
+
+	return r.From.Time, nil
+}
+
+func (r Range) ToAsTime() (time.Time, error) {
+	if r.To == nil {
+		return time.Time{}, rangeTimeNotSet
+	}
+
+	return r.To.Time, nil
+}
+
 func (r Range) MinAsInt64() (int64, error) {
 	if r.Min == nil {
 		return 0, rangeBoundNotSet

pkg/genlib/config/config_test.go

@@ -31,6 +31,118 @@ func TestLoadConfig(t *testing.T) {
 	assert.Equal(t, "foobar", f.Value.(string))
 }
 
+func TestIsValidForDateField(t *testing.T) {
+	testCases := []struct {
+		scenario string
+		config   string
+		hasError bool
+	}{
+		{
+			scenario: "no range",
+			config:   "name: field",
+			hasError: false,
+		},
+		{
+			scenario: "zero period",
+			config:   "name: field\nrange:\n  period: 0",
+			hasError: false,
+		},
+		{
+			scenario: "positive period",
+			config:   "name: field\nrange:\n  period: 1",
+			hasError: false,
+		},
+		{
+			scenario: "negative period",
+			config:   "name: field\nrange:\n  period: -1",
+			hasError: false,
+		},
+		{
+			scenario: "form only",
+			config:   "name: field\nrange:\n  from: \"2006-01-02T15:04:05+07:00\"",
+			hasError: false,
+		},
+		{
+			scenario: "form and zero period",
+			config:   "name: field\nrange:\n  period: 0\n  from: \"2006-01-02T15:04:05+07:00\"",
+			hasError: false,
+		},
+		{
+			scenario: "form and positive period",
+			config:   "name: field\nrange:\n  period: 1\n  from: \"2006-01-02T15:04:05+07:00\"",
+			hasError: true,
+		},
+		{
+			scenario: "form and negative period",
+			config:   "name: field\nrange:\n  period: -1\n  from: \"2006-01-02T15:04:05+07:00\"",
+			hasError: true,
+		},
+		{
+			scenario: "to only",
+			config:   "name: field\nrange:\n  to: \"2006-01-02T15:04:05-07:00\"",
+			hasError: false,
+		},
+		{
+			scenario: "to and zero period",
+			config:   "name: field\nrange:\n  period: 0\n  to: \"2006-01-02T15:04:05-07:00\"",
+			hasError: false,
+		},
+		{
+			scenario: "to and positive period",
+			config:   "name: field\nrange:\n  period: 1\n  to: \"2006-01-02T15:04:05-07:00\"",
+			hasError: true,
+		},
+		{
+			scenario: "to and negative period",
+			config:   "name: field\nrange:\n  period: -1\n  to: \"2006-01-02T15:04:05-07:00\"",
+			hasError: true,
+		},
+		{
+			scenario: "from and to only",
+			config:   "name: field\nrange:\n  from: \"2006-01-02T15:04:05-07:00\"\n  to: \"2006-01-02T15:04:05+07:00\"",
+			hasError: false,
+		},
+		{
+			scenario: "from and to and zero period",
+			config:   "name: field\nrange:\n  period: 0\n  from: \"2006-01-02T15:04:05-07:00\"\n\n  to: \"2006-01-02T15:04:05+07:00\"",
+			hasError: false,
+		},
+		{
+			scenario: "from and to and positive period",
+			config:   "name: field\nrange:\n  period: 1\n  from: \"2006-01-02T15:04:05-07:00\"\n  to: \"2006-01-02T15:04:05+07:00\"",
+			hasError: true,
+		},
+		{
+			scenario: "from and to and negative period",
+			config:   "name: field\nrange:\n  period: -1\n  from: \"2006-01-02T15:04:05-07:00\"\n  to: \"2006-01-02T15:04:05+07:00\"",
+			hasError: true,
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.scenario, func(t *testing.T) {
+			cfg, err := yaml.NewConfig([]byte(testCase.config))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var config ConfigField
+			err = cfg.Unpack(&config)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = config.ValidForDateField()
+			if testCase.hasError && err == nil {
+
+			}
+
+			if !testCase.hasError && err != nil {
+
+			}
+		})
+	}
+}
+
 func TestRange_MaxAsFloat64(t *testing.T) {
 	testCases := []struct {
 		scenario  string
@@ -259,6 +371,112 @@ func TestRange_MinAsInt64(t *testing.T) {
 	}
 }
 
+func TestRange_FromAsTime(t *testing.T) {
+	from, err := time.Parse("2006-01-02T15:04:05.999999999-07:00", "2023-11-23T08:35:38+00:00")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testCases := []struct {
+		scenario  string
+		rangeYaml string
+		expected  time.Time
+		hasError  bool
+	}{
+		{
+			scenario:  "from nil",
+			rangeYaml: "to: 2023-11-23T08:35:38+00:00",
+			expected:  time.Time{},
+			hasError:  true,
+		},
+		{
+			scenario:  "from not nil",
+			rangeYaml: "from: 2023-11-23T08:35:38-00:00",
+			expected:  from,
+			hasError:  false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.scenario, func(t *testing.T) {
+			cfg, err := yaml.NewConfig([]byte(testCase.rangeYaml))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var rangeCfg Range
+			err = cfg.Unpack(&rangeCfg)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			v, err := rangeCfg.FromAsTime()
+			if testCase.hasError && err == nil {
+				t.Fatal("expected error but got nil")
+			}
+			if !testCase.hasError && err != nil {
+				t.Fatal("expected no error but got one")
+			}
+			if testCase.expected != v {
+				t.Fatalf("expected %v, got %v", testCase.expected, v)
+			}
+		})
+	}
+}
+
+func TestRange_ToAsTime(t *testing.T) {
+	to, err := time.Parse("2006-01-02T15:04:05.999999999-07:00", "2023-11-23T08:35:38-00:00")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testCases := []struct {
+		scenario  string
+		rangeYaml string
+		expected  time.Time
+		hasError  bool
+	}{
+		{
+			scenario:  "to nil",
+			rangeYaml: "from: 2023-11-23T08:35:38+00:00",
+			expected:  time.Time{},
+			hasError:  true,
+		},
+		{
+			scenario:  "to not nil",
+			rangeYaml: "to: 2023-11-23T08:35:38-00:00",
+			expected:  to,
+			hasError:  false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.scenario, func(t *testing.T) {
+			cfg, err := yaml.NewConfig([]byte(testCase.rangeYaml))
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			var rangeCfg Range
+			err = cfg.Unpack(&rangeCfg)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			v, err := rangeCfg.ToAsTime()
+			if testCase.hasError && err == nil {
+				t.Fatal("expected error but got nil")
+			}
+			if !testCase.hasError && err != nil {
+				t.Fatal("expected no error but got one")
+			}
+			if testCase.expected != v {
+				t.Fatalf("expected %v, got %v", testCase.expected, v)
+			}
+		})
+	}
+}
+
 func TestPeriod(t *testing.T) {
 	testCases := []struct {
 		scenario   string

pkg/genlib/generator.go

@@ -121,7 +121,7 @@ func generateTemplateFromField(cfg Config, fields Fields, templateEngine int) ([
 				fieldVariableName += "Var"
 				if field.Type == FieldTypeDate {
 					if templateEngine == textTemplateEngine {
-						fieldTemplate = fmt.Sprintf(`{{ $%s := generate "%s.%s" }}"%s.%s": %s{{$%s.Format "2006-01-02T15:04:05.999999Z07:00"}}%s%s`, fieldVariableName, fieldNameRoot, rNoun, fieldNameRoot, rNoun, fieldWrap, fieldVariableName, fieldWrap, fieldTrailer)
+						fieldTemplate = fmt.Sprintf(`{{ $%s := generate "%s.%s" }}"%s.%s": %s{{$%s.Format "2006-01-02T15:04:05.999999999Z07:00"}}%s%s`, fieldVariableName, fieldNameRoot, rNoun, fieldNameRoot, rNoun, fieldWrap, fieldVariableName, fieldWrap, fieldTrailer)
 					} else if templateEngine == customTemplateEngine {
 						fieldTemplate = fmt.Sprintf(`"%s.%s": %s{{.%s.%s}}%s%s`, fieldNameRoot, rNoun, fieldWrap, fieldNameRoot, rNoun, fieldWrap, fieldTrailer)
 					}
@@ -146,7 +146,7 @@ func generateTemplateFromField(cfg Config, fields Fields, templateEngine int) ([
 			fieldVariableName += "Var"
 			if field.Type == FieldTypeDate {
 				if templateEngine == textTemplateEngine {
-					fieldTemplate = fmt.Sprintf(`{{ $%s := generate "%s" }}"%s": %s{{$%s.Format "2006-01-02T15:04:05.999999Z07:00"}}%s%s`, fieldVariableName, field.Name, field.Name, fieldWrap, fieldVariableName, fieldWrap, fieldTrailer)
+					fieldTemplate = fmt.Sprintf(`{{ $%s := generate "%s" }}"%s": %s{{$%s.Format "2006-01-02T15:04:05.999999999Z07:00"}}%s%s`, fieldVariableName, field.Name, field.Name, fieldWrap, fieldVariableName, fieldWrap, fieldTrailer)
 				} else if templateEngine == customTemplateEngine {
 					fieldTemplate = fmt.Sprintf(`"%s": %s{{.%s}}%s%s`, field.Name, fieldWrap, field.Name, fieldWrap, fieldTrailer)
 				}