From 8efeddcb6f5d97a5a6a20786353268847072f07a Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Fri, 14 Jun 2024 19:00:33 +0300 Subject: [PATCH 1/4] add formatting pattern for keyword type --- pkg/genlib/config/config.go | 19 ++++++------- pkg/genlib/generator_interface.go | 45 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/pkg/genlib/config/config.go b/pkg/genlib/config/config.go index 3b79b3b..2b8f349 100644 --- a/pkg/genlib/config/config.go +++ b/pkg/genlib/config/config.go @@ -39,15 +39,16 @@ type Config struct { } type ConfigField struct { - Name string `config:"name"` - Fuzziness float64 `config:"fuzziness"` - Range Range `config:"range"` - Cardinality int `config:"cardinality"` - Period time.Duration `config:"period"` - Enum []string `config:"enum"` - ObjectKeys []string `config:"object_keys"` - Value any `config:"value"` - Counter bool `config:"counter"` + Name string `config:"name"` + Fuzziness float64 `config:"fuzziness"` + Range Range `config:"range"` + Cardinality int `config:"cardinality"` + Period time.Duration `config:"period"` + Enum []string `config:"enum"` + ObjectKeys []string `config:"object_keys"` + Value any `config:"value"` + Counter bool `config:"counter"` + FormattingPattern string `config:"formatting_pattern"` } func (cf ConfigField) ValidForDateField() error { diff --git a/pkg/genlib/generator_interface.go b/pkg/genlib/generator_interface.go index 5ac92f3..7a6a496 100644 --- a/pkg/genlib/generator_interface.go +++ b/pkg/genlib/generator_interface.go @@ -98,6 +98,40 @@ func newGenState() *genState { } } +type patternFn func() string + +var patternGenerators = func() map[string]patternFn { + return map[string]patternFn{ + "string": func() string { + return randomdata.Noun() + }, + "ipv4": func() string { + return randomdata.IpV4Address() + }, + "ipv6": func() string { + return randomdata.IpV6Address() + }, + "port": func() string { + return fmt.Sprintf("%d", randomdata.Number(1024, 65535)) + }, + } +} + +func replacePattern(pattern string) string { + // Regular expression to find {generator} patterns + re := regexp.MustCompile(`\{(.*?)}`) + matches := re.FindAllStringSubmatch(pattern, -1) + + for _, match := range matches { + key := match[1] + if generator, exists := patternGenerators()[key]; exists { + pattern = strings.Replace(pattern, match[0], generator(), 1) + } + } + + return pattern +} + func bindField(cfg Config, field Field, fieldMap map[string]any, withReturn bool) error { // Check for hardcoded field value @@ -844,6 +878,17 @@ func bindConstantKeywordWithReturn(field Field, fieldMap map[string]any) error { } func bindKeywordWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]any) error { + if fieldCfg.FormattingPattern != "" { + var emitF emitF + emitF = func(state *genState) any { + res := replacePattern(fieldCfg.FormattingPattern) + return res + } + + fieldMap[field.Name] = emitF + return nil + } + if len(fieldCfg.Enum) > 0 { var emitF emitF emitF = func(state *genState) any { From d5c084073f894076a9ab0d70d4408d82b72d7ee9 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Mon, 17 Jun 2024 19:59:34 +0300 Subject: [PATCH 2/4] add tests --- .../generator_with_text_template_test.go | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/pkg/genlib/generator_with_text_template_test.go b/pkg/genlib/generator_with_text_template_test.go index ac1ec24..53a33d1 100644 --- a/pkg/genlib/generator_with_text_template_test.go +++ b/pkg/genlib/generator_with_text_template_test.go @@ -5,6 +5,7 @@ import ( "fmt" "math/rand" "net" + "regexp" "strconv" "strings" "testing" @@ -755,6 +756,104 @@ func Test_FieldIPWithTextTemplate(t *testing.T) { } } +func Test_FieldKeywordFormattingPatternPathWithTextTemplate(t *testing.T) { + fld := Field{ + Name: "path", + Type: FieldTypeKeyword, + } + + template := []byte(`{{$path := generate "path"}}{"path":"{{$path}}"}`) + configYaml := []byte(`fields: +- name: path + cardinality: 25 + formatting_pattern: "/home/{string}/{string}/{string}"`) + t.Logf("with template: %s", string(template)) + + cfg, err := config.LoadConfigFromYaml(configYaml) + if err != nil { + t.Fatal(err) + } + + g := makeGeneratorWithTextTemplate(t, cfg, []Field{fld}, template, 10) + + var buf bytes.Buffer + + pathRegex := regexp.MustCompile(`^/home/[^/]+/[^/]+/[^/]+$`) + + nSpins := int64(10) + + for i := int64(0); i < nSpins; i++ { + if err := g.Emit(&buf); err != nil { + t.Fatal(err) + } + + m := unmarshalJSONT[string](t, buf.Bytes()) + buf.Reset() + + if len(m) != 1 { + t.Errorf("Expected map size 1, got %d", len(m)) + } + + v, ok := m[fld.Name] + if !ok { + t.Errorf("Missing key %v", fld.Name) + } + + if !pathRegex.MatchString(v) { + t.Errorf("Generated path %v does not match expected format", v) + } + } +} + +func Test_FieldKeywordFormattingPatternHostIPWithTextTemplate(t *testing.T) { + fld := Field{ + Name: "hostIP", + Type: FieldTypeKeyword, + } + + template := []byte(`{{$hostIP := generate "hostIP"}}{"hostIP":"{{$hostIP}}"}`) + configYaml := []byte(`fields: +- name: hostIP + cardinality: 25 + formatting_pattern: "{ipv4}:{port}"`) + t.Logf("with template: %s", string(template)) + + cfg, err := config.LoadConfigFromYaml(configYaml) + if err != nil { + t.Fatal(err) + } + + g := makeGeneratorWithTextTemplate(t, cfg, []Field{fld}, template, 10) + + var buf bytes.Buffer + + ipv4PortRegex := regexp.MustCompile(`^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}$`) + + nSpins := int64(10) + + for i := int64(0); i < nSpins; i++ { + if err := g.Emit(&buf); err != nil { + t.Fatal(err) + } + + m := unmarshalJSONT[string](t, buf.Bytes()) + buf.Reset() + + if len(m) != 1 { + t.Errorf("Expected map size 1, got %d", len(m)) + } + + v, ok := m[fld.Name] + if !ok { + t.Errorf("Missing key %v", fld.Name) + } + + if !ipv4PortRegex.MatchString(v) { + t.Errorf("Generated pattern %v does not match expected format", v) + } + } +} + func Test_FieldFloatsWithTextTemplate(t *testing.T) { _testNumericWithTextTemplate[float64](t, FieldTypeDouble) _testNumericWithTextTemplate[float32](t, FieldTypeFloat) From b469e754214a44277d25b86f37943a6a580ca5a2 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 6 Aug 2024 14:54:56 +0300 Subject: [PATCH 3/4] refactor replacePattern --- pkg/genlib/generator_interface.go | 32 ++++++++----------- .../generator_with_text_template_test.go | 6 ++-- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/pkg/genlib/generator_interface.go b/pkg/genlib/generator_interface.go index 7a6a496..5884169 100644 --- a/pkg/genlib/generator_interface.go +++ b/pkg/genlib/generator_interface.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "math" + "math/rand" "regexp" "strconv" "strings" @@ -98,38 +99,33 @@ func newGenState() *genState { } } -type patternFn func() string +// replacePattern replaces placeholders in a pattern with random data. +func replacePattern(pattern string) string { + options := strings.Split(pattern, "|") + chosenOption := options[rand.Intn(len(options))] -var patternGenerators = func() map[string]patternFn { - return map[string]patternFn{ - "string": func() string { + replacements := map[string]func() string{ + "{string}": func() string { return randomdata.Noun() }, - "ipv4": func() string { + "{ipv4}": func() string { return randomdata.IpV4Address() }, - "ipv6": func() string { + "{ipv6}": func() string { return randomdata.IpV6Address() }, - "port": func() string { + "{port}": func() string { return fmt.Sprintf("%d", randomdata.Number(1024, 65535)) }, } -} -func replacePattern(pattern string) string { - // Regular expression to find {generator} patterns - re := regexp.MustCompile(`\{(.*?)}`) - matches := re.FindAllStringSubmatch(pattern, -1) - - for _, match := range matches { - key := match[1] - if generator, exists := patternGenerators()[key]; exists { - pattern = strings.Replace(pattern, match[0], generator(), 1) + for placeholder, replacementFunc := range replacements { + if strings.Contains(chosenOption, placeholder) { + chosenOption = strings.Replace(chosenOption, placeholder, replacementFunc(), -1) } } - return pattern + return chosenOption } func bindField(cfg Config, field Field, fieldMap map[string]any, withReturn bool) error { diff --git a/pkg/genlib/generator_with_text_template_test.go b/pkg/genlib/generator_with_text_template_test.go index 53a33d1..bd97ac7 100644 --- a/pkg/genlib/generator_with_text_template_test.go +++ b/pkg/genlib/generator_with_text_template_test.go @@ -815,7 +815,7 @@ func Test_FieldKeywordFormattingPatternHostIPWithTextTemplate(t *testing.T) { configYaml := []byte(`fields: - name: hostIP cardinality: 25 - formatting_pattern: "{ipv4}:{port}"`) + formatting_pattern: "{ipv4}:{port}|{ipv6}"`) t.Logf("with template: %s", string(template)) cfg, err := config.LoadConfigFromYaml(configYaml) @@ -827,7 +827,7 @@ func Test_FieldKeywordFormattingPatternHostIPWithTextTemplate(t *testing.T) { var buf bytes.Buffer - ipv4PortRegex := regexp.MustCompile(`^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}$`) + ipRegex := regexp.MustCompile(`^((\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5})|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}))$`) nSpins := int64(10) @@ -848,7 +848,7 @@ func Test_FieldKeywordFormattingPatternHostIPWithTextTemplate(t *testing.T) { t.Errorf("Missing key %v", fld.Name) } - if !ipv4PortRegex.MatchString(v) { + if !ipRegex.MatchString(v) { t.Errorf("Generated pattern %v does not match expected format", v) } } From 01ffb6e7c9acc1621e2b3381f9e18f2f090076cc Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Fri, 20 Sep 2024 14:46:31 +0300 Subject: [PATCH 4/4] add docs for formatting_pattern --- docs/fields-configuration.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/fields-configuration.md b/docs/fields-configuration.md index f65e305..68b812f 100644 --- a/docs/fields-configuration.md +++ b/docs/fields-configuration.md @@ -22,8 +22,11 @@ For each config entry the following fields are available: - `"after_n"`: resets the counter after a specific number of iterations. - `probability` *required when strategy is "probabilistic"*: an integer between 1 and 100 representing the percentage chance of reset for each generated value. - `reset_after_n` *required when strategy is "after_n"*: an integer specifying the number of values to generate before resetting the counter. - -Note: The `counter_reset` configuration is only applicable when `counter` is set to `true`. +- `formatting_pattern` *optional (applicable to `string` type fields)*: a string that defines a pattern for generating formatted string values. The pattern can include static text and placeholders that will be replaced with random values. Multiple options can be provided, separated by `|`, from which one will be randomly selected for each generated value. Available placeholders are: + - `{string}`: replaced with a random noun + - `{ipv4}`: replaced with a random IPv4 address + - `{ipv6}`: replaced with a random IPv6 address + - `{port}`: replaced with a random port number between 1024 and 65535 - `period` *optional (`date` type only)*: values will be evenly generated between `time.Now()` and `time.Now().Add(period)`, where period is expressed as `time.Duration`. It accepts also a negative duration: in this case values will be evenly generated between `time.Now().Add(period)` and `time.Now()`. If both `period` and at least one of `range.from` or `range.to` settings are defined an error will be returned and the generator will stop. - `object_keys` *optional (`object` type only)*: list of field names to generate in a object field type; if not specified a random number of field names will be generated in the object filed type - `value` *optional*: hardcoded value to set for the field (any `cardinality` will be ignored)