From c91c393df44ce246a1330f0365439b367bfda065 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 11 Jun 2024 22:27:49 +0300 Subject: [PATCH 1/4] add fields mappings in config --- pkg/genlib/config/config.go | 89 +++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/pkg/genlib/config/config.go b/pkg/genlib/config/config.go index 3b79b3b..46fce2c 100644 --- a/pkg/genlib/config/config.go +++ b/pkg/genlib/config/config.go @@ -2,6 +2,9 @@ package config import ( "errors" + "regexp" + "sort" + "strings" "time" "math" @@ -38,8 +41,94 @@ type Config struct { m map[string]ConfigField } +// HasFieldsMappings checks if all fields have a type defined. +// This is useful to determine if a fields definitions file is required. +func (c Config) HasFieldsMappings() bool { + for _, f := range c.m { + if f.Type == "" { + return false + } + } + return true +} + +// FieldMapping represents the mapping of a field. +// It defines the structure and properties of a field, including its name, +// data type, associated object type, example value, and the actual value. +type FieldMapping struct { + Name string + Type string + ObjectType string + Example string + Value any +} + +// FieldsMappings is a collection of FieldMapping. +type FieldsMappings []FieldMapping + +func (f FieldsMappings) Len() int { return len(f) } +func (f FieldsMappings) Less(i, j int) bool { return f[i].Name < f[j].Name } +func (f FieldsMappings) Swap(i, j int) { f[i], f[j] = f[j], f[i] } + +func normaliseFields(fields FieldsMappings) (FieldsMappings, error) { + sort.Sort(fields) + normalisedFields := make(FieldsMappings, 0, len(fields)) + for _, field := range fields { + if !strings.Contains(field.Name, "*") { + normalisedFields = append(normalisedFields, field) + continue + } + + normalizationPattern := strings.NewReplacer(".", "\\.", "*", ".+").Replace(field.Name) + re, err := regexp.Compile(normalizationPattern) + if err != nil { + return nil, err + } + + hasMatch := false + for _, otherField := range fields { + if otherField.Name == field.Name { + continue + } + + if re.MatchString(otherField.Name) { + hasMatch = true + break + } + } + + if !hasMatch { + normalisedFields = append(normalisedFields, field) + } + } + + sort.Sort(normalisedFields) + return normalisedFields, nil +} + +// LoadFieldsMappings creates the fields mappings from the config itself. +// It has to be called after the config is loaded. +func (c Config) LoadFieldsMappings() (FieldsMappings, error) { + var mappings FieldsMappings + + for _, f := range c.m { + mappings = append(mappings, FieldMapping{ + Name: f.Name, + Type: f.Type, + ObjectType: f.ObjectType, + Example: f.Example, + Value: f.Value, + }) + } + + return normaliseFields(mappings) +} + type ConfigField struct { Name string `config:"name"` + Type string `config:"type"` + ObjectType string `config:"object_type"` + Example string `config:"example"` Fuzziness float64 `config:"fuzziness"` Range Range `config:"range"` Cardinality int `config:"cardinality"` From 106dcd7d871565090e2d755ff7e19b1f69ccd01c Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 11 Jun 2024 22:29:15 +0300 Subject: [PATCH 2/4] change field value to any --- pkg/genlib/fields/fields.go | 4 ++-- pkg/genlib/generator.go | 2 +- pkg/genlib/generator_interface.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/genlib/fields/fields.go b/pkg/genlib/fields/fields.go index 0230c3a..38a9aed 100644 --- a/pkg/genlib/fields/fields.go +++ b/pkg/genlib/fields/fields.go @@ -17,7 +17,7 @@ type Field struct { Type string ObjectType string Example string - Value string + Value any } func (fields Fields) merge(fieldsToMerge ...Field) Fields { @@ -32,7 +32,7 @@ func (fields Fields) merge(fieldsToMerge ...Field) Fields { field.Example = currentField.Example } - if currentField.Value > field.Value { + if currentField.Value != nil && field.Value == nil { field.Value = currentField.Value } diff --git a/pkg/genlib/generator.go b/pkg/genlib/generator.go index 34eeafb..179ad8e 100644 --- a/pkg/genlib/generator.go +++ b/pkg/genlib/generator.go @@ -22,7 +22,7 @@ const ( ) func fieldValueWrapByType(field Field) string { - if len(field.Value) > 0 { + if field.Value != nil { return "" } diff --git a/pkg/genlib/generator_interface.go b/pkg/genlib/generator_interface.go index 5ac92f3..a0cc46b 100644 --- a/pkg/genlib/generator_interface.go +++ b/pkg/genlib/generator_interface.go @@ -101,7 +101,7 @@ func newGenState() *genState { func bindField(cfg Config, field Field, fieldMap map[string]any, withReturn bool) error { // Check for hardcoded field value - if len(field.Value) > 0 { + if field.Value != nil { if withReturn { return bindStaticWithReturn(field, field.Value, fieldMap) } else { From f4d7eef2d42ec3310fadc3ed1944b273e737f1dd Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 11 Jun 2024 22:30:04 +0300 Subject: [PATCH 3/4] create fields definitions from config or path --- internal/corpus/generator.go | 41 +++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/internal/corpus/generator.go b/internal/corpus/generator.go index 9facc76..28d6c17 100644 --- a/internal/corpus/generator.go +++ b/internal/corpus/generator.go @@ -214,13 +214,25 @@ func (gc GeneratorCorpus) GenerateWithTemplate(templatePath, fieldsDefinitionPat return "", errors.New("you must provide a non empty template content") } - ctx := context.Background() - flds, err := fields.LoadFieldsWithTemplate(ctx, fieldsDefinitionPath) - if err != nil { - return "", err + var fieldsDefinitions Fields + + // If fieldsDefinitionPath is not provided, we use the fields types from the config + if gc.config.HasFieldsMappings() && fieldsDefinitionPath == "" { + configFieldsDefinitions, err := gc.config.LoadFieldsMappings() + if err != nil { + return "", err + } + + fieldsDefinitions = mapFields(configFieldsDefinitions) + } else { + ctx := context.Background() + fieldsDefinitions, err = fields.LoadFieldsWithTemplate(ctx, fieldsDefinitionPath) + if err != nil { + return "", err + } } - err = gc.eventsPayloadFromFields(template, flds, totEvents, timeNow, randSeed, nil, f) + err = gc.eventsPayloadFromFields(template, fieldsDefinitions, totEvents, timeNow, randSeed, nil, f) if err != nil { return "", err } @@ -242,3 +254,22 @@ func sanitizeFilename(s string) string { s = strings.Replace(s, "\\", "-", -1) return s } + +// mapFields converts config fields mappings to generator fields. +func mapFields(fieldDefinitions config.FieldsMappings) Fields { + fieldsDefinitions := make(Fields, 0, len(fieldDefinitions)) + + for _, f := range fieldDefinitions { + tempField := fields.Field{ + Name: f.Name, + Type: f.Type, + ObjectType: f.ObjectType, + Example: f.Example, + Value: f.Value, + } + + fieldsDefinitions = append(fieldsDefinitions, tempField) + } + + return fieldsDefinitions +} From b1b4f4291d716af0888caafd98edcfb7deff2a5f Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 11 Jun 2024 22:30:33 +0300 Subject: [PATCH 4/4] make fields definitions path arg optional --- cmd/generate_with_template.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/cmd/generate_with_template.go b/cmd/generate_with_template.go index 654512a..d7d7f2d 100644 --- a/cmd/generate_with_template.go +++ b/cmd/generate_with_template.go @@ -24,11 +24,11 @@ func GenerateWithTemplateCmd() *cobra.Command { generateWithTemplateCmd := &cobra.Command{ Use: "generate-with-template template-path fields-definition-path", Short: "Generate a corpus", - Long: "Generate a bulk request corpus given a template path and a fields definition path", + Long: "Generate a bulk request corpus given a template path. The fields definition path is optional as long as the config has fields definitions.", Args: func(cmd *cobra.Command, args []string) error { var errs []error - if len(args) != 2 { - return errors.New("you must pass the template path and the fields definition path") + if len(args) != 1 { + return errors.New("you must pass the template path, fields definitions path is optional") } templatePath = args[0] @@ -36,9 +36,13 @@ func GenerateWithTemplateCmd() *cobra.Command { errs = append(errs, errors.New("you must provide a not empty template path argument")) } - fieldsDefinitionPath = args[1] - if fieldsDefinitionPath == "" { - errs = append(errs, errors.New("you must provide a not empty fields definition path argument")) + if len(args) > 1 { + fieldsDefinitionPath = args[1] + if fieldsDefinitionPath == "" { + errs = append(errs, errors.New("you must provide a not empty fields definition path argument")) + } + } else { + fmt.Println("Fields definition path not provided, using fields definitions from config") } if len(errs) > 0 { @@ -56,6 +60,14 @@ func GenerateWithTemplateCmd() *cobra.Command { return err } + if !cfg.HasFieldsMappings() && fieldsDefinitionPath == "" { + return errors.New("fields types not found in config and fields definition path not provided") + } + + if cfg.HasFieldsMappings() && fieldsDefinitionPath != "" { + return errors.New("fields types found in config and fields definition path provided, use only one") + } + fc, err := corpus.NewGeneratorWithTemplate(cfg, fs, location, templateType) if err != nil { return err