Add seed flag (#98)

* add --seed flag to cmds

* add randSeed to internal

* move randSeed and timeNow to init function in pkg

* add randSeed to internal

* fix aws.ec2_logs not deterministc reproducible schema-b template

* refactor deprecated rand.Seed call

* TestMain for init customRand

Author: Andrea Spacca

assets/templates/aws.ec2_logs/schema-b/fields.yml

@@ -1,4 +1,4 @@
-- name: timestamp
+- name: ts
   type: date
 - name: process.name
   type: keyword
@@ -12,8 +12,6 @@
   type: keyword
 - name: host.name
   type: keyword
-- name: aws.cloudwatch.ingestion_time
-  type: date
 - name: agent.id
   type: keyword
 - name: event.id

assets/templates/aws.ec2_logs/schema-b/gotext.tpl

@@ -1,4 +1,4 @@
-{{- $ts := now }}
+{{- $ts := generate "ts" }}
 {{- $ip := generate "aws.ec2.ip_address" }}
 {{- $pname := generate "process.name" }}
 {{- $logstream := generate "aws.cloudwatch.log_stream" }}

cmd/generate.go

@@ -18,7 +18,6 @@ import (
 var integrationPackage string
 var dataStream string
 var packageVersion string
-var timeNowAsString string
 
 func GenerateCmd() *cobra.Command {
 	generateCmd := &cobra.Command{
@@ -75,7 +74,7 @@ func GenerateCmd() *cobra.Command {
 				return err
 			}
 
-			payloadFilename, err := fc.Generate(packageRegistryBaseURL, integrationPackage, dataStream, packageVersion, totEvents, timeNow)
+			payloadFilename, err := fc.Generate(packageRegistryBaseURL, integrationPackage, dataStream, packageVersion, totEvents, timeNow, randSeed)
 			if err != nil {
 				return err
 			}
@@ -90,5 +89,7 @@ func GenerateCmd() *cobra.Command {
 	generateCmd.Flags().StringVarP(&configFile, "config-file", "c", "", "path to config file for generator settings")
 	generateCmd.Flags().Uint64VarP(&totEvents, "tot-events", "t", 1, "total events of the corpus to generate")
 	generateCmd.Flags().StringVarP(&timeNowAsString, "now", "n", "", "time to use for generation based on now (`date` type)")
+	generateCmd.Flags().Int64VarP(&randSeed, "seed", "s", 1, "seed to set as source of rand")
+
 	return generateCmd
 }

cmd/generate_common.go

@@ -10,6 +10,8 @@ import (
 var packageRegistryBaseURL string
 var configFile string
 var totEvents uint64
+var timeNowAsString string
+var randSeed int64
 
 func getTimeNowFromFlag(timeNowAsString string) (time.Time, error) {
 	if len(timeNowAsString) > 0 {

cmd/generate_with_template.go

@@ -66,7 +66,7 @@ func GenerateWithTemplateCmd() *cobra.Command {
 				return err
 			}
 
-			payloadFilename, err := fc.GenerateWithTemplate(templatePath, fieldsDefinitionPath, totEvents, timeNow)
+			payloadFilename, err := fc.GenerateWithTemplate(templatePath, fieldsDefinitionPath, totEvents, timeNow, randSeed)
 			if err != nil {
 				return err
 			}
@@ -81,5 +81,7 @@ func GenerateWithTemplateCmd() *cobra.Command {
 	generateWithTemplateCmd.Flags().StringVarP(&templateType, "template-type", "y", "placeholder", "either 'placeholder' or 'gotext'")
 	generateWithTemplateCmd.Flags().Uint64VarP(&totEvents, "tot-events", "t", 1, "total events of the corpus to generate")
 	generateWithTemplateCmd.Flags().StringVarP(&timeNowAsString, "now", "n", "", "time to use for generation based on now (`date` type)")
+	generateWithTemplateCmd.Flags().Int64VarP(&randSeed, "seed", "s", 1, "seed to set as source of rand")
+
 	return generateWithTemplateCmd
 }

cmd/local-template.go

@@ -85,7 +85,7 @@ func TemplateCmd() *cobra.Command {
 				return err
 			}
 
-			payloadFilename, err := fc.GenerateWithTemplate(templatePath, fieldsDefinitionPath, totEvents, timeNow)
+			payloadFilename, err := fc.GenerateWithTemplate(templatePath, fieldsDefinitionPath, totEvents, timeNow, randSeed)
 			if err != nil {
 				return err
 			}
@@ -101,5 +101,7 @@ func TemplateCmd() *cobra.Command {
 	command.Flags().Uint64VarP(&totEvents, "tot-events", "t", 1, "total events of the corpus to generate")
 	command.Flags().StringVarP(&flagSchema, "schema", "", "b", "schema to generate data for; valid values: a, b")
 	command.Flags().StringVarP(&timeNowAsString, "now", "n", "", "time to use for generation based on now (`date` type)")
+
+	command.Flags().Int64VarP(&randSeed, "seed", "s", 1, "seed to set as source of rand")
 	return command
 }

internal/corpus/generator.go

@@ -106,17 +106,19 @@ func (gc GeneratorCorpus) bulkPayloadFilenameWithTemplate(templatePath string) s
 var corpusLocPerm = os.FileMode(0770)
 var corpusPerm = os.FileMode(0660)
 
-func (gc GeneratorCorpus) eventsPayloadFromFields(template []byte, fields Fields, totEvents uint64, timeNow time.Time, createPayload []byte, f afero.File) error {
+func (gc GeneratorCorpus) eventsPayloadFromFields(template []byte, fields Fields, totEvents uint64, timeNow time.Time, randSeed int64, createPayload []byte, f afero.File) error {
+	genlib.InitGeneratorTimeNow(timeNow)
+	genlib.InitGeneratorRandSeed(randSeed)
 
 	var evgen genlib.Generator
 	var err error
 	if len(template) == 0 {
-		evgen, err = genlib.NewGenerator(gc.config, fields, totEvents, timeNow)
+		evgen, err = genlib.NewGenerator(gc.config, fields, totEvents)
 	} else {
 		if gc.templateType == templateTypeCustom {
-			evgen, err = genlib.NewGeneratorWithCustomTemplate(template, gc.config, fields, totEvents, timeNow)
+			evgen, err = genlib.NewGeneratorWithCustomTemplate(template, gc.config, fields, totEvents)
 		} else if gc.templateType == templateTypeGoText {
-			evgen, err = genlib.NewGeneratorWithTextTemplate(template, gc.config, fields, totEvents, timeNow)
+			evgen, err = genlib.NewGeneratorWithTextTemplate(template, gc.config, fields, totEvents)
 		} else {
 			return ErrNotValidTemplate
 		}
@@ -162,7 +164,7 @@ func (gc GeneratorCorpus) eventsPayloadFromFields(template []byte, fields Fields
 }
 
 // Generate generates a bulk request corpus and persist it to file.
-func (gc GeneratorCorpus) Generate(packageRegistryBaseURL, integrationPackage, dataStream, packageVersion string, totEvents uint64, timeNow time.Time) (string, error) {
+func (gc GeneratorCorpus) Generate(packageRegistryBaseURL, integrationPackage, dataStream, packageVersion string, totEvents uint64, timeNow time.Time, randSeed int64) (string, error) {
 	if err := gc.fs.MkdirAll(gc.location, corpusLocPerm); err != nil {
 		return "", fmt.Errorf("cannot generate corpus location folder: %v", err)
 	}
@@ -181,7 +183,7 @@ func (gc GeneratorCorpus) Generate(packageRegistryBaseURL, integrationPackage, d
 
 	createPayload := []byte(`{ "create" : { "_index": "` + dataStreamType + `-` + integrationPackage + `.` + dataStream + `-default" } }` + "\n")
 
-	err = gc.eventsPayloadFromFields(nil, flds, totEvents, timeNow, createPayload, f)
+	err = gc.eventsPayloadFromFields(nil, flds, totEvents, timeNow, randSeed, createPayload, f)
 	if err != nil {
 		return "", err
 	}
@@ -194,7 +196,7 @@ func (gc GeneratorCorpus) Generate(packageRegistryBaseURL, integrationPackage, d
 }
 
 // GenerateWithTemplate generates a template based corpus and persist it to file.
-func (gc GeneratorCorpus) GenerateWithTemplate(templatePath, fieldsDefinitionPath string, totEvents uint64, timeNow time.Time) (string, error) {
+func (gc GeneratorCorpus) GenerateWithTemplate(templatePath, fieldsDefinitionPath string, totEvents uint64, timeNow time.Time, randSeed int64) (string, error) {
 	if err := gc.fs.MkdirAll(gc.location, corpusLocPerm); err != nil {
 		return "", fmt.Errorf("cannot generate corpus location folder: %v", err)
 	}
@@ -220,7 +222,7 @@ func (gc GeneratorCorpus) GenerateWithTemplate(templatePath, fieldsDefinitionPat
 		return "", err
 	}
 
-	err = gc.eventsPayloadFromFields(template, flds, totEvents, timeNow, nil, f)
+	err = gc.eventsPayloadFromFields(template, flds, totEvents, timeNow, randSeed, nil, f)
 	if err != nil {
 		return "", err
 	}

pkg/genlib/generator.go

@@ -14,6 +14,8 @@ import (
 	"time"
 )
 
+var customRand *rand.Rand
+
 const (
 	textTemplateEngine = iota
 	customTemplateEngine
@@ -163,9 +165,22 @@ func generateTemplateFromField(cfg Config, fields Fields, templateEngine int) ([
 	return templateBuffer.Bytes(), objectKeysField
 }
 
-func NewGenerator(cfg Config, flds Fields, totEvents uint64, timeNow time.Time) (Generator, error) {
+func NewGenerator(cfg Config, flds Fields, totEvents uint64) (Generator, error) {
 	template, objectKeysField := generateCustomTemplateFromField(cfg, flds)
 	flds = append(flds, objectKeysField...)
 
-	return NewGeneratorWithCustomTemplate(template, cfg, flds, totEvents, timeNow)
+	return NewGeneratorWithCustomTemplate(template, cfg, flds, totEvents)
+}
+
+// InitGeneratorTimeNow sets base timeNow for `date` field
+func InitGeneratorTimeNow(timeNow time.Time) {
+	// set timeNowToBind to --now flag (already parsed or now)
+	timeNowToBind = timeNow
+}
+
+// InitGeneratorRandSeed sets rand seed
+func InitGeneratorRandSeed(randSeed int64) {
+	// set rand and randomdata seed to --seed flag (custom or 1)
+	customRand = rand.New(rand.NewSource(randSeed))
+	randomdata.CustomRand(customRand)
 }

pkg/genlib/generator_interface.go

@@ -13,7 +13,6 @@ import (
 	"github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config"
 	"github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/fields"
 	"math"
-	"math/rand"
 	"regexp"
 	"strconv"
 	"strings"
@@ -235,14 +234,14 @@ func makeFloatFunc(fieldCfg ConfigField, field Field) func() float64 {
 
 	switch {
 	case maxValue > 0:
-		dummyFunc = func() float64 { return minValue + rand.Float64()*(maxValue-minValue) }
+		dummyFunc = func() float64 { return minValue + customRand.Float64()*(maxValue-minValue) }
 	case len(field.Example) == 0:
-		dummyFunc = func() float64 { return rand.Float64() * 10 }
+		dummyFunc = func() float64 { return customRand.Float64() * 10 }
 	default:
 		totDigit := len(field.Example)
 		max := math.Pow10(totDigit)
 		dummyFunc = func() float64 {
-			return rand.Float64() * max
+			return customRand.Float64() * max
 		}
 	}
 
@@ -261,14 +260,14 @@ func makeIntFunc(fieldCfg ConfigField, field Field) func() int64 {
 
 	switch {
 	case maxValue > 0:
-		dummyFunc = func() int64 { return rand.Int63n(maxValue-minValue) + minValue }
+		dummyFunc = func() int64 { return customRand.Int63n(maxValue-minValue) + minValue }
 	case len(field.Example) == 0:
-		dummyFunc = func() int64 { return rand.Int63n(10) }
+		dummyFunc = func() int64 { return customRand.Int63n(10) }
 	default:
 		totDigit := len(field.Example)
 		max := int64(math.Pow10(totDigit))
 		dummyFunc = func() int64 {
-			return rand.Int63n(max)
+			return customRand.Int63n(max)
 		}
 	}
 
@@ -340,30 +339,30 @@ func genNounsNWithReturn(n int) string {
 }
 
 func randGeoPoint(buf *bytes.Buffer) error {
-	lat := rand.Intn(181) - 90
+	lat := customRand.Intn(181) - 90
 	var latD int
 	if lat != -90 && lat != 90 {
-		latD = rand.Intn(100)
+		latD = customRand.Intn(100)
 	}
 	var longD int
-	long := rand.Intn(361) - 180
+	long := customRand.Intn(361) - 180
 	if long != -180 && long != 180 {
-		longD = rand.Intn(100)
+		longD = customRand.Intn(100)
 	}
 	_, err := fmt.Fprintf(buf, "%d.%d,%d.%d", lat, latD, long, longD)
 	return err
 }
 
 func randGeoPointWithReturn() string {
-	lat := rand.Intn(181) - 90
+	lat := customRand.Intn(181) - 90
 	var latD int
 	if lat != -90 && lat != 90 {
-		latD = rand.Intn(100)
+		latD = customRand.Intn(100)
 	}
 	var longD int
-	long := rand.Intn(361) - 180
+	long := customRand.Intn(361) - 180
 	if long != -180 && long != 180 {
-		longD = rand.Intn(100)
+		longD = customRand.Intn(100)
 	}
 
 	return fmt.Sprintf("%d.%d,%d.%d", lat, latD, long, longD)
@@ -390,7 +389,7 @@ func bindKeyword(fieldCfg ConfigField, field Field, fieldMap map[string]any) err
 	if len(fieldCfg.Enum) > 0 {
 		var emitFNotReturn emitFNotReturn
 		emitFNotReturn = func(state *GenState, buf *bytes.Buffer) error {
-			idx := rand.Intn(len(fieldCfg.Enum))
+			idx := customRand.Intn(len(fieldCfg.Enum))
 			buf.WriteString(fieldCfg.Enum[idx])
 			return nil
 		}
@@ -462,7 +461,7 @@ func bindStatic(field Field, v any, fieldMap map[string]any) error {
 func bindBool(field Field, fieldMap map[string]any) error {
 	var emitFNotReturn emitFNotReturn
 	emitFNotReturn = func(state *GenState, buf *bytes.Buffer) error {
-		switch rand.Int() % 2 {
+		switch customRand.Int() % 2 {
 		case 0:
 			buf.WriteString("false")
 		case 1:
@@ -488,7 +487,7 @@ func bindGeoPoint(field Field, fieldMap map[string]any) error {
 func bindWordN(field Field, n int, fieldMap map[string]any) error {
 	var emitFNotReturn emitFNotReturn
 	emitFNotReturn = func(state *GenState, buf *bytes.Buffer) error {
-		genNounsN(rand.Intn(n), buf)
+		genNounsN(customRand.Intn(n), buf)
 		return nil
 	}
 
@@ -503,7 +502,7 @@ func bindNearTime(fieldCfg ConfigField, field Field, fieldMap map[string]any) er
 		if fieldCfg.Period > 0 && state.totEvents > 0 {
 			offset = time.Duration((fieldCfg.Period.Nanoseconds() / int64(state.totEvents)) * int64(state.counter))
 		} else {
-			offset = time.Duration(rand.Intn(FieldTypeTimeRange)*-1) * time.Second
+			offset = time.Duration(customRand.Intn(FieldTypeTimeRange)*-1) * time.Second
 		}
 
 		newTime := timeNowToBind.Add(offset)
@@ -518,10 +517,10 @@ func bindNearTime(fieldCfg ConfigField, field Field, fieldMap map[string]any) er
 func bindIP(field Field, fieldMap map[string]any) error {
 	var emitFNotReturn emitFNotReturn
 	emitFNotReturn = func(state *GenState, buf *bytes.Buffer) error {
-		i0 := rand.Intn(255)
-		i1 := rand.Intn(255)
-		i2 := rand.Intn(255)
-		i3 := rand.Intn(255)
+		i0 := customRand.Intn(255)
+		i1 := customRand.Intn(255)
+		i2 := customRand.Intn(255)
+		i3 := customRand.Intn(255)
 
 		_, err := fmt.Fprintf(buf, "%d.%d.%d.%d", i0, i1, i2, i3)
 		return err
@@ -537,7 +536,7 @@ func fuzzyInt(previous int64, fuzziness, min, max float64) int64 {
 	higherBound := float64(previous) * (1 + fuzziness)
 	lowerBound = math.Max(lowerBound, min)
 	higherBound = math.Min(higherBound, max)
-	return rand.Int63n(int64(math.Ceil(higherBound-lowerBound))) + int64(lowerBound)
+	return customRand.Int63n(int64(math.Ceil(higherBound-lowerBound))) + int64(lowerBound)
 }
 
 func bindLong(fieldCfg ConfigField, field Field, fieldMap map[string]any) error {
@@ -587,7 +586,7 @@ func fuzzyFloat(previous, fuzziness, min, max float64) float64 {
 	higherBound := previous * (1 + fuzziness)
 	lowerBound = math.Max(lowerBound, min)
 	higherBound = math.Min(higherBound, max)
-	return lowerBound + rand.Float64()*(higherBound-lowerBound)
+	return lowerBound + customRand.Float64()*(higherBound-lowerBound)
 }
 
 func bindDouble(fieldCfg ConfigField, field Field, fieldMap map[string]any) error {
@@ -739,7 +738,7 @@ func bindKeywordWithReturn(fieldCfg ConfigField, field Field, fieldMap map[strin
 	if len(fieldCfg.Enum) > 0 {
 		var emitF EmitF
 		emitF = func(state *GenState) any {
-			idx := rand.Intn(len(fieldCfg.Enum))
+			idx := customRand.Intn(len(fieldCfg.Enum))
 			return fieldCfg.Enum[idx]
 		}
 
@@ -804,7 +803,7 @@ func bindStaticWithReturn(field Field, v any, fieldMap map[string]any) error {
 func bindBoolWithReturn(field Field, fieldMap map[string]any) error {
 	var emitF EmitF
 	emitF = func(state *GenState) any {
-		switch rand.Int() % 2 {
+		switch customRand.Int() % 2 {
 		case 0:
 			return false
 		default:
@@ -830,7 +829,7 @@ func bindGeoPointWithReturn(field Field, fieldMap map[string]any) error {
 func bindWordNWithReturn(field Field, n int, fieldMap map[string]any) error {
 	var emitF EmitF
 	emitF = func(state *GenState) any {
-		return genNounsNWithReturn(rand.Intn(n))
+		return genNounsNWithReturn(customRand.Intn(n))
 	}
 	fieldMap[field.Name] = emitF
 	return nil
@@ -843,7 +842,7 @@ func bindNearTimeWithReturn(fieldCfg ConfigField, field Field, fieldMap map[stri
 		if fieldCfg.Period > 0 {
 			offset = time.Duration((fieldCfg.Period.Nanoseconds() / int64(state.totEvents)) * int64(state.counter))
 		} else {
-			offset = time.Duration(rand.Intn(FieldTypeTimeRange)*-1) * time.Second
+			offset = time.Duration(customRand.Intn(FieldTypeTimeRange)*-1) * time.Second
 		}
 
 		newTime := timeNowToBind.Add(offset)
@@ -857,10 +856,10 @@ func bindNearTimeWithReturn(fieldCfg ConfigField, field Field, fieldMap map[stri
 func bindIPWithReturn(field Field, fieldMap map[string]any) error {
 	var emitF EmitF
 	emitF = func(state *GenState) any {
-		i0 := rand.Intn(255)
-		i1 := rand.Intn(255)
-		i2 := rand.Intn(255)
-		i3 := rand.Intn(255)
+		i0 := customRand.Intn(255)
+		i1 := customRand.Intn(255)
+		i2 := customRand.Intn(255)
+		i3 := customRand.Intn(255)
 
 		return fmt.Sprintf("%d.%d.%d.%d", i0, i1, i2, i3)
 	}