Skip to content

Improve generation of integer fields #167

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
+475 −35
Draft

Improve generation of integer fields #167

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e901f6b
Elasticsearch integer type is 32-bit wide
cavokz Nov 12, 2025
2eb8bea
Add `long` type to the cardinality tests
cavokz Nov 12, 2025
9b3edc8
Add generation of `short` and `byte` fields
cavokz Nov 10, 2025
24c2ed9
Fix integers generation according to type
cavokz Nov 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/genlib/generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func fieldValueWrapByType(field Field) string {
return "\""
case FieldTypeDouble, FieldTypeFloat, FieldTypeHalfFloat, FieldTypeScaledFloat:
return ""
case FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong:
case FieldTypeByte, FieldTypeShort, FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong:
return ""
case FieldTypeConstantKeyword:
return "\""
Expand Down
151 changes: 136 additions & 15 deletions pkg/genlib/generator_interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ const (
FieldTypeFloat = "float"
FieldTypeHalfFloat = "half_float"
FieldTypeScaledFloat = "scaled_float"
FieldTypeByte = "byte"
FieldTypeShort = "short"
FieldTypeInteger = "integer"
FieldTypeLong = "long"
FieldTypeUnsignedLong = "unsigned_long"
Expand All @@ -59,6 +61,36 @@ var (
keywordRegex = regexp.MustCompile("(\\.|-|_|\\s){1,1}")
)

// getIntTypeBounds returns the min and max values for a given integer field type
func getIntTypeBounds(fieldType string) (min int64, max int64) {
switch fieldType {
case FieldTypeByte:
return math.MinInt8, math.MaxInt8
case FieldTypeShort:
return math.MinInt16, math.MaxInt16
case FieldTypeInteger:
return math.MinInt32, math.MaxInt32
case FieldTypeLong:
return math.MinInt64, math.MaxInt64
case FieldTypeUnsignedLong:
return 0, math.MaxInt64 // Go int64 max; actual ES unsigned_long goes to 2^64-1
default:
// Default to long bounds
return math.MinInt64, math.MaxInt64
}
}

// clampInt64 clamps a value between min and max
func clampInt64(value, min, max int64) int64 {
if value < min {
return min
}
if value > max {
return max
}
return value
}

// This is the emit function for the custom template engine where we stream content directly to the output buffer and no need a return value
type emitFNotReturn func(state *genState, buf *bytes.Buffer) error

Expand Down Expand Up @@ -173,7 +205,7 @@ func bindByType(cfg Config, field Field, fieldMap map[string]any) (err error) {
err = bindIP(field, fieldMap)
case FieldTypeDouble, FieldTypeFloat, FieldTypeHalfFloat, FieldTypeScaledFloat:
err = bindDouble(fieldCfg, field, fieldMap)
case FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: // TODO: generate > 63 bit values for unsigned_long
case FieldTypeByte, FieldTypeShort, FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: // TODO: generate > 63 bit values for unsigned_long
err = bindLong(fieldCfg, field, fieldMap)
case FieldTypeConstantKeyword:
err = bindConstantKeyword(field, fieldMap)
Expand Down Expand Up @@ -202,7 +234,7 @@ func bindByTypeWithReturn(cfg Config, field Field, fieldMap map[string]any) (err
err = bindIPWithReturn(field, fieldMap)
case FieldTypeDouble, FieldTypeFloat, FieldTypeHalfFloat, FieldTypeScaledFloat:
err = bindDoubleWithReturn(fieldCfg, field, fieldMap)
case FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: // TODO: generate > 63 bit values for unsigned_long
case FieldTypeByte, FieldTypeShort, FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: // TODO: generate > 63 bit values for unsigned_long
err = bindLongWithReturn(fieldCfg, field, fieldMap)
case FieldTypeConstantKeyword:
err = bindConstantKeywordWithReturn(field, fieldMap)
Expand Down Expand Up @@ -248,25 +280,64 @@ func makeFloatFunc(fieldCfg ConfigField, field Field) func() float64 {
}

func makeIntFunc(fieldCfg ConfigField, field Field) func() int64 {
// Get type-specific bounds
typeMin, typeMax := getIntTypeBounds(field.Type)

minValue, _ := fieldCfg.Range.MinAsInt64()
maxValue, err := fieldCfg.Range.MaxAsInt64()
// maxValue not set, let's set it to 0 for the sake of the switch above
if err != nil {
maxValue = 0

// Determine if a range was explicitly configured
hasConfiguredRange := err == nil && maxValue > 0

// Clamp configured range to type-specific bounds
if minValue < typeMin {
minValue = typeMin
}
if hasConfiguredRange {
if maxValue > typeMax {
maxValue = typeMax
}
} else {
// No configured range, use type bounds or simple defaults
maxValue = typeMax
}

var dummyFunc func() int64

switch {
case maxValue > 0:
dummyFunc = func() int64 { return customRand.Int63n(maxValue-minValue) + minValue }
case hasConfiguredRange && maxValue > minValue:
// User specified a range, respect it (clamped to type bounds)
rangeSize := maxValue - minValue + 1
if rangeSize > 0 {
dummyFunc = func() int64 { return customRand.Int63n(rangeSize) + minValue }
} else {
dummyFunc = func() int64 { return minValue }
}
case len(field.Example) == 0:
dummyFunc = func() int64 { return customRand.Int63n(10) }
// No range and no example, generate small values within type bounds
rangeBound := typeMax
if typeMin < 0 {
// For signed types, keep it simple and use 0-10
dummyFunc = func() int64 { return customRand.Int63n(10) }
} else if rangeBound > 10 {
// For unsigned types with large bounds, use 0-10
dummyFunc = func() int64 { return customRand.Int63n(10) }
} else {
// For small unsigned types (byte), use full range
dummyFunc = func() int64 { return customRand.Int63n(rangeBound + 1) }
}
default:
// Use example length to determine magnitude
totDigit := len(field.Example)
max := int64(math.Pow10(totDigit))
dummyFunc = func() int64 {
return customRand.Int63n(max)
// Clamp to type max
if max > typeMax {
max = typeMax
}
if max > 0 {
dummyFunc = func() int64 { return customRand.Int63n(max) }
} else {
dummyFunc = func() int64 { return 0 }
}
}

Expand Down Expand Up @@ -566,9 +637,14 @@ func fuzzyInt(previous int64, fuzziness, min, max float64) int64 {
return customRand.Int63n(int64(math.Ceil(higherBound-lowerBound))) + int64(lowerBound)
}

func fuzzyIntCounter(previous int64, fuzziness float64) int64 {
func fuzzyIntCounter(previous int64, fuzziness, max float64) int64 {
lowerBound := float64(previous)
higherBound := float64(previous) * (1 + fuzziness)
// Clamp to max bound
higherBound = math.Min(higherBound, max)
if higherBound <= lowerBound {
return previous
}
return customRand.Int63n(int64(math.Ceil(higherBound-lowerBound))) + int64(lowerBound)
}

Expand All @@ -578,6 +654,9 @@ func bindLong(fieldCfg ConfigField, field Field, fieldMap map[string]any) error
}

if fieldCfg.Counter {
// Get type-specific bounds for counter
_, typeMax := getIntTypeBounds(field.Type)

var emitFNotReturn emitFNotReturn
emitFNotReturn = func(state *genState, buf *bytes.Buffer) error {
previous := int64(1)
Expand All @@ -593,7 +672,7 @@ func bindLong(fieldCfg ConfigField, field Field, fieldMap map[string]any) error

dummyInt = dummyFunc()
} else {
dummyInt = fuzzyIntCounter(previous, fieldCfg.Fuzziness)
dummyInt = fuzzyIntCounter(previous, fieldCfg.Fuzziness, float64(typeMax))
}

state.prevCache[field.Name] = dummyInt
Expand Down Expand Up @@ -624,9 +703,25 @@ func bindLong(fieldCfg ConfigField, field Field, fieldMap map[string]any) error
return nil
}

// Get type-specific bounds
typeMin, typeMax := getIntTypeBounds(field.Type)

min, _ := fieldCfg.Range.MinAsFloat64()
max, _ := fieldCfg.Range.MaxAsFloat64()

// If no range specified or range exceeds type bounds, use type bounds
if min == 0 && max == 0 {
min = float64(typeMin)
max = float64(typeMax)
} else {
if min < float64(typeMin) {
min = float64(typeMin)
}
if max > float64(typeMax) || max == 0 {
max = float64(typeMax)
}
}

var emitFNotReturn emitFNotReturn
emitFNotReturn = func(state *genState, buf *bytes.Buffer) error {
var dummyInt int64
Expand Down Expand Up @@ -999,6 +1094,9 @@ func bindLongWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]a
}

if fieldCfg.Counter {
// Get type-specific bounds for counter
_, typeMax := getIntTypeBounds(field.Type)

var emitF emitF

emitF = func(state *genState) any {
Expand All @@ -1015,7 +1113,7 @@ func bindLongWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]a

dummyInt = dummyFunc()
} else {
dummyInt = fuzzyIntCounter(previous, fieldCfg.Fuzziness)
dummyInt = fuzzyIntCounter(previous, fieldCfg.Fuzziness, float64(typeMax))
}

if fieldCfg.CounterReset != nil {
Expand Down Expand Up @@ -1059,9 +1157,25 @@ func bindLongWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]a
return nil
}

// Get type-specific bounds
typeMin, typeMax := getIntTypeBounds(field.Type)

min, _ := fieldCfg.Range.MinAsFloat64()
max, _ := fieldCfg.Range.MaxAsFloat64()

// If no range specified or range exceeds type bounds, use type bounds
if min == 0 && max == 0 {
min = float64(typeMin)
max = float64(typeMax)
} else {
if min < float64(typeMin) {
min = float64(typeMin)
}
if max > float64(typeMax) || max == 0 {
max = float64(typeMax)
}
}

var emitF emitF
emitF = func(state *genState) any {
var dummyInt int64
Expand All @@ -1082,16 +1196,23 @@ func bindLongWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]a
}

func makeIntCounterFunc(previousDummyInt int64, field Field) func() int64 {
// Get type-specific bounds
_, typeMax := getIntTypeBounds(field.Type)

var dummyFunc func() int64

switch {
case len(field.Example) == 0:
dummyFunc = func() int64 { return previousDummyInt + customRand.Int63n(10) }
dummyFunc = func() int64 {
nextValue := previousDummyInt + customRand.Int63n(10)
return clampInt64(nextValue, previousDummyInt, typeMax)
}
default:
totDigit := len(field.Example)
max := int64(math.Pow10(totDigit))
dummyFunc = func() int64 {
return previousDummyInt + customRand.Int63n(max)
nextValue := previousDummyInt + customRand.Int63n(max)
return clampInt64(nextValue, previousDummyInt, typeMax)
}
}

Expand Down
Loading