diff --git a/pkg/genlib/generator.go b/pkg/genlib/generator.go index 34eeafb..d7747b4 100644 --- a/pkg/genlib/generator.go +++ b/pkg/genlib/generator.go @@ -31,7 +31,7 @@ func fieldValueWrapByType(field Field) string { return "\"" case FieldTypeDouble, FieldTypeFloat, FieldTypeHalfFloat, FieldTypeScaledFloat: return "" - case FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: + case FieldTypeByte, FieldTypeShort, FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: return "" case FieldTypeConstantKeyword: return "\"" diff --git a/pkg/genlib/generator_interface.go b/pkg/genlib/generator_interface.go index 45e718f..3e0f1bb 100644 --- a/pkg/genlib/generator_interface.go +++ b/pkg/genlib/generator_interface.go @@ -41,6 +41,8 @@ const ( FieldTypeFloat = "float" FieldTypeHalfFloat = "half_float" FieldTypeScaledFloat = "scaled_float" + FieldTypeByte = "byte" + FieldTypeShort = "short" FieldTypeInteger = "integer" FieldTypeLong = "long" FieldTypeUnsignedLong = "unsigned_long" @@ -59,6 +61,36 @@ var ( keywordRegex = regexp.MustCompile("(\\.|-|_|\\s){1,1}") ) +// getIntTypeBounds returns the min and max values for a given integer field type +func getIntTypeBounds(fieldType string) (min int64, max int64) { + switch fieldType { + case FieldTypeByte: + return math.MinInt8, math.MaxInt8 + case FieldTypeShort: + return math.MinInt16, math.MaxInt16 + case FieldTypeInteger: + return math.MinInt32, math.MaxInt32 + case FieldTypeLong: + return math.MinInt64, math.MaxInt64 + case FieldTypeUnsignedLong: + return 0, math.MaxInt64 // Go int64 max; actual ES unsigned_long goes to 2^64-1 + default: + // Default to long bounds + return math.MinInt64, math.MaxInt64 + } +} + +// clampInt64 clamps a value between min and max +func clampInt64(value, min, max int64) int64 { + if value < min { + return min + } + if value > max { + return max + } + return value +} + // This is the emit function for the custom template engine where we stream content directly to the output buffer and no need a return value type emitFNotReturn func(state *genState, buf *bytes.Buffer) error @@ -173,7 +205,7 @@ func bindByType(cfg Config, field Field, fieldMap map[string]any) (err error) { err = bindIP(field, fieldMap) case FieldTypeDouble, FieldTypeFloat, FieldTypeHalfFloat, FieldTypeScaledFloat: err = bindDouble(fieldCfg, field, fieldMap) - case FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: // TODO: generate > 63 bit values for unsigned_long + case FieldTypeByte, FieldTypeShort, FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: // TODO: generate > 63 bit values for unsigned_long err = bindLong(fieldCfg, field, fieldMap) case FieldTypeConstantKeyword: err = bindConstantKeyword(field, fieldMap) @@ -202,7 +234,7 @@ func bindByTypeWithReturn(cfg Config, field Field, fieldMap map[string]any) (err err = bindIPWithReturn(field, fieldMap) case FieldTypeDouble, FieldTypeFloat, FieldTypeHalfFloat, FieldTypeScaledFloat: err = bindDoubleWithReturn(fieldCfg, field, fieldMap) - case FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: // TODO: generate > 63 bit values for unsigned_long + case FieldTypeByte, FieldTypeShort, FieldTypeInteger, FieldTypeLong, FieldTypeUnsignedLong: // TODO: generate > 63 bit values for unsigned_long err = bindLongWithReturn(fieldCfg, field, fieldMap) case FieldTypeConstantKeyword: err = bindConstantKeywordWithReturn(field, fieldMap) @@ -248,25 +280,64 @@ func makeFloatFunc(fieldCfg ConfigField, field Field) func() float64 { } func makeIntFunc(fieldCfg ConfigField, field Field) func() int64 { + // Get type-specific bounds + typeMin, typeMax := getIntTypeBounds(field.Type) + minValue, _ := fieldCfg.Range.MinAsInt64() maxValue, err := fieldCfg.Range.MaxAsInt64() - // maxValue not set, let's set it to 0 for the sake of the switch above - if err != nil { - maxValue = 0 + + // Determine if a range was explicitly configured + hasConfiguredRange := err == nil && maxValue > 0 + + // Clamp configured range to type-specific bounds + if minValue < typeMin { + minValue = typeMin + } + if hasConfiguredRange { + if maxValue > typeMax { + maxValue = typeMax + } + } else { + // No configured range, use type bounds or simple defaults + maxValue = typeMax } var dummyFunc func() int64 switch { - case maxValue > 0: - dummyFunc = func() int64 { return customRand.Int63n(maxValue-minValue) + minValue } + case hasConfiguredRange && maxValue > minValue: + // User specified a range, respect it (clamped to type bounds) + rangeSize := maxValue - minValue + 1 + if rangeSize > 0 { + dummyFunc = func() int64 { return customRand.Int63n(rangeSize) + minValue } + } else { + dummyFunc = func() int64 { return minValue } + } case len(field.Example) == 0: - dummyFunc = func() int64 { return customRand.Int63n(10) } + // No range and no example, generate small values within type bounds + rangeBound := typeMax + if typeMin < 0 { + // For signed types, keep it simple and use 0-10 + dummyFunc = func() int64 { return customRand.Int63n(10) } + } else if rangeBound > 10 { + // For unsigned types with large bounds, use 0-10 + dummyFunc = func() int64 { return customRand.Int63n(10) } + } else { + // For small unsigned types (byte), use full range + dummyFunc = func() int64 { return customRand.Int63n(rangeBound + 1) } + } default: + // Use example length to determine magnitude totDigit := len(field.Example) max := int64(math.Pow10(totDigit)) - dummyFunc = func() int64 { - return customRand.Int63n(max) + // Clamp to type max + if max > typeMax { + max = typeMax + } + if max > 0 { + dummyFunc = func() int64 { return customRand.Int63n(max) } + } else { + dummyFunc = func() int64 { return 0 } } } @@ -566,9 +637,14 @@ func fuzzyInt(previous int64, fuzziness, min, max float64) int64 { return customRand.Int63n(int64(math.Ceil(higherBound-lowerBound))) + int64(lowerBound) } -func fuzzyIntCounter(previous int64, fuzziness float64) int64 { +func fuzzyIntCounter(previous int64, fuzziness, max float64) int64 { lowerBound := float64(previous) higherBound := float64(previous) * (1 + fuzziness) + // Clamp to max bound + higherBound = math.Min(higherBound, max) + if higherBound <= lowerBound { + return previous + } return customRand.Int63n(int64(math.Ceil(higherBound-lowerBound))) + int64(lowerBound) } @@ -578,6 +654,9 @@ func bindLong(fieldCfg ConfigField, field Field, fieldMap map[string]any) error } if fieldCfg.Counter { + // Get type-specific bounds for counter + _, typeMax := getIntTypeBounds(field.Type) + var emitFNotReturn emitFNotReturn emitFNotReturn = func(state *genState, buf *bytes.Buffer) error { previous := int64(1) @@ -593,7 +672,7 @@ func bindLong(fieldCfg ConfigField, field Field, fieldMap map[string]any) error dummyInt = dummyFunc() } else { - dummyInt = fuzzyIntCounter(previous, fieldCfg.Fuzziness) + dummyInt = fuzzyIntCounter(previous, fieldCfg.Fuzziness, float64(typeMax)) } state.prevCache[field.Name] = dummyInt @@ -624,9 +703,25 @@ func bindLong(fieldCfg ConfigField, field Field, fieldMap map[string]any) error return nil } + // Get type-specific bounds + typeMin, typeMax := getIntTypeBounds(field.Type) + min, _ := fieldCfg.Range.MinAsFloat64() max, _ := fieldCfg.Range.MaxAsFloat64() + // If no range specified or range exceeds type bounds, use type bounds + if min == 0 && max == 0 { + min = float64(typeMin) + max = float64(typeMax) + } else { + if min < float64(typeMin) { + min = float64(typeMin) + } + if max > float64(typeMax) || max == 0 { + max = float64(typeMax) + } + } + var emitFNotReturn emitFNotReturn emitFNotReturn = func(state *genState, buf *bytes.Buffer) error { var dummyInt int64 @@ -999,6 +1094,9 @@ func bindLongWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]a } if fieldCfg.Counter { + // Get type-specific bounds for counter + _, typeMax := getIntTypeBounds(field.Type) + var emitF emitF emitF = func(state *genState) any { @@ -1015,7 +1113,7 @@ func bindLongWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]a dummyInt = dummyFunc() } else { - dummyInt = fuzzyIntCounter(previous, fieldCfg.Fuzziness) + dummyInt = fuzzyIntCounter(previous, fieldCfg.Fuzziness, float64(typeMax)) } if fieldCfg.CounterReset != nil { @@ -1059,9 +1157,25 @@ func bindLongWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]a return nil } + // Get type-specific bounds + typeMin, typeMax := getIntTypeBounds(field.Type) + min, _ := fieldCfg.Range.MinAsFloat64() max, _ := fieldCfg.Range.MaxAsFloat64() + // If no range specified or range exceeds type bounds, use type bounds + if min == 0 && max == 0 { + min = float64(typeMin) + max = float64(typeMax) + } else { + if min < float64(typeMin) { + min = float64(typeMin) + } + if max > float64(typeMax) || max == 0 { + max = float64(typeMax) + } + } + var emitF emitF emitF = func(state *genState) any { var dummyInt int64 @@ -1082,16 +1196,23 @@ func bindLongWithReturn(fieldCfg ConfigField, field Field, fieldMap map[string]a } func makeIntCounterFunc(previousDummyInt int64, field Field) func() int64 { + // Get type-specific bounds + _, typeMax := getIntTypeBounds(field.Type) + var dummyFunc func() int64 switch { case len(field.Example) == 0: - dummyFunc = func() int64 { return previousDummyInt + customRand.Int63n(10) } + dummyFunc = func() int64 { + nextValue := previousDummyInt + customRand.Int63n(10) + return clampInt64(nextValue, previousDummyInt, typeMax) + } default: totDigit := len(field.Example) max := int64(math.Pow10(totDigit)) dummyFunc = func() int64 { - return previousDummyInt + customRand.Int63n(max) + nextValue := previousDummyInt + customRand.Int63n(max) + return clampInt64(nextValue, previousDummyInt, typeMax) } } diff --git a/pkg/genlib/generator_type_bounds_test.go b/pkg/genlib/generator_type_bounds_test.go new file mode 100644 index 0000000..85c9c38 --- /dev/null +++ b/pkg/genlib/generator_type_bounds_test.go @@ -0,0 +1,232 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package genlib + +import ( + "bytes" + "math" + "testing" + + "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" +) + +// Test that byte values respect the -128 to 127 range +func Test_ByteFieldRespectsBounds(t *testing.T) { + fld := Field{ + Name: "test_byte", + Type: FieldTypeByte, + } + + template := []byte(`{"test_byte":{{.test_byte}}}`) + cfg, err := config.LoadConfigFromYaml([]byte("")) + if err != nil { + t.Fatal(err) + } + + nSpins := 10000 + g := makeGeneratorWithCustomTemplate(t, cfg, []Field{fld}, template, uint64(nSpins)) + + for i := 0; i < nSpins; i++ { + var buf bytes.Buffer + if err := g.Emit(&buf); err != nil { + t.Fatal(err) + } + + m := unmarshalJSONT[int8](t, buf.Bytes()) + val := m["test_byte"] + + if val < math.MinInt8 || val > math.MaxInt8 { + t.Errorf("Byte value %d is outside valid range [%d, %d]", val, math.MinInt8, math.MaxInt8) + } + } +} + +// Test that short values respect the -32768 to 32767 range +func Test_ShortFieldRespectsBounds(t *testing.T) { + fld := Field{ + Name: "test_short", + Type: FieldTypeShort, + } + + template := []byte(`{"test_short":{{.test_short}}}`) + cfg, err := config.LoadConfigFromYaml([]byte("")) + if err != nil { + t.Fatal(err) + } + + nSpins := 10000 + g := makeGeneratorWithCustomTemplate(t, cfg, []Field{fld}, template, uint64(nSpins)) + + for i := 0; i < nSpins; i++ { + var buf bytes.Buffer + if err := g.Emit(&buf); err != nil { + t.Fatal(err) + } + + m := unmarshalJSONT[int16](t, buf.Bytes()) + val := m["test_short"] + + if val < math.MinInt16 || val > math.MaxInt16 { + t.Errorf("Short value %d is outside valid range [%d, %d]", val, math.MinInt16, math.MaxInt16) + } + } +} + +// Test that integer values respect the -2^31 to 2^31-1 range +func Test_IntegerFieldRespectsBounds(t *testing.T) { + fld := Field{ + Name: "test_integer", + Type: FieldTypeInteger, + } + + template := []byte(`{"test_integer":{{.test_integer}}}`) + cfg, err := config.LoadConfigFromYaml([]byte("")) + if err != nil { + t.Fatal(err) + } + + nSpins := 10000 + g := makeGeneratorWithCustomTemplate(t, cfg, []Field{fld}, template, uint64(nSpins)) + + for i := 0; i < nSpins; i++ { + var buf bytes.Buffer + if err := g.Emit(&buf); err != nil { + t.Fatal(err) + } + + m := unmarshalJSONT[int32](t, buf.Bytes()) + val := m["test_integer"] + + if val < math.MinInt32 || val > math.MaxInt32 { + t.Errorf("Integer value %d is outside valid range [%d, %d]", val, math.MinInt32, math.MaxInt32) + } + } +} + +// Test that configured ranges get clamped to type bounds +func Test_ByteFieldWithLargeRangeGetsClamped(t *testing.T) { + fld := Field{ + Name: "test_byte", + Type: FieldTypeByte, + } + + template := []byte(`{"test_byte":{{.test_byte}}}`) + // Try to configure a range larger than byte can support + configYaml := []byte(`fields: + - name: test_byte + range: + min: 0 + max: 1000`) + + cfg, err := config.LoadConfigFromYaml(configYaml) + if err != nil { + t.Fatal(err) + } + + nSpins := 1000 + g := makeGeneratorWithCustomTemplate(t, cfg, []Field{fld}, template, uint64(nSpins)) + + for i := 0; i < nSpins; i++ { + var buf bytes.Buffer + if err := g.Emit(&buf); err != nil { + t.Fatal(err) + } + + m := unmarshalJSONT[int8](t, buf.Bytes()) + val := m["test_byte"] + + // Value should be clamped to byte range + if val < 0 || val > math.MaxInt8 { + t.Errorf("Byte value %d exceeds clamped range [0, %d]", val, math.MaxInt8) + } + } +} + +// Test that counter mode respects type bounds +func Test_ByteCounterRespectsBounds(t *testing.T) { + fld := Field{ + Name: "test_byte_counter", + Type: FieldTypeByte, + } + + template := []byte(`{"test_byte_counter":{{.test_byte_counter}}}`) + configYaml := []byte(`fields: + - name: test_byte_counter + counter: true`) + + cfg, err := config.LoadConfigFromYaml(configYaml) + if err != nil { + t.Fatal(err) + } + + nSpins := 200 + g := makeGeneratorWithCustomTemplate(t, cfg, []Field{fld}, template, uint64(nSpins)) + + var lastVal int8 = 0 + for i := 0; i < nSpins; i++ { + var buf bytes.Buffer + if err := g.Emit(&buf); err != nil { + t.Fatal(err) + } + + m := unmarshalJSONT[int8](t, buf.Bytes()) + val := m["test_byte_counter"] + + if val < math.MinInt8 || val > math.MaxInt8 { + t.Errorf("Byte counter value %d is outside valid range [%d, %d]", val, math.MinInt8, math.MaxInt8) + } + + // Counter should be monotonically increasing (or stay at max) + if val < lastVal { + t.Errorf("Counter decreased from %d to %d", lastVal, val) + } + + // Once it hits the max, it should stay there or reset + if lastVal == math.MaxInt8 && val != math.MaxInt8 && val != 0 { + t.Logf("Counter hit max (%d) and reset to %d", math.MaxInt8, val) + } + + lastVal = val + } +} + +// Test that fuzziness respects type bounds +func Test_ByteFuzzinessRespectsBounds(t *testing.T) { + fld := Field{ + Name: "test_byte_fuzzy", + Type: FieldTypeByte, + } + + template := []byte(`{"test_byte_fuzzy":{{.test_byte_fuzzy}}}`) + // Start near max and use fuzziness - should not exceed max + configYaml := []byte(`fields: + - name: test_byte_fuzzy + fuzziness: 0.5 + range: + min: 100 + max: 127`) + + cfg, err := config.LoadConfigFromYaml(configYaml) + if err != nil { + t.Fatal(err) + } + + nSpins := 1000 + g := makeGeneratorWithCustomTemplate(t, cfg, []Field{fld}, template, uint64(nSpins)) + + for i := 0; i < nSpins; i++ { + var buf bytes.Buffer + if err := g.Emit(&buf); err != nil { + t.Fatal(err) + } + + m := unmarshalJSONT[int8](t, buf.Bytes()) + val := m["test_byte_fuzzy"] + + if val < 100 || val > math.MaxInt8 { + t.Errorf("Byte fuzzy value %d is outside valid range [100, %d]", val, math.MaxInt8) + } + } +} diff --git a/pkg/genlib/generator_with_custom_template_test.go b/pkg/genlib/generator_with_custom_template_test.go index ef95a8b..c48348e 100644 --- a/pkg/genlib/generator_with_custom_template_test.go +++ b/pkg/genlib/generator_with_custom_template_test.go @@ -3,13 +3,14 @@ package genlib import ( "bytes" "fmt" - "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" "math/rand" "net" "strconv" "strings" "testing" "time" + + "github.com/elastic/elastic-integration-corpus-generator-tool/pkg/genlib/config" ) func Test_ParseTemplate(t *testing.T) { @@ -195,7 +196,10 @@ func Test_EmptyCaseWithCustomTemplate(t *testing.T) { func Test_CardinalityWithCustomTemplate(t *testing.T) { test_CardinalityTWithCustomTemplate[string](t, FieldTypeKeyword) - test_CardinalityTWithCustomTemplate[int](t, FieldTypeInteger) + test_CardinalityTWithCustomTemplate[int8](t, FieldTypeByte) + test_CardinalityTWithCustomTemplate[int16](t, FieldTypeShort) + test_CardinalityTWithCustomTemplate[int32](t, FieldTypeInteger) + test_CardinalityTWithCustomTemplate[int64](t, FieldTypeLong) test_CardinalityTWithCustomTemplate[float64](t, FieldTypeFloat) test_CardinalityTWithCustomTemplate[string](t, FieldTypeGeoPoint) test_CardinalityTWithCustomTemplate[string](t, FieldTypeIP) @@ -204,7 +208,7 @@ func Test_CardinalityWithCustomTemplate(t *testing.T) { func test_CardinalityTWithCustomTemplate[T any](t *testing.T, ty string) { template := []byte(`{"alpha":"{{.alpha}}", "beta":"{{.beta}}"}`) - if ty == FieldTypeInteger || ty == FieldTypeFloat { + if ty == FieldTypeByte || ty == FieldTypeShort || ty == FieldTypeInteger || ty == FieldTypeLong || ty == FieldTypeFloat { template = []byte(`{"alpha":{{.alpha}}, "beta":{{.beta}}}`) } @@ -228,8 +232,25 @@ func test_CardinalityTWithCustomTemplate[T any](t *testing.T, ty string) { rangeTrailing = "." } - rangeMin := rand.Intn(100) - rangeMax := rand.Intn(10000-rangeMin) + rangeMin + // Adjust range based on field type to ensure cardinality is achievable + rangeMin := 0 + rangeMax := 10000 + + // For integer types with limited ranges, ensure we have enough values for max cardinality (200) + switch ty { + case FieldTypeByte: + // byte: -128 to 127 (256 values), use 0-255 but will be clamped to 0-127 + rangeMin = 0 + rangeMax = 255 + case FieldTypeShort: + // short: -32768 to 32767, plenty of range + rangeMin = 0 + rangeMax = 10000 + case FieldTypeInteger, FieldTypeLong: + // Plenty of range for any cardinality + rangeMin = rand.Intn(100) + rangeMax = rand.Intn(10000-rangeMin) + rangeMin + } // Add the range to get some variety in integers tmpl := "fields:\n - name: alpha\n cardinality: %d\n range:\n min: %d%s\n max: %d%s\n" @@ -277,11 +298,32 @@ func test_CardinalityTWithCustomTemplate[T any](t *testing.T, ty string) { vmapBeta[v] = vmapBeta[v] + 1 } - if len(vmapAlpha) != 1000/cardinality { - t.Errorf("Expected cardinality of %d got %d", 1000/cardinality, len(vmapAlpha)) + expectedAlpha := 1000 / cardinality + expectedBeta := 2000 / cardinality + + // For byte type, the maximum achievable cardinality is limited by the type range + // byte range 0-127 = 128 possible values, so cap expectations at 128 + // Also allow for small variance due to collision detection retry limits + tolerance := 0 + if ty == FieldTypeByte { + maxByteCardinality := 128 + if expectedAlpha > maxByteCardinality { + expectedAlpha = maxByteCardinality + } + if expectedBeta > maxByteCardinality { + expectedBeta = maxByteCardinality + } + // For byte type near the limit, allow slight variance due to collision retries + if expectedAlpha > 95 || expectedBeta > 95 { + tolerance = 2 + } + } + + if len(vmapAlpha) < expectedAlpha-tolerance || len(vmapAlpha) > expectedAlpha+tolerance { + t.Errorf("Expected cardinality of %d (±%d) got %d", expectedAlpha, tolerance, len(vmapAlpha)) } - if len(vmapBeta) != 2000/cardinality { - t.Errorf("Expected cardinality of %d got %d", 2000/cardinality, len(vmapBeta)) + if len(vmapBeta) < expectedBeta-tolerance || len(vmapBeta) > expectedBeta+tolerance { + t.Errorf("Expected cardinality of %d (±%d) got %d", expectedBeta, tolerance, len(vmapBeta)) } } } @@ -924,7 +966,9 @@ func Test_FieldFloatsWithCustomTemplate(t *testing.T) { } func Test_FieldIntegersWithCustomTemplate(t *testing.T) { - _testNumericWithCustomTemplate[int](t, FieldTypeInteger) + _testNumericWithCustomTemplate[int8](t, FieldTypeByte) + _testNumericWithCustomTemplate[int16](t, FieldTypeShort) + _testNumericWithCustomTemplate[int32](t, FieldTypeInteger) _testNumericWithCustomTemplate[int64](t, FieldTypeLong) _testNumericWithCustomTemplate[uint64](t, FieldTypeUnsignedLong) } diff --git a/pkg/genlib/generator_with_text_template_test.go b/pkg/genlib/generator_with_text_template_test.go index 5e7999b..e67f80f 100644 --- a/pkg/genlib/generator_with_text_template_test.go +++ b/pkg/genlib/generator_with_text_template_test.go @@ -32,7 +32,10 @@ func Test_EmptyCaseWithTextTemplate(t *testing.T) { func Test_CardinalityWithTextTemplate(t *testing.T) { test_CardinalityTWithTextTemplate[string](t, FieldTypeKeyword) - test_CardinalityTWithTextTemplate[int](t, FieldTypeInteger) + test_CardinalityTWithTextTemplate[int8](t, FieldTypeByte) + test_CardinalityTWithTextTemplate[int16](t, FieldTypeShort) + test_CardinalityTWithTextTemplate[int32](t, FieldTypeInteger) + test_CardinalityTWithTextTemplate[int64](t, FieldTypeLong) test_CardinalityTWithTextTemplate[float64](t, FieldTypeFloat) test_CardinalityTWithTextTemplate[string](t, FieldTypeGeoPoint) test_CardinalityTWithTextTemplate[string](t, FieldTypeIP) @@ -41,7 +44,7 @@ func Test_CardinalityWithTextTemplate(t *testing.T) { func test_CardinalityTWithTextTemplate[T any](t *testing.T, ty string) { template := []byte(`{"alpha":"{{generate "alpha"}}", "beta":"{{generate "beta"}}"}`) - if ty == FieldTypeInteger || ty == FieldTypeFloat { + if ty == FieldTypeByte || ty == FieldTypeShort || ty == FieldTypeInteger || ty == FieldTypeLong || ty == FieldTypeFloat { template = []byte(`{"alpha":{{generate "alpha"}}, "beta":{{generate "beta"}}}`) } @@ -66,8 +69,25 @@ func test_CardinalityTWithTextTemplate[T any](t *testing.T, ty string) { rangeTrailing = "." } - rangeMin := rand.Intn(100) - rangeMax := rand.Intn(10000-rangeMin) + rangeMin + // Adjust range based on field type to ensure cardinality is achievable + rangeMin := 0 + rangeMax := 10000 + + // For integer types with limited ranges, ensure we have enough values for max cardinality (200) + switch ty { + case FieldTypeByte: + // byte: -128 to 127 (256 values), use 0-255 but will be clamped to 0-127 + rangeMin = 0 + rangeMax = 255 + case FieldTypeShort: + // short: -32768 to 32767, plenty of range + rangeMin = 0 + rangeMax = 10000 + case FieldTypeInteger, FieldTypeLong: + // Plenty of range for any cardinality + rangeMin = rand.Intn(100) + rangeMax = rand.Intn(10000-rangeMin) + rangeMin + } // Add the range to get some variety in integers tmpl := "fields:\n - name: alpha\n cardinality: %d\n range:\n min: %d%s\n max: %d%s\n" @@ -116,12 +136,33 @@ func test_CardinalityTWithTextTemplate[T any](t *testing.T, ty string) { vmapBeta[v] = vmapBeta[v] + 1 } - if len(vmapAlpha) != 1000/cardinality { - t.Errorf("Expected cardinality of %d got %d", 1000/cardinality, len(vmapAlpha)) + expectedAlpha := 1000 / cardinality + expectedBeta := 2000 / cardinality + + // For byte type, the maximum achievable cardinality is limited by the type range + // byte range 0-127 = 128 possible values, so cap expectations at 128 + // Also allow for small variance due to collision detection retry limits + tolerance := 0 + if ty == FieldTypeByte { + maxByteCardinality := 128 + if expectedAlpha > maxByteCardinality { + expectedAlpha = maxByteCardinality + } + if expectedBeta > maxByteCardinality { + expectedBeta = maxByteCardinality + } + // For byte type near the limit, allow slight variance due to collision retries + if expectedAlpha > 95 || expectedBeta > 95 { + tolerance = 2 + } + } + + if len(vmapAlpha) < expectedAlpha-tolerance || len(vmapAlpha) > expectedAlpha+tolerance { + t.Errorf("Expected cardinality of %d (±%d) got %d", expectedAlpha, tolerance, len(vmapAlpha)) } - if len(vmapBeta) != 2000/cardinality { - t.Errorf("Expected cardinality of %d got %d", 2000/cardinality, len(vmapBeta)) + if len(vmapBeta) < expectedBeta-tolerance || len(vmapBeta) > expectedBeta+tolerance { + t.Errorf("Expected cardinality of %d (±%d) got %d", expectedBeta, tolerance, len(vmapBeta)) } } } @@ -827,7 +868,9 @@ func Test_FieldFloatsWithTextTemplate(t *testing.T) { } func Test_FieldIntegersWithTextTemplate(t *testing.T) { - _testNumericWithTextTemplate[int](t, FieldTypeInteger) + _testNumericWithTextTemplate[int8](t, FieldTypeByte) + _testNumericWithTextTemplate[int16](t, FieldTypeShort) + _testNumericWithTextTemplate[int32](t, FieldTypeInteger) _testNumericWithTextTemplate[int64](t, FieldTypeLong) _testNumericWithTextTemplate[uint64](t, FieldTypeUnsignedLong) }