Skip to content

Introduce multiple service group categories and maximumMaxReplicas for each of them #428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AVSBharadwaj wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Introduce multiple service group categories and maximumMaxReplicas for each of them #428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d759134
Introduce multiple service group categories and maximumMaxReplicas fo…
AVSBharadwaj Jan 9, 2025
a438211
Change design to include backward compatibility
AVSBharadwaj Jan 15, 2025
e93ac1b
Field name change from Namespace to Selectors
AVSBharadwaj Jan 16, 2025
00134fe
Added test cases for scenarios with sericeGroups and MaximumMaxReplic…
AVSBharadwaj Jan 21, 2025
c15336c
fix call to New()
AVSBharadwaj Jan 22, 2025
0862823
Merge branch 'main' into multiple-service-groups-feature
AVSBharadwaj Jun 13, 2025
44f709d
add config validations and tests for these validations
AVSBharadwaj Jun 20, 2025
4ea6b5d
Add test cases for serviceGroups and maxReplicasPerService
AVSBharadwaj Jun 23, 2025
f36f7d5
Add support for label based matching as well plus test cases for it
AVSBharadwaj Jun 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/autoscaling/v2/webhook_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ var _ = BeforeSuite(func() {
eventRecorder := mgr.GetEventRecorderFor("tortoise-controller")
tortoiseService, err := tortoise.New(mgr.GetClient(), eventRecorder, config.RangeOfMinMaxReplicasRecommendationHours, config.TimeZone, config.TortoiseUpdateInterval, config.GatheringDataPeriodType)
Expect(err).NotTo(HaveOccurred())
hpaService, err := hpa.New(mgr.GetClient(), eventRecorder, config.ReplicaReductionFactor, config.MaximumTargetResourceUtilization, 100, time.Hour, nil, 1000, 10000, 3, "")
hpaService, err := hpa.New(mgr.GetClient(), eventRecorder, config.ReplicaReductionFactor, config.MaximumTargetResourceUtilization, 100, time.Hour, nil, 1000, 10000, 3, config.ServiceGroups, config.MaximumMaxReplicasPerService, "")
Expect(err).NotTo(HaveOccurred())

hpaWebhook := New(tortoiseService, hpaService)
Expand Down
2 changes: 1 addition & 1 deletion cmd/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ func main() {
os.Exit(1)
}

hpaService, err := hpa.New(mgr.GetClient(), eventRecorder, config.ReplicaReductionFactor, config.MaximumTargetResourceUtilization, config.HPATargetUtilizationMaxIncrease, config.HPATargetUtilizationUpdateInterval, config.DefaultHPABehavior, config.MaximumMinReplicas, config.MaximumMaxReplicas, int32(config.MinimumMinReplicas), config.HPAExternalMetricExclusionRegex)
hpaService, err := hpa.New(mgr.GetClient(), eventRecorder, config.ReplicaReductionFactor, config.MaximumTargetResourceUtilization, config.HPATargetUtilizationMaxIncrease, config.HPATargetUtilizationUpdateInterval, config.DefaultHPABehavior, config.MaximumMinReplicas, config.MaximumMaxReplicas, int32(config.MinimumMinReplicas), config.ServiceGroups, config.MaximumMaxReplicasPerService, config.HPAExternalMetricExclusionRegex)
if err != nil {
setupLog.Error(err, "unable to start hpa service")
os.Exit(1)
Expand Down
3 changes: 2 additions & 1 deletion internal/controller/tortoise_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,8 @@ func startController(ctx context.Context) func() {
Expect(err).ShouldNot(HaveOccurred())
cli, err := vpa.New(mgr.GetConfig(), recorder)
Expect(err).ShouldNot(HaveOccurred())
hpaS, err := hpa.New(mgr.GetClient(), recorder, 0.95, 90, 25, time.Hour, nil, 1000, 10000, 3, ".*-exclude-metric")

hpaS, err := hpa.New(mgr.GetClient(), recorder, 0.95, 90, 25, time.Hour, nil, 1000, 10000, 3, nil, nil, ".*-exclude-metric")
Expect(err).ShouldNot(HaveOccurred())
reconciler := &TortoiseReconciler{
Scheme: scheme,
Expand Down
87 changes: 85 additions & 2 deletions pkg/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"gopkg.in/yaml.v3"
v2 "k8s.io/api/autoscaling/v2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/mercari/tortoise/pkg/features"
)
Expand Down Expand Up @@ -285,11 +286,38 @@ type Config struct {
// IstioSidecarProxyDefaultMemory is the default Memory resource request of the istio sidecar proxy (default: 200Mi)
IstioSidecarProxyDefaultMemory string `yaml:"IstioSidecarProxyDefaultMemory"`

// ServiceGroups defines a list of service category names.
ServiceGroups []ServiceGroup `yaml:"ServiceGroups"`
// MaximumMaxReplicasPerService is the maximum maxReplicas that tortoise can give to the HPA per service category.
// If the service category is not found in this list, tortoise uses the default value which is the value set in MaximumMaxReplicas.
MaximumMaxReplicasPerService []MaximumMaxReplicasPerGroup `yaml:"MaximumMaxReplicasPerService"`

// FeatureFlags is the list of feature flags (default: empty = all alpha features are disabled)
// See the list of feature flags in features.go
FeatureFlags []features.FeatureFlag `yaml:"FeatureFlags"`
}

type MaximumMaxReplicasPerGroup struct {
// ServiceGroupName refers to one ServiceGroup at Config.ServiceGroups
ServiceGroupName string `yaml:"ServiceGroupName"`

MaximumMaxReplica int32 `yaml:"MaximumMaxReplica"`
}

// Selector selects a group of pods by matching its namespace and labels.
type Selector struct {
Namespace string `yaml:"Namespace"` // Namespace name
LabelSelectors []*metav1.LabelSelector `yaml:"Labels,omitempty"` // Slice of label selectors within this namespace
}

// ServiceGroup represents a collection of services grouped together with namespace awareness.
type ServiceGroup struct {
// Name is the group's name (e.g., big-service, fintech-service, etc).
Name string `yaml:"Name"`
// Selectors represent multiple pod groups that belong to this ServiceGroup.
Selectors []Selector `yaml:"Selectors"` // Slice of namespaces and labels within this service group
}

func defaultConfig() *Config {
return &Config{
RangeOfMinMaxReplicasRecommendationHours: 1,
Expand Down Expand Up @@ -318,6 +346,8 @@ func defaultConfig() *Config {
IstioSidecarProxyDefaultMemory: "200Mi",
MinimumCPULimit: "0",
ResourceLimitMultiplier: map[string]int64{},
ServiceGroups: []ServiceGroup{},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{},
BufferRatioOnVerticalResource: 0.1,
}
}
Expand Down Expand Up @@ -414,11 +444,64 @@ func validate(config *Config) error {
if config.MinimumMinReplicas >= int(config.MaximumMinReplicas) {
return fmt.Errorf("MinimumMinReplicas should be less than MaximumMinReplicas")
}

// Find the minimum value of MaximumMaxReplicas across all service groups
minOfMaximumMaxReplicas := config.MaximumMaxReplicas // Start with the default value of MaximumMaxReplicas
for _, group := range config.MaximumMaxReplicasPerService {
if group.MaximumMaxReplica < minOfMaximumMaxReplicas {
minOfMaximumMaxReplicas = group.MaximumMaxReplica
}
}

// Check for non-negative values
if minOfMaximumMaxReplicas < 0 {
return fmt.Errorf("MaximumMaxReplicas should contain non-negative values")
}

// Ensure ServiceGroupNames in MaximumMaxReplicas match defined ServiceGroups
// Ensure no duplicates in ServiceGroups
seenServiceGroups := make(map[string]bool)
serviceGroupMap := make(map[string]bool)
for _, sg := range config.ServiceGroups {
if sg.Name == "" {
return fmt.Errorf("name of the service group should not be empty")
}
if seenServiceGroups[sg.Name] {
return fmt.Errorf("duplicate ServiceGroupName found: %s", sg.Name)
}
seenServiceGroups[sg.Name] = true
serviceGroupMap[sg.Name] = true
}

// Ensure MaximumMaxReplicasPerService is defined in ServiceGroups
// Check all entries in MaximumMaxReplicasPerService have non-nil ServiceGroupName
for _, maxReplicas := range config.MaximumMaxReplicasPerService {
if maxReplicas.ServiceGroupName == "" {
return fmt.Errorf("ServiceGroupName should not be nil in MaximumMaxReplicasPerService entries")
}
if _, exists := serviceGroupMap[maxReplicas.ServiceGroupName]; !exists {
return fmt.Errorf("ServiceGroupName %s in MaximumMaxReplicas is not defined in ServiceGroups", maxReplicas.ServiceGroupName)
}
}

// Validate MaximumMinReplicas against default first, then service groups
if config.MaximumMinReplicas > config.MaximumMaxReplicas {
return fmt.Errorf("MaximumMinReplicas should be less than or equal to MaximumMaxReplicas")
return fmt.Errorf("MaximumMinReplicas (%v) should be less than or equal to MaximumMaxReplicas (%v)", config.MaximumMinReplicas, config.MaximumMaxReplicas)
}
for _, group := range config.MaximumMaxReplicasPerService {
if config.MaximumMinReplicas > group.MaximumMaxReplica {
return fmt.Errorf("MaximumMinReplicas (%v) should be less than or equal to MaximumMaxReplica (%v) for service group %s", config.MaximumMinReplicas, group.MaximumMaxReplica, group.ServiceGroupName)
}
}

// Validate PreferredMaxReplicas against default first, then service groups
if config.PreferredMaxReplicas >= int(config.MaximumMaxReplicas) {
return fmt.Errorf("PreferredMaxReplicas should be less than MaximumMaxReplicas")
return fmt.Errorf("PreferredMaxReplicas (%v) should be less than MaximumMaxReplicas (%v)", config.PreferredMaxReplicas, config.MaximumMaxReplicas)
}
for _, group := range config.MaximumMaxReplicasPerService {
if config.PreferredMaxReplicas >= int(group.MaximumMaxReplica) {
return fmt.Errorf("PreferredMaxReplicas (%v) should be less than MaximumMaxReplica (%v) for service group %s", config.PreferredMaxReplicas, group.MaximumMaxReplica, group.ServiceGroupName)
}
}
if config.PreferredMaxReplicas <= config.MinimumMinReplicas {
return fmt.Errorf("PreferredMaxReplicas should be greater than or equal to MinimumMinReplicas")
Expand Down
192 changes: 191 additions & 1 deletion pkg/config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ func TestParseConfig(t *testing.T) {
IstioSidecarProxyDefaultCPU: "100m",
IstioSidecarProxyDefaultMemory: "200Mi",
MaxAllowedScalingDownRatio: 0.5,
ServiceGroups: []ServiceGroup{},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{},
MinimumCPURequestPerContainer: map[string]string{
"istio-proxy": "100m",
"hoge-agent": "120m",
Expand Down Expand Up @@ -95,6 +97,8 @@ func TestParseConfig(t *testing.T) {
MinimumCPURequestPerContainer: map[string]string{},
MinimumMemoryRequestPerContainer: map[string]string{},
ResourceLimitMultiplier: map[string]int64{},
ServiceGroups: []ServiceGroup{},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{},
BufferRatioOnVerticalResource: 0.1,
},
},
Expand Down Expand Up @@ -137,6 +141,8 @@ func TestParseConfig(t *testing.T) {
MinimumCPURequestPerContainer: map[string]string{},
MinimumMemoryRequestPerContainer: map[string]string{},
ResourceLimitMultiplier: map[string]int64{},
ServiceGroups: []ServiceGroup{},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{},
BufferRatioOnVerticalResource: 0.1,
},
},
Expand Down Expand Up @@ -226,7 +232,7 @@ func Test_validate(t *testing.T) {
wantErr: true,
},
{
name: "invalid PreferredMaxReplicas",
name: "invalid PreferredMaxReplicas less than MinimumMinReplicas",
config: &Config{
RangeOfMinMaxReplicasRecommendationHours: 2,
GatheringDataPeriodType: "daily",
Expand Down Expand Up @@ -572,6 +578,190 @@ func Test_validate(t *testing.T) {
},
wantErr: true,
},
// ServiceGroup validation test cases
{
name: "invalid ServiceGroup - empty service group name",
config: &Config{
RangeOfMinMaxReplicasRecommendationHours: 1,
GatheringDataPeriodType: "weekly",
HPATargetUtilizationMaxIncrease: 5,
MinimumMinReplicas: 3,
MaximumMinReplicas: 10,
MaximumMaxReplicas: 100,
PreferredMaxReplicas: 30,
MaxAllowedScalingDownRatio: 0.8,
ServiceGroups: []ServiceGroup{
{
Name: "", // Empty name should be invalid
},
},
},
wantErr: true,
},
{
name: "invalid ServiceGroup - duplicate service group names",
config: &Config{
RangeOfMinMaxReplicasRecommendationHours: 1,
GatheringDataPeriodType: "weekly",
HPATargetUtilizationMaxIncrease: 5,
MinimumMinReplicas: 3,
MaximumMinReplicas: 10,
MaximumMaxReplicas: 100,
PreferredMaxReplicas: 30,
MaxAllowedScalingDownRatio: 0.8,
ServiceGroups: []ServiceGroup{
{Name: "frontend"},
{Name: "frontend"}, // Duplicate name should be invalid
},
},
wantErr: true,
},
// MaximumMaxReplicasPerGroup validation test cases
{
name: "invalid MaximumMaxReplicasPerService - empty ServiceGroupName",
config: &Config{
RangeOfMinMaxReplicasRecommendationHours: 1,
GatheringDataPeriodType: "weekly",
HPATargetUtilizationMaxIncrease: 5,
MinimumMinReplicas: 3,
MaximumMinReplicas: 10,
MaximumMaxReplicas: 100,
PreferredMaxReplicas: 30,
MaxAllowedScalingDownRatio: 0.8,
ServiceGroups: []ServiceGroup{
{Name: "frontend"},
},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{
{
ServiceGroupName: "", // Empty ServiceGroupName should be invalid
MaximumMaxReplica: 50,
},
},
},
wantErr: true,
},
{
name: "invalid MaximumMaxReplicasPerService - ServiceGroupName not defined in ServiceGroups",
config: &Config{
RangeOfMinMaxReplicasRecommendationHours: 1,
GatheringDataPeriodType: "weekly",
HPATargetUtilizationMaxIncrease: 5,
MinimumMinReplicas: 3,
MaximumMinReplicas: 10,
MaximumMaxReplicas: 100,
PreferredMaxReplicas: 30,
MaxAllowedScalingDownRatio: 0.8,
ServiceGroups: []ServiceGroup{
{Name: "frontend"},
},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{
{
ServiceGroupName: "backend", // Not defined in ServiceGroups
MaximumMaxReplica: 50,
},
},
},
wantErr: true,
},
{
name: "invalid MaximumMaxReplicasPerService - negative MaximumMaxReplica",
config: &Config{
RangeOfMinMaxReplicasRecommendationHours: 1,
GatheringDataPeriodType: "weekly",
HPATargetUtilizationMaxIncrease: 5,
MinimumMinReplicas: 3,
MaximumMinReplicas: 10,
MaximumMaxReplicas: 100,
PreferredMaxReplicas: 30,
MaxAllowedScalingDownRatio: 0.8,
ServiceGroups: []ServiceGroup{
{Name: "frontend"},
},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{
{
ServiceGroupName: "frontend",
MaximumMaxReplica: -5, // Negative value should be invalid
},
},
},
wantErr: true,
},
{
name: "invalid MaximumMinReplicas greater than service group MaximumMaxReplica",
config: &Config{
RangeOfMinMaxReplicasRecommendationHours: 1,
GatheringDataPeriodType: "weekly",
HPATargetUtilizationMaxIncrease: 5,
MinimumMinReplicas: 3,
MaximumMinReplicas: 15, // Greater than service group MaximumMaxReplica
MaximumMaxReplicas: 100,
PreferredMaxReplicas: 30,
MaxAllowedScalingDownRatio: 0.8,
ServiceGroups: []ServiceGroup{
{Name: "frontend"},
},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{
{
ServiceGroupName: "frontend",
MaximumMaxReplica: 10, // Less than MaximumMinReplicas
},
},
},
wantErr: true,
},
{
name: "invalid PreferredMaxReplicas greater than service group MaximumMaxReplica",
config: &Config{
RangeOfMinMaxReplicasRecommendationHours: 1,
GatheringDataPeriodType: "weekly",
HPATargetUtilizationMaxIncrease: 5,
MinimumMinReplicas: 3,
MaximumMinReplicas: 10,
MaximumMaxReplicas: 100,
PreferredMaxReplicas: 25, // Greater than service group MaximumMaxReplica
MaxAllowedScalingDownRatio: 0.8,
ServiceGroups: []ServiceGroup{
{Name: "frontend"},
},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{
{
ServiceGroupName: "frontend",
MaximumMaxReplica: 20, // Less than PreferredMaxReplicas
},
},
},
wantErr: true,
},
{
name: "valid ServiceGroups and MaximumMaxReplicasPerService configuration",
config: &Config{
RangeOfMinMaxReplicasRecommendationHours: 1,
GatheringDataPeriodType: "weekly",
HPATargetUtilizationMaxIncrease: 5,
MinimumTargetResourceUtilization: 65,
MaximumTargetResourceUtilization: 90,
MinimumMinReplicas: 3,
MaximumMinReplicas: 10,
MaximumMaxReplicas: 100,
PreferredMaxReplicas: 30,
MaxAllowedScalingDownRatio: 0.8,
ServiceGroups: []ServiceGroup{
{Name: "frontend"},
{Name: "backend"},
},
MaximumMaxReplicasPerService: []MaximumMaxReplicasPerGroup{
{
ServiceGroupName: "frontend",
MaximumMaxReplica: 50,
},
{
ServiceGroupName: "backend",
MaximumMaxReplica: 80,
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
Loading