|
| 1 | +/* |
| 2 | +Copyright 2025 Tim Ebert. |
| 3 | +
|
| 4 | +Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +you may not use this file except in compliance with the License. |
| 6 | +You may obtain a copy of the License at |
| 7 | +
|
| 8 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +
|
| 10 | +Unless required by applicable law or agreed to in writing, software |
| 11 | +distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +See the License for the specific language governing permissions and |
| 14 | +limitations under the License. |
| 15 | +*/ |
| 16 | + |
| 17 | +package chaos |
| 18 | + |
| 19 | +import ( |
| 20 | + "context" |
| 21 | + "fmt" |
| 22 | + "time" |
| 23 | + |
| 24 | + "golang.org/x/time/rate" |
| 25 | + corev1 "k8s.io/api/core/v1" |
| 26 | + "sigs.k8s.io/controller-runtime/pkg/client" |
| 27 | + logf "sigs.k8s.io/controller-runtime/pkg/log" |
| 28 | + |
| 29 | + webhostingv1alpha1 "github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/pkg/apis/webhosting/v1alpha1" |
| 30 | + "github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/pkg/experiment" |
| 31 | + "github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/pkg/experiment/generator" |
| 32 | + "github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/pkg/experiment/scenario/base" |
| 33 | + "github.com/timebertt/kubernetes-controller-sharding/webhosting-operator/pkg/utils" |
| 34 | +) |
| 35 | + |
| 36 | +const ScenarioName = "chaos" |
| 37 | + |
| 38 | +func init() { |
| 39 | + s := &scenario{} |
| 40 | + s.Scenario = &base.Scenario{ |
| 41 | + ScenarioName: ScenarioName, |
| 42 | + Delegate: s, |
| 43 | + } |
| 44 | + |
| 45 | + experiment.RegisterScenario(s) |
| 46 | +} |
| 47 | + |
| 48 | +type scenario struct { |
| 49 | + *base.Scenario |
| 50 | +} |
| 51 | + |
| 52 | +func (s *scenario) Description() string { |
| 53 | + return "Create 4.5k websites over 15 minutes and terminate a random shard every 5 minutes" |
| 54 | +} |
| 55 | + |
| 56 | +func (s *scenario) LongDescription() string { |
| 57 | + return `The ` + ScenarioName + ` scenario generates load and chaos for the webhosting-operator: |
| 58 | +- website creation: 4500 over 15m |
| 59 | +- website spec changes: 0.5/m per object, max 37.5/s |
| 60 | +- shard termination (pod deletion): 1/m |
| 61 | +` |
| 62 | +} |
| 63 | + |
| 64 | +func (s *scenario) Prepare(ctx context.Context) error { |
| 65 | + s.Log.Info("Preparing themes") |
| 66 | + if err := generator.CreateThemes(ctx, s.Client, 50, generator.WithLabels(s.Labels), generator.WithOwnerReference(s.OwnerRef)); err != nil { |
| 67 | + return err |
| 68 | + } |
| 69 | + |
| 70 | + s.Log.Info("Preparing projects") |
| 71 | + if err := generator.CreateProjects(ctx, s.Client, 20, generator.WithLabels(s.Labels), generator.WithOwnerReference(s.OwnerRef)); err != nil { |
| 72 | + return err |
| 73 | + } |
| 74 | + |
| 75 | + return nil |
| 76 | +} |
| 77 | + |
| 78 | +func (s *scenario) Run(ctx context.Context) error { |
| 79 | + // website-generator: creates about 4500 websites over 15 minutes |
| 80 | + if err := (&generator.Every{ |
| 81 | + Name: "website-generator", |
| 82 | + Do: func(ctx context.Context, c client.Client) error { |
| 83 | + return generator.CreateWebsite(ctx, c, generator.WithLabels(s.Labels)) |
| 84 | + }, |
| 85 | + Rate: rate.Limit(5), |
| 86 | + }).AddToManager(s.Manager); err != nil { |
| 87 | + return fmt.Errorf("error adding website-generator: %w", err) |
| 88 | + } |
| 89 | + |
| 90 | + // trigger individual spec changes for website every other minute |
| 91 | + // => peaks at about 37.5 spec changes per second at the end of the experiment |
| 92 | + // (triggers roughly double the reconciliation rate in website controller because of deployment watches) |
| 93 | + if err := (&generator.ForEach[*webhostingv1alpha1.Website]{ |
| 94 | + Name: "website-mutator", |
| 95 | + Do: func(ctx context.Context, c client.Client, obj *webhostingv1alpha1.Website) error { |
| 96 | + return client.IgnoreNotFound(generator.MutateWebsite(ctx, c, obj, s.Labels)) |
| 97 | + }, |
| 98 | + Every: 2 * time.Minute, |
| 99 | + }).AddToManager(s.Manager); err != nil { |
| 100 | + return fmt.Errorf("error adding website-mutator: %w", err) |
| 101 | + } |
| 102 | + |
| 103 | + // Terminate a random shard every 5 minutes |
| 104 | + if err := (&generator.Every{ |
| 105 | + Name: "shard-terminator", |
| 106 | + Do: terminateRandomShard, |
| 107 | + Rate: rate.Every(5 * time.Minute), |
| 108 | + }).AddToManager(s.Manager); err != nil { |
| 109 | + return fmt.Errorf("error adding shard-terminator: %w", err) |
| 110 | + } |
| 111 | + |
| 112 | + return s.Wait(ctx, 15*time.Minute) |
| 113 | +} |
| 114 | + |
| 115 | +func terminateRandomShard(ctx context.Context, c client.Client) error { |
| 116 | + log := logf.FromContext(ctx) |
| 117 | + |
| 118 | + podList := &corev1.PodList{} |
| 119 | + if err := c.List(ctx, podList, |
| 120 | + client.InNamespace(webhostingv1alpha1.NamespaceSystem), |
| 121 | + client.MatchingLabels{"app.kubernetes.io/name": webhostingv1alpha1.WebhostingOperatorName}, |
| 122 | + ); err != nil { |
| 123 | + return err |
| 124 | + } |
| 125 | + |
| 126 | + if len(podList.Items) == 0 { |
| 127 | + log.Info("No shards found, skipping termination") |
| 128 | + return nil |
| 129 | + } |
| 130 | + |
| 131 | + pod := utils.PickRandom(podList.Items) |
| 132 | + if err := c.Delete(ctx, &pod); err != nil { |
| 133 | + return err |
| 134 | + } |
| 135 | + |
| 136 | + log.Info("Terminated shard", "pod", client.ObjectKeyFromObject(&pod)) |
| 137 | + return nil |
| 138 | +} |
0 commit comments