diff --git a/tests/integration/godog/features/experiment/model_traffic_split.feature b/tests/integration/godog/features/experiment/model_traffic_split.feature new file mode 100644 index 0000000000..81c768836a --- /dev/null +++ b/tests/integration/godog/features/experiment/model_traffic_split.feature @@ -0,0 +1,54 @@ +@ExperimentTrafficSplit @Functional @Experiments +Feature: Experiment traffic splitting + In order to perform A/B testing + As a model user + I need to create an Experiment resource that splits traffic 50/50 between models + + Scenario: Success - Create experiment with 50/50 traffic split between two iris models + Given I deploy model spec with timeout "10s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Model + metadata: + name: experiment-1 + spec: + replicas: 1 + requirements: + - sklearn + - mlserver + storageUri: gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn + """ + When the model "experiment-1" should eventually become Ready with timeout "20s" + Given I deploy model spec with timeout "10s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Model + metadata: + name: experiment-2 + spec: + replicas: 1 + requirements: + - sklearn + - mlserver + storageUri: gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn + """ + When the model "experiment-2" should eventually become Ready with timeout "20s" + When I deploy experiment spec with timeout "60s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Experiment + metadata: + name: experiment-50-50 + spec: + default: experiment-1 + candidates: + - name: experiment-1 + weight: 50 + - name: experiment-2 + weight: 50 + """ + Then the experiment should eventually become Ready with timeout "60s" + When I send "20" HTTP inference requests to the experiment and expect all models in response, with payoad: + """ + {"inputs": [{"name": "predict", "shape": [1, 4], "datatype": "FP32", "data": [[1, 2, 3, 4]]}]} + """ \ No newline at end of file diff --git a/tests/integration/godog/features/experiment/server_setup.feature b/tests/integration/godog/features/experiment/server_setup.feature new file mode 100644 index 0000000000..ae3df802bf --- /dev/null +++ b/tests/integration/godog/features/experiment/server_setup.feature @@ -0,0 +1,32 @@ +@ServerSetup +Feature: Server setup + Deploys an mlserver with one replica. We ensure the pods + become ready and remove any other server pods for different + servers. + + @ServerSetup @ServerSetupMLServer + Scenario: Deploy mlserver Server and remove other servers + Given I deploy server spec with timeout "10s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Server + metadata: + name: godog-mlserver + spec: + replicas: 1 + serverConfig: mlserver + """ + When the server should eventually become Ready with timeout "30s" + Then ensure only "1" pod(s) are deployed for server and they are Ready + + + @ServerSetup @ServerClean + Scenario: Remove any other pre-existing servers + Given I remove any other server deployments which are not "godog-mlserver,godog-triton" + +# TODO decide if we want to keep this, if we keep testers will need to ensure they don't run this tag when running all +# all features in this directory, as tests will fail when server is deleted. We can not delete and it's up to the +# feature dir server setup to ensure ONLY the required servers exist, like above. +# @ServerTeardown +# Scenario: Delete mlserver Server +# Given I delete server "godog-mlserver" with timeout "10s" \ No newline at end of file diff --git a/tests/integration/godog/go.mod b/tests/integration/godog/go.mod index 4d3bf828ef..d66c7762a3 100644 --- a/tests/integration/godog/go.mod +++ b/tests/integration/godog/go.mod @@ -9,6 +9,8 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.7 google.golang.org/grpc v1.73.0 + google.golang.org/protobuf v1.36.6 + k8s.io/api v0.34.3 k8s.io/apiextensions-apiserver v0.34.3 k8s.io/apimachinery v0.34.3 k8s.io/client-go v0.34.3 @@ -68,11 +70,9 @@ require ( golang.org/x/time v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/protobuf v1.36.6 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.34.3 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect diff --git a/tests/integration/godog/k8sclient/client.go b/tests/integration/godog/k8sclient/client.go index 376cbf4c77..51dbcc6275 100644 --- a/tests/integration/godog/k8sclient/client.go +++ b/tests/integration/godog/k8sclient/client.go @@ -129,6 +129,15 @@ func (k8s *K8sClient) DeleteScenarioResources(ctx context.Context, labels client return fmt.Errorf("failed to delete Pipelines: %w", err) } + if err := k8s.KubeClient.DeleteAllOf( + ctx, + &mlopsv1alpha1.Experiment{}, + client.InNamespace(k8s.namespace), + labels, + ); err != nil { + return fmt.Errorf("failed to delete Experiments: %w", err) + } + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() @@ -163,7 +172,18 @@ func (k8s *K8sClient) DeleteScenarioResources(ctx context.Context, labels client return fmt.Errorf("failed to list Pipelines: %w", err) } - if len(modelList.Items) == 0 && len(pipelineList.Items) == 0 { + // Check Experiments + var experimentList mlopsv1alpha1.ExperimentList + if err := k8s.KubeClient.List( + ctx, + &experimentList, + client.InNamespace(k8s.namespace), + labels, + ); err != nil { + return fmt.Errorf("failed to list Experiments: %w", err) + } + + if len(modelList.Items) == 0 && len(pipelineList.Items) == 0 && len(experimentList.Items) == 0 { return nil } } diff --git a/tests/integration/godog/k8sclient/watcher_store.go b/tests/integration/godog/k8sclient/watcher_store.go index acdf1dc6b8..e8701b9d77 100644 --- a/tests/integration/godog/k8sclient/watcher_store.go +++ b/tests/integration/godog/k8sclient/watcher_store.go @@ -27,6 +27,7 @@ import ( type WatcherStorage interface { WaitForObjectCondition(ctx context.Context, obj runtime.Object, cond ConditionFunc) error WaitForModelCondition(ctx context.Context, modelName string, cond ConditionFunc) error + WaitForExperimentCondition(ctx context.Context, experimentName string, cond ConditionFunc) error WaitForPipelineCondition(ctx context.Context, modelName string, cond ConditionFunc) error Clear() Start() @@ -36,18 +37,20 @@ type WatcherStorage interface { type objectKind string const ( - model objectKind = "Model" - pipeline objectKind = "Pipeline" + model objectKind = "Model" + pipeline objectKind = "Pipeline" + experiment objectKind = "Experiment" ) type WatcherStore struct { - namespace string - label string - mlopsClient v1alpha1.MlopsV1alpha1Interface - modelWatcher watch.Interface - pipelineWatcher watch.Interface - logger log.FieldLogger - scheme *runtime.Scheme + namespace string + label string + mlopsClient v1alpha1.MlopsV1alpha1Interface + modelWatcher watch.Interface + pipelineWatcher watch.Interface + experimentWatcher watch.Interface + logger log.FieldLogger + scheme *runtime.Scheme mu sync.RWMutex store map[string]runtime.Object // key: "namespace/name" @@ -80,21 +83,27 @@ func NewWatcherStore(namespace string, label string, mlopsClient v1alpha1.MlopsV return nil, fmt.Errorf("failed to create pipeline watcher: %w", err) } + experimentWatcher, err := mlopsClient.Experiments(namespace).Watch(context.Background(), v1.ListOptions{LabelSelector: DefaultCRDTestSuiteLabel}) + if err != nil { + return nil, fmt.Errorf("failed to create experiment watcher: %w", err) + } + // Base scheme + register your CRDs s := runtime.NewScheme() _ = scheme.AddToScheme(s) // core k8s types (optional but fine) _ = mlopsscheme.AddToScheme(s) // <-- this is the key line for your CRDs return &WatcherStore{ - namespace: namespace, - label: label, - mlopsClient: mlopsClient, - modelWatcher: modelWatcher, - pipelineWatcher: pipelineWatcher, - logger: logger.WithField("client", "watcher_store"), - store: make(map[string]runtime.Object), - doneChan: make(chan struct{}), - scheme: s, + namespace: namespace, + label: label, + mlopsClient: mlopsClient, + modelWatcher: modelWatcher, + experimentWatcher: experimentWatcher, + pipelineWatcher: pipelineWatcher, + logger: logger.WithField("client", "watcher_store"), + store: make(map[string]runtime.Object), + doneChan: make(chan struct{}), + scheme: s, }, nil } @@ -172,6 +181,42 @@ func (s *WatcherStore) Start() { } } }() + go func() { + for { + select { + case event, ok := <-s.experimentWatcher.ResultChan(): + if !ok { + // channel closed: watcher terminated + return + } + + accessor, err := meta.Accessor(event.Object) + if err != nil { + s.logger.WithError(err).Error("failed to access experiment watcher") + } else { + s.logger.WithField("event", event).Tracef("new experiment watch event with name: %s on namespace: %s", accessor.GetName(), accessor.GetNamespace()) + } + + if event.Object == nil { + continue + } + + switch event.Type { + case watch.Added, watch.Modified: + s.put(event.Object) + case watch.Deleted: + s.delete(event.Object) + case watch.Error: + fmt.Printf("experiment watch error: %v\n", event.Object) + } + + case <-s.doneChan: + // Stop underlying watcher and exit + s.experimentWatcher.Stop() + return + } + } + }() } // Stop terminates the watcher loop. @@ -308,6 +353,12 @@ func (s *WatcherStore) WaitForObjectCondition(ctx context.Context, obj runtime.O return err } } + +func (s *WatcherStore) WaitForExperimentCondition(ctx context.Context, experimentName string, cond ConditionFunc) error { + key := fmt.Sprintf("%s/%s/%s", s.namespace, experiment, experimentName) + return s.waitForKey(ctx, key, cond) +} + func (s *WatcherStore) WaitForModelCondition(ctx context.Context, modelName string, cond ConditionFunc) error { key := fmt.Sprintf("%s/%s/%s", s.namespace, model, modelName) return s.waitForKey(ctx, key, cond) diff --git a/tests/integration/godog/steps/assertions/model.go b/tests/integration/godog/steps/assertions/model.go index d2be8b0af3..68535d3aa1 100644 --- a/tests/integration/godog/steps/assertions/model.go +++ b/tests/integration/godog/steps/assertions/model.go @@ -26,11 +26,20 @@ func ModelReady(obj runtime.Object) (bool, error) { return false, fmt.Errorf("unexpected type %T, expected *v1alpha1.Model", obj) } - if model.Status.IsReady() { - return true, nil + return model.Status.IsReady(), nil +} + +func ExperimentReady(obj runtime.Object) (bool, error) { + if obj == nil { + return false, nil + } + + experiment, ok := obj.(*v1alpha1.Experiment) + if !ok { + return false, fmt.Errorf("unexpected type %T, expected *v1alpha1.Experiment", obj) } - return false, nil + return experiment.Status.IsReady(), nil } //func ModelReadyMessageCondition(expectedMessage string) k8sclient.ConditionFunc { diff --git a/tests/integration/godog/steps/custom_model_steps.go b/tests/integration/godog/steps/custom_model_steps.go index e3f9398d3a..12ddbfbdcf 100644 --- a/tests/integration/godog/steps/custom_model_steps.go +++ b/tests/integration/godog/steps/custom_model_steps.go @@ -14,12 +14,31 @@ import ( "errors" "fmt" + "github.com/cucumber/godog" "github.com/seldonio/seldon-core/tests/integration/godog/steps/assertions" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" ) +func LoadCustomModelSteps(scenario *godog.ScenarioContext, w *World) { + scenario.Step(`^I deploy model spec with timeout "([^"]+)":$`, func(timeout string, spec *godog.DocString) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.currentModel.deployModelSpec(ctx, spec) + }) + }) + scenario.Step(`^the model "([^"]+)" should eventually become Ready with timeout "([^"]+)"$`, func(model, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.currentModel.waitForModelNameReady(ctx, model) + }) + }) + scenario.Step(`^delete the model "([^"]+)" with timeout "([^"]+)"$`, func(model, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.currentModel.deleteModel(ctx, model) + }) + }) +} + // deleteModel we have to wait for model to be deleted, as there is a finalizer attached so the scheduler can confirm // when model has been unloaded from inference pod, model-gw, dataflow-engine, pipeline-gw and controller will remove // finalizer so deletion can complete. diff --git a/tests/integration/godog/steps/experiment_steps.go b/tests/integration/godog/steps/experiment_steps.go new file mode 100644 index 0000000000..bf1e4b3dc1 --- /dev/null +++ b/tests/integration/godog/steps/experiment_steps.go @@ -0,0 +1,155 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package steps + +import ( + "context" + "encoding/json" + "fmt" + "io" + "maps" + "net/http" + "strings" + + "github.com/cucumber/godog" + mlopsv1alpha1 "github.com/seldonio/seldon-core/operator/v2/apis/mlops/v1alpha1" + "github.com/seldonio/seldon-core/operator/v2/pkg/generated/clientset/versioned" + "github.com/seldonio/seldon-core/tests/integration/godog/k8sclient" + "github.com/seldonio/seldon-core/tests/integration/godog/steps/assertions" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +type Experiment struct { + namespace string + label map[string]string + experiment *mlopsv1alpha1.Experiment + k8sClient versioned.Interface + watcherStorage k8sclient.WatcherStorage + log logrus.FieldLogger +} + +func newExperiment(label map[string]string, namespace string, k8sClient versioned.Interface, log logrus.FieldLogger, watcherStorage k8sclient.WatcherStorage) *Experiment { + return &Experiment{label: label, experiment: &mlopsv1alpha1.Experiment{}, log: log, namespace: namespace, k8sClient: k8sClient, watcherStorage: watcherStorage} +} + +func LoadExperimentSteps(scenarioCtx *godog.ScenarioContext, w *World) { + scenarioCtx.Step(`^I deploy experiment spec with timeout "([^"]+)":$`, func(timeout string, docString *godog.DocString) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.currentExperiment.deployExperiment(ctx, docString.Content) + }) + }) + scenarioCtx.Step(`^the experiment should eventually become Ready with timeout "([^"]+)"$`, func(timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.currentExperiment.waitForExperimentReady(ctx) + }) + }) + scenarioCtx.Step(`^I send "([^"]+)" HTTP inference requests to the experiment and expect all models in response, with payoad:$`, func(count int, docString *godog.DocString) error { + return withTimeoutCtx("30s", func(ctx context.Context) error { + return w.currentExperiment.sendHTTPInferenceRequestsToExperiment(ctx, count, docString.Content, w.infer) + }) + }) +} + +func (e *Experiment) deployExperiment(ctx context.Context, yamlSpec string) error { + var experiment mlopsv1alpha1.Experiment + if err := yaml.Unmarshal([]byte(yamlSpec), &experiment); err != nil { + return fmt.Errorf("failed to unmarshal experiment spec: %w", err) + } + + experiment.Namespace = e.namespace + e.experiment = &experiment + e.applyScenarioLabel() + + _, err := e.k8sClient.MlopsV1alpha1().Experiments(e.namespace).Create( + ctx, + e.experiment, + metav1.CreateOptions{}, + ) + if err != nil { + return fmt.Errorf("failed to create experiment: %w", err) + } + + return nil +} + +func (e *Experiment) applyScenarioLabel() { + if e.experiment.Labels == nil { + e.experiment.Labels = make(map[string]string) + } + + maps.Copy(e.experiment.Labels, e.label) + + // todo: change this approach + for k, v := range k8sclient.DefaultCRDTestSuiteLabelMap { + e.experiment.Labels[k] = v + } +} + +func (e *Experiment) waitForExperimentReady(ctx context.Context) error { + return e.watcherStorage.WaitForExperimentCondition( + ctx, + e.experiment.Name, + assertions.ExperimentReady) +} + +func (e *Experiment) sendHTTPInferenceRequestsToExperiment(ctx context.Context, count int, body string, infer inference) error { + modelRespCount := make(map[string]uint) + + type respJSON struct { + Model string `json:"model_name"` + } + + for i := 0; i < count; i++ { + if err := infer.doHTTPExperimentInferenceRequest(ctx, e.experiment.Name, body); err != nil { + return fmt.Errorf("request %d failed: %w", i+1, err) + } + + if infer.lastHTTPResponse.StatusCode != http.StatusOK { + return fmt.Errorf("request failed with status %d", infer.lastHTTPResponse.StatusCode) + } + + body, err := io.ReadAll(infer.lastHTTPResponse.Body) + if err != nil { + return fmt.Errorf("failed reading resp body: %w", err) + } + + e.log.Debugf("Got response HTTP body %+v", string(body)) + + var resp respJSON + if err := json.Unmarshal(body, &resp); err != nil { + return fmt.Errorf("failed unmarshalling response body: %w", err) + } + + if resp.Model == "" { + return fmt.Errorf("response has no model name: %s", string(body)) + } + + index := strings.Index(resp.Model, "_") + if index == -1 { + return fmt.Errorf("response model name missing _: %s", string(body)) + } + gotModel := resp.Model[0:index] + modelRespCount[gotModel]++ + } + + for _, candidate := range e.experiment.Spec.Candidates { + count, ok := modelRespCount[candidate.Name] + if !ok { + return fmt.Errorf("model %s not found in any HTTP response", candidate.Name) + } + if count == 0 { + return fmt.Errorf("model %s response is zero", candidate.Name) + } + } + + return nil +} diff --git a/tests/integration/godog/steps/infer_steps.go b/tests/integration/godog/steps/infer_steps.go index 1c84ad465e..053c9bf3c9 100644 --- a/tests/integration/godog/steps/infer_steps.go +++ b/tests/integration/godog/steps/infer_steps.go @@ -74,13 +74,13 @@ func LoadInferenceSteps(scenario *godog.ScenarioContext, w *World) { }) } -func (i *inference) doHTTPModelInferenceRequest(ctx context.Context, modelName, body string) error { +func (i *inference) doHTTPInferenceRequest(ctx context.Context, resourceName, headerName, body string) error { url := fmt.Sprintf( "%s://%s:%d/v2/models/%s/infer", httpScheme(i.ssl), i.host, i.httpPort, - modelName, + resourceName, ) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, strings.NewReader(body)) @@ -90,7 +90,7 @@ func (i *inference) doHTTPModelInferenceRequest(ctx context.Context, modelName, req.Header.Add("Content-Type", "application/json") req.Header.Add("Host", "seldon-mesh.inference.seldon") - req.Header.Add("Seldon-model", modelName) + req.Header.Add("Seldon-model", headerName) resp, err := i.http.Do(req) if err != nil { @@ -101,6 +101,14 @@ func (i *inference) doHTTPModelInferenceRequest(ctx context.Context, modelName, return nil } +func (i *inference) doHTTPExperimentInferenceRequest(ctx context.Context, experimentName, body string) error { + return i.doHTTPInferenceRequest(ctx, experimentName, fmt.Sprintf("%s.experiment", experimentName), body) +} + +func (i *inference) doHTTPModelInferenceRequest(ctx context.Context, modelName, body string) error { + return i.doHTTPInferenceRequest(ctx, modelName, modelName, body) +} + // Used from steps that pass an explicit payload (DocString) func (i *inference) sendHTTPModelInferenceRequest(ctx context.Context, model string, payload *godog.DocString) error { return i.doHTTPModelInferenceRequest(ctx, model, payload.Content) diff --git a/tests/integration/godog/steps/model_steps.go b/tests/integration/godog/steps/model_steps.go index 1790bcc213..fcc4b73567 100644 --- a/tests/integration/godog/steps/model_steps.go +++ b/tests/integration/godog/steps/model_steps.go @@ -183,24 +183,6 @@ func LoadTemplateModelSteps(scenario *godog.ScenarioContext, w *World) { scenario.Step(`^the model status message eventually should be "([^"]+)"$`, w.currentModel.AssertModelStatus) } -func LoadCustomModelSteps(scenario *godog.ScenarioContext, w *World) { - scenario.Step(`^I deploy model spec with timeout "([^"]+)":$`, func(timeout string, spec *godog.DocString) error { - return withTimeoutCtx(timeout, func(ctx context.Context) error { - return w.currentModel.deployModelSpec(ctx, spec) - }) - }) - scenario.Step(`^the model "([^"]+)" should eventually become Ready with timeout "([^"]+)"$`, func(model, timeout string) error { - return withTimeoutCtx(timeout, func(ctx context.Context) error { - return w.currentModel.waitForModelNameReady(ctx, model) - }) - }) - scenario.Step(`^delete the model "([^"]+)" with timeout "([^"]+)"$`, func(model, timeout string) error { - return withTimeoutCtx(timeout, func(ctx context.Context) error { - return w.currentModel.deleteModel(ctx, model) - }) - }) -} - func (m *Model) deployModelSpec(ctx context.Context, spec *godog.DocString) error { modelSpec := &mlopsv1alpha1.Model{} if err := yaml.Unmarshal([]byte(spec.Content), &modelSpec); err != nil { @@ -284,6 +266,7 @@ func (m *Model) IHaveAModel(model string) error { return nil } + func newModel(label map[string]string, namespace string, k8sClient versioned.Interface, log logrus.FieldLogger, watcherStorage k8sclient.WatcherStorage) *Model { return &Model{label: label, model: &mlopsv1alpha1.Model{}, log: log, namespace: namespace, k8sClient: k8sClient, watcherStorage: watcherStorage} } diff --git a/tests/integration/godog/steps/world.go b/tests/integration/godog/steps/world.go index e1ffdef5fb..3338e6e43a 100644 --- a/tests/integration/godog/steps/world.go +++ b/tests/integration/godog/steps/world.go @@ -26,12 +26,13 @@ type World struct { StartingClusterState string //todo: this will be a combination of starting state awareness of core 2 such as the //todo: server config,seldon config and seldon runtime to be able to reconcile to starting state should we change //todo: the state such as reducing replicas to 0 of scheduler to test unavailability - currentModel *Model - currentPipeline *Pipeline - server *server - infer inference - logger log.FieldLogger - Label map[string]string + currentModel *Model + currentPipeline *Pipeline + currentExperiment *Experiment + server *server + infer inference + logger log.FieldLogger + Label map[string]string } type Config struct { @@ -58,12 +59,13 @@ func NewWorld(c Config) (*World, error) { } w := &World{ - namespace: c.Namespace, - kubeClient: c.KubeClient, - watcherStorage: c.WatcherStorage, - currentModel: newModel(label, c.Namespace, c.K8sClient, c.Logger, c.WatcherStorage), - currentPipeline: newPipeline(label, c.Namespace, c.K8sClient, c.Logger, c.WatcherStorage), - server: newServer(label, c.Namespace, c.K8sClient, c.Logger, c.KubeClient), + namespace: c.Namespace, + kubeClient: c.KubeClient, + watcherStorage: c.WatcherStorage, + currentModel: newModel(label, c.Namespace, c.K8sClient, c.Logger, c.WatcherStorage), + currentExperiment: newExperiment(label, c.Namespace, c.K8sClient, c.Logger, c.WatcherStorage), + currentPipeline: newPipeline(label, c.Namespace, c.K8sClient, c.Logger, c.WatcherStorage), + server: newServer(label, c.Namespace, c.K8sClient, c.Logger, c.KubeClient), infer: inference{ host: c.IngressHost, http: &http.Client{}, diff --git a/tests/integration/godog/suite/suite.go b/tests/integration/godog/suite/suite.go index 219a8de334..03326aac5c 100644 --- a/tests/integration/godog/suite/suite.go +++ b/tests/integration/godog/suite/suite.go @@ -166,5 +166,6 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { steps.LoadInferenceSteps(scenarioCtx, world) steps.LoadServerSteps(scenarioCtx, world) steps.LoadCustomPipelineSteps(scenarioCtx, world) + steps.LoadExperimentSteps(scenarioCtx, world) // TODO: load other steps, e.g. pipeline, experiment, etc. }