diff --git a/tests/integration/godog/features/model/deployment.feature b/tests/integration/godog/features/model/deployment.feature index 76b6b7429e..993b61ed6f 100644 --- a/tests/integration/godog/features/model/deployment.feature +++ b/tests/integration/godog/features/model/deployment.feature @@ -53,3 +53,4 @@ Feature: Model deployment Then the model eventually becomes not Ready And the model status message should eventually be "ModelFailed" + diff --git a/tests/integration/godog/features/model/explicit_model_deployment.feature b/tests/integration/godog/features/model/explicit_model_deployment.feature new file mode 100644 index 0000000000..aebb9082d0 --- /dev/null +++ b/tests/integration/godog/features/model/explicit_model_deployment.feature @@ -0,0 +1,85 @@ +@ModelDeployment @Functional @Models @Explicit +Feature: Explicit Model deployment + I deploy a custom model spec, wait for model to be deployed to the servers + and send an inference request to that model + + Scenario: Load model and send inference request to envoy + Given I deploy model spec: + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Model + metadata: + name: iris + spec: + replicas: 1 + requirements: + - sklearn + - mlserver + storageUri: gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn + """ + When the model "iris" should eventually become Ready with timeout "20s" + Then send HTTP inference request with timeout "20s" to model "iris" with payload: + """ + { + "inputs": [ + { + "name": "predict", + "shape": [1, 4], + "datatype": "FP32", + "data": [[1, 2, 3, 4]] + } + ] + } + """ + And expect http response status code "200" + And expect http response body to contain JSON: + """ + { "outputs": [ + { + "name": "predict", + "shape": [ + 1, + 1 + ], + "datatype": "INT64", + "parameters": { + "content_type": "np" + }, + "data": [ + 2 + ] + } + ] } + """ + Then send gRPC inference request with timeout "20s" to model "iris" with payload: + """ + { + "inputs": [ + { + "name": "predict", + "shape": [1, 4], + "datatype": "FP32", + "contents": { + "int64_contents" : [1, 2, 3, 4] + } + } + ] + } + """ + And expect gRPC response body to contain JSON: + """ + { "outputs": [ + { + "name": "predict", + "shape": [ + 1, + 1 + ], + "datatype": "INT64", + "parameters": { + "content_type": {"ParameterChoice":{"StringParam":"np"}} + }, + "contents": {"int64_contents" : [2]} + } + ] } + """ \ No newline at end of file diff --git a/tests/integration/godog/go.mod b/tests/integration/godog/go.mod index 994d5c0061..49cdd6f721 100644 --- a/tests/integration/godog/go.mod +++ b/tests/integration/godog/go.mod @@ -4,7 +4,13 @@ go 1.24.4 require ( github.com/cucumber/godog v0.15.1 + github.com/seldonio/seldon-core/apis/go/v2 v2.9.1 github.com/seldonio/seldon-core/operator/v2 v2.10.1 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/pflag v1.0.7 + google.golang.org/grpc v1.73.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.34.2 k8s.io/apimachinery v0.34.2 k8s.io/client-go v0.34.2 sigs.k8s.io/controller-runtime v0.22.4 @@ -36,8 +42,6 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/seldonio/seldon-core/apis/go/v2 v2.9.1 // indirect - github.com/spf13/pflag v1.0.7 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect @@ -49,11 +53,8 @@ require ( golang.org/x/time v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/grpc v1.73.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.34.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect diff --git a/tests/integration/godog/go.sum b/tests/integration/godog/go.sum index d6e958da23..6f84ff438c 100644 --- a/tests/integration/godog/go.sum +++ b/tests/integration/godog/go.sum @@ -115,6 +115,8 @@ github.com/seldonio/seldon-core/apis/go/v2 v2.9.1 h1:9UcxnTFRuCDApZqy7cy3Rm6B/aa github.com/seldonio/seldon-core/apis/go/v2 v2.9.1/go.mod h1:ptbV8xxTT6DI5hWGcOx74bizYhms/LhXBJ/04RD41jk= github.com/seldonio/seldon-core/operator/v2 v2.10.1 h1:Btn8xcFt5rPd4+xCMFAKwcuXGHAq4/nzE5EuYuNg0uI= github.com/seldonio/seldon-core/operator/v2 v2.10.1/go.mod h1:WMy17S3Q6QZTR2IP1OaIgRdh36RiNboT8jqCajJ6X9A= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= @@ -125,6 +127,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -186,6 +189,7 @@ golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= diff --git a/tests/integration/godog/k8sclient/client.go b/tests/integration/godog/k8sclient/client.go index 1e4b41757d..9139b51e97 100644 --- a/tests/integration/godog/k8sclient/client.go +++ b/tests/integration/godog/k8sclient/client.go @@ -1,3 +1,12 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + package k8sclient import ( diff --git a/tests/integration/godog/k8sclient/watcher.go b/tests/integration/godog/k8sclient/watcher.go index 29267ca8a7..21e5c4f5f1 100644 --- a/tests/integration/godog/k8sclient/watcher.go +++ b/tests/integration/godog/k8sclient/watcher.go @@ -1,3 +1,12 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + package k8sclient import ( diff --git a/tests/integration/godog/main_test.go b/tests/integration/godog/main_test.go index f973621791..6bdb502ad3 100644 --- a/tests/integration/godog/main_test.go +++ b/tests/integration/godog/main_test.go @@ -1,3 +1,12 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + package main import ( diff --git a/tests/integration/godog/scenario/assertions/model.go b/tests/integration/godog/scenario/assertions/model.go index 9885f71caa..eafc5b2a02 100644 --- a/tests/integration/godog/scenario/assertions/model.go +++ b/tests/integration/godog/scenario/assertions/model.go @@ -1,3 +1,12 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + package assertions import ( diff --git a/tests/integration/godog/scenario/scenario.go b/tests/integration/godog/scenario/scenario.go index 3a2af99265..b5d1f09be7 100644 --- a/tests/integration/godog/scenario/scenario.go +++ b/tests/integration/godog/scenario/scenario.go @@ -1,3 +1,12 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + package scenario import ( @@ -7,6 +16,7 @@ import ( "github.com/cucumber/godog" "github.com/seldonio/seldon-core/godog/k8sclient" "github.com/seldonio/seldon-core/godog/steps" + "github.com/sirupsen/logrus" ) type SuiteDeps struct { @@ -72,13 +82,21 @@ func InitializeTestSuite(ctx *godog.TestSuiteContext) { func InitializeScenario(scenarioCtx *godog.ScenarioContext) { // Create the world with long-lived deps once per scenario context - world := &steps.World{ + world, err := steps.NewWorld(steps.Config{ + Namespace: "seldon-mesh", //TODO configurable + Logger: logrus.New().WithField("test_type", "godog"), KubeClient: suiteDeps.K8sClient, WatcherStorage: suiteDeps.WatcherStore, - // initialise any other long-lived deps here, e.g. loggers, config, etc. + IngressHost: "localhost", //TODO configurable + HTTPPort: 9000, //TODO configurable + GRPCPort: 9000, //TODO configurable + SSL: false, //TODO configurable + }) + if err != nil { + panic(fmt.Errorf("failed to create world: %w", err)) } - world.CurrentModel = steps.NewModel(world) + world.CurrentModel = steps.NewModel() // Before: reset state and prep cluster before each scenario scenarioCtx.Before(func(ctx context.Context, scenario *godog.Scenario) (context.Context, error) { @@ -104,7 +122,9 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { }) // Register step definitions with access to world + k8sClient - steps.LoadModelSteps(scenarioCtx, world.CurrentModel) + steps.LoadModelSteps(scenarioCtx, world) + steps.LoadExplicitModelSteps(scenarioCtx, world) + steps.LoadInferenceSteps(scenarioCtx, world) // TODO: load other steps, e.g. pipeline, experiment, etc. } diff --git a/tests/integration/godog/steps/explicit_model_steps.go b/tests/integration/godog/steps/explicit_model_steps.go new file mode 100644 index 0000000000..204f7588ec --- /dev/null +++ b/tests/integration/godog/steps/explicit_model_steps.go @@ -0,0 +1,63 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package steps + +import "context" + +func (m *Model) waitForModelReady(ctx context.Context, model string) error { + // TODO: uncomment when auto-gen k8s client merged + + //foundModel, err := w.k8sClient.MlopsV1alpha1().Models(w.namespace).Get(ctx, model, metav1.GetOptions{}) + //if err != nil { + // return fmt.Errorf("failed getting model: %w", err) + //} + // + //if foundModel.Status.IsReady() { + // return nil + //} + // + //watcher, err := w.k8sClient.MlopsV1alpha1().Models(w.namespace).Watch(ctx, metav1.ListOptions{ + // FieldSelector: fmt.Sprintf("metadata.name=%s", model), + // ResourceVersion: foundModel.ResourceVersion, + // Watch: true, + //}) + //if err != nil { + // return fmt.Errorf("failed subscribed to watch model: %w", err) + //} + //defer watcher.Stop() + // + //for { + // select { + // case <-ctx.Done(): + // return ctx.Err() + // case event, ok := <-watcher.ResultChan(): + // if !ok { + // return fmt.Errorf("watch channel closed") + // } + // + // if event.Type == watch.Error { + // return fmt.Errorf("watch error: %v", event.Object) + // } + // + // if event.Type == watch.Added || event.Type == watch.Modified { + // model := event.Object.(*v1alpha1.Model) + // if model.Status.IsReady() { + // return nil + // } + // } + // + // if event.Type == watch.Deleted { + // return fmt.Errorf("resource was deleted") + // } + // } + //} + + return nil +} diff --git a/tests/integration/godog/steps/infer.go b/tests/integration/godog/steps/infer.go new file mode 100644 index 0000000000..5ec7bb95db --- /dev/null +++ b/tests/integration/godog/steps/infer.go @@ -0,0 +1,181 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + +package steps + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "reflect" + "strings" + "time" + + "github.com/cucumber/godog" + "github.com/seldonio/seldon-core/apis/go/v2/mlops/v2_dataplane" + "google.golang.org/grpc/metadata" +) + +func (i *inference) sendHTTPModelInferenceRequest(ctx context.Context, model string, payload *godog.DocString) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("%s://%s:%d/v2/models/%s/infer", httpScheme(i.ssl), i.host, i.httpPort, model), strings.NewReader(payload.Content)) + if err != nil { + return fmt.Errorf("could not create http request: %w", err) + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Host", "seldon-mesh.inference.seldon") + req.Header.Add("Seldon-model", model) + + resp, err := i.http.Do(req) + if err != nil { + return fmt.Errorf("could not send http request: %w", err) + } + i.lastHTTPResponse = resp + return nil +} + +func httpScheme(useSSL bool) string { + if useSSL { + return "https" + } + return "http" +} + +func (i *inference) sendGRPCModelInferenceRequest(ctx context.Context, model string, payload *godog.DocString) error { + var msg *v2_dataplane.ModelInferRequest + if err := json.Unmarshal([]byte(payload.Content), &msg); err != nil { + return fmt.Errorf("could not unmarshal gRPC json payload: %w", err) + } + msg.ModelName = model + + md := metadata.Pairs("seldon-model", model) + ctx = metadata.NewOutgoingContext(context.Background(), md) + resp, err := i.grpc.ModelInfer(ctx, msg) + if err != nil { + return fmt.Errorf("could not send grpc model inference: %w", err) + } + + i.lastGRPCResponse = resp + return nil +} + +func withTimeoutCtx(timeout string, callback func(ctx context.Context) error) error { + timeoutDuration, err := time.ParseDuration(timeout) + if err != nil { + return fmt.Errorf("invalid timeout %s: %w", timeout, err) + } + ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + defer cancel() + return callback(ctx) +} + +func isSubset(needle, hay any) bool { + nObj, nOK := needle.(map[string]any) + hObj, hOK := hay.(map[string]any) + if nOK && hOK { + for k, nv := range nObj { + hv, exists := hObj[k] + if !exists || !isSubset(nv, hv) { + return false + } + } + return true + } + + return reflect.DeepEqual(needle, hay) +} + +func containsSubset(needle, hay any) bool { + if isSubset(needle, hay) { + return true + } + switch h := hay.(type) { + case map[string]any: + for _, v := range h { + if containsSubset(needle, v) { + return true + } + } + case []any: + for _, v := range h { + if containsSubset(needle, v) { + return true + } + } + } + return false +} + +func jsonContainsObjectSubset(jsonStr, needleStr string) (bool, error) { + var hay, needle any + if err := json.Unmarshal([]byte(jsonStr), &hay); err != nil { + return false, fmt.Errorf("could not unmarshal hay json %s: %w", jsonStr, err) + } + if err := json.Unmarshal([]byte(needleStr), &needle); err != nil { + return false, fmt.Errorf("could not unmarshal needle json %s: %w", needleStr, err) + } + return containsSubset(needle, hay), nil +} + +func (i *inference) gRPCRespCheckBodyContainsJSON(expectJSON *godog.DocString) error { + if i.lastGRPCResponse == nil { + return errors.New("no gRPC response found") + } + + gotJson, err := json.Marshal(i.lastGRPCResponse) + if err != nil { + return fmt.Errorf("could not marshal gRPC json: %w", err) + } + + ok, err := jsonContainsObjectSubset(string(gotJson), expectJSON.Content) + if err != nil { + return fmt.Errorf("could not check if json contains object: %w", err) + } + + if !ok { + return fmt.Errorf("%s does not contain %s", string(gotJson), expectJSON) + } + + return nil +} + +func (i *inference) httpRespCheckBodyContainsJSON(expectJSON *godog.DocString) error { + if i.lastHTTPResponse == nil { + return errors.New("no http response found") + } + + body, err := io.ReadAll(i.lastHTTPResponse.Body) + if err != nil { + return fmt.Errorf("could not read response body: %w", err) + } + + ok, err := jsonContainsObjectSubset(string(body), expectJSON.Content) + if err != nil { + return fmt.Errorf("could not check if json contains object: %w", err) + } + + if !ok { + return fmt.Errorf("%s does not contain %s", string(body), expectJSON) + } + + return nil +} + +func (i *inference) httpRespCheckStatus(status int) error { + if i.lastHTTPResponse == nil { + return errors.New("no http response found") + } + if status != i.lastHTTPResponse.StatusCode { + return fmt.Errorf("expected http response status code %d, got %d", status, i.lastHTTPResponse.StatusCode) + } + return nil +} diff --git a/tests/integration/godog/steps/model_steps.go b/tests/integration/godog/steps/model_steps.go index 52595a005c..098d81b93e 100644 --- a/tests/integration/godog/steps/model_steps.go +++ b/tests/integration/godog/steps/model_steps.go @@ -1,3 +1,12 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + package steps import ( @@ -6,14 +15,15 @@ import ( "time" "github.com/cucumber/godog" + "github.com/seldonio/seldon-core/godog/k8sclient" "github.com/seldonio/seldon-core/godog/scenario/assertions" mlopsv1alpha1 "github.com/seldonio/seldon-core/operator/v2/apis/mlops/v1alpha1" + "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type Model struct { model *mlopsv1alpha1.Model - world *World } type TestModelConfig struct { @@ -37,18 +47,61 @@ var testModels = map[string]TestModelConfig{ }, } -func LoadModelSteps(ctx *godog.ScenarioContext, w *Model) { +func LoadModelSteps(scenario *godog.ScenarioContext, w *World) { // Model Operations - ctx.Step(`^I have an? "([^"]+)" model$`, w.IHaveAModel) - ctx.Step(`^the model has "(\d+)" min replicas$`, w.SetMinReplicas) - ctx.Step(`^the model has "(\d+)" max replicas$`, w.SetMaxReplicas) - ctx.Step(`^the model has "(\d+)" replicas$`, w.SetReplicas) + scenario.Step(`^I have an? "([^"]+)" model$`, w.CurrentModel.IHaveAModel) + scenario.Step(`^the model has "(\d+)" min replicas$`, w.CurrentModel.SetMinReplicas) + scenario.Step(`^the model has "(\d+)" max replicas$`, w.CurrentModel.SetMaxReplicas) + scenario.Step(`^the model has "(\d+)" replicas$`, w.CurrentModel.SetReplicas) // Model Deployments - ctx.Step(`^the model is applied$`, w.ApplyModel) + scenario.Step(`^the model is applied$`, func() error { + return w.CurrentModel.ApplyModel(w.KubeClient) + }) // Model Assertions - ctx.Step(`^the model (?:should )?eventually become(?:s)? Ready$`, w.ModelReady) - ctx.Step(`^the model status message should be "([^"]+)"$`, w.AssertModelStatus) + scenario.Step(`^the model (?:should )?eventually become(?:s)? Ready$`, func() error { + return w.CurrentModel.ModelReady(w.WatcherStorage) + }) + scenario.Step(`^the model status message should be "([^"]+)"$`, w.CurrentModel.AssertModelStatus) +} + +func LoadExplicitModelSteps(scenario *godog.ScenarioContext, w *World) { + scenario.Step(`^I deploy model spec:$`, func(spec *godog.DocString) error { + return w.CurrentModel.deployModelSpec(spec, w.namespace, w.KubeClient) + }) + scenario.Step(`^the model "([^"]+)" should eventually become Ready with timeout "([^"]+)"$`, func(model, timeout string) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.CurrentModel.waitForModelReady(ctx, model) + }) + }) +} +func LoadInferenceSteps(scenario *godog.ScenarioContext, w *World) { + scenario.Step(`^send HTTP inference request with timeout "([^"]+)" to model "([^"]+)" with payload:$`, func(timeout, model string, payload *godog.DocString) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.infer.sendHTTPModelInferenceRequest(ctx, model, payload) + }) + }) + scenario.Step(`^send gRPC inference request with timeout "([^"]+)" to model "([^"]+)" with payload:$`, func(timeout, model string, payload *godog.DocString) error { + return withTimeoutCtx(timeout, func(ctx context.Context) error { + return w.infer.sendGRPCModelInferenceRequest(ctx, model, payload) + }) + }) + scenario.Step(`^expect http response status code "([^"]*)"$`, w.infer.httpRespCheckStatus) + scenario.Step(`^expect http response body to contain JSON:$`, w.infer.httpRespCheckBodyContainsJSON) + scenario.Step(`^expect gRPC response body to contain JSON:$`, w.infer.gRPCRespCheckBodyContainsJSON) +} + +func (m *Model) deployModelSpec(spec *godog.DocString, namespace string, _ *k8sclient.K8sClient) error { + modelSpec := &mlopsv1alpha1.Model{} + if err := yaml.Unmarshal([]byte(spec.Content), &modelSpec); err != nil { + return fmt.Errorf("failed unmarshalling model spec: %w", err) + } + modelSpec.Namespace = namespace + // TODO: uncomment when auto-gen k8s client merged + //if _, err := w.k8sClient.MlopsV1alpha1().Models(w.namespace).Create(context.TODO(), modelSpec, metav1.CreateOptions{}); err != nil { + // return fmt.Errorf("failed creating model: %w", err) + //} + return nil } func (m *Model) IHaveAModel(model string) error { @@ -106,13 +159,12 @@ func (m *Model) IHaveAModel(model string) error { return nil } -func NewModel(world *World) *Model { - return &Model{model: &mlopsv1alpha1.Model{}, world: world} +func NewModel() *Model { + return &Model{model: &mlopsv1alpha1.Model{}} } func (m *Model) Reset(world *World) { - m.world.CurrentModel.model = &mlopsv1alpha1.Model{} - m.world.CurrentModel.world = world + world.CurrentModel.model = &mlopsv1alpha1.Model{} } func (m *Model) SetMinReplicas(replicas int) { @@ -124,28 +176,27 @@ func (m *Model) SetMaxReplicas(replicas int) {} func (m *Model) SetReplicas(replicas int) {} // ApplyModel model is aware of namespace and testsuite config and it might add extra information to the cr that the step hasn't added like namespace -func (m *Model) ApplyModel() error { +func (m *Model) ApplyModel(k *k8sclient.K8sClient) error { // retrieve current model and apply in k8s - err := m.world.KubeClient.ApplyModel(m.world.CurrentModel.model) + err := k.ApplyModel(m.model) if err != nil { return err } return nil - } -func (m *Model) ModelReady() error { +func (m *Model) ModelReady(w k8sclient.WatcherStorage) error { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() // m.world.CurrentModel.model is assumed to be *mlopsv1alpha1.Model // which implements runtime.Object, so no cast needed. - return m.world.WatcherStorage.WaitFor( + return w.WaitFor( ctx, - m.world.CurrentModel.model, + m.model, assertions.ModelReady, ) } diff --git a/tests/integration/godog/steps/util.go b/tests/integration/godog/steps/util.go index 2f1cba672e..53a4a6448b 100644 --- a/tests/integration/godog/steps/util.go +++ b/tests/integration/godog/steps/util.go @@ -1,3 +1,12 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + package steps import ( diff --git a/tests/integration/godog/steps/world.go b/tests/integration/godog/steps/world.go index 063190125c..392c52b23b 100644 --- a/tests/integration/godog/steps/world.go +++ b/tests/integration/godog/steps/world.go @@ -1,7 +1,23 @@ +/* +Copyright (c) 2024 Seldon Technologies Ltd. + +Use of this software is governed BY +(1) the license included in the LICENSE file or +(2) if the license included in the LICENSE file is the Business Source License 1.1, +the Change License after the Change Date as each is defined in accordance with the LICENSE file. +*/ + package steps import ( + "fmt" + "net/http" + + "github.com/seldonio/seldon-core/apis/go/v2/mlops/v2_dataplane" "github.com/seldonio/seldon-core/godog/k8sclient" + "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) type World struct { @@ -12,4 +28,57 @@ type World struct { //todo: server config,seldon config and seldon runtime to be able to reconcile to starting state should we change //todo: the state such as reducing replicas to 0 of scheduler to test unavailability CurrentModel *Model + infer inference + logger *logrus.Entry +} + +type Config struct { + Namespace string + Logger *logrus.Entry + KubeClient *k8sclient.K8sClient + WatcherStorage k8sclient.WatcherStorage + IngressHost string + HTTPPort uint + GRPCPort uint + SSL bool +} + +type inference struct { + ssl bool + host string + http *http.Client + grpc v2_dataplane.GRPCInferenceServiceClient + httpPort uint + lastHTTPResponse *http.Response + lastGRPCResponse *v2_dataplane.ModelInferResponse +} + +func NewWorld(c Config) (*World, error) { + // TODO TLS for gRPC when c.SSL == true + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + } + + conn, err := grpc.NewClient(fmt.Sprintf("%s:%d", c.IngressHost, c.GRPCPort), opts...) + if err != nil { + return nil, fmt.Errorf("could not create grpc client: %w", err) + } + grpcClient := v2_dataplane.NewGRPCInferenceServiceClient(conn) + + w := &World{ + namespace: c.Namespace, + KubeClient: c.KubeClient, + WatcherStorage: c.WatcherStorage, + infer: inference{ + host: c.IngressHost, + http: &http.Client{}, + httpPort: c.HTTPPort, + grpc: grpcClient, + ssl: c.SSL}, + } + + if c.Logger != nil { + w.logger = c.Logger + } + return w, nil }