diff --git a/tests/integration/godog/features/model/custom_model_deployment.feature b/tests/integration/godog/features/model/custom_model_deployment.feature index a615669ff4..5ac81eb437 100644 --- a/tests/integration/godog/features/model/custom_model_deployment.feature +++ b/tests/integration/godog/features/model/custom_model_deployment.feature @@ -19,7 +19,7 @@ Feature: Explicit Model deployment storageUri: gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn """ When the model "alpha-1" should eventually become Ready with timeout "20s" - Then send HTTP inference request with timeout "20s" to model "alpha-1" with payload: + Then I send HTTP inference request with timeout "20s" to model "alpha-1" with payload: """ { "inputs": [ @@ -52,7 +52,7 @@ Feature: Explicit Model deployment } ] } """ - Then send gRPC inference request with timeout "20s" to model "alpha-1" with payload: + Then I send gRPC inference request with timeout "20s" to model "alpha-1" with payload: """ { "inputs": [ @@ -85,7 +85,7 @@ Feature: Explicit Model deployment ] } """ Then delete the model "alpha-1" with timeout "10s" - Then send HTTP inference request with timeout "20s" to model "alpha-1" with payload: + Then I send HTTP inference request with timeout "20s" to model "alpha-1" with payload: """ { "inputs": [ @@ -99,7 +99,7 @@ Feature: Explicit Model deployment } """ And expect http response status code "404" - Then send gRPC inference request with timeout "20s" to model "alpha-1" with payload: + Then I send gRPC inference request with timeout "20s" to model "alpha-1" with payload: """ { "inputs": [ diff --git a/tests/integration/godog/features/model/over_commit.feature b/tests/integration/godog/features/model/over_commit.feature index 278e180831..0c6887cf36 100644 --- a/tests/integration/godog/features/model/over_commit.feature +++ b/tests/integration/godog/features/model/over_commit.feature @@ -51,7 +51,7 @@ Feature: Explicit Model deployment storageUri: gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn """ When the model "overcommit-3" should eventually become Ready with timeout "20s" - Then send HTTP inference request with timeout "20s" to model "overcommit-1" with payload: + Then I send HTTP inference request with timeout "20s" to model "overcommit-1" with payload: """ { "inputs": [ @@ -65,7 +65,7 @@ Feature: Explicit Model deployment } """ And expect http response status code "200" - Then send HTTP inference request with timeout "20s" to model "overcommit-2" with payload: + Then I send HTTP inference request with timeout "20s" to model "overcommit-2" with payload: """ { "inputs": [ @@ -79,7 +79,7 @@ Feature: Explicit Model deployment } """ And expect http response status code "200" - Then send HTTP inference request with timeout "20s" to model "overcommit-3" with payload: + Then I send HTTP inference request with timeout "20s" to model "overcommit-3" with payload: """ { "inputs": [ diff --git a/tests/integration/godog/features/pipeline/conditional.feature b/tests/integration/godog/features/pipeline/conditional.feature index eae88cd495..2992e7672d 100644 --- a/tests/integration/godog/features/pipeline/conditional.feature +++ b/tests/integration/godog/features/pipeline/conditional.feature @@ -1,8 +1,10 @@ -@PipelineDeployment @Functional @Pipelines @Conditional +@PipelineConditional @Functional @Pipelines @Conditional Feature: Conditional pipeline with branching models - This pipeline uses a conditional model to route data to either add10 or mul10. + In order to support decision-based inference + As a model user + I need a conditional pipeline that directs inputs to one of multiple models based on a condition - Scenario: Deploy tfsimple-conditional pipeline and wait for readiness + Scenario: Deploy a conditional pipeline, run inference, and verify the output Given I deploy model spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 @@ -43,7 +45,7 @@ Feature: Conditional pipeline with branching models And the model "add10-nbsl" should eventually become Ready with timeout "20s" And the model "mul10-nbsl" should eventually become Ready with timeout "20s" - And I deploy pipeline spec with timeout "30s": + When I deploy a pipeline spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 kind: Pipeline @@ -69,3 +71,70 @@ Feature: Conditional pipeline with branching models stepsJoin: any """ Then the pipeline "tfsimple-conditional-nbsl" should eventually become Ready with timeout "40s" + Then I send gRPC inference request with timeout "20s" to pipeline "tfsimple-conditional-nbsl" with payload: + """ + { + "model_name": "conditional-nbsl", + "inputs": [ + { + "name": "CHOICE", + "contents": { + "int_contents": [ + 0 + ] + }, + "datatype": "INT32", + "shape": [ + 1 + ] + }, + { + "name": "INPUT0", + "contents": { + "fp32_contents": [ + 1, + 2, + 3, + 4 + ] + }, + "datatype": "FP32", + "shape": [ + 4 + ] + }, + { + "name": "INPUT1", + "contents": { + "fp32_contents": [ + 1, + 2, + 3, + 4 + ] + }, + "datatype": "FP32", + "shape": [ + 4 + ] + } + ] + } + """ + And expect gRPC response body to contain JSON: + """ + { + "outputs": [ + { + "name": "OUTPUT", + "datatype": "FP32", + "shape": [ + 4 + ] + } + ], + "raw_output_contents": [ + "AAAgQQAAoEEAAPBBAAAgQg==" + ] + } + """ diff --git a/tests/integration/godog/features/pipeline/input_chaining.feature b/tests/integration/godog/features/pipeline/input_chaining.feature index 90ff394d40..cde09d8239 100644 --- a/tests/integration/godog/features/pipeline/input_chaining.feature +++ b/tests/integration/godog/features/pipeline/input_chaining.feature @@ -1,14 +1,16 @@ -@PipelineDeployment @Functional @Pipelines @ModelChainingFromInputs +@PipelineModelChainingFromInputs @Functional @Pipelines @ModelChainingFromInputs Feature: Pipeline model chaining using inputs and outputs - This pipeline chains tfsimple1 into tfsimple2 using both inputs and outputs. + In order to build multi-stage inference workflows + As a model user + I need a pipeline that chains models together by passing outputs from one stage into the next - Scenario: Deploy tfsimples-input pipeline and wait for readiness + Scenario: Scenario: Deploy a model-chaining pipeline, run inference, and verify the output Given I deploy model spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 kind: Model metadata: - name: chain-from-input-tfsimple1-yhjo + name: tfsimple1-yhjo spec: storageUri: "gs://seldon-models/triton/simple" requirements: @@ -21,33 +23,243 @@ Feature: Pipeline model chaining using inputs and outputs apiVersion: mlops.seldon.io/v1alpha1 kind: Model metadata: - name: chain-from-input-tfsimple2-yhjo + name: tfsimple2-yhjo spec: storageUri: "gs://seldon-models/triton/simple" requirements: - tensorflow memory: 100Ki """ - Then the model "chain-from-input-tfsimple1-yhjo" should eventually become Ready with timeout "20s" - Then the model "chain-from-input-tfsimple2-yhjo" should eventually become Ready with timeout "20s" - - And I deploy pipeline spec with timeout "30s": + Then the model "tfsimple1-yhjo" should eventually become Ready with timeout "20s" + Then the model "tfsimple2-yhjo" should eventually become Ready with timeout "20s" + And I deploy a pipeline spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 kind: Pipeline metadata: - name: chain-from-input-tfsimples-input-yhjo + name: chain-from-input-yhjo spec: steps: - - name: chain-from-input-tfsimple1-yhjo - - name: chain-from-input-tfsimple2-yhjo + - name: tfsimple1-yhjo + - name: tfsimple2-yhjo inputs: - - chain-from-input-tfsimple1-yhjo.inputs.INPUT0 - - chain-from-input-tfsimple1-yhjo.outputs.OUTPUT1 + - tfsimple1-yhjo.inputs.INPUT0 + - tfsimple1-yhjo.outputs.OUTPUT1 tensorMap: - chain-from-input-tfsimple1-yhjo.outputs.OUTPUT1: INPUT1 + tfsimple1-yhjo.outputs.OUTPUT1: INPUT1 output: steps: - - chain-from-input-tfsimple2-yhjo + - tfsimple2-yhjo + """ + Then the pipeline "chain-from-input-yhjo" should eventually become Ready with timeout "40s" + When I send HTTP inference request with timeout "20s" to pipeline "chain-from-input-yhjo" with payload: + """ + { + "inputs": [ + { + "name": "INPUT0", + "data": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ], + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + }, + { + "name": "INPUT1", + "data": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ], + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + } + ] + } + """ + And expect http response status code "200" + And expect http response body to contain JSON: + """ + { + "model_name": "", + "outputs": [ + { + "data": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ], + "name": "OUTPUT0", + "shape": [ + 1, + 16 + ], + "datatype": "INT32" + }, + { + "data": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ], + "name": "OUTPUT1", + "shape": [ + 1, + 16 + ], + "datatype": "INT32" + } + ] + } + """ + Then I send gRPC inference request with timeout "20s" to pipeline "chain-from-input-yhjo" with payload: + """ + { + "model_name": "simple", + "inputs": [ + { + "name": "INPUT0", + "contents": { + "int_contents": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ] + }, + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + }, + { + "name": "INPUT1", + "contents": { + "int_contents": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ] + }, + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + } + ] + } + """ + And expect gRPC response body to contain JSON: + """ + { + "outputs": [ + { + "name": "OUTPUT0", + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + }, + { + "name": "OUTPUT1", + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + } + ], + "raw_output_contents": [ + "AQAAAAIAAAADAAAABAAAAAUAAAAGAAAABwAAAAgAAAAJAAAACgAAAAsAAAAMAAAADQAAAA4AAAAPAAAAEAAAAA==", + "AQAAAAIAAAADAAAABAAAAAUAAAAGAAAABwAAAAgAAAAJAAAACgAAAAsAAAAMAAAADQAAAA4AAAAPAAAAEAAAAA==" + ] + } """ - Then the pipeline "chain-from-input-tfsimples-input-yhjo" should eventually become Ready with timeout "40s" diff --git a/tests/integration/godog/features/pipeline/input_tensors.feature b/tests/integration/godog/features/pipeline/input_tensors.feature index aa7778e927..00cab6fea2 100644 --- a/tests/integration/godog/features/pipeline/input_tensors.feature +++ b/tests/integration/godog/features/pipeline/input_tensors.feature @@ -1,8 +1,10 @@ -@PipelineDeployment @Functional @Pipelines @PipelineInputTensors +@PipelineInputTensors @Functional @Pipelines @PipelineInputTensors Feature: Pipeline using direct input tensors - This pipeline directly routes pipeline input tensors INPUT0 and INPUT1 into separate models. + In order to build pipelines that dispatch inputs to multiple models + As a model user + I need a pipeline that routes individual input tensors directly to different model stages - Scenario: Deploy pipeline-inputs pipeline and wait for readiness + Scenario: Deploy a pipeline that routes individual input tensors directly to different model stages, run inference, and verify the output Given I deploy model spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 @@ -30,7 +32,7 @@ Feature: Pipeline using direct input tensors Then the model "mul10-tw2x" should eventually become Ready with timeout "20s" And the model "add10-tw2x" should eventually become Ready with timeout "20s" - And I deploy pipeline spec with timeout "30s": + And I deploy a pipeline spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 kind: Pipeline @@ -54,3 +56,67 @@ Feature: Pipeline using direct input tensors - add10-tw2x """ Then the pipeline "pipeline-inputs-tw2x" should eventually become Ready with timeout "20s" + When I send gRPC inference request with timeout "20s" to pipeline "pipeline-inputs-tw2x" with payload: + """ + { + "model_name": "pipeline", + "inputs": [ + { + "name": "INPUT0", + "contents": { + "fp32_contents": [ + 1, + 2, + 3, + 4 + ] + }, + "datatype": "FP32", + "shape": [ + 4 + ] + }, + { + "name": "INPUT1", + "contents": { + "fp32_contents": [ + 1, + 2, + 3, + 4 + ] + }, + "datatype": "FP32", + "shape": [ + 4 + ] + } + ] + } + + """ + And expect gRPC response body to contain JSON: + """ + { + "outputs": [ + { + "name": "OUTPUT", + "datatype": "FP32", + "shape": [ + 4 + ] + }, + { + "name": "OUTPUT", + "datatype": "FP32", + "shape": [ + 4 + ] + } + ], + "raw_output_contents": [ + "AAAgQQAAoEEAAPBBAAAgQg==", + "AAAwQQAAQEEAAFBBAABgQQ==" + ] + } + """ diff --git a/tests/integration/godog/features/pipeline/join.feature b/tests/integration/godog/features/pipeline/join.feature index b5eda176e1..aa88a0ed31 100644 --- a/tests/integration/godog/features/pipeline/join.feature +++ b/tests/integration/godog/features/pipeline/join.feature @@ -1,8 +1,10 @@ -@PipelineDeployment @Functional @Pipelines @ModelJoin +@PipelineModelJoin @Functional @Pipelines @ModelJoin Feature: Pipeline model join - This pipeline joins outputs from tfsimple1 and tfsimple2 and feeds them into tfsimple3. + In order to perform inference that depends on multiple upstream computations + As a model user + I need a pipeline that merges the outputs of multiple models into a single input for a subsequent model - Scenario: Deploy tfsimples-join pipeline and wait for readiness + Scenario: Deploy a a pipeline that merges the outputs of multiple models into a single input for a subsequent model, run inference, and verify the output Given I deploy model spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 @@ -43,7 +45,7 @@ Feature: Pipeline model join And the model "join-tfsimple2-w4e3" should eventually become Ready with timeout "20s" And the model "join-tfsimple3-w4e3" should eventually become Ready with timeout "20s" - And I deploy pipeline spec with timeout "30s": + And I deploy a pipeline spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 kind: Pipeline @@ -65,3 +67,94 @@ Feature: Pipeline model join - join-tfsimple3-w4e3 """ Then the pipeline "join-pipeline-w4e3" should eventually become Ready with timeout "40s" + Then I send gRPC inference request with timeout "20s" to pipeline "join-pipeline-w4e3" with payload: + """ + { + "model_name": "simple", + "inputs": [ + { + "name": "INPUT0", + "contents": { + "int_contents": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ] + }, + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + }, + { + "name": "INPUT1", + "contents": { + "int_contents": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ] + }, + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + } + ] + } + """ + And expect gRPC response body to contain JSON: + """ + { + "outputs": [ + { + "name": "OUTPUT0", + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + }, + { + "name": "OUTPUT1", + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + } + ], + "raw_output_contents": [ + "AgAAAAQAAAAGAAAACAAAAAoAAAAMAAAADgAAABAAAAASAAAAFAAAABYAAAAYAAAAGgAAABwAAAAeAAAAIAAAAA==", + "AgAAAAQAAAAGAAAACAAAAAoAAAAMAAAADgAAABAAAAASAAAAFAAAABYAAAAYAAAAGgAAABwAAAAeAAAAIAAAAA==" + ] + } + """ diff --git a/tests/integration/godog/features/pipeline/model_chaining.feature b/tests/integration/godog/features/pipeline/model_chaining.feature index 5af153759e..4089805e91 100644 --- a/tests/integration/godog/features/pipeline/model_chaining.feature +++ b/tests/integration/godog/features/pipeline/model_chaining.feature @@ -1,8 +1,10 @@ -@ModelChaining @Functional @Pipelines +@ModelModelChaining @Functional @Pipelines Feature: Pipeline model chaining - This pipeline chains tfsimple1 into tfsimple2 using tensorMap. + In order to compose models that rely on each other's outputs + As a model user + I need a pipeline that maps specific output tensors from an upstream model into the inputs of a downstream model - Scenario: Deploy tfsimples pipeline and wait for readiness + Scenario: Deploy a chaining pipeline, run inference, and verify the output Given I deploy model spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 @@ -30,7 +32,7 @@ Feature: Pipeline model chaining """ Then the model "model-chain-tfsimple1-iuw3" should eventually become Ready with timeout "20s" Then the model "model-chain-tfsimple2-iuw3" should eventually become Ready with timeout "20s" - When I deploy pipeline spec with timeout "20s": + When I deploy a pipeline spec with timeout "20s": """ apiVersion: mlops.seldon.io/v1alpha1 kind: Pipeline @@ -50,3 +52,155 @@ Feature: Pipeline model chaining - model-chain-tfsimple2-iuw3 """ Then the pipeline "model-chain-tfsimples-iuw3" should eventually become Ready with timeout "40s" + When I send HTTP inference request with timeout "20s" to pipeline "model-chain-tfsimples-iuw3" with payload: + """ + { + "inputs": [ + { + "name": "INPUT0", + "data": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ], + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + }, + { + "name": "INPUT1", + "data": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ], + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + } + ] + } + """ + And expect http response status code "200" + Then I send gRPC inference request with timeout "20s" to pipeline "model-chain-tfsimples-iuw3" with payload: + """ + { + "model_name": "simple", + "inputs": [ + { + "name": "INPUT0", + "contents": { + "int_contents": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ] + }, + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + }, + { + "name": "INPUT1", + "contents": { + "int_contents": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ] + }, + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + } + ] + } + + """ + And expect gRPC response body to contain JSON: + """ + { + "outputs": [ + { + "name": "OUTPUT0", + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + }, + { + "name": "OUTPUT1", + "datatype": "INT32", + "shape": [ + 1, + 16 + ] + } + ], + "raw_output_contents": [ + "AgAAAAQAAAAGAAAACAAAAAoAAAAMAAAADgAAABAAAAASAAAAFAAAABYAAAAYAAAAGgAAABwAAAAeAAAAIAAAAA==", + "AgAAAAQAAAAGAAAACAAAAAoAAAAMAAAADgAAABAAAAASAAAAFAAAABYAAAAYAAAAGgAAABwAAAAeAAAAIAAAAA==" + ] + } + """ diff --git a/tests/integration/godog/features/pipeline/pipeline_setup.feature b/tests/integration/godog/features/pipeline/pipeline_setup.feature new file mode 100644 index 0000000000..a88546b8fb --- /dev/null +++ b/tests/integration/godog/features/pipeline/pipeline_setup.feature @@ -0,0 +1,47 @@ +@ServerSetup +Feature: Server setup + Deploys an mlserver with one replica. We ensure the pods + become ready and remove any other server pods for different + servers. + + @ServerSetup @ServerSetupMLServer + Scenario: Deploy mlserver Server and remove other servers + Given I deploy server spec with timeout "10s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Server + metadata: + name: godog-mlserver + spec: + replicas: 1 + serverConfig: mlserver + """ + When the server should eventually become Ready with timeout "30s" + Then ensure only "1" pod(s) are deployed for server and they are Ready + + @ServerSetup @ServerSetupTritonServer + Scenario: Deploy triton Server + Given I deploy server spec with timeout "10s": + """ + apiVersion: mlops.seldon.io/v1alpha1 + kind: Server + metadata: + name: godog-triton + spec: + replicas: 1 + serverConfig: triton + """ + When the server should eventually become Ready with timeout "30s" + Then ensure only "1" pod(s) are deployed for server and they are Ready + + + @ServerSetup @ServerClean + Scenario: Remove any other pre-existing servers + Given I remove any other server deployments which are not "godog-mlserver,godog-triton" + +# TODO decide if we want to keep this, if we keep testers will need to ensure they don't run this tag when running all +# all features in this directory, as tests will fail when server is deleted. We can not delete and it's up to the +# feature dir server setup to ensure ONLY the required servers exist, like above. +# @ServerTeardown +# Scenario: Delete mlserver Server +# Given I delete server "godog-mlserver" with timeout "10s" \ No newline at end of file diff --git a/tests/integration/godog/features/pipeline/trigger_joins.feature b/tests/integration/godog/features/pipeline/trigger_joins.feature index 89de371066..55da178dd6 100644 --- a/tests/integration/godog/features/pipeline/trigger_joins.feature +++ b/tests/integration/godog/features/pipeline/trigger_joins.feature @@ -1,8 +1,10 @@ -@PipelineDeployment @Functional @Pipelines @TriggerJoins +@PipelineTriggerJoins @Functional @Pipelines @TriggerJoins Feature: Pipeline using trigger joins - This pipeline uses trigger joins to decide whether mul10 or add10 should run. + In order to control which model stages execute during inference + As a model user + I need a pipeline that evaluates trigger conditions and runs model stages when their associated triggers are satisfied - Scenario: Deploy trigger-joins pipeline and wait for readiness + Scenario: Deploy a trigger pipeline, run inference, and verify the output Given I deploy model spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 @@ -29,8 +31,7 @@ Feature: Pipeline using trigger joins """ Then the model "mul10-99lo" should eventually become Ready with timeout "20s" And the model "add10-99lo" should eventually become Ready with timeout "20s" - - And I deploy pipeline spec with timeout "30s": + And I deploy a pipeline spec with timeout "30s": """ apiVersion: mlops.seldon.io/v1alpha1 kind: Pipeline @@ -57,3 +58,55 @@ Feature: Pipeline using trigger joins stepsJoin: any """ Then the pipeline "trigger-joins-99lo" should eventually become Ready with timeout "20s" + When I send gRPC inference request with timeout "20s" to pipeline "trigger-joins-99lo" with payload: + """ + { + "model_name": "pipeline", + "inputs": [ + { + "name": "ok1", + "contents": { + "fp32_contents": [ + 1 + ] + }, + "datatype": "FP32", + "shape": [ + 1 + ] + }, + { + "name": "INPUT", + "contents": { + "fp32_contents": [ + 1, + 2, + 3, + 4 + ] + }, + "datatype": "FP32", + "shape": [ + 4 + ] + } + ] + } + """ + And expect gRPC response body to contain JSON: + """ + { + "outputs": [ + { + "name": "OUTPUT", + "datatype": "FP32", + "shape": [ + 4 + ] + } + ], + "raw_output_contents": [ + "AAAgQQAAoEEAAPBBAAAgQg==" + ] + } + """ diff --git a/tests/integration/godog/go.mod b/tests/integration/godog/go.mod index d66c7762a3..9984280067 100644 --- a/tests/integration/godog/go.mod +++ b/tests/integration/godog/go.mod @@ -8,6 +8,7 @@ require ( github.com/seldonio/seldon-core/operator/v2 v2.10.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.7 + github.com/stretchr/testify v1.10.0 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 k8s.io/api v0.34.3 diff --git a/tests/integration/godog/godog-config.json b/tests/integration/godog/godog-config.json index 9111146777..2c04056468 100644 --- a/tests/integration/godog/godog-config.json +++ b/tests/integration/godog/godog-config.json @@ -2,6 +2,7 @@ "namespace": "seldon-mesh", "log_level": "debug", "skip_cleanup" : false, + "skip_cleanup_on_error": false, "inference" : { "host": "localhost", "httpPort": 9000, diff --git a/tests/integration/godog/steps/assertions/pipeline_test.go b/tests/integration/godog/steps/assertions/pipeline_test.go new file mode 100644 index 0000000000..e212d72175 --- /dev/null +++ b/tests/integration/godog/steps/assertions/pipeline_test.go @@ -0,0 +1,75 @@ +package assertions + +import ( + "testing" + + "github.com/seldonio/seldon-core/operator/v2/apis/mlops/v1alpha1" + "github.com/stretchr/testify/require" + "sigs.k8s.io/yaml" +) + +func TestPipelineReady(t *testing.T) { + pipelineYAML := ` +apiVersion: mlops.seldon.io/v1alpha1 +kind: Pipeline +metadata: + name: model-chain-tfsimples-iuw3 + namespace: seldon-mesh +status: + conditions: + - type: ModelsReady + status: "True" + - type: PipelineGwReady + status: "True" + - type: PipelineReady + status: "True" + - type: Ready + status: "True" +` + + pipeline := &v1alpha1.Pipeline{} + + // Unmarshal YAML into the Pipeline struct + err := yaml.Unmarshal([]byte(pipelineYAML), pipeline) + require.NoError(t, err) + + // Call the function under test + ready, err := PipelineReady(pipeline) + + // Assertions + require.NoError(t, err) + require.True(t, ready) +} + +func TestPipelineNotReady(t *testing.T) { + pipelineYAML := ` +apiVersion: mlops.seldon.io/v1alpha1 +kind: Pipeline +metadata: + name: model-chain-tfsimples-iuw3 + namespace: seldon-mesh +status: + conditions: + - type: ModelsReady + status: "False" + - type: PipelineGwReady + status: "True" + - type: PipelineReady + status: "True" + - type: Ready + status: "False" +` + + pipeline := &v1alpha1.Pipeline{} + + // Unmarshal YAML into the Pipeline struct + err := yaml.Unmarshal([]byte(pipelineYAML), pipeline) + require.NoError(t, err) + + // Call the function under test + ready, err := PipelineReady(pipeline) + + // Assertions + require.NoError(t, err) + require.False(t, ready) +} diff --git a/tests/integration/godog/steps/infer_steps.go b/tests/integration/godog/steps/infer_steps.go index 053c9bf3c9..111ee911f3 100644 --- a/tests/integration/godog/steps/infer_steps.go +++ b/tests/integration/godog/steps/infer_steps.go @@ -38,23 +38,46 @@ type inference struct { log logrus.FieldLogger } +// todo: this is to avoid 503s since the route at times isn't ready when the model is ready +// todo: this might be related to issue: https://seldonio.atlassian.net/browse/INFRA-1576?search_id=4def05ca-ec64-436e-b824-49e39f1c94c4 +// todo: this issue relates to how we should route inference request via IP instead of DNS +const inferenceRequestDelay = 200 * time.Millisecond + func LoadInferenceSteps(scenario *godog.ScenarioContext, w *World) { - scenario.Step(`^send HTTP inference request with timeout "([^"]+)" to model "([^"]+)" with payload:$`, func(timeout, model string, payload *godog.DocString) error { + scenario.Step(`^(?:I )send HTTP inference request with timeout "([^"]+)" to (model|pipeline) "([^"]+)" with payload:$`, func(timeout, kind, resourceName string, payload *godog.DocString) error { + time.Sleep(inferenceRequestDelay) return withTimeoutCtx(timeout, func(ctx context.Context) error { - return w.infer.sendHTTPModelInferenceRequest(ctx, model, payload) + switch kind { + case "model": + return w.infer.doHTTPModelInferenceRequest(ctx, resourceName, payload.Content) + case "pipeline": + return w.infer.doHTTPPipelineInferenceRequest(ctx, resourceName, payload.Content) + default: + return fmt.Errorf("unknown target type: %s", kind) + } }) }) - scenario.Step(`^send gRPC inference request with timeout "([^"]+)" to model "([^"]+)" with payload:$`, func(timeout, model string, payload *godog.DocString) error { + scenario.Step(`^(?:I )send gRPC inference request with timeout "([^"]+)" to (model|pipeline) "([^"]+)" with payload:$`, func(timeout, kind, resourceName string, payload *godog.DocString) error { + time.Sleep(inferenceRequestDelay) return withTimeoutCtx(timeout, func(ctx context.Context) error { - return w.infer.sendGRPCModelInferenceRequest(ctx, model, payload) + switch kind { + case "model": + return w.infer.doGRPCInferenceRequest(ctx, resourceName, payload.Content) + case "pipeline": + return w.infer.doGRPCInferenceRequest(ctx, fmt.Sprintf("%s.pipeline", resourceName), payload.Content) + default: + return fmt.Errorf("unknown target type: %s", kind) + } }) }) scenario.Step(`^(?:I )send a valid gRPC inference request with timeout "([^"]+)"`, func(timeout string) error { + time.Sleep(inferenceRequestDelay) return withTimeoutCtx(timeout, func(ctx context.Context) error { return w.infer.sendGRPCModelInferenceRequestFromModel(ctx, w.currentModel) }) }) scenario.Step(`^(?:I )send a valid HTTP inference request with timeout "([^"]+)"`, func(timeout string) error { + time.Sleep(inferenceRequestDelay) return withTimeoutCtx(timeout, func(ctx context.Context) error { return w.infer.sendHTTPModelInferenceRequestFromModel(ctx, w.currentModel) }) @@ -105,6 +128,10 @@ func (i *inference) doHTTPExperimentInferenceRequest(ctx context.Context, experi return i.doHTTPInferenceRequest(ctx, experimentName, fmt.Sprintf("%s.experiment", experimentName), body) } +func (i *inference) doHTTPPipelineInferenceRequest(ctx context.Context, pipelineName, body string) error { + return i.doHTTPInferenceRequest(ctx, pipelineName, fmt.Sprintf("%s.pipeline", pipelineName), body) +} + func (i *inference) doHTTPModelInferenceRequest(ctx context.Context, modelName, body string) error { return i.doHTTPInferenceRequest(ctx, modelName, modelName, body) } @@ -132,7 +159,7 @@ func httpScheme(useSSL bool) string { } func (i *inference) sendGRPCModelInferenceRequest(ctx context.Context, model string, payload *godog.DocString) error { - return i.doGRPCModelInferenceRequest(ctx, model, payload.Content) + return i.doGRPCInferenceRequest(ctx, model, payload.Content) } func (i *inference) sendGRPCModelInferenceRequestFromModel(ctx context.Context, m *Model) error { @@ -140,21 +167,21 @@ func (i *inference) sendGRPCModelInferenceRequestFromModel(ctx context.Context, if !ok { return fmt.Errorf("could not find test model %s", m.model.Name) } - return i.doGRPCModelInferenceRequest(ctx, m.modelName, testModel.ValidGRPCInferenceRequest) + return i.doGRPCInferenceRequest(ctx, m.modelName, testModel.ValidGRPCInferenceRequest) } -func (i *inference) doGRPCModelInferenceRequest( +func (i *inference) doGRPCInferenceRequest( ctx context.Context, - model string, + resourceName string, payload string, ) error { var req v2_dataplane.ModelInferRequest if err := protojson.Unmarshal([]byte(payload), &req); err != nil { return fmt.Errorf("could not unmarshal gRPC json payload: %w", err) } - req.ModelName = model + req.ModelName = resourceName - md := metadata.Pairs("seldon-model", model) + md := metadata.Pairs("seldon-model", resourceName) ctx = metadata.NewOutgoingContext(ctx, md) i.log.Debugf("sending gRPC model inference %+v", &req) @@ -257,6 +284,8 @@ func (i *inference) gRPCRespContainsError(err string) error { return fmt.Errorf("error %s does not contain %s", i.lastGRPCResponse.err.Error(), err) } +// todo: in the future we should also check for raw output contents the function needed is in the command line cli of seldon on the infer method +// todo: https://github.com/SeldonIO/seldon-core/blob/ce6deb2433927e7632f56b471dfe3c0a0fb1210c/operator/pkg/cli/infer.go#L586 func (i *inference) gRPCRespCheckBodyContainsJSON(expectJSON *godog.DocString) error { if i.lastGRPCResponse.response == nil { if i.lastGRPCResponse.err != nil { diff --git a/tests/integration/godog/steps/pipeline_steps.go b/tests/integration/godog/steps/pipeline_steps.go index 14f20a414c..ca63313b2b 100644 --- a/tests/integration/godog/steps/pipeline_steps.go +++ b/tests/integration/godog/steps/pipeline_steps.go @@ -39,7 +39,7 @@ func newPipeline(label map[string]string, namespace string, k8sClient versioned. } func LoadCustomPipelineSteps(scenario *godog.ScenarioContext, w *World) { - scenario.Step(`^I deploy pipeline spec with timeout "([^"]+)":$`, func(timeout string, spec *godog.DocString) error { + scenario.Step(`^I deploy a pipeline spec with timeout "([^"]+)":$`, func(timeout string, spec *godog.DocString) error { return withTimeoutCtx(timeout, func(ctx context.Context) error { return w.currentPipeline.deployPipelineSpec(ctx, spec) }) diff --git a/tests/integration/godog/suite/config.go b/tests/integration/godog/suite/config.go index f65f222522..b6a92d5381 100644 --- a/tests/integration/godog/suite/config.go +++ b/tests/integration/godog/suite/config.go @@ -16,10 +16,11 @@ import ( ) type GodogConfig struct { - Namespace string `json:"namespace"` - LogLevel string `json:"log_level"` - SkipCleanup bool `json:"skip_cleanup"` - Inference Inference `json:"inference"` + Namespace string `json:"namespace"` + LogLevel string `json:"log_level"` + SkipCleanup bool `json:"skip_cleanup"` + SkipCleanUpOnError bool `json:"skip_clean_up_on_error"` + Inference Inference `json:"inference"` } type Inference struct { diff --git a/tests/integration/godog/suite/suite.go b/tests/integration/godog/suite/suite.go index 03326aac5c..6bf2925576 100644 --- a/tests/integration/godog/suite/suite.go +++ b/tests/integration/godog/suite/suite.go @@ -148,6 +148,11 @@ func InitializeScenario(scenarioCtx *godog.ScenarioContext) { // After: optional cleanup / rollback scenarioCtx.After(func(ctx context.Context, scenario *godog.Scenario, err error) (context.Context, error) { + if err != nil && suiteDeps.Config.SkipCleanUpOnError { + log.WithField("scenario", scenario.Name).Debugf("Skipping cleanup of resources for scenario with err %v", err) + // don't clean up resources for scenarios that fail + return ctx, nil + } if suiteDeps.Config.SkipCleanup { log.WithField("scenario", scenario.Name).Debug("Skipping cleanup") return ctx, nil