test model over-commit (#7021)

Author: domsolutions

tests/integration/godog/features/model/over_commit.feature

@@ -0,0 +1,95 @@
+@ModelDeployment @Functional @Models @CustomModelSpec @OverCommit
+Feature: Explicit Model deployment
+  I deploy 3 iris models, each requiring 334MB. The server memory capacity is 1GB,
+  with a 10% allowance for over-commit. The third model should be evicted to disk. Send
+  inference requests to all models, expected them all to pass, as the agent will
+  automatically load the evicted model on-the-fly when req received
+
+  Scenario: Deploy 3 identical models and send inference
+    Given I deploy model spec with timeout "10s":
+    """
+    apiVersion: mlops.seldon.io/v1alpha1
+    kind: Model
+    metadata:
+      name: "overcommit-1"
+    spec:
+      replicas: 1
+      requirements:
+      - sklearn
+      - mlserver
+      memory: 334000000
+      storageUri: gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn
+    """
+    When the model "overcommit-1" should eventually become Ready with timeout "20s"
+    Given I deploy model spec with timeout "10s":
+    """
+    apiVersion: mlops.seldon.io/v1alpha1
+    kind: Model
+    metadata:
+      name: "overcommit-2"
+    spec:
+      replicas: 1
+      requirements:
+      - sklearn
+      - mlserver
+      memory: 334000000
+      storageUri: gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn
+    """
+    When the model "overcommit-2" should eventually become Ready with timeout "20s"
+    Given I deploy model spec with timeout "10s":
+    """
+    apiVersion: mlops.seldon.io/v1alpha1
+    kind: Model
+    metadata:
+      name: "overcommit-3"
+    spec:
+      replicas: 1
+      requirements:
+      - sklearn
+      - mlserver
+      memory: 334000000
+      storageUri: gs://seldon-models/scv2/samples/mlserver_1.3.5/iris-sklearn
+    """
+    When the model "overcommit-3" should eventually become Ready with timeout "20s"
+    Then send HTTP inference request with timeout "20s" to model "overcommit-1" with payload:
+    """
+    {
+        "inputs": [
+          {
+            "name": "predict",
+            "shape": [1, 4],
+            "datatype": "FP32",
+            "data": [[1, 2, 3, 4]]
+          }
+        ]
+    }
+    """
+    And expect http response status code "200"
+    Then send HTTP inference request with timeout "20s" to model "overcommit-2" with payload:
+    """
+    {
+        "inputs": [
+          {
+            "name": "predict",
+            "shape": [1, 4],
+            "datatype": "FP32",
+            "data": [[1, 2, 3, 4]]
+          }
+        ]
+    }
+    """
+    And expect http response status code "200"
+    Then send HTTP inference request with timeout "20s" to model "overcommit-3" with payload:
+    """
+    {
+        "inputs": [
+          {
+            "name": "predict",
+            "shape": [1, 4],
+            "datatype": "FP32",
+            "data": [[1, 2, 3, 4]]
+          }
+        ]
+    }
+    """
+    And expect http response status code "200"

tests/integration/godog/features/model/server_setup.feature

@@ -15,6 +15,14 @@ Feature: Server setup
     spec:
       replicas: 1
       serverConfig: mlserver
+      podSpec:
+        containers:
+          - name: agent
+            env:
+              - name: SELDON_OVERCOMMIT_PERCENTAGE
+                value: "10"
+              - name: MEMORY_REQUEST
+                value: "1073741824"
     """
     When the server should eventually become Ready with timeout "30s"
     Then ensure only "1" pod(s) are deployed for server and they are Ready

tests/integration/godog/main_test.go

@@ -24,7 +24,7 @@ const cmdOptPrefix = "godog."
 
 var opts = godog.Options{
 	Output: colors.Colored(os.Stdout),
-	Format: "progress", // can define default values
+	Format: "pretty", // can define default values
 }
 
 func init() {

tests/integration/godog/steps/server_steps.go

@@ -103,20 +103,23 @@ func (s *server) deployServerSpec(ctx context.Context, spec *godog.DocString) er
 	serverSpec.Namespace = s.namespace
 	s.currentServer = serverSpec
 	s.applyScenarioLabel()
+
+	s.log.Debugf("Attempts to create server %s", serverSpec.Name)
+
 	if _, err := s.seldonK8sClient.MlopsV1alpha1().Servers(s.namespace).Create(ctx, s.currentServer, metav1.CreateOptions{}); err != nil {
 		if k8serrors.IsAlreadyExists(err) {
 			s.log.Debugf("server %s already exists, checking if equal", s.currentServer.Name)
-			deployerServer, err := s.seldonK8sClient.MlopsV1alpha1().Servers(s.namespace).Get(ctx, s.currentServer.Name, metav1.GetOptions{})
+			deployedServer, err := s.seldonK8sClient.MlopsV1alpha1().Servers(s.namespace).Get(ctx, s.currentServer.Name, metav1.GetOptions{})
 			if err != nil {
 				return fmt.Errorf("failed getting server: %w", err)
 			}
-			if equality.Semantic.DeepEqual(serverSpec.Spec, deployerServer.Spec) {
+			if equality.Semantic.DeepEqual(serverSpec.Spec, deployedServer.Spec) {
 				s.log.Debugf("server %s deployed spec equals desired spec", s.currentServer.Name)
 				return nil
 			}
 			s.log.Debugf("server %s deployed spec needs updating to desired spec", s.currentServer.Name)
-			deployerServer.Spec = s.currentServer.Spec
-			if _, err := s.seldonK8sClient.MlopsV1alpha1().Servers(s.namespace).Update(ctx, deployerServer, metav1.UpdateOptions{}); err != nil {
+			deployedServer.Spec = s.currentServer.Spec
+			if _, err := s.seldonK8sClient.MlopsV1alpha1().Servers(s.namespace).Update(ctx, deployedServer, metav1.UpdateOptions{}); err != nil {
 				return fmt.Errorf("failed updating server: %w", err)
 			}
 			return nil