Making inferenceModel optional (#1024)

Author: kfswain

pkg/epp/requestcontrol/director.go

@@ -91,12 +91,16 @@ func (d *Director) HandleRequest(ctx context.Context, reqCtx *handlers.RequestCo
 		return reqCtx, err
 	}
 
-	// NOTE: The nil checking for the modelObject means that we DO allow passthrough currently.
-	// This might be a security risk in the future where adapters not registered in the InferenceModel
-	// are able to be requested by using their distinct name.
 	modelObj := d.datastore.ModelGet(reqCtx.Model)
 	if modelObj == nil {
-		return reqCtx, errutil.Error{Code: errutil.BadConfiguration, Msg: fmt.Sprintf("error finding a model object in InferenceModel for input %v", reqCtx.Model)}
+		logger.Info("No associated inferenceModel found, using default", "model", reqCtx.Model)
+		sheddable := v1alpha2.Sheddable
+		modelObj = &v1alpha2.InferenceModel{
+			Spec: v1alpha2.InferenceModelSpec{
+				ModelName:   reqCtx.Model,
+				Criticality: &sheddable,
+			},
+		}
 	}
 
 	reqCtx.ResolvedTargetModel = reqCtx.Model

pkg/epp/requestcontrol/director_test.go

@@ -268,6 +268,27 @@ func TestDirector_HandleRequest(t *testing.T) {
 			},
 			wantMutatedBodyModel: "resolved-target-model-A",
 		},
+		{
+			name: "nonexistent target defined, use default inference model",
+			schedulerMockSetup: func(m *mockScheduler) {
+				m.scheduleResults = defaultSuccessfulScheduleResults
+			},
+			wantReqCtx: &handlers.RequestContext{
+				Model:               "food-review-1",
+				ResolvedTargetModel: "food-review-1",
+				TargetPod: &backend.Pod{
+					NamespacedName: types.NamespacedName{Namespace: "default", Name: "pod1"},
+					Address:        "192.168.1.100",
+				},
+				TargetEndpoint: "192.168.1.100:8000",
+			},
+			wantMutatedBodyModel: "food-review-1",
+			reqBodyMap: map[string]interface{}{
+				"model":  "food-review-1",
+				"prompt": "test prompt",
+			},
+			mockSaturationDetector: &mockSaturationDetector{isSaturated: false},
+		},
 		{
 
 			name: "request dropped (sheddable, saturated)",
@@ -298,24 +319,6 @@ func TestDirector_HandleRequest(t *testing.T) {
 			},
 			wantErrCode: errutil.BadRequest,
 		},
-		{
-			name: "invalid model defined, expect err",
-			reqBodyMap: map[string]interface{}{
-				"model":  "non-existent-model",
-				"prompt": "test prompt",
-			},
-			mockSaturationDetector: &mockSaturationDetector{isSaturated: false},
-			wantErrCode:            errutil.BadConfiguration,
-		},
-		{
-			name: "invalid target defined, expect err",
-			reqBodyMap: map[string]interface{}{
-				"model":  "food-review-1",
-				"prompt": "test prompt",
-			},
-			mockSaturationDetector: &mockSaturationDetector{isSaturated: false},
-			wantErrCode:            errutil.BadConfiguration,
-		},
 		{
 			name: "scheduler returns error",
 			reqBodyMap: map[string]interface{}{