Some fixes for endpoints (#283)

Author: yunfeng-scale

charts/model-engine/templates/service_template_config_map.yaml

@@ -127,6 +127,7 @@ data:
                   port: ${FORWARDER_PORT}
                 initialDelaySeconds: ${READINESS_INITIAL_DELAY}
                 periodSeconds: 5
+                timeoutSeconds: 5
               resources:
                 requests:
                   cpu: 0.1
@@ -172,6 +173,7 @@ data:
                   port: ${FORWARDER_PORT}
                 initialDelaySeconds: ${READINESS_INITIAL_DELAY}
                 periodSeconds: 5
+                timeoutSeconds: 5
               resources:
                 requests:
                   cpu: 0.1
@@ -285,11 +287,6 @@ data:
                   port: ${USER_CONTAINER_PORT}
                 initialDelaySeconds: ${READINESS_INITIAL_DELAY}
                 periodSeconds: 5
-              livenessProbe:
-                httpGet:
-                  path: ${HEALTHCHECK_ROUTE}
-                  port: ${USER_CONTAINER_PORT}
-                initialDelaySeconds: ${READINESS_INITIAL_DELAY}
                 timeoutSeconds: 5
               resources:
                 requests:

model-engine/model_engine_server/common/resource_limits.py

@@ -43,8 +43,8 @@
     GpuType.NVIDIA_AMPERE_A100E: A100_INSTANCE_LIMITS,
 }
 
-FORWARDER_CPU_USAGE = 0.5
-FORWARDER_MEMORY_USAGE = "1Gi"
+FORWARDER_CPU_USAGE = 1
+FORWARDER_MEMORY_USAGE = "2Gi"
 FORWARDER_STORAGE_USAGE = "1G"
 
 logger = make_logger(filename_wo_ext(__name__))

model-engine/model_engine_server/inference/configs/service--http_forwarder.yaml

@@ -16,4 +16,4 @@ forwarder:
     batch_route: null
     model_engine_unwrap: true
     serialize_results_as_string: false
-  max_concurrency: 20
+  max_concurrency: 100