Increase default concurrency to 100 for http forwarder (#552)

* increase default concurrency to 50 for http forwarder

* even more hehe

* codecoverage

Author: seanshi-scale

model-engine/model_engine_server/inference/forwarding/http_forwarder.py

@@ -51,7 +51,7 @@ def get_streaming_forwarder_loader():
 @lru_cache()
 def get_concurrency_limiter():
     config = get_config()
-    concurrency = int(config.get("max_concurrency", 5))
+    concurrency = int(config.get("max_concurrency", 100))
     return MultiprocessingConcurrencyLimiter(
         concurrency=concurrency, fail_on_concurrency_limit=True
     )

model-engine/tests/unit/inference/test_http_forwarder.py

@@ -10,6 +10,7 @@
 from model_engine_server.inference.forwarding.forwarding import Forwarder
 from model_engine_server.inference.forwarding.http_forwarder import (
     MultiprocessingConcurrencyLimiter,
+    get_concurrency_limiter,
     predict,
 )
 from model_engine_server.inference.infra.gateways.datadog_inference_monitoring_metrics_gateway import (
@@ -57,6 +58,32 @@ def json(self) -> dict:
     return mocked_static_json()
 
 
+def mocked_get_config():
+    return {
+        "sync": {
+            "user_port": 5005,
+            "user_hostname": "localhost",
+            "use_grpc": False,
+            "predict_route": "/predict",
+            "healthcheck_route": "/readyz",
+            "batch_route": None,
+            "model_engine_unwrap": True,
+            "serialize_results_as_string": True,
+            "forward_http_status": True,
+        },
+        "stream": {
+            "user_port": 5005,
+            "user_hostname": "localhost",
+            "predict_route": "/stream",
+            "healthcheck_route": "/readyz",
+            "batch_route": None,
+            "model_engine_unwrap": True,
+            "serialize_results_as_string": False,
+        },
+        "max_concurrency": 42,
+    }
+
+
 @pytest.fixture
 def post_inference_hooks_handler():
     handler = PostInferenceHooksHandler(
@@ -108,6 +135,13 @@ def mock_request():
     )
 
 
+@mock.patch("model_engine_server.inference.forwarding.http_forwarder.get_config", mocked_get_config)
+def test_get_concurrency_limiter():
+    limiter = get_concurrency_limiter()
+    assert isinstance(limiter, MultiprocessingConcurrencyLimiter)
+    assert limiter.concurrency == 42
+
+
 @mock.patch("requests.post", mocked_post)
 @mock.patch("requests.get", mocked_get)
 def test_http_service_429(mock_request, post_inference_hooks_handler):