Fix HIGH_WATER_MARK to match actual per-worker capacity

Each eventlet worker can handle 8 concurrent connections, not 32.
The previous HIGH_WATER_MARK of 26 would never trigger since workers
max out at 8 concurrent requests.

Changes:
- Update HIGH_WATER_MARK default from 26 to 6 (75% of 8 connections)
- Fix fallback value in is_worker_overloaded() from 26 to 6
- Update test mocks to use realistic values (3-8 instead of 10-30)

Author: corlettb

app/config.py

@@ -164,8 +164,8 @@ class Config:
 
     # Load Shedding Configuration
     LOAD_SHEDDING_ENABLED = os.getenv("LOAD_SHEDDING_ENABLED", "false").lower() == "true"
-    # High water mark: 80% of capacity (26 out of 32 concurrent requests per worker)
-    HIGH_WATER_MARK = int(os.getenv("HIGH_WATER_MARK", "26"))
+    # High water mark: 75% of capacity (6 out of 8 concurrent requests per worker)
+    HIGH_WATER_MARK = int(os.getenv("HIGH_WATER_MARK", "6"))
     # Throttle services contributing this % or more of total request volume
     THROTTLE_CONTRIBUTION_PCT = int(os.getenv("THROTTLE_CONTRIBUTION_PCT", "20"))
     # Only apply contribution-based throttling when enough services are active

app/load_shedding.py

@@ -185,7 +185,7 @@ def is_worker_overloaded() -> bool:
 
     try:
         current_load = concurrent_request_counter.get()
-        high_water_mark = current_app.config.get("HIGH_WATER_MARK", 26)
+        high_water_mark = current_app.config.get("HIGH_WATER_MARK", 6)
 
         is_overloaded = current_load > high_water_mark
 

tests/app/test_load_shedding.py

@@ -31,7 +31,7 @@ def test_returns_false_when_worker_not_overloaded(self, notify_api: "Flask", moc
 
     def test_returns_false_when_below_high_water_mark(self, notify_api: "Flask", mocker: MockerFixture) -> None:
         mock_counter = mocker.patch("app.load_shedding.concurrent_request_counter")
-        mock_counter.get.return_value = 20  # Below HIGH_WATER_MARK of 26
+        mock_counter.get.return_value = 4  # Below HIGH_WATER_MARK of 6
         mock_gauge = mocker.patch("app.load_shedding.WORKER_LOAD_SHEDDING")
 
         assert is_worker_overloaded() is False
@@ -41,7 +41,7 @@ def test_returns_false_when_below_high_water_mark(self, notify_api: "Flask", moc
 
     def test_returns_true_when_above_high_water_mark(self, notify_api: "Flask", mocker: MockerFixture) -> None:
         mock_counter = mocker.patch("app.load_shedding.concurrent_request_counter")
-        mock_counter.get.return_value = 28  # Above HIGH_WATER_MARK of 26
+        mock_counter.get.return_value = 7  # Above HIGH_WATER_MARK of 6
         mock_gauge = mocker.patch("app.load_shedding.WORKER_LOAD_SHEDDING")
 
         assert is_worker_overloaded() is True
@@ -51,7 +51,7 @@ def test_returns_true_when_above_high_water_mark(self, notify_api: "Flask", mock
 
     def test_returns_false_at_boundary(self, notify_api: "Flask", mocker: MockerFixture) -> None:
         mock_counter = mocker.patch("app.load_shedding.concurrent_request_counter")
-        mock_counter.get.return_value = 26  # At HIGH_WATER_MARK
+        mock_counter.get.return_value = 6  # At HIGH_WATER_MARK
         mock_gauge = mocker.patch("app.load_shedding.WORKER_LOAD_SHEDDING")
 
         # At the threshold should NOT trigger (uses > not >=)
@@ -73,32 +73,32 @@ def test_increments_activation_counter_on_state_transition(
 
         ls_module._is_currently_load_shedding = False
 
-        # First call: healthy (10 < 26)
-        mock_counter.get.return_value = 10
+        # First call: healthy (3 < 6)
+        mock_counter.get.return_value = 3
         is_worker_overloaded()
         assert mock_activation_counter.inc.call_count == 0
         assert mock_gauge.set.call_args[0][0] == 0  # Gauge set to 0
 
-        # Second call: enter overload (28 > 26) - should increment
-        mock_counter.get.return_value = 28
+        # Second call: enter overload (7 > 6) - should increment
+        mock_counter.get.return_value = 7
         is_worker_overloaded()
         assert mock_activation_counter.inc.call_count == 1
         assert mock_gauge.set.call_args[0][0] == 1  # Gauge set to 1
 
-        # Third call: stay overloaded (30 > 26) - should NOT increment again
-        mock_counter.get.return_value = 30
+        # Third call: stay overloaded (8 > 6) - should NOT increment again
+        mock_counter.get.return_value = 8
         is_worker_overloaded()
         assert mock_activation_counter.inc.call_count == 1  # Still 1, not 2
         assert mock_gauge.set.call_args[0][0] == 1  # Gauge still 1
 
-        # Fourth call: exit overload (10 < 26) - should NOT increment
-        mock_counter.get.return_value = 10
+        # Fourth call: exit overload (3 < 6) - should NOT increment
+        mock_counter.get.return_value = 3
         is_worker_overloaded()
         assert mock_activation_counter.inc.call_count == 1
         assert mock_gauge.set.call_args[0][0] == 0  # Gauge back to 0
 
-        # Fifth call: re-enter overload (28 > 26) - should increment again
-        mock_counter.get.return_value = 28
+        # Fifth call: re-enter overload (7 > 6) - should increment again
+        mock_counter.get.return_value = 7
         is_worker_overloaded()
         assert mock_activation_counter.inc.call_count == 2
         assert mock_gauge.set.call_args[0][0] == 1  # Gauge set to 1 again
@@ -115,7 +115,7 @@ def test_logs_activation_and_deactivation(self, notify_api: "Flask", mocker: Moc
         ls_module._is_currently_load_shedding = False
 
         # Enter load shedding - should log warning
-        mock_counter.get.return_value = 28
+        mock_counter.get.return_value = 7
         with notify_api.app_context():
             mock_logger = mocker.patch("app.load_shedding.current_app.logger")
             is_worker_overloaded()
@@ -125,7 +125,7 @@ def test_logs_activation_and_deactivation(self, notify_api: "Flask", mocker: Moc
             assert "ACTIVATED" in mock_logger.warning.call_args[0][0]
 
         # Exit load shedding - should log info
-        mock_counter.get.return_value = 10
+        mock_counter.get.return_value = 3
         with notify_api.app_context():
             mock_logger = mocker.patch("app.load_shedding.current_app.logger")
             is_worker_overloaded()