MB-51665 Don't raise false diskspace worker stuck

We periodically run a check of diskspace and that check can possibly
hang and so we have logic to detect if it has hung. In addition we might
be running on a laptop which has been closed and then opened after some
time. We currently cannot tell if the lack of a periodic diskspace check
is due to it being wedged or due to the time jumping.

We now will not declare the diskspace check to be wedged until it has
not updated the diskspace info for three times the periodic interval. If we
wake up from a laptop being reopened then enough time may have elapsed
such that we mark the diskspace check as pending stale and when the
second interval elapses and the check doesn't occur we assume it's wedged.
We allow three misses to account for possible races between the process
that updates the last checked time and the process doing the check.

Change-Id: I3e227f8588000eee5a3fc5d0afcf62fb18c21f3a
Reviewed-on: https://review.couchbase.org/c/ns_server/+/174401
Well-Formed: Restriction Checker
Well-Formed: Build Bot <[email protected]>
Tested-by: Build Bot <[email protected]>
Reviewed-by: Timofey Barmin <[email protected]>
Reviewed-by: Bryan McCoid <[email protected]>

Author: stevewatanabe

src/menelaus_web_alerts_srv.erl

@@ -15,6 +15,8 @@
 %% needed to mock ns_config in tests
 -include("ns_config.hrl").
 
+-define(MAX_DISK_USAGE_MISSED_CHECKS, 2).
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 -endif.
@@ -304,15 +306,26 @@ check(ip, Opaque, _History, _Stats) ->
     Opaque;
 
 check(disk_usage_analyzer_stuck, Opaque, _History, _Stats) ->
-    case ns_disksup:is_stale() of
-        true ->
-            global_alert(disk_usage_analyzer_stuck,
-                         fmt_to_bin(
-                           errors(disk_usage_analyzer_stuck), [node()]));
-        false -> ok
-    end,
-
-    Opaque;
+    IsStale = ns_disksup:is_stale(),
+    Missed = case dict:find(disk_usage_missed_checks, Opaque) of
+                 {ok, M} -> M;
+                 error -> 0
+             end,
+
+    NewMissed = case {IsStale, Missed} of
+                    {false, _} ->
+                        0;
+                    {true, Missed}
+                      when Missed < ?MAX_DISK_USAGE_MISSED_CHECKS ->
+                        Missed + 1;
+                    {true, Missed} ->
+                        global_alert(disk_usage_analyzer_stuck,
+                                     fmt_to_bin(
+                                       errors(disk_usage_analyzer_stuck),
+                                       [node()])),
+                        Missed + 1
+                end,
+    dict:store(disk_usage_missed_checks, NewMissed, Opaque);
 
 %% @doc check the capacity of the drives used for db and log files
 check(disk, Opaque, _History, _Stats) ->