safe/fast mode: add tnt_crud_router_cache_clear_ts metric to router

This metric can help user to monitor routers status
in the cluster when switching it back to fast mode.

Also changed metrics callbacks to prevent duplicate metrics
if init function is called multiple times.

Author: ita-sammann

CHANGELOG.md

@@ -7,9 +7,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Added
+* Add `tnt_crud_router_cache_clear_ts` metric to router to help user correctly disable safe mode in cluster.
+
 ### Fixed
 * Return bucket_ref error as array in `crud.*_many` methods.
 * Move bucket_unref out of transaction.
+* Prevent duplicate metrics if init function is called multiple times.
 
 ## [1.7.2] - 28-01-26
 

crud.lua

@@ -167,6 +167,8 @@ crud.rebalance.router_cache_clear = rebalance.router_api.cache_clear
 
 function crud.init_router()
     rawset(_G, 'crud', crud)
+
+    rebalance.init_router_metrics()
 end
 
 function crud.stop_router()

crud/common/rebalance.lua

@@ -1,4 +1,5 @@
 local log = require('log')
+local fiber = require('fiber')
 local trigger = require('internal.trigger')
 local vshard_consts = require('vshard.consts')
 local schema = require('crud.schema')
@@ -11,6 +12,12 @@ local rebalance = {
     safe_mode = false,
     -- Trigger is run with one argument: true if safe mode is enabled and false if disabled.
     on_safe_mode_toggle = trigger.new('_crud.safe_mode_toggle'),
+    router_cache_clear_ts = 0,
+
+    _metrics = {
+        safe_mode_enabled_gauge = nil,
+        router_cache_clear_ts_gauge = nil,
+    },
 }
 
 local function safe_mode_bucket_trigger(_, new, space, op)
@@ -98,17 +105,7 @@ function rebalance.init()
         _safe_mode_disable()
     end
 
-    --- Rebalance related metrics
-    if has_metrics_module then
-        local safe_mode_enabled_gauge = metrics.gauge(
-                'tnt_crud_storage_safe_mode_enabled',
-                "is safe mode enabled on this storage instance"
-        )
-
-        metrics.register_callback(function()
-            safe_mode_enabled_gauge:set(rebalance.safe_mode_status() and 1 or 0)
-        end)
-    end
+    rebalance.init_storage_metrics()
 end
 
 function rebalance.safe_mode_status()
@@ -139,7 +136,37 @@ function rebalance.router_api.cache_clear()
         log.warn("Router is not initialized yet")
         return
     end
-    return router:_route_map_clear()
+    router:_route_map_clear()
+    rebalance.router_cache_clear_ts = fiber.time()
+end
+
+--- Rebalance related metrics
+function rebalance._metrics.storage_callback()
+    if not rebalance._metrics.safe_mode_enabled_gauge then return end
+    rebalance._metrics.safe_mode_enabled_gauge:set(rebalance.safe_mode_status() and 1 or 0)
+end
+
+function rebalance.init_storage_metrics()
+    if not has_metrics_module then return end
+    rebalance._metrics.safe_mode_enabled_gauge = metrics.gauge(
+            'tnt_crud_storage_safe_mode_enabled',
+            "is safe mode enabled on this storage instance"
+    )
+    metrics.register_callback(rebalance._metrics.storage_callback)
+end
+
+function rebalance._metrics.router_callback()
+    if not rebalance._metrics.router_cache_clear_ts_gauge then return end
+    rebalance._metrics.router_cache_clear_ts_gauge:set(rebalance.router_cache_clear_ts)
+end
+
+function rebalance.init_router_metrics()
+    if not has_metrics_module then return end
+    rebalance._metrics.router_cache_clear_ts_gauge = metrics.gauge(
+            'tnt_crud_router_cache_clear_ts',
+            "when route cache was last cleared on this router instance"
+    )
+    metrics.register_callback(rebalance._metrics.router_callback)
 end
 
 return rebalance

test/integration/metrics_test.lua

@@ -1,3 +1,4 @@
+local fiber = require('fiber')
 local helpers = require('test.helper')
 local t = require('luatest')
 
@@ -83,3 +84,60 @@ pgroup.test_safe_mode_storage_metrics = function(g)
         end
     end)
 end
+
+pgroup.test_router_cache_metrics = function(g)
+    local has_metrics_module = require('metrics')
+    t.skip_if(not has_metrics_module, 'No metrics module in current version')
+
+    -- Check router cache metric initial value on router
+    local observed = g.router:eval("return require('metrics').collect({ invoke_callbacks = true })")
+    local has_metric = false
+    for _, m in pairs(observed) do
+        if m.metric_name == 'tnt_crud_router_cache_clear_ts' then
+            t.assert_equals(m.value, 0, 'Cache never cleared')
+            has_metric = true
+            break
+        end
+    end
+    if not has_metric then
+        t.fail('No tnt_crud_router_cache_clear_ts metric found on router')
+    end
+
+    -- Check no router cache metric on storage
+    helpers.call_on_storages(g.cluster, function(server)
+        observed = server:eval("return require('metrics').collect({ invoke_callbacks = true })")
+        for _, m in pairs(observed) do
+            if m.metric_name == 'tnt_crud_router_cache_clear_ts' then
+                t.fail('tnt_crud_router_cache_clear_ts metric found on storage')
+            end
+        end
+    end)
+
+    -- Clear router cache
+    local expected_ts = fiber.time()
+    g.router:call("crud.rebalance.router_cache_clear")
+
+    -- Check router cache metric new value on router
+    observed = g.router:eval("return require('metrics').collect({ invoke_callbacks = true })")
+    has_metric = false
+    for _, m in pairs(observed) do
+        if m.metric_name == 'tnt_crud_router_cache_clear_ts' then
+            t.assert_almost_equals(m.value, expected_ts, 5, 'Cache never cleared')
+            has_metric = true
+            break
+        end
+    end
+    if not has_metric then
+        t.fail('No tnt_crud_router_cache_clear_ts metric found on router')
+    end
+
+    -- Check no router cache metric appeared on storage
+    helpers.call_on_storages(g.cluster, function(server)
+        observed = server:eval("return require('metrics').collect({ invoke_callbacks = true })")
+        for _, m in pairs(observed) do
+            if m.metric_name == 'tnt_crud_router_cache_clear_ts' then
+                t.fail('tnt_crud_router_cache_clear_ts metric found on storage')
+            end
+        end
+    end)
+end