fiber: don't hang on uncancellable client fiber

Similar to handling of uncancellable iproto request let's also panic on
uncancellable client fiber on shutdown.

Now `sql-tap/in2.test` start to fail under ASAN. `metrics_collector`
fiber of feedback_daemon is not cancelled in due time on shutdown.
Server panics and test fails. Turned out the fiber is cancelled during
metrics collection which is done under pcall. So cancelling is ignored
and we start the loop again going to sleep. fiber.sleep() will detect
cancel state after sleep but we panic before that moment.

We tried to make fiber.sleep() cancellation point but 2 tests start to
spin in busy loop (one of them is the test added in the patch).  So this
approach looks dangeous.

Part of tarantool#8423

NO_CHANGELOG=internal
NO_DOC=internal

Author: nshy

src/box/box.cc

@@ -6005,7 +6005,10 @@ box_storage_shutdown()
 	 * we won't need to handle requests from client fibers after/during
 	 * subsystem shutdown.
 	 */
-	fiber_shutdown();
+	if (fiber_shutdown(box_shutdown_timeout) != 0) {
+		diag_log();
+		panic("cannot gracefully shutdown client fibers");
+	}
 	replication_shutdown();
 	gc_shutdown();
 	engine_shutdown();

src/box/lua/feedback_daemon.lua

@@ -426,6 +426,9 @@ local function metrics_collect_loop(self)
         if new_metric ~= nil then
             insert_metric(self, new_metric)
         end
+        if not pcall(fiber.testcancel) then
+            break
+        end
     end
     self.shutdown:put("stopped")
 end

src/lib/core/fiber.c

@@ -2377,8 +2377,8 @@ fiber_set_system(struct fiber *f, bool yesno)
 	}
 }
 
-void
-fiber_shutdown(void)
+int
+fiber_shutdown(double timeout)
 {
 	assert(cord()->shutdown_fiber == NULL);
 	cord()->is_shutdown = true;
@@ -2388,7 +2388,13 @@ fiber_shutdown(void)
 			fiber_cancel(fiber);
 	}
 	cord()->shutdown_fiber = fiber();
-	while (cord()->client_fiber_count != 0)
-		fiber_yield();
+	double deadline = ev_monotonic_now(loop()) + timeout;
+	while (cord()->client_fiber_count != 0) {
+		if (fiber_yield_deadline(deadline)) {
+			diag_set(TimedOut);
+			break;
+		}
+	}
 	cord()->shutdown_fiber = NULL;
+	return cord()->client_fiber_count == 0 ? 0 : -1;
 }

src/lib/core/fiber.h

@@ -1254,9 +1254,17 @@ fiber_lua_state(struct fiber *f);
 void
 fiber_set_system(struct fiber *f, bool yesno);
 
-/** Cancel all client (non system) fibers and wait until they finished. */
-void
-fiber_shutdown(void);
+/**
+ * Cancel all client (non system) fibers and wait until they finished.
+ *
+ * If not finished in given timeout then failure result code is returned.
+ *
+ * Return:
+ *   0 - success
+ *  -1 - failure (diag is set)
+ */
+int
+fiber_shutdown(double timeout);
 
 #if defined(__cplusplus)
 } /* extern "C" */

test/box-luatest/shutdown_test.lua

@@ -119,3 +119,27 @@ g.test_shutdown_of_hanging_iproto_request = function(cg)
     t.assert(cg.server:grep_log('cannot gracefully shutdown iproto', nil,
              {filename = log}))
 end
+
+-- Test shutdown does not hang if there is client fiber that
+-- cannot be cancelled.
+g.test_shutdown_of_hanging_client_fiber = function(cg)
+    cg.server:exec(function()
+        local log = require('log')
+        local fiber = require('fiber')
+        local tweaks = require('internal.tweaks')
+        tweaks.box_shutdown_timeout = 1.0
+        fiber.new(function()
+            log.info('going to sleep for test')
+            while true do
+                pcall(fiber.sleep, 1000)
+            end
+        end)
+    end)
+    t.helpers.retrying({}, function()
+        t.assert(cg.server:grep_log('going to sleep for test'))
+    end)
+    local log = fio.pathjoin(cg.server.workdir, cg.server.alias .. '.log')
+    test_no_hang_on_shutdown(cg.server)
+    t.assert(cg.server:grep_log('cannot gracefully shutdown client fibers', nil,
+             {filename = log}))
+end

test/unit/fiber.cc

@@ -722,7 +722,8 @@ fiber_test_shutdown(void)
 	fail_unless(fiber4 != NULL);
 	fiber_set_joinable(fiber4, true);
 
-	fiber_shutdown();
+	int rc = fiber_shutdown(1000.0);
+	fail_unless(rc == 0);
 
 	fail_unless((fiber1->flags & FIBER_IS_DEAD) != 0);
 	fail_unless((fiber2->flags & FIBER_IS_DEAD) == 0);