t: check pending requests at end of kvs tests

Problem: Now that the KVS tracks pending requests, it would be easy
to introduce an error in the future where pending requests are not
tracked correctly.

Ensure that pending requests are 0 (or the expected number) at the
end of each kvs test file.

Author: chu11

t/t1000-kvs.t

@@ -1273,6 +1273,15 @@ test_expect_success 'setroot-unpause request with empty payload fails with EPROT
 	${RPC} kvs.setroot-unpause 71 </dev/null
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests before module removal' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 #
 # module corner cases
 #

t/t1001-kvs-internals.t

@@ -522,6 +522,18 @@ test_expect_success 'kvs: test invalid lookup rpc' '
 	grep "flux_future_get: Protocol error" lookup_invalid_output
 '
 
+#
+# ensure pending requests are the expected number
+#
+
+# fence invalid tests above linger a pending request on both rank 0 and rank 1
+test_expect_success 'kvs: 1 pending requests at end of tests before module removal' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 1 &&
+	pendingcount1=$(flux exec -n -r 1 sh -c "flux module stats -p pending_requests kvs") &&
+	test $pendingcount1 -eq 1
+'
+
 #
 # test ENOSYS on unfinished requests when unloading the KVS module
 #

t/t1003-kvs-stress.t

@@ -123,4 +123,13 @@ test_expect_success 'kvs: transaction-merge disabling works' '
 	test "$OUTPUT" = "${THREADS}"
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done

t/t1004-kvs-namespace.t

@@ -546,4 +546,13 @@ test_expect_success NO_CHAIN_LINT 'kvs: wait in different namespaces works' '
         test_expect_code 0 wait $testkvswaitpid
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done

t/t1005-kvs-security.t

@@ -433,4 +433,13 @@ test_expect_success 'kvs: stats clear fails (user)' '
         unset_userid
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done

t/t1007-kvs-lookup-watch.t

@@ -670,4 +670,14 @@ test_expect_success 'kvs-watch.lookup request non-streaming fails with EPROTO(71
 	echo "{\"namespace\":"foo", \"key\":\"bar\", \"flags\":4}" \
 		${RPC} kvs-watch.lookup 71
 '
+
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done

t/t1008-kvs-eventlog.t

@@ -236,4 +236,13 @@ test_expect_success 'flux kvs eventlog wait-event fails on bad input' '
 	test_must_fail flux kvs eventlog wait-event
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done

t/t1009-kvs-copy.t

@@ -187,4 +187,13 @@ test_expect_success 'kvs-move nonexistent dst namespace fails' '
 	test_must_fail flux kvs move --dst-namespace=nons test.foo test.foo
 '
 
+#
+# ensure no lingering pending requests
+#
+
+test_expect_success 'kvs: no pending requests at end of tests' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_done

t/t1010-kvs-commit-sync.t

@@ -77,6 +77,11 @@ test_expect_success 'kvs: sync without backing store fails' '
         test_must_fail flux kvs put --sync d=4
 '
 
+test_expect_success 'kvs: no pending requests at end of tests before module removal' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_expect_success 'kvs: unload kvs' '
 	flux module remove kvs
 '

t/t1011-kvs-checkpoint-period.t

@@ -156,6 +156,11 @@ test_expect_success 'kvs: checkpoint of kvs-primary should change in time (4)' '
 	test_cmp checkpoint5.out blob5.out
 '
 
+test_expect_success 'kvs: no pending requests at end of tests before module removal' '
+	pendingcount=$(flux module stats -p pending_requests kvs) &&
+	test $pendingcount -eq 0
+'
+
 test_expect_success 'kvs: remove modules' '
 	flux module remove kvs &&
 	flux module remove content-sqlite &&