t: add content ENOSPC tests

Problem: There are no tests for when space runs out on disk.

Add some initial tests in the new t0090-content-enospc.t.

Author: chu11

t/Makefile.am

@@ -95,6 +95,7 @@ TESTSCRIPTS = \
 	t0022-jj-reader.t \
 	t0023-jobspec1-validate.t \
 	t0026-flux-R.t \
+	t0090-content-enospc.t \
 	t1000-kvs.t \
 	t1001-kvs-internals.t \
 	t1003-kvs-stress.t \

t/t0090-content-enospc.t

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+test_description='Test content ENOSPC corner cases'
+
+. `dirname $0`/sharness.sh
+
+if ! ls /test/tmpfs-1m; then
+	skip_all='skipping ENOSPC tests, no small tmpfs directory mounted'
+	test_done
+fi
+
+test_expect_success 'create script to fill statedir' '
+	cat >fillstatedir.sh <<-EOT &&
+	#!/bin/sh
+	while true ; do
+	      flux run echo 0123456789 > /dev/null 2>&1
+	      if flux dmesg | grep -q "No space left on device"; then
+		   break
+	      fi
+	done
+	EOT
+	chmod +x fillstatedir.sh
+'
+
+test_expect_success 'clear & setup test statedir' '
+	rm -rf /test/tmpfs-1m/* &&
+	mkdir /test/tmpfs-1m/statedir
+'
+
+# rc3 currently hangs under ENOSPC, so disable it. flux start will fail for time being
+test_expect_success 'flux still operates with content-sqlite running out of space' '
+	test_must_fail flux start \
+	    -o,-Scontent.backing-module=content-sqlite \
+	    -o,-Sstatedir=/test/tmpfs-1m/statedir \
+	    -o,-Sbroker.rc3_path= \
+	    "./fillstatedir.sh; flux dmesg; flux run echo helloworld" > sql.out 2> sql.err &&
+        grep -q "No space left on device" sql.out &&
+        grep "helloworld" sql.out
+'
+
+test_expect_success 'clear & setup test statedir' '
+	rm -rf /test/tmpfs-1m/* &&
+	mkdir /test/tmpfs-1m/statedir
+'
+
+# rc3 currently hangs under ENOSPC, so disable it. flux start will fail for time being
+test_expect_success 'flux still operates with content-files running out of space' '
+	test_must_fail flux start \
+	    -o,-Scontent.backing-module=content-files \
+	    -o,-Sstatedir=/test/tmpfs-1m/statedir \
+	    -o,-Sbroker.rc3_path= \
+	    "./fillstatedir.sh; flux dmesg; flux run echo helloworld" > files.out 2> files.err &&
+        grep -q "No space left on device" files.out &&
+        grep "helloworld" files.out
+'
+
+test_done