Merge pull request #6127 from chu11/test_enospc

t: add initial ENOSPC tests

Author: mergify[bot]

src/test/docker/docker-run-checks.sh

@@ -205,6 +205,7 @@ else
     docker run --rm \
         --workdir=$WORKDIR \
         --volume=$TOP:$WORKDIR \
+        --mount type=tmpfs,destination=/test/tmpfs-1m,tmpfs-size=1048576 \
         ${PLATFORM} \
         $MOUNT_HOME_ARGS \
         -e PLATFORM \

src/test/docker/docker-run-systest.sh

@@ -116,6 +116,7 @@ checks_group "Launching system instance container $NAME" \
     --volume=$TOP:$WORKDIR \
     --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \
     --tmpfs=/run \
+    --mount=type=tmpfs,destination=/test/tmpfs-1m,tmpfs-size=1048576 \
     --cap-add SYS_PTRACE \
     --name=flux-system-test-$$ \
     --network=host \

t/Makefile.am

@@ -95,6 +95,7 @@ TESTSCRIPTS = \
 	t0022-jj-reader.t \
 	t0023-jobspec1-validate.t \
 	t0026-flux-R.t \
+	t0090-content-enospc.t \
 	t1000-kvs.t \
 	t1001-kvs-internals.t \
 	t1003-kvs-stress.t \

t/t0090-content-enospc.t

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+test_description='Test content ENOSPC corner cases'
+
+. `dirname $0`/sharness.sh
+
+if ! ls /test/tmpfs-1m; then
+	skip_all='skipping ENOSPC tests, no small tmpfs directory mounted'
+	test_done
+fi
+
+test_expect_success 'create script to fill statedir' '
+	cat >fillstatedir.sh <<-EOT &&
+	#!/bin/sh
+	while true ; do
+	      flux run echo 0123456789 > /dev/null 2>&1
+	      if flux dmesg | grep -q "No space left on device"; then
+		   break
+	      fi
+	done
+	EOT
+	chmod +x fillstatedir.sh
+'
+
+test_expect_success 'clear & setup test statedir' '
+	rm -rf /test/tmpfs-1m/* &&
+	mkdir /test/tmpfs-1m/statedir
+'
+
+# rc3 currently hangs under ENOSPC, so disable it. flux start will fail for time being
+test_expect_success 'flux still operates with content-sqlite running out of space' '
+	test_must_fail flux start \
+	    -o,-Scontent.backing-module=content-sqlite \
+	    -o,-Sstatedir=/test/tmpfs-1m/statedir \
+	    -o,-Sbroker.rc3_path= \
+	    "./fillstatedir.sh; flux dmesg; flux run echo helloworld" > sql.out 2> sql.err &&
+        grep -q "No space left on device" sql.out &&
+        grep "helloworld" sql.out
+'
+
+test_expect_success 'clear & setup test statedir' '
+	rm -rf /test/tmpfs-1m/* &&
+	mkdir /test/tmpfs-1m/statedir
+'
+
+# rc3 currently hangs under ENOSPC, so disable it. flux start will fail for time being
+test_expect_success 'flux still operates with content-files running out of space' '
+	test_must_fail flux start \
+	    -o,-Scontent.backing-module=content-files \
+	    -o,-Sstatedir=/test/tmpfs-1m/statedir \
+	    -o,-Sbroker.rc3_path= \
+	    "./fillstatedir.sh; flux dmesg; flux run echo helloworld" > files.out 2> files.err &&
+        grep -q "No space left on device" files.out &&
+        grep "helloworld" files.out
+'
+
+test_done