testsuite: cover flub bootstrap

Problem: there is no test coverage for adding brokers to
a flux instance.

Add some tests.

Author: garlick

t/Makefile.am

@@ -90,6 +90,7 @@ TESTSCRIPTS = \
 	t0023-jobspec1-validate.t \
 	t0026-flux-R.t \
 	t0033-size-override.t \
+	t0034-flub.t \
 	t1000-kvs.t \
 	t1001-kvs-internals.t \
 	t1003-kvs-stress.t \

t/t0034-flub.t

@@ -0,0 +1,143 @@
+#!/bin/sh
+#
+
+test_description='Test flub bootstrap method'
+
+. `dirname $0`/sharness.sh
+
+test_under_flux 8 full
+
+export FLUX_SSH="${SHARNESS_TEST_SRCDIR}/scripts/tssh"
+
+test_expect_success 'broker fails with bad broker.boot-server' '
+	test_must_fail flux broker \
+	    -Sbroker.rc1_path= -Sbroker.rc3_path= \
+	    -Sbroker.boot-server=local://noexist/path \
+	    /bin/true 2>server.err &&
+	grep "was not found" server.err
+'
+
+test_expect_success 'start a one node job with no extra ranks' '
+	id=$(flux batch -N1 --wrap sleep inf) &&
+	flux job wait-event -t10 $id memo &&
+	flux uri $id >test1.uri
+'
+test_expect_success 'job has size 1' '
+	size=$(flux proxy $(cat test1.uri) flux getattr size) &&
+	test $size -eq 1
+'
+test_expect_success 'flub bootstrap fails with no available ranks' '
+	test_must_fail flux broker \
+	    -Sbroker.boot-server=$(cat test1.uri) 2>noranks.err
+	grep "no available ranks" noranks.err
+'
+test_expect_success 'clean up' '
+	flux cancel --all
+'
+
+#
+# Start 2 node batch job with one extra slot.
+# Submit 1 node broker job that fills the slot.
+# Run a parallel job across all three nodes in the batch job.
+#
+
+test_expect_success 'create config with fake resources' '
+	cat >fake2.toml <<-EOT
+	[resource]
+	noverify = true
+	[[resource.config]]
+	hosts = "a,b,c"
+	cores = "0-3"
+	EOT
+'
+test_expect_success 'start a two node job with one extra rank' '
+	id=$(flux batch -N2 \
+	    --broker-opts=--config-path=fake2.toml \
+	    --broker-opts=-Ssize=3 \
+	    --broker-opts=-Sbroker.quorum=2 \
+	    --broker-opts=-Stbon.topo=kary:0 \
+	    --wrap sleep inf) &&
+	flux job wait-event -t10 $id memo &&
+	flux uri $id >test2.uri
+'
+test_expect_success 'job has size 3' '
+	size=$(flux proxy $(cat test2.uri) flux getattr size) &&
+	test $size -eq 3
+'
+test_expect_success 'overlay status shows extra node offline' '
+	flux proxy $(cat test2.uri) \
+	    flux overlay status --wait=partial --no-pretty >ov2.out &&
+	grep "2 extra0: offline" ov2.out
+'
+test_expect_success 'submit a job that starts the extra broker' '
+	flux submit -N1 flux broker \
+	    --config-path=fake2.toml \
+	    -Stbon.topo=kary:0 \
+	    -Sbroker.boot-server=$(cat test2.uri)
+'
+test_expect_success 'overlay status shows extra rank connected' '
+	run_timeout 20 flux proxy $(cat test2.uri) \
+	    flux overlay status --wait=full --no-pretty >ov2b.out &&
+	grep "2 extra0: full" ov2b.out
+'
+test_expect_success 'run a 3 node job in the expanded instance' '
+	run_timeout 30 flux proxy $(cat test2.uri) \
+	    flux run --label-io -N3 flux pmi barrier
+'
+test_expect_success 'clean up' '
+	flux cancel --all
+'
+
+#
+# Start 3 node batch job with four extra slots (kary:2).
+# Submit 4 node broker job that fills the slots.
+# Run a parallel job across all seven nodes in the batch job.
+#
+
+test_expect_success 'create config with fake resources' '
+	cat >fake3.toml <<-EOT
+	[resource]
+	noverify = true
+	[[resource.config]]
+	hosts = "a,b,c,d,e,f,g"
+	cores = "0-3"
+	EOT
+'
+test_expect_success 'start a two node job with one extra rank' '
+	id=$(flux batch -N3 \
+	    --broker-opts=--config-path=fake3.toml \
+	    --broker-opts=-Ssize=7 \
+	    --broker-opts=-Sbroker.quorum=3 \
+	    --broker-opts=-Stbon.topo=kary:2 \
+	    --wrap sleep inf) &&
+	flux job wait-event -t10 $id memo &&
+	flux uri $id >test3.uri
+'
+test_expect_success 'job has size 7' '
+	size=$(flux proxy $(cat test3.uri) flux getattr size) &&
+	test $size -eq 7
+'
+test_expect_success 'show overlay status' '
+	flux proxy $(cat test3.uri) \
+	    flux overlay status
+'
+test_expect_success 'submit a job that starts extra brokers' '
+	flux submit -N4 flux broker \
+	    --config-path=fake3.toml \
+	    -Stbon.topo=kary:2 \
+	    -Sbroker.boot-server=$(cat test3.uri)
+'
+test_expect_success 'overlay status shows extra ranks connected' '
+	run_timeout 20 flux proxy $(cat test3.uri) \
+	    flux overlay status --wait=full
+'
+test_expect_success 'run a 7 node job in the expanded instance' '
+	run_timeout 30 flux proxy $(cat test3.uri) \
+	    flux run --label-io -N7 flux pmi barrier
+'
+test_expect_success 'clean up' '
+	flux cancel --all
+'
+
+
+test_done