testsuite: rework archive command sharness test

Problem: t0029-filemap-cmd.t only covers flux-filemap, but that
command is being replaced with flux-archive.

Rename the script to t0029-archive-mmap.t and port the tests to
the new tooling.

Essentially this covers 'flux archive create --mmap'.  Another test
script will be added to more fully cover flux archive without the focus
on the --mmap option.

Author: garlick

t/Makefile.am

@@ -76,7 +76,7 @@ TESTSCRIPTS = \
 	t0012-content-sqlite.t \
 	t0024-content-s3.t \
 	t0028-content-backing-none.t \
-	t0029-filemap-cmd.t \
+	t0029-archive-mmap.t \
 	t0030-marshall.t \
 	t0031-constraint-parser.t \
 	t0032-directives-parser.t \

t/t0029-filemap-cmd.t t/t0029-archive-mmap.tt/t0029-filemap-cmd.t renamed to t/t0029-archive-mmap.t

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-test_description='Test flux-filemap'
+test_description='Test flux-archive'
 
 . `dirname $0`/content/content-helper.sh
 
@@ -33,16 +33,17 @@ if havesparse testholes; then
 	test_set_prereq HAVE_SPARSE
 fi
 
-test_under_flux 2 minimal
+test_under_flux 2 kvs
 
 # after test_under_flux is launched, cannot assume what umask is.  An
 # unexpected umask could affect tests below.  Hard code to 022 for
 # these tests.
 umask 022
 
-test_expect_success 'load content module' '
-	flux exec flux module load content
-'
+# Usage: list_mapped_files tag
+list_mapped_files() {
+	flux module stats content | jq -r ".mmap.tags[\"$1\"][]"
+}
 
 test_expect_success 'create copy directory' '
 	mkdir -p copydir
@@ -53,25 +54,40 @@ test_expect_success 'create test file' '
 	chmod go=r testfile
 '
 test_expect_success 'map nonexistent file fails' '
-	test_must_fail flux filemap map notafile
+	test_must_fail flux archive create --mmap notafile
+'
+test_expect_success 'map fails on rank != 0' '
+	test_must_fail flux exec -r 1 flux archive create --mmap testfile
+'
+test_expect_success 'map fails with --preserve' '
+	test_must_fail flux archive create --mmap --preserve testfile
+'
+test_expect_success 'map fails with --no-force-primary' '
+	test_must_fail flux archive create --mmap --no-force-primary testfile
 '
 test_expect_success 'map unreadable file fails with appropriate error' '
 	touch unreadable &&
 	chmod ugo-r unreadable &&
-	test_must_fail flux filemap map unreadable 2>unreadable.err &&
+	test_must_fail flux archive create --mmap unreadable 2>unreadable.err &&
 	grep "Permission denied" unreadable.err
 '
 test_expect_success 'map test file' '
-	flux filemap map ./testfile
+	flux archive create --mmap ./testfile &&
+	flux kvs get --raw archive.main | jq
+'
+test_expect_success 'content stats show mapped file' '
+	list_mapped_files main >stats.out &&
+	realpath ./testfile >stats.exp &&
+	test_cmp stats.exp stats.out
 '
 test_expect_success 'file can be listed' '
-	flux filemap list
+	flux archive list
 '
 test_expect_success 'file can be listed in long form' '
-	flux filemap list --long
+	flux archive list --long
 '
 test_expect_success 'test file can be read through content cache on rank 0' '
-	flux filemap get -C copydir
+	flux archive extract -C copydir
 '
 test_expect_success 'file content is correct' '
 	test_cmp testfile copydir/testfile
@@ -83,36 +99,38 @@ test_expect_success 'file permissions are correct' '
 '
 test_expect_success 'test file can be read through content cache on rank 1' '
 	rm -f copydir/testfile &&
-	flux exec -r 1 flux filemap get -C copydir &&
+	flux exec -r 1 flux archive extract -C copydir &&
 	test_cmp testfile copydir/testfile
 '
 test_expect_success 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
+'
+test_expect_success 'content stats no longer show mapped file' '
+	test_must_fail list_mapped_files archive.main
 '
 test_expect_success 'drop the cache' '
 	flux exec -r 1 flux content dropcache &&
 	flux content dropcache
 '
 test_expect_success 'test file is not extracted' '
 	rm -f copydir/testfile &&
-	flux filemap get -C copydir &&
-	test_must_fail test -f copydir/testfile
+	test_must_fail flux archive extract
 '
-test_expect_success 'map test file with small blobsize' '
-	flux filemap map --chunksize=10 ./testfile
+test_expect_success 'map test file with small chunksize' '
+	flux archive create --mmap --chunksize=10 ./testfile
 '
 test_expect_success 'test file can be read through content cache on rank 0' '
 	rm -f copydir/testfile &&
-	flux filemap get -C copydir &&
+	flux archive extract -C copydir &&
 	test_cmp testfile copydir/testfile
 '
 test_expect_success 'test file can be read through content cache on rank 1' '
 	rm -f copydir/testfile &&
-	flux exec -r 1 flux filemap get -C copydir &&
+	flux exec -r 1 flux archive extract -C copydir &&
 	test_cmp testfile copydir/testfile
 '
 test_expect_success 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
 '
 test_expect_success 'drop the cache' '
 	flux exec -r 1 flux content dropcache &&
@@ -121,43 +139,39 @@ test_expect_success 'drop the cache' '
 test_expect_success 'create test file' '
 	echo abcdefghijklmnopqrstuvwxyz >testfile2
 '
-test_expect_success 'map test file and get its blobref' '
-	flux filemap map ./testfile2 &&
-	flux filemap list --blobref >testfile2.blobref
+test_expect_success 'map test file with small small-file-threshold' '
+	flux archive create --mmap --small-file-threshold=10 ./testfile2
 '
-test_expect_success 'show raw object' '
-	flux filemap list --raw | jq .
-'
-test_expect_success 'test file can be read through content cache on rank 1' '
-	flux exec -r 1 flux filemap get -C copydir &&
-	test_cmp testfile2 copydir/testfile2
-'
-test_expect_success 'store the duplicate blob on rank 1' '
-	flux content load <testfile2.blobref >testfile2.fileref &&
-	flux exec -r 1 flux content store <testfile2.fileref \
+test_expect_success 'extract blobref from mapped file' "
+	flux kvs get --raw archive.main >testfile2.json &&
+	jq -r '.[0].data[0][2]' <testfile2.json >testfile2.blobref
+"
+test_expect_success 'store the duplicate blob to the mapped one on rank 1' '
+	flux content load <testfile2.blobref >testfile2.data &&
+	flux exec -r 1 flux content store <testfile2.data \
 	    >testfile2.blobref2  &&
 	test_cmp testfile2.blobref testfile2.blobref2
 '
 test_expect_success 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
 '
 test_expect_success 'drop the cache' '
 	flux exec -r 1 flux content dropcache &&
 	flux content dropcache
 '
 test_expect_success 'blob can still be read through content cache on rank 1' '
 	flux exec -r 1 flux content load <testfile2.blobref2 \
-		>testfile2.fileref2 &&
-	test_cmp testfile2.fileref testfile2.fileref2
+		>testfile2.data2 &&
+	test_cmp testfile2.data testfile2.data2
 '
 test_expect_success HAVE_SPARSE 'create sparse test file' '
 	truncate --size=8192 testfile3
 '
 test_expect_success HAVE_SPARSE 'map test file' '
-	flux filemap map ./testfile3
+	flux archive create --mmap ./testfile3
 '
 test_expect_success HAVE_SPARSE 'test file can be read through content cache' '
-	flux filemap get -C copydir &&
+	flux archive extract -C copydir &&
 	test_cmp testfile3 copydir/testfile3
 '
 test_expect_success HAVE_SPARSE 'holes were preserved' '
@@ -166,17 +180,17 @@ test_expect_success HAVE_SPARSE 'holes were preserved' '
 	test_cmp blocks.exp blocks.out
 '
 test_expect_success HAVE_SPARSE 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
 '
 test_expect_success HAVE_SPARSE 'create sparse test file with data' '
 	truncate --size=8192 testfile3b &&
 	echo more-data >>testfile3b
 '
 test_expect_success HAVE_SPARSE 'map test file' '
-	flux filemap map ./testfile3b
+	flux archive create --mmap ./testfile3b
 '
 test_expect_success HAVE_SPARSE 'test file can be read through content cache' '
-	flux filemap get -C copydir &&
+	flux archive extract -C copydir &&
 	test_cmp testfile3b copydir/testfile3b
 '
 test_expect_success HAVE_SPARSE 'holes were preserved' '
@@ -185,155 +199,112 @@ test_expect_success HAVE_SPARSE 'holes were preserved' '
 	test_cmp blocks3b.exp blocks3b.out
 '
 test_expect_success HAVE_SPARSE 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
 '
 test_expect_success 'create test symlink' '
 	ln -s /a/b/c/d testfile4
 '
 test_expect_success 'map test file' '
-	flux filemap map ./testfile4
+	flux archive create --mmap ./testfile4
 '
 test_expect_success 'show raw object' '
-	flux filemap list --raw | jq .
+	flux kvs get --raw archive.main | jq .
 '
-test_expect_success 'test file can be read through content cache' '
-	flux filemap get -vv -C copydir
+test_expect_success 'test file can be extracted' '
+	flux archive extract -v -C copydir
 '
 test_expect_success 'copy is a symlink with expected target' '
 	readlink testfile4 >link.exp &&
 	readlink copydir/testfile4 >link.out &&
 	test_cmp link.exp link.out
 '
 test_expect_success 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
 '
 test_expect_success 'map file by absolute path' '
-	flux filemap map /etc/group
+	flux archive create --mmap /etc/group
 '
 test_expect_success 'test file lists with relative path' '
 	cat >list.exp <<-EOT &&
 	etc/group
 	EOT
-	flux filemap list >list.out &&
+	flux archive list >list.out &&
 	test_cmp list.exp list.out
 '
 test_expect_success 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
 '
 test_expect_success 'create test directory' '
 	mkdir testfile5
 '
 test_expect_success 'map test file' '
-	flux filemap map ./testfile5
+	flux archive create --mmap ./testfile5
 '
 test_expect_success 'test file can be read through content cache' '
-	flux filemap get -vv -C copydir
+	flux archive extract -v -C copydir
 '
 test_expect_success 'copy is a directory' '
 	test -d copydir/testfile5
 '
 test_expect_success 'unmap test file' '
-	flux filemap unmap
-'
-test_expect_success 'map testfile with tags=red,blue' '
-	flux filemap map --tags=red,blue ./testfile
-'
-test_expect_success 'map testfile with tags=green' '
-	flux filemap map --tags=green ./testfile
-'
-test_expect_success 'map testfile with tags=green (again)' '
-	flux filemap map --tags=green ./testfile
-'
-test_expect_success 'map testfile2 with tags=green' '
-	flux filemap map --tags=green ./testfile2
-'
-test_expect_success 'list tags=red,blue,green reports two files' '
-	test $(flux filemap list --tags=red,blue,green | wc -l) -eq 2
-'
-test_expect_success 'unmap red tag' '
-	flux filemap unmap --tags=red
-'
-test_expect_success 'list tags=red,blue,green reports two files' '
-	test $(flux filemap list --tags=red,blue,green | wc -l) -eq 2
-'
-test_expect_success 'unmap tags=blue,green' '
-	flux filemap unmap --tags=blue,green
-'
-test_expect_success 'flux filemap list reports no files' '
-	test $(flux filemap list --tags red,blue,green | wc -l) -eq 0
-'
-test_expect_success 'map test file without mmap' '
-	rm -f copydir/testfile &&
-	flux filemap map --disable-mmap ./testfile
-'
-test_expect_success 'test file did not use blobvec encoding' '
-	flux filemap list --raw | jq -e ".encoding != \"blobvec\""
-'
-test_expect_success 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
 '
 test_expect_success 'map small test file with reduced small file threshold' '
-	flux filemap map --small-file-threshold=0 ./testfile2
+	flux archive create --mmap --small-file-threshold=0 ./testfile2
 '
 test_expect_success 'test file used blobvec encoding' '
-	flux filemap list --raw | jq -e ".encoding = \"blobvec\""
+	flux kvs get --raw archive.main | jq -e ".[0].encoding == \"blobvec\""
 '
 test_expect_success 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
 '
-
 test_expect_success 'map test file' '
 	rm -f copydir/testfile &&
-	flux filemap map ./testfile
+	flux archive create --mmap ./testfile
 '
 test_expect_success 'modify mapped test file without reducing its size' '
 	dd if=/dev/zero of=testfile bs=4096 count=1 conv=notrunc
 '
 test_expect_success 'content change should cause an error' '
 	rm -f copydir/testfile &&
-	test_must_fail flux filemap get -C copydir 2>changed.err &&
+	test_must_fail flux archive extract -C copydir 2>changed.err &&
 	grep changed changed.err
 '
 test_expect_success 'drop cache and unmap test file' '
 	flux content dropcache &&
-	flux filemap unmap
+	flux archive remove
 '
 test_expect_success 'map test file' '
 	rm -f copydir/testfile &&
-	flux filemap map ./testfile
+	flux archive create --mmap ./testfile
 '
 test_expect_success 'truncate mapped test file' '
 	cp /dev/null testfile
 '
 test_expect_success 'size reduction should cause an error' '
 	rm -f copydir/testfile &&
-	test_must_fail flux filemap get -C copydir 2>reduced.err &&
+	test_must_fail flux archive extract -C copydir 2>reduced.err &&
 	grep changed reduced.err
 '
 test_expect_success 'unmap test file' '
-	flux filemap unmap
+	flux archive remove
 '
-
 test_expect_success 're-create and map test file' '
 	${LPTEST} >testfile &&
-	flux filemap map ./testfile
+	flux archive create --mmap ./testfile
 '
-test_expect_success 'get copydir/testfile' '
+test_expect_success 'extract copydir/testfile' '
 	rm -f copydir/testfile &&
-	flux filemap get -C copydir
+	flux archive extract -C copydir
 '
-test_expect_success 'get --overwrite works' '
-	flux filemap get --overwrite -C copydir
+test_expect_success 'extract --overwrite works' '
+	flux archive extract --overwrite -C copydir
 '
-test_expect_success 'get refuses to overwrite without explicit option' '
-	test_must_fail flux filemap get -C copydir
+test_expect_success 'extract refuses to overwrite without explicit option' '
+	test_must_fail flux archive extract -C copydir
 '
 test_expect_success 'unmap test file' '
-	flux filemap unmap
-'
-
-test_expect_success 'remove content module' '
-	flux exec flux module remove content
+	flux archive remove
 '
 
 test_done