Multi-job based data verification

Makefile

@@ -63,7 +63,7 @@ SOURCE :=	$(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \
 		profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \
 		workqueue.c rate-submit.c optgroup.c helper_thread.c \
 		steadystate.c zone-dist.c zbd.c dedupe.c dataplacement.c \
-		sprandom.c
+		sprandom.c skiplist.c
 
 ifdef CONFIG_LIBHDFS
   HDFSFLAGS= -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(FIO_LIBHDFS_INCLUDE)

backend.c

@@ -586,6 +586,22 @@ static void do_verify(struct thread_data *td, uint64_t verify_bytes)
 
 	dprint(FD_VERIFY, "starting loop\n");
 
+	/*
+	 * If using shared verify table, wait for all write jobs to complete
+	 */
+	if (td->shared_verify_table) {
+		int active = atomic_load(&td->shared_verify_table->write_jobs_active);
+		int done = atomic_load(&td->shared_verify_table->write_jobs_done);
+
+		if (active > 0 && done < active) {
+			while (atomic_load(&td->shared_verify_table->write_jobs_done) < active) {
+				usleep(10000);
+				if (td->terminate)
+					return;
+			}
+		}
+	}
+
 	/*
 	 * sync io first and invalidate cache, to make sure we really
 	 * read from disk.
@@ -936,7 +952,11 @@ void log_inflight(struct thread_data *td, struct io_u *io_u)
 		abort();
 	}
 
-	if (td->inflight_issued != io_u->numberio) {
+	/*
+	 * With shared verify tables, numberio is allocated globally across jobs,
+	 * so skip this check as td->inflight_issued is per-job
+	 */
+	if (!td->shared_verify_table && td->inflight_issued != io_u->numberio) {
 		log_err("inflight_issued does not match: numberio=%"PRIu64", inflight_issued=%"PRIu64"\n",
 			io_u->numberio, td->inflight_issued);
 		abort();
@@ -1125,7 +1145,10 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)
 		if (io_u->ddir == DDIR_WRITE && td->flags & TD_F_DO_VERIFY) {
 			if (!(io_u->flags & IO_U_F_PATTERN_DONE)) {
 				io_u_set(td, io_u, IO_U_F_PATTERN_DONE);
-				io_u->numberio = td->io_issues[io_u->ddir];
+				if (td->shared_verify_table)
+					io_u->numberio = atomic_fetch_add(&td->shared_verify_table->shared_numberio, 1);
+				else
+					io_u->numberio = td->io_issues[io_u->ddir];
 				populate_verify_io_u(td, io_u);
 				log_inflight(td, io_u);
 			}
@@ -1172,11 +1195,18 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)
 		 * order of it. The logged unit will track when the IO has
 		 * completed.
 		 */
-		if (td_write(td) && io_u->ddir == DDIR_WRITE &&
+		if (((td_write(td) && io_u->ddir == DDIR_WRITE) || io_u->ddir == DDIR_TRIM) &&
 		    td->o.do_verify &&
 		    td->o.verify != VERIFY_NONE &&
-		    !td->o.experimental_verify)
-			log_io_piece(td, io_u);
+		    !td->o.experimental_verify) {
+			if (!log_io_piece(td, io_u)) {
+				dprint(FD_IO, "Overlap detected, skipping...\n");
+				invalidate_inflight(td, io_u);
+				clear_io_u(td, io_u);
+				ret = FIO_Q_BUSY;
+				goto reap;
+			}
+		}
 
 		if (td->o.io_submit_mode == IO_MODE_OFFLOAD) {
 			const unsigned long long blen = io_u->xfer_buflen;
@@ -1434,10 +1464,11 @@ int init_io_u_buffers(struct thread_data *td)
 
 	/*
 	 * For reads, writes, and multi-range trim operations we need a
-	 * data buffer
+	 * data buffer. Also need buffer if we're verifying trimmed data.
 	 */
 	if (td_ioengine_flagged(td, FIO_NOIO) ||
-	    !(td_read(td) || td_write(td) || (td_trim(td) && td->o.num_range > 1)))
+	    !(td_read(td) || td_write(td) || (td_trim(td) && td->o.num_range > 1) ||
+	      (td_trim(td) && td->o.do_verify && td->o.trim_zero)))
 		data_xfer = 0;
 
 	/*
@@ -1780,6 +1811,17 @@ static void *thread_main(void *data)
 		goto err;
 	}
 
+	if (o->verify_table_id > 0) {
+		td->shared_verify_table = get_shared_verify_table(o->verify_table_id);
+		if (!td->shared_verify_table) {
+			td_verror(td, ENOMEM, "failed to get shared verify table");
+			goto err;
+		}
+
+		if ((td_write(td) || td_trim(td)) && o->do_verify)
+			atomic_fetch_add(&td->shared_verify_table->write_jobs_active, 1);
+	}
+
 	td_set_runstate(td, TD_INITIALIZED);
 	dprint(FD_MUTEX, "up startup_sem\n");
 	fio_sem_up(startup_sem);
@@ -2056,6 +2098,9 @@ static void *thread_main(void *data)
 		fio_gettime(&td->start, NULL);
 		fio_sem_up(stat_sem);
 
+		if (td->shared_verify_table && (td_write(td) || td_trim(td)))
+			atomic_fetch_add(&td->shared_verify_table->write_jobs_done, 1);
+
 		if (td->error || td->terminate)
 			break;
 
@@ -2064,6 +2109,18 @@ static void *thread_main(void *data)
 		    td_ioengine_flagged(td, FIO_UNIDIR))
 			continue;
 
+		/*
+		 * If --verify_table_id= is given, only one job can win the
+		 * race among write jobs and verify the entire io pieces at
+		 * once.  Until the verify job is done, wait here.
+		 */
+		if (td->shared_verify_table) {
+			int expected = 0;
+
+			if (!atomic_compare_exchange_strong(&td->shared_verify_table->verify_done, &expected, 1))
+				continue;
+		}
+
 		clear_io_state(td, 0);
 
 		fio_gettime(&td->start, NULL);
@@ -2140,6 +2197,11 @@ static void *thread_main(void *data)
 	if (o->verify_async)
 		verify_async_exit(td);
 
+	if (td->shared_verify_table) {
+		put_shared_verify_table(td->shared_verify_table);
+		td->shared_verify_table = NULL;
+	}
+
 	close_and_free_files(td);
 	cleanup_io_u(td);
 	close_ioengine(td);

engines/io_uring.c

@@ -1609,6 +1609,16 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
 		io_u->engine_data = pi_data;
 	}
 
+	if (ld->is_uring_cmd_eng) {
+		if (ld->write_opcode == nvme_cmd_write_zeroes) {
+			if (o->deac)
+				io_u_set(td, io_u, IO_U_F_TRIMMED);
+			else
+				io_u_set(td, io_u, IO_U_F_ZEROED);
+		} else if (ld->write_opcode == nvme_cmd_write_uncor)
+			io_u_set(td, io_u, IO_U_F_ERRORED);
+	}
+
 	return 0;
 }
 

file.h

@@ -163,6 +163,8 @@ struct fio_file {
 	enum fio_file_flags flags;
 
 	struct disk_util *du;
+
+	uint64_t file_name_hash;
 };
 
 #define FILE_ENG_DATA(f)		((f)->engine_data)

filesetup.c

@@ -1794,6 +1794,7 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc)
 	char file_name[PATH_MAX];
 	struct fio_file *f;
 	int len = 0;
+	uint64_t hash = 0;
 
 	dprint(FD_FILE, "add file %s\n", fname);
 
@@ -1847,6 +1848,15 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc)
 	/* can't handle smalloc failure from here */
 	assert(f->file_name);
 
+	/*
+	 * Get hashed value from the file name.
+	 */
+	while (*fname) {
+		hash = (hash << 5) - hash + *fname;
+		fname++;
+	}
+	f->file_name_hash = hash;
+
 	if (td->o.filetype)
 		f->filetype = td->o.filetype;
 	else

fio.h

@@ -451,6 +451,7 @@ struct thread_data {
 	struct rb_root io_hist_tree;
 	struct flist_head io_hist_list;
 	unsigned long io_hist_len;
+	struct shared_verify_table *shared_verify_table;
 
 	/*
 	 * For IO replaying

io_u.h

@@ -24,6 +24,8 @@ enum {
 	IO_U_F_PATTERN_DONE	= 1 << 8,
 	IO_U_F_DEVICE_ERROR	= 1 << 9,
 	IO_U_F_VER_IN_DEV	= 1 << 10, /* Verify data in device */
+	IO_U_F_ZEROED		= 1 << 11, /* Zeroed data */
+	IO_U_F_ERRORED		= 1 << 12, /* Errored offset */
 };
 
 /*