dperf support --sync flag for synchronous I/O

Author: harshavardhana

cmd/cmd.go

@@ -45,6 +45,7 @@ var (
 	serial     = false
 	writeOnly  = false
 	verbose    = false
+	syncMode   = false
 	blockSize  = "4MiB"
 	fileSize   = "1GiB"
 	cpuNode    = 0
@@ -61,32 +62,44 @@ var dperfCmd = &cobra.Command{
 MinIO drive performance utility
 --------------------------------
   dperf measures throughput of each of the drives mounted at PATH...
+
+  By default, dperf uses O_DIRECT for block sizes >= 4KiB to bypass the page cache.
+  For block sizes < 4KiB, it automatically switches to O_DSYNC/O_SYNC mode.
+  You can explicitly enable sync mode for any block size using the --sync flag.
 `,
 	SilenceUsage:  true,
 	SilenceErrors: true,
 	Args:          cobra.MinimumNArgs(1),
 	Version:       Version,
 	Example: `
-# run dpref on drive mounted at /mnt/drive1
+# run dperf on drive mounted at /mnt/drive1
 $ dperf /mnt/drive1
 
 # run dperf on drives 1 to 6. Output will be sorted by throughput. Fastest drive is at the top.
 $ dperf /mnt/drive{1..6}
 
 # run dperf on drives one-by-one
 $ dperf --serial /mnt/drive{1..6}
+
+# run dperf with 1KiB block size (automatically uses O_DSYNC)
+$ dperf -b 1KiB /mnt/drive1
+
+# run dperf with 4KiB block size using O_DSYNC instead of O_DIRECT
+$ dperf --sync -b 4KiB /mnt/drive1
 `,
 	RunE: func(c *cobra.Command, args []string) error {
 		bs, err := humanize.ParseBytes(blockSize)
 		if err != nil {
 			return fmt.Errorf("Invalid blocksize format: %v", err)
 		}
 
-		if bs < alignSize {
-			return fmt.Errorf("Invalid blocksize must greater than 4k: %d", bs)
+		if bs == 0 {
+			return fmt.Errorf("Invalid blocksize must be greater than 0: %d", bs)
 		}
 
-		if bs%alignSize != 0 {
+		// For block sizes < 4KiB, we'll use O_DSYNC instead of O_DIRECT
+		// For block sizes >= 4KiB, we require alignment for O_DIRECT
+		if bs >= alignSize && bs%alignSize != 0 {
 			return fmt.Errorf("Invalid blocksize must be multiples of 4k: %d", bs)
 		}
 
@@ -95,25 +108,31 @@ $ dperf --serial /mnt/drive{1..6}
 			return fmt.Errorf("Invalid filesize format: %v", err)
 		}
 
-		if fs < alignSize {
-			return fmt.Errorf("Invalid filesize must greater than 4k: %d", fs)
+		if fs == 0 {
+			return fmt.Errorf("Invalid filesize must be greater than 0: %d", fs)
 		}
 
-		if fs%alignSize != 0 {
+		// For file sizes with small block sizes, we relax the alignment requirement
+		// For file sizes with block sizes >= 4KiB, we require alignment for O_DIRECT
+		if bs >= alignSize && fs%alignSize != 0 {
 			return fmt.Errorf("Invalid filesize must multiples of 4k: %d", fs)
 		}
 
 		if ioPerDrive <= 0 {
 			return fmt.Errorf("Invalid ioperdrive must greater than 0: %d", ioPerDrive)
 		}
 
+		// Use sync mode if explicitly requested or if block size < 4KiB
+		useSyncMode := syncMode || bs < alignSize
+
 		perf := &dperf.DrivePerf{
 			Serial:     serial,
 			BlockSize:  bs,
 			FileSize:   fs,
 			Verbose:    verbose,
 			IOPerDrive: ioPerDrive,
 			WriteOnly:  writeOnly,
+			SyncMode:   useSyncMode,
 		}
 		paths := make([]string, 0, len(args))
 		for _, arg := range args {
@@ -215,6 +234,8 @@ func init() {
 		"write-only", "", writeOnly, "run write only tests")
 	dperfCmd.PersistentFlags().BoolVarP(&verbose,
 		"verbose", "v", verbose, "print READ/WRITE for each paths independently, default only prints aggregated")
+	dperfCmd.PersistentFlags().BoolVarP(&syncMode,
+		"sync", "", syncMode, "use O_DSYNC for writes and O_SYNC for reads instead of O_DIRECT (automatically enabled for block sizes < 4KiB)")
 	dperfCmd.PersistentFlags().StringVarP(&blockSize,
 		"blocksize", "b", blockSize, "read/write block size")
 	dperfCmd.PersistentFlags().StringVarP(&fileSize,

pkg/dperf/perf.go

@@ -35,6 +35,7 @@ type DrivePerf struct {
 	FileSize   uint64
 	IOPerDrive int
 	WriteOnly  bool
+	SyncMode   bool // Use O_DSYNC/O_SYNC instead of O_DIRECT
 }
 
 // mustGetUUID - get a random UUID.
@@ -54,8 +55,13 @@ func (d *DrivePerf) runTests(ctx context.Context, path string, testUUID string)
 
 	dataBuffers := make([][]byte, d.IOPerDrive)
 	for i := 0; i < d.IOPerDrive; i++ {
-		// Read Aligned block upto a multiple of BlockSize
-		dataBuffers[i] = alignedBlock(int(d.BlockSize))
+		// In sync mode or with small block sizes, use regular buffer allocation
+		// Otherwise, use aligned blocks for O_DIRECT
+		if d.SyncMode {
+			dataBuffers[i] = make([]byte, d.BlockSize)
+		} else {
+			dataBuffers[i] = alignedBlock(int(d.BlockSize))
+		}
 	}
 
 	testUUIDPath := filepath.Join(path, testUUID)

pkg/dperf/run_linux.go

@@ -39,13 +39,24 @@ func (n nullWriter) Write(b []byte) (int, error) {
 
 func (d *DrivePerf) runReadTest(ctx context.Context, path string, data []byte) (uint64, error) {
 	startTime := time.Now()
-	r, err := os.OpenFile(path, syscall.O_DIRECT|os.O_RDONLY, 0o400)
+
+	// Choose flags based on sync mode
+	var flags int
+	if d.SyncMode {
+		// Use O_SYNC for synchronized reads (for small block sizes or when --sync is specified)
+		flags = syscall.O_SYNC | os.O_RDONLY
+	} else {
+		// Use O_DIRECT for direct I/O (bypasses page cache)
+		flags = syscall.O_DIRECT | os.O_RDONLY
+	}
+
+	r, err := os.OpenFile(path, flags, 0o400)
 	if err != nil {
 		return 0, err
 	}
 	unix.Fadvise(int(r.Fd()), 0, int64(d.FileSize), unix.FADV_SEQUENTIAL)
 
-	n, err := copyAligned(&nullWriter{}, r, data, int64(d.FileSize), r.Fd())
+	n, err := copyAligned(&nullWriter{}, r, data, int64(d.FileSize), r.Fd(), d.SyncMode)
 	r.Close()
 	if err != nil {
 		return 0, err
@@ -125,7 +136,10 @@ const DirectioAlignSize = 4096
 // used with DIRECT I/O based file descriptor and it is expected that
 // input writer *os.File not a generic io.Writer. Make sure to have
 // the file opened for writes with syscall.O_DIRECT flag.
-func copyAligned(w io.Writer, r io.Reader, alignedBuf []byte, totalSize int64, fd uintptr) (int64, error) {
+//
+// When syncMode is true, alignment checks are skipped as O_DSYNC/O_SYNC
+// is used instead of O_DIRECT.
+func copyAligned(w io.Writer, r io.Reader, alignedBuf []byte, totalSize int64, fd uintptr, syncMode bool) (int64, error) {
 	if totalSize == 0 {
 		return 0, nil
 	}
@@ -140,7 +154,8 @@ func copyAligned(w io.Writer, r io.Reader, alignedBuf []byte, totalSize int64, f
 			}
 		}
 
-		if len(buf)%DirectioAlignSize != 0 {
+		// In sync mode, we don't need to worry about alignment since we're not using O_DIRECT
+		if !syncMode && len(buf)%DirectioAlignSize != 0 {
 			// Disable O_DIRECT on fd's on unaligned buffer
 			// perform an amortized Fdatasync(fd) on the fd at
 			// the end, this is performed by the caller before
@@ -164,8 +179,9 @@ func copyAligned(w io.Writer, r io.Reader, alignedBuf []byte, totalSize int64, f
 		)
 
 		remain := len(buf) % DirectioAlignSize
-		if remain == 0 {
-			// buf is aligned for directio write()
+		// In sync mode, treat all buffers as "aligned" (no special handling needed)
+		if syncMode || remain == 0 {
+			// buf is aligned for directio write() or we're in sync mode
 			n, err = w.Write(buf)
 			nw = int64(n)
 		} else {
@@ -225,12 +241,23 @@ func (d *DrivePerf) runWriteTest(ctx context.Context, path string, data []byte)
 	}
 
 	startTime := time.Now()
-	w, err := os.OpenFile(path, syscall.O_DIRECT|os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o600)
+
+	// Choose flags based on sync mode
+	var flags int
+	if d.SyncMode {
+		// Use O_DSYNC for synchronized writes (for small block sizes or when --sync is specified)
+		flags = syscall.O_DSYNC | os.O_RDWR | os.O_CREATE | os.O_TRUNC
+	} else {
+		// Use O_DIRECT for direct I/O (bypasses page cache)
+		flags = syscall.O_DIRECT | os.O_RDWR | os.O_CREATE | os.O_TRUNC
+	}
+
+	w, err := os.OpenFile(path, flags, 0o600)
 	if err != nil {
 		return 0, err
 	}
 
-	n, err := copyAligned(w, newRandomReader(ctx), data, int64(d.FileSize), w.Fd())
+	n, err := copyAligned(w, newRandomReader(ctx), data, int64(d.FileSize), w.Fd(), d.SyncMode)
 	if err != nil {
 		w.Close()
 		return 0, err