Feature(format): support fomat with blocksize and chunksize.

Signed-off-by: Hanqing Wu <[email protected]>

Author: wu-hanqing

internal/configure/format.go

@@ -32,6 +32,12 @@ import (
 
 const (
 	DEFAULT_CONTAINER_IMAGE = "opencurvedocker/curvebs:v1.2"
+	DEFAULT_BLOCK_SIZE      = 4096
+	DEFAULT_CHUNK_SIZE      = 16 * 1024 * 1024
+)
+
+var (
+	VALID_BLOCK_SIZE = [2]int{512, 4096}
 )
 
 /*
@@ -51,12 +57,16 @@ type (
 		Device         string
 		MountPoint     string
 		FormtPercent   int
+		BlockSize      int
+		ChunkSize      int
 	}
 
 	Format struct {
 		ContainerImage string   `mapstructure:"container_image"`
 		Hosts          []string `mapstructure:"host"`
 		Disks          []string `mapstructure:"disk"`
+		BlockSize      int      `mapstructure:"block_size"`
+		ChunkSize      int      `mapstructure:"chunk_size"`
 	}
 )
 
@@ -93,6 +103,16 @@ func newFormatConfig(containerImage, host, disk string) (*FormatConfig, error) {
 	}, nil
 }
 
+func isValidBlockSize(blocksize int) bool {
+	for _, bs := range VALID_BLOCK_SIZE {
+		if bs == blocksize {
+			return true
+		}
+	}
+
+	return false
+}
+
 func ParseFormat(filename string) ([]*FormatConfig, error) {
 	if !utils.PathExist(filename) {
 		return nil, errno.ERR_FORMAT_CONFIGURE_FILE_NOT_EXIST.
@@ -107,7 +127,10 @@ func ParseFormat(filename string) ([]*FormatConfig, error) {
 		return nil, errno.ERR_PARSE_FORMAT_CONFIGURE_FAILED.E(err)
 	}
 
-	format := &Format{}
+	format := &Format{
+		BlockSize: DEFAULT_BLOCK_SIZE,
+		ChunkSize: DEFAULT_CHUNK_SIZE,
+	}
 	err = parser.Unmarshal(format)
 	if err != nil {
 		return nil, errno.ERR_PARSE_FORMAT_CONFIGURE_FAILED.E(err)
@@ -118,13 +141,19 @@ func ParseFormat(filename string) ([]*FormatConfig, error) {
 		containerImage = format.ContainerImage
 	}
 
+	if !isValidBlockSize(format.BlockSize) {
+		return nil, errno.ERR_INVALID_BLOCK_SIZE.F("block_size: %d", format.BlockSize)
+	}
+
 	fcs := []*FormatConfig{}
 	for _, host := range format.Hosts {
 		for _, disk := range format.Disks {
 			fc, err := newFormatConfig(containerImage, host, disk)
 			if err != nil {
 				return nil, err
 			}
+			fc.BlockSize = format.BlockSize
+			fc.ChunkSize = format.ChunkSize
 			fcs = append(fcs, fc)
 		}
 	}
@@ -137,3 +166,5 @@ func (fc *FormatConfig) GetHost() string           { return fc.Host }
 func (fc *FormatConfig) GetDevice() string         { return fc.Device }
 func (fc *FormatConfig) GetMountPoint() string     { return fc.MountPoint }
 func (fc *FormatConfig) GetFormatPercent() int     { return fc.FormtPercent }
+func (fc *FormatConfig) GetBlockSize() int         { return fc.BlockSize }
+func (fc *FormatConfig) GetChunkSize() int         { return fc.ChunkSize }

internal/errno/errno.go

@@ -333,6 +333,7 @@ var (
 	ERR_MOUNT_POINT_REQUIRE_ABSOLUTE_PATH        = EC(341002, "mount point must be an absolute path")
 	ERR_FORMAT_PERCENT_REQUIRES_INTERGET         = EC(341003, "format percentage requires an integer")
 	ERR_FORMAT_PERCENT_MUST_BE_BETWEEN_1_AND_100 = EC(341004, "format percentage must be between 1 and 100")
+	ERR_INVALID_BLOCK_SIZE                       = EC(341005, "invalid block size, support 512,4096")
 
 	// 350: configure (client.yaml: parse failed)
 	ERR_PARSE_CLIENT_CONFIGURE_FAILED  = EC(350000, "parse client configure failed")

internal/task/scripts/shell/format.sh

@@ -9,11 +9,14 @@ percent=$2
 chunkfile_size=$3
 chunkfile_pool_dir=$4
 chunkfile_pool_meta_path=$5
+chunkfile_block_size=$6
 
 mkdir -p $chunkfile_pool_dir
 $binary \
   -allocatePercent=$percent \
   -fileSize=$chunkfile_size \
   -filePoolDir=$chunkfile_pool_dir \
   -filePoolMetaPath=$chunkfile_pool_meta_path \
-  -fileSystemPath=$chunkfile_pool_dir
+  -fileSystemPath=$chunkfile_pool_dir \
+  -blockSize=$chunkfile_block_size \
+  -undefok blockSize

internal/task/task/bs/format.go

@@ -178,6 +178,8 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf
 	device := fc.GetDevice()
 	mountPoint := fc.GetMountPoint()
 	usagePercent := fc.GetFormatPercent()
+	blockSize := fc.GetBlockSize()
+	chunkSize := fc.GetChunkSize()
 	subname := fmt.Sprintf("host=%s device=%s mountPoint=%s usage=%d%%",
 		fc.GetHost(), device, mountPoint, usagePercent)
 	t := task.NewTask("Start Format Chunkfile Pool", subname, hc.GetSSHConfig())
@@ -190,8 +192,8 @@ func NewFormatChunkfilePoolTask(curveadm *cli.CurveAdm, fc *configure.FormatConf
 	chunkfilePoolRootDir := layout.ChunkfilePoolRootDir
 	formatScript := scripts.FORMAT
 	formatScriptPath := fmt.Sprintf("%s/format.sh", layout.ToolsBinDir)
-	formatCommand := fmt.Sprintf("%s %s %d %d %s %s", formatScriptPath, layout.FormatBinaryPath,
-		usagePercent, DEFAULT_CHUNKFILE_SIZE, layout.ChunkfilePoolDir, layout.ChunkfilePoolMetaPath)
+	formatCommand := fmt.Sprintf("%s %s %d %d %s %s %d", formatScriptPath, layout.FormatBinaryPath,
+		usagePercent, chunkSize, layout.ChunkfilePoolDir, layout.ChunkfilePoolMetaPath, blockSize)
 
 	// 1: skip if formating container exist
 	t.AddStep(&step.ListContainers{