add dax support

Signed-off-by: Lai Jiangshan <[email protected]>

Author: laijs

api/descriptions.pb.go

@@ -88,6 +88,7 @@ message VolumeOption {
     string keyring = 3;
     int32 bytesPerSec = 4;
     int32 iops = 5;
+    bool daxBlock = 6;
 }
 
 message UserGroupInfo {

api/descriptions.proto

@@ -3,7 +3,7 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then
+if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3."* ]]; then
   echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
   echo "install the platform appropriate Protobuf package for your OS: "
   echo

hack/update-generated-proto.sh

@@ -18,6 +18,7 @@ import (
 type VmHwStatus struct {
 	PciAddr  int    //next available pci addr for pci hotplug
 	ScsiId   int    //next available scsi id for scsi hotplug
+	PmemId   int    //next available pmem id for nvdimm hotplug
 	AttachId uint64 //next available attachId for attached tty
 	GuestCid uint32 //vsock guest cid
 }
@@ -48,6 +49,7 @@ type VmContext struct {
 
 	pciAddr int //next available pci addr for pci hotplug
 	scsiId  int //next available scsi id for scsi hotplug
+	pmemId  int //next available pmem id for nvdimm hotplug
 
 	//	InterfaceCount int
 
@@ -187,6 +189,14 @@ func (ctx *VmContext) unsetTimeout() {
 	}
 }
 
+func (ctx *VmContext) nextPmemId() int {
+	ctx.idLock.Lock()
+	id := ctx.pmemId
+	ctx.pmemId++
+	ctx.idLock.Unlock()
+	return id
+}
+
 func (ctx *VmContext) nextScsiId() int {
 	ctx.idLock.Lock()
 	id := ctx.scsiId

hypervisor/context.go

@@ -20,6 +20,8 @@ type DiskDescriptor struct {
 	DockerVolume bool
 	ReadOnly     bool
 	Options      map[string]string
+	Dax          bool
+	PmemId       int
 }
 
 func (d *DiskDescriptor) IsDir() bool {
@@ -67,6 +69,8 @@ func NewDiskContext(ctx *VmContext, vol *api.VolumeDescription) *DiskContext {
 			"bytespersec": strconv.Itoa(int(vol.Options.BytesPerSec)),
 			"iops":        strconv.Itoa(int(vol.Options.Iops)),
 		}
+	} else if vol.Options != nil && vol.Options.DaxBlock {
+		dc.DiskDescriptor.Dax = true
 	}
 	return dc
 }
@@ -80,7 +84,11 @@ func (dc *DiskContext) insert(result chan api.Result) {
 		return
 	}
 
-	dc.ScsiId = dc.sandbox.nextScsiId()
+	if dc.Dax {
+		dc.PmemId = dc.sandbox.nextPmemId()
+	} else {
+		dc.ScsiId = dc.sandbox.nextScsiId()
+	}
 	usage := "volume"
 	if dc.isRootVol {
 		usage = "image"

hypervisor/disk.go

@@ -442,7 +442,7 @@ func (lc *LibvirtContext) domainXml(ctx *hypervisor.VmContext) (string, error) {
 
 	dom.OS.Supported = "yes"
 	dom.OS.Type.Arch = "x86_64"
-	dom.OS.Type.Machine = "pc-i440fx-2.1"
+	dom.OS.Type.Machine = "pc-i440fx-2.1,nvdimm"
 	dom.OS.Type.Content = "hvm"
 
 	dom.SecLabel.Type = "none"
@@ -917,6 +917,33 @@ func (lc *LibvirtContext) AddDisk(ctx *hypervisor.VmContext, sourceType string,
 		return
 	}
 
+	if blockInfo.Dax {
+		// get the size
+		fi, e := os.Stat(blockInfo.Filename)
+		if e != nil {
+			result <- &hypervisor.DeviceFailed{}
+			return
+		}
+		size := fi.Size()
+		// compose hmp
+		hmp := fmt.Sprintf("object_add memory-backend-file,id=mem%d,share=on,mem-path=%s,size=%d", blockInfo.PmemId, blockInfo.Filename, size)
+		err := exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
+		if err != nil {
+			result <- &hypervisor.DeviceFailed{}
+			return
+		}
+		hmp = fmt.Sprintf("device_add nvdimm,id=nvdimm%d,memdev=mem%d", blockInfo.PmemId, blockInfo.PmemId)
+		err = exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
+		if err != nil {
+			result <- &hypervisor.DeviceFailed{}
+			return
+		}
+		result <- &hypervisor.BlockdevInsertedEvent{
+			DeviceName: "pmem" + strconv.Itoa(blockInfo.PmemId),
+		}
+		return
+	}
+
 	secretUUID, err := lc.diskSecretUUID(blockInfo)
 	if err != nil {
 		glog.Error("generate disk-get-secret failed, ", err.Error())

hypervisor/libvirt/libvirt.go

@@ -136,6 +136,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
 	return &VmHwStatus{
 		PciAddr:  ctx.pciAddr,
 		ScsiId:   ctx.scsiId,
+		PmemId:   ctx.pmemId,
 		AttachId: ctx.hyperstart.LastStreamSeq(),
 		GuestCid: ctx.GuestCid,
 	}
@@ -144,6 +145,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
 func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error {
 	ctx.pciAddr = pinfo.HwStat.PciAddr
 	ctx.scsiId = pinfo.HwStat.ScsiId
+	ctx.pmemId = pinfo.HwStat.PmemId
 	ctx.GuestCid = pinfo.HwStat.GuestCid
 	if ctx.GuestCid != 0 {
 		if !VsockCidManager.MarkCidInuse(ctx.GuestCid) {

hypervisor/persistence.go

@@ -248,7 +248,63 @@ func (qc *QemuContext) AddDisk(ctx *hypervisor.VmContext, sourceType string, blo
 		}
 	}
 
-	newDiskAddSession(ctx, qc, filename, format, id, readonly, result)
+	commands := make([]*QmpCommand, 2)
+	devName := scsiId2Name(id)
+	if !blockInfo.Dax {
+		hmp := "drive_add dummy file=" + filename + ",if=none,id=" + "drive" + strconv.Itoa(id) + ",format=" + format + ",cache=writeback"
+		if readonly {
+			hmp += ",readonly"
+		}
+		commands[0] = &QmpCommand{
+			Execute: "human-monitor-command",
+			Arguments: map[string]interface{}{
+				"command-line": hmp,
+			},
+		}
+		commands[1] = &QmpCommand{
+			Execute: "device_add",
+			Arguments: map[string]interface{}{
+				"driver": "scsi-hd", "bus": "scsi0.0", "scsi-id": strconv.Itoa(id),
+				"drive": "drive" + strconv.Itoa(id), "id": "scsi-disk" + strconv.Itoa(id),
+			},
+		}
+	} else {
+		// compose qmp commands
+		// hmp: object_add memory-backend-file,id=mem2,share=on,mem-path=/path/to/dax.img,size=10G
+		// hmp: device_add nvdimm,id=nvdimm2,memdev=mem2
+		hmp := "object_add memory-backend-file,id=mem" + strconv.Itoa(blockInfo.PmemId) + ",mem-path=" + filename
+		if readonly {
+			hmp += ",share=off"
+		} else {
+			hmp += ",share=on"
+		}
+		// get the size
+		fi, e := os.Stat(filename)
+		if e != nil {
+			result <- &hypervisor.DeviceFailed{}
+			return
+		}
+		hmp += ",size=" + strconv.FormatInt(fi.Size(), 10)
+		commands[0] = &QmpCommand{
+			Execute: "human-monitor-command",
+			Arguments: map[string]interface{}{
+				"command-line": hmp,
+			},
+		}
+		commands[1] = &QmpCommand{
+			Execute: "device_add",
+			Arguments: map[string]interface{}{
+				"driver": "nvdimm", "memdev": "mem" + strconv.Itoa(blockInfo.PmemId), "id": "nvdimm" + strconv.Itoa(blockInfo.PmemId),
+			},
+		}
+		devName = "pmem" + strconv.Itoa(blockInfo.PmemId)
+	}
+	qc.qmp <- &QmpSession{
+		commands: commands,
+		respond: defaultRespond(result, &hypervisor.BlockdevInsertedEvent{
+			DeviceName: devName,
+		}),
+	}
 }
 
 func (qc *QemuContext) RemoveDisk(ctx *hypervisor.VmContext, blockInfo *hypervisor.DiskDescriptor, callback hypervisor.VmEvent, result chan<- hypervisor.VmEvent) {

hypervisor/qemu/qemu.go

@@ -27,7 +27,7 @@ func (qc *QemuContext) arguments(ctx *hypervisor.VmContext) []string {
 	qc.cpus = boot.CPU
 
 	var machineClass, memParams, cpuParams string
-	machineClass = "pc-i440fx-2.1"
+	machineClass = "pc-i440fx-2.1,nvdimm"
 	memParams = fmt.Sprintf("size=%d,slots=1,maxmem=%dM", boot.Memory, hypervisor.DefaultMaxMem) // TODO set maxmem to the total memory of the system
 	cpuParams = fmt.Sprintf("cpus=%d,maxcpus=%d", boot.CPU, hypervisor.DefaultMaxCpus)           // TODO set it to the cpus of the system
 

hypervisor/qemu/qemu_amd64.go

@@ -33,34 +33,6 @@ func defaultRespond(result chan<- hypervisor.VmEvent, callback hypervisor.VmEven
 	}
 }
 
-func newDiskAddSession(ctx *hypervisor.VmContext, qc *QemuContext, filename, format string, id int, readonly bool, result chan<- hypervisor.VmEvent) {
-	args := "drive_add dummy file=" + filename + ",if=none,id=" + "drive" + strconv.Itoa(id) + ",format=" + format + ",cache=writeback"
-	if readonly {
-		args += ",readonly"
-	}
-	commands := make([]*QmpCommand, 2)
-	commands[0] = &QmpCommand{
-		Execute: "human-monitor-command",
-		Arguments: map[string]interface{}{
-			"command-line": args,
-		},
-	}
-	commands[1] = &QmpCommand{
-		Execute: "device_add",
-		Arguments: map[string]interface{}{
-			"driver": "scsi-hd", "bus": "scsi0.0", "scsi-id": strconv.Itoa(id),
-			"drive": "drive" + strconv.Itoa(id), "id": "scsi-disk" + strconv.Itoa(id),
-		},
-	}
-	devName := scsiId2Name(id)
-	qc.qmp <- &QmpSession{
-		commands: commands,
-		respond: defaultRespond(result, &hypervisor.BlockdevInsertedEvent{
-			DeviceName: devName,
-		}),
-	}
-}
-
 func newDiskDelSession(ctx *hypervisor.VmContext, qc *QemuContext, id int, callback hypervisor.VmEvent, result chan<- hypervisor.VmEvent) {
 	commands := make([]*QmpCommand, 2)
 	commands[1] = &QmpCommand{