Skip to content
This repository was archived by the owner on Feb 8, 2021. It is now read-only.

Commit aa0162a

Browse files
laijslaijs
committed
add dax support
Signed-off-by: Lai Jiangshan <[email protected]>
1 parent e286c66 commit aa0162a

10 files changed

+171
-94
lines changed

‎api/descriptions.pb.go

Lines changed: 70 additions & 64 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/descriptions.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ message VolumeOption {
8787
string keyring = 3;
8888
int32 bytesPerSec = 4;
8989
int32 iops = 5;
90+
bool daxBlock = 6;
9091
}
9192

9293
message UserGroupInfo {

‎hack/update-generated-proto.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ set -o errexit
33
set -o nounset
44
set -o pipefail
55

6-
if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then
6+
if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3."* ]]; then
77
echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
88
echo "install the platform appropriate Protobuf package for your OS: "
99
echo

‎hypervisor/context.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
type VmHwStatus struct {
1919
PciAddr int //next available pci addr for pci hotplug
2020
ScsiId int //next available scsi id for scsi hotplug
21+
PmemId int //next available pmem id for nvdimm hotplug
2122
AttachId uint64 //next available attachId for attached tty
2223
GuestCid uint32 //vsock guest cid
2324
}
@@ -50,6 +51,7 @@ type VmContext struct {
5051

5152
pciAddr int //next available pci addr for pci hotplug
5253
scsiId int //next available scsi id for scsi hotplug
54+
pmemId int //next available pmem id for nvdimm hotplug
5355

5456
// InterfaceCount int
5557

@@ -185,6 +187,14 @@ func (ctx *VmContext) unsetTimeout() {
185187
}
186188
}
187189

190+
func (ctx *VmContext) nextPmemId() int {
191+
ctx.idLock.Lock()
192+
id := ctx.pmemId
193+
ctx.pmemId++
194+
ctx.idLock.Unlock()
195+
return id
196+
}
197+
188198
func (ctx *VmContext) nextScsiId() int {
189199
ctx.idLock.Lock()
190200
id := ctx.scsiId

‎hypervisor/disk.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ type DiskDescriptor struct {
1919
ScsiAddr string
2020
DockerVolume bool
2121
Options map[string]string
22+
Dax bool
23+
PmemId int
2224
}
2325

2426
func (d *DiskDescriptor) IsDir() bool {
@@ -65,6 +67,8 @@ func NewDiskContext(ctx *VmContext, vol *api.VolumeDescription) *DiskContext {
6567
"bytespersec": strconv.Itoa(int(vol.Options.BytesPerSec)),
6668
"iops": strconv.Itoa(int(vol.Options.Iops)),
6769
}
70+
} else if vol.Options != nil && vol.Options.DaxBlock {
71+
dc.DiskDescriptor.Dax = true
6872
}
6973
return dc
7074
}
@@ -78,7 +82,11 @@ func (dc *DiskContext) insert(result chan api.Result) {
7882
return
7983
}
8084

81-
dc.ScsiId = dc.sandbox.nextScsiId()
85+
if dc.Dax {
86+
dc.PmemId = dc.sandbox.nextPmemId()
87+
} else {
88+
dc.ScsiId = dc.sandbox.nextScsiId()
89+
}
8290
usage := "volume"
8391
if dc.isRootVol {
8492
usage = "image"

‎hypervisor/libvirt/libvirt.go

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ func (lc *LibvirtContext) domainXml(ctx *hypervisor.VmContext) (string, error) {
442442

443443
dom.OS.Supported = "yes"
444444
dom.OS.Type.Arch = "x86_64"
445-
dom.OS.Type.Machine = "pc-i440fx-2.1"
445+
dom.OS.Type.Machine = "pc-i440fx-2.1,nvdimm"
446446
dom.OS.Type.Content = "hvm"
447447

448448
dom.SecLabel.Type = "none"
@@ -912,6 +912,33 @@ func (lc *LibvirtContext) AddDisk(ctx *hypervisor.VmContext, sourceType string,
912912
return
913913
}
914914

915+
if blockInfo.Dax {
916+
// get the size
917+
fi, e := os.Stat(blockInfo.Filename)
918+
if e != nil {
919+
result <- &hypervisor.DeviceFailed{}
920+
return
921+
}
922+
size := fi.Size()
923+
// compose hmp
924+
hmp := fmt.Sprintf("object_add memory-backend-file,id=mem%d,share=on,mem-path=%s,size=%d", blockInfo.PmemId, blockInfo.Filename, size)
925+
err := exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
926+
if err != nil {
927+
result <- &hypervisor.DeviceFailed{}
928+
return
929+
}
930+
hmp = fmt.Sprintf("device_add nvdimm,id=nvdimm%d,memdev=mem%d", blockInfo.PmemId, blockInfo.PmemId)
931+
err = exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
932+
if err != nil {
933+
result <- &hypervisor.DeviceFailed{}
934+
return
935+
}
936+
result <- &hypervisor.BlockdevInsertedEvent{
937+
DeviceName: "pmem" + strconv.Itoa(blockInfo.PmemId),
938+
}
939+
return
940+
}
941+
915942
secretUUID, err := lc.diskSecretUUID(blockInfo)
916943
if err != nil {
917944
glog.Error("generate disk-get-secret failed, ", err.Error())

‎hypervisor/persistence.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
136136
return &VmHwStatus{
137137
PciAddr: ctx.pciAddr,
138138
ScsiId: ctx.scsiId,
139+
PmemId: ctx.pmemId,
139140
AttachId: ctx.hyperstart.LastStreamSeq(),
140141
GuestCid: ctx.GuestCid,
141142
}
@@ -144,6 +145,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
144145
func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error {
145146
ctx.pciAddr = pinfo.HwStat.PciAddr
146147
ctx.scsiId = pinfo.HwStat.ScsiId
148+
ctx.pmemId = pinfo.HwStat.PmemId
147149
ctx.GuestCid = pinfo.HwStat.GuestCid
148150
if ctx.GuestCid != 0 {
149151
if !VsockCidManager.MarkCidInuse(ctx.GuestCid) {

‎hypervisor/qemu/qemu.go

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,55 @@ func (qc *QemuContext) AddDisk(ctx *hypervisor.VmContext, sourceType string, blo
247247
}
248248
}
249249

250-
newDiskAddSession(ctx, qc, filename, format, id, result)
250+
commands := make([]*QmpCommand, 2)
251+
devName := scsiId2Name(id)
252+
if !blockInfo.Dax {
253+
commands[0] = &QmpCommand{
254+
Execute: "human-monitor-command",
255+
Arguments: map[string]interface{}{
256+
"command-line": "drive_add dummy file=" +
257+
filename + ",if=none,id=" + "drive" + strconv.Itoa(id) + ",format=" + format + ",cache=writeback",
258+
},
259+
}
260+
commands[1] = &QmpCommand{
261+
Execute: "device_add",
262+
Arguments: map[string]interface{}{
263+
"driver": "scsi-hd", "bus": "scsi0.0", "scsi-id": strconv.Itoa(id),
264+
"drive": "drive" + strconv.Itoa(id), "id": "scsi-disk" + strconv.Itoa(id),
265+
},
266+
}
267+
} else {
268+
// get the size
269+
fi, e := os.Stat(filename)
270+
if e != nil {
271+
result <- &hypervisor.DeviceFailed{}
272+
return
273+
}
274+
size := fi.Size()
275+
// compose qmp commands
276+
// hmp: object_add memory-backend-file,id=mem2,share=off,mem-path=/path/to/dax.img,size=10G
277+
// hmp: device_add nvdimm,id=nvdimm2,memdev=mem2
278+
commands[0] = &QmpCommand{
279+
Execute: "human-monitor-command",
280+
Arguments: map[string]interface{}{
281+
"command-line": "object_add memory-backend-file,id=mem" + strconv.Itoa(blockInfo.PmemId) +
282+
",share=on,mem-path=" + filename + ",size=" + strconv.FormatInt(size, 10),
283+
},
284+
}
285+
commands[1] = &QmpCommand{
286+
Execute: "device_add",
287+
Arguments: map[string]interface{}{
288+
"driver": "nvdimm", "memdev": "mem" + strconv.Itoa(blockInfo.PmemId), "id": "nvdimm" + strconv.Itoa(blockInfo.PmemId),
289+
},
290+
}
291+
devName = "pmem" + strconv.Itoa(blockInfo.PmemId)
292+
}
293+
qc.qmp <- &QmpSession{
294+
commands: commands,
295+
respond: defaultRespond(result, &hypervisor.BlockdevInsertedEvent{
296+
DeviceName: devName,
297+
}),
298+
}
251299
}
252300

253301
func (qc *QemuContext) RemoveDisk(ctx *hypervisor.VmContext, blockInfo *hypervisor.DiskDescriptor, callback hypervisor.VmEvent, result chan<- hypervisor.VmEvent) {

0 commit comments

Comments
 (0)