Skip to content
This repository was archived by the owner on Feb 8, 2021. It is now read-only.

Commit bff1b98

Browse files
laijslaijs
committed
add dax support
Signed-off-by: Lai Jiangshan <[email protected]>
1 parent c8033af commit bff1b98

10 files changed

+179
-500
lines changed

‎api/descriptions.pb.go

Lines changed: 70 additions & 467 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/descriptions.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ message VolumeOption {
8888
string keyring = 3;
8989
int32 bytesPerSec = 4;
9090
int32 iops = 5;
91+
bool daxBlock = 6;
9192
}
9293

9394
message UserGroupInfo {

‎hack/update-generated-proto.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ set -o errexit
33
set -o nounset
44
set -o pipefail
55

6-
if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then
6+
if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3."* ]]; then
77
echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
88
echo "install the platform appropriate Protobuf package for your OS: "
99
echo

‎hypervisor/context.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
type VmHwStatus struct {
1919
PciAddr int //next available pci addr for pci hotplug
2020
ScsiId int //next available scsi id for scsi hotplug
21+
PmemId int //next available pmem id for nvdimm hotplug
2122
AttachId uint64 //next available attachId for attached tty
2223
GuestCid uint32 //vsock guest cid
2324
}
@@ -48,6 +49,7 @@ type VmContext struct {
4849

4950
pciAddr int //next available pci addr for pci hotplug
5051
scsiId int //next available scsi id for scsi hotplug
52+
pmemId int //next available pmem id for nvdimm hotplug
5153

5254
// InterfaceCount int
5355

@@ -187,6 +189,14 @@ func (ctx *VmContext) unsetTimeout() {
187189
}
188190
}
189191

192+
func (ctx *VmContext) nextPmemId() int {
193+
ctx.idLock.Lock()
194+
id := ctx.pmemId
195+
ctx.pmemId++
196+
ctx.idLock.Unlock()
197+
return id
198+
}
199+
190200
func (ctx *VmContext) nextScsiId() int {
191201
ctx.idLock.Lock()
192202
id := ctx.scsiId

‎hypervisor/disk.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ type DiskDescriptor struct {
2020
DockerVolume bool
2121
ReadOnly bool
2222
Options map[string]string
23+
Dax bool
24+
PmemId int
2325
}
2426

2527
func (d *DiskDescriptor) IsDir() bool {
@@ -67,6 +69,8 @@ func NewDiskContext(ctx *VmContext, vol *api.VolumeDescription) *DiskContext {
6769
"bytespersec": strconv.Itoa(int(vol.Options.BytesPerSec)),
6870
"iops": strconv.Itoa(int(vol.Options.Iops)),
6971
}
72+
} else if vol.Options != nil && vol.Options.DaxBlock {
73+
dc.DiskDescriptor.Dax = true
7074
}
7175
return dc
7276
}
@@ -80,7 +84,11 @@ func (dc *DiskContext) insert(result chan api.Result) {
8084
return
8185
}
8286

83-
dc.ScsiId = dc.sandbox.nextScsiId()
87+
if dc.Dax {
88+
dc.PmemId = dc.sandbox.nextPmemId()
89+
} else {
90+
dc.ScsiId = dc.sandbox.nextScsiId()
91+
}
8492
usage := "volume"
8593
if dc.isRootVol {
8694
usage = "image"

‎hypervisor/libvirt/libvirt.go

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ func (lc *LibvirtContext) domainXml(ctx *hypervisor.VmContext) (string, error) {
442442

443443
dom.OS.Supported = "yes"
444444
dom.OS.Type.Arch = "x86_64"
445-
dom.OS.Type.Machine = "pc-i440fx-2.1"
445+
dom.OS.Type.Machine = "pc-i440fx-2.1,nvdimm"
446446
dom.OS.Type.Content = "hvm"
447447

448448
dom.SecLabel.Type = "none"
@@ -917,6 +917,33 @@ func (lc *LibvirtContext) AddDisk(ctx *hypervisor.VmContext, sourceType string,
917917
return
918918
}
919919

920+
if blockInfo.Dax {
921+
// get the size
922+
fi, e := os.Stat(blockInfo.Filename)
923+
if e != nil {
924+
result <- &hypervisor.DeviceFailed{}
925+
return
926+
}
927+
size := fi.Size()
928+
// compose hmp
929+
hmp := fmt.Sprintf("object_add memory-backend-file,id=mem%d,share=on,mem-path=%s,size=%d", blockInfo.PmemId, blockInfo.Filename, size)
930+
err := exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
931+
if err != nil {
932+
result <- &hypervisor.DeviceFailed{}
933+
return
934+
}
935+
hmp = fmt.Sprintf("device_add nvdimm,id=nvdimm%d,memdev=mem%d", blockInfo.PmemId, blockInfo.PmemId)
936+
err = exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
937+
if err != nil {
938+
result <- &hypervisor.DeviceFailed{}
939+
return
940+
}
941+
result <- &hypervisor.BlockdevInsertedEvent{
942+
DeviceName: "pmem" + strconv.Itoa(blockInfo.PmemId),
943+
}
944+
return
945+
}
946+
920947
secretUUID, err := lc.diskSecretUUID(blockInfo)
921948
if err != nil {
922949
glog.Error("generate disk-get-secret failed, ", err.Error())

‎hypervisor/persistence.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
136136
return &VmHwStatus{
137137
PciAddr: ctx.pciAddr,
138138
ScsiId: ctx.scsiId,
139+
PmemId: ctx.pmemId,
139140
AttachId: ctx.hyperstart.LastStreamSeq(),
140141
GuestCid: ctx.GuestCid,
141142
}
@@ -144,6 +145,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
144145
func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error {
145146
ctx.pciAddr = pinfo.HwStat.PciAddr
146147
ctx.scsiId = pinfo.HwStat.ScsiId
148+
ctx.pmemId = pinfo.HwStat.PmemId
147149
ctx.GuestCid = pinfo.HwStat.GuestCid
148150
if ctx.GuestCid != 0 {
149151
if !VsockCidManager.MarkCidInuse(ctx.GuestCid) {

‎hypervisor/qemu/qemu.go

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,63 @@ func (qc *QemuContext) AddDisk(ctx *hypervisor.VmContext, sourceType string, blo
248248
}
249249
}
250250

251-
newDiskAddSession(ctx, qc, filename, format, id, readonly, result)
251+
commands := make([]*QmpCommand, 2)
252+
devName := scsiId2Name(id)
253+
if !blockInfo.Dax {
254+
hmp := "drive_add dummy file=" + filename + ",if=none,id=" + "drive" + strconv.Itoa(id) + ",format=" + format + ",cache=writeback"
255+
if readonly {
256+
hmp += ",readonly"
257+
}
258+
commands[0] = &QmpCommand{
259+
Execute: "human-monitor-command",
260+
Arguments: map[string]interface{}{
261+
"command-line": hmp,
262+
},
263+
}
264+
commands[1] = &QmpCommand{
265+
Execute: "device_add",
266+
Arguments: map[string]interface{}{
267+
"driver": "scsi-hd", "bus": "scsi0.0", "scsi-id": strconv.Itoa(id),
268+
"drive": "drive" + strconv.Itoa(id), "id": "scsi-disk" + strconv.Itoa(id),
269+
},
270+
}
271+
} else {
272+
// compose qmp commands
273+
// hmp: object_add memory-backend-file,id=mem2,share=on,mem-path=/path/to/dax.img,size=10G
274+
// hmp: device_add nvdimm,id=nvdimm2,memdev=mem2
275+
hmp := "object_add memory-backend-file,id=mem" + strconv.Itoa(blockInfo.PmemId) + ",mem-path=" + filename
276+
if readonly {
277+
hmp += ",share=off"
278+
} else {
279+
hmp += ",share=on"
280+
}
281+
// get the size
282+
fi, e := os.Stat(filename)
283+
if e != nil {
284+
result <- &hypervisor.DeviceFailed{}
285+
return
286+
}
287+
hmp += ",size=" + strconv.FormatInt(fi.Size(), 10)
288+
commands[0] = &QmpCommand{
289+
Execute: "human-monitor-command",
290+
Arguments: map[string]interface{}{
291+
"command-line": hmp,
292+
},
293+
}
294+
commands[1] = &QmpCommand{
295+
Execute: "device_add",
296+
Arguments: map[string]interface{}{
297+
"driver": "nvdimm", "memdev": "mem" + strconv.Itoa(blockInfo.PmemId), "id": "nvdimm" + strconv.Itoa(blockInfo.PmemId),
298+
},
299+
}
300+
devName = "pmem" + strconv.Itoa(blockInfo.PmemId)
301+
}
302+
qc.qmp <- &QmpSession{
303+
commands: commands,
304+
respond: defaultRespond(result, &hypervisor.BlockdevInsertedEvent{
305+
DeviceName: devName,
306+
}),
307+
}
252308
}
253309

254310
func (qc *QemuContext) RemoveDisk(ctx *hypervisor.VmContext, blockInfo *hypervisor.DiskDescriptor, callback hypervisor.VmEvent, result chan<- hypervisor.VmEvent) {

‎hypervisor/qemu/qemu_amd64.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ func (qc *QemuContext) arguments(ctx *hypervisor.VmContext) []string {
2727
qc.cpus = boot.CPU
2828

2929
var machineClass, memParams, cpuParams string
30-
machineClass = "pc-i440fx-2.1"
30+
machineClass = "pc-i440fx-2.1,nvdimm"
3131
memParams = fmt.Sprintf("size=%d,slots=1,maxmem=%dM", boot.Memory, hypervisor.DefaultMaxMem) // TODO set maxmem to the total memory of the system
3232
cpuParams = fmt.Sprintf("cpus=%d,maxcpus=%d", boot.CPU, hypervisor.DefaultMaxCpus) // TODO set it to the cpus of the system
3333

‎hypervisor/qemu/qmp_wrapper.go

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -33,34 +33,6 @@ func defaultRespond(result chan<- hypervisor.VmEvent, callback hypervisor.VmEven
3333
}
3434
}
3535

36-
func newDiskAddSession(ctx *hypervisor.VmContext, qc *QemuContext, filename, format string, id int, readonly bool, result chan<- hypervisor.VmEvent) {
37-
args := "drive_add dummy file=" + filename + ",if=none,id=" + "drive" + strconv.Itoa(id) + ",format=" + format + ",cache=writeback"
38-
if readonly {
39-
args += ",readonly"
40-
}
41-
commands := make([]*QmpCommand, 2)
42-
commands[0] = &QmpCommand{
43-
Execute: "human-monitor-command",
44-
Arguments: map[string]interface{}{
45-
"command-line": args,
46-
},
47-
}
48-
commands[1] = &QmpCommand{
49-
Execute: "device_add",
50-
Arguments: map[string]interface{}{
51-
"driver": "scsi-hd", "bus": "scsi0.0", "scsi-id": strconv.Itoa(id),
52-
"drive": "drive" + strconv.Itoa(id), "id": "scsi-disk" + strconv.Itoa(id),
53-
},
54-
}
55-
devName := scsiId2Name(id)
56-
qc.qmp <- &QmpSession{
57-
commands: commands,
58-
respond: defaultRespond(result, &hypervisor.BlockdevInsertedEvent{
59-
DeviceName: devName,
60-
}),
61-
}
62-
}
63-
6436
func newDiskDelSession(ctx *hypervisor.VmContext, qc *QemuContext, id int, callback hypervisor.VmEvent, result chan<- hypervisor.VmEvent) {
6537
commands := make([]*QmpCommand, 2)
6638
commands[1] = &QmpCommand{

0 commit comments

Comments
 (0)