Skip to content
This repository was archived by the owner on Feb 8, 2021. It is now read-only.

Commit ee420f6

Browse files
laijslaijs
committed
add dax support
Signed-off-by: Lai Jiangshan <[email protected]>
1 parent 73016ca commit ee420f6

10 files changed

+177
-98
lines changed

‎api/descriptions.pb.go

Lines changed: 70 additions & 64 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/descriptions.proto

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ message VolumeOption {
8686
string keyring = 3;
8787
int32 bytesPerSec = 4;
8888
int32 iops = 5;
89+
bool daxBlock = 6;
8990
}
9091

9192
message UserGroupInfo {
@@ -110,4 +111,4 @@ message Process {
110111
repeated string Args = 7;
111112
repeated string Envs = 8;
112113
string Workdir = 9;
113-
}
114+
}

‎hack/update-generated-proto.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ set -o errexit
33
set -o nounset
44
set -o pipefail
55

6-
if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3.0."* ]]; then
6+
if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3."* ]]; then
77
echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
88
echo "install the platform appropriate Protobuf package for your OS: "
99
echo

‎hypervisor/context.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
type VmHwStatus struct {
1919
PciAddr int //next available pci addr for pci hotplug
2020
ScsiId int //next available scsi id for scsi hotplug
21+
PmemId int //next available pmem id for nvdimm hotplug
2122
AttachId uint64 //next available attachId for attached tty
2223
GuestCid uint32 //vsock guest cid
2324
}
@@ -50,6 +51,7 @@ type VmContext struct {
5051

5152
pciAddr int //next available pci addr for pci hotplug
5253
scsiId int //next available scsi id for scsi hotplug
54+
pmemId int //next available pmem id for nvdimm hotplug
5355

5456
// InterfaceCount int
5557

@@ -185,6 +187,14 @@ func (ctx *VmContext) unsetTimeout() {
185187
}
186188
}
187189

190+
func (ctx *VmContext) nextPmemId() int {
191+
ctx.idLock.Lock()
192+
id := ctx.pmemId
193+
ctx.pmemId++
194+
ctx.idLock.Unlock()
195+
return id
196+
}
197+
188198
func (ctx *VmContext) nextScsiId() int {
189199
ctx.idLock.Lock()
190200
id := ctx.scsiId

‎hypervisor/disk.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ type DiskDescriptor struct {
1919
ScsiAddr string
2020
DockerVolume bool
2121
Options map[string]string
22+
Dax bool
23+
PmemId int
2224
}
2325

2426
func (d *DiskDescriptor) IsDir() bool {
@@ -65,6 +67,8 @@ func NewDiskContext(ctx *VmContext, vol *api.VolumeDescription) *DiskContext {
6567
"bytespersec": strconv.Itoa(int(vol.Options.BytesPerSec)),
6668
"iops": strconv.Itoa(int(vol.Options.Iops)),
6769
}
70+
} else if vol.Options != nil && vol.Options.DaxBlock && ctx.Boot.HotAddCpuMem {
71+
dc.DiskDescriptor.Dax = true
6872
}
6973
return dc
7074
}
@@ -78,7 +82,11 @@ func (dc *DiskContext) insert(result chan api.Result) {
7882
return
7983
}
8084

81-
dc.ScsiId = dc.sandbox.nextScsiId()
85+
if dc.Dax {
86+
dc.PmemId = dc.sandbox.nextPmemId()
87+
} else {
88+
dc.ScsiId = dc.sandbox.nextScsiId()
89+
}
8290
usage := "volume"
8391
if dc.isRootVol {
8492
usage = "image"

‎hypervisor/libvirt/libvirt.go

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -461,7 +461,7 @@ func (lc *LibvirtContext) domainXml(ctx *hypervisor.VmContext) (string, error) {
461461
}
462462

463463
if ctx.Boot.HotAddCpuMem {
464-
dom.OS.Type.Machine = "pc-i440fx-2.1"
464+
dom.OS.Type.Machine = "pc-i440fx-2.1,nvdimm"
465465
dom.VCpu.Content = hypervisor.DefaultMaxCpus
466466
dom.MaxMem = &maxmem{Unit: "MiB", Slots: "1", Content: hypervisor.DefaultMaxMem}
467467

@@ -918,6 +918,35 @@ func (lc *LibvirtContext) AddDisk(ctx *hypervisor.VmContext, sourceType string,
918918
return
919919
}
920920

921+
if blockInfo.Dax {
922+
// get the size
923+
fi, e := os.Stat(blockInfo.Filename)
924+
if e != nil {
925+
result <- &hypervisor.DeviceFailed{}
926+
return
927+
}
928+
size := fi.Size()
929+
// compose hmp
930+
hmp := fmt.Sprintf("object_add memory-backend-file,id=mem%d,share=off,mem-path=%s,size=%d", blockInfo.PmemId, blockInfo.Filename, size)
931+
err := exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
932+
if err != nil {
933+
result <- &hypervisor.DeviceFailed{}
934+
return
935+
}
936+
hmp = fmt.Sprintf("device_add nvdimm,id=nvdimm%d,memdev=mem%d", blockInfo.PmemId, blockInfo.PmemId)
937+
err = exec.Command("virsh", "-c", LibvirtdAddress, "qemu-monitor-command", ctx.Id, "--hmp", hmp).Run()
938+
if err != nil {
939+
result <- &hypervisor.DeviceFailed{}
940+
return
941+
}
942+
result <- &hypervisor.BlockdevInsertedEvent{
943+
Name: name,
944+
SourceType: sourceType,
945+
DeviceName: "pmem" + strconv.Itoa(blockInfo.PmemId),
946+
}
947+
return
948+
}
949+
921950
secretUUID, err := lc.diskSecretUUID(blockInfo)
922951
if err != nil {
923952
glog.Error("generate disk-get-secret failed, ", err.Error())

‎hypervisor/persistence.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
136136
return &VmHwStatus{
137137
PciAddr: ctx.pciAddr,
138138
ScsiId: ctx.scsiId,
139+
PmemId: ctx.pmemId,
139140
AttachId: ctx.hyperstart.LastStreamSeq(),
140141
GuestCid: ctx.GuestCid,
141142
}
@@ -144,6 +145,7 @@ func (ctx *VmContext) dumpHwInfo() *VmHwStatus {
144145
func (ctx *VmContext) loadHwStatus(pinfo *PersistInfo) error {
145146
ctx.pciAddr = pinfo.HwStat.PciAddr
146147
ctx.scsiId = pinfo.HwStat.ScsiId
148+
ctx.pmemId = pinfo.HwStat.PmemId
147149
ctx.GuestCid = pinfo.HwStat.GuestCid
148150
if ctx.GuestCid != 0 {
149151
if !VsockCidManager.MarkCidInuse(ctx.GuestCid) {

‎hypervisor/qemu/qemu.go

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,7 +247,58 @@ func (qc *QemuContext) AddDisk(ctx *hypervisor.VmContext, sourceType string, blo
247247
}
248248
}
249249

250-
newDiskAddSession(ctx, qc, name, sourceType, filename, format, id, result)
250+
commands := make([]*QmpCommand, 2)
251+
devName := scsiId2Name(id)
252+
if !blockInfo.Dax {
253+
commands[0] = &QmpCommand{
254+
Execute: "human-monitor-command",
255+
Arguments: map[string]interface{}{
256+
"command-line": "drive_add dummy file=" +
257+
filename + ",if=none,id=" + "drive" + strconv.Itoa(id) + ",format=" + format + ",cache=writeback",
258+
},
259+
}
260+
commands[1] = &QmpCommand{
261+
Execute: "device_add",
262+
Arguments: map[string]interface{}{
263+
"driver": "scsi-hd", "bus": "scsi0.0", "scsi-id": strconv.Itoa(id),
264+
"drive": "drive" + strconv.Itoa(id), "id": "scsi-disk" + strconv.Itoa(id),
265+
},
266+
}
267+
} else {
268+
// get the size
269+
fi, e := os.Stat(filename)
270+
if e != nil {
271+
result <- &hypervisor.DeviceFailed{}
272+
return
273+
}
274+
size := fi.Size()
275+
// compose qmp commands
276+
// hmp: object_add memory-backend-file,id=mem2,share=off,mem-path=/path/to/dax.img,size=10G
277+
// hmp: device_add nvdimm,id=nvdimm2,memdev=mem2
278+
commands[0] = &QmpCommand{
279+
Execute: "human-monitor-command",
280+
Arguments: map[string]interface{}{
281+
"command-line": "object_add memory-backend-file,id=mem" + strconv.Itoa(blockInfo.PmemId) +
282+
",share=off,mem-path=" + filename + ",size=" + strconv.FormatInt(size, 10),
283+
},
284+
}
285+
commands[1] = &QmpCommand{
286+
Execute: "device_add",
287+
Arguments: map[string]interface{}{
288+
"driver": "nvdimm", "memdev": "mem" + strconv.Itoa(blockInfo.PmemId), "id": "nvdimm" + strconv.Itoa(blockInfo.PmemId),
289+
},
290+
}
291+
devName = "pmem" + strconv.Itoa(blockInfo.PmemId)
292+
}
293+
qc.qmp <- &QmpSession{
294+
commands: commands,
295+
respond: defaultRespond(result, &hypervisor.BlockdevInsertedEvent{
296+
Name: name,
297+
SourceType: sourceType,
298+
DeviceName: devName,
299+
ScsiId: id,
300+
}),
301+
}
251302
}
252303

253304
func (qc *QemuContext) RemoveDisk(ctx *hypervisor.VmContext, blockInfo *hypervisor.DiskDescriptor, callback hypervisor.VmEvent, result chan<- hypervisor.VmEvent) {

0 commit comments

Comments
 (0)