qemu: Add command-line to generate SGX EPC memory backend

According to the result parsing from xml, add the argument of
SGX EPC memory backend into QEMU command line.

$ qemu-system-x86_64 \
    ...... \
    -object '{"qom-type":"memory-backend-epc","id":"memepc0","prealloc":true,"size":67108864,"host-nodes":[0,1],"policy":"bind"}' \
    -object '{"qom-type":"memory-backend-epc","id":"memepc1","prealloc":true,"size":16777216,"host-nodes":[2,3],"policy":"bind"}' \
    -machine sgx-epc.0.memdev=memepc0,sgx-epc.0.node=0,sgx-epc.1.memdev=memepc1,sgx-epc.1.node=1

Signed-off-by: Lin Yang <lin.a.yang@intel.com>
Signed-off-by: Haibin Huang <haibin.huang@intel.com>
Signed-off-by: Michal Privoznik <mprivozn@redhat.com>
Reviewed-by: Michal Privoznik <mprivozn@redhat.com>

src/qemu/qemu_alias.c

@@ -464,7 +464,8 @@ qemuDeviceMemoryGetAliasID(virDomainDef *def,
      * valid */
     if (!oldAlias &&
         mem->model != VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM &&
-        mem->model != VIR_DOMAIN_MEMORY_MODEL_VIRTIO_MEM)
+        mem->model != VIR_DOMAIN_MEMORY_MODEL_VIRTIO_MEM &&
+        mem->model != VIR_DOMAIN_MEMORY_MODEL_SGX_EPC)
         return mem->info.addr.dimm.slot;
 
     for (i = 0; i < def->nmems; i++) {

src/qemu/qemu_command.c

@@ -3307,7 +3307,11 @@ qemuBuildMemoryBackendProps(virJSONValue **backendProps,
 
     props = virJSONValueNewObject();
 
-    if (!mem->nvdimmPath &&
+    if (mem->model == VIR_DOMAIN_MEMORY_MODEL_SGX_EPC) {
+        backendType = "memory-backend-epc";
+        if (!priv->memPrealloc)
+            prealloc = true;
+    } else if (!mem->nvdimmPath &&
         def->mem.source == VIR_DOMAIN_MEMORY_SOURCE_MEMFD) {
         backendType = "memory-backend-memfd";
 
@@ -3322,7 +3326,6 @@ qemuBuildMemoryBackendProps(virJSONValue **backendProps,
 
         if (systemMemory)
             disableCanonicalPath = true;
-
     } else if (useHugepage || mem->nvdimmPath || memAccess ||
         def->mem.source == VIR_DOMAIN_MEMORY_SOURCE_FILE) {
 
@@ -6600,6 +6603,8 @@ qemuAppendDomainMemoryMachineParams(virBuffer *buf,
                                     virQEMUCaps *qemuCaps)
 {
     virTristateSwitch dump = def->mem.dump_core;
+    bool nvdimmAdded = false;
+    int epcNum = 0;
     size_t i;
 
     if (dump == VIR_TRISTATE_SWITCH_ABSENT)
@@ -6611,8 +6616,36 @@ qemuAppendDomainMemoryMachineParams(virBuffer *buf,
         virBufferAddLit(buf, ",mem-merge=off");
 
     for (i = 0; i < def->nmems; i++) {
-        if (def->mems[i]->model == VIR_DOMAIN_MEMORY_MODEL_NVDIMM) {
-            virBufferAddLit(buf, ",nvdimm=on");
+        int targetNode = def->mems[i]->targetNode;
+
+        switch (def->mems[i]->model) {
+        case VIR_DOMAIN_MEMORY_MODEL_NVDIMM:
+            if (!nvdimmAdded) {
+                virBufferAddLit(buf, ",nvdimm=on");
+                nvdimmAdded = true;
+            }
+            break;
+
+        case VIR_DOMAIN_MEMORY_MODEL_SGX_EPC:
+            /* add sgx epc memory to -machine parameter */
+
+            if (targetNode < 0) {
+                /* set NUMA target node to 0 by default if user doesn't
+                 * specify it. */
+                targetNode = 0;
+            }
+
+            virBufferAsprintf(buf, ",sgx-epc.%d.memdev=mem%s,sgx-epc.%d.node=%d",
+                              epcNum, def->mems[i]->info.alias, epcNum, targetNode);
+
+            epcNum++;
+            break;
+
+        case VIR_DOMAIN_MEMORY_MODEL_DIMM:
+        case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
+        case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_MEM:
+        case VIR_DOMAIN_MEMORY_MODEL_NONE:
+        case VIR_DOMAIN_MEMORY_MODEL_LAST:
             break;
         }
     }
@@ -7322,11 +7355,27 @@ qemuBuildMemoryDeviceCommandLine(virCommand *cmd,
         if (qemuBuildMemoryDimmBackendStr(cmd, def->mems[i], def, cfg, priv) < 0)
             return -1;
 
-        if (!(props = qemuBuildMemoryDeviceProps(cfg, priv, def, def->mems[i])))
-            return -1;
+        switch (def->mems[i]->model) {
+        case VIR_DOMAIN_MEMORY_MODEL_NVDIMM:
+        case VIR_DOMAIN_MEMORY_MODEL_DIMM:
+        case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_PMEM:
+        case VIR_DOMAIN_MEMORY_MODEL_VIRTIO_MEM:
+            if (!(props = qemuBuildMemoryDeviceProps(cfg, priv, def, def->mems[i])))
+                return -1;
 
-        if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, priv->qemuCaps) < 0)
-            return -1;
+            if (qemuBuildDeviceCommandlineFromJSON(cmd, props, def, priv->qemuCaps) < 0)
+                return -1;
+
+            break;
+
+        /* sgx epc memory will be added to -machine parameter, so skip here */
+        case VIR_DOMAIN_MEMORY_MODEL_SGX_EPC:
+            break;
+
+        case VIR_DOMAIN_MEMORY_MODEL_NONE:
+        case VIR_DOMAIN_MEMORY_MODEL_LAST:
+            break;
+        }
     }
 
     return 0;

src/qemu/qemu_monitor_json.c

@@ -7208,13 +7208,25 @@ qemuMonitorJSONGetMemoryDeviceInfo(qemuMonitor *mon,
             return -1;
         }
 
-        /* While 'id' attribute is marked as optional in QEMU's QAPI
-         * specification, Libvirt always sets it. Thus we can fail if not
-         * present. */
-        if (!(devalias = virJSONValueObjectGetString(dimminfo, "id"))) {
-            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
-                           _("dimm memory info data is missing 'id'"));
-            return -1;
+        if (STREQ(type, "dimm") || STREQ(type, "nvdimm") || STREQ(type, "virtio-mem")) {
+            /* While 'id' attribute is marked as optional in QEMU's QAPI
+            * specification, Libvirt always sets it. Thus we can fail if not
+            * present. */
+            if (!(devalias = virJSONValueObjectGetString(dimminfo, "id"))) {
+                virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                            _("dimm memory info data is missing 'id'"));
+                return -1;
+            }
+        } else if (STREQ(type, "sgx-epc")) {
+            if (!(devalias = virJSONValueObjectGetString(dimminfo, "memdev"))) {
+                virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                            _("sgx-epc memory info data is missing 'memdev'"));
+                return -1;
+            }
+        } else {
+            virReportError(VIR_ERR_INTERNAL_ERROR,
+                        _("%s memory device info is not handled yet"), type);
+                return -1;
         }
 
         meminfo = g_new0(qemuMonitorMemoryDeviceInfo, 1);
@@ -7258,6 +7270,21 @@ qemuMonitorJSONGetMemoryDeviceInfo(qemuMonitor *mon,
                                _("malformed/missing size in virtio memory info"));
                 return -1;
             }
+        } else if (STREQ(type, "sgx-epc")) {
+            /* sgx-epc memory devices */
+            if (virJSONValueObjectGetNumberUlong(dimminfo, "memaddr",
+                                                 &meminfo->address) < 0) {
+                virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                               _("malformed/missing memaddr in sgx-epc memory info"));
+                return -1;
+            }
+
+            if (virJSONValueObjectGetNumberUlong(dimminfo, "size",
+                                                 &meminfo->size) < 0) {
+                virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                               _("malformed/missing size in sgx-epc memory info"));
+                return -1;
+            }
         } else {
             /* type not handled yet */
             continue;

src/qemu/qemu_validate.c

@@ -4984,6 +4984,9 @@ static int
 qemuValidateDomainDeviceDefMemory(virDomainMemoryDef *mem,
                                   virQEMUCaps *qemuCaps)
 {
+    virSGXCapability *sgxCaps;
+    ssize_t node = -1;
+
     switch (mem->model) {
     case VIR_DOMAIN_MEMORY_MODEL_DIMM:
         if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_PC_DIMM)) {
@@ -5031,6 +5034,35 @@ qemuValidateDomainDeviceDefMemory(virDomainMemoryDef *mem,
                            _("sgx epc isn't supported by this QEMU binary"));
             return -1;
         }
+
+        sgxCaps = virQEMUCapsGetSGXCapabilities(qemuCaps);
+
+        if (sgxCaps->nSgxSections == 0) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                           _("this QEMU version didn't provide SGX EPC NUMA info"));
+            return -1;
+        }
+
+        if (mem->sourceNodes) {
+            while ((node = virBitmapNextSetBit(mem->sourceNodes, node)) >= 0) {
+                if (mem->size > sgxCaps->sgxSections[node].size) {
+                    virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                                   _("sgx epc size %lld on host node %ld is less than requested size %lld"),
+                                   sgxCaps->sgxSections[node].size, node, mem->size);
+                    return -1;
+                }
+            }
+        } else {
+            /* allocate epc from host node 0 by default if user doesn't
+             * specify it. */
+            if (mem->size > sgxCaps->sgxSections[0].size) {
+                virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                               _("sgx epc size %lld on host node %d is less than requested size %lld"),
+                               sgxCaps->sgxSections[0].size, 0, mem->size);
+                return -1;
+            }
+        }
+
         break;
 
     case VIR_DOMAIN_MEMORY_MODEL_NONE:

tests/qemuxml2argvdata/sgx-epc.x86_64-7.0.0.args

@@ -0,0 +1,40 @@
+LC_ALL=C \
+PATH=/bin \
+HOME=/tmp/lib/domain--1-QEMUGuest1 \
+USER=test \
+LOGNAME=test \
+XDG_DATA_HOME=/tmp/lib/domain--1-QEMUGuest1/.local/share \
+XDG_CACHE_HOME=/tmp/lib/domain--1-QEMUGuest1/.cache \
+XDG_CONFIG_HOME=/tmp/lib/domain--1-QEMUGuest1/.config \
+/usr/bin/qemu-system-x86_64 \
+-name guest=QEMUGuest1,debug-threads=on \
+-S \
+-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/tmp/lib/domain--1-QEMUGuest1/master-key.aes"}' \
+-machine pc-q35-7.0,usb=off,dump-guest-core=off,sgx-epc.0.memdev=memepc0,sgx-epc.0.node=0,sgx-epc.1.memdev=memepc1,sgx-epc.1.node=1 \
+-accel tcg \
+-cpu qemu64 \
+-m size=1048576k,slots=16,maxmem=1130496k \
+-overcommit mem-lock=off \
+-smp 2,sockets=2,cores=1,threads=1 \
+-object '{"qom-type":"memory-backend-ram","id":"ram-node0","size":536870912}' \
+-numa node,nodeid=0,cpus=0,memdev=ram-node0 \
+-object '{"qom-type":"memory-backend-ram","id":"ram-node1","size":536870912}' \
+-numa node,nodeid=1,cpus=1,memdev=ram-node1 \
+-uuid c7a5fdbd-edaf-9455-926a-d65c16db1809 \
+-display none \
+-no-user-config \
+-nodefaults \
+-chardev socket,id=charmonitor,fd=1729,server=on,wait=off \
+-mon chardev=charmonitor,id=monitor,mode=control \
+-rtc base=utc \
+-no-shutdown \
+-no-acpi \
+-boot strict=on \
+-device '{"driver":"pcie-root-port","port":8,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x1"}' \
+-device '{"driver":"pcie-root-port","port":9,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x1.0x1"}' \
+-object '{"qom-type":"memory-backend-epc","id":"memepc0","prealloc":true,"size":67108864,"host-nodes":[0,1],"policy":"bind"}' \
+-object '{"qom-type":"memory-backend-epc","id":"memepc1","prealloc":true,"size":16777216,"host-nodes":[0,1],"policy":"bind"}' \
+-audiodev '{"id":"audio1","driver":"none"}' \
+-device '{"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.1","addr":"0x0"}' \
+-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
+-msg timestamp=on

tests/qemuxml2argvtest.c

@@ -2967,6 +2967,8 @@ mymain(void)
     DO_TEST_PARSE_ERROR("cpu-phys-bits-emulate3", QEMU_CAPS_KVM);
     DO_TEST_PARSE_ERROR("cpu-phys-bits-passthrough2", QEMU_CAPS_KVM);
 
+    DO_TEST_CAPS_VER("sgx-epc", "7.0.0");
+
     if (getenv("LIBVIRT_SKIP_CLEANUP") == NULL)
         virFileDeleteTree(fakerootdir);