conf, vmx: check for OOM after calling xmlBufferCreate()

Although libvirt itself uses g_malloc0() and friends, which exit when
there isn't enouogh memory, libxml2 uses standard malloc(), which just
returns NULL on OOM - this means we must check for NULL on return from
any libxml2 functions that allocate memory.

xmlBufferCreate(), for example, might return NULL, and we don't always
check for it. This patch adds checks where it isn't already done.

(NB: Although libxml2 has a provision for changing behavior on OOM (by
calling xmlMemSetup() to change what functions are used to
allocating/freeing memory), we can't use that, since parts of libvirt
code end up in libvirt.so, which is linked and called directly by
applications that may themselves use libxml2 (and may have already set
their own alternate malloc()), e.g. drivers like esx which live totally
in the library rather than a separate process.)

Signed-off-by: Laine Stump <laine@redhat.com>
Reviewed-by: Ján Tomko <jtomko@redhat.com>

src/conf/domain_conf.c

@@ -29589,7 +29589,11 @@ virDomainDefFormatInternalSetRootName(virDomainDefPtr def,
          * Thankfully, libxml maps what looks like globals into
          * thread-local uses, so we are thread-safe.  */
         xmlIndentTreeOutput = 1;
-        xmlbuf = xmlBufferCreate();
+        if (!(xmlbuf = xmlBufferCreate())) {
+            virReportOOMError();
+            goto error;
+        }
+
         if (xmlNodeDump(xmlbuf, def->metadata->doc, def->metadata,
                         virBufferGetIndent(buf) / 2, 1) < 0) {
             xmlBufferFree(xmlbuf);

src/conf/network_conf.c

@@ -2518,7 +2518,11 @@ virNetworkDefFormatBuf(virBufferPtr buf,
          * Thankfully, libxml maps what looks like globals into
          * thread-local uses, so we are thread-safe.  */
         xmlIndentTreeOutput = 1;
-        xmlbuf = xmlBufferCreate();
+        if (!(xmlbuf = xmlBufferCreate())) {
+            virReportOOMError();
+            return -1;
+        }
+
         if (xmlNodeDump(xmlbuf, def->metadata->doc, def->metadata,
                         virBufferGetIndent(buf) / 2, 1) < 0) {
             xmlBufferFree(xmlbuf);

src/vmx/vmx.c

@@ -697,8 +697,8 @@ virVMXConvertToUTF8(const char *encoding, const char *string)
 {
     char *result = NULL;
     xmlCharEncodingHandlerPtr handler;
-    xmlBufferPtr input;
-    xmlBufferPtr utf8;
+    xmlBufferPtr input = NULL;
+    xmlBufferPtr utf8 = NULL;
 
     handler = xmlFindCharEncodingHandler(encoding);
 
@@ -708,8 +708,11 @@ virVMXConvertToUTF8(const char *encoding, const char *string)
         return NULL;
     }
 
-    input = xmlBufferCreateStatic((char *)string, strlen(string));
-    utf8 = xmlBufferCreate();
+    if (!(input = xmlBufferCreateStatic((char *)string, strlen(string))) ||
+        !(utf8 = xmlBufferCreate())) {
+        virReportOOMError();
+        goto cleanup;
+    }
 
     if (xmlCharEncInFunc(handler, utf8, input) < 0) {
         virReportError(VIR_ERR_INTERNAL_ERROR,