diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c
index 7105b820f1..9fdf66a840 100644
--- a/src/qemu/qemu_hotplug.c
+++ b/src/qemu/qemu_hotplug.c
@@ -3039,6 +3039,9 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver,
             VIR_WARN("Failed to restore host device labelling");
     }
 
+    if (qemuTeardownHostdevCgroup(vm, hostdev) < 0)
+        VIR_WARN("Failed to remove host device cgroup ACL");
+
     switch ((virDomainHostdevSubsysType) hostdev->source.subsys.type) {
     case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
         qemuDomainRemovePCIHostDevice(driver, vm, hostdev);
@@ -3057,9 +3060,6 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver,
         break;
     }
 
-    if (qemuTeardownHostdevCgroup(vm, hostdev) < 0)
-        VIR_WARN("Failed to remove host device cgroup ACL");
-
     virDomainHostdevDefFree(hostdev);
 
     if (net) {