diff --git a/src/rpc/virnetsocket.c b/src/rpc/virnetsocket.c
index 51f94d4bf1..6153e0e7cb 100644
--- a/src/rpc/virnetsocket.c
+++ b/src/rpc/virnetsocket.c
@@ -610,7 +610,8 @@ int virNetSocketNewConnectUNIX(const char *path,
 
     while (retries &&
            connect(fd, &remoteAddr.data.sa, remoteAddr.len) < 0) {
-        if (!(spawnDaemon && errno == ENOENT)) {
+        if (!(spawnDaemon && (errno == ENOENT ||
+                              errno == ECONNREFUSED))) {
             virReportSystemError(errno, _("Failed to connect socket to '%s'"),
                                  path);
             goto cleanup;