diff --git a/src/rpc/virnetsocket.c b/src/rpc/virnetsocket.c
index 3d7508110e..106d09aaf9 100644
--- a/src/rpc/virnetsocket.c
+++ b/src/rpc/virnetsocket.c
@@ -681,7 +681,8 @@ int virNetSocketNewConnectUNIX(const char *path,
 
     while (retries &&
            connect(fd, &remoteAddr.data.sa, remoteAddr.len) < 0) {
-        if (!(spawnDaemon && errno == ENOENT)) {
+        if (!(spawnDaemon && (errno == ENOENT ||
+                              errno == ECONNREFUSED))) {
             virReportSystemError(errno, _("Failed to connect socket to '%s'"),
                                  path);
             goto cleanup;