diff --git a/kernel/exit.c b/kernel/exit.c
index c764d16328f6..56d3a099825f 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -433,6 +433,8 @@ retry:
 	 * Search through everything else, we should not get here often.
 	 */
 	for_each_process(g) {
+		if (atomic_read(&mm->mm_users) <= 1)
+			break;
 		if (g->flags & PF_KTHREAD)
 			continue;
 		for_each_thread(g, c) {