diff --git a/kernel/exit.c b/kernel/exit.c
index 21a59a6e1f2e..3540b2c9b1b6 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -485,6 +485,8 @@ retry:
 	 * Search through everything else, we should not get here often.
 	 */
 	for_each_process(g) {
+		if (atomic_read(&mm->mm_users) <= 1)
+			break;
 		if (g->flags & PF_KTHREAD)
 			continue;
 		for_each_thread(g, c) {