diff --git a/kernel/exit.c b/kernel/exit.c
index bccfa4218356..156283b3c1bf 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -481,6 +481,8 @@ void mm_update_next_owner(struct mm_struct *mm)
 	 * Search through everything else, we should not get here often.
 	 */
 	for_each_process(g) {
+		if (atomic_read(&mm->mm_users) <= 1)
+			break;
 		if (g->flags & PF_KTHREAD)
 			continue;
 		for_each_thread(g, c) {