diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 1e5c2f94e8a3..9f0ad2a4f102 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -197,22 +197,25 @@ retry: * retry, dst_vma will be set to NULL and we must lookup again. */ if (!dst_vma) { - err = -EINVAL; + err = -ENOENT; dst_vma = find_vma(dst_mm, dst_start); if (!dst_vma || !is_vm_hugetlb_page(dst_vma)) goto out_unlock; - - if (vma_hpagesize != vma_kernel_pagesize(dst_vma)) + /* + * Only allow __mcopy_atomic_hugetlb on userfaultfd + * registered ranges. + */ + if (!dst_vma->vm_userfaultfd_ctx.ctx) goto out_unlock; - /* - * Make sure the remaining dst range is both valid and - * fully within a single existing vma. - */ if (dst_start < dst_vma->vm_start || dst_start + len > dst_vma->vm_end) goto out_unlock; + err = -EINVAL; + if (vma_hpagesize != vma_kernel_pagesize(dst_vma)) + goto out_unlock; + vm_shared = dst_vma->vm_flags & VM_SHARED; } @@ -220,12 +223,6 @@ retry: (len - copied) & (vma_hpagesize - 1))) goto out_unlock; - /* - * Only allow __mcopy_atomic_hugetlb on userfaultfd registered ranges. - */ - if (!dst_vma->vm_userfaultfd_ctx.ctx) - goto out_unlock; - /* * If not shared, ensure the dst_vma has a anon_vma. */ @@ -404,29 +401,10 @@ retry: * Make sure the vma is not shared, that the dst range is * both valid and fully within a single existing vma. */ - err = -EINVAL; + err = -ENOENT; dst_vma = find_vma(dst_mm, dst_start); if (!dst_vma) goto out_unlock; - /* - * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but - * it will overwrite vm_ops, so vma_is_anonymous must return false. - */ - if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) && - dst_vma->vm_flags & VM_SHARED)) - goto out_unlock; - - if (dst_start < dst_vma->vm_start || - dst_start + len > dst_vma->vm_end) - goto out_unlock; - - /* - * If this is a HUGETLB vma, pass off to appropriate routine - */ - if (is_vm_hugetlb_page(dst_vma)) - return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start, - src_start, len, zeropage); - /* * Be strict and only allow __mcopy_atomic on userfaultfd * registered ranges to prevent userland errors going @@ -439,6 +417,26 @@ retry: if (!dst_vma->vm_userfaultfd_ctx.ctx) goto out_unlock; + if (dst_start < dst_vma->vm_start || + dst_start + len > dst_vma->vm_end) + goto out_unlock; + + err = -EINVAL; + /* + * shmem_zero_setup is invoked in mmap for MAP_ANONYMOUS|MAP_SHARED but + * it will overwrite vm_ops, so vma_is_anonymous must return false. + */ + if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) && + dst_vma->vm_flags & VM_SHARED)) + goto out_unlock; + + /* + * If this is a HUGETLB vma, pass off to appropriate routine + */ + if (is_vm_hugetlb_page(dst_vma)) + return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start, + src_start, len, zeropage); + if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma)) goto out_unlock;