iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mm: clarify that the function operates on hugepage pte

Browse Source 
We have confusing functions to clear pmd, pmd_clear_* and pmd_clear.  Add
_huge_ to pmdp_clear functions so that we are clear that they operate on
hugepage pte.

We don't bother about other functions like pmdp_set_wrprotect,
pmdp_clear_flush_young, because they operate on PTE bits and hence
indicate they are operating on hugepage ptes

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Aneesh Kumar K.V 2015-06-24 16:57:44 -07:00 committed by Linus Torvalds
parent f28b6ff8c3
commit 8809aa2d28
13 changed files with 65 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
arch/mips/include/asm/pgtable.h
View File

@ -568,12 +568,12 @@ static inline pmd_t pmd_mknotpresent(pmd_t pmd)
} }
/* /*
* The generic version pmdp_get_and_clear uses a version of pmd_clear() with a * The generic version pmdp_huge_get_and_clear uses a version of pmd_clear() with a
* different prototype. * different prototype.
*/ */
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address, pmd_t *pmdp) unsigned long address, pmd_t *pmdp)
{ {
pmd_t old = *pmdp; pmd_t old = *pmdp;

6
arch/powerpc/include/asm/pgtable-ppc64.h
View File

@ -569,9 +569,9 @@ extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
extern int pmdp_clear_flush_young(struct vm_area_struct *vma, extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp); unsigned long address, pmd_t *pmdp);
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
extern pmd_t pmdp_get_and_clear(struct mm_struct *mm, extern pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp); unsigned long addr, pmd_t *pmdp);
#define __HAVE_ARCH_PMDP_SET_WRPROTECT #define __HAVE_ARCH_PMDP_SET_WRPROTECT
static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,

4
arch/powerpc/mm/pgtable_64.c
View File

@ -812,8 +812,8 @@ void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
return; return;
} }
pmd_t pmdp_get_and_clear(struct mm_struct *mm, pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp) unsigned long addr, pmd_t *pmdp)
{ {
pmd_t old_pmd; pmd_t old_pmd;
pgtable_t pgtable; pgtable_t pgtable;

24
arch/s390/include/asm/pgtable.h
View File

@ -1498,9 +1498,9 @@ static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
return pmd_young(pmd); return pmd_young(pmd);
} }
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address, pmd_t *pmdp) unsigned long address, pmd_t *pmdp)
{ {
pmd_t pmd = *pmdp; pmd_t pmd = *pmdp;
@ -1509,10 +1509,10 @@ static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
return pmd; return pmd;
} }
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR_FULL #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
static inline pmd_t pmdp_get_and_clear_full(struct mm_struct *mm, static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm,
unsigned long address, unsigned long address,
pmd_t *pmdp, int full) pmd_t *pmdp, int full)
{ {
pmd_t pmd = *pmdp; pmd_t pmd = *pmdp;
@ -1522,11 +1522,11 @@ static inline pmd_t pmdp_get_and_clear_full(struct mm_struct *mm,
return pmd; return pmd;
} }
#define __HAVE_ARCH_PMDP_CLEAR_FLUSH #define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
static inline pmd_t pmdp_clear_flush(struct vm_area_struct *vma, static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp) unsigned long address, pmd_t *pmdp)
{ {
return pmdp_get_and_clear(vma->vm_mm, address, pmdp); return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
} }
#define __HAVE_ARCH_PMDP_INVALIDATE #define __HAVE_ARCH_PMDP_INVALIDATE
@ -1552,7 +1552,7 @@ static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma,
unsigned long address, unsigned long address,
pmd_t *pmdp) pmd_t *pmdp)
{ {
return pmdp_get_and_clear(vma->vm_mm, address, pmdp); return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
} }
#define pmdp_collapse_flush pmdp_collapse_flush #define pmdp_collapse_flush pmdp_collapse_flush

8
arch/sparc/include/asm/pgtable_64.h
View File

@ -865,10 +865,10 @@ static inline unsigned long pud_pfn(pud_t pud)
void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
pte_t *ptep, pte_t orig, int fullmm); pte_t *ptep, pte_t orig, int fullmm);
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long addr, unsigned long addr,
pmd_t *pmdp) pmd_t *pmdp)
{ {
pmd_t pmd = *pmdp; pmd_t pmd = *pmdp;
set_pmd_at(mm, addr, pmdp, __pmd(0UL)); set_pmd_at(mm, addr, pmdp, __pmd(0UL));

8
arch/tile/include/asm/pgtable.h
View File

@ -414,10 +414,10 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm,
} }
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address, unsigned long address,
pmd_t *pmdp) pmd_t *pmdp)
{ {
return pte_pmd(ptep_get_and_clear(mm, address, pmdp_ptep(pmdp))); return pte_pmd(ptep_get_and_clear(mm, address, pmdp_ptep(pmdp)));
} }

4
arch/x86/include/asm/pgtable.h
View File

@ -805,8 +805,8 @@ static inline int pmd_write(pmd_t pmd)
return pmd_flags(pmd) & _PAGE_RW; return pmd_flags(pmd) & _PAGE_RW;
} }
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, unsigned long addr, static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp) pmd_t *pmdp)
{ {
pmd_t pmd = native_pmdp_get_and_clear(pmdp); pmd_t pmd = native_pmdp_get_and_clear(pmdp);

18
include/asm-generic/pgtable.h
View File

@ -96,11 +96,11 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
} }
#endif #endif
#ifndef __HAVE_ARCH_PMDP_GET_AND_CLEAR #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
unsigned long address, unsigned long address,
pmd_t *pmdp) pmd_t *pmdp)
{ {
pmd_t pmd = *pmdp; pmd_t pmd = *pmdp;
pmd_clear(pmdp); pmd_clear(pmdp);
@ -109,13 +109,13 @@ static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif #endif
#ifndef __HAVE_ARCH_PMDP_GET_AND_CLEAR_FULL #ifndef __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline pmd_t pmdp_get_and_clear_full(struct mm_struct *mm, static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm,
unsigned long address, pmd_t *pmdp, unsigned long address, pmd_t *pmdp,
int full) int full)
{ {
return pmdp_get_and_clear(mm, address, pmdp); return pmdp_huge_get_and_clear(mm, address, pmdp);
} }
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
#endif #endif
@ -152,8 +152,8 @@ extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
pte_t *ptep); pte_t *ptep);
#endif #endif
#ifndef __HAVE_ARCH_PMDP_CLEAR_FLUSH #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
extern pmd_t pmdp_clear_flush(struct vm_area_struct *vma, extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
unsigned long address, unsigned long address,
pmd_t *pmdp); pmd_t *pmdp);
#endif #endif

12
include/linux/mmu_notifier.h
View File

@ -324,25 +324,25 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
___pte; \ ___pte; \
}) })
#define pmdp_clear_flush_notify(__vma, __haddr, __pmd) \ #define pmdp_huge_clear_flush_notify(__vma, __haddr, __pmd) \
({ \ ({ \
unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \ unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
struct mm_struct *___mm = (__vma)->vm_mm; \ struct mm_struct *___mm = (__vma)->vm_mm; \
pmd_t ___pmd; \ pmd_t ___pmd; \
\ \
___pmd = pmdp_clear_flush(__vma, __haddr, __pmd); \ ___pmd = pmdp_huge_clear_flush(__vma, __haddr, __pmd); \
mmu_notifier_invalidate_range(___mm, ___haddr, \ mmu_notifier_invalidate_range(___mm, ___haddr, \
___haddr + HPAGE_PMD_SIZE); \ ___haddr + HPAGE_PMD_SIZE); \
\ \
___pmd; \ ___pmd; \
}) })
#define pmdp_get_and_clear_notify(__mm, __haddr, __pmd) \ #define pmdp_huge_get_and_clear_notify(__mm, __haddr, __pmd) \
({ \ ({ \
unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \ unsigned long ___haddr = __haddr & HPAGE_PMD_MASK; \
pmd_t ___pmd; \ pmd_t ___pmd; \
\ \
___pmd = pmdp_get_and_clear(__mm, __haddr, __pmd); \ ___pmd = pmdp_huge_get_and_clear(__mm, __haddr, __pmd); \
mmu_notifier_invalidate_range(__mm, ___haddr, \ mmu_notifier_invalidate_range(__mm, ___haddr, \
___haddr + HPAGE_PMD_SIZE); \ ___haddr + HPAGE_PMD_SIZE); \
\ \
@ -428,8 +428,8 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
#define ptep_clear_flush_young_notify ptep_clear_flush_young #define ptep_clear_flush_young_notify ptep_clear_flush_young
#define pmdp_clear_flush_young_notify pmdp_clear_flush_young #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
#define ptep_clear_flush_notify ptep_clear_flush #define ptep_clear_flush_notify ptep_clear_flush
#define pmdp_clear_flush_notify pmdp_clear_flush #define pmdp_huge_clear_flush_notify pmdp_huge_clear_flush
#define pmdp_get_and_clear_notify pmdp_get_and_clear #define pmdp_huge_get_and_clear_notify pmdp_huge_get_and_clear
#define set_pte_at_notify set_pte_at #define set_pte_at_notify set_pte_at
#endif /* CONFIG_MMU_NOTIFIER */ #endif /* CONFIG_MMU_NOTIFIER */

16
mm/huge_memory.c
View File

@ -1031,7 +1031,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
goto out_free_pages; goto out_free_pages;
VM_BUG_ON_PAGE(!PageHead(page), page); VM_BUG_ON_PAGE(!PageHead(page), page);
pmdp_clear_flush_notify(vma, haddr, pmd); pmdp_huge_clear_flush_notify(vma, haddr, pmd);
/* leave pmd empty until pte is filled */ /* leave pmd empty until pte is filled */
pgtable = pgtable_trans_huge_withdraw(mm, pmd); pgtable = pgtable_trans_huge_withdraw(mm, pmd);
@ -1174,7 +1174,7 @@ alloc:
pmd_t entry; pmd_t entry;
entry = mk_huge_pmd(new_page, vma->vm_page_prot); entry = mk_huge_pmd(new_page, vma->vm_page_prot);
entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
pmdp_clear_flush_notify(vma, haddr, pmd); pmdp_huge_clear_flush_notify(vma, haddr, pmd);
page_add_new_anon_rmap(new_page, vma, haddr); page_add_new_anon_rmap(new_page, vma, haddr);
mem_cgroup_commit_charge(new_page, memcg, false); mem_cgroup_commit_charge(new_page, memcg, false);
lru_cache_add_active_or_unevictable(new_page, vma); lru_cache_add_active_or_unevictable(new_page, vma);
@ -1396,12 +1396,12 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
pmd_t orig_pmd; pmd_t orig_pmd;
/* /*
* For architectures like ppc64 we look at deposited pgtable * For architectures like ppc64 we look at deposited pgtable
* when calling pmdp_get_and_clear. So do the * when calling pmdp_huge_get_and_clear. So do the
* pgtable_trans_huge_withdraw after finishing pmdp related * pgtable_trans_huge_withdraw after finishing pmdp related
* operations. * operations.
*/ */
orig_pmd = pmdp_get_and_clear_full(tlb->mm, addr, pmd, orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
tlb->fullmm); tlb->fullmm);
tlb_remove_pmd_tlb_entry(tlb, pmd, addr); tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd); pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
if (is_huge_zero_pmd(orig_pmd)) { if (is_huge_zero_pmd(orig_pmd)) {
@ -1459,7 +1459,7 @@ int move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma,
new_ptl = pmd_lockptr(mm, new_pmd); new_ptl = pmd_lockptr(mm, new_pmd);
if (new_ptl != old_ptl) if (new_ptl != old_ptl)
spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
pmd = pmdp_get_and_clear(mm, old_addr, old_pmd); pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
VM_BUG_ON(!pmd_none(*new_pmd)); VM_BUG_ON(!pmd_none(*new_pmd));
if (pmd_move_must_withdraw(new_ptl, old_ptl)) { if (pmd_move_must_withdraw(new_ptl, old_ptl)) {
@ -1505,7 +1505,7 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
} }
if (!prot_numa || !pmd_protnone(*pmd)) { if (!prot_numa || !pmd_protnone(*pmd)) {
entry = pmdp_get_and_clear_notify(mm, addr, pmd); entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd);
entry = pmd_modify(entry, newprot); entry = pmd_modify(entry, newprot);
if (preserve_write) if (preserve_write)
entry = pmd_mkwrite(entry); entry = pmd_mkwrite(entry);
@ -2863,7 +2863,7 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
pmd_t _pmd; pmd_t _pmd;
int i; int i;
pmdp_clear_flush_notify(vma, haddr, pmd); pmdp_huge_clear_flush_notify(vma, haddr, pmd);
/* leave pmd empty until pte is filled */ /* leave pmd empty until pte is filled */
pgtable = pgtable_trans_huge_withdraw(mm, pmd); pgtable = pgtable_trans_huge_withdraw(mm, pmd);

2
mm/migrate.c
View File

@ -1799,7 +1799,7 @@ fail_putback:
*/ */
flush_cache_range(vma, mmun_start, mmun_end); flush_cache_range(vma, mmun_start, mmun_end);
page_add_anon_rmap(new_page, vma, mmun_start); page_add_anon_rmap(new_page, vma, mmun_start);
pmdp_clear_flush_notify(vma, mmun_start, pmd); pmdp_huge_clear_flush_notify(vma, mmun_start, pmd);
set_pmd_at(mm, mmun_start, pmd, entry); set_pmd_at(mm, mmun_start, pmd, entry);
flush_tlb_range(vma, mmun_start, mmun_end); flush_tlb_range(vma, mmun_start, mmun_end);
update_mmu_cache_pmd(vma, address, &entry); update_mmu_cache_pmd(vma, address, &entry);

14
mm/pgtable-generic.c
View File

@ -119,15 +119,15 @@ pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
} }
#endif #endif
#ifndef __HAVE_ARCH_PMDP_CLEAR_FLUSH #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address, pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp) pmd_t *pmdp)
{ {
pmd_t pmd; pmd_t pmd;
VM_BUG_ON(address & ~HPAGE_PMD_MASK); VM_BUG_ON(address & ~HPAGE_PMD_MASK);
VM_BUG_ON(!pmd_trans_huge(*pmdp)); VM_BUG_ON(!pmd_trans_huge(*pmdp));
pmd = pmdp_get_and_clear(vma->vm_mm, address, pmdp); pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return pmd; return pmd;
} }
@ -205,11 +205,15 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp) pmd_t *pmdp)
{ {
/*
* pmd and hugepage pte format are same. So we could
* use the same function.
*/
pmd_t pmd; pmd_t pmd;
VM_BUG_ON(address & ~HPAGE_PMD_MASK); VM_BUG_ON(address & ~HPAGE_PMD_MASK);
VM_BUG_ON(pmd_trans_huge(*pmdp)); VM_BUG_ON(pmd_trans_huge(*pmdp));
pmd = pmdp_get_and_clear(vma->vm_mm, address, pmdp); pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
return pmd; return pmd;
} }

2
mm/rmap.c
View File

@ -625,7 +625,7 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
pmd = pmd_offset(pud, address); pmd = pmd_offset(pud, address);
/* /*
* Some THP functions use the sequence pmdp_clear_flush(), set_pmd_at() * Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at()
* without holding anon_vma lock for write. So when looking for a * without holding anon_vma lock for write. So when looking for a
* genuine pmde (in which to find pte), test present and !THP together. * genuine pmde (in which to find pte), test present and !THP together.
*/ */