powerpc/mm: Add new "set" flag argument to pte/pmd update function
pte_update() is a powerpc-ism used to change the bits of a PTE when the access permission is being restricted (a flush is potentially needed). It uses atomic operations on when needed and handles the hash synchronization on hash based processors. It is currently only used to clear PTE bits and so the current implementation doesn't provide a way to also set PTE bits. The new _PAGE_NUMA bit, when set, is actually restricting access so it must use that function too, so this change adds the ability for pte_update() to also set bits. We will use this later to set the _PAGE_NUMA bit. Acked-by: Mel Gorman <mgorman@suse.de> Acked-by: Rik van Riel <riel@redhat.com> Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
This commit is contained in:
Aneesh Kumar K.V
Benjamin Herrenschmidt
parent
49d9684a54
commit
88247e8d7b
@ -127,7 +127,7 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long addr, pte_t *ptep)
|
||||
{
|
||||
#ifdef CONFIG_PPC64
|
||||
return __pte(pte_update(mm, addr, ptep, ~0UL, 1));
|
||||
return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1));
|
||||
#else
|
||||
return __pte(pte_update(ptep, ~0UL, 0));
|
||||
#endif
|
||||
|
||||
@ -195,6 +195,7 @@ extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
|
||||
static inline unsigned long pte_update(struct mm_struct *mm,
|
||||
unsigned long addr,
|
||||
pte_t *ptep, unsigned long clr,
|
||||
unsigned long set,
|
||||
int huge)
|
||||
{
|
||||
#ifdef PTE_ATOMIC_UPDATES
|
||||
@ -205,14 +206,15 @@ static inline unsigned long pte_update(struct mm_struct *mm,
|
||||
andi. %1,%0,%6\n\
|
||||
bne- 1b \n\
|
||||
andc %1,%0,%4 \n\
|
||||
or %1,%1,%7\n\
|
||||
stdcx. %1,0,%3 \n\
|
||||
bne- 1b"
|
||||
: "=&r" (old), "=&r" (tmp), "=m" (*ptep)
|
||||
: "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY)
|
||||
: "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY), "r" (set)
|
||||
: "cc" );
|
||||
#else
|
||||
unsigned long old = pte_val(*ptep);
|
||||
*ptep = __pte(old & ~clr);
|
||||
*ptep = __pte((old & ~clr) | set);
|
||||
#endif
|
||||
/* huge pages use the old page table lock */
|
||||
if (!huge)
|
||||
@ -231,9 +233,9 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
|
||||
{
|
||||
unsigned long old;
|
||||
|
||||
if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
|
||||
if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
|
||||
return 0;
|
||||
old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0);
|
||||
old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0, 0);
|
||||
return (old & _PAGE_ACCESSED) != 0;
|
||||
}
|
||||
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
|
||||
@ -252,7 +254,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
|
||||
if ((pte_val(*ptep) & _PAGE_RW) == 0)
|
||||
return;
|
||||
|
||||
pte_update(mm, addr, ptep, _PAGE_RW, 0);
|
||||
pte_update(mm, addr, ptep, _PAGE_RW, 0, 0);
|
||||
}
|
||||
|
||||
static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
|
||||
@ -261,7 +263,7 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
|
||||
if ((pte_val(*ptep) & _PAGE_RW) == 0)
|
||||
return;
|
||||
|
||||
pte_update(mm, addr, ptep, _PAGE_RW, 1);
|
||||
pte_update(mm, addr, ptep, _PAGE_RW, 0, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -284,14 +286,14 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
|
||||
static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long addr, pte_t *ptep)
|
||||
{
|
||||
unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0);
|
||||
unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0, 0);
|
||||
return __pte(old);
|
||||
}
|
||||
|
||||
static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
|
||||
pte_t * ptep)
|
||||
{
|
||||
pte_update(mm, addr, ptep, ~0UL, 0);
|
||||
pte_update(mm, addr, ptep, ~0UL, 0, 0);
|
||||
}
|
||||
|
||||
|
||||
@ -506,7 +508,9 @@ extern int pmdp_set_access_flags(struct vm_area_struct *vma,
|
||||
|
||||
extern unsigned long pmd_hugepage_update(struct mm_struct *mm,
|
||||
unsigned long addr,
|
||||
pmd_t *pmdp, unsigned long clr);
|
||||
pmd_t *pmdp,
|
||||
unsigned long clr,
|
||||
unsigned long set);
|
||||
|
||||
static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
|
||||
unsigned long addr, pmd_t *pmdp)
|
||||
@ -515,7 +519,7 @@ static inline int __pmdp_test_and_clear_young(struct mm_struct *mm,
|
||||
|
||||
if ((pmd_val(*pmdp) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
|
||||
return 0;
|
||||
old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED);
|
||||
old = pmd_hugepage_update(mm, addr, pmdp, _PAGE_ACCESSED, 0);
|
||||
return ((old & _PAGE_ACCESSED) != 0);
|
||||
}
|
||||
|
||||
@ -542,7 +546,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr,
|
||||
if ((pmd_val(*pmdp) & _PAGE_RW) == 0)
|
||||
return;
|
||||
|
||||
pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW);
|
||||
pmd_hugepage_update(mm, addr, pmdp, _PAGE_RW, 0);
|
||||
}
|
||||
|
||||
#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
|
||||
|
||||
@ -510,7 +510,8 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
|
||||
}
|
||||
|
||||
unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
|
||||
pmd_t *pmdp, unsigned long clr)
|
||||
pmd_t *pmdp, unsigned long clr,
|
||||
unsigned long set)
|
||||
{
|
||||
|
||||
unsigned long old, tmp;
|
||||
@ -526,14 +527,15 @@ unsigned long pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
|
||||
andi. %1,%0,%6\n\
|
||||
bne- 1b \n\
|
||||
andc %1,%0,%4 \n\
|
||||
or %1,%1,%7\n\
|
||||
stdcx. %1,0,%3 \n\
|
||||
bne- 1b"
|
||||
: "=&r" (old), "=&r" (tmp), "=m" (*pmdp)
|
||||
: "r" (pmdp), "r" (clr), "m" (*pmdp), "i" (_PAGE_BUSY)
|
||||
: "r" (pmdp), "r" (clr), "m" (*pmdp), "i" (_PAGE_BUSY), "r" (set)
|
||||
: "cc" );
|
||||
#else
|
||||
old = pmd_val(*pmdp);
|
||||
*pmdp = __pmd(old & ~clr);
|
||||
*pmdp = __pmd((old & ~clr) | set);
|
||||
#endif
|
||||
if (old & _PAGE_HASHPTE)
|
||||
hpte_do_hugepage_flush(mm, addr, pmdp);
|
||||
@ -708,7 +710,7 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
|
||||
void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
|
||||
pmd_t *pmdp)
|
||||
{
|
||||
pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT);
|
||||
pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -835,7 +837,7 @@ pmd_t pmdp_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long old;
|
||||
pgtable_t *pgtable_slot;
|
||||
|
||||
old = pmd_hugepage_update(mm, addr, pmdp, ~0UL);
|
||||
old = pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
|
||||
old_pmd = __pmd(old);
|
||||
/*
|
||||
* We have pmd == none and we are holding page_table_lock.
|
||||
|
||||
@ -78,7 +78,7 @@ static void hpte_flush_range(struct mm_struct *mm, unsigned long addr,
|
||||
pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
|
||||
arch_enter_lazy_mmu_mode();
|
||||
for (; npages > 0; --npages) {
|
||||
pte_update(mm, addr, pte, 0, 0);
|
||||
pte_update(mm, addr, pte, 0, 0, 0);
|
||||
addr += PAGE_SIZE;
|
||||
++pte;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user