iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
baed14de78
linux/arch/powerpc/mm/nohash/tlb_low.S
Christophe Leroy a97dd9e2f7 powerpc/fsl_booke: Enable reloading of TLBCAM without switching to AS1 
Avoid switching to AS1 when reloading TLBCAM after init for
STRICT_KERNEL_RWX.

When we setup AS1 we expect the entire accessible memory to be mapped
through one entry, this is not the case anymore at the end of init.

We are not changing the size of TLBCAMs, only flags, so no need to
switch to AS1.

So change loadcam_multi() to not switch to AS1 when the given
temporary tlb entry in 0.

Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/a9d517fbfbc940f56103c46b323f6eb8f4485571.1634292136.git.christophe.leroy@csgroup.eu
2021-10-22 15:22:03 +11:00

473 lines
9.6 KiB
ArmAsm
Raw Blame History

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* This file contains low-level functions for performing various
* types of TLB invalidations on various processors with no hash
* table.
*
* This file implements the following functions for all no-hash
* processors. Some aren't implemented for some variants. Some
* are inline in tlbflush.h
*
* - tlbil_va
* - tlbil_pid
* - tlbil_all
* - tlbivax_bcast
*
* Code mostly moved over from misc_32.S
*
* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
*
* Partially rewritten by Cort Dougan (cort@cs.nmt.edu)
* Paul Mackerras, Kumar Gala and Benjamin Herrenschmidt.
*/
#include <asm/reg.h>
#include <asm/page.h>
#include <asm/cputable.h>
#include <asm/mmu.h>
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/processor.h>
#include <asm/bug.h>
#include <asm/asm-compat.h>
#include <asm/feature-fixups.h>
#if defined(CONFIG_40x)
/*
* 40x implementation needs only tlbil_va
*/
_GLOBAL(__tlbil_va)
/* We run the search with interrupts disabled because we have to change
* the PID and I don't want to preempt when that happens.
*/
mfmsr r5
mfspr r6,SPRN_PID
wrteei 0
mtspr SPRN_PID,r4
tlbsx. r3, 0, r3
mtspr SPRN_PID,r6
wrtee r5
bne 1f
sync
/* There are only 64 TLB entries, so r3 < 64, which means bit 25 is
* clear. Since 25 is the V bit in the TLB_TAG, loading this value
* will invalidate the TLB entry. */
tlbwe r3, r3, TLB_TAG
isync
1: blr
#elif defined(CONFIG_PPC_8xx)
/*
* Nothing to do for 8xx, everything is inline
*/
#elif defined(CONFIG_44x) /* Includes 47x */
/*
* 440 implementation uses tlbsx/we for tlbil_va and a full sweep
* of the TLB for everything else.
*/
_GLOBAL(__tlbil_va)
mfspr r5,SPRN_MMUCR
mfmsr r10
/*
* We write 16 bits of STID since 47x supports that much, we
* will never be passed out of bounds values on 440 (hopefully)
*/
rlwimi r5,r4,0,16,31
/* We have to run the search with interrupts disabled, otherwise
* an interrupt which causes a TLB miss can clobber the MMUCR
* between the mtspr and the tlbsx.
*
* Critical and Machine Check interrupts take care of saving
* and restoring MMUCR, so only normal interrupts have to be
* taken care of.
*/
wrteei 0
mtspr SPRN_MMUCR,r5
tlbsx. r6,0,r3
bne 10f
sync
#ifndef CONFIG_PPC_47x
/* On 440 There are only 64 TLB entries, so r3 < 64, which means bit
* 22, is clear. Since 22 is the V bit in the TLB_PAGEID, loading this
* value will invalidate the TLB entry.
*/
tlbwe r6,r6,PPC44x_TLB_PAGEID
#else
oris r7,r6,0x8000 /* specify way explicitly */
clrrwi r4,r3,12 /* get an EPN for the hashing with V = 0 */
ori r4,r4,PPC47x_TLBE_SIZE
tlbwe r4,r7,0 /* write it */
#endif /* !CONFIG_PPC_47x */
isync
10: wrtee r10
blr
_GLOBAL(_tlbil_all)
_GLOBAL(_tlbil_pid)
#ifndef CONFIG_PPC_47x
li r3,0
sync
/* Load high watermark */
lis r4,tlb_44x_hwater@ha
lwz r5,tlb_44x_hwater@l(r4)
1: tlbwe r3,r3,PPC44x_TLB_PAGEID
addi r3,r3,1
cmpw 0,r3,r5
ble 1b
isync
blr
#else
/* 476 variant. There's not simple way to do this, hopefully we'll
* try to limit the amount of such full invalidates
*/
mfmsr r11 /* Interrupts off */
wrteei 0
li r3,-1 /* Current set */
lis r10,tlb_47x_boltmap@h
ori r10,r10,tlb_47x_boltmap@l
lis r7,0x8000 /* Specify way explicitly */
b 9f /* For each set */
1: li r9,4 /* Number of ways */
li r4,0 /* Current way */
li r6,0 /* Default entry value 0 */
andi. r0,r8,1 /* Check if way 0 is bolted */
mtctr r9 /* Load way counter */
bne- 3f /* Bolted, skip loading it */
2: /* For each way */
or r5,r3,r4 /* Make way|index for tlbre */
rlwimi r5,r5,16,8,15 /* Copy index into position */
tlbre r6,r5,0 /* Read entry */
3: addis r4,r4,0x2000 /* Next way */
andi. r0,r6,PPC47x_TLB0_VALID /* Valid entry ? */
beq 4f /* Nope, skip it */
rlwimi r7,r5,0,1,2 /* Insert way number */
rlwinm r6,r6,0,21,19 /* Clear V */
tlbwe r6,r7,0 /* Write it */
4: bdnz 2b /* Loop for each way */
srwi r8,r8,1 /* Next boltmap bit */
9: cmpwi cr1,r3,255 /* Last set done ? */
addi r3,r3,1 /* Next set */
beq cr1,1f /* End of loop */
andi. r0,r3,0x1f /* Need to load a new boltmap word ? */
bne 1b /* No, loop */
lwz r8,0(r10) /* Load boltmap entry */
addi r10,r10,4 /* Next word */
b 1b /* Then loop */
1: isync /* Sync shadows */
wrtee r11
blr
#endif /* !CONFIG_PPC_47x */
#ifdef CONFIG_PPC_47x
/*
* _tlbivax_bcast is only on 47x. We don't bother doing a runtime
* check though, it will blow up soon enough if we mistakenly try
* to use it on a 440.
*/
_GLOBAL(_tlbivax_bcast)
mfspr r5,SPRN_MMUCR
mfmsr r10
rlwimi r5,r4,0,16,31
wrteei 0
mtspr SPRN_MMUCR,r5
isync
PPC_TLBIVAX(0, R3)
isync
eieio
tlbsync
BEGIN_FTR_SECTION
b 1f
END_FTR_SECTION_IFSET(CPU_FTR_476_DD2)
sync
wrtee r10
blr
/*
* DD2 HW could hang if in instruction fetch happens before msync completes.
* Touch enough instruction cache lines to ensure cache hits
*/
1: mflr r9
bcl 20,31,$+4
2: mflr r6
li r7,32
PPC_ICBT(0,R6,R7) /* touch next cache line */
add r6,r6,r7
PPC_ICBT(0,R6,R7) /* touch next cache line */
add r6,r6,r7
PPC_ICBT(0,R6,R7) /* touch next cache line */
sync
nop
nop
nop
nop
nop
nop
nop
nop
mtlr r9
wrtee r10
blr
#endif /* CONFIG_PPC_47x */
#elif defined(CONFIG_FSL_BOOKE)
/*
* FSL BookE implementations.
*
* Since feature sections are using _SECTION_ELSE we need
* to have the larger code path before the _SECTION_ELSE
*/
/*
* Flush MMU TLB on the local processor
*/
_GLOBAL(_tlbil_all)
BEGIN_MMU_FTR_SECTION
li r3,(MMUCSR0_TLBFI)@l
mtspr SPRN_MMUCSR0, r3
1:
mfspr r3,SPRN_MMUCSR0
andi. r3,r3,MMUCSR0_TLBFI@l
bne 1b
MMU_FTR_SECTION_ELSE
PPC_TLBILX_ALL(0,R0)
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
msync
isync
blr
_GLOBAL(_tlbil_pid)
BEGIN_MMU_FTR_SECTION
slwi r3,r3,16
mfmsr r10
wrteei 0
mfspr r4,SPRN_MAS6 /* save MAS6 */
mtspr SPRN_MAS6,r3
PPC_TLBILX_PID(0,R0)
mtspr SPRN_MAS6,r4 /* restore MAS6 */
wrtee r10
MMU_FTR_SECTION_ELSE
li r3,(MMUCSR0_TLBFI)@l
mtspr SPRN_MMUCSR0, r3
1:
mfspr r3,SPRN_MMUCSR0
andi. r3,r3,MMUCSR0_TLBFI@l
bne 1b
ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBILX)
msync
isync
blr
/*
* Flush MMU TLB for a particular address, but only on the local processor
* (no broadcast)
*/
_GLOBAL(__tlbil_va)
mfmsr r10
wrteei 0
slwi r4,r4,16
ori r4,r4,(MAS6_ISIZE(BOOK3E_PAGESZ_4K))@l
mtspr SPRN_MAS6,r4 /* assume AS=0 for now */
BEGIN_MMU_FTR_SECTION
tlbsx 0,r3
mfspr r4,SPRN_MAS1 /* check valid */
andis. r3,r4,MAS1_VALID@h
beq 1f
rlwinm r4,r4,0,1,31
mtspr SPRN_MAS1,r4
tlbwe
MMU_FTR_SECTION_ELSE
PPC_TLBILX_VA(0,R3)
ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_USE_TLBILX)
msync
isync
1: wrtee r10
blr
#elif defined(CONFIG_PPC_BOOK3E)
/*
* New Book3E (>= 2.06) implementation
*
* Note: We may be able to get away without the interrupt masking stuff
* if we save/restore MAS6 on exceptions that might modify it
*/
_GLOBAL(_tlbil_pid)
slwi r4,r3,MAS6_SPID_SHIFT
mfmsr r10
wrteei 0
mtspr SPRN_MAS6,r4
PPC_TLBILX_PID(0,R0)
wrtee r10
msync
isync
blr
_GLOBAL(_tlbil_pid_noind)
slwi r4,r3,MAS6_SPID_SHIFT
mfmsr r10
ori r4,r4,MAS6_SIND
wrteei 0
mtspr SPRN_MAS6,r4
PPC_TLBILX_PID(0,R0)
wrtee r10
msync
isync
blr
_GLOBAL(_tlbil_all)
PPC_TLBILX_ALL(0,R0)
msync
isync
blr
_GLOBAL(_tlbil_va)
mfmsr r10
wrteei 0
cmpwi cr0,r6,0
slwi r4,r4,MAS6_SPID_SHIFT
rlwimi r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
beq 1f
rlwimi r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
1: mtspr SPRN_MAS6,r4 /* assume AS=0 for now */
PPC_TLBILX_VA(0,R3)
msync
isync
wrtee r10
blr
_GLOBAL(_tlbivax_bcast)
mfmsr r10
wrteei 0
cmpwi cr0,r6,0
slwi r4,r4,MAS6_SPID_SHIFT
rlwimi r4,r5,MAS6_ISIZE_SHIFT,MAS6_ISIZE_MASK
beq 1f
rlwimi r4,r6,MAS6_SIND_SHIFT,MAS6_SIND
1: mtspr SPRN_MAS6,r4 /* assume AS=0 for now */
PPC_TLBIVAX(0,R3)
eieio
tlbsync
sync
wrtee r10
blr
#else
#error Unsupported processor type !
#endif
#if defined(CONFIG_PPC_FSL_BOOK3E)
/*
* extern void loadcam_entry(unsigned int index)
*
* Load TLBCAM[index] entry in to the L2 CAM MMU
* Must preserve r7, r8, r9, r10, r11, r12
*/
_GLOBAL(loadcam_entry)
mflr r5
LOAD_REG_ADDR_PIC(r4, TLBCAM)
mtlr r5
mulli r5,r3,TLBCAM_SIZE
add r3,r5,r4
lwz r4,TLBCAM_MAS0(r3)
mtspr SPRN_MAS0,r4
lwz r4,TLBCAM_MAS1(r3)
mtspr SPRN_MAS1,r4
PPC_LL r4,TLBCAM_MAS2(r3)
mtspr SPRN_MAS2,r4
lwz r4,TLBCAM_MAS3(r3)
mtspr SPRN_MAS3,r4
BEGIN_MMU_FTR_SECTION
lwz r4,TLBCAM_MAS7(r3)
mtspr SPRN_MAS7,r4
END_MMU_FTR_SECTION_IFSET(MMU_FTR_BIG_PHYS)
isync
tlbwe
isync
blr
/*
* Load multiple TLB entries at once, using an alternate-space
* trampoline so that we don't have to care about whether the same
* TLB entry maps us before and after.
*
* r3 = first entry to write
* r4 = number of entries to write
* r5 = temporary tlb entry (0 means no switch to AS1)
*/
_GLOBAL(loadcam_multi)
mflr r8
/* Don't switch to AS=1 if already there */
mfmsr r11
andi. r11,r11,MSR_IS
bne 10f
mr. r12, r5
beq 10f
/*
* Set up temporary TLB entry that is the same as what we're
* running from, but in AS=1.
*/
bcl 20,31,$+4
1: mflr r6
tlbsx 0,r8
mfspr r6,SPRN_MAS1
ori r6,r6,MAS1_TS
mtspr SPRN_MAS1,r6
mfspr r6,SPRN_MAS0
rlwimi r6,r5,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
mr r7,r5
mtspr SPRN_MAS0,r6
isync
tlbwe
isync
/* Switch to AS=1 */
mfmsr r6
ori r6,r6,MSR_IS|MSR_DS
mtmsr r6
isync
10:
mr r9,r3
add r10,r3,r4
2: bl loadcam_entry
addi r9,r9,1
cmpw r9,r10
mr r3,r9
blt 2b
/* Don't return to AS=0 if we were in AS=1 at function start */
andi. r11,r11,MSR_IS
bne 3f
cmpwi r12, 0
beq 3f
/* Return to AS=0 and clear the temporary entry */
mfmsr r6
rlwinm. r6,r6,0,~(MSR_IS|MSR_DS)
mtmsr r6
isync
li r6,0
mtspr SPRN_MAS1,r6
rlwinm r6,r7,MAS0_ESEL_SHIFT,MAS0_ESEL_MASK
oris r6,r6,MAS0_TLBSEL(1)@h
mtspr SPRN_MAS0,r6
isync
tlbwe
isync
3:
mtlr r8
blr
#endif
Reference in New Issue View Git Blame Copy Permalink