2005-04-17 02:20:36 +04:00
/*
* pSeries_lpar . c
* Copyright ( C ) 2001 Todd Inglett , IBM Corporation
*
* pSeries LPAR support .
*
* This program is free software ; you can redistribute it and / or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation ; either version 2 of the License , or
* ( at your option ) any later version .
*
* This program is distributed in the hope that it will be useful ,
* but WITHOUT ANY WARRANTY ; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
* GNU General Public License for more details .
*
* You should have received a copy of the GNU General Public License
* along with this program ; if not , write to the Free Software
* Foundation , Inc . , 59 Temple Place , Suite 330 , Boston , MA 02111 - 1307 USA
*/
2008-04-24 09:13:19 +04:00
/* Enables debugging of low-level hash table routines - careful! */
# undef DEBUG
2005-04-17 02:20:36 +04:00
# include <linux/kernel.h>
# include <linux/dma-mapping.h>
2005-11-23 09:56:06 +03:00
# include <linux/console.h>
2011-05-27 18:46:24 +04:00
# include <linux/export.h>
2014-07-03 09:52:03 +04:00
# include <linux/static_key.h>
2005-04-17 02:20:36 +04:00
# include <asm/processor.h>
# include <asm/mmu.h>
# include <asm/page.h>
# include <asm/pgtable.h>
# include <asm/machdep.h>
# include <asm/mmu_context.h>
# include <asm/iommu.h>
# include <asm/tlbflush.h>
# include <asm/tlb.h>
# include <asm/prom.h>
# include <asm/cputable.h>
2005-11-07 01:49:43 +03:00
# include <asm/udbg.h>
2005-11-07 05:18:13 +03:00
# include <asm/smp.h>
2009-10-26 21:50:29 +03:00
# include <asm/trace.h>
2012-03-15 22:18:00 +04:00
# include <asm/firmware.h>
2013-08-22 13:53:52 +04:00
# include <asm/plpar_wrappers.h>
2005-11-03 07:33:31 +03:00
2008-04-16 07:51:48 +04:00
# include "pseries.h"
2005-04-17 02:20:36 +04:00
2013-06-20 13:00:27 +04:00
/* Flag bits for H_BULK_REMOVE */
# define HBR_REQUEST 0x4000000000000000UL
# define HBR_RESPONSE 0x8000000000000000UL
# define HBR_END 0xc000000000000000UL
# define HBR_AVPN 0x0200000000000000UL
# define HBR_ANDCOND 0x0100000000000000UL
2005-04-17 02:20:36 +04:00
2006-07-19 02:01:28 +04:00
/* in hvCall.S */
2005-04-17 02:20:36 +04:00
EXPORT_SYMBOL ( plpar_hcall ) ;
2006-07-19 02:01:28 +04:00
EXPORT_SYMBOL ( plpar_hcall9 ) ;
2005-04-17 02:20:36 +04:00
EXPORT_SYMBOL ( plpar_hcall_norets ) ;
2006-07-19 02:01:28 +04:00
2005-04-17 02:20:36 +04:00
void vpa_init ( int cpu )
{
int hwcpu = get_hard_smp_processor_id ( cpu ) ;
2006-08-07 10:19:19 +04:00
unsigned long addr ;
2005-04-17 02:20:36 +04:00
long ret ;
powerpc: Account time using timebase rather than PURR
Currently, when CONFIG_VIRT_CPU_ACCOUNTING is enabled, we use the
PURR register for measuring the user and system time used by
processes, as well as other related times such as hardirq and
softirq times. This turns out to be quite confusing for users
because it means that a program will often be measured as taking
less time when run on a multi-threaded processor (SMT2 or SMT4 mode)
than it does when run on a single-threaded processor (ST mode), even
though the program takes longer to finish. The discrepancy is
accounted for as stolen time, which is also confusing, particularly
when there are no other partitions running.
This changes the accounting to use the timebase instead, meaning that
the reported user and system times are the actual number of real-time
seconds that the program was executing on the processor thread,
regardless of which SMT mode the processor is in. Thus a program will
generally show greater user and system times when run on a
multi-threaded processor than on a single-threaded processor.
On pSeries systems on POWER5 or later processors, we measure the
stolen time (time when this partition wasn't running) using the
hypervisor dispatch trace log. We check for new entries in the
log on every entry from user mode and on every transition from
kernel process context to soft or hard IRQ context (i.e. when
account_system_vtime() gets called). So that we can correctly
distinguish time stolen from user time and time stolen from system
time, without having to check the log on every exit to user mode,
we store separate timestamps for exit to user mode and entry from
user mode.
On systems that have a SPURR (POWER6 and POWER7), we read the SPURR
in account_system_vtime() (as before), and then apportion the SPURR
ticks since the last time we read it between scaled user time and
scaled system time according to the relative proportions of user
time and system time over the same interval. This avoids having to
read the SPURR on every kernel entry and exit. On systems that have
PURR but not SPURR (i.e., POWER5), we do the same using the PURR
rather than the SPURR.
This disables the DTL user interface in /sys/debug/kernel/powerpc/dtl
for now since it conflicts with the use of the dispatch trace log
by the time accounting code.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2010-08-26 23:56:43 +04:00
struct paca_struct * pp ;
struct dtl_entry * dtl ;
2005-09-04 02:55:59 +04:00
2013-08-15 09:22:15 +04:00
/*
* The spec says it " may be problematic " if CPU x registers the VPA of
* CPU y . We should never do that , but wail if we ever do .
*/
WARN_ON ( cpu ! = smp_processor_id ( ) ) ;
2005-09-04 02:55:59 +04:00
if ( cpu_has_feature ( CPU_FTR_ALTIVEC ) )
2010-08-13 00:18:15 +04:00
lppaca_of ( cpu ) . vmxregs_in_use = 1 ;
2005-09-04 02:55:59 +04:00
2013-06-28 12:15:18 +04:00
if ( cpu_has_feature ( CPU_FTR_ARCH_207S ) )
lppaca_of ( cpu ) . ebb_regs_in_use = 1 ;
2010-08-13 00:18:15 +04:00
addr = __pa ( & lppaca_of ( cpu ) ) ;
2006-08-07 10:19:19 +04:00
ret = register_vpa ( hwcpu , addr ) ;
2005-04-17 02:20:36 +04:00
2006-08-07 10:19:19 +04:00
if ( ret ) {
2011-07-25 05:46:33 +04:00
pr_err ( " WARNING: VPA registration for cpu %d (hw %d) of area "
" %lx failed with %ld \n " , cpu , hwcpu , addr , ret ) ;
2006-08-07 10:19:19 +04:00
return ;
}
/*
* PAPR says this feature is SLB - Buffer but firmware never
* reports that . All SPLPAR support SLB shadow buffer .
*/
2013-12-05 07:31:08 +04:00
addr = __pa ( paca [ cpu ] . slb_shadow_ptr ) ;
2006-08-07 10:19:19 +04:00
if ( firmware_has_feature ( FW_FEATURE_SPLPAR ) ) {
ret = register_slb_shadow ( hwcpu , addr ) ;
if ( ret )
2011-07-25 05:46:33 +04:00
pr_err ( " WARNING: SLB shadow buffer registration for "
" cpu %d (hw %d) of area %lx failed with %ld \n " ,
cpu , hwcpu , addr , ret ) ;
2006-08-07 10:19:19 +04:00
}
powerpc: Account time using timebase rather than PURR
Currently, when CONFIG_VIRT_CPU_ACCOUNTING is enabled, we use the
PURR register for measuring the user and system time used by
processes, as well as other related times such as hardirq and
softirq times. This turns out to be quite confusing for users
because it means that a program will often be measured as taking
less time when run on a multi-threaded processor (SMT2 or SMT4 mode)
than it does when run on a single-threaded processor (ST mode), even
though the program takes longer to finish. The discrepancy is
accounted for as stolen time, which is also confusing, particularly
when there are no other partitions running.
This changes the accounting to use the timebase instead, meaning that
the reported user and system times are the actual number of real-time
seconds that the program was executing on the processor thread,
regardless of which SMT mode the processor is in. Thus a program will
generally show greater user and system times when run on a
multi-threaded processor than on a single-threaded processor.
On pSeries systems on POWER5 or later processors, we measure the
stolen time (time when this partition wasn't running) using the
hypervisor dispatch trace log. We check for new entries in the
log on every entry from user mode and on every transition from
kernel process context to soft or hard IRQ context (i.e. when
account_system_vtime() gets called). So that we can correctly
distinguish time stolen from user time and time stolen from system
time, without having to check the log on every exit to user mode,
we store separate timestamps for exit to user mode and entry from
user mode.
On systems that have a SPURR (POWER6 and POWER7), we read the SPURR
in account_system_vtime() (as before), and then apportion the SPURR
ticks since the last time we read it between scaled user time and
scaled system time according to the relative proportions of user
time and system time over the same interval. This avoids having to
read the SPURR on every kernel entry and exit. On systems that have
PURR but not SPURR (i.e., POWER5), we do the same using the PURR
rather than the SPURR.
This disables the DTL user interface in /sys/debug/kernel/powerpc/dtl
for now since it conflicts with the use of the dispatch trace log
by the time accounting code.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2010-08-26 23:56:43 +04:00
/*
* Register dispatch trace log , if one has been allocated .
*/
pp = & paca [ cpu ] ;
dtl = pp - > dispatch_log ;
if ( dtl ) {
pp - > dtl_ridx = 0 ;
pp - > dtl_curr = dtl ;
lppaca_of ( cpu ) . dtl_idx = 0 ;
/* hypervisor reads buffer length from this field */
2013-08-06 20:01:46 +04:00
dtl - > enqueue_to_dispatch_time = cpu_to_be32 ( DISPATCH_LOG_BYTES ) ;
powerpc: Account time using timebase rather than PURR
Currently, when CONFIG_VIRT_CPU_ACCOUNTING is enabled, we use the
PURR register for measuring the user and system time used by
processes, as well as other related times such as hardirq and
softirq times. This turns out to be quite confusing for users
because it means that a program will often be measured as taking
less time when run on a multi-threaded processor (SMT2 or SMT4 mode)
than it does when run on a single-threaded processor (ST mode), even
though the program takes longer to finish. The discrepancy is
accounted for as stolen time, which is also confusing, particularly
when there are no other partitions running.
This changes the accounting to use the timebase instead, meaning that
the reported user and system times are the actual number of real-time
seconds that the program was executing on the processor thread,
regardless of which SMT mode the processor is in. Thus a program will
generally show greater user and system times when run on a
multi-threaded processor than on a single-threaded processor.
On pSeries systems on POWER5 or later processors, we measure the
stolen time (time when this partition wasn't running) using the
hypervisor dispatch trace log. We check for new entries in the
log on every entry from user mode and on every transition from
kernel process context to soft or hard IRQ context (i.e. when
account_system_vtime() gets called). So that we can correctly
distinguish time stolen from user time and time stolen from system
time, without having to check the log on every exit to user mode,
we store separate timestamps for exit to user mode and entry from
user mode.
On systems that have a SPURR (POWER6 and POWER7), we read the SPURR
in account_system_vtime() (as before), and then apportion the SPURR
ticks since the last time we read it between scaled user time and
scaled system time according to the relative proportions of user
time and system time over the same interval. This avoids having to
read the SPURR on every kernel entry and exit. On systems that have
PURR but not SPURR (i.e., POWER5), we do the same using the PURR
rather than the SPURR.
This disables the DTL user interface in /sys/debug/kernel/powerpc/dtl
for now since it conflicts with the use of the dispatch trace log
by the time accounting code.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2010-08-26 23:56:43 +04:00
ret = register_dtl ( hwcpu , __pa ( dtl ) ) ;
if ( ret )
2011-07-25 05:46:33 +04:00
pr_err ( " WARNING: DTL registration of cpu %d (hw %d) "
" failed with %ld \n " , smp_processor_id ( ) ,
hwcpu , ret ) ;
powerpc: Account time using timebase rather than PURR
Currently, when CONFIG_VIRT_CPU_ACCOUNTING is enabled, we use the
PURR register for measuring the user and system time used by
processes, as well as other related times such as hardirq and
softirq times. This turns out to be quite confusing for users
because it means that a program will often be measured as taking
less time when run on a multi-threaded processor (SMT2 or SMT4 mode)
than it does when run on a single-threaded processor (ST mode), even
though the program takes longer to finish. The discrepancy is
accounted for as stolen time, which is also confusing, particularly
when there are no other partitions running.
This changes the accounting to use the timebase instead, meaning that
the reported user and system times are the actual number of real-time
seconds that the program was executing on the processor thread,
regardless of which SMT mode the processor is in. Thus a program will
generally show greater user and system times when run on a
multi-threaded processor than on a single-threaded processor.
On pSeries systems on POWER5 or later processors, we measure the
stolen time (time when this partition wasn't running) using the
hypervisor dispatch trace log. We check for new entries in the
log on every entry from user mode and on every transition from
kernel process context to soft or hard IRQ context (i.e. when
account_system_vtime() gets called). So that we can correctly
distinguish time stolen from user time and time stolen from system
time, without having to check the log on every exit to user mode,
we store separate timestamps for exit to user mode and entry from
user mode.
On systems that have a SPURR (POWER6 and POWER7), we read the SPURR
in account_system_vtime() (as before), and then apportion the SPURR
ticks since the last time we read it between scaled user time and
scaled system time according to the relative proportions of user
time and system time over the same interval. This avoids having to
read the SPURR on every kernel entry and exit. On systems that have
PURR but not SPURR (i.e., POWER5), we do the same using the PURR
rather than the SPURR.
This disables the DTL user interface in /sys/debug/kernel/powerpc/dtl
for now since it conflicts with the use of the dispatch trace log
by the time accounting code.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
2010-08-26 23:56:43 +04:00
lppaca_of ( cpu ) . dtl_enable_mask = 2 ;
}
2005-04-17 02:20:36 +04:00
}
2006-10-05 22:35:10 +04:00
static long pSeries_lpar_hpte_insert ( unsigned long hpte_group ,
2012-09-10 06:52:50 +04:00
unsigned long vpn , unsigned long pa ,
unsigned long rflags , unsigned long vflags ,
2013-04-28 13:37:35 +04:00
int psize , int apsize , int ssize )
2005-04-17 02:20:36 +04:00
{
unsigned long lpar_rc ;
unsigned long flags ;
unsigned long slot ;
2005-07-13 12:11:42 +04:00
unsigned long hpte_v , hpte_r ;
2005-04-17 02:20:36 +04:00
2005-11-07 03:06:55 +03:00
if ( ! ( vflags & HPTE_V_BOLTED ) )
2012-09-10 06:52:50 +04:00
pr_devel ( " hpte_insert(group=%lx, vpn=%016lx, "
" pa=%016lx, rflags=%lx, vflags=%lx, psize=%d) \n " ,
hpte_group , vpn , pa , rflags , vflags , psize ) ;
2005-11-07 03:06:55 +03:00
2013-04-28 13:37:35 +04:00
hpte_v = hpte_encode_v ( vpn , psize , apsize , ssize ) | vflags | HPTE_V_VALID ;
hpte_r = hpte_encode_r ( pa , psize , apsize ) | rflags ;
2005-11-07 03:06:55 +03:00
if ( ! ( vflags & HPTE_V_BOLTED ) )
2009-06-17 22:13:50 +04:00
pr_devel ( " hpte_v=%016lx, hpte_r=%016lx \n " , hpte_v , hpte_r ) ;
2005-11-07 03:06:55 +03:00
2005-04-17 02:20:36 +04:00
/* Now fill in the actual HPTE */
/* Set CEC cookie to 0 */
/* Zero page = 0 */
/* I-cache Invalidate = 0 */
/* I-cache synchronize = 0 */
/* Exact = 0 */
flags = 0 ;
2005-11-07 03:06:55 +03:00
/* Make pHyp happy */
2013-07-23 15:28:03 +04:00
if ( ( rflags & _PAGE_NO_CACHE ) & & ! ( rflags & _PAGE_WRITETHRU ) )
2013-11-18 13:28:09 +04:00
hpte_r & = ~ HPTE_R_M ;
2011-05-04 10:01:20 +04:00
if ( firmware_has_feature ( FW_FEATURE_XCMO ) & & ! ( hpte_r & HPTE_R_N ) )
flags | = H_COALESCE_CAND ;
2005-04-17 02:20:36 +04:00
2006-07-19 02:01:28 +04:00
lpar_rc = plpar_pte_enter ( flags , hpte_group , hpte_v , hpte_r , & slot ) ;
2006-03-30 16:49:40 +04:00
if ( unlikely ( lpar_rc = = H_PTEG_FULL ) ) {
2005-11-07 03:06:55 +03:00
if ( ! ( vflags & HPTE_V_BOLTED ) )
2009-06-17 22:13:50 +04:00
pr_devel ( " full \n " ) ;
2005-04-17 02:20:36 +04:00
return - 1 ;
2005-11-07 03:06:55 +03:00
}
2005-04-17 02:20:36 +04:00
/*
* Since we try and ioremap PHBs we don ' t own , the pte insert
* will fail . However we must catch the failure in hash_page
* or we will loop forever , so return - 2 in this case .
*/
2006-03-30 16:49:40 +04:00
if ( unlikely ( lpar_rc ! = H_SUCCESS ) ) {
2005-11-07 03:06:55 +03:00
if ( ! ( vflags & HPTE_V_BOLTED ) )
2013-04-28 13:37:25 +04:00
pr_devel ( " lpar err %ld \n " , lpar_rc ) ;
2005-04-17 02:20:36 +04:00
return - 2 ;
2005-11-07 03:06:55 +03:00
}
if ( ! ( vflags & HPTE_V_BOLTED ) )
2009-06-17 22:13:50 +04:00
pr_devel ( " -> slot: %lu \n " , slot & 7 ) ;
2005-04-17 02:20:36 +04:00
/* Because of iSeries, we have to pass down the secondary
* bucket bit here as well
*/
2005-07-13 12:11:42 +04:00
return ( slot & 7 ) | ( ! ! ( vflags & HPTE_V_SECONDARY ) < < 3 ) ;
2005-04-17 02:20:36 +04:00
}
static DEFINE_SPINLOCK ( pSeries_lpar_tlbie_lock ) ;
static long pSeries_lpar_hpte_remove ( unsigned long hpte_group )
{
unsigned long slot_offset ;
unsigned long lpar_rc ;
int i ;
unsigned long dummy1 , dummy2 ;
/* pick a random slot to start at */
slot_offset = mftb ( ) & 0x7 ;
for ( i = 0 ; i < HPTES_PER_GROUP ; i + + ) {
/* don't remove a bolted entry */
lpar_rc = plpar_pte_remove ( H_ANDCOND , hpte_group + slot_offset ,
( 0x1UL < < 4 ) , & dummy1 , & dummy2 ) ;
2006-03-30 16:49:40 +04:00
if ( lpar_rc = = H_SUCCESS )
2005-04-17 02:20:36 +04:00
return i ;
2013-04-05 14:41:40 +04:00
/*
* The test for adjunct partition is performed before the
* ANDCOND test . H_RESOURCE may be returned , so we need to
* check for that as well .
*/
BUG_ON ( lpar_rc ! = H_NOT_FOUND & & lpar_rc ! = H_RESOURCE ) ;
2005-04-17 02:20:36 +04:00
slot_offset + + ;
slot_offset & = 0x7 ;
}
return - 1 ;
}
static void pSeries_lpar_hptab_clear ( void )
{
unsigned long size_bytes = 1UL < < ppc64_pft_size ;
unsigned long hpte_count = size_bytes > > 4 ;
2010-05-11 00:28:26 +04:00
struct {
unsigned long pteh ;
unsigned long ptel ;
} ptes [ 4 ] ;
2007-06-14 09:31:34 +04:00
long lpar_rc ;
2011-07-26 22:15:03 +04:00
unsigned long i , j ;
2010-05-11 00:28:26 +04:00
/* Read in batches of 4,
* invalidate only valid entries not in the VRMA
* hpte_count will be a multiple of 4
*/
for ( i = 0 ; i < hpte_count ; i + = 4 ) {
lpar_rc = plpar_pte_read_4_raw ( 0 , i , ( void * ) ptes ) ;
if ( lpar_rc ! = H_SUCCESS )
continue ;
for ( j = 0 ; j < 4 ; j + + ) {
if ( ( ptes [ j ] . pteh & HPTE_V_VRMA_MASK ) = =
HPTE_V_VRMA_MASK )
continue ;
if ( ptes [ j ] . pteh & HPTE_V_VALID )
plpar_pte_remove_raw ( 0 , i + j , 0 ,
& ( ptes [ j ] . pteh ) , & ( ptes [ j ] . ptel ) ) ;
2007-06-14 09:31:34 +04:00
}
}
2013-11-20 15:14:59 +04:00
# ifdef __LITTLE_ENDIAN__
/* Reset exceptions to big endian */
if ( firmware_has_feature ( FW_FEATURE_SET_MODE ) ) {
long rc ;
rc = pseries_big_endian_exceptions ( ) ;
/*
* At this point it is unlikely panic ( ) will get anything
* out to the user , but at least this will stop us from
* continuing on further and creating an even more
* difficult to debug situation .
*/
if ( rc )
panic ( " Could not enable big endian exceptions " ) ;
}
# endif
2005-04-17 02:20:36 +04:00
}
/*
* NOTE : for updatepp ops we are fortunate that the linux " newpp " bits and
* the low 3 bits of flags happen to line up . So no transform is needed .
* We can probably optimize here and assume the high bits of newpp are
* already zero . For now I am paranoid .
*/
2005-11-07 03:06:55 +03:00
static long pSeries_lpar_hpte_updatepp ( unsigned long slot ,
unsigned long newpp ,
2012-09-10 06:52:50 +04:00
unsigned long vpn ,
2013-06-20 13:00:13 +04:00
int psize , int apsize ,
int ssize , int local )
2005-04-17 02:20:36 +04:00
{
unsigned long lpar_rc ;
unsigned long flags = ( newpp & 7 ) | H_AVPN ;
2005-11-07 03:06:55 +03:00
unsigned long want_v ;
2005-04-17 02:20:36 +04:00
2012-09-10 06:52:50 +04:00
want_v = hpte_encode_avpn ( vpn , psize , ssize ) ;
2005-04-17 02:20:36 +04:00
2009-06-17 22:13:50 +04:00
pr_devel ( " update: avpnv=%016lx, hash=%016lx, f=%lx, psize: %d ... " ,
2008-04-24 09:13:19 +04:00
want_v , slot , flags , psize ) ;
2005-04-17 02:20:36 +04:00
2007-10-11 14:37:10 +04:00
lpar_rc = plpar_pte_protect ( flags , slot , want_v ) ;
2005-11-07 03:06:55 +03:00
2006-03-30 16:49:40 +04:00
if ( lpar_rc = = H_NOT_FOUND ) {
2009-06-17 22:13:50 +04:00
pr_devel ( " not found ! \n " ) ;
2005-04-17 02:20:36 +04:00
return - 1 ;
2005-11-07 03:06:55 +03:00
}
2009-06-17 22:13:50 +04:00
pr_devel ( " ok \n " ) ;
2005-04-17 02:20:36 +04:00
2006-03-30 16:49:40 +04:00
BUG_ON ( lpar_rc ! = H_SUCCESS ) ;
2005-04-17 02:20:36 +04:00
return 0 ;
}
static unsigned long pSeries_lpar_hpte_getword0 ( unsigned long slot )
{
unsigned long dword0 ;
unsigned long lpar_rc ;
unsigned long dummy_word1 ;
unsigned long flags ;
/* Read 1 pte at a time */
/* Do not need RPN to logical page translation */
/* No cross CEC PFT access */
flags = 0 ;
lpar_rc = plpar_pte_read ( flags , slot , & dword0 , & dummy_word1 ) ;
2006-03-30 16:49:40 +04:00
BUG_ON ( lpar_rc ! = H_SUCCESS ) ;
2005-04-17 02:20:36 +04:00
return dword0 ;
}
2012-09-10 06:52:50 +04:00
static long pSeries_lpar_hpte_find ( unsigned long vpn , int psize , int ssize )
2005-04-17 02:20:36 +04:00
{
unsigned long hash ;
2007-10-11 14:37:10 +04:00
unsigned long i ;
2005-04-17 02:20:36 +04:00
long slot ;
2005-11-07 03:06:55 +03:00
unsigned long want_v , hpte_v ;
2005-04-17 02:20:36 +04:00
2012-09-10 06:52:50 +04:00
hash = hpt_hash ( vpn , mmu_psize_defs [ psize ] . shift , ssize ) ;
want_v = hpte_encode_avpn ( vpn , psize , ssize ) ;
2007-10-11 14:37:10 +04:00
/* Bolted entries are always in the primary group */
slot = ( hash & htab_hash_mask ) * HPTES_PER_GROUP ;
for ( i = 0 ; i < HPTES_PER_GROUP ; i + + ) {
hpte_v = pSeries_lpar_hpte_getword0 ( slot ) ;
if ( HPTE_V_COMPARE ( hpte_v , want_v ) & & ( hpte_v & HPTE_V_VALID ) )
/* HPTE matches */
return slot ;
+ + slot ;
2005-04-17 02:20:36 +04:00
}
return - 1 ;
}
static void pSeries_lpar_hpte_updateboltedpp ( unsigned long newpp ,
2005-11-07 03:06:55 +03:00
unsigned long ea ,
2007-10-11 14:37:10 +04:00
int psize , int ssize )
2005-04-17 02:20:36 +04:00
{
2012-09-10 06:52:50 +04:00
unsigned long vpn ;
unsigned long lpar_rc , slot , vsid , flags ;
2005-04-17 02:20:36 +04:00
2007-10-11 14:37:10 +04:00
vsid = get_kernel_vsid ( ea , ssize ) ;
2012-09-10 06:52:50 +04:00
vpn = hpt_vpn ( ea , vsid , ssize ) ;
2005-04-17 02:20:36 +04:00
2012-09-10 06:52:50 +04:00
slot = pSeries_lpar_hpte_find ( vpn , psize , ssize ) ;
2005-04-17 02:20:36 +04:00
BUG_ON ( slot = = - 1 ) ;
flags = newpp & 7 ;
lpar_rc = plpar_pte_protect ( flags , slot , 0 ) ;
2006-03-30 16:49:40 +04:00
BUG_ON ( lpar_rc ! = H_SUCCESS ) ;
2005-04-17 02:20:36 +04:00
}
2012-09-10 06:52:50 +04:00
static void pSeries_lpar_hpte_invalidate ( unsigned long slot , unsigned long vpn ,
2013-06-20 13:00:13 +04:00
int psize , int apsize ,
int ssize , int local )
2005-04-17 02:20:36 +04:00
{
2005-11-07 03:06:55 +03:00
unsigned long want_v ;
2005-04-17 02:20:36 +04:00
unsigned long lpar_rc ;
unsigned long dummy1 , dummy2 ;
2012-09-10 06:52:50 +04:00
pr_devel ( " inval : slot=%lx, vpn=%016lx, psize: %d, local: %d \n " ,
slot , vpn , psize , local ) ;
2005-04-17 02:20:36 +04:00
2012-09-10 06:52:50 +04:00
want_v = hpte_encode_avpn ( vpn , psize , ssize ) ;
2007-10-11 14:37:10 +04:00
lpar_rc = plpar_pte_remove ( H_AVPN , slot , want_v , & dummy1 , & dummy2 ) ;
2006-03-30 16:49:40 +04:00
if ( lpar_rc = = H_NOT_FOUND )
2005-04-17 02:20:36 +04:00
return ;
2006-03-30 16:49:40 +04:00
BUG_ON ( lpar_rc ! = H_SUCCESS ) ;
2005-04-17 02:20:36 +04:00
}
2013-06-20 13:00:27 +04:00
/*
* Limit iterations holding pSeries_lpar_tlbie_lock to 3. We also need
* to make sure that we avoid bouncing the hypervisor tlbie lock .
*/
# define PPC64_HUGE_HPTE_BATCH 12
static void __pSeries_lpar_hugepage_invalidate ( unsigned long * slot ,
unsigned long * vpn , int count ,
int psize , int ssize )
{
unsigned long param [ 8 ] ;
int i = 0 , pix = 0 , rc ;
unsigned long flags = 0 ;
int lock_tlbie = ! mmu_has_feature ( MMU_FTR_LOCKLESS_TLBIE ) ;
if ( lock_tlbie )
spin_lock_irqsave ( & pSeries_lpar_tlbie_lock , flags ) ;
for ( i = 0 ; i < count ; i + + ) {
if ( ! firmware_has_feature ( FW_FEATURE_BULK_REMOVE ) ) {
pSeries_lpar_hpte_invalidate ( slot [ i ] , vpn [ i ] , psize , 0 ,
ssize , 0 ) ;
} else {
param [ pix ] = HBR_REQUEST | HBR_AVPN | slot [ i ] ;
param [ pix + 1 ] = hpte_encode_avpn ( vpn [ i ] , psize , ssize ) ;
pix + = 2 ;
if ( pix = = 8 ) {
rc = plpar_hcall9 ( H_BULK_REMOVE , param ,
param [ 0 ] , param [ 1 ] , param [ 2 ] ,
param [ 3 ] , param [ 4 ] , param [ 5 ] ,
param [ 6 ] , param [ 7 ] ) ;
BUG_ON ( rc ! = H_SUCCESS ) ;
pix = 0 ;
}
}
}
if ( pix ) {
param [ pix ] = HBR_END ;
rc = plpar_hcall9 ( H_BULK_REMOVE , param , param [ 0 ] , param [ 1 ] ,
param [ 2 ] , param [ 3 ] , param [ 4 ] , param [ 5 ] ,
param [ 6 ] , param [ 7 ] ) ;
BUG_ON ( rc ! = H_SUCCESS ) ;
}
if ( lock_tlbie )
spin_unlock_irqrestore ( & pSeries_lpar_tlbie_lock , flags ) ;
}
2014-08-13 11:01:58 +04:00
static void pSeries_lpar_hugepage_invalidate ( unsigned long vsid ,
unsigned long addr ,
unsigned char * hpte_slot_array ,
int psize , int ssize )
2013-06-20 13:00:27 +04:00
{
2014-08-13 11:01:58 +04:00
int i , index = 0 ;
2013-06-20 13:00:27 +04:00
unsigned long s_addr = addr ;
unsigned int max_hpte_count , valid ;
unsigned long vpn_array [ PPC64_HUGE_HPTE_BATCH ] ;
unsigned long slot_array [ PPC64_HUGE_HPTE_BATCH ] ;
2014-08-13 11:01:58 +04:00
unsigned long shift , hidx , vpn = 0 , hash , slot ;
2013-06-20 13:00:27 +04:00
shift = mmu_psize_defs [ psize ] . shift ;
max_hpte_count = 1U < < ( PMD_SHIFT - shift ) ;
for ( i = 0 ; i < max_hpte_count ; i + + ) {
valid = hpte_valid ( hpte_slot_array , i ) ;
if ( ! valid )
continue ;
hidx = hpte_hash_index ( hpte_slot_array , i ) ;
/* get the vpn */
addr = s_addr + ( i * ( 1ul < < shift ) ) ;
vpn = hpt_vpn ( addr , vsid , ssize ) ;
hash = hpt_hash ( vpn , shift , ssize ) ;
if ( hidx & _PTEIDX_SECONDARY )
hash = ~ hash ;
slot = ( hash & htab_hash_mask ) * HPTES_PER_GROUP ;
slot + = hidx & _PTEIDX_GROUP_IX ;
slot_array [ index ] = slot ;
vpn_array [ index ] = vpn ;
if ( index = = PPC64_HUGE_HPTE_BATCH - 1 ) {
/*
* Now do a bluk invalidate
*/
__pSeries_lpar_hugepage_invalidate ( slot_array ,
vpn_array ,
PPC64_HUGE_HPTE_BATCH ,
psize , ssize ) ;
index = 0 ;
} else
index + + ;
}
if ( index )
__pSeries_lpar_hugepage_invalidate ( slot_array , vpn_array ,
index , psize , ssize ) ;
}
2008-01-29 01:19:24 +03:00
static void pSeries_lpar_hpte_removebolted ( unsigned long ea ,
int psize , int ssize )
{
2012-09-10 06:52:50 +04:00
unsigned long vpn ;
unsigned long slot , vsid ;
2008-01-29 01:19:24 +03:00
vsid = get_kernel_vsid ( ea , ssize ) ;
2012-09-10 06:52:50 +04:00
vpn = hpt_vpn ( ea , vsid , ssize ) ;
2008-01-29 01:19:24 +03:00
2012-09-10 06:52:50 +04:00
slot = pSeries_lpar_hpte_find ( vpn , psize , ssize ) ;
2008-01-29 01:19:24 +03:00
BUG_ON ( slot = = - 1 ) ;
2013-06-20 13:00:13 +04:00
/*
* lpar doesn ' t use the passed actual page size
*/
pSeries_lpar_hpte_invalidate ( slot , vpn , psize , 0 , ssize , 0 ) ;
2008-01-29 01:19:24 +03:00
}
2005-04-17 02:20:36 +04:00
/*
* Take a spinlock around flushes to avoid bouncing the hypervisor tlbie
* lock .
*/
2006-10-05 22:35:10 +04:00
static void pSeries_lpar_flush_hash_range ( unsigned long number , int local )
2005-04-17 02:20:36 +04:00
{
2012-09-10 06:52:50 +04:00
unsigned long vpn ;
2007-02-06 13:10:31 +03:00
unsigned long i , pix , rc ;
2007-02-08 07:02:35 +03:00
unsigned long flags = 0 ;
2005-04-17 02:20:36 +04:00
struct ppc64_tlb_batch * batch = & __get_cpu_var ( ppc64_tlb_batch ) ;
2011-04-06 23:48:50 +04:00
int lock_tlbie = ! mmu_has_feature ( MMU_FTR_LOCKLESS_TLBIE ) ;
2007-02-06 13:10:31 +03:00
unsigned long param [ 9 ] ;
unsigned long hash , index , shift , hidx , slot ;
real_pte_t pte ;
2007-10-11 14:37:10 +04:00
int psize , ssize ;
2005-04-17 02:20:36 +04:00
if ( lock_tlbie )
spin_lock_irqsave ( & pSeries_lpar_tlbie_lock , flags ) ;
2007-02-06 13:10:31 +03:00
psize = batch - > psize ;
2007-10-11 14:37:10 +04:00
ssize = batch - > ssize ;
2007-02-06 13:10:31 +03:00
pix = 0 ;
for ( i = 0 ; i < number ; i + + ) {
2012-09-10 06:52:50 +04:00
vpn = batch - > vpn [ i ] ;
2007-02-06 13:10:31 +03:00
pte = batch - > pte [ i ] ;
2012-09-10 06:52:50 +04:00
pte_iterate_hashed_subpages ( pte , psize , vpn , index , shift ) {
hash = hpt_hash ( vpn , shift , ssize ) ;
2007-02-06 13:10:31 +03:00
hidx = __rpte_to_hidx ( pte , index ) ;
if ( hidx & _PTEIDX_SECONDARY )
hash = ~ hash ;
slot = ( hash & htab_hash_mask ) * HPTES_PER_GROUP ;
slot + = hidx & _PTEIDX_GROUP_IX ;
2007-02-08 07:02:35 +03:00
if ( ! firmware_has_feature ( FW_FEATURE_BULK_REMOVE ) ) {
2013-06-20 13:00:13 +04:00
/*
* lpar doesn ' t use the passed actual page size
*/
2012-09-10 06:52:50 +04:00
pSeries_lpar_hpte_invalidate ( slot , vpn , psize ,
2013-06-20 13:00:13 +04:00
0 , ssize , local ) ;
2007-02-08 07:02:35 +03:00
} else {
param [ pix ] = HBR_REQUEST | HBR_AVPN | slot ;
2012-09-10 06:52:50 +04:00
param [ pix + 1 ] = hpte_encode_avpn ( vpn , psize ,
2007-10-11 14:37:10 +04:00
ssize ) ;
2007-02-08 07:02:35 +03:00
pix + = 2 ;
if ( pix = = 8 ) {
rc = plpar_hcall9 ( H_BULK_REMOVE , param ,
2007-02-06 13:10:31 +03:00
param [ 0 ] , param [ 1 ] , param [ 2 ] ,
param [ 3 ] , param [ 4 ] , param [ 5 ] ,
param [ 6 ] , param [ 7 ] ) ;
2007-02-08 07:02:35 +03:00
BUG_ON ( rc ! = H_SUCCESS ) ;
pix = 0 ;
}
2007-02-06 13:10:31 +03:00
}
} pte_iterate_hashed_end ( ) ;
}
if ( pix ) {
param [ pix ] = HBR_END ;
rc = plpar_hcall9 ( H_BULK_REMOVE , param , param [ 0 ] , param [ 1 ] ,
param [ 2 ] , param [ 3 ] , param [ 4 ] , param [ 5 ] ,
param [ 6 ] , param [ 7 ] ) ;
BUG_ON ( rc ! = H_SUCCESS ) ;
}
2005-04-17 02:20:36 +04:00
if ( lock_tlbie )
spin_unlock_irqrestore ( & pSeries_lpar_tlbie_lock , flags ) ;
}
2010-09-28 19:33:12 +04:00
static int __init disable_bulk_remove ( char * str )
{
if ( strcmp ( str , " off " ) = = 0 & &
firmware_has_feature ( FW_FEATURE_BULK_REMOVE ) ) {
printk ( KERN_INFO " Disabling BULK_REMOVE firmware feature " ) ;
powerpc_firmware_features & = ~ FW_FEATURE_BULK_REMOVE ;
}
return 1 ;
}
__setup ( " bulk_remove= " , disable_bulk_remove ) ;
2006-06-23 12:16:38 +04:00
void __init hpte_init_lpar ( void )
2005-04-17 02:20:36 +04:00
{
ppc_md . hpte_invalidate = pSeries_lpar_hpte_invalidate ;
ppc_md . hpte_updatepp = pSeries_lpar_hpte_updatepp ;
ppc_md . hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp ;
ppc_md . hpte_insert = pSeries_lpar_hpte_insert ;
ppc_md . hpte_remove = pSeries_lpar_hpte_remove ;
2008-01-29 01:19:24 +03:00
ppc_md . hpte_removebolted = pSeries_lpar_hpte_removebolted ;
2005-04-17 02:20:36 +04:00
ppc_md . flush_hash_range = pSeries_lpar_flush_hash_range ;
ppc_md . hpte_clear_all = pSeries_lpar_hptab_clear ;
2013-06-20 13:00:27 +04:00
ppc_md . hugepage_invalidate = pSeries_lpar_hugepage_invalidate ;
2005-04-17 02:20:36 +04:00
}
2009-04-15 09:55:32 +04:00
# ifdef CONFIG_PPC_SMLPAR
# define CMO_FREE_HINT_DEFAULT 1
static int cmo_free_hint_flag = CMO_FREE_HINT_DEFAULT ;
static int __init cmo_free_hint ( char * str )
{
char * parm ;
parm = strstrip ( str ) ;
if ( strcasecmp ( parm , " no " ) = = 0 | | strcasecmp ( parm , " off " ) = = 0 ) {
printk ( KERN_INFO " cmo_free_hint: CMO free page hinting is not active. \n " ) ;
cmo_free_hint_flag = 0 ;
return 1 ;
}
cmo_free_hint_flag = 1 ;
printk ( KERN_INFO " cmo_free_hint: CMO free page hinting is active. \n " ) ;
if ( strcasecmp ( parm , " yes " ) = = 0 | | strcasecmp ( parm , " on " ) = = 0 )
return 1 ;
return 0 ;
}
__setup ( " cmo_free_hint= " , cmo_free_hint ) ;
static void pSeries_set_page_state ( struct page * page , int order ,
unsigned long state )
{
int i , j ;
unsigned long cmo_page_sz , addr ;
cmo_page_sz = cmo_get_page_size ( ) ;
addr = __pa ( ( unsigned long ) page_address ( page ) ) ;
for ( i = 0 ; i < ( 1 < < order ) ; i + + , addr + = PAGE_SIZE ) {
for ( j = 0 ; j < PAGE_SIZE ; j + = cmo_page_sz )
plpar_hcall_norets ( H_PAGE_INIT , state , addr + j , 0 ) ;
}
}
void arch_free_page ( struct page * page , int order )
{
if ( ! cmo_free_hint_flag | | ! firmware_has_feature ( FW_FEATURE_CMO ) )
return ;
pSeries_set_page_state ( page , order , H_PAGE_SET_UNUSED ) ;
}
EXPORT_SYMBOL ( arch_free_page ) ;
# endif
2009-10-26 21:50:29 +03:00
# ifdef CONFIG_TRACEPOINTS
2014-08-21 06:41:23 +04:00
# ifdef HAVE_JUMP_LABEL
2014-07-03 09:52:03 +04:00
struct static_key hcall_tracepoint_key = STATIC_KEY_INIT ;
void hcall_tracepoint_regfunc ( void )
{
static_key_slow_inc ( & hcall_tracepoint_key ) ;
}
void hcall_tracepoint_unregfunc ( void )
{
static_key_slow_dec ( & hcall_tracepoint_key ) ;
}
# else
2009-10-26 21:50:29 +03:00
/*
* We optimise our hcall path by placing hcall_tracepoint_refcount
* directly in the TOC so we can check if the hcall tracepoints are
* enabled via a single load .
*/
/* NB: reg/unreg are called while guarded with the tracepoints_mutex */
extern long hcall_tracepoint_refcount ;
void hcall_tracepoint_regfunc ( void )
{
hcall_tracepoint_refcount + + ;
}
void hcall_tracepoint_unregfunc ( void )
{
hcall_tracepoint_refcount - - ;
}
2014-07-03 09:52:03 +04:00
# endif
/*
* Since the tracing code might execute hcalls we need to guard against
* recursion . One example of this are spinlocks calling H_YIELD on
* shared processor partitions .
*/
static DEFINE_PER_CPU ( unsigned int , hcall_trace_depth ) ;
2009-10-26 21:50:29 +03:00
2009-10-26 21:51:09 +03:00
void __trace_hcall_entry ( unsigned long opcode , unsigned long * args )
2009-10-26 21:50:29 +03:00
{
2010-10-21 04:52:12 +04:00
unsigned long flags ;
unsigned int * depth ;
2012-01-09 18:29:15 +04:00
/*
* We cannot call tracepoints inside RCU idle regions which
* means we must not trace H_CEDE .
*/
if ( opcode = = H_CEDE )
return ;
2010-10-21 04:52:12 +04:00
local_irq_save ( flags ) ;
depth = & __get_cpu_var ( hcall_trace_depth ) ;
if ( * depth )
goto out ;
( * depth ) + + ;
2011-12-18 20:03:04 +04:00
preempt_disable ( ) ;
2009-10-26 21:51:09 +03:00
trace_hcall_entry ( opcode , args ) ;
2010-10-21 04:52:12 +04:00
( * depth ) - - ;
out :
local_irq_restore ( flags ) ;
2009-10-26 21:50:29 +03:00
}
2009-10-26 21:51:09 +03:00
void __trace_hcall_exit ( long opcode , unsigned long retval ,
unsigned long * retbuf )
2009-10-26 21:50:29 +03:00
{
2010-10-21 04:52:12 +04:00
unsigned long flags ;
unsigned int * depth ;
2012-01-09 18:29:15 +04:00
if ( opcode = = H_CEDE )
return ;
2010-10-21 04:52:12 +04:00
local_irq_save ( flags ) ;
depth = & __get_cpu_var ( hcall_trace_depth ) ;
if ( * depth )
goto out ;
( * depth ) + + ;
2009-10-26 21:51:09 +03:00
trace_hcall_exit ( opcode , retval , retbuf ) ;
2011-12-18 20:03:04 +04:00
preempt_enable ( ) ;
2010-10-21 04:52:12 +04:00
( * depth ) - - ;
out :
local_irq_restore ( flags ) ;
2009-10-26 21:50:29 +03:00
}
# endif
2011-05-04 10:01:20 +04:00
/**
* h_get_mpp
* H_GET_MPP hcall returns info in 7 parms
*/
int h_get_mpp ( struct hvcall_mpp_data * mpp_data )
{
int rc ;
unsigned long retbuf [ PLPAR_HCALL9_BUFSIZE ] ;
rc = plpar_hcall9 ( H_GET_MPP , retbuf ) ;
mpp_data - > entitled_mem = retbuf [ 0 ] ;
mpp_data - > mapped_mem = retbuf [ 1 ] ;
mpp_data - > group_num = ( retbuf [ 2 ] > > 2 * 8 ) & 0xffff ;
mpp_data - > pool_num = retbuf [ 2 ] & 0xffff ;
mpp_data - > mem_weight = ( retbuf [ 3 ] > > 7 * 8 ) & 0xff ;
mpp_data - > unallocated_mem_weight = ( retbuf [ 3 ] > > 6 * 8 ) & 0xff ;
2013-08-06 20:01:24 +04:00
mpp_data - > unallocated_entitlement = retbuf [ 3 ] & 0xffffffffffffUL ;
2011-05-04 10:01:20 +04:00
mpp_data - > pool_size = retbuf [ 4 ] ;
mpp_data - > loan_request = retbuf [ 5 ] ;
mpp_data - > backing_mem = retbuf [ 6 ] ;
return rc ;
}
EXPORT_SYMBOL ( h_get_mpp ) ;
int h_get_mpp_x ( struct hvcall_mpp_x_data * mpp_x_data )
{
int rc ;
unsigned long retbuf [ PLPAR_HCALL9_BUFSIZE ] = { 0 } ;
rc = plpar_hcall9 ( H_GET_MPP_X , retbuf ) ;
mpp_x_data - > coalesced_bytes = retbuf [ 0 ] ;
mpp_x_data - > pool_coalesced_bytes = retbuf [ 1 ] ;
mpp_x_data - > pool_purr_cycles = retbuf [ 2 ] ;
mpp_x_data - > pool_spurr_cycles = retbuf [ 3 ] ;
return rc ;
}
