2012-04-10 03:09:27 +04:00
/*
* Frontswap frontend
*
* This code provides the generic " frontend " layer to call a matching
* " backend " driver implementation of frontswap . See
* Documentation / vm / frontswap . txt for more information .
*
* Copyright ( C ) 2009 - 2012 Oracle Corp . All rights reserved .
* Author : Dan Magenheimer
*
* This work is licensed under the terms of the GNU GPL , version 2.
*/
# include <linux/mman.h>
# include <linux/swap.h>
# include <linux/swapops.h>
# include <linux/security.h>
# include <linux/module.h>
# include <linux/debugfs.h>
# include <linux/frontswap.h>
# include <linux/swapfile.h>
/*
* frontswap_ops is set by frontswap_register_ops to contain the pointers
* to the frontswap " backend " implementation functions .
*/
static struct frontswap_ops frontswap_ops __read_mostly ;
/*
* This global enablement flag reduces overhead on systems where frontswap_ops
* has not been registered , so is preferred to the slower alternative : a
* function call that checks a non - global .
*/
bool frontswap_enabled __read_mostly ;
EXPORT_SYMBOL ( frontswap_enabled ) ;
/*
2012-05-15 19:32:15 +04:00
* If enabled , frontswap_store will return failure even on success . As
2012-04-10 03:09:27 +04:00
* a result , the swap subsystem will always write the page to swap , in
* effect converting frontswap into a writethrough cache . In this mode ,
* there is no direct reduction in swap writes , but a frontswap backend
* can unilaterally " reclaim " any pages in use with no data loss , thus
* providing increases control over maximum memory usage due to frontswap .
*/
static bool frontswap_writethrough_enabled __read_mostly ;
# ifdef CONFIG_DEBUG_FS
/*
* Counters available via / sys / kernel / debug / frontswap ( if debugfs is
* properly configured ) . These are for information only so are not protected
* against increment races .
*/
2012-05-15 19:32:15 +04:00
static u64 frontswap_loads ;
static u64 frontswap_succ_stores ;
static u64 frontswap_failed_stores ;
2012-04-10 03:09:27 +04:00
static u64 frontswap_invalidates ;
2012-05-15 19:32:15 +04:00
static inline void inc_frontswap_loads ( void ) {
frontswap_loads + + ;
2012-04-10 03:09:27 +04:00
}
2012-05-15 19:32:15 +04:00
static inline void inc_frontswap_succ_stores ( void ) {
frontswap_succ_stores + + ;
2012-04-10 03:09:27 +04:00
}
2012-05-15 19:32:15 +04:00
static inline void inc_frontswap_failed_stores ( void ) {
frontswap_failed_stores + + ;
2012-04-10 03:09:27 +04:00
}
static inline void inc_frontswap_invalidates ( void ) {
frontswap_invalidates + + ;
}
# else
2012-05-15 19:32:15 +04:00
static inline void inc_frontswap_loads ( void ) { }
static inline void inc_frontswap_succ_stores ( void ) { }
static inline void inc_frontswap_failed_stores ( void ) { }
2012-04-10 03:09:27 +04:00
static inline void inc_frontswap_invalidates ( void ) { }
# endif
/*
* Register operations for frontswap , returning previous thus allowing
* detection of multiple backends and possible nesting .
*/
struct frontswap_ops frontswap_register_ops ( struct frontswap_ops * ops )
{
struct frontswap_ops old = frontswap_ops ;
frontswap_ops = * ops ;
frontswap_enabled = true ;
return old ;
}
EXPORT_SYMBOL ( frontswap_register_ops ) ;
/*
* Enable / disable frontswap writethrough ( see above ) .
*/
void frontswap_writethrough ( bool enable )
{
frontswap_writethrough_enabled = enable ;
}
EXPORT_SYMBOL ( frontswap_writethrough ) ;
/*
* Called when a swap device is swapon ' d .
*/
void __frontswap_init ( unsigned type )
{
struct swap_info_struct * sis = swap_info [ type ] ;
BUG_ON ( sis = = NULL ) ;
if ( sis - > frontswap_map = = NULL )
return ;
2012-06-10 14:51:05 +04:00
frontswap_ops . init ( type ) ;
2012-04-10 03:09:27 +04:00
}
EXPORT_SYMBOL ( __frontswap_init ) ;
2012-06-10 14:51:07 +04:00
static inline void __frontswap_clear ( struct swap_info_struct * sis , pgoff_t offset )
{
frontswap_clear ( sis , offset ) ;
atomic_dec ( & sis - > frontswap_pages ) ;
}
2012-04-10 03:09:27 +04:00
/*
2012-05-15 19:32:15 +04:00
* " Store " data from a page to frontswap and associate it with the page ' s
2012-04-10 03:09:27 +04:00
* swaptype and offset . Page must be locked and in the swap cache .
* If frontswap already contains a page with matching swaptype and
2012-06-16 16:37:48 +04:00
* offset , the frontswap implementation may either overwrite the data and
2012-04-10 03:09:27 +04:00
* return success or invalidate the page from frontswap and return failure .
*/
2012-05-15 19:32:15 +04:00
int __frontswap_store ( struct page * page )
2012-04-10 03:09:27 +04:00
{
int ret = - 1 , dup = 0 ;
swp_entry_t entry = { . val = page_private ( page ) , } ;
int type = swp_type ( entry ) ;
struct swap_info_struct * sis = swap_info [ type ] ;
pgoff_t offset = swp_offset ( entry ) ;
BUG_ON ( ! PageLocked ( page ) ) ;
BUG_ON ( sis = = NULL ) ;
if ( frontswap_test ( sis , offset ) )
dup = 1 ;
2012-06-10 14:50:59 +04:00
ret = frontswap_ops . store ( type , offset , page ) ;
2012-04-10 03:09:27 +04:00
if ( ret = = 0 ) {
frontswap_set ( sis , offset ) ;
2012-05-15 19:32:15 +04:00
inc_frontswap_succ_stores ( ) ;
2012-04-10 03:09:27 +04:00
if ( ! dup )
atomic_inc ( & sis - > frontswap_pages ) ;
2012-06-10 14:51:04 +04:00
} else {
2012-04-10 03:09:27 +04:00
/*
failed dup always results in automatic invalidate of
the ( older ) page from frontswap
*/
2012-05-15 19:32:15 +04:00
inc_frontswap_failed_stores ( ) ;
2012-06-10 14:51:07 +04:00
if ( dup )
__frontswap_clear ( sis , offset ) ;
2012-06-10 14:51:00 +04:00
}
2012-04-10 03:09:27 +04:00
if ( frontswap_writethrough_enabled )
/* report failure so swap also writes to swap device */
ret = - 1 ;
return ret ;
}
2012-05-15 19:32:15 +04:00
EXPORT_SYMBOL ( __frontswap_store ) ;
2012-04-10 03:09:27 +04:00
/*
* " Get " data from frontswap associated with swaptype and offset that were
* specified when the data was put to frontswap and use it to fill the
* specified page with data . Page must be locked and in the swap cache .
*/
2012-05-15 19:32:15 +04:00
int __frontswap_load ( struct page * page )
2012-04-10 03:09:27 +04:00
{
int ret = - 1 ;
swp_entry_t entry = { . val = page_private ( page ) , } ;
int type = swp_type ( entry ) ;
struct swap_info_struct * sis = swap_info [ type ] ;
pgoff_t offset = swp_offset ( entry ) ;
BUG_ON ( ! PageLocked ( page ) ) ;
BUG_ON ( sis = = NULL ) ;
if ( frontswap_test ( sis , offset ) )
2012-06-10 14:50:59 +04:00
ret = frontswap_ops . load ( type , offset , page ) ;
2012-04-10 03:09:27 +04:00
if ( ret = = 0 )
2012-05-15 19:32:15 +04:00
inc_frontswap_loads ( ) ;
2012-04-10 03:09:27 +04:00
return ret ;
}
2012-05-15 19:32:15 +04:00
EXPORT_SYMBOL ( __frontswap_load ) ;
2012-04-10 03:09:27 +04:00
/*
* Invalidate any data from frontswap associated with the specified swaptype
* and offset so that a subsequent " get " will fail .
*/
void __frontswap_invalidate_page ( unsigned type , pgoff_t offset )
{
struct swap_info_struct * sis = swap_info [ type ] ;
BUG_ON ( sis = = NULL ) ;
if ( frontswap_test ( sis , offset ) ) {
2012-06-10 14:50:59 +04:00
frontswap_ops . invalidate_page ( type , offset ) ;
2012-06-10 14:51:07 +04:00
__frontswap_clear ( sis , offset ) ;
2012-04-10 03:09:27 +04:00
inc_frontswap_invalidates ( ) ;
}
}
EXPORT_SYMBOL ( __frontswap_invalidate_page ) ;
/*
* Invalidate all data from frontswap associated with all offsets for the
* specified swaptype .
*/
void __frontswap_invalidate_area ( unsigned type )
{
struct swap_info_struct * sis = swap_info [ type ] ;
BUG_ON ( sis = = NULL ) ;
if ( sis - > frontswap_map = = NULL )
return ;
2012-06-10 14:50:59 +04:00
frontswap_ops . invalidate_area ( type ) ;
2012-04-10 03:09:27 +04:00
atomic_set ( & sis - > frontswap_pages , 0 ) ;
memset ( sis - > frontswap_map , 0 , sis - > max / sizeof ( long ) ) ;
}
EXPORT_SYMBOL ( __frontswap_invalidate_area ) ;
2012-06-10 14:51:01 +04:00
static unsigned long __frontswap_curr_pages ( void )
{
int type ;
unsigned long totalpages = 0 ;
struct swap_info_struct * si = NULL ;
assert_spin_locked ( & swap_lock ) ;
for ( type = swap_list . head ; type > = 0 ; type = si - > next ) {
si = swap_info [ type ] ;
totalpages + = atomic_read ( & si - > frontswap_pages ) ;
}
return totalpages ;
}
2012-06-10 14:51:02 +04:00
static int __frontswap_unuse_pages ( unsigned long total , unsigned long * unused ,
int * swapid )
{
int ret = - EINVAL ;
struct swap_info_struct * si = NULL ;
int si_frontswap_pages ;
unsigned long total_pages_to_unuse = total ;
unsigned long pages = 0 , pages_to_unuse = 0 ;
int type ;
assert_spin_locked ( & swap_lock ) ;
for ( type = swap_list . head ; type > = 0 ; type = si - > next ) {
si = swap_info [ type ] ;
si_frontswap_pages = atomic_read ( & si - > frontswap_pages ) ;
if ( total_pages_to_unuse < si_frontswap_pages ) {
pages = pages_to_unuse = total_pages_to_unuse ;
} else {
pages = si_frontswap_pages ;
pages_to_unuse = 0 ; /* unuse all */
}
/* ensure there is enough RAM to fetch pages from frontswap */
if ( security_vm_enough_memory_mm ( current - > mm , pages ) ) {
ret = - ENOMEM ;
continue ;
}
vm_unacct_memory ( pages ) ;
* unused = pages_to_unuse ;
* swapid = type ;
ret = 0 ;
break ;
}
return ret ;
}
2012-06-10 14:51:03 +04:00
static int __frontswap_shrink ( unsigned long target_pages ,
unsigned long * pages_to_unuse ,
int * type )
{
unsigned long total_pages = 0 , total_pages_to_unuse ;
assert_spin_locked ( & swap_lock ) ;
total_pages = __frontswap_curr_pages ( ) ;
if ( total_pages < = target_pages ) {
/* Nothing to do */
* pages_to_unuse = 0 ;
return 0 ;
}
total_pages_to_unuse = total_pages - target_pages ;
return __frontswap_unuse_pages ( total_pages_to_unuse , pages_to_unuse , type ) ;
}
2012-04-10 03:09:27 +04:00
/*
* Frontswap , like a true swap device , may unnecessarily retain pages
* under certain circumstances ; " shrink " frontswap is essentially a
* " partial swapoff " and works by calling try_to_unuse to attempt to
* unuse enough frontswap pages to attempt to - - subject to memory
* constraints - - reduce the number of pages in frontswap to the
* number given in the parameter target_pages .
*/
void frontswap_shrink ( unsigned long target_pages )
{
2012-06-10 14:51:02 +04:00
unsigned long pages_to_unuse = 0 ;
int type , ret ;
2012-04-10 03:09:27 +04:00
/*
* we don ' t want to hold swap_lock while doing a very
* lengthy try_to_unuse , but swap_list may change
* so restart scan from swap_list . head each time
*/
spin_lock ( & swap_lock ) ;
2012-06-10 14:51:03 +04:00
ret = __frontswap_shrink ( target_pages , & pages_to_unuse , & type ) ;
2012-04-10 03:09:27 +04:00
spin_unlock ( & swap_lock ) ;
2012-06-10 14:51:03 +04:00
if ( ret = = 0 & & pages_to_unuse )
try_to_unuse ( type , true , pages_to_unuse ) ;
2012-04-10 03:09:27 +04:00
return ;
}
EXPORT_SYMBOL ( frontswap_shrink ) ;
/*
* Count and return the number of frontswap pages across all
* swap devices . This is exported so that backend drivers can
* determine current usage without reading debugfs .
*/
unsigned long frontswap_curr_pages ( void )
{
unsigned long totalpages = 0 ;
spin_lock ( & swap_lock ) ;
2012-06-10 14:51:01 +04:00
totalpages = __frontswap_curr_pages ( ) ;
2012-04-10 03:09:27 +04:00
spin_unlock ( & swap_lock ) ;
2012-06-10 14:51:01 +04:00
2012-04-10 03:09:27 +04:00
return totalpages ;
}
EXPORT_SYMBOL ( frontswap_curr_pages ) ;
static int __init init_frontswap ( void )
{
# ifdef CONFIG_DEBUG_FS
struct dentry * root = debugfs_create_dir ( " frontswap " , NULL ) ;
if ( root = = NULL )
return - ENXIO ;
2012-05-15 19:32:15 +04:00
debugfs_create_u64 ( " loads " , S_IRUGO , root , & frontswap_loads ) ;
debugfs_create_u64 ( " succ_stores " , S_IRUGO , root , & frontswap_succ_stores ) ;
debugfs_create_u64 ( " failed_stores " , S_IRUGO , root ,
& frontswap_failed_stores ) ;
2012-04-10 03:09:27 +04:00
debugfs_create_u64 ( " invalidates " , S_IRUGO ,
root , & frontswap_invalidates ) ;
# endif
return 0 ;
}
module_init ( init_frontswap ) ;