2018-06-06 05:42:14 +03:00
// SPDX-License-Identifier: GPL-2.0
2005-04-17 02:20:36 +04:00
/*
2005-11-02 06:58:39 +03:00
* Copyright ( c ) 2000 - 2005 Silicon Graphics , Inc .
* All Rights Reserved .
2005-04-17 02:20:36 +04:00
*/
# include "xfs.h"
2010-02-15 12:44:46 +03:00
# include "xfs_fs.h"
2013-10-23 03:36:05 +04:00
# include "xfs_shared.h"
2013-10-23 03:51:50 +04:00
# include "xfs_format.h"
2013-10-23 03:50:10 +04:00
# include "xfs_log_format.h"
# include "xfs_trans_resv.h"
2005-04-17 02:20:36 +04:00
# include "xfs_mount.h"
# include "xfs_inode.h"
2013-10-23 03:50:10 +04:00
# include "xfs_trans.h"
2010-02-15 12:44:48 +03:00
# include "xfs_inode_item.h"
2010-02-15 12:44:46 +03:00
# include "xfs_bmap.h"
2013-08-12 14:49:45 +04:00
# include "xfs_bmap_util.h"
2013-08-12 14:49:37 +04:00
# include "xfs_dir2.h"
2013-08-12 14:49:45 +04:00
# include "xfs_dir2_priv.h"
2008-12-03 15:55:34 +03:00
# include "xfs_ioctl.h"
2010-02-15 12:44:46 +03:00
# include "xfs_trace.h"
2013-10-23 03:50:10 +04:00
# include "xfs_log.h"
2014-07-24 13:49:28 +04:00
# include "xfs_icache.h"
2015-02-16 03:59:50 +03:00
# include "xfs_pnfs.h"
2016-06-21 02:53:44 +03:00
# include "xfs_iomap.h"
2016-10-03 19:11:37 +03:00
# include "xfs_reflink.h"
2005-04-17 02:20:36 +04:00
2011-01-14 15:07:43 +03:00
# include <linux/falloc.h>
2015-05-23 00:13:32 +03:00
# include <linux/backing-dev.h>
2017-11-01 18:36:47 +03:00
# include <linux/mman.h>
2019-08-29 19:04:12 +03:00
# include <linux/fadvise.h>
2021-01-21 16:19:58 +03:00
# include <linux/mount.h>
2005-04-17 02:20:36 +04:00
2009-09-27 22:29:37 +04:00
static const struct vm_operations_struct xfs_file_vm_ops ;
2005-04-17 02:20:36 +04:00
2020-10-10 02:42:59 +03:00
/*
* Decide if the given file range is aligned to the size of the fundamental
* allocation unit for the file .
*/
static bool
xfs_is_falloc_aligned (
struct xfs_inode * ip ,
loff_t pos ,
long long int len )
{
struct xfs_mount * mp = ip - > i_mount ;
uint64_t mask ;
if ( XFS_IS_REALTIME_INODE ( ip ) ) {
if ( ! is_power_of_2 ( mp - > m_sb . sb_rextsize ) ) {
u64 rextbytes ;
u32 mod ;
rextbytes = XFS_FSB_TO_B ( mp , mp - > m_sb . sb_rextsize ) ;
div_u64_rem ( pos , rextbytes , & mod ) ;
if ( mod )
return false ;
div_u64_rem ( len , rextbytes , & mod ) ;
return mod = = 0 ;
}
mask = XFS_FSB_TO_B ( mp , mp - > m_sb . sb_rextsize ) - 1 ;
} else {
mask = mp - > m_sb . sb_blocksize - 1 ;
}
return ! ( ( pos | len ) & mask ) ;
}
2015-02-02 01:53:56 +03:00
int
xfs_update_prealloc_flags (
struct xfs_inode * ip ,
enum xfs_prealloc_flags flags )
{
struct xfs_trans * tp ;
int error ;
2016-04-06 02:19:55 +03:00
error = xfs_trans_alloc ( ip - > i_mount , & M_RES ( ip - > i_mount ) - > tr_writeid ,
0 , 0 , 0 , & tp ) ;
if ( error )
2015-02-02 01:53:56 +03:00
return error ;
xfs_ilock ( ip , XFS_ILOCK_EXCL ) ;
xfs_trans_ijoin ( tp , ip , XFS_ILOCK_EXCL ) ;
if ( ! ( flags & XFS_PREALLOC_INVISIBLE ) ) {
2016-02-09 08:54:58 +03:00
VFS_I ( ip ) - > i_mode & = ~ S_ISUID ;
if ( VFS_I ( ip ) - > i_mode & S_IXGRP )
VFS_I ( ip ) - > i_mode & = ~ S_ISGID ;
2015-02-02 01:53:56 +03:00
xfs_trans_ichgtime ( tp , ip , XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG ) ;
}
if ( flags & XFS_PREALLOC_SET )
2021-03-29 21:11:44 +03:00
ip - > i_diflags | = XFS_DIFLAG_PREALLOC ;
2015-02-02 01:53:56 +03:00
if ( flags & XFS_PREALLOC_CLEAR )
2021-03-29 21:11:44 +03:00
ip - > i_diflags & = ~ XFS_DIFLAG_PREALLOC ;
2015-02-02 01:53:56 +03:00
xfs_trans_log_inode ( tp , ip , XFS_ILOG_CORE ) ;
if ( flags & XFS_PREALLOC_SYNC )
xfs_trans_set_sync ( tp ) ;
2015-06-04 06:48:08 +03:00
return xfs_trans_commit ( tp ) ;
2015-02-02 01:53:56 +03:00
}
2011-10-02 18:25:16 +04:00
/*
* Fsync operations on directories are much simpler than on regular files ,
* as there is no file data to flush , and thus also no need for explicit
* cache flush operations , and there are no non - transaction metadata updates
* on directories either .
*/
STATIC int
xfs_dir_fsync (
struct file * file ,
loff_t start ,
loff_t end ,
int datasync )
{
struct xfs_inode * ip = XFS_I ( file - > f_mapping - > host ) ;
trace_xfs_dir_fsync ( ip ) ;
2020-04-03 21:45:37 +03:00
return xfs_log_force_inode ( ip ) ;
2011-10-02 18:25:16 +04:00
}
2021-06-18 18:21:52 +03:00
static xfs_csn_t
xfs_fsync_seq (
2021-01-23 03:48:25 +03:00
struct xfs_inode * ip ,
bool datasync )
{
if ( ! xfs_ipincount ( ip ) )
return 0 ;
if ( datasync & & ! ( ip - > i_itemp - > ili_fsync_fields & ~ XFS_ILOG_TIMESTAMP ) )
return 0 ;
2021-06-18 18:21:52 +03:00
return ip - > i_itemp - > ili_commit_seq ;
2021-01-23 03:48:25 +03:00
}
/*
* All metadata updates are logged , which means that we just have to flush the
* log up to the latest LSN that touched the inode .
*
* If we have concurrent fsync / fdatasync ( ) calls , we need them to all block on
* the log force before we clear the ili_fsync_fields field . This ensures that
* we don ' t get a racing sync operation that does not wait for the metadata to
* hit the journal before returning . If we race with clearing ili_fsync_fields ,
* then all that will happen is the log force will do nothing as the lsn will
* already be on disk . We can ' t race with setting ili_fsync_fields because that
* is done under XFS_ILOCK_EXCL , and that can ' t happen because we hold the lock
* shared until after the ili_fsync_fields is cleared .
*/
static int
xfs_fsync_flush_log (
struct xfs_inode * ip ,
bool datasync ,
int * log_flushed )
{
int error = 0 ;
2021-06-18 18:21:52 +03:00
xfs_csn_t seq ;
2021-01-23 03:48:25 +03:00
xfs_ilock ( ip , XFS_ILOCK_SHARED ) ;
2021-06-18 18:21:52 +03:00
seq = xfs_fsync_seq ( ip , datasync ) ;
if ( seq ) {
error = xfs_log_force_seq ( ip - > i_mount , seq , XFS_LOG_SYNC ,
2021-01-23 03:48:25 +03:00
log_flushed ) ;
spin_lock ( & ip - > i_itemp - > ili_lock ) ;
ip - > i_itemp - > ili_fsync_fields = 0 ;
spin_unlock ( & ip - > i_itemp - > ili_lock ) ;
}
xfs_iunlock ( ip , XFS_ILOCK_SHARED ) ;
return error ;
}
2010-02-15 12:44:48 +03:00
STATIC int
xfs_file_fsync (
struct file * file ,
2011-07-17 04:44:56 +04:00
loff_t start ,
loff_t end ,
2010-02-15 12:44:48 +03:00
int datasync )
{
2021-01-23 03:48:25 +03:00
struct xfs_inode * ip = XFS_I ( file - > f_mapping - > host ) ;
2011-06-16 16:02:23 +04:00
struct xfs_mount * mp = ip - > i_mount ;
2010-02-15 12:44:48 +03:00
int error = 0 ;
int log_flushed = 0 ;
2010-06-24 05:57:09 +04:00
trace_xfs_file_fsync ( ip ) ;
2010-02-15 12:44:48 +03:00
2017-07-06 14:02:30 +03:00
error = file_write_and_wait_range ( file , start , end ) ;
2011-07-17 04:44:56 +04:00
if ( error )
return error ;
2011-06-16 16:02:23 +04:00
if ( XFS_FORCED_SHUTDOWN ( mp ) )
2014-06-22 09:04:54 +04:00
return - EIO ;
2010-02-15 12:44:48 +03:00
xfs_iflags_clear ( ip , XFS_ITRUNCATED ) ;
2016-12-09 08:49:54 +03:00
/*
* If we have an RT and / or log subvolume we need to make sure to flush
* the write cache the device used for file data first . This is to
* ensure newly written file data make it to disk before logging the new
* inode size in case of an extending write .
*/
if ( XFS_IS_REALTIME_INODE ( ip ) )
2021-06-18 18:21:49 +03:00
blkdev_issue_flush ( mp - > m_rtdev_targp - > bt_bdev ) ;
2016-12-09 08:49:54 +03:00
else if ( mp - > m_logdev_targp ! = mp - > m_ddev_targp )
2021-06-18 18:21:49 +03:00
blkdev_issue_flush ( mp - > m_ddev_targp - > bt_bdev ) ;
2011-06-16 16:02:23 +04:00
2010-02-15 12:44:48 +03:00
/*
2021-01-23 03:48:25 +03:00
* Any inode that has dirty modifications in the log is pinned . The
* racy check here for a pinned inode while not catch modifications
* that happen concurrently to the fsync call , but fsync semantics
* only require to sync previously completed I / O .
2010-02-15 12:44:48 +03:00
*/
2021-01-23 03:48:25 +03:00
if ( xfs_ipincount ( ip ) )
error = xfs_fsync_flush_log ( ip , datasync , & log_flushed ) ;
2011-09-19 18:55:51 +04:00
2011-06-16 16:02:23 +04:00
/*
* If we only have a single device , and the log force about was
* a no - op we might have to flush the data device cache here .
* This can only happen for fdatasync / O_DSYNC if we were overwriting
* an already allocated file and thus do not have any metadata to
* commit .
*/
2016-12-09 08:49:54 +03:00
if ( ! log_flushed & & ! XFS_IS_REALTIME_INODE ( ip ) & &
mp - > m_logdev_targp = = mp - > m_ddev_targp )
2021-06-18 18:21:49 +03:00
blkdev_issue_flush ( mp - > m_ddev_targp - > bt_bdev ) ;
2010-02-15 12:44:48 +03:00
2014-06-25 08:58:08 +04:00
return error ;
2010-02-15 12:44:48 +03:00
}
2021-01-23 21:06:27 +03:00
static int
xfs_ilock_iocb (
struct kiocb * iocb ,
unsigned int lock_mode )
{
struct xfs_inode * ip = XFS_I ( file_inode ( iocb - > ki_filp ) ) ;
if ( iocb - > ki_flags & IOCB_NOWAIT ) {
if ( ! xfs_ilock_nowait ( ip , lock_mode ) )
return - EAGAIN ;
} else {
xfs_ilock ( ip , lock_mode ) ;
}
return 0 ;
}
2010-02-15 12:44:47 +03:00
STATIC ssize_t
2021-01-23 21:06:28 +03:00
xfs_file_dio_read (
2010-02-15 12:44:46 +03:00
struct kiocb * iocb ,
2014-04-02 22:37:59 +04:00
struct iov_iter * to )
2010-02-15 12:44:46 +03:00
{
2016-11-30 06:37:15 +03:00
struct xfs_inode * ip = XFS_I ( file_inode ( iocb - > ki_filp ) ) ;
ssize_t ret ;
2010-02-15 12:44:46 +03:00
2021-01-23 21:06:29 +03:00
trace_xfs_file_direct_read ( iocb , to ) ;
2010-02-15 12:44:46 +03:00
2021-01-23 21:06:29 +03:00
if ( ! iov_iter_count ( to ) )
2016-07-20 04:36:57 +03:00
return 0 ; /* skip atime */
2010-02-15 12:44:46 +03:00
2016-10-03 01:47:34 +03:00
file_accessed ( iocb - > ki_filp ) ;
2021-01-23 21:06:27 +03:00
ret = xfs_ilock_iocb ( iocb , XFS_IOLOCK_SHARED ) ;
if ( ret )
return ret ;
2021-01-23 21:06:09 +03:00
ret = iomap_dio_rw ( iocb , to , & xfs_read_iomap_ops , NULL , 0 ) ;
2016-11-30 06:33:25 +03:00
xfs_iunlock ( ip , XFS_IOLOCK_SHARED ) ;
2016-11-30 06:37:15 +03:00
2016-07-20 04:38:55 +03:00
return ret ;
}
2016-07-22 02:50:55 +03:00
static noinline ssize_t
2016-07-20 04:38:55 +03:00
xfs_file_dax_read (
struct kiocb * iocb ,
struct iov_iter * to )
{
2016-09-19 04:28:38 +03:00
struct xfs_inode * ip = XFS_I ( iocb - > ki_filp - > f_mapping - > host ) ;
2016-07-20 04:38:55 +03:00
ssize_t ret = 0 ;
2021-01-23 21:06:29 +03:00
trace_xfs_file_dax_read ( iocb , to ) ;
2016-07-20 04:38:55 +03:00
2021-01-23 21:06:29 +03:00
if ( ! iov_iter_count ( to ) )
2016-07-20 04:38:55 +03:00
return 0 ; /* skip atime */
2021-01-23 21:06:27 +03:00
ret = xfs_ilock_iocb ( iocb , XFS_IOLOCK_SHARED ) ;
if ( ret )
return ret ;
2019-10-19 19:09:45 +03:00
ret = dax_iomap_rw ( iocb , to , & xfs_read_iomap_ops ) ;
2016-11-30 06:33:25 +03:00
xfs_iunlock ( ip , XFS_IOLOCK_SHARED ) ;
2016-07-20 04:35:42 +03:00
2016-07-20 04:36:57 +03:00
file_accessed ( iocb - > ki_filp ) ;
2016-07-20 04:35:42 +03:00
return ret ;
}
STATIC ssize_t
2021-01-23 21:06:28 +03:00
xfs_file_buffered_read (
2016-07-20 04:35:42 +03:00
struct kiocb * iocb ,
struct iov_iter * to )
{
struct xfs_inode * ip = XFS_I ( file_inode ( iocb - > ki_filp ) ) ;
ssize_t ret ;
2021-01-23 21:06:29 +03:00
trace_xfs_file_buffered_read ( iocb , to ) ;
2010-02-15 12:44:46 +03:00
2021-01-23 21:06:27 +03:00
ret = xfs_ilock_iocb ( iocb , XFS_IOLOCK_SHARED ) ;
if ( ret )
return ret ;
2014-04-02 22:37:59 +04:00
ret = generic_file_read_iter ( iocb , to ) ;
2016-11-30 06:33:25 +03:00
xfs_iunlock ( ip , XFS_IOLOCK_SHARED ) ;
2016-07-20 04:35:42 +03:00
return ret ;
}
STATIC ssize_t
xfs_file_read_iter (
struct kiocb * iocb ,
struct iov_iter * to )
{
2016-07-20 04:38:55 +03:00
struct inode * inode = file_inode ( iocb - > ki_filp ) ;
struct xfs_mount * mp = XFS_I ( inode ) - > i_mount ;
2016-07-20 04:35:42 +03:00
ssize_t ret = 0 ;
XFS_STATS_INC ( mp , xs_read_calls ) ;
if ( XFS_FORCED_SHUTDOWN ( mp ) )
return - EIO ;
2016-07-20 04:38:55 +03:00
if ( IS_DAX ( inode ) )
ret = xfs_file_dax_read ( iocb , to ) ;
else if ( iocb - > ki_flags & IOCB_DIRECT )
2021-01-23 21:06:28 +03:00
ret = xfs_file_dio_read ( iocb , to ) ;
2016-07-20 04:31:42 +03:00
else
2021-01-23 21:06:28 +03:00
ret = xfs_file_buffered_read ( iocb , to ) ;
2010-02-15 12:44:46 +03:00
if ( ret > 0 )
2015-10-12 10:21:22 +03:00
XFS_STATS_ADD ( mp , xs_read_bytes , ret ) ;
2010-02-15 12:44:46 +03:00
return ret ;
}
2011-01-11 02:23:42 +03:00
/*
* Common pre - write limit and setup checks .
*
2011-12-19 00:00:13 +04:00
* Called with the iolocked held either shared and exclusive according to
* @ iolock , and returns with it held . Might upgrade the iolock to exclusive
* if called for a direct write beyond i_size .
2011-01-11 02:23:42 +03:00
*/
STATIC ssize_t
2021-01-23 21:06:28 +03:00
xfs_file_write_checks (
2015-04-07 21:25:18 +03:00
struct kiocb * iocb ,
struct iov_iter * from ,
2011-01-11 02:23:42 +03:00
int * iolock )
{
2015-04-07 21:25:18 +03:00
struct file * file = iocb - > ki_filp ;
2011-01-11 02:23:42 +03:00
struct inode * inode = file - > f_mapping - > host ;
struct xfs_inode * ip = XFS_I ( inode ) ;
2015-04-09 19:55:47 +03:00
ssize_t error = 0 ;
2015-04-07 21:25:18 +03:00
size_t count = iov_iter_count ( from ) ;
2015-10-12 08:02:05 +03:00
bool drained_dio = false ;
2018-03-14 09:15:32 +03:00
loff_t isize ;
2011-01-11 02:23:42 +03:00
xfs: don't serialise adjacent concurrent direct IO appending writes
For append write workloads, extending the file requires a certain
amount of exclusive locking to be done up front to ensure sanity in
things like ensuring that we've zeroed any allocated regions
between the old EOF and the start of the new IO.
For single threads, this typically isn't a problem, and for large
IOs we don't serialise enough for it to be a problem for two
threads on really fast block devices. However for smaller IO and
larger thread counts we have a problem.
Take 4 concurrent sequential, single block sized and aligned IOs.
After the first IO is submitted but before it completes, we end up
with this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
And the IO is done without exclusive locking because offset <=
ip->i_size. When we submit IO 2, we see offset > ip->i_size, and
grab the IO lock exclusive, because there is a chance we need to do
EOF zeroing. However, there is already an IO in progress that avoids
the need for IO zeroing because offset <= ip->i_new_size. hence we
could avoid holding the IO lock exlcusive for this. Hence after
submission of the second IO, we'd end up this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
There is no need to grab the i_mutex of the IO lock in exclusive
mode if we don't need to invalidate the page cache. Taking these
locks on every direct IO effective serialises them as taking the IO
lock in exclusive mode has to wait for all shared holders to drop
the lock. That only happens when IO is complete, so effective it
prevents dispatch of concurrent direct IO writes to the same inode.
And so you can see that for the third concurrent IO, we'd avoid
exclusive locking for the same reason we avoided the exclusive lock
for the second IO.
Fixing this is a bit more complex than that, because we need to hold
a write-submission local value of ip->i_new_size to that clearing
the value is only done if no other thread has updated it before our
IO completes.....
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Alex Elder <aelder@sgi.com>
2011-08-25 11:17:02 +04:00
restart :
2015-04-09 19:55:47 +03:00
error = generic_write_checks ( iocb , from ) ;
if ( error < = 0 )
2011-01-11 02:23:42 +03:00
return error ;
2021-01-23 21:06:28 +03:00
if ( iocb - > ki_flags & IOCB_NOWAIT ) {
error = break_layout ( inode , false ) ;
if ( error = = - EWOULDBLOCK )
error = - EAGAIN ;
} else {
error = xfs_break_layouts ( inode , iolock , BREAK_WRITE ) ;
}
2015-02-16 03:59:50 +03:00
if ( error )
return error ;
2016-11-30 06:33:25 +03:00
/*
* For changing security info in file_remove_privs ( ) we need i_rwsem
* exclusively .
*/
2015-05-21 17:05:56 +03:00
if ( * iolock = = XFS_IOLOCK_SHARED & & ! IS_NOSEC ( inode ) ) {
2016-11-30 06:33:25 +03:00
xfs_iunlock ( ip , * iolock ) ;
2015-05-21 17:05:56 +03:00
* iolock = XFS_IOLOCK_EXCL ;
2021-01-23 21:06:28 +03:00
error = xfs_ilock_iocb ( iocb , * iolock ) ;
if ( error ) {
* iolock = 0 ;
return error ;
}
2015-05-21 17:05:56 +03:00
goto restart ;
}
2021-06-03 01:00:38 +03:00
2011-01-11 02:23:42 +03:00
/*
* If the offset is beyond the size of the file , we need to zero any
* blocks that fall between the existing EOF and the start of this
2021-06-03 01:00:38 +03:00
* write . If zeroing is needed and we are currently holding the iolock
* shared , we need to update it to exclusive which implies having to
* redo all checks before .
*
* We need to serialise against EOF updates that occur in IO completions
* here . We want to make sure that nobody is changing the size while we
* do this check until we have placed an IO barrier ( i . e . hold the
* XFS_IOLOCK_EXCL ) that prevents new IO from being dispatched . The
* spinlock effectively forms a memory barrier once we have the
* XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value and
* hence be able to correctly determine if we need to run zeroing .
2015-04-16 15:03:07 +03:00
*
2021-06-03 01:00:38 +03:00
* We can do an unlocked check here safely as IO completion can only
* extend EOF . Truncate is locked out at this point , so the EOF can
* not move backwards , only forwards . Hence we only need to take the
* slow path and spin locks when we are at or beyond the current EOF .
2011-01-11 02:23:42 +03:00
*/
2021-06-03 01:00:38 +03:00
if ( iocb - > ki_pos < = i_size_read ( inode ) )
goto out ;
2015-04-16 15:03:07 +03:00
spin_lock ( & ip - > i_flags_lock ) ;
2018-03-14 09:15:32 +03:00
isize = i_size_read ( inode ) ;
if ( iocb - > ki_pos > isize ) {
2015-04-16 15:03:07 +03:00
spin_unlock ( & ip - > i_flags_lock ) ;
2021-01-23 21:06:28 +03:00
if ( iocb - > ki_flags & IOCB_NOWAIT )
return - EAGAIN ;
2015-10-12 08:02:05 +03:00
if ( ! drained_dio ) {
if ( * iolock = = XFS_IOLOCK_SHARED ) {
2016-11-30 06:33:25 +03:00
xfs_iunlock ( ip , * iolock ) ;
2015-10-12 08:02:05 +03:00
* iolock = XFS_IOLOCK_EXCL ;
2016-11-30 06:33:25 +03:00
xfs_ilock ( ip , * iolock ) ;
2015-10-12 08:02:05 +03:00
iov_iter_reexpand ( from , count ) ;
}
2015-04-16 15:03:17 +03:00
/*
* We now have an IO submission barrier in place , but
* AIO can do EOF updates during IO completion and hence
* we now need to wait for all of them to drain . Non - AIO
* DIO will have drained before we are given the
* XFS_IOLOCK_EXCL , and so for most cases this wait is a
* no - op .
*/
inode_dio_wait ( inode ) ;
2015-10-12 08:02:05 +03:00
drained_dio = true ;
xfs: don't serialise adjacent concurrent direct IO appending writes
For append write workloads, extending the file requires a certain
amount of exclusive locking to be done up front to ensure sanity in
things like ensuring that we've zeroed any allocated regions
between the old EOF and the start of the new IO.
For single threads, this typically isn't a problem, and for large
IOs we don't serialise enough for it to be a problem for two
threads on really fast block devices. However for smaller IO and
larger thread counts we have a problem.
Take 4 concurrent sequential, single block sized and aligned IOs.
After the first IO is submitted but before it completes, we end up
with this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
And the IO is done without exclusive locking because offset <=
ip->i_size. When we submit IO 2, we see offset > ip->i_size, and
grab the IO lock exclusive, because there is a chance we need to do
EOF zeroing. However, there is already an IO in progress that avoids
the need for IO zeroing because offset <= ip->i_new_size. hence we
could avoid holding the IO lock exlcusive for this. Hence after
submission of the second IO, we'd end up this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
There is no need to grab the i_mutex of the IO lock in exclusive
mode if we don't need to invalidate the page cache. Taking these
locks on every direct IO effective serialises them as taking the IO
lock in exclusive mode has to wait for all shared holders to drop
the lock. That only happens when IO is complete, so effective it
prevents dispatch of concurrent direct IO writes to the same inode.
And so you can see that for the third concurrent IO, we'd avoid
exclusive locking for the same reason we avoided the exclusive lock
for the second IO.
Fixing this is a bit more complex than that, because we need to hold
a write-submission local value of ip->i_new_size to that clearing
the value is only done if no other thread has updated it before our
IO completes.....
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Alex Elder <aelder@sgi.com>
2011-08-25 11:17:02 +04:00
goto restart ;
}
2021-06-03 01:00:38 +03:00
2018-03-14 09:15:32 +03:00
trace_xfs_zero_eof ( ip , isize , iocb - > ki_pos - isize ) ;
error = iomap_zero_range ( inode , isize , iocb - > ki_pos - isize ,
2019-10-19 19:09:46 +03:00
NULL , & xfs_buffered_write_iomap_ops ) ;
2012-03-27 18:34:47 +04:00
if ( error )
return error ;
2015-04-16 15:03:07 +03:00
} else
spin_unlock ( & ip - > i_flags_lock ) ;
2011-01-11 02:23:42 +03:00
2021-06-03 01:00:38 +03:00
out :
2019-06-05 18:04:50 +03:00
return file_modified ( file ) ;
2011-01-11 02:23:42 +03:00
}
2016-11-30 06:37:15 +03:00
static int
xfs_dio_write_end_io (
struct kiocb * iocb ,
ssize_t size ,
2019-09-20 01:32:44 +03:00
int error ,
2016-11-30 06:37:15 +03:00
unsigned flags )
{
struct inode * inode = file_inode ( iocb - > ki_filp ) ;
struct xfs_inode * ip = XFS_I ( inode ) ;
loff_t offset = iocb - > ki_pos ;
2019-06-29 05:31:38 +03:00
unsigned int nofs_flag ;
2016-11-30 06:37:15 +03:00
trace_xfs_end_io_direct_write ( ip , offset , size ) ;
if ( XFS_FORCED_SHUTDOWN ( ip - > i_mount ) )
return - EIO ;
2019-09-20 01:32:44 +03:00
if ( error )
return error ;
if ( ! size )
return 0 ;
2016-11-30 06:37:15 +03:00
2018-05-02 22:54:52 +03:00
/*
* Capture amount written on completion as we can ' t reliably account
* for it on submission .
*/
XFS_STATS_ADD ( ip - > i_mount , xs_write_bytes , size ) ;
2019-06-29 05:31:38 +03:00
/*
* We can allocate memory here while doing writeback on behalf of
* memory reclaim . To avoid memory allocation deadlocks set the
* task - wide nofs context for the following operations .
*/
nofs_flag = memalloc_nofs_save ( ) ;
2017-09-21 21:26:18 +03:00
if ( flags & IOMAP_DIO_COW ) {
error = xfs_reflink_end_cow ( ip , offset , size ) ;
if ( error )
2019-06-29 05:31:38 +03:00
goto out ;
2017-09-21 21:26:18 +03:00
}
/*
* Unwritten conversion updates the in - core isize after extent
* conversion but before updating the on - disk size . Updating isize any
* earlier allows a racing dio read to find unwritten extents before
* they are converted .
*/
2019-06-29 05:31:38 +03:00
if ( flags & IOMAP_DIO_UNWRITTEN ) {
error = xfs_iomap_write_unwritten ( ip , offset , size , true ) ;
goto out ;
}
2017-09-21 21:26:18 +03:00
2016-11-30 06:37:15 +03:00
/*
* We need to update the in - core inode size here so that we don ' t end up
* with the on - disk inode size being outside the in - core inode size . We
* have no other method of updating EOF for AIO , so always do it here
* if necessary .
*
* We need to lock the test / set EOF update as we can be racing with
* other IO completions here to update the EOF . Failing to serialise
* here can result in EOF moving backwards and Bad Things Happen when
* that occurs .
2021-06-03 01:00:38 +03:00
*
* As IO completion only ever extends EOF , we can do an unlocked check
* here to avoid taking the spinlock . If we land within the current EOF ,
* then we do not need to do an extending update at all , and we don ' t
* need to take the lock to check this . If we race with an update moving
* EOF , then we ' ll either still be beyond EOF and need to take the lock ,
* or we ' ll be within EOF and we don ' t need to take it at all .
2016-11-30 06:37:15 +03:00
*/
2021-06-03 01:00:38 +03:00
if ( offset + size < = i_size_read ( inode ) )
goto out ;
2016-11-30 06:37:15 +03:00
spin_lock ( & ip - > i_flags_lock ) ;
if ( offset + size > i_size_read ( inode ) ) {
i_size_write ( inode , offset + size ) ;
2017-09-21 21:26:18 +03:00
spin_unlock ( & ip - > i_flags_lock ) ;
2016-11-30 06:37:15 +03:00
error = xfs_setfilesize ( ip , offset , size ) ;
2017-09-21 21:26:18 +03:00
} else {
spin_unlock ( & ip - > i_flags_lock ) ;
}
2016-11-30 06:37:15 +03:00
2019-06-29 05:31:38 +03:00
out :
memalloc_nofs_restore ( nofs_flag ) ;
2016-11-30 06:37:15 +03:00
return error ;
}
2019-09-20 01:32:45 +03:00
static const struct iomap_dio_ops xfs_dio_write_ops = {
. end_io = xfs_dio_write_end_io ,
} ;
2011-01-11 02:15:36 +03:00
/*
2021-01-23 21:06:30 +03:00
* Handle block aligned direct I / O writes
2011-01-11 02:15:36 +03:00
*/
2021-01-23 21:06:30 +03:00
static noinline ssize_t
xfs_file_dio_write_aligned (
struct xfs_inode * ip ,
2011-01-11 02:15:36 +03:00
struct kiocb * iocb ,
2014-04-02 15:06:30 +04:00
struct iov_iter * from )
2011-01-11 02:15:36 +03:00
{
2021-01-23 21:06:30 +03:00
int iolock = XFS_IOLOCK_SHARED ;
ssize_t ret ;
2011-01-11 02:15:36 +03:00
2021-01-23 21:06:30 +03:00
ret = xfs_ilock_iocb ( iocb , iolock ) ;
if ( ret )
return ret ;
ret = xfs_file_write_checks ( iocb , from , & iolock ) ;
if ( ret )
goto out_unlock ;
2011-01-11 02:15:36 +03:00
xfs: don't serialise adjacent concurrent direct IO appending writes
For append write workloads, extending the file requires a certain
amount of exclusive locking to be done up front to ensure sanity in
things like ensuring that we've zeroed any allocated regions
between the old EOF and the start of the new IO.
For single threads, this typically isn't a problem, and for large
IOs we don't serialise enough for it to be a problem for two
threads on really fast block devices. However for smaller IO and
larger thread counts we have a problem.
Take 4 concurrent sequential, single block sized and aligned IOs.
After the first IO is submitted but before it completes, we end up
with this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
And the IO is done without exclusive locking because offset <=
ip->i_size. When we submit IO 2, we see offset > ip->i_size, and
grab the IO lock exclusive, because there is a chance we need to do
EOF zeroing. However, there is already an IO in progress that avoids
the need for IO zeroing because offset <= ip->i_new_size. hence we
could avoid holding the IO lock exlcusive for this. Hence after
submission of the second IO, we'd end up this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
There is no need to grab the i_mutex of the IO lock in exclusive
mode if we don't need to invalidate the page cache. Taking these
locks on every direct IO effective serialises them as taking the IO
lock in exclusive mode has to wait for all shared holders to drop
the lock. That only happens when IO is complete, so effective it
prevents dispatch of concurrent direct IO writes to the same inode.
And so you can see that for the third concurrent IO, we'd avoid
exclusive locking for the same reason we avoided the exclusive lock
for the second IO.
Fixing this is a bit more complex than that, because we need to hold
a write-submission local value of ip->i_new_size to that clearing
the value is only done if no other thread has updated it before our
IO completes.....
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Alex Elder <aelder@sgi.com>
2011-08-25 11:17:02 +04:00
/*
2021-01-23 21:06:30 +03:00
* We don ' t need to hold the IOLOCK exclusively across the IO , so demote
* the iolock back to shared if we had to take the exclusive lock in
* xfs_file_write_checks ( ) for other reasons .
xfs: don't serialise adjacent concurrent direct IO appending writes
For append write workloads, extending the file requires a certain
amount of exclusive locking to be done up front to ensure sanity in
things like ensuring that we've zeroed any allocated regions
between the old EOF and the start of the new IO.
For single threads, this typically isn't a problem, and for large
IOs we don't serialise enough for it to be a problem for two
threads on really fast block devices. However for smaller IO and
larger thread counts we have a problem.
Take 4 concurrent sequential, single block sized and aligned IOs.
After the first IO is submitted but before it completes, we end up
with this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
And the IO is done without exclusive locking because offset <=
ip->i_size. When we submit IO 2, we see offset > ip->i_size, and
grab the IO lock exclusive, because there is a chance we need to do
EOF zeroing. However, there is already an IO in progress that avoids
the need for IO zeroing because offset <= ip->i_new_size. hence we
could avoid holding the IO lock exlcusive for this. Hence after
submission of the second IO, we'd end up this state:
IO 1 IO 2 IO 3 IO 4
+-------+-------+-------+-------+
^ ^
| |
| |
| |
| \- ip->i_new_size
\- ip->i_size
There is no need to grab the i_mutex of the IO lock in exclusive
mode if we don't need to invalidate the page cache. Taking these
locks on every direct IO effective serialises them as taking the IO
lock in exclusive mode has to wait for all shared holders to drop
the lock. That only happens when IO is complete, so effective it
prevents dispatch of concurrent direct IO writes to the same inode.
And so you can see that for the third concurrent IO, we'd avoid
exclusive locking for the same reason we avoided the exclusive lock
for the second IO.
Fixing this is a bit more complex than that, because we need to hold
a write-submission local value of ip->i_new_size to that clearing
the value is only done if no other thread has updated it before our
IO completes.....
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Signed-off-by: Alex Elder <aelder@sgi.com>
2011-08-25 11:17:02 +04:00
*/
2021-01-23 21:06:30 +03:00
if ( iolock = = XFS_IOLOCK_EXCL ) {
xfs_ilock_demote ( ip , XFS_IOLOCK_EXCL ) ;
2011-12-19 00:00:14 +04:00
iolock = XFS_IOLOCK_SHARED ;
2011-08-27 18:42:53 +04:00
}
2021-01-23 21:06:30 +03:00
trace_xfs_file_direct_write ( iocb , from ) ;
ret = iomap_dio_rw ( iocb , from , & xfs_direct_write_iomap_ops ,
& xfs_dio_write_ops , 0 ) ;
out_unlock :
if ( iolock )
xfs_iunlock ( ip , iolock ) ;
return ret ;
}
2011-01-11 02:15:36 +03:00
2021-01-23 21:06:30 +03:00
/*
* Handle block unaligned direct I / O writes
*
* In most cases direct I / O writes will be done holding IOLOCK_SHARED , allowing
* them to be done in parallel with reads and other direct I / O writes . However ,
* if the I / O is not aligned to filesystem blocks , the direct I / O layer may need
* to do sub - block zeroing and that requires serialisation against other direct
* I / O to the same block . In this case we need to serialise the submission of
* the unaligned I / O so that we don ' t get racing block zeroing in the dio layer .
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 21:06:31 +03:00
* In the case where sub - block zeroing is not required , we can do concurrent
* sub - block dios to the same block successfully .
2021-01-23 21:06:30 +03:00
*
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 21:06:31 +03:00
* Optimistically submit the I / O using the shared lock first , but use the
* IOMAP_DIO_OVERWRITE_ONLY flag to tell the lower layers to return - EAGAIN
* if block allocation or partial block zeroing would be required . In that case
* we try again with the exclusive lock .
2021-01-23 21:06:30 +03:00
*/
static noinline ssize_t
xfs_file_dio_write_unaligned (
struct xfs_inode * ip ,
struct kiocb * iocb ,
struct iov_iter * from )
{
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 21:06:31 +03:00
size_t isize = i_size_read ( VFS_I ( ip ) ) ;
size_t count = iov_iter_count ( from ) ;
int iolock = XFS_IOLOCK_SHARED ;
unsigned int flags = IOMAP_DIO_OVERWRITE_ONLY ;
2021-01-23 21:06:30 +03:00
ssize_t ret ;
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 21:06:31 +03:00
/*
* Extending writes need exclusivity because of the sub - block zeroing
* that the DIO code always does for partial tail blocks beyond EOF , so
* don ' t even bother trying the fast path in this case .
*/
if ( iocb - > ki_pos > isize | | iocb - > ki_pos + count > = isize ) {
retry_exclusive :
if ( iocb - > ki_flags & IOCB_NOWAIT )
return - EAGAIN ;
iolock = XFS_IOLOCK_EXCL ;
flags = IOMAP_DIO_FORCE_WAIT ;
}
ret = xfs_ilock_iocb ( iocb , iolock ) ;
if ( ret )
return ret ;
2021-01-23 21:06:30 +03:00
/*
* We can ' t properly handle unaligned direct I / O to reflink files yet ,
* as we can ' t unshare a partial block .
*/
if ( xfs_is_cow_inode ( ip ) ) {
trace_xfs_reflink_bounce_dio_write ( iocb , from ) ;
ret = - ENOTBLK ;
goto out_unlock ;
2017-06-20 15:05:48 +03:00
}
2016-10-20 07:44:14 +03:00
2021-01-23 21:06:28 +03:00
ret = xfs_file_write_checks ( iocb , from , & iolock ) ;
2011-01-11 02:23:42 +03:00
if ( ret )
2021-01-23 21:06:30 +03:00
goto out_unlock ;
2011-01-11 02:15:36 +03:00
2011-01-11 02:22:40 +03:00
/*
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 21:06:31 +03:00
* If we are doing exclusive unaligned I / O , this must be the only I / O
* in - flight . Otherwise we risk data corruption due to unwritten extent
* conversions from the AIO end_io handler . Wait for all other I / O to
* drain first .
2011-01-11 02:22:40 +03:00
*/
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 21:06:31 +03:00
if ( flags & IOMAP_DIO_FORCE_WAIT )
inode_dio_wait ( VFS_I ( ip ) ) ;
2011-01-11 02:15:36 +03:00
2021-01-23 21:06:29 +03:00
trace_xfs_file_direct_write ( iocb , from ) ;
2019-10-19 19:09:46 +03:00
ret = iomap_dio_rw ( iocb , from , & xfs_direct_write_iomap_ops ,
xfs: reduce exclusive locking on unaligned dio
Attempt shared locking for unaligned DIO, but only if the the
underlying extent is already allocated and in written state. On
failure, retry with the existing exclusive locking.
Test case is fio randrw of 512 byte IOs using AIO and an iodepth of
32 IOs.
Vanilla:
READ: bw=4560KiB/s (4670kB/s), 4560KiB/s-4560KiB/s (4670kB/s-4670kB/s), io=134MiB (140MB), run=30001-30001msec
WRITE: bw=4567KiB/s (4676kB/s), 4567KiB/s-4567KiB/s (4676kB/s-4676kB/s), io=134MiB (140MB), run=30001-30001msec
Patched:
READ: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1127MiB (1182MB), run=30002-30002msec
WRITE: bw=37.6MiB/s (39.4MB/s), 37.6MiB/s-37.6MiB/s (39.4MB/s-39.4MB/s), io=1128MiB (1183MB), run=30002-30002msec
That's an improvement from ~18k IOPS to a ~150k IOPS, which is
about the IOPS limit of the VM block device setup I'm testing on.
4kB block IO comparison:
READ: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8868MiB (9299MB), run=30002-30002msec
WRITE: bw=296MiB/s (310MB/s), 296MiB/s-296MiB/s (310MB/s-310MB/s), io=8878MiB (9309MB), run=30002-30002msec
Which is ~150k IOPS, same as what the test gets for sub-block
AIO+DIO writes with this patch.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
[hch: rebased, split unaligned from nowait]
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Darrick J. Wong <djwong@kernel.org>
2021-01-23 21:06:31 +03:00
& xfs_dio_write_ops , flags ) ;
/*
* Retry unaligned I / O with exclusive blocking semantics if the DIO
* layer rejected it for mapping or locking reasons . If we are doing
* nonblocking user I / O , propagate the error .
*/
if ( ret = = - EAGAIN & & ! ( iocb - > ki_flags & IOCB_NOWAIT ) ) {
ASSERT ( flags & IOMAP_DIO_OVERWRITE_ONLY ) ;
xfs_iunlock ( ip , iolock ) ;
goto retry_exclusive ;
}
2021-01-23 21:06:30 +03:00
out_unlock :
2021-01-23 21:06:28 +03:00
if ( iolock )
xfs_iunlock ( ip , iolock ) ;
2016-07-20 04:38:55 +03:00
return ret ;
}
2021-01-23 21:06:30 +03:00
static ssize_t
xfs_file_dio_write (
struct kiocb * iocb ,
struct iov_iter * from )
{
struct xfs_inode * ip = XFS_I ( file_inode ( iocb - > ki_filp ) ) ;
struct xfs_buftarg * target = xfs_inode_buftarg ( ip ) ;
size_t count = iov_iter_count ( from ) ;
/* direct I/O must be aligned to device logical sector size */
if ( ( iocb - > ki_pos | count ) & target - > bt_logical_sectormask )
return - EINVAL ;
if ( ( iocb - > ki_pos | count ) & ip - > i_mount - > m_blockmask )
return xfs_file_dio_write_unaligned ( ip , iocb , from ) ;
return xfs_file_dio_write_aligned ( ip , iocb , from ) ;
}
2016-07-22 02:50:55 +03:00
static noinline ssize_t
2016-07-20 04:38:55 +03:00
xfs_file_dax_write (
struct kiocb * iocb ,
struct iov_iter * from )
{
2016-09-19 04:28:38 +03:00
struct inode * inode = iocb - > ki_filp - > f_mapping - > host ;
2016-07-20 04:38:55 +03:00
struct xfs_inode * ip = XFS_I ( inode ) ;
2016-09-19 04:24:50 +03:00
int iolock = XFS_IOLOCK_EXCL ;
2016-09-19 04:28:38 +03:00
ssize_t ret , error = 0 ;
loff_t pos ;
2016-07-20 04:38:55 +03:00
2021-01-23 21:06:27 +03:00
ret = xfs_ilock_iocb ( iocb , iolock ) ;
if ( ret )
return ret ;
2021-01-23 21:06:28 +03:00
ret = xfs_file_write_checks ( iocb , from , & iolock ) ;
2016-07-20 04:38:55 +03:00
if ( ret )
goto out ;
2016-09-19 04:28:38 +03:00
pos = iocb - > ki_pos ;
2016-08-17 01:31:33 +03:00
2021-01-23 21:06:29 +03:00
trace_xfs_file_dax_write ( iocb , from ) ;
2019-10-19 19:09:46 +03:00
ret = dax_iomap_rw ( iocb , from , & xfs_direct_write_iomap_ops ) ;
2016-09-19 04:28:38 +03:00
if ( ret > 0 & & iocb - > ki_pos > i_size_read ( inode ) ) {
i_size_write ( inode , iocb - > ki_pos ) ;
error = xfs_setfilesize ( ip , pos , ret ) ;
2016-07-20 04:38:55 +03:00
}
out :
2021-01-23 21:06:28 +03:00
if ( iolock )
xfs_iunlock ( ip , iolock ) ;
2018-05-02 22:54:52 +03:00
if ( error )
return error ;
if ( ret > 0 ) {
XFS_STATS_ADD ( ip - > i_mount , xs_write_bytes , ret ) ;
/* Handle various SYNC-type writes */
ret = generic_write_sync ( iocb , ret ) ;
}
return ret ;
2011-01-11 02:15:36 +03:00
}
2010-02-15 12:44:47 +03:00
STATIC ssize_t
2021-01-23 21:06:28 +03:00
xfs_file_buffered_write (
2010-02-15 12:44:46 +03:00
struct kiocb * iocb ,
2014-04-02 15:06:30 +04:00
struct iov_iter * from )
2010-02-15 12:44:46 +03:00
{
struct file * file = iocb - > ki_filp ;
struct address_space * mapping = file - > f_mapping ;
struct inode * inode = mapping - > host ;
2010-02-15 12:44:47 +03:00
struct xfs_inode * ip = XFS_I ( inode ) ;
2011-01-11 02:17:30 +03:00
ssize_t ret ;
2021-01-23 03:48:34 +03:00
bool cleared_space = false ;
2017-01-28 10:22:56 +03:00
int iolock ;
2010-02-15 12:44:46 +03:00
2017-08-29 17:13:20 +03:00
if ( iocb - > ki_flags & IOCB_NOWAIT )
return - EOPNOTSUPP ;
2017-01-28 10:22:56 +03:00
write_retry :
iolock = XFS_IOLOCK_EXCL ;
2016-11-30 06:33:25 +03:00
xfs_ilock ( ip , iolock ) ;
2010-02-15 12:44:46 +03:00
2021-01-23 21:06:28 +03:00
ret = xfs_file_write_checks ( iocb , from , & iolock ) ;
2011-01-11 02:23:42 +03:00
if ( ret )
2011-12-19 00:00:14 +04:00
goto out ;
2010-02-15 12:44:46 +03:00
/* We can write back this queue in page reclaim */
2015-01-14 12:42:36 +03:00
current - > backing_dev_info = inode_to_bdi ( inode ) ;
2010-02-15 12:44:46 +03:00
2021-01-23 21:06:29 +03:00
trace_xfs_file_buffered_write ( iocb , from ) ;
2019-10-19 19:09:46 +03:00
ret = iomap_file_buffered_write ( iocb , from ,
& xfs_buffered_write_iomap_ops ) ;
2014-02-12 07:25:22 +04:00
if ( likely ( ret > = 0 ) )
2015-04-07 21:25:18 +03:00
iocb - > ki_pos + = ret ;
2014-07-24 13:49:28 +04:00
2011-01-11 02:17:30 +03:00
/*
2014-07-24 13:49:28 +04:00
* If we hit a space limit , try to free up some lingering preallocated
* space before returning an error . In the case of ENOSPC , first try to
* write back all dirty inodes to free up some of the excess reserved
* metadata space . This reduces the chances that the eofblocks scan
* waits on dirty mappings . Since xfs_flush_inodes ( ) is serialized , this
* also behaves as a filter to prevent too many eofblocks scans from
2021-01-23 03:48:36 +03:00
* running at the same time . Use a synchronous scan to increase the
* effectiveness of the scan .
2011-01-11 02:17:30 +03:00
*/
2021-01-23 03:48:34 +03:00
if ( ret = = - EDQUOT & & ! cleared_space ) {
2017-01-28 10:22:56 +03:00
xfs_iunlock ( ip , iolock ) ;
2021-06-07 19:34:51 +03:00
xfs_blockgc_free_quota ( ip , XFS_ICWALK_FLAG_SYNC ) ;
2021-01-23 03:48:36 +03:00
cleared_space = true ;
goto write_retry ;
2021-01-23 03:48:34 +03:00
} else if ( ret = = - ENOSPC & & ! cleared_space ) {
2021-06-07 19:34:51 +03:00
struct xfs_icwalk icw = { 0 } ;
2014-07-24 13:49:28 +04:00
2021-01-23 03:48:34 +03:00
cleared_space = true ;
2012-10-08 14:56:04 +04:00
xfs_flush_inodes ( ip - > i_mount ) ;
2017-01-28 10:22:56 +03:00
xfs_iunlock ( ip , iolock ) ;
2021-06-07 19:34:51 +03:00
icw . icw_flags = XFS_ICWALK_FLAG_SYNC ;
xfs_blockgc_free_space ( ip - > i_mount , & icw ) ;
2012-10-08 14:56:04 +04:00
goto write_retry ;
2010-02-15 12:44:46 +03:00
}
2011-12-19 00:00:14 +04:00
2010-02-15 12:44:46 +03:00
current - > backing_dev_info = NULL ;
2011-12-19 00:00:14 +04:00
out :
2017-01-28 10:22:56 +03:00
if ( iolock )
xfs_iunlock ( ip , iolock ) ;
2018-05-02 22:54:52 +03:00
if ( ret > 0 ) {
XFS_STATS_ADD ( ip - > i_mount , xs_write_bytes , ret ) ;
/* Handle various SYNC-type writes */
ret = generic_write_sync ( iocb , ret ) ;
}
2011-01-11 02:17:30 +03:00
return ret ;
}
STATIC ssize_t
2014-04-03 22:20:23 +04:00
xfs_file_write_iter (
2011-01-11 02:17:30 +03:00
struct kiocb * iocb ,
2014-04-03 22:20:23 +04:00
struct iov_iter * from )
2011-01-11 02:17:30 +03:00
{
struct file * file = iocb - > ki_filp ;
struct address_space * mapping = file - > f_mapping ;
struct inode * inode = mapping - > host ;
struct xfs_inode * ip = XFS_I ( inode ) ;
ssize_t ret ;
2014-04-03 22:20:23 +04:00
size_t ocount = iov_iter_count ( from ) ;
2011-01-11 02:17:30 +03:00
2015-10-12 10:21:22 +03:00
XFS_STATS_INC ( ip - > i_mount , xs_write_calls ) ;
2011-01-11 02:17:30 +03:00
if ( ocount = = 0 )
return 0 ;
2014-04-03 22:20:23 +04:00
if ( XFS_FORCED_SHUTDOWN ( ip - > i_mount ) )
return - EIO ;
2011-01-11 02:17:30 +03:00
2016-07-20 04:38:55 +03:00
if ( IS_DAX ( inode ) )
2018-05-02 22:54:52 +03:00
return xfs_file_dax_write ( iocb , from ) ;
if ( iocb - > ki_flags & IOCB_DIRECT ) {
2016-10-03 19:11:37 +03:00
/*
* Allow a directio write to fall back to a buffered
* write * only * in the case that we ' re doing a reflink
* CoW . In all other directio scenarios we do not
* allow an operation to fall back to buffered mode .
*/
2021-01-23 21:06:28 +03:00
ret = xfs_file_dio_write ( iocb , from ) ;
2020-07-24 08:45:58 +03:00
if ( ret ! = - ENOTBLK )
2018-05-02 22:54:52 +03:00
return ret ;
2016-10-03 19:11:37 +03:00
}
2010-02-15 12:44:46 +03:00
2021-01-23 21:06:28 +03:00
return xfs_file_buffered_write ( iocb , from ) ;
2010-02-15 12:44:46 +03:00
}
2018-05-10 01:47:49 +03:00
static void
xfs_wait_dax_page (
2018-08-10 18:48:18 +03:00
struct inode * inode )
2018-05-10 01:47:49 +03:00
{
struct xfs_inode * ip = XFS_I ( inode ) ;
xfs_iunlock ( ip , XFS_MMAPLOCK_EXCL ) ;
schedule ( ) ;
xfs_ilock ( ip , XFS_MMAPLOCK_EXCL ) ;
}
static int
xfs_break_dax_layouts (
struct inode * inode ,
2018-08-10 18:48:18 +03:00
bool * retry )
2018-05-10 01:47:49 +03:00
{
struct page * page ;
ASSERT ( xfs_isilocked ( XFS_I ( inode ) , XFS_MMAPLOCK_EXCL ) ) ;
page = dax_layout_busy_page ( inode - > i_mapping ) ;
if ( ! page )
return 0 ;
2018-08-10 18:48:18 +03:00
* retry = true ;
2018-05-10 01:47:49 +03:00
return ___wait_var_event ( & page - > _refcount ,
atomic_read ( & page - > _refcount ) = = 1 , TASK_INTERRUPTIBLE ,
2018-08-10 18:48:18 +03:00
0 , 0 , xfs_wait_dax_page ( inode ) ) ;
2018-05-10 01:47:49 +03:00
}
2018-03-21 00:42:38 +03:00
int
xfs_break_layouts (
struct inode * inode ,
uint * iolock ,
enum layout_break_reason reason )
{
bool retry ;
2018-05-10 01:47:49 +03:00
int error ;
2018-03-21 00:42:38 +03:00
ASSERT ( xfs_isilocked ( XFS_I ( inode ) , XFS_IOLOCK_SHARED | XFS_IOLOCK_EXCL ) ) ;
2018-05-10 01:47:49 +03:00
do {
retry = false ;
switch ( reason ) {
case BREAK_UNMAP :
2018-07-12 08:26:36 +03:00
error = xfs_break_dax_layouts ( inode , & retry ) ;
2018-05-10 01:47:49 +03:00
if ( error | | retry )
break ;
2021-04-21 01:54:36 +03:00
fallthrough ;
2018-05-10 01:47:49 +03:00
case BREAK_WRITE :
error = xfs_break_leased_layouts ( inode , iolock , & retry ) ;
break ;
default :
WARN_ON_ONCE ( 1 ) ;
error = - EINVAL ;
}
} while ( error = = 0 & & retry ) ;
return error ;
2018-03-21 00:42:38 +03:00
}
2015-03-25 07:08:56 +03:00
# define XFS_FALLOC_FL_SUPPORTED \
( FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | \
2016-10-03 19:11:43 +03:00
FALLOC_FL_INSERT_RANGE | FALLOC_FL_UNSHARE_RANGE )
2015-03-25 07:08:56 +03:00
2011-01-14 15:07:43 +03:00
STATIC long
xfs_file_fallocate (
2013-10-12 11:55:07 +04:00
struct file * file ,
int mode ,
loff_t offset ,
loff_t len )
2011-01-14 15:07:43 +03:00
{
2013-10-12 11:55:07 +04:00
struct inode * inode = file_inode ( file ) ;
struct xfs_inode * ip = XFS_I ( inode ) ;
long error ;
2015-02-02 01:53:56 +03:00
enum xfs_prealloc_flags flags = 0 ;
2018-03-13 00:12:29 +03:00
uint iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL ;
2013-10-12 11:55:07 +04:00
loff_t new_size = 0 ;
2017-10-09 21:38:54 +03:00
bool do_file_insert = false ;
2011-01-14 15:07:43 +03:00
2013-10-12 11:55:07 +04:00
if ( ! S_ISREG ( inode - > i_mode ) )
return - EINVAL ;
2015-03-25 07:08:56 +03:00
if ( mode & ~ XFS_FALLOC_FL_SUPPORTED )
2011-01-14 15:07:43 +03:00
return - EOPNOTSUPP ;
2015-02-16 03:59:50 +03:00
xfs_ilock ( ip , iolock ) ;
2018-03-21 00:42:38 +03:00
error = xfs_break_layouts ( inode , & iolock , BREAK_UNMAP ) ;
2015-02-16 03:59:50 +03:00
if ( error )
goto out_unlock ;
xfs: properly serialise fallocate against AIO+DIO
AIO+DIO can extend the file size on IO completion, and it holds
no inode locks while the IO is in flight. Therefore, a race
condition exists in file size updates if we do something like this:
aio-thread fallocate-thread
lock inode
submit IO beyond inode->i_size
unlock inode
.....
lock inode
break layouts
if (off + len > inode->i_size)
new_size = off + len
.....
inode_dio_wait()
<blocks>
.....
completes
inode->i_size updated
inode_dio_done()
....
<wakes>
<does stuff no long beyond EOF>
if (new_size)
xfs_vn_setattr(inode, new_size)
Yup, that attempt to extend the file size in the fallocate code
turns into a truncate - it removes the whatever the aio write
allocated and put to disk, and reduced the inode size back down to
where the fallocate operation ends.
Fundamentally, xfs_file_fallocate() not compatible with racing
AIO+DIO completions, so we need to move the inode_dio_wait() call
up to where the lock the inode and break the layouts.
Secondly, storing the inode size and then using it unchecked without
holding the ILOCK is not safe; we can only do such a thing if we've
locked out and drained all IO and other modification operations,
which we don't do initially in xfs_file_fallocate.
It should be noted that some of the fallocate operations are
compound operations - they are made up of multiple manipulations
that may zero data, and so we may need to flush and invalidate the
file multiple times during an operation. However, we only need to
lock out IO and other space manipulation operations once, as that
lockout is maintained until the entire fallocate operation has been
completed.
Signed-off-by: Dave Chinner <dchinner@redhat.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Brian Foster <bfoster@redhat.com>
Reviewed-by: Darrick J. Wong <darrick.wong@oracle.com>
Signed-off-by: Darrick J. Wong <darrick.wong@oracle.com>
2019-10-29 23:04:32 +03:00
/*
* Must wait for all AIO to complete before we continue as AIO can
* change the file size on completion without holding any locks we
* currently hold . We must do this first because AIO can update both
* the on disk and in memory inode sizes , and the operations that follow
* require the in - memory size to be fully up - to - date .
*/
inode_dio_wait ( inode ) ;
/*
* Now AIO and DIO has drained we flush and ( if necessary ) invalidate
* the cached range over the first operation we are about to run .
*
* We care about zero and collapse here because they both run a hole
* punch over the range first . Because that can zero data , and the range
* of invalidation for the shift operations is much larger , we still do
* the required flush for collapse in xfs_prepare_shift ( ) .
*
* Insert has the same range requirements as collapse , and we extend the
* file first which can zero data . Hence insert has the same
* flush / invalidate requirements as collapse and so they are both
* handled at the right time by xfs_prepare_shift ( ) .
*/
if ( mode & ( FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE |
FALLOC_FL_COLLAPSE_RANGE ) ) {
error = xfs_flush_unmap_range ( ip , offset , len ) ;
if ( error )
goto out_unlock ;
}
2013-10-12 11:55:07 +04:00
if ( mode & FALLOC_FL_PUNCH_HOLE ) {
error = xfs_free_file_space ( ip , offset , len ) ;
if ( error )
goto out_unlock ;
2014-02-24 03:58:19 +04:00
} else if ( mode & FALLOC_FL_COLLAPSE_RANGE ) {
2020-10-10 02:42:59 +03:00
if ( ! xfs_is_falloc_aligned ( ip , offset , len ) ) {
2014-06-25 08:58:08 +04:00
error = - EINVAL ;
2014-02-24 03:58:19 +04:00
goto out_unlock ;
}
2014-04-12 17:56:41 +04:00
/*
* There is no need to overlap collapse range with EOF ,
* in which case it is effectively a truncate operation
*/
if ( offset + len > = i_size_read ( inode ) ) {
2014-06-25 08:58:08 +04:00
error = - EINVAL ;
2014-04-12 17:56:41 +04:00
goto out_unlock ;
}
2014-02-24 03:58:19 +04:00
new_size = i_size_read ( inode ) - len ;
error = xfs_collapse_file_space ( ip , offset , len ) ;
if ( error )
goto out_unlock ;
2015-03-25 07:08:56 +03:00
} else if ( mode & FALLOC_FL_INSERT_RANGE ) {
2018-04-17 09:07:45 +03:00
loff_t isize = i_size_read ( inode ) ;
2015-03-25 07:08:56 +03:00
2020-10-10 02:42:59 +03:00
if ( ! xfs_is_falloc_aligned ( ip , offset , len ) ) {
2015-03-25 07:08:56 +03:00
error = - EINVAL ;
goto out_unlock ;
}
2018-04-17 09:07:45 +03:00
/*
* New inode size must not exceed - > s_maxbytes , accounting for
* possible signed overflow .
*/
if ( inode - > i_sb - > s_maxbytes - isize < len ) {
2015-03-25 07:08:56 +03:00
error = - EFBIG ;
goto out_unlock ;
}
2018-04-17 09:07:45 +03:00
new_size = isize + len ;
2015-03-25 07:08:56 +03:00
/* Offset should be less than i_size */
2018-04-17 09:07:45 +03:00
if ( offset > = isize ) {
2015-03-25 07:08:56 +03:00
error = - EINVAL ;
goto out_unlock ;
}
2017-10-09 21:38:54 +03:00
do_file_insert = true ;
2013-10-12 11:55:07 +04:00
} else {
2015-02-02 01:53:56 +03:00
flags | = XFS_PREALLOC_SET ;
2013-10-12 11:55:07 +04:00
if ( ! ( mode & FALLOC_FL_KEEP_SIZE ) & &
offset + len > i_size_read ( inode ) ) {
new_size = offset + len ;
2014-06-25 08:58:08 +04:00
error = inode_newsize_ok ( inode , new_size ) ;
2013-10-12 11:55:07 +04:00
if ( error )
goto out_unlock ;
}
2011-01-14 15:07:43 +03:00
2019-02-18 20:38:49 +03:00
if ( mode & FALLOC_FL_ZERO_RANGE ) {
2019-10-25 08:26:27 +03:00
/*
* Punch a hole and prealloc the range . We use a hole
* punch rather than unwritten extent conversion for two
* reasons :
*
* 1. ) Hole punch handles partial block zeroing for us .
* 2. ) If prealloc returns ENOSPC , the file range is
* still zero - valued by virtue of the hole punch .
*/
unsigned int blksize = i_blocksize ( inode ) ;
trace_xfs_zero_file_space ( ip ) ;
error = xfs_free_file_space ( ip , offset , len ) ;
if ( error )
goto out_unlock ;
len = round_up ( offset + len , blksize ) -
round_down ( offset , blksize ) ;
offset = round_down ( offset , blksize ) ;
2019-02-18 20:38:49 +03:00
} else if ( mode & FALLOC_FL_UNSHARE_RANGE ) {
error = xfs_reflink_unshare ( ip , offset , len ) ;
if ( error )
goto out_unlock ;
} else {
/*
* If always_cow mode we can ' t use preallocations and
* thus should not create them .
*/
if ( xfs_is_always_cow_inode ( ip ) ) {
error = - EOPNOTSUPP ;
goto out_unlock ;
}
2019-10-25 08:26:27 +03:00
}
2019-02-18 20:38:49 +03:00
2019-10-25 08:26:27 +03:00
if ( ! xfs_is_always_cow_inode ( ip ) ) {
2014-03-13 12:07:58 +04:00
error = xfs_alloc_file_space ( ip , offset , len ,
XFS_BMAPI_PREALLOC ) ;
2019-10-25 08:26:27 +03:00
if ( error )
goto out_unlock ;
2016-10-03 19:11:43 +03:00
}
2011-01-14 15:07:43 +03:00
}
2013-10-12 11:55:07 +04:00
if ( file - > f_flags & O_DSYNC )
2015-02-02 01:53:56 +03:00
flags | = XFS_PREALLOC_SYNC ;
error = xfs_update_prealloc_flags ( ip , flags ) ;
2011-01-14 15:07:43 +03:00
if ( error )
goto out_unlock ;
/* Change file size if needed */
if ( new_size ) {
struct iattr iattr ;
iattr . ia_valid = ATTR_SIZE ;
iattr . ia_size = new_size ;
2021-01-21 16:19:58 +03:00
error = xfs_vn_setattr_size ( file_mnt_user_ns ( file ) ,
file_dentry ( file ) , & iattr ) ;
2015-03-25 07:08:56 +03:00
if ( error )
goto out_unlock ;
2011-01-14 15:07:43 +03:00
}
2015-03-25 07:08:56 +03:00
/*
* Perform hole insertion now that the file size has been
* updated so that if we crash during the operation we don ' t
* leave shifted extents past EOF and hence losing access to
* the data that is contained within them .
*/
if ( do_file_insert )
error = xfs_insert_file_space ( ip , offset , len ) ;
2011-01-14 15:07:43 +03:00
out_unlock :
2015-02-16 03:59:50 +03:00
xfs_iunlock ( ip , iolock ) ;
2014-06-25 08:58:08 +04:00
return error ;
2011-01-14 15:07:43 +03:00
}
2019-08-29 19:04:12 +03:00
STATIC int
xfs_file_fadvise (
struct file * file ,
loff_t start ,
loff_t end ,
int advice )
{
struct xfs_inode * ip = XFS_I ( file_inode ( file ) ) ;
int ret ;
int lockflags = 0 ;
/*
* Operations creating pages in page cache need protection from hole
* punching and similar ops
*/
if ( advice = = POSIX_FADV_WILLNEED ) {
lockflags = XFS_IOLOCK_SHARED ;
xfs_ilock ( ip , lockflags ) ;
}
ret = generic_fadvise ( file , start , end , advice ) ;
if ( lockflags )
xfs_iunlock ( ip , lockflags ) ;
return ret ;
}
2018-10-30 02:47:26 +03:00
2020-09-04 20:20:16 +03:00
/* Does this file, inode, or mount want synchronous writes? */
static inline bool xfs_file_sync_writes ( struct file * filp )
{
struct xfs_inode * ip = XFS_I ( file_inode ( filp ) ) ;
if ( ip - > i_mount - > m_flags & XFS_MOUNT_WSYNC )
return true ;
if ( filp - > f_flags & ( __O_SYNC | O_DSYNC ) )
return true ;
if ( IS_SYNC ( file_inode ( filp ) ) )
return true ;
return false ;
}
2018-11-15 08:48:18 +03:00
STATIC loff_t
2018-10-30 02:41:21 +03:00
xfs_file_remap_range (
2018-10-30 02:47:26 +03:00
struct file * file_in ,
loff_t pos_in ,
struct file * file_out ,
loff_t pos_out ,
loff_t len ,
unsigned int remap_flags )
2016-10-03 19:11:40 +03:00
{
2018-10-30 02:47:26 +03:00
struct inode * inode_in = file_inode ( file_in ) ;
struct xfs_inode * src = XFS_I ( inode_in ) ;
struct inode * inode_out = file_inode ( file_out ) ;
struct xfs_inode * dest = XFS_I ( inode_out ) ;
struct xfs_mount * mp = src - > i_mount ;
loff_t remapped = 0 ;
xfs_extlen_t cowextsize ;
int ret ;
2018-10-30 02:41:21 +03:00
if ( remap_flags & ~ ( REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY ) )
return - EINVAL ;
2016-10-03 19:11:41 +03:00
2018-10-30 02:47:26 +03:00
if ( ! xfs_sb_version_hasreflink ( & mp - > m_sb ) )
return - EOPNOTSUPP ;
if ( XFS_FORCED_SHUTDOWN ( mp ) )
return - EIO ;
/* Prepare and then clone file data. */
ret = xfs_reflink_remap_prep ( file_in , pos_in , file_out , pos_out ,
& len , remap_flags ) ;
2020-06-30 00:47:20 +03:00
if ( ret | | len = = 0 )
2018-10-30 02:47:26 +03:00
return ret ;
trace_xfs_reflink_remap_range ( src , pos_in , len , dest , pos_out ) ;
ret = xfs_reflink_remap_blocks ( src , pos_in , dest , pos_out , len ,
& remapped ) ;
if ( ret )
goto out_unlock ;
/*
* Carry the cowextsize hint from src to dest if we ' re sharing the
* entire source file to the entire destination file , the source file
* has a cowextsize hint , and the destination file does not .
*/
cowextsize = 0 ;
if ( pos_in = = 0 & & len = = i_size_read ( inode_in ) & &
2021-03-29 21:11:45 +03:00
( src - > i_diflags2 & XFS_DIFLAG2_COWEXTSIZE ) & &
2018-10-30 02:47:26 +03:00
pos_out = = 0 & & len > = i_size_read ( inode_out ) & &
2021-03-29 21:11:45 +03:00
! ( dest - > i_diflags2 & XFS_DIFLAG2_COWEXTSIZE ) )
2021-03-29 21:11:42 +03:00
cowextsize = src - > i_cowextsize ;
2018-10-30 02:47:26 +03:00
ret = xfs_reflink_update_dest ( dest , pos_out + len , cowextsize ,
remap_flags ) ;
2020-04-03 21:45:37 +03:00
if ( ret )
goto out_unlock ;
2018-10-30 02:47:26 +03:00
2020-09-04 20:20:16 +03:00
if ( xfs_file_sync_writes ( file_in ) | | xfs_file_sync_writes ( file_out ) )
2020-04-03 21:45:37 +03:00
xfs_log_force_inode ( dest ) ;
2018-10-30 02:47:26 +03:00
out_unlock :
2020-06-30 00:47:20 +03:00
xfs_iunlock2_io_mmap ( src , dest ) ;
2018-10-30 02:47:26 +03:00
if ( ret )
trace_xfs_reflink_remap_range_error ( dest , ret , _RET_IP_ ) ;
return remapped > 0 ? remapped : ret ;
2016-10-03 19:11:40 +03:00
}
2011-01-14 15:07:43 +03:00
2005-04-17 02:20:36 +04:00
STATIC int
2006-03-14 06:00:35 +03:00
xfs_file_open (
2005-04-17 02:20:36 +04:00
struct inode * inode ,
2008-11-28 06:23:32 +03:00
struct file * file )
2005-04-17 02:20:36 +04:00
{
2008-11-28 06:23:32 +03:00
if ( ! ( file - > f_flags & O_LARGEFILE ) & & i_size_read ( inode ) > MAX_NON_LFS )
2005-04-17 02:20:36 +04:00
return - EFBIG ;
2008-11-28 06:23:32 +03:00
if ( XFS_FORCED_SHUTDOWN ( XFS_M ( inode - > i_sb ) ) )
return - EIO ;
2020-05-22 18:27:33 +03:00
file - > f_mode | = FMODE_NOWAIT | FMODE_BUF_RASYNC ;
2008-11-28 06:23:32 +03:00
return 0 ;
}
STATIC int
xfs_dir_open (
struct inode * inode ,
struct file * file )
{
struct xfs_inode * ip = XFS_I ( inode ) ;
int mode ;
int error ;
error = xfs_file_open ( inode , file ) ;
if ( error )
return error ;
/*
* If there are any blocks , read - ahead block 0 as we ' re almost
* certain to have the next operation be a read there .
*/
2013-12-07 00:30:09 +04:00
mode = xfs_ilock_data_map_shared ( ip ) ;
2020-05-18 20:27:22 +03:00
if ( ip - > i_df . if_nextents > 0 )
2019-11-20 20:46:02 +03:00
error = xfs_dir3_data_readahead ( ip , 0 , 0 ) ;
2008-11-28 06:23:32 +03:00
xfs_iunlock ( ip , mode ) ;
2017-02-03 02:13:58 +03:00
return error ;
2005-04-17 02:20:36 +04:00
}
STATIC int
2006-03-14 06:00:35 +03:00
xfs_file_release (
2005-04-17 02:20:36 +04:00
struct inode * inode ,
struct file * filp )
{
2014-06-25 08:58:08 +04:00
return xfs_release ( XFS_I ( inode ) ) ;
2005-04-17 02:20:36 +04:00
}
STATIC int
2006-03-14 06:00:35 +03:00
xfs_file_readdir (
2013-05-23 01:07:56 +04:00
struct file * file ,
struct dir_context * ctx )
2005-04-17 02:20:36 +04:00
{
2013-05-23 01:07:56 +04:00
struct inode * inode = file_inode ( file ) ;
2007-08-29 04:58:01 +04:00
xfs_inode_t * ip = XFS_I ( inode ) ;
2007-08-28 07:58:24 +04:00
size_t bufsize ;
/*
* The Linux API doesn ' t pass down the total size of the buffer
* we read into down to the filesystem . With the filldir concept
* it ' s not needed for correct information , but the XFS dir2 leaf
* code wants an estimate of the buffer size to calculate it ' s
* readahead window and size the buffers used for mapping to
* physical blocks .
*
* Try to give it an estimate that ' s good enough , maybe at some
* point we can change the - > readdir prototype to include the
2010-02-03 20:50:13 +03:00
* buffer size . For now we use the current glibc buffer size .
2007-08-28 07:58:24 +04:00
*/
2021-03-29 21:11:40 +03:00
bufsize = ( size_t ) min_t ( loff_t , XFS_READDIR_BUFSIZE , ip - > i_disk_size ) ;
2007-08-28 07:58:24 +04:00
2017-06-16 21:00:14 +03:00
return xfs_readdir ( NULL , ip , ctx , bufsize ) ;
2012-05-10 17:29:17 +04:00
}
STATIC loff_t
xfs_file_llseek (
struct file * file ,
loff_t offset ,
2014-09-09 05:57:10 +04:00
int whence )
2012-05-10 17:29:17 +04:00
{
2017-06-29 21:43:21 +03:00
struct inode * inode = file - > f_mapping - > host ;
if ( XFS_FORCED_SHUTDOWN ( XFS_I ( inode ) - > i_mount ) )
return - EIO ;
2014-09-09 05:57:10 +04:00
switch ( whence ) {
2017-06-29 21:43:21 +03:00
default :
2014-09-09 05:57:10 +04:00
return generic_file_llseek ( file , offset , whence ) ;
2012-05-10 17:29:17 +04:00
case SEEK_HOLE :
2019-02-18 20:38:46 +03:00
offset = iomap_seek_hole ( inode , offset , & xfs_seek_iomap_ops ) ;
2017-06-29 21:43:21 +03:00
break ;
2014-09-09 05:56:48 +04:00
case SEEK_DATA :
2019-02-18 20:38:46 +03:00
offset = iomap_seek_data ( inode , offset , & xfs_seek_iomap_ops ) ;
2017-06-29 21:43:21 +03:00
break ;
2012-05-10 17:29:17 +04:00
}
2017-06-29 21:43:21 +03:00
if ( offset < 0 )
return offset ;
return vfs_setpos ( file , offset , inode - > i_sb - > s_maxbytes ) ;
2012-05-10 17:29:17 +04:00
}
2015-02-23 13:44:19 +03:00
/*
* Locking for serialisation of IO during page faults . This results in a lock
* ordering of :
*
2020-06-09 07:33:54 +03:00
* mmap_lock ( MM )
2015-06-04 02:18:53 +03:00
* sb_start_pagefault ( vfs , freeze )
2015-11-03 04:37:02 +03:00
* i_mmaplock ( XFS - truncate serialisation )
2015-06-04 02:18:53 +03:00
* page_lock ( MM )
* i_lock ( XFS - extent map serialisation )
2015-02-23 13:44:19 +03:00
*/
2018-05-29 20:39:03 +03:00
static vm_fault_t
2017-08-29 20:08:41 +03:00
__xfs_filemap_fault (
struct vm_fault * vmf ,
enum page_entry_size pe_size ,
bool write_fault )
2015-02-23 13:44:19 +03:00
{
2017-02-25 01:56:41 +03:00
struct inode * inode = file_inode ( vmf - > vma - > vm_file ) ;
2017-08-29 20:08:41 +03:00
struct xfs_inode * ip = XFS_I ( inode ) ;
2018-05-29 20:39:03 +03:00
vm_fault_t ret ;
2015-02-23 13:44:19 +03:00
2017-08-29 20:08:41 +03:00
trace_xfs_filemap_fault ( ip , pe_size , write_fault ) ;
2015-02-23 13:44:19 +03:00
2017-08-29 20:08:41 +03:00
if ( write_fault ) {
sb_start_pagefault ( inode - > i_sb ) ;
file_update_time ( vmf - > vma - > vm_file ) ;
}
2015-02-23 13:44:19 +03:00
2017-08-29 20:08:41 +03:00
xfs_ilock ( XFS_I ( inode ) , XFS_MMAPLOCK_SHARED ) ;
2015-06-04 02:18:53 +03:00
if ( IS_DAX ( inode ) ) {
2017-11-01 18:36:47 +03:00
pfn_t pfn ;
2019-10-19 19:09:45 +03:00
ret = dax_iomap_fault ( vmf , pe_size , & pfn , NULL ,
( write_fault & & ! vmf - > cow_page ) ?
2019-10-19 19:09:46 +03:00
& xfs_direct_write_iomap_ops :
& xfs_read_iomap_ops ) ;
2017-11-01 18:36:47 +03:00
if ( ret & VM_FAULT_NEEDDSYNC )
ret = dax_finish_sync_fault ( vmf , pe_size , pfn ) ;
2015-06-04 02:18:53 +03:00
} else {
2017-08-29 20:08:41 +03:00
if ( write_fault )
2019-10-19 19:09:46 +03:00
ret = iomap_page_mkwrite ( vmf ,
& xfs_buffered_write_iomap_ops ) ;
2017-08-29 20:08:41 +03:00
else
ret = filemap_fault ( vmf ) ;
2015-06-04 02:18:53 +03:00
}
xfs_iunlock ( XFS_I ( inode ) , XFS_MMAPLOCK_SHARED ) ;
2017-08-29 20:08:41 +03:00
if ( write_fault )
sb_end_pagefault ( inode - > i_sb ) ;
2015-06-04 02:18:53 +03:00
return ret ;
2015-02-23 13:44:19 +03:00
}
2020-09-05 15:13:02 +03:00
static inline bool
xfs_is_write_fault (
struct vm_fault * vmf )
{
return ( vmf - > flags & FAULT_FLAG_WRITE ) & &
( vmf - > vma - > vm_flags & VM_SHARED ) ;
}
2018-05-29 20:39:03 +03:00
static vm_fault_t
2015-06-04 02:18:53 +03:00
xfs_filemap_fault (
2015-02-23 13:44:54 +03:00
struct vm_fault * vmf )
{
2015-06-04 02:18:53 +03:00
/* DAX can shortcut the normal fault path on write faults! */
2017-08-29 20:08:41 +03:00
return __xfs_filemap_fault ( vmf , PE_SIZE_PTE ,
IS_DAX ( file_inode ( vmf - > vma - > vm_file ) ) & &
2020-09-05 15:13:02 +03:00
xfs_is_write_fault ( vmf ) ) ;
2015-06-04 02:18:53 +03:00
}
2018-05-29 20:39:03 +03:00
static vm_fault_t
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-25 01:56:59 +03:00
xfs_filemap_huge_fault (
2017-02-25 01:57:08 +03:00
struct vm_fault * vmf ,
enum page_entry_size pe_size )
2015-09-09 00:59:06 +03:00
{
2017-08-29 20:08:41 +03:00
if ( ! IS_DAX ( file_inode ( vmf - > vma - > vm_file ) ) )
2015-09-09 00:59:06 +03:00
return VM_FAULT_FALLBACK ;
2017-08-29 20:08:41 +03:00
/* DAX can shortcut the normal fault path on write faults! */
return __xfs_filemap_fault ( vmf , pe_size ,
2020-09-05 15:13:02 +03:00
xfs_is_write_fault ( vmf ) ) ;
2017-08-29 20:08:41 +03:00
}
2015-09-09 00:59:06 +03:00
2018-05-29 20:39:03 +03:00
static vm_fault_t
2017-08-29 20:08:41 +03:00
xfs_filemap_page_mkwrite (
struct vm_fault * vmf )
{
return __xfs_filemap_fault ( vmf , PE_SIZE_PTE , true ) ;
2015-09-09 00:59:06 +03:00
}
2015-11-03 04:37:02 +03:00
/*
2017-11-01 18:36:46 +03:00
* pfn_mkwrite was originally intended to ensure we capture time stamp updates
* on write faults . In reality , it needs to serialise against truncate and
* prepare memory for writing so handle is as standard write fault .
2015-11-03 04:37:02 +03:00
*/
2018-05-29 20:39:03 +03:00
static vm_fault_t
2015-11-03 04:37:02 +03:00
xfs_filemap_pfn_mkwrite (
struct vm_fault * vmf )
{
2017-11-01 18:36:46 +03:00
return __xfs_filemap_fault ( vmf , PE_SIZE_PTE , true ) ;
2015-09-09 00:59:06 +03:00
}
2020-12-19 15:19:23 +03:00
static vm_fault_t
2020-06-30 21:28:53 +03:00
xfs_filemap_map_pages (
struct vm_fault * vmf ,
pgoff_t start_pgoff ,
pgoff_t end_pgoff )
{
struct inode * inode = file_inode ( vmf - > vma - > vm_file ) ;
2020-12-19 15:19:23 +03:00
vm_fault_t ret ;
2020-06-30 21:28:53 +03:00
xfs_ilock ( XFS_I ( inode ) , XFS_MMAPLOCK_SHARED ) ;
2020-12-19 15:19:23 +03:00
ret = filemap_map_pages ( vmf , start_pgoff , end_pgoff ) ;
2020-06-30 21:28:53 +03:00
xfs_iunlock ( XFS_I ( inode ) , XFS_MMAPLOCK_SHARED ) ;
2020-12-19 15:19:23 +03:00
return ret ;
2020-06-30 21:28:53 +03:00
}
2015-06-04 02:18:53 +03:00
static const struct vm_operations_struct xfs_file_vm_ops = {
. fault = xfs_filemap_fault ,
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-25 01:56:59 +03:00
. huge_fault = xfs_filemap_huge_fault ,
2020-06-30 21:28:53 +03:00
. map_pages = xfs_filemap_map_pages ,
2015-06-04 02:18:53 +03:00
. page_mkwrite = xfs_filemap_page_mkwrite ,
2015-11-03 04:37:02 +03:00
. pfn_mkwrite = xfs_filemap_pfn_mkwrite ,
2015-06-04 02:18:53 +03:00
} ;
STATIC int
xfs_file_mmap (
2019-10-25 08:25:38 +03:00
struct file * file ,
struct vm_area_struct * vma )
2015-06-04 02:18:53 +03:00
{
2019-10-25 08:25:38 +03:00
struct inode * inode = file_inode ( file ) ;
struct xfs_buftarg * target = xfs_inode_buftarg ( XFS_I ( inode ) ) ;
2019-07-05 17:03:28 +03:00
2017-11-01 18:36:47 +03:00
/*
2019-07-05 17:03:28 +03:00
* We don ' t support synchronous mappings for non - DAX files and
* for DAX files if underneath dax_device is not synchronous .
2017-11-01 18:36:47 +03:00
*/
2019-10-25 08:25:38 +03:00
if ( ! daxdev_mapping_supported ( vma , target - > bt_daxdev ) )
2017-11-01 18:36:47 +03:00
return - EOPNOTSUPP ;
2019-10-25 08:25:38 +03:00
file_accessed ( file ) ;
2015-06-04 02:18:53 +03:00
vma - > vm_ops = & xfs_file_vm_ops ;
2019-10-25 08:25:38 +03:00
if ( IS_DAX ( inode ) )
2018-08-18 01:43:40 +03:00
vma - > vm_flags | = VM_HUGEPAGE ;
2015-06-04 02:18:53 +03:00
return 0 ;
2015-02-23 13:44:54 +03:00
}
2006-03-28 13:56:42 +04:00
const struct file_operations xfs_file_operations = {
2012-05-10 17:29:17 +04:00
. llseek = xfs_file_llseek ,
2014-04-02 22:37:59 +04:00
. read_iter = xfs_file_read_iter ,
2014-04-03 22:20:23 +04:00
. write_iter = xfs_file_write_iter ,
2016-09-23 06:35:42 +03:00
. splice_read = generic_file_splice_read ,
2014-04-05 12:27:08 +04:00
. splice_write = iter_file_splice_write ,
2018-12-04 21:12:08 +03:00
. iopoll = iomap_dio_iopoll ,
2006-03-14 06:00:35 +03:00
. unlocked_ioctl = xfs_file_ioctl ,
2005-04-17 02:20:36 +04:00
# ifdef CONFIG_COMPAT
2006-03-14 06:00:35 +03:00
. compat_ioctl = xfs_file_compat_ioctl ,
2005-04-17 02:20:36 +04:00
# endif
2006-03-14 06:00:35 +03:00
. mmap = xfs_file_mmap ,
2017-11-01 18:36:47 +03:00
. mmap_supported_flags = MAP_SYNC ,
2006-03-14 06:00:35 +03:00
. open = xfs_file_open ,
. release = xfs_file_release ,
. fsync = xfs_file_fsync ,
2016-10-08 02:59:59 +03:00
. get_unmapped_area = thp_get_unmapped_area ,
2011-01-14 15:07:43 +03:00
. fallocate = xfs_file_fallocate ,
2019-08-29 19:04:12 +03:00
. fadvise = xfs_file_fadvise ,
2018-10-30 02:41:21 +03:00
. remap_file_range = xfs_file_remap_range ,
2005-04-17 02:20:36 +04:00
} ;
2006-03-28 13:56:42 +04:00
const struct file_operations xfs_dir_file_operations = {
2008-11-28 06:23:32 +03:00
. open = xfs_dir_open ,
2005-04-17 02:20:36 +04:00
. read = generic_read_dir ,
2016-04-21 06:42:46 +03:00
. iterate_shared = xfs_file_readdir ,
2008-08-24 15:24:41 +04:00
. llseek = generic_file_llseek ,
2006-03-14 06:00:35 +03:00
. unlocked_ioctl = xfs_file_ioctl ,
2005-05-06 17:44:46 +04:00
# ifdef CONFIG_COMPAT
2006-03-14 06:00:35 +03:00
. compat_ioctl = xfs_file_compat_ioctl ,
2005-05-06 17:44:46 +04:00
# endif
2011-10-02 18:25:16 +04:00
. fsync = xfs_dir_fsync ,
2005-04-17 02:20:36 +04:00
} ;