iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

filemap: Return only folios from find_get_entries()

Browse Source 
The callers have all been converted to work on folios, so convert
find_get_entries() to return a batch of folios instead of pages.
We also now return multiple large folios in a single call.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: Jan Kara <jack@suse.cz>
Reviewed-by: William Kucharski <william.kucharski@oracle.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
This commit is contained in:
Matthew Wilcox (Oracle) 2020-09-01 23:17:50 -04:00
parent 25d6a23e8d
commit 0e499ed3d7
5 changed files with 59 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/pagemap.h
View File

@ -592,8 +592,6 @@ static inline struct page *find_subpage(struct page *head, pgoff_t index)
return head + (index & (thp_nr_pages(head) - 1)); return head + (index & (thp_nr_pages(head) - 1));
} }
unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
pgoff_t end, struct pagevec *pvec, pgoff_t *indices);
unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
pgoff_t end, unsigned int nr_pages, pgoff_t end, unsigned int nr_pages,
struct page **pages); struct page **pages);

43
mm/filemap.c
View File

@ -2015,57 +2015,36 @@ reset:
* @mapping: The address_space to search * @mapping: The address_space to search
* @start: The starting page cache index * @start: The starting page cache index
* @end: The final page index (inclusive). * @end: The final page index (inclusive).
* @pvec: Where the resulting entries are placed. * @fbatch: Where the resulting entries are placed.
* @indices: The cache indices corresponding to the entries in @entries * @indices: The cache indices corresponding to the entries in @entries
* *
* find_get_entries() will search for and return a batch of entries in * find_get_entries() will search for and return a batch of entries in
* the mapping. The entries are placed in @pvec. find_get_entries() * the mapping. The entries are placed in @fbatch. find_get_entries()
* takes a reference on any actual pages it returns. * takes a reference on any actual folios it returns.
* *
* The search returns a group of mapping-contiguous page cache entries * The entries have ascending indexes. The indices may not be consecutive
* with ascending indexes. There may be holes in the indices due to * due to not-present entries or large folios.
* not-present pages.
* *
* Any shadow entries of evicted pages, or swap entries from * Any shadow entries of evicted folios, or swap entries from
* shmem/tmpfs, are included in the returned array. * shmem/tmpfs, are included in the returned array.
* *
* If it finds a Transparent Huge Page, head or tail, find_get_entries() * Return: The number of entries which were found.
* stops at that page: the caller is likely to have a better way to handle
* the compound page as a whole, and then skip its extent, than repeatedly
* calling find_get_entries() to return all its tails.
*
* Return: the number of pages and shadow entries which were found.
*/ */
unsigned find_get_entries(struct address_space *mapping, pgoff_t start, unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
pgoff_t end, struct pagevec *pvec, pgoff_t *indices) pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices)
{ {
XA_STATE(xas, &mapping->i_pages, start); XA_STATE(xas, &mapping->i_pages, start);
struct folio *folio; struct folio *folio;
unsigned int ret = 0;
unsigned nr_entries = PAGEVEC_SIZE;
rcu_read_lock(); rcu_read_lock();
while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) { while ((folio = find_get_entry(&xas, end, XA_PRESENT)) != NULL) {
struct page *page = &folio->page; indices[fbatch->nr] = xas.xa_index;
/* if (!folio_batch_add(fbatch, folio))
* Terminate early on finding a THP, to allow the caller to
* handle it all at once; but continue if this is hugetlbfs.
*/
if (!xa_is_value(folio) && folio_test_large(folio) &&
!folio_test_hugetlb(folio)) {
page = folio_file_page(folio, xas.xa_index);
nr_entries = ret + 1;
}
indices[ret] = xas.xa_index;
pvec->pages[ret] = page;
if (++ret == nr_entries)
break; break;
} }
rcu_read_unlock(); rcu_read_unlock();
pvec->nr = ret; return folio_batch_count(fbatch);
return ret;
} }
/** /**

4
mm/internal.h
View File

@ -12,6 +12,8 @@
#include <linux/pagemap.h> #include <linux/pagemap.h>
#include <linux/tracepoint-defs.h> #include <linux/tracepoint-defs.h>
struct folio_batch;
/* /*
* The set of flags that only affect watermark checking and reclaim * The set of flags that only affect watermark checking and reclaim
* behaviour. This is used by the MM to obey the caller constraints * behaviour. This is used by the MM to obey the caller constraints
@ -92,6 +94,8 @@ static inline void force_page_cache_readahead(struct address_space *mapping,
unsigned find_lock_entries(struct address_space *mapping, pgoff_t start, unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
pgoff_t end, struct pagevec *pvec, pgoff_t *indices); pgoff_t end, struct pagevec *pvec, pgoff_t *indices);
unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);
void filemap_free_folio(struct address_space *mapping, struct folio *folio); void filemap_free_folio(struct address_space *mapping, struct folio *folio);
int truncate_inode_folio(struct address_space *mapping, struct folio *folio); int truncate_inode_folio(struct address_space *mapping, struct folio *folio);

36
mm/shmem.c
View File

@ -920,6 +920,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
unsigned int partial_start = lstart & (PAGE_SIZE - 1); unsigned int partial_start = lstart & (PAGE_SIZE - 1);
unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
struct pagevec pvec; struct pagevec pvec;
struct folio_batch fbatch;
pgoff_t indices[PAGEVEC_SIZE]; pgoff_t indices[PAGEVEC_SIZE];
long nr_swaps_freed = 0; long nr_swaps_freed = 0;
pgoff_t index; pgoff_t index;
@ -987,11 +988,12 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
if (start >= end) if (start >= end)
return; return;
folio_batch_init(&fbatch);
index = start; index = start;
while (index < end) { while (index < end) {
cond_resched(); cond_resched();
if (!find_get_entries(mapping, index, end - 1, &pvec, if (!find_get_entries(mapping, index, end - 1, &fbatch,
indices)) { indices)) {
/* If all gone or hole-punch or unfalloc, we're done */ /* If all gone or hole-punch or unfalloc, we're done */
if (index == start || end != -1) if (index == start || end != -1)
@ -1000,14 +1002,14 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
index = start; index = start;
continue; continue;
} }
for (i = 0; i < pagevec_count(&pvec); i++) { for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct page *page = pvec.pages[i]; struct folio *folio = fbatch.folios[i];
index = indices[i]; index = indices[i];
if (xa_is_value(page)) { if (xa_is_value(folio)) {
if (unfalloc) if (unfalloc)
continue; continue;
if (shmem_free_swap(mapping, index, page)) { if (shmem_free_swap(mapping, index, folio)) {
/* Swap was replaced by page: retry */ /* Swap was replaced by page: retry */
index--; index--;
break; break;
@ -1016,33 +1018,35 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
continue; continue;
} }
lock_page(page); folio_lock(folio);
if (!unfalloc || !PageUptodate(page)) { if (!unfalloc || !folio_test_uptodate(folio)) {
if (page_mapping(page) != mapping) { struct page *page = folio_file_page(folio,
index);
if (folio_mapping(folio) != mapping) {
/* Page was replaced by swap: retry */ /* Page was replaced by swap: retry */
unlock_page(page); folio_unlock(folio);
index--; index--;
break; break;
} }
VM_BUG_ON_PAGE(PageWriteback(page), page); VM_BUG_ON_FOLIO(folio_test_writeback(folio),
folio);
if (shmem_punch_compound(page, start, end)) if (shmem_punch_compound(page, start, end))
truncate_inode_folio(mapping, truncate_inode_folio(mapping, folio);
page_folio(page));
else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
/* Wipe the page and don't get stuck */ /* Wipe the page and don't get stuck */
clear_highpage(page); clear_highpage(page);
flush_dcache_page(page); flush_dcache_page(page);
set_page_dirty(page); folio_mark_dirty(folio);
if (index < if (index <
round_up(start, HPAGE_PMD_NR)) round_up(start, HPAGE_PMD_NR))
start = index + 1; start = index + 1;
} }
} }
unlock_page(page); folio_unlock(folio);
} }
pagevec_remove_exceptionals(&pvec); folio_batch_remove_exceptionals(&fbatch);
pagevec_release(&pvec); folio_batch_release(&fbatch);
index++; index++;
} }

43
mm/truncate.c
View File

@ -108,6 +108,13 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping,
pvec->nr = j; pvec->nr = j;
} }
static void truncate_folio_batch_exceptionals(struct address_space *mapping,
struct folio_batch *fbatch, pgoff_t *indices)
{
truncate_exceptional_pvec_entries(mapping, (struct pagevec *)fbatch,
indices);
}
/* /*
* Invalidate exceptional entry if easily possible. This handles exceptional * Invalidate exceptional entry if easily possible. This handles exceptional
* entries for invalidate_inode_pages(). * entries for invalidate_inode_pages().
@ -297,6 +304,7 @@ void truncate_inode_pages_range(struct address_space *mapping,
unsigned int partial_start; /* inclusive */ unsigned int partial_start; /* inclusive */
unsigned int partial_end; /* exclusive */ unsigned int partial_end; /* exclusive */
struct pagevec pvec; struct pagevec pvec;
struct folio_batch fbatch;
pgoff_t indices[PAGEVEC_SIZE]; pgoff_t indices[PAGEVEC_SIZE];
pgoff_t index; pgoff_t index;
int i; int i;
@ -379,10 +387,11 @@ void truncate_inode_pages_range(struct address_space *mapping,
if (start >= end) if (start >= end)
goto out; goto out;
folio_batch_init(&fbatch);
index = start; index = start;
for ( ; ; ) { for ( ; ; ) {
cond_resched(); cond_resched();
if (!find_get_entries(mapping, index, end - 1, &pvec, if (!find_get_entries(mapping, index, end - 1, &fbatch,
indices)) { indices)) {
/* If all gone from start onwards, we're done */ /* If all gone from start onwards, we're done */
if (index == start) if (index == start)
@ -392,16 +401,14 @@ void truncate_inode_pages_range(struct address_space *mapping,
continue; continue;
} }
for (i = 0; i < pagevec_count(&pvec); i++) { for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct page *page = pvec.pages[i]; struct folio *folio = fbatch.folios[i];
struct folio *folio;
/* We rely upon deletion not changing page->index */ /* We rely upon deletion not changing page->index */
index = indices[i]; index = indices[i];
if (xa_is_value(page)) if (xa_is_value(folio))
continue; continue;
folio = page_folio(page);
folio_lock(folio); folio_lock(folio);
VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio);
@ -410,8 +417,8 @@ void truncate_inode_pages_range(struct address_space *mapping,
folio_unlock(folio); folio_unlock(folio);
index = folio_index(folio) + folio_nr_pages(folio) - 1; index = folio_index(folio) + folio_nr_pages(folio) - 1;
} }
truncate_exceptional_pvec_entries(mapping, &pvec, indices); truncate_folio_batch_exceptionals(mapping, &fbatch, indices);
pagevec_release(&pvec); folio_batch_release(&fbatch);
index++; index++;
} }
@ -625,7 +632,7 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
pgoff_t start, pgoff_t end) pgoff_t start, pgoff_t end)
{ {
pgoff_t indices[PAGEVEC_SIZE]; pgoff_t indices[PAGEVEC_SIZE];
struct pagevec pvec; struct folio_batch fbatch;
pgoff_t index; pgoff_t index;
int i; int i;
int ret = 0; int ret = 0;
@ -635,23 +642,21 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
if (mapping_empty(mapping)) if (mapping_empty(mapping))
goto out; goto out;
pagevec_init(&pvec); folio_batch_init(&fbatch);
index = start; index = start;
while (find_get_entries(mapping, index, end, &pvec, indices)) { while (find_get_entries(mapping, index, end, &fbatch, indices)) {
for (i = 0; i < pagevec_count(&pvec); i++) { for (i = 0; i < folio_batch_count(&fbatch); i++) {
struct page *page = pvec.pages[i]; struct folio *folio = fbatch.folios[i];
struct folio *folio;
/* We rely upon deletion not changing folio->index */ /* We rely upon deletion not changing folio->index */
index = indices[i]; index = indices[i];
if (xa_is_value(page)) { if (xa_is_value(folio)) {
if (!invalidate_exceptional_entry2(mapping, if (!invalidate_exceptional_entry2(mapping,
index, page)) index, folio))
ret = -EBUSY; ret = -EBUSY;
continue; continue;
} }
folio = page_folio(page);
if (!did_range_unmap && folio_mapped(folio)) { if (!did_range_unmap && folio_mapped(folio)) {
/* /*
@ -684,8 +689,8 @@ int invalidate_inode_pages2_range(struct address_space *mapping,
ret = ret2; ret = ret2;
folio_unlock(folio); folio_unlock(folio);
} }
pagevec_remove_exceptionals(&pvec); folio_batch_remove_exceptionals(&fbatch);
pagevec_release(&pvec); folio_batch_release(&fbatch);
cond_resched(); cond_resched();
index++; index++;
} }