iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mm/damon/paddr: introduce DAMOS_MIGRATE_COLD action for demotion

Browse Source 
This patch introduces DAMOS_MIGRATE_COLD action, which is similar to
DAMOS_PAGEOUT, but migrate folios to the given 'target_nid' in the sysfs
instead of swapping them out.

The 'target_nid' sysfs knob informs the migration target node ID.

Here is one of the example usage of this 'migrate_cold' action.

  $ cd /sys/kernel/mm/damon/admin/kdamonds/<N>
  $ cat contexts/<N>/schemes/<N>/action
  migrate_cold
  $ echo 2 > contexts/<N>/schemes/<N>/target_nid
  $ echo commit > state
  $ numactl -p 0 ./hot_cold 500M 600M &
  $ numastat -c -p hot_cold

  Per-node process memory usage (in MBs)
  PID             Node 0 Node 1 Node 2 Total
  --------------  ------ ------ ------ -----
  701 (hot_cold)     501      0    601  1101

Since there are some common routines with pageout, many functions have
similar logics between pageout and migrate cold.

damon_pa_migrate_folio_list() is a minimized version of
shrink_folio_list().

Link: https://lkml.kernel.org/r/20240614030010.751-6-honggyu.kim@sk.com
Signed-off-by: Honggyu Kim <honggyu.kim@sk.com>
Signed-off-by: Hyeongtak Ji <hyeongtak.ji@sk.com>
Signed-off-by: SeongJae Park <sj@kernel.org>
Cc: Gregory Price <gregory.price@memverge.com>
Cc: Hyeonggon Yoo <42.hyeyoo@gmail.com>
Cc: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Rakie Kim <rakie.kim@sk.com>
Cc: Steven Rostedt (Google) <rostedt@goodmis.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Honggyu Kim 2024-06-14 12:00:07 +09:00 committed by Andrew Morton
parent ced816a76b
commit b51820ebea
3 changed files with 157 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/damon.h
View File

@ -105,6 +105,7 @@ struct damon_target {
* @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE.
* @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists.
* @DAMOS_LRU_DEPRIO: Deprioritize the region on its LRU lists.
* @DAMOS_MIGRATE_COLD: Migrate the regions prioritizing colder regions.
* @DAMOS_STAT: Do nothing but count the stat.
* @NR_DAMOS_ACTIONS: Total number of DAMOS actions
*
@ -122,6 +123,7 @@ enum damos_action {
DAMOS_NOHUGEPAGE,
DAMOS_LRU_PRIO,
DAMOS_LRU_DEPRIO,
DAMOS_MIGRATE_COLD,
DAMOS_STAT, /* Do nothing but only record the stat */
NR_DAMOS_ACTIONS,
};

154
mm/damon/paddr.c
View File

@ -12,6 +12,9 @@
#include <linux/pagemap.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/memory-tiers.h>
#include <linux/migrate.h>
#include <linux/mm_inline.h>
#include "../internal.h"
#include "ops-common.h"
@ -325,6 +328,153 @@ static unsigned long damon_pa_deactivate_pages(struct damon_region *r,
return damon_pa_mark_accessed_or_deactivate(r, s, false);
}
static unsigned int __damon_pa_migrate_folio_list(
struct list_head *migrate_folios, struct pglist_data *pgdat,
int target_nid)
{
unsigned int nr_succeeded;
nodemask_t allowed_mask = NODE_MASK_NONE;
struct migration_target_control mtc = {
/*
* Allocate from 'node', or fail quickly and quietly.
* When this happens, 'page' will likely just be discarded
* instead of migrated.
*/
.gfp_mask = (GFP_HIGHUSER_MOVABLE & ~__GFP_RECLAIM) |
__GFP_NOWARN | __GFP_NOMEMALLOC | GFP_NOWAIT,
.nid = target_nid,
.nmask = &allowed_mask
};
if (pgdat->node_id == target_nid || target_nid == NUMA_NO_NODE)
return 0;
if (list_empty(migrate_folios))
return 0;
/* Migration ignores all cpuset and mempolicy settings */
migrate_pages(migrate_folios, alloc_migrate_folio, NULL,
(unsigned long)&mtc, MIGRATE_ASYNC, MR_DAMON,
&nr_succeeded);
return nr_succeeded;
}
static unsigned int damon_pa_migrate_folio_list(struct list_head *folio_list,
struct pglist_data *pgdat,
int target_nid)
{
unsigned int nr_migrated = 0;
struct folio *folio;
LIST_HEAD(ret_folios);
LIST_HEAD(migrate_folios);
while (!list_empty(folio_list)) {
struct folio *folio;
cond_resched();
folio = lru_to_folio(folio_list);
list_del(&folio->lru);
if (!folio_trylock(folio))
goto keep;
/* Relocate its contents to another node. */
list_add(&folio->lru, &migrate_folios);
folio_unlock(folio);
continue;
keep:
list_add(&folio->lru, &ret_folios);
}
/* 'folio_list' is always empty here */
/* Migrate folios selected for migration */
nr_migrated += __damon_pa_migrate_folio_list(
&migrate_folios, pgdat, target_nid);
/*
* Folios that could not be migrated are still in @migrate_folios. Add
* those back on @folio_list
*/
if (!list_empty(&migrate_folios))
list_splice_init(&migrate_folios, folio_list);
try_to_unmap_flush();
list_splice(&ret_folios, folio_list);
while (!list_empty(folio_list)) {
folio = lru_to_folio(folio_list);
list_del(&folio->lru);
folio_putback_lru(folio);
}
return nr_migrated;
}
static unsigned long damon_pa_migrate_pages(struct list_head *folio_list,
int target_nid)
{
int nid;
unsigned long nr_migrated = 0;
LIST_HEAD(node_folio_list);
unsigned int noreclaim_flag;
if (list_empty(folio_list))
return nr_migrated;
noreclaim_flag = memalloc_noreclaim_save();
nid = folio_nid(lru_to_folio(folio_list));
do {
struct folio *folio = lru_to_folio(folio_list);
if (nid == folio_nid(folio)) {
list_move(&folio->lru, &node_folio_list);
continue;
}
nr_migrated += damon_pa_migrate_folio_list(&node_folio_list,
NODE_DATA(nid),
target_nid);
nid = folio_nid(lru_to_folio(folio_list));
} while (!list_empty(folio_list));
nr_migrated += damon_pa_migrate_folio_list(&node_folio_list,
NODE_DATA(nid),
target_nid);
memalloc_noreclaim_restore(noreclaim_flag);
return nr_migrated;
}
static unsigned long damon_pa_migrate(struct damon_region *r, struct damos *s)
{
unsigned long addr, applied;
LIST_HEAD(folio_list);
for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
struct folio *folio = damon_get_folio(PHYS_PFN(addr));
if (!folio)
continue;
if (damos_pa_filter_out(s, folio))
goto put_folio;
if (!folio_isolate_lru(folio))
goto put_folio;
list_add(&folio->lru, &folio_list);
put_folio:
folio_put(folio);
}
applied = damon_pa_migrate_pages(&folio_list, s->target_nid);
cond_resched();
return applied * PAGE_SIZE;
}
static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
struct damon_target *t, struct damon_region *r,
struct damos *scheme)
@ -336,6 +486,8 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
return damon_pa_mark_accessed(r, scheme);
case DAMOS_LRU_DEPRIO:
return damon_pa_deactivate_pages(r, scheme);
case DAMOS_MIGRATE_COLD:
return damon_pa_migrate(r, scheme);
case DAMOS_STAT:
break;
default:
@ -356,6 +508,8 @@ static int damon_pa_scheme_score(struct damon_ctx *context,
return damon_hot_score(context, r, scheme);
case DAMOS_LRU_DEPRIO:
return damon_cold_score(context, r, scheme);
case DAMOS_MIGRATE_COLD:
return damon_cold_score(context, r, scheme);
default:
break;
}

1
mm/damon/sysfs-schemes.c
View File

@ -1458,6 +1458,7 @@ static const char * const damon_sysfs_damos_action_strs[] = {
"nohugepage",
"lru_prio",
"lru_deprio",
"migrate_cold",
"stat",
};