Skip to content

Commit

Permalink
memcg: consolidate memory cgroup lru stat functions
Browse files Browse the repository at this point in the history 
In mm/memcontrol.c, there are many lru stat functions as..

  mem_cgroup_zone_nr_lru_pages
  mem_cgroup_node_nr_file_lru_pages
  mem_cgroup_nr_file_lru_pages
  mem_cgroup_node_nr_anon_lru_pages
  mem_cgroup_nr_anon_lru_pages
  mem_cgroup_node_nr_unevictable_lru_pages
  mem_cgroup_nr_unevictable_lru_pages
  mem_cgroup_node_nr_lru_pages
  mem_cgroup_nr_lru_pages
  mem_cgroup_get_local_zonestat

Some of them are under #ifdef MAX_NUMNODES >1 and others are not.
This seems bad. This patch consolidates all functions into

  mem_cgroup_zone_nr_lru_pages()
  mem_cgroup_node_nr_lru_pages()
  mem_cgroup_nr_lru_pages()

For these functions, "which LRU?" information is passed by a mask.

example:
  mem_cgroup_nr_lru_pages(mem, BIT(LRU_ACTIVE_ANON))

And I added some macro as ALL_LRU, ALL_LRU_FILE, ALL_LRU_ANON.

example:
  mem_cgroup_nr_lru_pages(mem, ALL_LRU)

BTW, considering layout of NUMA memory placement of counters, this patch seems
to be better.

Now, when we gather all LRU information, we scan in following orer
    for_each_lru -> for_each_node -> for_each_zone.

This means we'll touch cache lines in different node in turn.

After patch, we'll scan
    for_each_node -> for_each_zone -> for_each_lru(mask)

Then, we'll gather information in the same cacheline at once.

[[email protected]: fix warnigns, build error]
Signed-off-by: KAMEZAWA Hiroyuki <[email protected]>
Cc: Daisuke Nishimura <[email protected]>
Cc: Balbir Singh <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Ying Han <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
@hkamezawa @torvalds
hkamezawa authored and torvalds committed Jul 26, 2011
1 parent 1f4c025 commit bb2a0de
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 132 deletions.
7 changes: 3 additions & 4 deletions include/linux/memcontrol.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,7 @@ int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg);
int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg);
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
struct zone *zone,
enum lru_list lru);
int nid, int zid, unsigned int lrumask);
struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
struct zone *zone);
struct zone_reclaim_stat*
Expand Down Expand Up @@ -313,8 +312,8 @@ mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
}

static inline unsigned long
mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, struct zone *zone,
enum lru_list lru)
mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
unsigned int lru_mask)
{
return 0;
}
Expand Down
6 changes: 6 additions & 0 deletions include/linux/mmzone.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,12 @@ static inline int is_unevictable_lru(enum lru_list l)
return (l == LRU_UNEVICTABLE);
}

/* Mask used at gathering information at once (see memcontrol.c) */
#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
#define LRU_ALL_EVICTABLE (LRU_ALL_FILE | LRU_ALL_ANON)
#define LRU_ALL ((1 << NR_LRU_LISTS) - 1)

enum zone_watermarks {
WMARK_MIN,
WMARK_LOW,
Expand Down
176 changes: 49 additions & 127 deletions mm/memcontrol.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -636,27 +636,44 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
preempt_enable();
}

static unsigned long
mem_cgroup_get_zonestat_node(struct mem_cgroup *mem, int nid, enum lru_list idx)
unsigned long
mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *mem, int nid, int zid,
unsigned int lru_mask)
{
struct mem_cgroup_per_zone *mz;
enum lru_list l;
unsigned long ret = 0;

mz = mem_cgroup_zoneinfo(mem, nid, zid);

for_each_lru(l) {
if (BIT(l) & lru_mask)
ret += MEM_CGROUP_ZSTAT(mz, l);
}
return ret;
}

static unsigned long
mem_cgroup_node_nr_lru_pages(struct mem_cgroup *mem,
int nid, unsigned int lru_mask)
{
u64 total = 0;
int zid;

for (zid = 0; zid < MAX_NR_ZONES; zid++) {
mz = mem_cgroup_zoneinfo(mem, nid, zid);
total += MEM_CGROUP_ZSTAT(mz, idx);
}
for (zid = 0; zid < MAX_NR_ZONES; zid++)
total += mem_cgroup_zone_nr_lru_pages(mem, nid, zid, lru_mask);

return total;
}
static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
enum lru_list idx)

static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *mem,
unsigned int lru_mask)
{
int nid;
u64 total = 0;

for_each_online_node(nid)
total += mem_cgroup_get_zonestat_node(mem, nid, idx);
for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_node_nr_lru_pages(mem, nid, lru_mask);
return total;
}

Expand Down Expand Up @@ -1077,8 +1094,8 @@ static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_
unsigned long gb;
unsigned long inactive_ratio;

inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
inactive = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_ANON));
active = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_ANON));

gb = (inactive + active) >> (30 - PAGE_SHIFT);
if (gb)
Expand Down Expand Up @@ -1117,109 +1134,12 @@ int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
unsigned long active;
unsigned long inactive;

inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
inactive = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_FILE));
active = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_FILE));

return (active > inactive);
}

unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
struct zone *zone,
enum lru_list lru)
{
int nid = zone_to_nid(zone);
int zid = zone_idx(zone);
struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);

return MEM_CGROUP_ZSTAT(mz, lru);
}

static unsigned long mem_cgroup_node_nr_file_lru_pages(struct mem_cgroup *memcg,
int nid)
{
unsigned long ret;

ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_FILE) +
mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_FILE);

return ret;
}

static unsigned long mem_cgroup_node_nr_anon_lru_pages(struct mem_cgroup *memcg,
int nid)
{
unsigned long ret;

ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_ANON) +
mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_ANON);
return ret;
}

#if MAX_NUMNODES > 1
static unsigned long mem_cgroup_nr_file_lru_pages(struct mem_cgroup *memcg)
{
u64 total = 0;
int nid;

for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_node_nr_file_lru_pages(memcg, nid);

return total;
}

static unsigned long mem_cgroup_nr_anon_lru_pages(struct mem_cgroup *memcg)
{
u64 total = 0;
int nid;

for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_node_nr_anon_lru_pages(memcg, nid);

return total;
}

static unsigned long
mem_cgroup_node_nr_unevictable_lru_pages(struct mem_cgroup *memcg, int nid)
{
return mem_cgroup_get_zonestat_node(memcg, nid, LRU_UNEVICTABLE);
}

static unsigned long
mem_cgroup_nr_unevictable_lru_pages(struct mem_cgroup *memcg)
{
u64 total = 0;
int nid;

for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_node_nr_unevictable_lru_pages(memcg, nid);

return total;
}

static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
int nid)
{
enum lru_list l;
u64 total = 0;

for_each_lru(l)
total += mem_cgroup_get_zonestat_node(memcg, nid, l);

return total;
}

static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg)
{
u64 total = 0;
int nid;

for_each_node_state(nid, N_HIGH_MEMORY)
total += mem_cgroup_node_nr_lru_pages(memcg, nid);

return total;
}
#endif /* CONFIG_NUMA */

struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
struct zone *zone)
{
Expand Down Expand Up @@ -1576,11 +1496,11 @@ mem_cgroup_select_victim(struct mem_cgroup *root_mem)
static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *mem,
int nid, bool noswap)
{
if (mem_cgroup_node_nr_file_lru_pages(mem, nid))
if (mem_cgroup_node_nr_lru_pages(mem, nid, LRU_ALL_FILE))
return true;
if (noswap || !total_swap_pages)
return false;
if (mem_cgroup_node_nr_anon_lru_pages(mem, nid))
if (mem_cgroup_node_nr_lru_pages(mem, nid, LRU_ALL_ANON))
return true;
return false;

Expand Down Expand Up @@ -4151,15 +4071,15 @@ mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
s->stat[MCS_PGMAJFAULT] += val;

/* per zone stat */
val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_INACTIVE_ANON));
s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_ACTIVE_ANON));
s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_INACTIVE_FILE));
s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_ACTIVE_FILE));
s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_UNEVICTABLE));
s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
}

Expand All @@ -4181,35 +4101,37 @@ static int mem_control_numa_stat_show(struct seq_file *m, void *arg)
struct cgroup *cont = m->private;
struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);

total_nr = mem_cgroup_nr_lru_pages(mem_cont);
total_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL);
seq_printf(m, "total=%lu", total_nr);
for_each_node_state(nid, N_HIGH_MEMORY) {
node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid);
node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, LRU_ALL);
seq_printf(m, " N%d=%lu", nid, node_nr);
}
seq_putc(m, '\n');

file_nr = mem_cgroup_nr_file_lru_pages(mem_cont);
file_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_FILE);
seq_printf(m, "file=%lu", file_nr);
for_each_node_state(nid, N_HIGH_MEMORY) {
node_nr = mem_cgroup_node_nr_file_lru_pages(mem_cont, nid);
node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
LRU_ALL_FILE);
seq_printf(m, " N%d=%lu", nid, node_nr);
}
seq_putc(m, '\n');

anon_nr = mem_cgroup_nr_anon_lru_pages(mem_cont);
anon_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_ANON);
seq_printf(m, "anon=%lu", anon_nr);
for_each_node_state(nid, N_HIGH_MEMORY) {
node_nr = mem_cgroup_node_nr_anon_lru_pages(mem_cont, nid);
node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
LRU_ALL_ANON);
seq_printf(m, " N%d=%lu", nid, node_nr);
}
seq_putc(m, '\n');

unevictable_nr = mem_cgroup_nr_unevictable_lru_pages(mem_cont);
unevictable_nr = mem_cgroup_nr_lru_pages(mem_cont, BIT(LRU_UNEVICTABLE));
seq_printf(m, "unevictable=%lu", unevictable_nr);
for_each_node_state(nid, N_HIGH_MEMORY) {
node_nr = mem_cgroup_node_nr_unevictable_lru_pages(mem_cont,
nid);
node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
BIT(LRU_UNEVICTABLE));
seq_printf(m, " N%d=%lu", nid, node_nr);
}
seq_putc(m, '\n');
Expand Down
3 changes: 2 additions & 1 deletion mm/vmscan.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,8 @@ static unsigned long zone_nr_lru_pages(struct zone *zone,
struct scan_control *sc, enum lru_list lru)
{
if (!scanning_global_lru(sc))
return mem_cgroup_zone_nr_lru_pages(sc->mem_cgroup, zone, lru);
return mem_cgroup_zone_nr_lru_pages(sc->mem_cgroup,
zone_to_nid(zone), zone_idx(zone), BIT(lru));

return zone_page_state(zone, NR_LRU_BASE + lru);
}
Expand Down

0 comments on commit bb2a0de

Please sign in to comment.