diff options
| -rw-r--r-- | drivers/base/node.c | 34 | ||||
| -rw-r--r-- | include/linux/mmzone.h | 17 | ||||
| -rw-r--r-- | include/linux/vmstat.h | 10 | ||||
| -rw-r--r-- | mm/mempolicy.c | 6 | ||||
| -rw-r--r-- | mm/page_alloc.c | 23 | ||||
| -rw-r--r-- | mm/vmstat.c | 73 |
6 files changed, 73 insertions, 90 deletions
diff --git a/drivers/base/node.c b/drivers/base/node.c index 772eadac57a7..d7de1753e094 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -94,28 +94,6 @@ static SYSDEV_ATTR(meminfo, S_IRUGO, node_read_meminfo, NULL); static ssize_t node_read_numastat(struct sys_device * dev, char * buf) { - unsigned long numa_hit, numa_miss, interleave_hit, numa_foreign; - unsigned long local_node, other_node; - int i, cpu; - pg_data_t *pg = NODE_DATA(dev->id); - numa_hit = 0; - numa_miss = 0; - interleave_hit = 0; - numa_foreign = 0; - local_node = 0; - other_node = 0; - for (i = 0; i < MAX_NR_ZONES; i++) { - struct zone *z = &pg->node_zones[i]; - for_each_online_cpu(cpu) { - struct per_cpu_pageset *ps = zone_pcp(z,cpu); - numa_hit += ps->numa_hit; - numa_miss += ps->numa_miss; - numa_foreign += ps->numa_foreign; - interleave_hit += ps->interleave_hit; - local_node += ps->local_node; - other_node += ps->other_node; - } - } return sprintf(buf, "numa_hit %lu\n" "numa_miss %lu\n" @@ -123,12 +101,12 @@ static ssize_t node_read_numastat(struct sys_device * dev, char * buf) "interleave_hit %lu\n" "local_node %lu\n" "other_node %lu\n", - numa_hit, - numa_miss, - numa_foreign, - interleave_hit, - local_node, - other_node); + node_page_state(dev->id, NUMA_HIT), + node_page_state(dev->id, NUMA_MISS), + node_page_state(dev->id, NUMA_FOREIGN), + node_page_state(dev->id, NUMA_INTERLEAVE_HIT), + node_page_state(dev->id, NUMA_LOCAL), + node_page_state(dev->id, NUMA_OTHER)); } static SYSDEV_ATTR(numastat, S_IRUGO, node_read_numastat, NULL); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 2dbeec1d2874..27e748eb72b0 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -57,6 +57,14 @@ enum zone_stat_item { NR_WRITEBACK, NR_UNSTABLE_NFS, /* NFS unstable pages */ NR_BOUNCE, +#ifdef CONFIG_NUMA + NUMA_HIT, /* allocated in intended node */ + NUMA_MISS, /* allocated in non intended node */ + NUMA_FOREIGN, /* was intended here, hit elsewhere */ + NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */ + NUMA_LOCAL, /* allocation from local node */ + NUMA_OTHER, /* allocation from other node */ +#endif NR_VM_ZONE_STAT_ITEMS }; struct per_cpu_pages { @@ -71,15 +79,6 @@ struct per_cpu_pageset { #ifdef CONFIG_SMP s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; #endif - -#ifdef CONFIG_NUMA - unsigned long numa_hit; /* allocated in intended node */ - unsigned long numa_miss; /* allocated in non intended node */ - unsigned long numa_foreign; /* was intended here, hit elsewhere */ - unsigned long interleave_hit; /* interleaver prefered this zone */ - unsigned long local_node; /* allocation from local node */ - unsigned long other_node; /* allocation from other node */ -#endif } ____cacheline_aligned_in_smp; #ifdef CONFIG_NUMA diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index 5fad1613e7d6..16173b63ee67 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -173,9 +173,15 @@ static inline unsigned long node_page_state(int node, #endif zone_page_state(&zones[ZONE_DMA], item); } + +extern void zone_statistics(struct zonelist *, struct zone *); + #else + #define node_page_state(node, item) global_page_state(item) -#endif +#define zone_statistics(_zl,_z) do { } while (0) + +#endif /* CONFIG_NUMA */ #define __add_zone_page_state(__z, __i, __d) \ __mod_zone_page_state(__z, __i, __d) @@ -190,6 +196,8 @@ static inline void zap_zone_vm_stats(struct zone *zone) memset(zone->vm_stat, 0, sizeof(zone->vm_stat)); } +extern void inc_zone_state(struct zone *, enum zone_stat_item); + #ifdef CONFIG_SMP void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int); void __inc_zone_page_state(struct page *, enum zone_stat_item); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 6b9740bbf4c0..e07e27e846a2 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1209,10 +1209,8 @@ static struct page *alloc_page_interleave(gfp_t gfp, unsigned order, zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp); page = __alloc_pages(gfp, order, zl); - if (page && page_zone(page) == zl->zones[0]) { - zone_pcp(zl->zones[0],get_cpu())->interleave_hit++; - put_cpu(); - } + if (page && page_zone(page) == zl->zones[0]) + inc_zone_page_state(page, NUMA_INTERLEAVE_HIT); return page; } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 6aa2c31f513b..d61671260f92 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -709,27 +709,6 @@ void drain_local_pages(void) } #endif /* CONFIG_PM */ -static void zone_statistics(struct zonelist *zonelist, struct zone *z, int cpu) -{ -#ifdef CONFIG_NUMA - pg_data_t *pg = z->zone_pgdat; - pg_data_t *orig = zonelist->zones[0]->zone_pgdat; - struct per_cpu_pageset *p; - - p = zone_pcp(z, cpu); - if (pg == orig) { - p->numa_hit++; - } else { - p->numa_miss++; - zone_pcp(zonelist->zones[0], cpu)->numa_foreign++; - } - if (pg == NODE_DATA(numa_node_id())) - p->local_node++; - else - p->other_node++; -#endif -} - /* * Free a 0-order page */ @@ -827,7 +806,7 @@ again: } __mod_page_state_zone(zone, pgalloc, 1 << order); - zone_statistics(zonelist, zone, cpu); + zone_statistics(zonelist, zone); local_irq_restore(flags); put_cpu(); diff --git a/mm/vmstat.c b/mm/vmstat.c index 06a6d1052198..ee7f89666250 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -185,9 +185,8 @@ EXPORT_SYMBOL(mod_zone_page_state); * in between and therefore the atomicity vs. interrupt cannot be exploited * in a useful way here. */ -void __inc_zone_page_state(struct page *page, enum zone_stat_item item) +static void __inc_zone_state(struct zone *zone, enum zone_stat_item item) { - struct zone *zone = page_zone(page); s8 *p = diff_pointer(zone, item); (*p)++; @@ -197,6 +196,11 @@ void __inc_zone_page_state(struct page *page, enum zone_stat_item item) *p = 0; } } + +void __inc_zone_page_state(struct page *page, enum zone_stat_item item) +{ + __inc_zone_state(page_zone(page), item); +} EXPORT_SYMBOL(__inc_zone_page_state); void __dec_zone_page_state(struct page *page, enum zone_stat_item item) @@ -213,22 +217,23 @@ void __dec_zone_page_state(struct page *page, enum zone_stat_item item) } EXPORT_SYMBOL(__dec_zone_page_state); +void inc_zone_state(struct zone *zone, enum zone_stat_item item) +{ + unsigned long flags; + + local_irq_save(flags); + __inc_zone_state(zone, item); + local_irq_restore(flags); +} + void inc_zone_page_state(struct page *page, enum zone_stat_item item) { unsigned long flags; struct zone *zone; - s8 *p; zone = page_zone(page); local_irq_save(flags); - p = diff_pointer(zone, item); - - (*p)++; - - if (unlikely(*p > STAT_THRESHOLD)) { - zone_page_state_add(*p, zone, item); - *p = 0; - } + __inc_zone_state(zone, item); local_irq_restore(flags); } EXPORT_SYMBOL(inc_zone_page_state); @@ -297,6 +302,28 @@ EXPORT_SYMBOL(refresh_vm_stats); #endif +#ifdef CONFIG_NUMA +/* + * zonelist = the list of zones passed to the allocator + * z = the zone from which the allocation occurred. + * + * Must be called with interrupts disabled. + */ +void zone_statistics(struct zonelist *zonelist, struct zone *z) +{ + if (z->zone_pgdat == zonelist->zones[0]->zone_pgdat) { + __inc_zone_state(z, NUMA_HIT); + } else { + __inc_zone_state(z, NUMA_MISS); + __inc_zone_state(zonelist->zones[0], NUMA_FOREIGN); + } + if (z->zone_pgdat == NODE_DATA(numa_node_id())) + __inc_zone_state(z, NUMA_LOCAL); + else + __inc_zone_state(z, NUMA_OTHER); +} +#endif + #ifdef CONFIG_PROC_FS #include <linux/seq_file.h> @@ -369,6 +396,15 @@ static char *vmstat_text[] = { "nr_unstable", "nr_bounce", +#ifdef CONFIG_NUMA + "numa_hit", + "numa_miss", + "numa_foreign", + "numa_interleave", + "numa_local", + "numa_other", +#endif + /* Event counters */ "pgpgin", "pgpgout", @@ -490,21 +526,6 @@ static int zoneinfo_show(struct seq_file *m, void *arg) pageset->pcp[j].high, pageset->pcp[j].batch); } -#ifdef CONFIG_NUMA - seq_printf(m, - "\n numa_hit: %lu" - "\n numa_miss: %lu" - "\n numa_foreign: %lu" - "\n interleave_hit: %lu" - "\n local_node: %lu" - "\n other_node: %lu", - pageset->numa_hit, - pageset->numa_miss, - pageset->numa_foreign, - pageset->interleave_hit, - pageset->local_node, - pageset->other_node); -#endif } seq_printf(m, "\n all_unreclaimable: %u" |