summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristoph Lameter <cl@linux.com>2010-12-06 11:16:28 -0600
committerTejun Heo <tj@kernel.org>2010-12-17 15:07:19 +0100
commitc7b92516a9c68fa5403879225a5a19974a801ef6 (patch)
tree1f72ebd85bca40806024ac6b8f5caab068b7c15d
parent5309665dcc1143d659d82568da8d00f0e08a58f9 (diff)
fs: Use this_cpu_xx operations in buffer.c
Optimize various per cpu area operations through these new percpu operations. These operations avoid address calculations through the use of segment prefixes and multiple memory references through RMW instructions etc. Reduces code size: Before: christoph@linux-2.6$ size fs/buffer.o text data bss dec hex filename 19169 80 28 19277 4b4d fs/buffer.o After: christoph@linux-2.6$ size fs/buffer.o text data bss dec hex filename 19138 80 28 19246 4b2e fs/buffer.o V3->V4: - Move the use of this_cpu_inc_return into a later patch so that this one can go in without percpu infrastructure changes. Cc: Wu Fengguang <fengguang.wu@intel.com> Cc: Christoph Hellwig <hch@lst.de> Acked-by: H. Peter Anvin <hpa@zytor.com> Signed-off-by: Christoph Lameter <cl@linux.com> Signed-off-by: Tejun Heo <tj@kernel.org>
-rw-r--r--fs/buffer.c35
1 files changed, 17 insertions, 18 deletions
diff --git a/fs/buffer.c b/fs/buffer.c
index 5930e382959b..137d9de00e24 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -1270,12 +1270,10 @@ static inline void check_irqs_on(void)
static void bh_lru_install(struct buffer_head *bh)
{
struct buffer_head *evictee = NULL;
- struct bh_lru *lru;
check_irqs_on();
bh_lru_lock();
- lru = &__get_cpu_var(bh_lrus);
- if (lru->bhs[0] != bh) {
+ if (__this_cpu_read(bh_lrus.bhs[0]) != bh) {
struct buffer_head *bhs[BH_LRU_SIZE];
int in;
int out = 0;
@@ -1283,7 +1281,8 @@ static void bh_lru_install(struct buffer_head *bh)
get_bh(bh);
bhs[out++] = bh;
for (in = 0; in < BH_LRU_SIZE; in++) {
- struct buffer_head *bh2 = lru->bhs[in];
+ struct buffer_head *bh2 =
+ __this_cpu_read(bh_lrus.bhs[in]);
if (bh2 == bh) {
__brelse(bh2);
@@ -1298,7 +1297,7 @@ static void bh_lru_install(struct buffer_head *bh)
}
while (out < BH_LRU_SIZE)
bhs[out++] = NULL;
- memcpy(lru->bhs, bhs, sizeof(bhs));
+ memcpy(__this_cpu_ptr(&bh_lrus.bhs), bhs, sizeof(bhs));
}
bh_lru_unlock();
@@ -1313,23 +1312,22 @@ static struct buffer_head *
lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
{
struct buffer_head *ret = NULL;
- struct bh_lru *lru;
unsigned int i;
check_irqs_on();
bh_lru_lock();
- lru = &__get_cpu_var(bh_lrus);
for (i = 0; i < BH_LRU_SIZE; i++) {
- struct buffer_head *bh = lru->bhs[i];
+ struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
if (bh && bh->b_bdev == bdev &&
bh->b_blocknr == block && bh->b_size == size) {
if (i) {
while (i) {
- lru->bhs[i] = lru->bhs[i - 1];
+ __this_cpu_write(bh_lrus.bhs[i],
+ __this_cpu_read(bh_lrus.bhs[i - 1]));
i--;
}
- lru->bhs[0] = bh;
+ __this_cpu_write(bh_lrus.bhs[0], bh);
}
get_bh(bh);
ret = bh;
@@ -3205,20 +3203,21 @@ static void recalc_bh_state(void)
if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
return;
- __get_cpu_var(bh_accounting).ratelimit = 0;
+ __this_cpu_write(bh_accounting.ratelimit, 0);
for_each_online_cpu(i)
tot += per_cpu(bh_accounting, i).nr;
buffer_heads_over_limit = (tot > max_buffer_heads);
}
-
+
struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
{
struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
if (ret) {
INIT_LIST_HEAD(&ret->b_assoc_buffers);
- get_cpu_var(bh_accounting).nr++;
+ preempt_disable();
+ __this_cpu_inc(bh_accounting.nr);
recalc_bh_state();
- put_cpu_var(bh_accounting);
+ preempt_enable();
}
return ret;
}
@@ -3228,9 +3227,10 @@ void free_buffer_head(struct buffer_head *bh)
{
BUG_ON(!list_empty(&bh->b_assoc_buffers));
kmem_cache_free(bh_cachep, bh);
- get_cpu_var(bh_accounting).nr--;
+ preempt_disable();
+ __this_cpu_dec(bh_accounting.nr);
recalc_bh_state();
- put_cpu_var(bh_accounting);
+ preempt_enable();
}
EXPORT_SYMBOL(free_buffer_head);
@@ -3243,9 +3243,8 @@ static void buffer_exit_cpu(int cpu)
brelse(b->bhs[i]);
b->bhs[i] = NULL;
}
- get_cpu_var(bh_accounting).nr += per_cpu(bh_accounting, cpu).nr;
+ this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
per_cpu(bh_accounting, cpu).nr = 0;
- put_cpu_var(bh_accounting);
}
static int buffer_cpu_notify(struct notifier_block *self,