From 3ab9bb7243489f9db3abf3d05521ddfc6b184c0a Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 9 Jun 2007 15:41:42 -0400 Subject: SUNRPC: Fix a memory leak in the auth credcache code The leak only affects the RPCSEC_GSS caches, since they are the only ones that are dynamically allocated... Rename the existing rpcauth_free_credcache() to rpcauth_clear_credcache() in order to better describe its role, then add a new function rpcauth_destroy_credcache() that actually frees the cache in addition to clearing it out. Also move the call to destroy the credcache in gss_destroy() to come before the rpc upcall pipe is unlinked. Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 9527f2bb1744..f6b6c81cbc3e 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -137,9 +137,8 @@ void rpcauth_destroy_credlist(struct hlist_head *head) * that are not referenced. */ void -rpcauth_free_credcache(struct rpc_auth *auth) +rpcauth_clear_credcache(struct rpc_cred_cache *cache) { - struct rpc_cred_cache *cache = auth->au_credcache; HLIST_HEAD(free); struct hlist_node *pos, *next; struct rpc_cred *cred; @@ -157,6 +156,21 @@ rpcauth_free_credcache(struct rpc_auth *auth) rpcauth_destroy_credlist(&free); } +/* + * Destroy the RPC credential cache + */ +void +rpcauth_destroy_credcache(struct rpc_auth *auth) +{ + struct rpc_cred_cache *cache = auth->au_credcache; + + if (cache) { + auth->au_credcache = NULL; + rpcauth_clear_credcache(cache); + kfree(cache); + } +} + static void rpcauth_prune_expired(struct rpc_auth *auth, struct rpc_cred *cred, struct hlist_head *free) { -- cgit v1.2.3 From fc1b356f566fe05929ec2a88ce2c7b15f8b6279f Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 9 Jun 2007 16:15:46 -0400 Subject: SUNRPC: Fix races in rpcauth_create See the FIXME: auth_flavors[] really needs a lock and module refcounting. Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index f6b6c81cbc3e..584f24311a80 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -18,6 +18,7 @@ # define RPCDBG_FACILITY RPCDBG_AUTH #endif +static DEFINE_SPINLOCK(rpc_authflavor_lock); static struct rpc_authops * auth_flavors[RPC_AUTH_MAXFLAVOR] = { &authnull_ops, /* AUTH_NULL */ &authunix_ops, /* AUTH_UNIX */ @@ -35,26 +36,34 @@ int rpcauth_register(struct rpc_authops *ops) { rpc_authflavor_t flavor; + int ret = -EPERM; if ((flavor = ops->au_flavor) >= RPC_AUTH_MAXFLAVOR) return -EINVAL; - if (auth_flavors[flavor] != NULL) - return -EPERM; /* what else? */ - auth_flavors[flavor] = ops; - return 0; + spin_lock(&rpc_authflavor_lock); + if (auth_flavors[flavor] == NULL) { + auth_flavors[flavor] = ops; + ret = 0; + } + spin_unlock(&rpc_authflavor_lock); + return ret; } int rpcauth_unregister(struct rpc_authops *ops) { rpc_authflavor_t flavor; + int ret = -EPERM; if ((flavor = ops->au_flavor) >= RPC_AUTH_MAXFLAVOR) return -EINVAL; - if (auth_flavors[flavor] != ops) - return -EPERM; /* what else? */ - auth_flavors[flavor] = NULL; - return 0; + spin_lock(&rpc_authflavor_lock); + if (auth_flavors[flavor] == ops) { + auth_flavors[flavor] = NULL; + ret = 0; + } + spin_unlock(&rpc_authflavor_lock); + return ret; } struct rpc_auth * @@ -68,15 +77,19 @@ rpcauth_create(rpc_authflavor_t pseudoflavor, struct rpc_clnt *clnt) if (flavor >= RPC_AUTH_MAXFLAVOR) goto out; - /* FIXME - auth_flavors[] really needs an rw lock, - * and module refcounting. */ #ifdef CONFIG_KMOD if ((ops = auth_flavors[flavor]) == NULL) request_module("rpc-auth-%u", flavor); #endif - if ((ops = auth_flavors[flavor]) == NULL) + spin_lock(&rpc_authflavor_lock); + ops = auth_flavors[flavor]; + if (ops == NULL || !try_module_get(ops->owner)) { + spin_unlock(&rpc_authflavor_lock); goto out; + } + spin_unlock(&rpc_authflavor_lock); auth = ops->create(clnt, pseudoflavor); + module_put(ops->owner); if (IS_ERR(auth)) return auth; if (clnt->cl_auth) -- cgit v1.2.3 From de7a8ce38aea529876db3890b61947bc4bc004da Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 23 Jun 2007 10:46:47 -0400 Subject: SUNRPC: Rename rpcauth_destroy() to rpcauth_release() Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 584f24311a80..1686dc74c6a9 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -93,7 +93,7 @@ rpcauth_create(rpc_authflavor_t pseudoflavor, struct rpc_clnt *clnt) if (IS_ERR(auth)) return auth; if (clnt->cl_auth) - rpcauth_destroy(clnt->cl_auth); + rpcauth_release(clnt->cl_auth); clnt->cl_auth = auth; out: @@ -101,7 +101,7 @@ out: } void -rpcauth_destroy(struct rpc_auth *auth) +rpcauth_release(struct rpc_auth *auth) { if (!atomic_dec_and_test(&auth->au_count)) return; -- cgit v1.2.3 From f1c0a8615090359d57e096157feb9f900cbb233c Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 23 Jun 2007 20:17:58 -0400 Subject: SUNRPC: Mark auth and cred operation tables as constant. Also do the same for gss_api operation tables. Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 1686dc74c6a9..d3f0f944c0b5 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -19,7 +19,7 @@ #endif static DEFINE_SPINLOCK(rpc_authflavor_lock); -static struct rpc_authops * auth_flavors[RPC_AUTH_MAXFLAVOR] = { +static const struct rpc_authops *auth_flavors[RPC_AUTH_MAXFLAVOR] = { &authnull_ops, /* AUTH_NULL */ &authunix_ops, /* AUTH_UNIX */ NULL, /* others can be loadable modules */ @@ -33,7 +33,7 @@ pseudoflavor_to_flavor(u32 flavor) { } int -rpcauth_register(struct rpc_authops *ops) +rpcauth_register(const struct rpc_authops *ops) { rpc_authflavor_t flavor; int ret = -EPERM; @@ -50,7 +50,7 @@ rpcauth_register(struct rpc_authops *ops) } int -rpcauth_unregister(struct rpc_authops *ops) +rpcauth_unregister(const struct rpc_authops *ops) { rpc_authflavor_t flavor; int ret = -EPERM; @@ -70,7 +70,7 @@ struct rpc_auth * rpcauth_create(rpc_authflavor_t pseudoflavor, struct rpc_clnt *clnt) { struct rpc_auth *auth; - struct rpc_authops *ops; + const struct rpc_authops *ops; u32 flavor = pseudoflavor_to_flavor(pseudoflavor); auth = ERR_PTR(-EINVAL); -- cgit v1.2.3 From 5fe4755e2526a2aa82b7ed8daeb3aed74a236925 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 23 Jun 2007 19:55:31 -0400 Subject: SUNRPC: Clean up rpc credential initialisation Add a helper rpc_cred_init() Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index d3f0f944c0b5..2156327da45b 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -264,13 +264,9 @@ retry: if (!cred) { new = auth->au_ops->crcreate(auth, acred, flags); - if (!IS_ERR(new)) { -#ifdef RPC_DEBUG - new->cr_magic = RPCAUTH_CRED_MAGIC; -#endif + if (!IS_ERR(new)) goto retry; - } else - cred = new; + cred = new; } else if ((cred->cr_flags & RPCAUTH_CRED_NEW) && cred->cr_ops->cr_init != NULL && !(flags & RPCAUTH_LOOKUP_NEW)) { @@ -302,6 +298,22 @@ rpcauth_lookupcred(struct rpc_auth *auth, int flags) return ret; } +void +rpcauth_init_cred(struct rpc_cred *cred, const struct auth_cred *acred, + struct rpc_auth *auth, const struct rpc_credops *ops) +{ + INIT_HLIST_NODE(&cred->cr_hash); + atomic_set(&cred->cr_count, 1); + cred->cr_auth = auth; + cred->cr_ops = ops; + cred->cr_expire = jiffies; +#ifdef RPC_DEBUG + cred->cr_magic = RPCAUTH_CRED_MAGIC; +#endif + cred->cr_uid = acred->uid; +} +EXPORT_SYMBOL(rpcauth_init_cred); + struct rpc_cred * rpcauth_bindcred(struct rpc_task *task) { -- cgit v1.2.3 From fc432dd90760a629c57026e57f65ff80a1a31d2f Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 25 Jun 2007 10:15:15 -0400 Subject: SUNRPC: Enforce atomic updates of rpc_cred->cr_flags Convert to the use of atomic bitops... Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 2156327da45b..4d7c78b05d1e 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -190,8 +190,8 @@ rpcauth_prune_expired(struct rpc_auth *auth, struct rpc_cred *cred, struct hlist if (atomic_read(&cred->cr_count) != 1) return; if (time_after(jiffies, cred->cr_expire + auth->au_credcache->expire)) - cred->cr_flags &= ~RPCAUTH_CRED_UPTODATE; - if (!(cred->cr_flags & RPCAUTH_CRED_UPTODATE)) { + clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags); + if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0) { __hlist_del(&cred->cr_hash); hlist_add_head(&cred->cr_hash, free); } @@ -267,7 +267,7 @@ retry: if (!IS_ERR(new)) goto retry; cred = new; - } else if ((cred->cr_flags & RPCAUTH_CRED_NEW) + } else if (test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags) && cred->cr_ops->cr_init != NULL && !(flags & RPCAUTH_LOOKUP_NEW)) { int res = cred->cr_ops->cr_init(auth, cred); @@ -440,17 +440,19 @@ rpcauth_refreshcred(struct rpc_task *task) void rpcauth_invalcred(struct rpc_task *task) { + struct rpc_cred *cred = task->tk_msg.rpc_cred; + dprintk("RPC: %5u invalidating %s cred %p\n", - task->tk_pid, task->tk_auth->au_ops->au_name, task->tk_msg.rpc_cred); - spin_lock(&rpc_credcache_lock); - if (task->tk_msg.rpc_cred) - task->tk_msg.rpc_cred->cr_flags &= ~RPCAUTH_CRED_UPTODATE; - spin_unlock(&rpc_credcache_lock); + task->tk_pid, task->tk_auth->au_ops->au_name, cred); + if (cred) + clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags); } int rpcauth_uptodatecred(struct rpc_task *task) { - return !(task->tk_msg.rpc_cred) || - (task->tk_msg.rpc_cred->cr_flags & RPCAUTH_CRED_UPTODATE); + struct rpc_cred *cred = task->tk_msg.rpc_cred; + + return cred == NULL || + test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0; } -- cgit v1.2.3 From e092bdcd939416ef911090890096fe07d0281a5e Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 23 Jun 2007 19:45:36 -0400 Subject: SUNRPC: cleanup rpc credential cache garbage collection Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 121 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 72 insertions(+), 49 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 4d7c78b05d1e..00f9649b0901 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -25,6 +25,8 @@ static const struct rpc_authops *auth_flavors[RPC_AUTH_MAXFLAVOR] = { NULL, /* others can be loadable modules */ }; +static LIST_HEAD(cred_unused); + static u32 pseudoflavor_to_flavor(u32 flavor) { if (flavor >= RPC_AUTH_MAXFLAVOR) @@ -134,13 +136,13 @@ rpcauth_init_credcache(struct rpc_auth *auth, unsigned long expire) * Destroy a list of credentials */ static inline -void rpcauth_destroy_credlist(struct hlist_head *head) +void rpcauth_destroy_credlist(struct list_head *head) { struct rpc_cred *cred; - while (!hlist_empty(head)) { - cred = hlist_entry(head->first, struct rpc_cred, cr_hash); - hlist_del_init(&cred->cr_hash); + while (!list_empty(head)) { + cred = list_entry(head->next, struct rpc_cred, cr_lru); + list_del_init(&cred->cr_lru); put_rpccred(cred); } } @@ -152,17 +154,20 @@ void rpcauth_destroy_credlist(struct hlist_head *head) void rpcauth_clear_credcache(struct rpc_cred_cache *cache) { - HLIST_HEAD(free); - struct hlist_node *pos, *next; + LIST_HEAD(free); + struct hlist_head *head; struct rpc_cred *cred; int i; spin_lock(&rpc_credcache_lock); for (i = 0; i < RPC_CREDCACHE_NR; i++) { - hlist_for_each_safe(pos, next, &cache->hashtable[i]) { - cred = hlist_entry(pos, struct rpc_cred, cr_hash); - __hlist_del(&cred->cr_hash); - hlist_add_head(&cred->cr_hash, &free); + head = &cache->hashtable[i]; + while (!hlist_empty(head)) { + cred = hlist_entry(head->first, struct rpc_cred, cr_hash); + get_rpccred(cred); + list_move_tail(&cred->cr_lru, &free); + smp_wmb(); + hlist_del_init(&cred->cr_hash); } } spin_unlock(&rpc_credcache_lock); @@ -184,38 +189,39 @@ rpcauth_destroy_credcache(struct rpc_auth *auth) } } +/* + * Remove stale credentials. Avoid sleeping inside the loop. + */ static void -rpcauth_prune_expired(struct rpc_auth *auth, struct rpc_cred *cred, struct hlist_head *free) +rpcauth_prune_expired(struct list_head *free) { - if (atomic_read(&cred->cr_count) != 1) - return; - if (time_after(jiffies, cred->cr_expire + auth->au_credcache->expire)) - clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags); - if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0) { - __hlist_del(&cred->cr_hash); - hlist_add_head(&cred->cr_hash, free); + struct rpc_cred *cred; + + while (!list_empty(&cred_unused)) { + cred = list_entry(cred_unused.next, struct rpc_cred, cr_lru); + if (time_after(jiffies, cred->cr_expire + + cred->cr_auth->au_credcache->expire)) + break; + list_del_init(&cred->cr_lru); + if (atomic_read(&cred->cr_count) != 0) + continue; + get_rpccred(cred); + list_add_tail(&cred->cr_lru, free); + smp_wmb(); + hlist_del_init(&cred->cr_hash); } } /* - * Remove stale credentials. Avoid sleeping inside the loop. + * Run garbage collector. */ static void -rpcauth_gc_credcache(struct rpc_auth *auth, struct hlist_head *free) +rpcauth_gc_credcache(struct rpc_cred_cache *cache, struct list_head *free) { - struct rpc_cred_cache *cache = auth->au_credcache; - struct hlist_node *pos, *next; - struct rpc_cred *cred; - int i; - - dprintk("RPC: gc'ing RPC credentials for auth %p\n", auth); - for (i = 0; i < RPC_CREDCACHE_NR; i++) { - hlist_for_each_safe(pos, next, &cache->hashtable[i]) { - cred = hlist_entry(pos, struct rpc_cred, cr_hash); - rpcauth_prune_expired(auth, cred, free); - } - } + if (time_before(jiffies, cache->nextgc)) + return; cache->nextgc = jiffies + cache->expire; + rpcauth_prune_expired(free); } /* @@ -225,39 +231,35 @@ struct rpc_cred * rpcauth_lookup_credcache(struct rpc_auth *auth, struct auth_cred * acred, int flags) { + LIST_HEAD(free); struct rpc_cred_cache *cache = auth->au_credcache; - HLIST_HEAD(free); - struct hlist_node *pos, *next; + struct hlist_node *pos; struct rpc_cred *new = NULL, - *cred = NULL; + *cred = NULL, + *entry; int nr = 0; if (!(flags & RPCAUTH_LOOKUP_ROOTCREDS)) nr = acred->uid & RPC_CREDCACHE_MASK; retry: spin_lock(&rpc_credcache_lock); - if (time_before(cache->nextgc, jiffies)) - rpcauth_gc_credcache(auth, &free); - hlist_for_each_safe(pos, next, &cache->hashtable[nr]) { - struct rpc_cred *entry; - entry = hlist_entry(pos, struct rpc_cred, cr_hash); - if (entry->cr_ops->crmatch(acred, entry, flags)) { - hlist_del(&entry->cr_hash); - cred = entry; - break; - } - rpcauth_prune_expired(auth, entry, &free); + hlist_for_each_entry(entry, pos, &cache->hashtable[nr], cr_hash) { + if (!entry->cr_ops->crmatch(acred, entry, flags)) + continue; + cred = get_rpccred(entry); + hlist_del(&entry->cr_hash); + break; } if (new) { if (cred) - hlist_add_head(&new->cr_hash, &free); + list_add_tail(&new->cr_lru, &free); else cred = new; } if (cred) { hlist_add_head(&cred->cr_hash, &cache->hashtable[nr]); - get_rpccred(cred); } + rpcauth_gc_credcache(cache, &free); spin_unlock(&rpc_credcache_lock); rpcauth_destroy_credlist(&free); @@ -303,6 +305,7 @@ rpcauth_init_cred(struct rpc_cred *cred, const struct auth_cred *acred, struct rpc_auth *auth, const struct rpc_credops *ops) { INIT_HLIST_NODE(&cred->cr_hash); + INIT_LIST_HEAD(&cred->cr_lru); atomic_set(&cred->cr_count, 1); cred->cr_auth = auth; cred->cr_ops = ops; @@ -353,9 +356,29 @@ rpcauth_holdcred(struct rpc_task *task) void put_rpccred(struct rpc_cred *cred) { - cred->cr_expire = jiffies; + /* Fast path for unhashed credentials */ + if (!hlist_unhashed(&cred->cr_hash)) + goto need_lock; + if (!atomic_dec_and_test(&cred->cr_count)) return; + goto out_destroy; + +need_lock: + if (!atomic_dec_and_lock(&cred->cr_count, &rpc_credcache_lock)) + return; + if (!list_empty(&cred->cr_lru)) + list_del_init(&cred->cr_lru); + if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0) + hlist_del(&cred->cr_hash); + else if (!hlist_unhashed(&cred->cr_hash)) { + cred->cr_expire = jiffies; + list_add_tail(&cred->cr_lru, &cred_unused); + spin_unlock(&rpc_credcache_lock); + return; + } + spin_unlock(&rpc_credcache_lock); +out_destroy: cred->cr_ops->crdestroy(cred); } -- cgit v1.2.3 From 31be5bf15f3dafffce110eb1afadccbf2e3067b4 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 24 Jun 2007 15:55:26 -0400 Subject: SUNRPC: Convert the credcache lookup code to use RCU Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 91 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 34 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 00f9649b0901..ad7bde2c437e 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -112,6 +112,14 @@ rpcauth_release(struct rpc_auth *auth) static DEFINE_SPINLOCK(rpc_credcache_lock); +static void +rpcauth_unhash_cred_locked(struct rpc_cred *cred) +{ + hlist_del_rcu(&cred->cr_hash); + smp_mb__before_clear_bit(); + clear_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags); +} + /* * Initialize RPC credential cache */ @@ -166,8 +174,7 @@ rpcauth_clear_credcache(struct rpc_cred_cache *cache) cred = hlist_entry(head->first, struct rpc_cred, cr_hash); get_rpccred(cred); list_move_tail(&cred->cr_lru, &free); - smp_wmb(); - hlist_del_init(&cred->cr_hash); + rpcauth_unhash_cred_locked(cred); } } spin_unlock(&rpc_credcache_lock); @@ -207,8 +214,7 @@ rpcauth_prune_expired(struct list_head *free) continue; get_rpccred(cred); list_add_tail(&cred->cr_lru, free); - smp_wmb(); - hlist_del_init(&cred->cr_hash); + rpcauth_unhash_cred_locked(cred); } } @@ -218,10 +224,12 @@ rpcauth_prune_expired(struct list_head *free) static void rpcauth_gc_credcache(struct rpc_cred_cache *cache, struct list_head *free) { - if (time_before(jiffies, cache->nextgc)) + if (list_empty(&cred_unused) || time_before(jiffies, cache->nextgc)) return; + spin_lock(&rpc_credcache_lock); cache->nextgc = jiffies + cache->expire; rpcauth_prune_expired(free); + spin_unlock(&rpc_credcache_lock); } /* @@ -234,42 +242,57 @@ rpcauth_lookup_credcache(struct rpc_auth *auth, struct auth_cred * acred, LIST_HEAD(free); struct rpc_cred_cache *cache = auth->au_credcache; struct hlist_node *pos; - struct rpc_cred *new = NULL, - *cred = NULL, - *entry; + struct rpc_cred *cred = NULL, + *entry, *new; int nr = 0; if (!(flags & RPCAUTH_LOOKUP_ROOTCREDS)) nr = acred->uid & RPC_CREDCACHE_MASK; -retry: - spin_lock(&rpc_credcache_lock); - hlist_for_each_entry(entry, pos, &cache->hashtable[nr], cr_hash) { + + rcu_read_lock(); + hlist_for_each_entry_rcu(entry, pos, &cache->hashtable[nr], cr_hash) { if (!entry->cr_ops->crmatch(acred, entry, flags)) continue; + spin_lock(&rpc_credcache_lock); + if (test_bit(RPCAUTH_CRED_HASHED, &entry->cr_flags) == 0) { + spin_unlock(&rpc_credcache_lock); + continue; + } cred = get_rpccred(entry); - hlist_del(&entry->cr_hash); + spin_unlock(&rpc_credcache_lock); break; } - if (new) { - if (cred) - list_add_tail(&new->cr_lru, &free); - else - cred = new; - } - if (cred) { - hlist_add_head(&cred->cr_hash, &cache->hashtable[nr]); + rcu_read_unlock(); + + if (cred != NULL) { + rpcauth_gc_credcache(cache, &free); + goto found; } - rpcauth_gc_credcache(cache, &free); - spin_unlock(&rpc_credcache_lock); - rpcauth_destroy_credlist(&free); + new = auth->au_ops->crcreate(auth, acred, flags); + if (IS_ERR(new)) { + cred = new; + goto out; + } - if (!cred) { - new = auth->au_ops->crcreate(auth, acred, flags); - if (!IS_ERR(new)) - goto retry; + spin_lock(&rpc_credcache_lock); + hlist_for_each_entry(entry, pos, &cache->hashtable[nr], cr_hash) { + if (!entry->cr_ops->crmatch(acred, entry, flags)) + continue; + cred = get_rpccred(entry); + break; + } + if (cred == NULL) { cred = new; - } else if (test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags) + set_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags); + hlist_add_head_rcu(&cred->cr_hash, &cache->hashtable[nr]); + } else + list_add_tail(&new->cr_lru, &free); + rpcauth_prune_expired(&free); + cache->nextgc = jiffies + cache->expire; + spin_unlock(&rpc_credcache_lock); +found: + if (test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags) && cred->cr_ops->cr_init != NULL && !(flags & RPCAUTH_LOOKUP_NEW)) { int res = cred->cr_ops->cr_init(auth, cred); @@ -278,8 +301,9 @@ retry: cred = ERR_PTR(res); } } - - return (struct rpc_cred *) cred; + rpcauth_destroy_credlist(&free); +out: + return cred; } struct rpc_cred * @@ -357,21 +381,20 @@ void put_rpccred(struct rpc_cred *cred) { /* Fast path for unhashed credentials */ - if (!hlist_unhashed(&cred->cr_hash)) + if (test_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags) != 0) goto need_lock; if (!atomic_dec_and_test(&cred->cr_count)) return; goto out_destroy; - need_lock: if (!atomic_dec_and_lock(&cred->cr_count, &rpc_credcache_lock)) return; if (!list_empty(&cred->cr_lru)) list_del_init(&cred->cr_lru); if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0) - hlist_del(&cred->cr_hash); - else if (!hlist_unhashed(&cred->cr_hash)) { + rpcauth_unhash_cred_locked(cred); + else if (test_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags) != 0) { cred->cr_expire = jiffies; list_add_tail(&cred->cr_lru, &cred_unused); spin_unlock(&rpc_credcache_lock); -- cgit v1.2.3 From 9499b4341b56935f61af9e7e354e7d11e70f5258 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 24 Jun 2007 15:57:57 -0400 Subject: SUNRPC: Give credential cache a local spinlock Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 46 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 14 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index ad7bde2c437e..cf1198d10ee9 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -120,6 +120,18 @@ rpcauth_unhash_cred_locked(struct rpc_cred *cred) clear_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags); } +static void +rpcauth_unhash_cred(struct rpc_cred *cred) +{ + spinlock_t *cache_lock; + + cache_lock = &cred->cr_auth->au_credcache->lock; + spin_lock(cache_lock); + if (atomic_read(&cred->cr_count) == 0) + rpcauth_unhash_cred_locked(cred); + spin_unlock(cache_lock); +} + /* * Initialize RPC credential cache */ @@ -134,6 +146,7 @@ rpcauth_init_credcache(struct rpc_auth *auth, unsigned long expire) return -ENOMEM; for (i = 0; i < RPC_CREDCACHE_NR; i++) INIT_HLIST_HEAD(&new->hashtable[i]); + spin_lock_init(&new->lock); new->expire = expire; new->nextgc = jiffies + (expire >> 1); auth->au_credcache = new; @@ -168,6 +181,7 @@ rpcauth_clear_credcache(struct rpc_cred_cache *cache) int i; spin_lock(&rpc_credcache_lock); + spin_lock(&cache->lock); for (i = 0; i < RPC_CREDCACHE_NR; i++) { head = &cache->hashtable[i]; while (!hlist_empty(head)) { @@ -177,6 +191,7 @@ rpcauth_clear_credcache(struct rpc_cred_cache *cache) rpcauth_unhash_cred_locked(cred); } } + spin_unlock(&cache->lock); spin_unlock(&rpc_credcache_lock); rpcauth_destroy_credlist(&free); } @@ -202,6 +217,7 @@ rpcauth_destroy_credcache(struct rpc_auth *auth) static void rpcauth_prune_expired(struct list_head *free) { + spinlock_t *cache_lock; struct rpc_cred *cred; while (!list_empty(&cred_unused)) { @@ -212,9 +228,14 @@ rpcauth_prune_expired(struct list_head *free) list_del_init(&cred->cr_lru); if (atomic_read(&cred->cr_count) != 0) continue; - get_rpccred(cred); - list_add_tail(&cred->cr_lru, free); - rpcauth_unhash_cred_locked(cred); + cache_lock = &cred->cr_auth->au_credcache->lock; + spin_lock(cache_lock); + if (atomic_read(&cred->cr_count) == 0) { + get_rpccred(cred); + list_add_tail(&cred->cr_lru, free); + rpcauth_unhash_cred_locked(cred); + } + spin_unlock(cache_lock); } } @@ -253,21 +274,19 @@ rpcauth_lookup_credcache(struct rpc_auth *auth, struct auth_cred * acred, hlist_for_each_entry_rcu(entry, pos, &cache->hashtable[nr], cr_hash) { if (!entry->cr_ops->crmatch(acred, entry, flags)) continue; - spin_lock(&rpc_credcache_lock); + spin_lock(&cache->lock); if (test_bit(RPCAUTH_CRED_HASHED, &entry->cr_flags) == 0) { - spin_unlock(&rpc_credcache_lock); + spin_unlock(&cache->lock); continue; } cred = get_rpccred(entry); - spin_unlock(&rpc_credcache_lock); + spin_unlock(&cache->lock); break; } rcu_read_unlock(); - if (cred != NULL) { - rpcauth_gc_credcache(cache, &free); + if (cred != NULL) goto found; - } new = auth->au_ops->crcreate(auth, acred, flags); if (IS_ERR(new)) { @@ -275,7 +294,7 @@ rpcauth_lookup_credcache(struct rpc_auth *auth, struct auth_cred * acred, goto out; } - spin_lock(&rpc_credcache_lock); + spin_lock(&cache->lock); hlist_for_each_entry(entry, pos, &cache->hashtable[nr], cr_hash) { if (!entry->cr_ops->crmatch(acred, entry, flags)) continue; @@ -288,9 +307,7 @@ rpcauth_lookup_credcache(struct rpc_auth *auth, struct auth_cred * acred, hlist_add_head_rcu(&cred->cr_hash, &cache->hashtable[nr]); } else list_add_tail(&new->cr_lru, &free); - rpcauth_prune_expired(&free); - cache->nextgc = jiffies + cache->expire; - spin_unlock(&rpc_credcache_lock); + spin_unlock(&cache->lock); found: if (test_bit(RPCAUTH_CRED_NEW, &cred->cr_flags) && cred->cr_ops->cr_init != NULL @@ -301,6 +318,7 @@ found: cred = ERR_PTR(res); } } + rpcauth_gc_credcache(cache, &free); rpcauth_destroy_credlist(&free); out: return cred; @@ -393,7 +411,7 @@ need_lock: if (!list_empty(&cred->cr_lru)) list_del_init(&cred->cr_lru); if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0) - rpcauth_unhash_cred_locked(cred); + rpcauth_unhash_cred(cred); else if (test_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags) != 0) { cred->cr_expire = jiffies; list_add_tail(&cred->cr_lru, &cred_unused); -- cgit v1.2.3 From f5c2187cfef628784d8a09b6d0f77888246d0c0f Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Mon, 25 Jun 2007 17:11:20 -0400 Subject: SUNRPC: Convert the credential garbage collector into a shrinker callback Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 63 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 18 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index cf1198d10ee9..81f4c776c558 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -26,6 +26,7 @@ static const struct rpc_authops *auth_flavors[RPC_AUTH_MAXFLAVOR] = { }; static LIST_HEAD(cred_unused); +static unsigned long number_cred_unused; static u32 pseudoflavor_to_flavor(u32 flavor) { @@ -136,7 +137,7 @@ rpcauth_unhash_cred(struct rpc_cred *cred) * Initialize RPC credential cache */ int -rpcauth_init_credcache(struct rpc_auth *auth, unsigned long expire) +rpcauth_init_credcache(struct rpc_auth *auth) { struct rpc_cred_cache *new; int i; @@ -147,8 +148,6 @@ rpcauth_init_credcache(struct rpc_auth *auth, unsigned long expire) for (i = 0; i < RPC_CREDCACHE_NR; i++) INIT_HLIST_HEAD(&new->hashtable[i]); spin_lock_init(&new->lock); - new->expire = expire; - new->nextgc = jiffies + (expire >> 1); auth->au_credcache = new; return 0; } @@ -187,7 +186,11 @@ rpcauth_clear_credcache(struct rpc_cred_cache *cache) while (!hlist_empty(head)) { cred = hlist_entry(head->first, struct rpc_cred, cr_hash); get_rpccred(cred); - list_move_tail(&cred->cr_lru, &free); + if (!list_empty(&cred->cr_lru)) { + list_del(&cred->cr_lru); + number_cred_unused--; + } + list_add_tail(&cred->cr_lru, &free); rpcauth_unhash_cred_locked(cred); } } @@ -214,18 +217,16 @@ rpcauth_destroy_credcache(struct rpc_auth *auth) /* * Remove stale credentials. Avoid sleeping inside the loop. */ -static void -rpcauth_prune_expired(struct list_head *free) +static int +rpcauth_prune_expired(struct list_head *free, int nr_to_scan) { spinlock_t *cache_lock; struct rpc_cred *cred; while (!list_empty(&cred_unused)) { cred = list_entry(cred_unused.next, struct rpc_cred, cr_lru); - if (time_after(jiffies, cred->cr_expire + - cred->cr_auth->au_credcache->expire)) - break; list_del_init(&cred->cr_lru); + number_cred_unused--; if (atomic_read(&cred->cr_count) != 0) continue; cache_lock = &cred->cr_auth->au_credcache->lock; @@ -234,23 +235,32 @@ rpcauth_prune_expired(struct list_head *free) get_rpccred(cred); list_add_tail(&cred->cr_lru, free); rpcauth_unhash_cred_locked(cred); + nr_to_scan--; } spin_unlock(cache_lock); + if (nr_to_scan == 0) + break; } + return nr_to_scan; } /* - * Run garbage collector. + * Run memory cache shrinker. */ -static void -rpcauth_gc_credcache(struct rpc_cred_cache *cache, struct list_head *free) +static int +rpcauth_cache_shrinker(int nr_to_scan, gfp_t gfp_mask) { - if (list_empty(&cred_unused) || time_before(jiffies, cache->nextgc)) - return; + LIST_HEAD(free); + int res; + + if (list_empty(&cred_unused)) + return 0; spin_lock(&rpc_credcache_lock); - cache->nextgc = jiffies + cache->expire; - rpcauth_prune_expired(free); + nr_to_scan = rpcauth_prune_expired(&free, nr_to_scan); + res = (number_cred_unused / 100) * sysctl_vfs_cache_pressure; spin_unlock(&rpc_credcache_lock); + rpcauth_destroy_credlist(&free); + return res; } /* @@ -318,7 +328,6 @@ found: cred = ERR_PTR(res); } } - rpcauth_gc_credcache(cache, &free); rpcauth_destroy_credlist(&free); out: return cred; @@ -408,13 +417,16 @@ put_rpccred(struct rpc_cred *cred) need_lock: if (!atomic_dec_and_lock(&cred->cr_count, &rpc_credcache_lock)) return; - if (!list_empty(&cred->cr_lru)) + if (!list_empty(&cred->cr_lru)) { + number_cred_unused--; list_del_init(&cred->cr_lru); + } if (test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) == 0) rpcauth_unhash_cred(cred); else if (test_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags) != 0) { cred->cr_expire = jiffies; list_add_tail(&cred->cr_lru, &cred_unused); + number_cred_unused++; spin_unlock(&rpc_credcache_lock); return; } @@ -520,3 +532,18 @@ rpcauth_uptodatecred(struct rpc_task *task) return cred == NULL || test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0; } + + +static struct shrinker *rpc_cred_shrinker; + +void __init rpcauth_init_module(void) +{ + rpc_init_authunix(); + rpc_cred_shrinker = set_shrinker(DEFAULT_SEEKS, rpcauth_cache_shrinker); +} + +void __exit rpcauth_remove_module(void) +{ + if (rpc_cred_shrinker != NULL) + remove_shrinker(rpc_cred_shrinker); +} -- cgit v1.2.3 From 1be27f36601973815171db684c711d30557cf50c Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Wed, 27 Jun 2007 14:29:04 -0400 Subject: SUNRPC: Remove the tk_auth macro... We should almost always be deferencing the rpc_auth struct by means of the credential's cr_auth field instead of the rpc_clnt->cl_auth anyway. Fix up that historical mistake, and remove the macro that propagated it. Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 81f4c776c558..74baf87ccff9 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -371,7 +371,7 @@ EXPORT_SYMBOL(rpcauth_init_cred); struct rpc_cred * rpcauth_bindcred(struct rpc_task *task) { - struct rpc_auth *auth = task->tk_auth; + struct rpc_auth *auth = task->tk_client->cl_auth; struct auth_cred acred = { .uid = current->fsuid, .gid = current->fsgid, @@ -381,7 +381,7 @@ rpcauth_bindcred(struct rpc_task *task) int flags = 0; dprintk("RPC: %5u looking up %s cred\n", - task->tk_pid, task->tk_auth->au_ops->au_name); + task->tk_pid, task->tk_client->cl_auth->au_ops->au_name); get_group_info(acred.group_info); if (task->tk_flags & RPC_TASK_ROOTCREDS) flags |= RPCAUTH_LOOKUP_ROOTCREDS; @@ -397,11 +397,12 @@ rpcauth_bindcred(struct rpc_task *task) void rpcauth_holdcred(struct rpc_task *task) { - dprintk("RPC: %5u holding %s cred %p\n", - task->tk_pid, task->tk_auth->au_ops->au_name, - task->tk_msg.rpc_cred); - if (task->tk_msg.rpc_cred) - get_rpccred(task->tk_msg.rpc_cred); + struct rpc_cred *cred = task->tk_msg.rpc_cred; + if (cred != NULL) { + get_rpccred(cred); + dprintk("RPC: %5u holding %s cred %p\n", task->tk_pid, + cred->cr_auth->au_ops->au_name, cred); + } } void @@ -441,7 +442,7 @@ rpcauth_unbindcred(struct rpc_task *task) struct rpc_cred *cred = task->tk_msg.rpc_cred; dprintk("RPC: %5u releasing %s cred %p\n", - task->tk_pid, task->tk_auth->au_ops->au_name, cred); + task->tk_pid, cred->cr_auth->au_ops->au_name, cred); put_rpccred(cred); task->tk_msg.rpc_cred = NULL; @@ -453,7 +454,7 @@ rpcauth_marshcred(struct rpc_task *task, __be32 *p) struct rpc_cred *cred = task->tk_msg.rpc_cred; dprintk("RPC: %5u marshaling %s cred %p\n", - task->tk_pid, task->tk_auth->au_ops->au_name, cred); + task->tk_pid, cred->cr_auth->au_ops->au_name, cred); return cred->cr_ops->crmarshal(task, p); } @@ -464,7 +465,7 @@ rpcauth_checkverf(struct rpc_task *task, __be32 *p) struct rpc_cred *cred = task->tk_msg.rpc_cred; dprintk("RPC: %5u validating %s cred %p\n", - task->tk_pid, task->tk_auth->au_ops->au_name, cred); + task->tk_pid, cred->cr_auth->au_ops->au_name, cred); return cred->cr_ops->crvalidate(task, p); } @@ -505,7 +506,7 @@ rpcauth_refreshcred(struct rpc_task *task) int err; dprintk("RPC: %5u refreshing %s cred %p\n", - task->tk_pid, task->tk_auth->au_ops->au_name, cred); + task->tk_pid, cred->cr_auth->au_ops->au_name, cred); err = cred->cr_ops->crrefresh(task); if (err < 0) @@ -519,7 +520,7 @@ rpcauth_invalcred(struct rpc_task *task) struct rpc_cred *cred = task->tk_msg.rpc_cred; dprintk("RPC: %5u invalidating %s cred %p\n", - task->tk_pid, task->tk_auth->au_ops->au_name, cred); + task->tk_pid, cred->cr_auth->au_ops->au_name, cred); if (cred) clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags); } -- cgit v1.2.3 From d8558f99fbc5ef5d4ae76b893784005056450f82 Mon Sep 17 00:00:00 2001 From: "J. Bruce Fields" Date: Tue, 10 Jul 2007 15:19:26 -0400 Subject: sunrpc: drop BKL around wrap and unwrap We don't need the BKL when wrapping and unwrapping; and experiments by Avishay Traeger have found that permitting multiple encryption and decryption operations to proceed in parallel can provide significant performance improvements. Signed-off-by: J. Bruce Fields Cc: Avishay Traeger Signed-off-by: Trond Myklebust --- net/sunrpc/auth.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'net/sunrpc/auth.c') diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 74baf87ccff9..aa55d0a03e6f 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -13,6 +13,7 @@ #include #include #include +#include #ifdef RPC_DEBUG # define RPCDBG_FACILITY RPCDBG_AUTH @@ -475,13 +476,17 @@ rpcauth_wrap_req(struct rpc_task *task, kxdrproc_t encode, void *rqstp, __be32 *data, void *obj) { struct rpc_cred *cred = task->tk_msg.rpc_cred; + int ret; dprintk("RPC: %5u using %s cred %p to wrap rpc data\n", task->tk_pid, cred->cr_ops->cr_name, cred); if (cred->cr_ops->crwrap_req) return cred->cr_ops->crwrap_req(task, encode, rqstp, data, obj); /* By default, we encode the arguments normally. */ - return encode(rqstp, data, obj); + lock_kernel(); + ret = encode(rqstp, data, obj); + unlock_kernel(); + return ret; } int @@ -489,6 +494,7 @@ rpcauth_unwrap_resp(struct rpc_task *task, kxdrproc_t decode, void *rqstp, __be32 *data, void *obj) { struct rpc_cred *cred = task->tk_msg.rpc_cred; + int ret; dprintk("RPC: %5u using %s cred %p to unwrap rpc data\n", task->tk_pid, cred->cr_ops->cr_name, cred); @@ -496,7 +502,10 @@ rpcauth_unwrap_resp(struct rpc_task *task, kxdrproc_t decode, void *rqstp, return cred->cr_ops->crunwrap_resp(task, decode, rqstp, data, obj); /* By default, we decode the arguments normally. */ - return decode(rqstp, data, obj); + lock_kernel(); + ret = decode(rqstp, data, obj); + unlock_kernel(); + return ret; } int -- cgit v1.2.3