From 407640de2152e33341ce1131dac269672c3d50f7 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 17 Mar 2015 18:32:26 -0700 Subject: inet: add sk_listener argument to inet_reqsk_alloc() listener socket can be used to set net pointer, and will be later used to hold a reference on listener. Add a const qualifier to first argument (struct request_sock_ops *), and factorize all write_pnet(&ireq->ireq_net, sock_net(sk)); Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/inet_sock.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index c9ed91891887..cf7abb00941b 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -244,16 +244,19 @@ static inline unsigned int __inet_ehashfn(const __be32 laddr, initval); } -static inline struct request_sock *inet_reqsk_alloc(struct request_sock_ops *ops) +static inline struct request_sock * +inet_reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener) { struct request_sock *req = reqsk_alloc(ops); - struct inet_request_sock *ireq = inet_rsk(req); - if (req != NULL) { + if (req) { + struct inet_request_sock *ireq = inet_rsk(req); + kmemcheck_annotate_bitfield(ireq, flags); ireq->opt = NULL; atomic64_set(&ireq->ir_cookie, 0); ireq->ireq_state = TCP_NEW_SYN_RECV; + write_pnet(&ireq->ireq_net, sock_net(sk_listener)); /* Following is temporary. It is coupled with debugging * helpers in reqsk_put() & reqsk_free() -- cgit v1.2.3 From e49bb337d77d54afebe4fe5b9008955e1337f83d Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 17 Mar 2015 18:32:27 -0700 Subject: inet: uninline inet_reqsk_alloc() inet_reqsk_alloc() is becoming fat and should not be inlined. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/inet_sock.h | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) (limited to 'include') diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index cf7abb00941b..6fec7343070f 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -244,28 +244,8 @@ static inline unsigned int __inet_ehashfn(const __be32 laddr, initval); } -static inline struct request_sock * -inet_reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener) -{ - struct request_sock *req = reqsk_alloc(ops); - - if (req) { - struct inet_request_sock *ireq = inet_rsk(req); - - kmemcheck_annotate_bitfield(ireq, flags); - ireq->opt = NULL; - atomic64_set(&ireq->ir_cookie, 0); - ireq->ireq_state = TCP_NEW_SYN_RECV; - write_pnet(&ireq->ireq_net, sock_net(sk_listener)); - - /* Following is temporary. It is coupled with debugging - * helpers in reqsk_put() & reqsk_free() - */ - atomic_set(&ireq->ireq_refcnt, 0); - } - - return req; -} +struct request_sock *inet_reqsk_alloc(const struct request_sock_ops *ops, + struct sock *sk_listener); static inline __u8 inet_sk_flowi_flags(const struct sock *sk) { -- cgit v1.2.3 From 4e9a578e5b6bdfa8b7fed7a41f28a86a7cffc85f Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 17 Mar 2015 18:32:28 -0700 Subject: inet: add rsk_listener field to struct request_sock Once we'll be able to lookup request sockets in ehash table, we'll need to get access to listener which created this request. This avoid doing a lookup to find the listener, which benefits for a more solid SO_REUSEPORT, and is needed once we no longer queue request sock into a listener private queue. Note that 'struct tcp_request_sock'->listener could be reduced to a single bit, as TFO listener should match req->rsk_listener. TFO will no longer need to hold a reference on the listener. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/request_sock.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/net/request_sock.h b/include/net/request_sock.h index 56dc2faba47e..723d1cbdf20e 100644 --- a/include/net/request_sock.h +++ b/include/net/request_sock.h @@ -52,6 +52,7 @@ struct request_sock { #define rsk_refcnt __req_common.skc_refcnt struct request_sock *dl_next; + struct sock *rsk_listener; u16 mss; u8 num_retrans; /* number of retransmits */ u8 cookie_ts:1; /* syncookie: encode tcpopts in timestamp */ @@ -67,13 +68,16 @@ struct request_sock { u32 peer_secid; }; -static inline struct request_sock *reqsk_alloc(const struct request_sock_ops *ops) +static inline struct request_sock * +reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener) { struct request_sock *req = kmem_cache_alloc(ops->slab, GFP_ATOMIC); - if (req != NULL) + if (req) { req->rsk_ops = ops; - + sock_hold(sk_listener); + req->rsk_listener = sk_listener; + } return req; } @@ -88,6 +92,8 @@ static inline void reqsk_free(struct request_sock *req) WARN_ON_ONCE(atomic_read(&req->rsk_refcnt) != 0); req->rsk_ops->destructor(req); + if (req->rsk_listener) + sock_put(req->rsk_listener); kmem_cache_free(req->rsk_ops->slab, req); } -- cgit v1.2.3 From 9439ce00f208d95703a6725e4ea986dd90e37ffd Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 17 Mar 2015 18:32:29 -0700 Subject: tcp: rename struct tcp_request_sock listener The listener field in struct tcp_request_sock is a pointer back to the listener. We now have req->rsk_listener, so TCP only needs one boolean and not a full pointer. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/linux/tcp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 97dbf16f7d9d..f869ae8afbaf 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -111,7 +111,7 @@ struct tcp_request_sock_ops; struct tcp_request_sock { struct inet_request_sock req; const struct tcp_request_sock_ops *af_specific; - struct sock *listener; /* needed for TFO */ + bool tfo_listener; u32 rcv_isn; u32 snt_isn; u32 snt_synack; /* synack sent time */ -- cgit v1.2.3 From 0470c8ca1d57927f2cc3e1d5add1fb2834609447 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 17 Mar 2015 18:32:31 -0700 Subject: inet: fix request sock refcounting While testing last patch series, I found req sock refcounting was wrong. We must set skc_refcnt to 1 for all request socks added in hashes, but also on request sockets created by FastOpen or syncookies. It is tricky because we need to defer this initialization so that future RCU lookups do not try to take a refcount on a not yet fully initialized request socket. Also get rid of ireq_refcnt alias. Signed-off-by: Eric Dumazet Fixes: 13854e5a6046 ("inet: add proper refcounting to request sock") Signed-off-by: David S. Miller --- include/net/inet_connection_sock.h | 5 ----- include/net/inet_sock.h | 1 - include/net/request_sock.h | 11 +++++++++++ 3 files changed, 11 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index 191feec60205..b9a6b0a94cc6 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -275,11 +275,6 @@ static inline void inet_csk_reqsk_queue_add(struct sock *sk, struct sock *child) { reqsk_queue_add(&inet_csk(sk)->icsk_accept_queue, req, sk, child); - /* before letting lookups find us, make sure all req fields - * are committed to memory. - */ - smp_wmb(); - atomic_set(&req->rsk_refcnt, 1); } void inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req, diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 6fec7343070f..b6c3737da4e9 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -81,7 +81,6 @@ struct inet_request_sock { #define ir_cookie req.__req_common.skc_cookie #define ireq_net req.__req_common.skc_net #define ireq_state req.__req_common.skc_state -#define ireq_refcnt req.__req_common.skc_refcnt #define ireq_family req.__req_common.skc_family kmemcheck_bitfield_begin(flags); diff --git a/include/net/request_sock.h b/include/net/request_sock.h index 723d1cbdf20e..3fa4f824900a 100644 --- a/include/net/request_sock.h +++ b/include/net/request_sock.h @@ -77,6 +77,11 @@ reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener) req->rsk_ops = ops; sock_hold(sk_listener); req->rsk_listener = sk_listener; + + /* Following is temporary. It is coupled with debugging + * helpers in reqsk_put() & reqsk_free() + */ + atomic_set(&req->rsk_refcnt, 0); } return req; } @@ -292,6 +297,12 @@ static inline void reqsk_queue_hash_req(struct request_sock_queue *queue, req->sk = NULL; req->dl_next = lopt->syn_table[hash]; + /* before letting lookups find us, make sure all req fields + * are committed to memory and refcnt initialized. + */ + smp_wmb(); + atomic_set(&req->rsk_refcnt, 1); + write_lock(&queue->syn_wait_lock); lopt->syn_table[hash] = req; write_unlock(&queue->syn_wait_lock); -- cgit v1.2.3