From 76ba0aae673075c77a8b775e9133c8e8b1a44563 Mon Sep 17 00:00:00 2001 From: Tom Herbert Date: Fri, 2 May 2014 16:29:18 -0700 Subject: net: Generalize checksum_init functions Create a general __skb_checksum_validate function (actually a macro) to subsume the various checksum_init functions. This function can either init the checksum, or do the full validation (logically checksum_init+skb_check_complete)-- a flag specifies if full vaidation is performed. Also, there is a flag to the function to indicate that zero checksums are allowed (to support optional UDP checksums). Added several stub functions for calling __skb_checksum_validate. Signed-off-by: Tom Herbert Signed-off-by: David S. Miller --- include/linux/skbuff.h | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 08074a810164..3ca0dda5a42e 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -2741,6 +2741,99 @@ static inline __sum16 skb_checksum_complete(struct sk_buff *skb) 0 : __skb_checksum_complete(skb); } +/* Check if we need to perform checksum complete validation. + * + * Returns true if checksum complete is needed, false otherwise + * (either checksum is unnecessary or zero checksum is allowed). + */ +static inline bool __skb_checksum_validate_needed(struct sk_buff *skb, + bool zero_okay, + __sum16 check) +{ + if (skb_csum_unnecessary(skb)) { + return false; + } else if (zero_okay && !check) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + return false; + } + + return true; +} + +/* For small packets <= CHECKSUM_BREAK peform checksum complete directly + * in checksum_init. + */ +#define CHECKSUM_BREAK 76 + +/* Validate (init) checksum based on checksum complete. + * + * Return values: + * 0: checksum is validated or try to in skb_checksum_complete. In the latter + * case the ip_summed will not be CHECKSUM_UNNECESSARY and the pseudo + * checksum is stored in skb->csum for use in __skb_checksum_complete + * non-zero: value of invalid checksum + * + */ +static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb, + bool complete, + __wsum psum) +{ + if (skb->ip_summed == CHECKSUM_COMPLETE) { + if (!csum_fold(csum_add(psum, skb->csum))) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + return 0; + } + } + + skb->csum = psum; + + if (complete || skb->len <= CHECKSUM_BREAK) + return __skb_checksum_complete(skb); + + return 0; +} + +static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto) +{ + return 0; +} + +/* Perform checksum validate (init). Note that this is a macro since we only + * want to calculate the pseudo header which is an input function if necessary. + * First we try to validate without any computation (checksum unnecessary) and + * then calculate based on checksum complete calling the function to compute + * pseudo header. + * + * Return values: + * 0: checksum is validated or try to in skb_checksum_complete + * non-zero: value of invalid checksum + */ +#define __skb_checksum_validate(skb, proto, complete, \ + zero_okay, check, compute_pseudo) \ +({ \ + __sum16 __ret = 0; \ + if (__skb_checksum_validate_needed(skb, zero_okay, check)) \ + __ret = __skb_checksum_validate_complete(skb, \ + complete, compute_pseudo(skb, proto)); \ + __ret; \ +}) + +#define skb_checksum_init(skb, proto, compute_pseudo) \ + __skb_checksum_validate(skb, proto, false, false, 0, compute_pseudo) + +#define skb_checksum_init_zero_check(skb, proto, check, compute_pseudo) \ + __skb_checksum_validate(skb, proto, false, true, check, compute_pseudo) + +#define skb_checksum_validate(skb, proto, compute_pseudo) \ + __skb_checksum_validate(skb, proto, true, false, 0, compute_pseudo) + +#define skb_checksum_validate_zero_check(skb, proto, check, \ + compute_pseudo) \ + __skb_checksum_validate_(skb, proto, true, true, check, compute_pseudo) + +#define skb_checksum_simple_validate(skb) \ + __skb_checksum_validate(skb, 0, true, false, 0, null_compute_pseudo) + #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) void nf_conntrack_destroy(struct nf_conntrack *nfct); static inline void nf_conntrack_put(struct nf_conntrack *nfct) -- cgit v1.2.3 From 60ff746739bf805a912484643c720b6124826140 Mon Sep 17 00:00:00 2001 From: WANG Cong Date: Sun, 4 May 2014 16:39:18 -0700 Subject: net: rename local_df to ignore_df MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As suggested by several people, rename local_df to ignore_df, since it means "ignore df bit if it is set". Cc: Maciej Żenczykowski Cc: Florian Westphal Cc: David S. Miller Cc: Eric Dumazet Signed-off-by: Cong Wang Acked-by: Maciej Żenczykowski Signed-off-by: David S. Miller --- include/linux/skbuff.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 3ca0dda5a42e..7a9beeb1c458 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -426,7 +426,7 @@ static inline u32 skb_mstamp_us_delta(const struct skb_mstamp *t1, * @csum_start: Offset from skb->head where checksumming should start * @csum_offset: Offset from csum_start where checksum should be stored * @priority: Packet queueing priority - * @local_df: allow local fragmentation + * @ignore_df: allow local fragmentation * @cloned: Head may be cloned (check refcnt to be sure) * @ip_summed: Driver fed us an IP checksum * @nohdr: Payload reference only, must not modify header @@ -514,7 +514,7 @@ struct sk_buff { }; __u32 priority; kmemcheck_bitfield_begin(flags1); - __u8 local_df:1, + __u8 ignore_df:1, cloned:1, ip_summed:2, nohdr:1, -- cgit v1.2.3 From 7e2b10c1e52ca37fb522be49f4be367f9311d0cd Mon Sep 17 00:00:00 2001 From: Tom Herbert Date: Wed, 4 Jun 2014 17:20:02 -0700 Subject: net: Support for multiple checksums with gso When creating a GSO packet segment we may need to set more than one checksum in the packet (for instance a TCP checksum and UDP checksum for VXLAN encapsulation). To be efficient, we want to do checksum calculation for any part of the packet at most once. This patch adds csum_start offset to skb_gso_cb. This tracks the starting offset for skb->csum which is initially set in skb_segment. When a protocol needs to compute a transport checksum it calls gso_make_checksum which computes the checksum value from the start of transport header to csum_start and then adds in skb->csum to get the full checksum. skb->csum and csum_start are then updated to reflect the checksum of the resultant packet starting from the transport header. This patch also adds a flag to skbuff, encap_hdr_csum, which is set in *gso_segment fucntions to indicate that a tunnel protocol needs checksum calculation Signed-off-by: Tom Herbert Signed-off-by: David S. Miller --- include/linux/skbuff.h | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 7a9beeb1c458..d8d397acb52c 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -567,7 +567,8 @@ struct sk_buff { * headers if needed */ __u8 encapsulation:1; - /* 6/8 bit hole (depending on ndisc_nodetype presence) */ + __u8 encap_hdr_csum:1; + /* 5/7 bit hole (depending on ndisc_nodetype presence) */ kmemcheck_bitfield_end(flags2); #if defined CONFIG_NET_DMA || defined CONFIG_NET_RX_BUSY_POLL @@ -2988,6 +2989,7 @@ static inline struct sec_path *skb_sec_path(struct sk_buff *skb) struct skb_gso_cb { int mac_offset; int encap_level; + __u16 csum_start; }; #define SKB_GSO_CB(skb) ((struct skb_gso_cb *)(skb)->cb) @@ -3012,6 +3014,28 @@ static inline int gso_pskb_expand_head(struct sk_buff *skb, int extra) return 0; } +/* Compute the checksum for a gso segment. First compute the checksum value + * from the start of transport header to SKB_GSO_CB(skb)->csum_start, and + * then add in skb->csum (checksum from csum_start to end of packet). + * skb->csum and csum_start are then updated to reflect the checksum of the + * resultant packet starting from the transport header-- the resultant checksum + * is in the res argument (i.e. normally zero or ~ of checksum of a pseudo + * header. + */ +static inline __sum16 gso_make_checksum(struct sk_buff *skb, __wsum res) +{ + int plen = SKB_GSO_CB(skb)->csum_start - skb_headroom(skb) - + skb_transport_offset(skb); + __u16 csum; + + csum = csum_fold(csum_partial(skb_transport_header(skb), + plen, skb->csum)); + skb->csum = res; + SKB_GSO_CB(skb)->csum_start -= plen; + + return csum; +} + static inline bool skb_is_gso(const struct sk_buff *skb) { return skb_shinfo(skb)->gso_size; -- cgit v1.2.3 From 0f4f4ffa7b7c3d29d0537a126145c9f8d8ed5dbc Mon Sep 17 00:00:00 2001 From: Tom Herbert Date: Wed, 4 Jun 2014 17:20:16 -0700 Subject: net: Add GSO support for UDP tunnels with checksum Added a new netif feature for GSO_UDP_TUNNEL_CSUM. This indicates that a device is capable of computing the UDP checksum in the encapsulating header of a UDP tunnel. Signed-off-by: Tom Herbert Signed-off-by: David S. Miller --- include/linux/skbuff.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index d8d397acb52c..5a6d10a538f5 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -345,6 +345,8 @@ enum { SKB_GSO_UDP_TUNNEL = 1 << 9, SKB_GSO_MPLS = 1 << 10, + + SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11, }; #if BITS_PER_LONG > 32 -- cgit v1.2.3 From 4749c09c37030ccdc44aecebe0f71b02a377fc14 Mon Sep 17 00:00:00 2001 From: Tom Herbert Date: Wed, 4 Jun 2014 17:20:23 -0700 Subject: gre: Call gso_make_checksum Call gso_make_checksum. This should have the benefit of using a checksum that may have been previously computed for the packet. This also adds NETIF_F_GSO_GRE_CSUM to differentiate devices that offload GRE GSO with and without the GRE checksum offloaed. Signed-off-by: Tom Herbert Signed-off-by: David S. Miller --- include/linux/skbuff.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 5a6d10a538f5..c705808bef9c 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -347,6 +347,8 @@ enum { SKB_GSO_MPLS = 1 << 10, SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11, + + SKB_GSO_GRE_CSUM = 1 << 12, }; #if BITS_PER_LONG > 32 -- cgit v1.2.3 From bad93e9d4eeb0d2d6b79204d6cedc7f2e7b256f1 Mon Sep 17 00:00:00 2001 From: Octavian Purdila Date: Thu, 12 Jun 2014 01:36:26 +0300 Subject: net: add __pskb_copy_fclone and pskb_copy_for_clone There are several instances where a pskb_copy or __pskb_copy is immediately followed by an skb_clone. Add a couple of new functions to allow the copy skb to be allocated from the fclone cache and thus speed up subsequent skb_clone calls. Cc: Alexander Smirnov Cc: Dmitry Eremin-Solenikov Cc: Marek Lindner Cc: Simon Wunderlich Cc: Antonio Quartulli Cc: Marcel Holtmann Cc: Gustavo Padovan Cc: Johan Hedberg Cc: Arvid Brodin Cc: Patrick McHardy Cc: Pablo Neira Ayuso Cc: Jozsef Kadlecsik Cc: Lauro Ramos Venancio Cc: Aloisio Almeida Jr Cc: Samuel Ortiz Cc: Jon Maloy Cc: Allan Stephens Cc: Andrew Hendry Cc: Eric Dumazet Reviewed-by: Christoph Paasch Signed-off-by: Octavian Purdila Signed-off-by: David S. Miller --- include/linux/skbuff.h | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index c705808bef9c..1f50bfe2243d 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -744,7 +744,13 @@ struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src); int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask); struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t priority); struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t priority); -struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, gfp_t gfp_mask); +struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, + gfp_t gfp_mask, bool fclone); +static inline struct sk_buff *__pskb_copy(struct sk_buff *skb, int headroom, + gfp_t gfp_mask) +{ + return __pskb_copy_fclone(skb, headroom, gfp_mask, false); +} int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, gfp_t gfp_mask); struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, @@ -2238,6 +2244,14 @@ static inline struct sk_buff *pskb_copy(struct sk_buff *skb, return __pskb_copy(skb, skb_headroom(skb), gfp_mask); } + +static inline struct sk_buff *pskb_copy_for_clone(struct sk_buff *skb, + gfp_t gfp_mask) +{ + return __pskb_copy_fclone(skb, skb_headroom(skb), gfp_mask, true); +} + + /** * skb_clone_writable - is the header of a clone writable * @skb: buffer to check -- cgit v1.2.3 From 5d0c2b95bc57cf8fdc0e7b3e9d7e751eb65ad052 Mon Sep 17 00:00:00 2001 From: Tom Herbert Date: Tue, 10 Jun 2014 18:54:13 -0700 Subject: net: Preserve CHECKSUM_COMPLETE at validation Currently when the first checksum in a packet is validated using CHECKSUM_COMPLETE, ip_summed is overwritten to be CHECKSUM_UNNECESSARY so that any subsequent checksums in the packet are not correctly validated. This patch adds csum_valid flag in sk_buff and uses that to indicate validated checksum instead of setting CHECKSUM_UNNECESSARY. The bit is set accordingly in the skb_checksum_validate_* functions. The flag is checked in skb_checksum_complete, so that validation is communicated between checksum_init and checksum_complete sequence in TCP and UDP. Signed-off-by: Tom Herbert Signed-off-by: David S. Miller --- include/linux/skbuff.h | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 1f50bfe2243d..72a53805858a 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -572,7 +572,8 @@ struct sk_buff { */ __u8 encapsulation:1; __u8 encap_hdr_csum:1; - /* 5/7 bit hole (depending on ndisc_nodetype presence) */ + __u8 csum_valid:1; + /* 4/6 bit hole (depending on ndisc_nodetype presence) */ kmemcheck_bitfield_end(flags2); #if defined CONFIG_NET_DMA || defined CONFIG_NET_RX_BUSY_POLL @@ -2735,7 +2736,7 @@ __sum16 __skb_checksum_complete(struct sk_buff *skb); static inline int skb_csum_unnecessary(const struct sk_buff *skb) { - return skb->ip_summed & CHECKSUM_UNNECESSARY; + return ((skb->ip_summed & CHECKSUM_UNNECESSARY) || skb->csum_valid); } /** @@ -2769,10 +2770,8 @@ static inline bool __skb_checksum_validate_needed(struct sk_buff *skb, bool zero_okay, __sum16 check) { - if (skb_csum_unnecessary(skb)) { - return false; - } else if (zero_okay && !check) { - skb->ip_summed = CHECKSUM_UNNECESSARY; + if (skb_csum_unnecessary(skb) || (zero_okay && !check)) { + skb->csum_valid = 1; return false; } @@ -2799,15 +2798,20 @@ static inline __sum16 __skb_checksum_validate_complete(struct sk_buff *skb, { if (skb->ip_summed == CHECKSUM_COMPLETE) { if (!csum_fold(csum_add(psum, skb->csum))) { - skb->ip_summed = CHECKSUM_UNNECESSARY; + skb->csum_valid = 1; return 0; } } skb->csum = psum; - if (complete || skb->len <= CHECKSUM_BREAK) - return __skb_checksum_complete(skb); + if (complete || skb->len <= CHECKSUM_BREAK) { + __sum16 csum; + + csum = __skb_checksum_complete(skb); + skb->csum_valid = !csum; + return csum; + } return 0; } @@ -2831,6 +2835,7 @@ static inline __wsum null_compute_pseudo(struct sk_buff *skb, int proto) zero_okay, check, compute_pseudo) \ ({ \ __sum16 __ret = 0; \ + skb->csum_valid = 0; \ if (__skb_checksum_validate_needed(skb, zero_okay, check)) \ __ret = __skb_checksum_validate_complete(skb, \ complete, compute_pseudo(skb, proto)); \ -- cgit v1.2.3 From 7e3cead5172927732f51fde77fef6f521e22f209 Mon Sep 17 00:00:00 2001 From: Tom Herbert Date: Tue, 10 Jun 2014 18:54:19 -0700 Subject: net: Save software checksum complete In skb_checksum complete, if we need to compute the checksum for the packet (via skb_checksum) save the result as CHECKSUM_COMPLETE. Subsequent checksum verification can use this. Also, added csum_complete_sw flag to distinguish between software and hardware generated checksum complete, we should always be able to trust the software computation. Signed-off-by: Tom Herbert Signed-off-by: David S. Miller --- include/linux/skbuff.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 72a53805858a..5b5cd3189c98 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -573,7 +573,8 @@ struct sk_buff { __u8 encapsulation:1; __u8 encap_hdr_csum:1; __u8 csum_valid:1; - /* 4/6 bit hole (depending on ndisc_nodetype presence) */ + __u8 csum_complete_sw:1; + /* 3/5 bit hole (depending on ndisc_nodetype presence) */ kmemcheck_bitfield_end(flags2); #if defined CONFIG_NET_DMA || defined CONFIG_NET_RX_BUSY_POLL -- cgit v1.2.3 From 4b28252cada3d0521ab59751f4240ecdfb9bba18 Mon Sep 17 00:00:00 2001 From: Tom Herbert Date: Sat, 14 Jun 2014 23:23:52 -0700 Subject: net: Fix GSO constants to match NETIF flags Joseph Gasparakis reported that VXLAN GSO offload stopped working with i40e device after recent UDP changes. The problem is that the SKB_GSO_* bits are out of sync with the corresponding NETIF flags. This patch fixes that. Also, we add BUILD_BUG_ONs in net_gso_ok for several GSO constants that were missing to avoid the problem in the future. Reported-by: Joseph Gasparakis Signed-off-by: Tom Herbert Signed-off-by: David S. Miller --- include/linux/skbuff.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 5b5cd3189c98..e13ed90be7c2 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -338,17 +338,18 @@ enum { SKB_GSO_GRE = 1 << 6, - SKB_GSO_IPIP = 1 << 7, + SKB_GSO_GRE_CSUM = 1 << 7, - SKB_GSO_SIT = 1 << 8, + SKB_GSO_IPIP = 1 << 8, - SKB_GSO_UDP_TUNNEL = 1 << 9, + SKB_GSO_SIT = 1 << 9, - SKB_GSO_MPLS = 1 << 10, + SKB_GSO_UDP_TUNNEL = 1 << 10, SKB_GSO_UDP_TUNNEL_CSUM = 1 << 11, - SKB_GSO_GRE_CSUM = 1 << 12, + SKB_GSO_MPLS = 1 << 12, + }; #if BITS_PER_LONG > 32 -- cgit v1.2.3 From e5eb4e30a51236079fb22bb9f75fcd31915b03c6 Mon Sep 17 00:00:00 2001 From: Tom Herbert Date: Sat, 14 Jun 2014 23:24:28 -0700 Subject: net: add skb_pop_rcv_encapsulation This function is used by UDP encapsulation protocols in RX when crossing encapsulation boundary. If ip_summed is set to CHECKSUM_UNNECESSARY and encapsulation is not set, change to CHECKSUM_NONE since the checksum has not been validated within the encapsulation. Clears csum_valid by the same rationale. Signed-off-by: Tom Herbert Signed-off-by: David S. Miller --- include/linux/skbuff.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'include/linux/skbuff.h') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index e13ed90be7c2..ec89301ada41 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1854,6 +1854,18 @@ static inline int pskb_network_may_pull(struct sk_buff *skb, unsigned int len) return pskb_may_pull(skb, skb_network_offset(skb) + len); } +static inline void skb_pop_rcv_encapsulation(struct sk_buff *skb) +{ + /* Only continue with checksum unnecessary if device indicated + * it is valid across encapsulation (skb->encapsulation was set). + */ + if (skb->ip_summed == CHECKSUM_UNNECESSARY && !skb->encapsulation) + skb->ip_summed = CHECKSUM_NONE; + + skb->encapsulation = 0; + skb->csum_valid = 0; +} + /* * CPUs often take a performance hit when accessing unaligned memory * locations. The actual performance hit varies, it can be small if the -- cgit v1.2.3