diff options
| author | Eric Dumazet <edumazet@google.com> | 2026-01-05 10:17:19 +0000 |
|---|---|---|
| committer | Jakub Kicinski <kuba@kernel.org> | 2026-01-06 17:06:03 -0800 |
| commit | e9cd04b2816f8ebae9b274f5c52145c8f40d4a6f (patch) | |
| tree | 35b95b8046ae2b3aea3dc5ed465d2970ca115077 | |
| parent | e4bc5dd53bf5d46cd58f081ffccc3809e2be5373 (diff) | |
udp: udplite is unlikely
Add some unlikely() annotations to speed up the fast path,
at least with clang compiler.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://patch.msgid.link/20260105101719.2378881-1-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
| -rw-r--r-- | include/linux/udp.h | 2 | ||||
| -rw-r--r-- | include/net/udp.h | 8 | ||||
| -rw-r--r-- | net/ipv4/udp.c | 5 | ||||
| -rw-r--r-- | net/ipv6/udp.c | 5 |
4 files changed, 11 insertions, 9 deletions
diff --git a/include/linux/udp.h b/include/linux/udp.h index 58795688a186..1cbf6b4d3aab 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -236,7 +236,7 @@ static inline void udp_allow_gso(struct sock *sk) hlist_nulls_for_each_entry_rcu(__up, node, list, udp_lrpa_node) #endif -#define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE) +#define IS_UDPLITE(__sk) (unlikely(__sk->sk_protocol == IPPROTO_UDPLITE)) static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6) { diff --git a/include/net/udp.h b/include/net/udp.h index a061d1b22ddc..700dbedcb15f 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -527,18 +527,18 @@ static inline int copy_linear_skb(struct sk_buff *skb, int len, int off, * SNMP statistics for UDP and UDP-Lite */ #define UDP_INC_STATS(net, field, is_udplite) do { \ - if (is_udplite) SNMP_INC_STATS((net)->mib.udplite_statistics, field); \ + if (unlikely(is_udplite)) SNMP_INC_STATS((net)->mib.udplite_statistics, field); \ else SNMP_INC_STATS((net)->mib.udp_statistics, field); } while(0) #define __UDP_INC_STATS(net, field, is_udplite) do { \ - if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_statistics, field); \ + if (unlikely(is_udplite)) __SNMP_INC_STATS((net)->mib.udplite_statistics, field); \ else __SNMP_INC_STATS((net)->mib.udp_statistics, field); } while(0) #define __UDP6_INC_STATS(net, field, is_udplite) do { \ - if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);\ + if (unlikely(is_udplite)) __SNMP_INC_STATS((net)->mib.udplite_stats_in6, field); \ else __SNMP_INC_STATS((net)->mib.udp_stats_in6, field); \ } while(0) #define UDP6_INC_STATS(net, field, __lite) do { \ - if (__lite) SNMP_INC_STATS((net)->mib.udplite_stats_in6, field); \ + if (unlikely(__lite)) SNMP_INC_STATS((net)->mib.udplite_stats_in6, field); \ else SNMP_INC_STATS((net)->mib.udp_stats_in6, field); \ } while(0) diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 591527ae811c..9c87067c74bc 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1193,7 +1193,7 @@ csum_partial: send: err = ip_send_skb(sock_net(sk), skb); - if (err) { + if (unlikely(err)) { if (err == -ENOBUFS && !inet_test_bit(RECVERR, sk)) { UDP_INC_STATS(sock_net(sk), @@ -2429,7 +2429,8 @@ static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb) /* * UDP-Lite specific tests, ignored on UDP sockets */ - if (udp_test_bit(UDPLITE_RECV_CC, sk) && UDP_SKB_CB(skb)->partial_cov) { + if (unlikely(udp_test_bit(UDPLITE_RECV_CC, sk) && + UDP_SKB_CB(skb)->partial_cov)) { u16 pcrlen = READ_ONCE(up->pcrlen); /* diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 794c13674e8a..010b909275dd 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -875,7 +875,8 @@ static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb) /* * UDP-Lite specific tests, ignored on UDP sockets (see net/ipv4/udp.c). */ - if (udp_test_bit(UDPLITE_RECV_CC, sk) && UDP_SKB_CB(skb)->partial_cov) { + if (unlikely(udp_test_bit(UDPLITE_RECV_CC, sk) && + UDP_SKB_CB(skb)->partial_cov)) { u16 pcrlen = READ_ONCE(up->pcrlen); if (pcrlen == 0) { /* full coverage was set */ @@ -1439,7 +1440,7 @@ csum_partial: send: err = ip6_send_skb(skb); - if (err) { + if (unlikely(err)) { if (err == -ENOBUFS && !inet6_test_bit(RECVERR6, sk)) { UDP6_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS, is_udplite); |