diff options
Diffstat (limited to 'include/net')
184 files changed, 5650 insertions, 4040 deletions
diff --git a/include/net/9p/9p.h b/include/net/9p/9p.h index 60cad0d200a4..fd7a034b8278 100644 --- a/include/net/9p/9p.h +++ b/include/net/9p/9p.h @@ -24,6 +24,8 @@ * @P9_DEBUG_PKT: packet marshalling/unmarshalling * @P9_DEBUG_FSC: FS-cache tracing * @P9_DEBUG_VPKT: Verbose packet debugging (full packet dump) + * @P9_DEBUG_CACHE: cache operations tracing + * @P9_DEBUG_MMAP: memory-mapped I/O tracing * * These flags are passed at mount time to turn on various levels of * verbosity and tracing which will be output to the system logs. @@ -68,13 +70,39 @@ void _p9_debug(enum p9_debug_flags level, const char *func, * @P9_RSYMLINK: make symlink response * @P9_TMKNOD: create a special file object request * @P9_RMKNOD: create a special file object response + * @P9_TLOPEN: open a file for I/O (9P2000.L) + * @P9_RLOPEN: response with qid and iounit (9P2000.L) * @P9_TLCREATE: prepare a handle for I/O on an new file for 9P2000.L * @P9_RLCREATE: response with file access information for 9P2000.L * @P9_TRENAME: rename request * @P9_RRENAME: rename response - * @P9_TMKDIR: create a directory request - * @P9_RMKDIR: create a directory response - * @P9_TVERSION: version handshake request + * @P9_TREADLINK: read symbolic link target (9P2000.L) + * @P9_RREADLINK: response with symbolic link target (9P2000.L) + * @P9_TGETATTR: get file attributes request (9P2000.L) + * @P9_RGETATTR: get file attributes response (9P2000.L) + * @P9_TSETATTR: set file attributes request (9P2000.L) + * @P9_RSETATTR: set file attributes response (9P2000.L) + * @P9_TXATTRWALK: prepare to read/list extended attributes (9P2000.L) + * @P9_RXATTRWALK: response with extended attribute size (9P2000.L) + * @P9_TXATTRCREATE: prepare to set extended attribute (9P2000.L) + * @P9_RXATTRCREATE: set extended attribute response (9P2000.L) + * @P9_TREADDIR: read directory entries request (9P2000.L) + * @P9_RREADDIR: read directory entries response (9P2000.L) + * @P9_TFSYNC: flush cached file data to storage request (9P2000.L) + * @P9_RFSYNC: flush cached file data to storage response (9P2000.L) + * @P9_TLOCK: acquire or release a POSIX record lock (9P2000.L) + * @P9_RLOCK: POSIX record lock response (9P2000.L) + * @P9_TGETLOCK: test for existence of POSIX record lock (9P2000.L) + * @P9_RGETLOCK: POSIX record lock test response (9P2000.L) + * @P9_TLINK: create a hard link (9P2000.L) + * @P9_RLINK: hard link response (9P2000.L) + * @P9_TRENAMEAT: safely rename across directories (9P2000.L) + * @P9_RRENAMEAT: rename response (9P2000.L) + * @P9_TUNLINKAT: unlink a file or directory (9P2000.L) + * @P9_RUNLINKAT: unlink response (9P2000.L) + * @P9_TMKDIR: create a directory request (9P2000.L) + * @P9_RMKDIR: create a directory response (9P2000.L) + * @P9_TVERSION: negotiate protocol version and message size * @P9_RVERSION: version handshake response * @P9_TAUTH: request to establish authentication channel * @P9_RAUTH: response with authentication information @@ -194,6 +222,10 @@ enum p9_msg_t { * @P9_ORCLOSE: remove the file when the file is closed * @P9_OAPPEND: open the file and seek to the end * @P9_OEXCL: only create a file, do not open it + * @P9L_MODE_MASK: mask for protocol mode bits (client-side only) + * @P9L_DIRECT: disable client-side caching for this file + * @P9L_NOWRITECACHE: disable write caching for this file + * @P9L_LOOSE: enable loose cache consistency * * 9P open modes differ slightly from Posix standard modes. * In particular, there are extra modes which specify different diff --git a/include/net/9p/client.h b/include/net/9p/client.h index 838a94218b59..55c6cb54bd25 100644 --- a/include/net/9p/client.h +++ b/include/net/9p/client.h @@ -192,6 +192,7 @@ struct p9_rdma_opts { * @dfltgid: default numeric groupid to mount hierarchy as * @uid: if %V9FS_ACCESS_SINGLE, the numeric uid which mounted the hierarchy * @session_lock_timeout: retry interval for blocking locks + * @ndentry_timeout_ms: Negative dentry lookup cache retention time in ms * * This strucure holds options which are parsed and will be transferred * to the v9fs_session_info structure when mounted, and therefore largely @@ -203,6 +204,7 @@ struct p9_session_opts { unsigned short debug; unsigned int afid; unsigned int cache; + unsigned int ndentry_timeout_ms; #ifdef CONFIG_9P_FSCACHE char *cachetag; #endif diff --git a/include/net/Space.h b/include/net/Space.h deleted file mode 100644 index ef42629f4258..000000000000 --- a/include/net/Space.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* A unified ethernet device probe. This is the easiest way to have every - * ethernet adaptor have the name "eth[0123...]". - */ - -struct net_device *ultra_probe(int unit); -struct net_device *wd_probe(int unit); -struct net_device *ne_probe(int unit); -struct net_device *smc_init(int unit); -struct net_device *cs89x0_probe(int unit); -struct net_device *tc515_probe(int unit); -struct net_device *lance_probe(int unit); diff --git a/include/net/act_api.h b/include/net/act_api.h index 91a24b5e0b93..fd03f6319e88 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -45,6 +45,7 @@ struct tc_action { struct tc_cookie __rcu *user_cookie; struct tcf_chain __rcu *goto_chain; u32 tcfa_flags; + struct rcu_head tcfa_rcu; u8 hw_stats; u8 used_hw_stats; bool used_hw_stats_valid; @@ -70,6 +71,7 @@ struct tc_action { #define TCA_ACT_FLAGS_REPLACE (1U << (TCA_ACT_FLAGS_USER_BITS + 2)) #define TCA_ACT_FLAGS_NO_RTNL (1U << (TCA_ACT_FLAGS_USER_BITS + 3)) #define TCA_ACT_FLAGS_AT_INGRESS (1U << (TCA_ACT_FLAGS_USER_BITS + 4)) +#define TCA_ACT_FLAGS_AT_INGRESS_OR_CLSACT (1U << (TCA_ACT_FLAGS_USER_BITS + 5)) /* Update lastuse only if needed, to avoid dirtying a cache line. * We use a temp variable to avoid fetching jiffies twice. @@ -159,7 +161,7 @@ int tc_action_net_init(struct net *net, struct tc_action_net *tn, { int err = 0; - tn->idrinfo = kmalloc(sizeof(*tn->idrinfo), GFP_KERNEL); + tn->idrinfo = kmalloc_obj(*tn->idrinfo); if (!tn->idrinfo) return -ENOMEM; tn->ops = ops; @@ -240,7 +242,7 @@ static inline void tcf_action_update_bstats(struct tc_action *a, static inline void tcf_action_inc_drop_qstats(struct tc_action *a) { if (likely(a->cpu_qstats)) { - qstats_drop_inc(this_cpu_ptr(a->cpu_qstats)); + qstats_cpu_drop_inc(a->cpu_qstats); return; } atomic_inc(&a->tcfa_drops); @@ -249,7 +251,7 @@ static inline void tcf_action_inc_drop_qstats(struct tc_action *a) static inline void tcf_action_inc_overlimit_qstats(struct tc_action *a) { if (likely(a->cpu_qstats)) { - qstats_overlimit_inc(this_cpu_ptr(a->cpu_qstats)); + qstats_cpu_overlimit_inc(a->cpu_qstats); return; } atomic_inc(&a->tcfa_overlimits); @@ -268,6 +270,25 @@ int tcf_action_check_ctrlact(int action, struct tcf_proto *tp, struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action, struct tcf_chain *newchain); +/* Range check for a control action supplied by user space. + * + * This is the same test tcf_action_check_ctrlact() applies to the primary + * control action, factored out for the *fallback* control actions + * (act_gact's TCA_GACT_PROB.paction and act_police's TCA_POLICE_RESULT), + * which must not reach tcf_action_check_ctrlact() because they have no + * goto_chain to allocate. Without it, user space can store kernel-internal + * verdicts such as TC_ACT_CONSUMED, which is TC_ACT_VALUE_MAX + 1 and is + * deliberately not part of the UAPI value range. + */ +static inline bool tcf_action_valid(int action) +{ + int opcode = TC_ACT_EXT_OPCODE(action); + + if (!opcode) + return action <= TC_ACT_VALUE_MAX; + return opcode <= TC_ACT_EXT_OPCODE_MAX || action == TC_ACT_UNSPEC; +} + #ifdef CONFIG_INET DECLARE_STATIC_KEY_FALSE(tcf_frag_xmit_count); #endif diff --git a/include/net/addrconf.h b/include/net/addrconf.h index 78e8b877fb25..e6764245995f 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -8,7 +8,8 @@ #define MIN_VALID_LIFETIME (2*3600) /* 2 hours */ -#define TEMP_VALID_LIFETIME (7*86400) /* 1 week */ +/* TEMP_VALID_LIFETIME default value as specified in RFC 8981 3.8 */ +#define TEMP_VALID_LIFETIME (2*86400) /* 2 days */ #define TEMP_PREFERRED_LIFETIME (86400) /* 24 hours */ #define REGEN_MIN_ADVANCE (2) /* 2 seconds */ #define REGEN_MAX_RETRY (3) @@ -368,8 +369,11 @@ static inline struct inet6_dev *__in6_dev_get_rtnl_net(const struct net_device * static inline struct inet6_dev *__in6_dev_stats_get(const struct net_device *dev, const struct sk_buff *skb) { - if (netif_is_l3_master(dev)) + if (netif_is_l3_master(dev)) { dev = dev_get_by_index_rcu(dev_net(dev), inet6_iif(skb)); + if (!dev) + return NULL; + } return __in6_dev_get(dev); } @@ -401,8 +405,8 @@ static inline struct inet6_dev *in6_dev_get(const struct net_device *dev) rcu_read_lock(); idev = rcu_dereference(dev->ip6_ptr); - if (idev) - refcount_inc(&idev->refcnt); + if (idev && !refcount_inc_not_zero(&idev->refcnt)) + idev = NULL; rcu_read_unlock(); return idev; } @@ -442,6 +446,11 @@ static inline void in6_dev_hold(struct inet6_dev *idev) refcount_inc(&idev->refcnt); } +static inline bool in6_dev_hold_safe(struct inet6_dev *idev) +{ + return refcount_inc_not_zero(&idev->refcnt); +} + /* called with rcu_read_lock held */ static inline bool ip6_ignore_linkdown(const struct net_device *dev) { diff --git a/include/net/af_unix.h b/include/net/af_unix.h index 34f53dde65ce..bb1b3dee02e8 100644 --- a/include/net/af_unix.h +++ b/include/net/af_unix.h @@ -49,6 +49,7 @@ struct unix_sock { struct scm_stat scm_stat; int inq_len; bool recvmsg_inq; + bool scm_rights_notrunc; #if IS_ENABLED(CONFIG_AF_UNIX_OOB) struct sk_buff *oob_skb; #endif diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h index d40e978126e3..5549298c1ec6 100644 --- a/include/net/af_vsock.h +++ b/include/net/af_vsock.h @@ -10,6 +10,7 @@ #include <linux/kernel.h> #include <linux/workqueue.h> +#include <net/netns/vsock.h> #include <net/sock.h> #include <uapi/linux/vm_sockets.h> @@ -51,13 +52,10 @@ struct vsock_sock { * The listening socket is the head for both lists. Sockets created * for connection requests are placed in the pending list until they * are connected, at which point they are put in the accept queue list - * so they can be accepted in accept(). If accept() cannot accept the - * connection, it is marked as rejected so the cleanup function knows - * to clean up the socket. + * so they can be accepted in accept(). */ struct list_head pending_links; struct list_head accept_queue; - bool rejected; struct delayed_work connect_work; struct delayed_work pending_work; struct delayed_work close_work; @@ -124,7 +122,7 @@ struct vsock_transport { size_t len, int flags); int (*dgram_enqueue)(struct vsock_sock *, struct sockaddr_vm *, struct msghdr *, size_t len); - bool (*dgram_allow)(u32 cid, u32 port); + bool (*dgram_allow)(struct vsock_sock *vsk, u32 cid, u32 port); /* STREAM. */ /* TODO: stream_bind() */ @@ -136,14 +134,14 @@ struct vsock_transport { s64 (*stream_has_space)(struct vsock_sock *); u64 (*stream_rcvhiwat)(struct vsock_sock *); bool (*stream_is_active)(struct vsock_sock *); - bool (*stream_allow)(u32 cid, u32 port); + bool (*stream_allow)(struct vsock_sock *vsk, u32 cid, u32 port); /* SEQ_PACKET. */ ssize_t (*seqpacket_dequeue)(struct vsock_sock *vsk, struct msghdr *msg, int flags); int (*seqpacket_enqueue)(struct vsock_sock *vsk, struct msghdr *msg, size_t len); - bool (*seqpacket_allow)(u32 remote_cid); + bool (*seqpacket_allow)(struct vsock_sock *vsk, u32 remote_cid); u32 (*seqpacket_has_data)(struct vsock_sock *vsk); /* Notification. */ @@ -178,6 +176,15 @@ struct vsock_transport { /* Addressing. */ u32 (*get_local_cid)(void); + /* Check if this transport serves a specific remote CID. + * For H2G transports: return true if the CID belongs to a registered + * guest. If not implemented, all CIDs > VMADDR_CID_HOST go to H2G. + * For G2H transports: return true if the transport can reach arbitrary + * CIDs via the hypervisor (i.e. supports the fallback overlay). VMCI + * does not implement this as it only serves CIDs 0 and 2. + */ + bool (*has_remote_cid)(struct vsock_sock *vsk, u32 remote_cid); + /* Read a single skb */ int (*read_skb)(struct vsock_sock *, skb_read_actor_t); @@ -210,12 +217,21 @@ static inline bool __vsock_in_connected_table(struct vsock_sock *vsk) void vsock_add_pending(struct sock *listener, struct sock *pending); void vsock_remove_pending(struct sock *listener, struct sock *pending); void vsock_enqueue_accept(struct sock *listener, struct sock *connected); +void vsock_pending_to_accept(struct sock *listener, struct sock *pending); void vsock_insert_connected(struct vsock_sock *vsk); void vsock_remove_bound(struct vsock_sock *vsk); void vsock_remove_connected(struct vsock_sock *vsk); struct sock *vsock_find_bound_socket(struct sockaddr_vm *addr); struct sock *vsock_find_connected_socket(struct sockaddr_vm *src, struct sockaddr_vm *dst); +struct sock *vsock_find_bound_socket_net(struct sockaddr_vm *addr, + struct net *net); +struct sock *vsock_find_connected_socket_net(struct sockaddr_vm *src, + struct sockaddr_vm *dst, + struct net *net); +bool vsock_check_source(const struct vsock_sock *vsk, + const struct vsock_transport *transport, + const struct sockaddr_vm *src); void vsock_remove_sock(struct vsock_sock *vsk); void vsock_for_each_connected_socket(struct vsock_transport *transport, void (*fn)(struct sock *sk)); @@ -256,4 +272,62 @@ static inline bool vsock_msgzerocopy_allow(const struct vsock_transport *t) { return t->msgzerocopy_allow && t->msgzerocopy_allow(); } + +static inline enum vsock_net_mode vsock_net_mode(struct net *net) +{ + if (!net) + return VSOCK_NET_MODE_GLOBAL; + + return READ_ONCE(net->vsock.mode); +} + +static inline bool vsock_net_mode_global(struct vsock_sock *vsk) +{ + return vsock_net_mode(sock_net(sk_vsock(vsk))) == VSOCK_NET_MODE_GLOBAL; +} + +static inline bool vsock_net_set_child_mode(struct net *net, + enum vsock_net_mode mode) +{ + int new_locked = mode + 1; + int old_locked = 0; /* unlocked */ + + if (try_cmpxchg(&net->vsock.child_ns_mode_locked, + &old_locked, new_locked)) { + WRITE_ONCE(net->vsock.child_ns_mode, mode); + return true; + } + + return old_locked == new_locked; +} + +static inline enum vsock_net_mode vsock_net_child_mode(struct net *net) +{ + return READ_ONCE(net->vsock.child_ns_mode); +} + +/* Return true if two namespaces pass the mode rules. Otherwise, return false. + * + * A NULL namespace is treated as VSOCK_NET_MODE_GLOBAL. + * + * Read more about modes in the comment header of net/vmw_vsock/af_vsock.c. + */ +static inline bool vsock_net_check_mode(struct net *ns0, struct net *ns1) +{ + enum vsock_net_mode mode0, mode1; + + /* Any vsocks within the same network namespace are always reachable, + * regardless of the mode. + */ + if (net_eq(ns0, ns1)) + return true; + + mode0 = vsock_net_mode(ns0); + mode1 = vsock_net_mode(ns1); + + /* Different namespaces are only reachable if they are both + * global mode. + */ + return mode0 == VSOCK_NET_MODE_GLOBAL && mode0 == mode1; +} #endif /* __AF_VSOCK_H__ */ diff --git a/include/net/amt.h b/include/net/amt.h index c881bc8b673b..a0255491f5b0 100644 --- a/include/net/amt.h +++ b/include/net/amt.h @@ -331,7 +331,7 @@ struct amt_dev { enum amt_status status; /* Generated key */ siphash_key_t key; - struct socket __rcu *sock; + struct sock __rcu *sk; u32 max_groups; u32 max_sources; u32 hash_buckets; diff --git a/include/net/atmclip.h b/include/net/atmclip.h deleted file mode 100644 index 70e350e0db3d..000000000000 --- a/include/net/atmclip.h +++ /dev/null @@ -1,53 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* net/atm/atmarp.h - RFC1577 ATM ARP */ - -/* Written 1995-2000 by Werner Almesberger, EPFL LRC/ICA */ - - -#ifndef _ATMCLIP_H -#define _ATMCLIP_H - -#include <linux/netdevice.h> -#include <linux/atm.h> -#include <linux/atmdev.h> -#include <linux/atmarp.h> -#include <linux/spinlock.h> -#include <net/neighbour.h> - - -#define CLIP_VCC(vcc) ((struct clip_vcc *) ((vcc)->user_back)) - -struct sk_buff; - -struct clip_vcc { - struct atm_vcc *vcc; /* VCC descriptor */ - struct atmarp_entry *entry; /* ATMARP table entry, NULL if IP addr. - isn't known yet */ - int xoff; /* 1 if send buffer is full */ - unsigned char encap; /* 0: NULL, 1: LLC/SNAP */ - unsigned long last_use; /* last send or receive operation */ - unsigned long idle_timeout; /* keep open idle for so many jiffies*/ - void (*old_push)(struct atm_vcc *vcc,struct sk_buff *skb); - /* keep old push fn for chaining */ - void (*old_pop)(struct atm_vcc *vcc,struct sk_buff *skb); - /* keep old pop fn for chaining */ - struct clip_vcc *next; /* next VCC */ -}; - - -struct atmarp_entry { - struct clip_vcc *vccs; /* active VCCs; NULL if resolution is - pending */ - unsigned long expires; /* entry expiration time */ - struct neighbour *neigh; /* neighbour back-pointer */ -}; - -#define PRIV(dev) ((struct clip_priv *) netdev_priv(dev)) - -struct clip_priv { - int number; /* for convenience ... */ - spinlock_t xoff_lock; /* ensures that pop is atomic (SMP) */ - struct net_device *next; /* next CLIP interface */ -}; - -#endif diff --git a/include/net/ax25.h b/include/net/ax25.h index a7bba42dde15..6b2f518facdb 100644 --- a/include/net/ax25.h +++ b/include/net/ax25.h @@ -1,486 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * Declarations of AX.25 type objects. - * - * Alan Cox (GW4PTS) 10/11/93 - */ #ifndef _AX25_H -#define _AX25_H +#define _AX25_H #include <linux/ax25.h> -#include <linux/spinlock.h> -#include <linux/timer.h> -#include <linux/list.h> -#include <linux/slab.h> -#include <linux/refcount.h> -#include <net/neighbour.h> -#include <net/sock.h> -#include <linux/seq_file.h> -#define AX25_T1CLAMPLO 1 -#define AX25_T1CLAMPHI (30 * HZ) - -#define AX25_BPQ_HEADER_LEN 16 -#define AX25_KISS_HEADER_LEN 1 - -#define AX25_HEADER_LEN 17 -#define AX25_ADDR_LEN 7 -#define AX25_DIGI_HEADER_LEN (AX25_MAX_DIGIS * AX25_ADDR_LEN) -#define AX25_MAX_HEADER_LEN (AX25_HEADER_LEN + AX25_DIGI_HEADER_LEN) - -/* AX.25 Protocol IDs */ -#define AX25_P_ROSE 0x01 -#define AX25_P_VJCOMP 0x06 /* Compressed TCP/IP packet */ - /* Van Jacobsen (RFC 1144) */ -#define AX25_P_VJUNCOMP 0x07 /* Uncompressed TCP/IP packet */ - /* Van Jacobsen (RFC 1144) */ -#define AX25_P_SEGMENT 0x08 /* Segmentation fragment */ -#define AX25_P_TEXNET 0xc3 /* TEXTNET datagram protocol */ -#define AX25_P_LQ 0xc4 /* Link Quality Protocol */ -#define AX25_P_ATALK 0xca /* Appletalk */ -#define AX25_P_ATALK_ARP 0xcb /* Appletalk ARP */ -#define AX25_P_IP 0xcc /* ARPA Internet Protocol */ -#define AX25_P_ARP 0xcd /* ARPA Address Resolution */ -#define AX25_P_FLEXNET 0xce /* FlexNet */ -#define AX25_P_NETROM 0xcf /* NET/ROM */ -#define AX25_P_TEXT 0xF0 /* No layer 3 protocol impl. */ - -/* AX.25 Segment control values */ -#define AX25_SEG_REM 0x7F -#define AX25_SEG_FIRST 0x80 - -#define AX25_CBIT 0x80 /* Command/Response bit */ -#define AX25_EBIT 0x01 /* HDLC Address Extension bit */ -#define AX25_HBIT 0x80 /* Has been repeated bit */ - -#define AX25_SSSID_SPARE 0x60 /* Unused bits in SSID for standard AX.25 */ -#define AX25_ESSID_SPARE 0x20 /* Unused bits in SSID for extended AX.25 */ -#define AX25_DAMA_FLAG 0x20 /* Well, it is *NOT* unused! (dl1bke 951121 */ - -#define AX25_COND_ACK_PENDING 0x01 -#define AX25_COND_REJECT 0x02 -#define AX25_COND_PEER_RX_BUSY 0x04 -#define AX25_COND_OWN_RX_BUSY 0x08 -#define AX25_COND_DAMA_MODE 0x10 - -#ifndef _LINUX_NETDEVICE_H -#include <linux/netdevice.h> -#endif - -/* Upper sub-layer (LAPB) definitions */ - -/* Control field templates */ -#define AX25_I 0x00 /* Information frames */ -#define AX25_S 0x01 /* Supervisory frames */ -#define AX25_RR 0x01 /* Receiver ready */ -#define AX25_RNR 0x05 /* Receiver not ready */ -#define AX25_REJ 0x09 /* Reject */ -#define AX25_U 0x03 /* Unnumbered frames */ -#define AX25_SABM 0x2f /* Set Asynchronous Balanced Mode */ -#define AX25_SABME 0x6f /* Set Asynchronous Balanced Mode Extended */ -#define AX25_DISC 0x43 /* Disconnect */ -#define AX25_DM 0x0f /* Disconnected mode */ -#define AX25_UA 0x63 /* Unnumbered acknowledge */ -#define AX25_FRMR 0x87 /* Frame reject */ -#define AX25_UI 0x03 /* Unnumbered information */ -#define AX25_XID 0xaf /* Exchange information */ -#define AX25_TEST 0xe3 /* Test */ - -#define AX25_PF 0x10 /* Poll/final bit for standard AX.25 */ -#define AX25_EPF 0x01 /* Poll/final bit for extended AX.25 */ - -#define AX25_ILLEGAL 0x100 /* Impossible to be a real frame type */ - -#define AX25_POLLOFF 0 -#define AX25_POLLON 1 - -/* AX25 L2 C-bit */ -#define AX25_COMMAND 1 -#define AX25_RESPONSE 2 - -/* Define Link State constants. */ - -enum { - AX25_STATE_0, /* Listening */ - AX25_STATE_1, /* SABM sent */ - AX25_STATE_2, /* DISC sent */ - AX25_STATE_3, /* Established */ - AX25_STATE_4 /* Recovery */ -}; - -#define AX25_MODULUS 8 /* Standard AX.25 modulus */ -#define AX25_EMODULUS 128 /* Extended AX.25 modulus */ - -enum { - AX25_PROTO_STD_SIMPLEX, - AX25_PROTO_STD_DUPLEX, -#ifdef CONFIG_AX25_DAMA_SLAVE - AX25_PROTO_DAMA_SLAVE, -#ifdef CONFIG_AX25_DAMA_MASTER - AX25_PROTO_DAMA_MASTER, -#define AX25_PROTO_MAX AX25_PROTO_DAMA_MASTER -#endif -#endif - __AX25_PROTO_MAX, - AX25_PROTO_MAX = __AX25_PROTO_MAX -1 -}; - -enum { - AX25_VALUES_IPDEFMODE, /* 0=DG 1=VC */ - AX25_VALUES_AXDEFMODE, /* 0=Normal 1=Extended Seq Nos */ - AX25_VALUES_BACKOFF, /* 0=None 1=Linear 2=Exponential */ - AX25_VALUES_CONMODE, /* Allow connected modes - 0=No 1=no "PID text" 2=all PIDs */ - AX25_VALUES_WINDOW, /* Default window size for standard AX.25 */ - AX25_VALUES_EWINDOW, /* Default window size for extended AX.25 */ - AX25_VALUES_T1, /* Default T1 timeout value */ - AX25_VALUES_T2, /* Default T2 timeout value */ - AX25_VALUES_T3, /* Default T3 timeout value */ - AX25_VALUES_IDLE, /* Connected mode idle timer */ - AX25_VALUES_N2, /* Default N2 value */ - AX25_VALUES_PACLEN, /* AX.25 MTU */ - AX25_VALUES_PROTOCOL, /* Std AX.25, DAMA Slave, DAMA Master */ -#ifdef CONFIG_AX25_DAMA_SLAVE - AX25_VALUES_DS_TIMEOUT, /* DAMA Slave timeout */ -#endif - AX25_MAX_VALUES /* THIS MUST REMAIN THE LAST ENTRY OF THIS LIST */ -}; - -#define AX25_DEF_IPDEFMODE 0 /* Datagram */ -#define AX25_DEF_AXDEFMODE 0 /* Normal */ -#define AX25_DEF_BACKOFF 1 /* Linear backoff */ -#define AX25_DEF_CONMODE 2 /* Connected mode allowed */ -#define AX25_DEF_WINDOW 2 /* Window=2 */ -#define AX25_DEF_EWINDOW 32 /* Module-128 Window=32 */ -#define AX25_DEF_T1 10000 /* T1=10s */ -#define AX25_DEF_T2 3000 /* T2=3s */ -#define AX25_DEF_T3 300000 /* T3=300s */ -#define AX25_DEF_N2 10 /* N2=10 */ -#define AX25_DEF_IDLE 0 /* Idle=None */ -#define AX25_DEF_PACLEN 256 /* Paclen=256 */ -#define AX25_DEF_PROTOCOL AX25_PROTO_STD_SIMPLEX /* Standard AX.25 */ -#define AX25_DEF_DS_TIMEOUT 180000 /* DAMA timeout 3 minutes */ - -typedef struct ax25_uid_assoc { - struct hlist_node uid_node; - refcount_t refcount; - kuid_t uid; - ax25_address call; -} ax25_uid_assoc; - -#define ax25_uid_for_each(__ax25, list) \ - hlist_for_each_entry(__ax25, list, uid_node) - -#define ax25_uid_hold(ax25) \ - refcount_inc(&((ax25)->refcount)) - -static inline void ax25_uid_put(ax25_uid_assoc *assoc) -{ - if (refcount_dec_and_test(&assoc->refcount)) { - kfree(assoc); - } -} - -typedef struct { - ax25_address calls[AX25_MAX_DIGIS]; - unsigned char repeated[AX25_MAX_DIGIS]; - unsigned char ndigi; - signed char lastrepeat; -} ax25_digi; - -typedef struct ax25_route { - struct ax25_route *next; - ax25_address callsign; - struct net_device *dev; - ax25_digi *digipeat; - char ip_mode; -} ax25_route; - -void __ax25_put_route(ax25_route *ax25_rt); - -extern rwlock_t ax25_route_lock; - -static inline void ax25_route_lock_use(void) -{ - read_lock(&ax25_route_lock); -} - -static inline void ax25_route_lock_unuse(void) -{ - read_unlock(&ax25_route_lock); -} - -typedef struct { - char slave; /* slave_mode? */ - struct timer_list slave_timer; /* timeout timer */ - unsigned short slave_timeout; /* when? */ -} ax25_dama_info; - -struct ctl_table; - -typedef struct ax25_dev { - struct list_head list; - - struct net_device *dev; - netdevice_tracker dev_tracker; - - struct net_device *forward; - struct ctl_table_header *sysheader; - int values[AX25_MAX_VALUES]; -#if defined(CONFIG_AX25_DAMA_SLAVE) || defined(CONFIG_AX25_DAMA_MASTER) - ax25_dama_info dama; -#endif - refcount_t refcount; - bool device_up; - struct rcu_head rcu; -} ax25_dev; - -typedef struct ax25_cb { - struct hlist_node ax25_node; - ax25_address source_addr, dest_addr; - ax25_digi *digipeat; - ax25_dev *ax25_dev; - netdevice_tracker dev_tracker; - unsigned char iamdigi; - unsigned char state, modulus, pidincl; - unsigned short vs, vr, va; - unsigned char condition, backoff; - unsigned char n2, n2count; - struct timer_list t1timer, t2timer, t3timer, idletimer; - unsigned long t1, t2, t3, idle, rtt; - unsigned short paclen, fragno, fraglen; - struct sk_buff_head write_queue; - struct sk_buff_head reseq_queue; - struct sk_buff_head ack_queue; - struct sk_buff_head frag_queue; - unsigned char window; - struct timer_list timer, dtimer; - struct sock *sk; /* Backlink to socket */ - refcount_t refcount; -} ax25_cb; - -struct ax25_sock { - struct sock sk; - struct ax25_cb *cb; -}; - -#define ax25_sk(ptr) container_of_const(ptr, struct ax25_sock, sk) - -static inline struct ax25_cb *sk_to_ax25(const struct sock *sk) -{ - return ax25_sk(sk)->cb; -} - -#define ax25_for_each(__ax25, list) \ - hlist_for_each_entry(__ax25, list, ax25_node) - -#define ax25_cb_hold(__ax25) \ - refcount_inc(&((__ax25)->refcount)) - -static __inline__ void ax25_cb_put(ax25_cb *ax25) -{ - if (refcount_dec_and_test(&ax25->refcount)) { - kfree(ax25->digipeat); - kfree(ax25); - } -} - -static inline void ax25_dev_hold(ax25_dev *ax25_dev) -{ - refcount_inc(&ax25_dev->refcount); -} - -static inline void ax25_dev_put(ax25_dev *ax25_dev) -{ - if (refcount_dec_and_test(&ax25_dev->refcount)) - kfree_rcu(ax25_dev, rcu); -} -static inline __be16 ax25_type_trans(struct sk_buff *skb, struct net_device *dev) -{ - skb->dev = dev; - skb_reset_mac_header(skb); - skb->pkt_type = PACKET_HOST; - return htons(ETH_P_AX25); -} - -/* af_ax25.c */ -extern struct hlist_head ax25_list; -extern spinlock_t ax25_list_lock; -void ax25_cb_add(ax25_cb *); -struct sock *ax25_find_listener(ax25_address *, int, struct net_device *, int); -struct sock *ax25_get_socket(ax25_address *, ax25_address *, int); -ax25_cb *ax25_find_cb(const ax25_address *, ax25_address *, ax25_digi *, - struct net_device *); -void ax25_send_to_raw(ax25_address *, struct sk_buff *, int); -void ax25_destroy_socket(ax25_cb *); -ax25_cb * __must_check ax25_create_cb(void); -void ax25_fillin_cb(ax25_cb *, ax25_dev *); -struct sock *ax25_make_new(struct sock *, struct ax25_dev *); - -/* ax25_addr.c */ -extern const ax25_address ax25_bcast; -extern const ax25_address ax25_defaddr; -extern const ax25_address null_ax25_address; -char *ax2asc(char *buf, const ax25_address *); -void asc2ax(ax25_address *addr, const char *callsign); -int ax25cmp(const ax25_address *, const ax25_address *); -int ax25digicmp(const ax25_digi *, const ax25_digi *); -const unsigned char *ax25_addr_parse(const unsigned char *, int, - ax25_address *, ax25_address *, ax25_digi *, int *, int *); -int ax25_addr_build(unsigned char *, const ax25_address *, - const ax25_address *, const ax25_digi *, int, int); -int ax25_addr_size(const ax25_digi *); -void ax25_digi_invert(const ax25_digi *, ax25_digi *); - -/* ax25_dev.c */ -extern spinlock_t ax25_dev_lock; - -#if IS_ENABLED(CONFIG_AX25) -static inline ax25_dev *ax25_dev_ax25dev(const struct net_device *dev) -{ - return rcu_dereference_rtnl(dev->ax25_ptr); -} -#endif - -ax25_dev *ax25_addr_ax25dev(ax25_address *); -void ax25_dev_device_up(struct net_device *); -void ax25_dev_device_down(struct net_device *); -int ax25_fwd_ioctl(unsigned int, struct ax25_fwd_struct *); -struct net_device *ax25_fwd_dev(struct net_device *); -void ax25_dev_free(void); - -/* ax25_ds_in.c */ -int ax25_ds_frame_in(ax25_cb *, struct sk_buff *, int); - -/* ax25_ds_subr.c */ -void ax25_ds_nr_error_recovery(ax25_cb *); -void ax25_ds_enquiry_response(ax25_cb *); -void ax25_ds_establish_data_link(ax25_cb *); -void ax25_dev_dama_off(ax25_dev *); -void ax25_dama_on(ax25_cb *); -void ax25_dama_off(ax25_cb *); - -/* ax25_ds_timer.c */ -void ax25_ds_setup_timer(ax25_dev *); -void ax25_ds_set_timer(ax25_dev *); -void ax25_ds_del_timer(ax25_dev *); -void ax25_ds_timer(ax25_cb *); -void ax25_ds_t1_timeout(ax25_cb *); -void ax25_ds_heartbeat_expiry(ax25_cb *); -void ax25_ds_t3timer_expiry(ax25_cb *); -void ax25_ds_idletimer_expiry(ax25_cb *); - -/* ax25_iface.c */ - -struct ax25_protocol { - struct ax25_protocol *next; - unsigned int pid; - int (*func)(struct sk_buff *, ax25_cb *); -}; - -void ax25_register_pid(struct ax25_protocol *ap); -void ax25_protocol_release(unsigned int); - -struct ax25_linkfail { - struct hlist_node lf_node; - void (*func)(ax25_cb *, int); -}; - -void ax25_linkfail_register(struct ax25_linkfail *lf); -void ax25_linkfail_release(struct ax25_linkfail *lf); -int __must_check ax25_listen_register(const ax25_address *, - struct net_device *); -void ax25_listen_release(const ax25_address *, struct net_device *); -int(*ax25_protocol_function(unsigned int))(struct sk_buff *, ax25_cb *); -int ax25_listen_mine(const ax25_address *, struct net_device *); -void ax25_link_failed(ax25_cb *, int); -int ax25_protocol_is_registered(unsigned int); - -/* ax25_in.c */ -int ax25_rx_iframe(ax25_cb *, struct sk_buff *); -int ax25_kiss_rcv(struct sk_buff *, struct net_device *, struct packet_type *, - struct net_device *); - -/* ax25_ip.c */ -netdev_tx_t ax25_ip_xmit(struct sk_buff *skb); -extern const struct header_ops ax25_header_ops; - -/* ax25_out.c */ -ax25_cb *ax25_send_frame(struct sk_buff *, int, const ax25_address *, - ax25_address *, ax25_digi *, struct net_device *); -void ax25_output(ax25_cb *, int, struct sk_buff *); -void ax25_kick(ax25_cb *); -void ax25_transmit_buffer(ax25_cb *, struct sk_buff *, int); -void ax25_queue_xmit(struct sk_buff *skb, struct net_device *dev); -int ax25_check_iframes_acked(ax25_cb *, unsigned short); - -/* ax25_route.c */ -void ax25_rt_device_down(struct net_device *); -int ax25_rt_ioctl(unsigned int, void __user *); -extern const struct seq_operations ax25_rt_seqops; -ax25_route *ax25_get_route(ax25_address *addr, struct net_device *dev); -struct sk_buff *ax25_rt_build_path(struct sk_buff *, ax25_address *, - ax25_address *, ax25_digi *); -void ax25_rt_free(void); - -/* ax25_std_in.c */ -int ax25_std_frame_in(ax25_cb *, struct sk_buff *, int); - -/* ax25_std_subr.c */ -void ax25_std_nr_error_recovery(ax25_cb *); -void ax25_std_establish_data_link(ax25_cb *); -void ax25_std_transmit_enquiry(ax25_cb *); -void ax25_std_enquiry_response(ax25_cb *); -void ax25_std_timeout_response(ax25_cb *); - -/* ax25_std_timer.c */ -void ax25_std_heartbeat_expiry(ax25_cb *); -void ax25_std_t1timer_expiry(ax25_cb *); -void ax25_std_t2timer_expiry(ax25_cb *); -void ax25_std_t3timer_expiry(ax25_cb *); -void ax25_std_idletimer_expiry(ax25_cb *); - -/* ax25_subr.c */ -void ax25_clear_queues(ax25_cb *); -void ax25_frames_acked(ax25_cb *, unsigned short); -void ax25_requeue_frames(ax25_cb *); -int ax25_validate_nr(ax25_cb *, unsigned short); -int ax25_decode(ax25_cb *, struct sk_buff *, int *, int *, int *); -void ax25_send_control(ax25_cb *, int, int, int); -void ax25_return_dm(struct net_device *, ax25_address *, ax25_address *, - ax25_digi *); -void ax25_calculate_t1(ax25_cb *); -void ax25_calculate_rtt(ax25_cb *); -void ax25_disconnect(ax25_cb *, int); - -/* ax25_timer.c */ -void ax25_setup_timers(ax25_cb *); -void ax25_start_heartbeat(ax25_cb *); -void ax25_start_t1timer(ax25_cb *); -void ax25_start_t2timer(ax25_cb *); -void ax25_start_t3timer(ax25_cb *); -void ax25_start_idletimer(ax25_cb *); -void ax25_stop_heartbeat(ax25_cb *); -void ax25_stop_t1timer(ax25_cb *); -void ax25_stop_t2timer(ax25_cb *); -void ax25_stop_t3timer(ax25_cb *); -void ax25_stop_idletimer(ax25_cb *); -int ax25_t1timer_running(ax25_cb *); -unsigned long ax25_display_timer(struct timer_list *); - -/* ax25_uid.c */ -extern int ax25_uid_policy; -ax25_uid_assoc *ax25_findbyuid(kuid_t); -int __must_check ax25_uid_ioctl(int, struct sockaddr_ax25 *); -extern const struct seq_operations ax25_uid_seqops; -void ax25_uid_free(void); - -/* sysctl_net_ax25.c */ -#ifdef CONFIG_SYSCTL -int ax25_register_dev_sysctl(ax25_dev *ax25_dev); -void ax25_unregister_dev_sysctl(ax25_dev *ax25_dev); -#else -static inline int ax25_register_dev_sysctl(ax25_dev *ax25_dev) { return 0; } -static inline void ax25_unregister_dev_sysctl(ax25_dev *ax25_dev) {} -#endif /* CONFIG_SYSCTL */ +#define AX25_ADDR_LEN 7 +#define AX25_P_IP 0xCC #endif diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h index d46ed9011ee5..b624da5026f5 100644 --- a/include/net/bluetooth/bluetooth.h +++ b/include/net/bluetooth/bluetooth.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* BlueZ - Bluetooth protocol stack for Linux Copyright (C) 2000-2001 Qualcomm Incorporated @@ -5,10 +6,6 @@ Written 2000,2001 by Maxim Krasnyansky <maxk@qualcomm.com> - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 as - published by the Free Software Foundation; - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. @@ -130,21 +127,30 @@ struct bt_voice { #define BT_RCVMTU 13 #define BT_PHY 14 -#define BT_PHY_BR_1M_1SLOT 0x00000001 -#define BT_PHY_BR_1M_3SLOT 0x00000002 -#define BT_PHY_BR_1M_5SLOT 0x00000004 -#define BT_PHY_EDR_2M_1SLOT 0x00000008 -#define BT_PHY_EDR_2M_3SLOT 0x00000010 -#define BT_PHY_EDR_2M_5SLOT 0x00000020 -#define BT_PHY_EDR_3M_1SLOT 0x00000040 -#define BT_PHY_EDR_3M_3SLOT 0x00000080 -#define BT_PHY_EDR_3M_5SLOT 0x00000100 -#define BT_PHY_LE_1M_TX 0x00000200 -#define BT_PHY_LE_1M_RX 0x00000400 -#define BT_PHY_LE_2M_TX 0x00000800 -#define BT_PHY_LE_2M_RX 0x00001000 -#define BT_PHY_LE_CODED_TX 0x00002000 -#define BT_PHY_LE_CODED_RX 0x00004000 +#define BT_PHY_BR_1M_1SLOT BIT(0) +#define BT_PHY_BR_1M_3SLOT BIT(1) +#define BT_PHY_BR_1M_5SLOT BIT(2) +#define BT_PHY_EDR_2M_1SLOT BIT(3) +#define BT_PHY_EDR_2M_3SLOT BIT(4) +#define BT_PHY_EDR_2M_5SLOT BIT(5) +#define BT_PHY_EDR_3M_1SLOT BIT(6) +#define BT_PHY_EDR_3M_3SLOT BIT(7) +#define BT_PHY_EDR_3M_5SLOT BIT(8) +#define BT_PHY_LE_1M_TX BIT(9) +#define BT_PHY_LE_1M_RX BIT(10) +#define BT_PHY_LE_2M_TX BIT(11) +#define BT_PHY_LE_2M_RX BIT(12) +#define BT_PHY_LE_CODED_TX BIT(13) +#define BT_PHY_LE_CODED_RX BIT(14) + +#define BT_PHY_BREDR_MASK (BT_PHY_BR_1M_1SLOT | BT_PHY_BR_1M_3SLOT | \ + BT_PHY_BR_1M_5SLOT | BT_PHY_EDR_2M_1SLOT | \ + BT_PHY_EDR_2M_3SLOT | BT_PHY_EDR_2M_5SLOT | \ + BT_PHY_EDR_3M_1SLOT | BT_PHY_EDR_3M_3SLOT | \ + BT_PHY_EDR_3M_5SLOT) +#define BT_PHY_LE_MASK (BT_PHY_LE_1M_TX | BT_PHY_LE_1M_RX | \ + BT_PHY_LE_2M_TX | BT_PHY_LE_2M_RX | \ + BT_PHY_LE_CODED_TX | BT_PHY_LE_CODED_RX) #define BT_MODE 15 @@ -173,7 +179,7 @@ struct bt_iso_io_qos { __u32 interval; __u16 latency; __u16 sdu; - __u8 phy; + __u8 phys; __u8 rtn; }; @@ -212,9 +218,9 @@ struct bt_iso_qos { }; }; -#define BT_ISO_PHY_1M 0x01 -#define BT_ISO_PHY_2M 0x02 -#define BT_ISO_PHY_CODED 0x04 +#define BT_ISO_PHY_1M BIT(0) +#define BT_ISO_PHY_2M BIT(1) +#define BT_ISO_PHY_CODED BIT(2) #define BT_ISO_PHY_ANY (BT_ISO_PHY_1M | BT_ISO_PHY_2M | \ BT_ISO_PHY_CODED) @@ -389,6 +395,7 @@ void baswap(bdaddr_t *dst, const bdaddr_t *src); struct bt_sock { struct sock sk; struct list_head accept_q; + spinlock_t accept_q_lock; /* protects accept_q */ struct sock *parent; unsigned long flags; void (*skb_msg_name)(struct sk_buff *, void *, int *); diff --git a/include/net/bluetooth/coredump.h b/include/net/bluetooth/coredump.h index 72f51b587a04..acc1849f66c0 100644 --- a/include/net/bluetooth/coredump.h +++ b/include/net/bluetooth/coredump.h @@ -8,6 +8,13 @@ #define DEVCOREDUMP_TIMEOUT msecs_to_jiffies(10000) /* 10 sec */ +/* + * Max header size, shared by both the devcoredump core and + * the dmp_hdr() registered by driver via hci_devcd_register() + */ +#define HCI_DEVCD_HDR_SIZE_MAX 512 +#define HCI_DEVCD_HDR_END_MARKER "--- Start dump ---\n" + typedef void (*coredump_t)(struct hci_dev *hdev); typedef void (*dmp_hdr_t)(struct hci_dev *hdev, struct sk_buff *skb); typedef void (*notify_change_t)(struct hci_dev *hdev, int state); @@ -60,7 +67,10 @@ struct hci_devcoredump { #ifdef CONFIG_DEV_COREDUMP +const char *hci_devcd_state_name(enum devcoredump_state state); + void hci_devcd_reset(struct hci_dev *hdev); +void hci_devcd_shutdown(struct hci_dev *hdev); void hci_devcd_rx(struct work_struct *work); void hci_devcd_timeout(struct work_struct *work); @@ -74,7 +84,13 @@ int hci_devcd_abort(struct hci_dev *hdev); #else +static inline const char *hci_devcd_state_name(enum devcoredump_state state) +{ + return ""; +} + static inline void hci_devcd_reset(struct hci_dev *hdev) {} +static inline void hci_devcd_shutdown(struct hci_dev *hdev) {} static inline void hci_devcd_rx(struct work_struct *work) {} static inline void hci_devcd_timeout(struct work_struct *work) {} diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index a27cd3626b87..1641d879dbda 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* BlueZ - Bluetooth protocol stack for Linux Copyright (C) 2000-2001 Qualcomm Incorporated @@ -5,10 +6,6 @@ Written 2000,2001 by Maxim Krasnyansky <maxk@qualcomm.com> - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 as - published by the Free Software Foundation; - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. @@ -654,6 +651,10 @@ enum { #define HCI_LE_ISO_BROADCASTER 0x40 #define HCI_LE_ISO_SYNC_RECEIVER 0x80 #define HCI_LE_LL_EXT_FEATURE 0x80 +#define HCI_LE_CS 0x40 +#define HCI_LE_CS_HOST 0x80 +#define HCI_LE_SCI 0x01 /* byte 9 - Shorter Connection Intervals */ +#define HCI_LE_SCI_HOST 0x02 /* byte 9 - Shorter Connection Intervals (Host) */ /* Connection modes */ #define HCI_CM_ACTIVE 0x0000 @@ -1466,8 +1467,12 @@ struct hci_rp_read_data_block_size { } __packed; #define HCI_OP_READ_LOCAL_CODECS 0x100b -struct hci_std_codecs { +struct hci_std_codecs_hdr { __u8 num; +} __packed; + +struct hci_std_codecs { + struct hci_std_codecs_hdr; __u8 codec[]; } __packed; @@ -1485,7 +1490,7 @@ struct hci_vnd_codecs { struct hci_rp_read_local_supported_codecs { __u8 status; - struct hci_std_codecs std_codecs; + struct hci_std_codecs_hdr std_codecs; struct hci_vnd_codecs vnd_codecs; } __packed; @@ -1502,8 +1507,12 @@ struct hci_std_codec_v2 { __u8 transport; } __packed; -struct hci_std_codecs_v2 { +struct hci_std_codecs_v2_hdr { __u8 num; +} __packed; + +struct hci_std_codecs_v2 { + struct hci_std_codecs_v2_hdr; struct hci_std_codec_v2 codec[]; } __packed; @@ -1520,7 +1529,7 @@ struct hci_vnd_codecs_v2 { struct hci_rp_read_local_supported_codecs_v2 { __u8 status; - struct hci_std_codecs_v2 std_codecs; + struct hci_std_codecs_v2_hdr std_codecs; struct hci_vnd_codecs_v2 vendor_codecs; } __packed; @@ -1883,6 +1892,15 @@ struct hci_cp_le_set_default_phy { #define HCI_LE_SET_PHY_2M 0x02 #define HCI_LE_SET_PHY_CODED 0x04 +#define HCI_OP_LE_SET_PHY 0x2032 +struct hci_cp_le_set_phy { + __le16 handle; + __u8 all_phys; + __u8 tx_phys; + __u8 rx_phys; + __le16 phy_opts; +} __packed; + #define HCI_OP_LE_SET_EXT_SCAN_PARAMS 0x2041 struct hci_cp_le_set_ext_scan_params { __u8 own_addr_type; @@ -2136,8 +2154,8 @@ struct hci_cis_params { __u8 cis_id; __le16 c_sdu; __le16 p_sdu; - __u8 c_phy; - __u8 p_phy; + __u8 c_phys; + __u8 p_phys; __u8 c_rtn; __u8 p_rtn; } __packed; @@ -2269,6 +2287,250 @@ struct hci_cp_le_read_all_remote_features { __u8 pages; } __packed; +/* Channel Sounding Commands */ +#define HCI_OP_LE_CS_RD_LOCAL_SUPP_CAP 0x2089 +struct hci_rp_le_cs_rd_local_supp_cap { + __u8 status; + __u8 num_config_supported; + __le16 max_consecutive_procedures_supported; + __u8 num_antennas_supported; + __u8 max_antenna_paths_supported; + __u8 roles_supported; + __u8 modes_supported; + __u8 rtt_capability; + __u8 rtt_aa_only_n; + __u8 rtt_sounding_n; + __u8 rtt_random_payload_n; + __le16 nadm_sounding_capability; + __le16 nadm_random_capability; + __u8 cs_sync_phys_supported; + __le16 subfeatures_supported; + __le16 t_ip1_times_supported; + __le16 t_ip2_times_supported; + __le16 t_fcs_times_supported; + __le16 t_pm_times_supported; + __u8 t_sw_time_supported; + __u8 tx_snr_capability; +} __packed; + +#define HCI_OP_LE_CS_RD_RMT_SUPP_CAP 0x208A +struct hci_cp_le_cs_rd_local_supp_cap { + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_WR_CACHED_RMT_SUPP_CAP 0x208B +struct hci_cp_le_cs_wr_cached_rmt_supp_cap { + __le16 handle; + __u8 num_config_supported; + __le16 max_consecutive_procedures_supported; + __u8 num_antennas_supported; + __u8 max_antenna_paths_supported; + __u8 roles_supported; + __u8 modes_supported; + __u8 rtt_capability; + __u8 rtt_aa_only_n; + __u8 rtt_sounding_n; + __u8 rtt_random_payload_n; + __le16 nadm_sounding_capability; + __le16 nadm_random_capability; + __u8 cs_sync_phys_supported; + __le16 subfeatures_supported; + __le16 t_ip1_times_supported; + __le16 t_ip2_times_supported; + __le16 t_fcs_times_supported; + __le16 t_pm_times_supported; + __u8 t_sw_time_supported; + __u8 tx_snr_capability; +} __packed; + +struct hci_rp_le_cs_wr_cached_rmt_supp_cap { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_SEC_ENABLE 0x208C +struct hci_cp_le_cs_sec_enable { + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_SET_DEFAULT_SETTINGS 0x208D +struct hci_cp_le_cs_set_default_settings { + __le16 handle; + __u8 role_enable; + __u8 cs_sync_ant_sel; + __s8 max_tx_power; +} __packed; + +struct hci_rp_le_cs_set_default_settings { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_RD_RMT_FAE_TABLE 0x208E +struct hci_cp_le_cs_rd_rmt_fae_table { + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_WR_CACHED_RMT_FAE_TABLE 0x208F +struct hci_cp_le_cs_wr_rmt_cached_fae_table { + __le16 handle; + __u8 remote_fae_table[72]; +} __packed; + +struct hci_rp_le_cs_wr_rmt_cached_fae_table { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_CREATE_CONFIG 0x2090 +struct hci_cp_le_cs_create_config { + __le16 handle; + __u8 config_id; + __u8 create_context; + __u8 main_mode_type; + __u8 sub_mode_type; + __u8 min_main_mode_steps; + __u8 max_main_mode_steps; + __u8 main_mode_repetition; + __u8 mode_0_steps; + __u8 role; + __u8 rtt_type; + __u8 cs_sync_phy; + __u8 channel_map[10]; + __u8 channel_map_repetition; + __u8 channel_selection_type; + __u8 ch3c_shape; + __u8 ch3c_jump; + __u8 reserved; +} __packed; + +#define HCI_OP_LE_CS_REMOVE_CONFIG 0x2091 +struct hci_cp_le_cs_remove_config { + __le16 handle; + __u8 config_id; +} __packed; + +#define HCI_OP_LE_CS_SET_CH_CLASSIFICATION 0x2092 +struct hci_cp_le_cs_set_ch_classification { + __u8 ch_classification[10]; +} __packed; + +struct hci_rp_le_cs_set_ch_classification { + __u8 status; +} __packed; + +#define HCI_OP_LE_CS_SET_PROC_PARAM 0x2093 +struct hci_cp_le_cs_set_proc_param { + __le16 handle; + __u8 config_id; + __le16 max_procedure_len; + __le16 min_procedure_interval; + __le16 max_procedure_interval; + __le16 max_procedure_count; + __u8 min_subevent_len[3]; + __u8 max_subevent_len[3]; + __u8 tone_antenna_config_selection; + __u8 phy; + __u8 tx_power_delta; + __u8 preferred_peer_antenna; + __u8 snr_control_initiator; + __u8 snr_control_reflector; +} __packed; + +struct hci_rp_le_cs_set_proc_param { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_CS_SET_PROC_ENABLE 0x2094 +struct hci_cp_le_cs_set_proc_enable { + __le16 handle; + __u8 config_id; + __u8 enable; +} __packed; + +#define HCI_OP_LE_CS_TEST 0x2095 +struct hci_cp_le_cs_test { + __u8 main_mode_type; + __u8 sub_mode_type; + __u8 main_mode_repetition; + __u8 mode_0_steps; + __u8 role; + __u8 rtt_type; + __u8 cs_sync_phy; + __u8 cs_sync_antenna_selection; + __u8 subevent_len[3]; + __le16 subevent_interval; + __u8 max_num_subevents; + __u8 transmit_power_level; + __u8 t_ip1_time; + __u8 t_ip2_time; + __u8 t_fcs_time; + __u8 t_pm_time; + __u8 t_sw_time; + __u8 tone_antenna_config_selection; + __u8 reserved; + __u8 snr_control_initiator; + __u8 snr_control_reflector; + __le16 drbg_nonce; + __u8 channel_map_repetition; + __le16 override_config; + __u8 override_parameters_length; + __u8 override_parameters_data[]; +} __packed; + +struct hci_rp_le_cs_test { + __u8 status; +} __packed; + +#define HCI_OP_LE_CS_TEST_END 0x2096 + +#define HCI_OP_LE_SET_HOST_FEATURE_V2 0x2097 +struct hci_cp_le_set_host_feature_v2 { + __le16 bit_number; + __u8 bit_value; +} __packed; + +#define HCI_OP_LE_CONN_RATE 0x20a1 +struct hci_cp_le_conn_rate { + __le16 handle; + __le16 interval_min; + __le16 interval_max; + __le16 subrate_min; + __le16 subrate_max; + __le16 max_latency; + __le16 cont_num; + __le16 supv_timeout; + __le16 min_ce_len; + __le16 max_ce_len; +} __packed; + +#define HCI_OP_LE_SET_DEF_RATE 0x20a2 +struct hci_cp_le_set_def_rate { + __le16 interval_min; + __le16 interval_max; + __le16 subrate_min; + __le16 subrate_max; + __le16 max_latency; + __le16 cont_num; + __le16 supv_timeout; + __le16 min_ce_len; + __le16 max_ce_len; +} __packed; + +#define HCI_OP_LE_READ_CONN_INTERVAL 0x20a3 +struct hci_le_conn_interval_group { + __le16 min; + __le16 max; + __le16 stride; +} __packed; + +struct hci_rp_le_read_conn_interval { + __u8 status; + __u8 num_grps; + struct hci_le_conn_interval_group grps[]; +} __packed; + /* ---- HCI Events ---- */ struct hci_ev_status { __u8 status; @@ -2960,6 +3222,140 @@ struct hci_evt_le_read_all_remote_features_complete { __u8 features[248]; } __packed; +/* Channel Sounding Events */ +#define HCI_EVT_LE_CS_READ_RMT_SUPP_CAP_COMPLETE 0x2C +struct hci_evt_le_cs_read_rmt_supp_cap_complete { + __u8 status; + __le16 handle; + __u8 num_configs_supp; + __le16 max_consec_proc_supp; + __u8 num_ant_supp; + __u8 max_ant_path_supp; + __u8 roles_supp; + __u8 modes_supp; + __u8 rtt_cap; + __u8 rtt_aa_only_n; + __u8 rtt_sounding_n; + __u8 rtt_rand_payload_n; + __le16 nadm_sounding_cap; + __le16 nadm_rand_cap; + __u8 cs_sync_phys_supp; + __le16 sub_feat_supp; + __le16 t_ip1_times_supp; + __le16 t_ip2_times_supp; + __le16 t_fcs_times_supp; + __le16 t_pm_times_supp; + __u8 t_sw_times_supp; + __u8 tx_snr_cap; +} __packed; + +#define HCI_EVT_LE_CS_READ_RMT_FAE_TABLE_COMPLETE 0x2D +struct hci_evt_le_cs_read_rmt_fae_table_complete { + __u8 status; + __le16 handle; + __u8 remote_fae_table[72]; +} __packed; + +#define HCI_EVT_LE_CS_SECURITY_ENABLE_COMPLETE 0x2E +struct hci_evt_le_cs_security_enable_complete { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_EVT_LE_CS_CONFIG_COMPLETE 0x2F +struct hci_evt_le_cs_config_complete { + __u8 status; + __le16 handle; + __u8 config_id; + __u8 action; + __u8 main_mode_type; + __u8 sub_mode_type; + __u8 min_main_mode_steps; + __u8 max_main_mode_steps; + __u8 main_mode_rep; + __u8 mode_0_steps; + __u8 role; + __u8 rtt_type; + __u8 cs_sync_phy; + __u8 channel_map[10]; + __u8 channel_map_rep; + __u8 channel_sel_type; + __u8 ch3c_shape; + __u8 ch3c_jump; + __u8 reserved; + __u8 t_ip1_time; + __u8 t_ip2_time; + __u8 t_fcs_time; + __u8 t_pm_time; +} __packed; + +#define HCI_EVT_LE_CS_PROCEDURE_ENABLE_COMPLETE 0x30 +struct hci_evt_le_cs_procedure_enable_complete { + __u8 status; + __le16 handle; + __u8 config_id; + __u8 state; + __u8 tone_ant_config_sel; + __s8 sel_tx_pwr; + __u8 sub_evt_len[3]; + __u8 sub_evts_per_evt; + __le16 sub_evt_intrvl; + __le16 evt_intrvl; + __le16 proc_intrvl; + __le16 proc_counter; + __le16 max_proc_len; +} __packed; + +#define HCI_EVT_LE_CS_SUBEVENT_RESULT 0x31 +struct hci_evt_le_cs_subevent_result { + __le16 handle; + __u8 config_id; + __le16 start_acl_conn_evt_counter; + __le16 proc_counter; + __le16 freq_comp; + __u8 ref_pwr_lvl; + __u8 proc_done_status; + __u8 subevt_done_status; + __u8 abort_reason; + __u8 num_ant_paths; + __u8 num_steps_reported; + __u8 step_mode[0]; /* depends on num_steps_reported */ + __u8 step_channel[0]; /* depends on num_steps_reported */ + __u8 step_data_length[0]; /* depends on num_steps_reported */ + __u8 step_data[0]; /* depends on num_steps_reported */ +} __packed; + +#define HCI_EVT_LE_CS_SUBEVENT_RESULT_CONTINUE 0x32 +struct hci_evt_le_cs_subevent_result_continue { + __le16 handle; + __u8 config_id; + __u8 proc_done_status; + __u8 subevt_done_status; + __u8 abort_reason; + __u8 num_ant_paths; + __u8 num_steps_reported; + __u8 step_mode[0]; /* depends on num_steps_reported */ + __u8 step_channel[0]; /* depends on num_steps_reported */ + __u8 step_data_length[0]; /* depends on num_steps_reported */ + __u8 step_data[0]; /* depends on num_steps_reported */ +} __packed; + +#define HCI_EVT_LE_CS_TEST_END_COMPLETE 0x33 +struct hci_evt_le_cs_test_end_complete { + __u8 status; +} __packed; + +#define HCI_EVT_LE_CONN_RATE_CHANGE 0x37 +struct hci_evt_le_conn_rate_change { + __u8 status; + __le16 handle; + __le16 interval; + __le16 subrate; + __le16 latency; + __le16 cont_number; + __le16 supv_timeout; +} __packed; + #define HCI_EV_VENDOR 0xff /* Internal events generated by Bluetooth stack */ @@ -2986,6 +3382,7 @@ struct hci_ev_si_security { /* ---- HCI Packet structures ---- */ #define HCI_COMMAND_HDR_SIZE 3 #define HCI_EVENT_HDR_SIZE 2 +#define HCI_MAX_EVENT_PLEN 255 #define HCI_ACL_HDR_SIZE 4 #define HCI_SCO_HDR_SIZE 3 #define HCI_ISO_HDR_SIZE 4 @@ -3064,14 +3461,25 @@ static inline struct hci_iso_hdr *hci_iso_hdr(const struct sk_buff *skb) #define hci_handle(h) (h & 0x0fff) #define hci_flags(h) (h >> 12) +static inline __u16 hci_acl_handle(const struct sk_buff *skb) +{ + return hci_handle(__le16_to_cpu(hci_acl_hdr(skb)->handle)); +} + +static inline __u16 hci_acl_dlen(const struct sk_buff *skb) +{ + return __le16_to_cpu(hci_acl_hdr(skb)->dlen); +} + /* ISO handle and flags pack/unpack */ #define hci_iso_flags_pb(f) (f & 0x0003) #define hci_iso_flags_ts(f) ((f >> 2) & 0x0001) #define hci_iso_flags_pack(pb, ts) ((pb & 0x03) | ((ts & 0x01) << 2)) /* ISO data length and flags pack/unpack */ -#define hci_iso_data_len_pack(h, f) ((__u16) ((h) | ((f) << 14))) -#define hci_iso_data_len(h) ((h) & 0x3fff) +#define hci_iso_data_len_pack(h, f) ((__u16) (((h) & 0x0fff) | \ + (((f) & 0x3) << 14))) +#define hci_iso_data_len(h) ((h) & 0x0fff) #define hci_iso_data_flags(h) ((h) >> 14) /* codec transport types */ diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 4263e71a23ef..4105c446ca98 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* BlueZ - Bluetooth protocol stack for Linux Copyright (c) 2000-2001, 2010, Code Aurora Forum. All rights reserved. @@ -5,10 +6,6 @@ Written 2000,2001 by Maxim Krasnyansky <maxk@qualcomm.com> - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 as - published by the Free Software Foundation; - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. @@ -419,6 +416,7 @@ struct hci_dev { __u16 le_conn_max_interval; __u16 le_conn_latency; __u16 le_supv_timeout; + __u16 le_min_rate_interval; __u16 le_def_tx_len; __u16 le_def_tx_time; __u16 le_max_tx_len; @@ -648,6 +646,8 @@ struct hci_dev { int (*setup)(struct hci_dev *hdev); int (*shutdown)(struct hci_dev *hdev); int (*send)(struct hci_dev *hdev, struct sk_buff *skb); + /* Handle HCI_EV_VENDOR; return true if handled, false otherwise */ + bool (*handle_ev_vendor)(struct hci_dev *hdev, struct sk_buff *skb); void (*notify)(struct hci_dev *hdev, unsigned int evt); void (*hw_error)(struct hci_dev *hdev, u8 code); int (*post_init)(struct hci_dev *hdev); @@ -723,6 +723,11 @@ struct hci_conn { __u16 le_conn_interval; __u16 le_conn_latency; __u16 le_supv_timeout; + __u16 le_rate_interval; + __u16 le_subrate; + __u16 le_rate_latency; + __u16 le_cont_num; + __u16 le_rate_supv_timeout; __u8 le_adv_data[HCI_MAX_EXT_AD_LENGTH]; __u8 le_adv_data_len; __u8 le_per_adv_data[HCI_MAX_PER_AD_TOT_LEN]; @@ -730,6 +735,8 @@ struct hci_conn { __u16 le_per_adv_data_offset; __u8 le_adv_phy; __u8 le_adv_sec_phy; + __u8 le_tx_def_phys; + __u8 le_rx_def_phys; __u8 le_tx_phy; __u8 le_rx_phy; __s8 rssi; @@ -768,9 +775,11 @@ struct hci_conn { struct dentry *debugfs; struct hci_dev *hdev; - void *l2cap_data; + + spinlock_t proto_lock; /* lock guarding protocol data */ + void *l2cap_data __guarded_by(&proto_lock, &hdev->lock); void *sco_data; - void *iso_data; + void *iso_data __guarded_by(&proto_lock); struct list_head link_list; struct hci_conn *parent; @@ -811,6 +820,14 @@ struct hci_conn_params { u16 conn_latency; u16 supervision_timeout; + u16 rate_min_interval; + u16 rate_max_interval; + u16 subrate_min; + u16 subrate_max; + u16 max_latency; + u16 cont_num; + u16 rate_supv_timeout; + enum { HCI_AUTO_CONN_DISABLED, HCI_AUTO_CONN_REPORT, @@ -918,9 +935,9 @@ static inline void hci_discovery_filter_clear(struct hci_dev *hdev) hdev->discovery.result_filtering = false; hdev->discovery.report_invalid_rssi = true; hdev->discovery.rssi = HCI_RSSI_INVALID; - hdev->discovery.uuid_count = 0; spin_lock(&hdev->discovery.lock); + hdev->discovery.uuid_count = 0; kfree(hdev->discovery.uuids); hdev->discovery.uuids = NULL; spin_unlock(&hdev->discovery.lock); @@ -986,6 +1003,7 @@ enum { HCI_CONN_AUTH_FAILURE, HCI_CONN_PER_ADV, HCI_CONN_BIG_CREATED, + HCI_CONN_CREATE, HCI_CONN_CREATE_CIS, HCI_CONN_CREATE_BIG_SYNC, HCI_CONN_BIG_SYNC, @@ -1769,7 +1787,13 @@ int hci_register_suspend_notifier(struct hci_dev *hdev); int hci_unregister_suspend_notifier(struct hci_dev *hdev); int hci_suspend_dev(struct hci_dev *hdev); int hci_resume_dev(struct hci_dev *hdev); -int hci_reset_dev(struct hci_dev *hdev); +int __hci_reset_dev(struct hci_dev *hdev, u8 hw_err_code); + +static inline int hci_reset_dev(struct hci_dev *hdev) +{ + return __hci_reset_dev(hdev, 0); +} + int hci_recv_frame(struct hci_dev *hdev, struct sk_buff *skb); int hci_recv_diag(struct hci_dev *hdev, struct sk_buff *skb); __printf(2, 3) void hci_set_hw_info(struct hci_dev *hdev, const char *fmt, ...); @@ -2071,6 +2095,17 @@ void hci_conn_del_sysfs(struct hci_conn *conn); #define ll_ext_feature_capable(dev) \ ((dev)->le_features[7] & HCI_LE_LL_EXT_FEATURE) +/* Channel sounding support */ +#define le_cs_capable(dev) \ + ((dev)->le_features[5] & HCI_LE_CS) +#define le_cs_host_capable(dev) \ + ((dev)->le_features[5] & HCI_LE_CS_HOST) + +#define le_sci_capable(dev) \ + ((dev)->le_features[9] & HCI_LE_SCI) +#define le_sci_enabled(dev) \ + (le_enabled(dev) && le_sci_capable(dev)) + #define mws_transport_config_capable(dev) (((dev)->commands[30] & 0x08) && \ (!hci_test_quirk((dev), HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG))) @@ -2334,6 +2369,7 @@ void *hci_sent_cmd_data(struct hci_dev *hdev, __u16 opcode); void *hci_recv_event_data(struct hci_dev *hdev, __u8 event); u32 hci_conn_get_phy(struct hci_conn *conn); +int hci_conn_set_phy(struct hci_conn *conn, u32 phys); /* ----- HCI Sockets ----- */ void hci_send_to_sock(struct hci_dev *hdev, struct sk_buff *skb); @@ -2423,6 +2459,7 @@ void mgmt_new_link_key(struct hci_dev *hdev, struct link_key *key, bool persistent); void mgmt_device_connected(struct hci_dev *hdev, struct hci_conn *conn, u8 *name, u8 name_len); +u8 hci_to_mgmt_reason(u8 err); void mgmt_device_disconnected(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type, u8 addr_type, u8 reason, bool mgmt_connected); @@ -2484,9 +2521,11 @@ void mgmt_advertising_removed(struct sock *sk, struct hci_dev *hdev, int mgmt_phy_configuration_changed(struct hci_dev *hdev, struct sock *skip); void mgmt_adv_monitor_device_lost(struct hci_dev *hdev, u16 handle, bdaddr_t *bdaddr, u8 addr_type); +void mgmt_conn_subrate_notify(struct hci_dev *hdev, struct hci_conn *conn, + u8 status); int hci_abort_conn(struct hci_conn *conn, u8 reason); -u8 hci_le_conn_update(struct hci_conn *conn, u16 min, u16 max, u16 latency, +void hci_le_conn_update(struct hci_conn *conn, u16 min, u16 max, u16 latency, u16 to_multiplier); void hci_le_start_enc(struct hci_conn *conn, __le16 ediv, __le64 rand, __u8 ltk[16], __u8 key_size); diff --git a/include/net/bluetooth/hci_mon.h b/include/net/bluetooth/hci_mon.h index bbd752494ef9..4b2a0af4ed58 100644 --- a/include/net/bluetooth/hci_mon.h +++ b/include/net/bluetooth/hci_mon.h @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* BlueZ - Bluetooth protocol stack for Linux Copyright (C) 2011-2012 Intel Corporation - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 as - published by the Free Software Foundation; - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. diff --git a/include/net/bluetooth/hci_sock.h b/include/net/bluetooth/hci_sock.h index 13e8cd4414a1..16f150b861be 100644 --- a/include/net/bluetooth/hci_sock.h +++ b/include/net/bluetooth/hci_sock.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* BlueZ - Bluetooth protocol stack for Linux Copyright (C) 2000-2001 Qualcomm Incorporated Written 2000,2001 by Maxim Krasnyansky <maxk@qualcomm.com> - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 as - published by the Free Software Foundation; - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h index 56076bbc981d..a6579a868678 100644 --- a/include/net/bluetooth/hci_sync.h +++ b/include/net/bluetooth/hci_sync.h @@ -59,6 +59,7 @@ int __hci_cmd_sync_status(struct hci_dev *hdev, u16 opcode, u32 plen, int __hci_cmd_sync_status_sk(struct hci_dev *hdev, u16 opcode, u32 plen, const void *param, u8 event, u32 timeout, struct sock *sk); +int __hci_reset_sync(struct hci_dev *hdev); int hci_cmd_sync_status(struct hci_dev *hdev, u16 opcode, u32 plen, const void *param, u32 timeout); @@ -84,9 +85,6 @@ void hci_cmd_sync_cancel_entry(struct hci_dev *hdev, struct hci_cmd_sync_work_entry *entry); bool hci_cmd_sync_dequeue(struct hci_dev *hdev, hci_cmd_sync_work_func_t func, void *data, hci_cmd_sync_work_destroy_t destroy); -bool hci_cmd_sync_dequeue_once(struct hci_dev *hdev, - hci_cmd_sync_work_func_t func, void *data, - hci_cmd_sync_work_destroy_t destroy); int hci_update_eir_sync(struct hci_dev *hdev); int hci_update_class_sync(struct hci_dev *hdev); @@ -185,9 +183,13 @@ int hci_connect_le_sync(struct hci_dev *hdev, struct hci_conn *conn); int hci_cancel_connect_sync(struct hci_dev *hdev, struct hci_conn *conn); int hci_le_conn_update_sync(struct hci_dev *hdev, struct hci_conn *conn, struct hci_conn_params *params); +int hci_le_conn_rate_request(struct hci_dev *hdev, struct hci_conn *conn); int hci_connect_pa_sync(struct hci_dev *hdev, struct hci_conn *conn); int hci_connect_big_sync(struct hci_dev *hdev, struct hci_conn *conn); int hci_past_sync(struct hci_conn *conn, struct hci_conn *le); int hci_le_read_remote_features(struct hci_conn *conn); + +int hci_acl_change_pkt_type(struct hci_conn *conn, u16 pkt_type); +int hci_le_set_phy(struct hci_conn *conn, u8 tx_phys, u8 rx_phys); diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 00e182a22720..3d9a32094347 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* BlueZ - Bluetooth protocol stack for Linux Copyright (C) 2000-2001 Qualcomm Incorporated @@ -6,10 +7,6 @@ Written 2000,2001 by Maxim Krasnyansky <maxk@qualcomm.com> - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 as - published by the Free Software Foundation; - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. @@ -33,6 +30,7 @@ /* L2CAP defaults */ #define L2CAP_DEFAULT_MTU 672 #define L2CAP_DEFAULT_MIN_MTU 48 +#define L2CAP_SIG_MTU 48 /* BR/EDR signaling MTU */ #define L2CAP_DEFAULT_FLUSH_TO 0xFFFF #define L2CAP_EFS_DEFAULT_FLUSH_TO 0xFFFFFFFF #define L2CAP_DEFAULT_TX_WINDOW 63 @@ -284,9 +282,9 @@ struct l2cap_conn_rsp { #define L2CAP_CR_LE_BAD_KEY_SIZE 0x0007 #define L2CAP_CR_LE_ENCRYPTION 0x0008 #define L2CAP_CR_LE_INVALID_SCID 0x0009 -#define L2CAP_CR_LE_SCID_IN_USE 0X000A -#define L2CAP_CR_LE_UNACCEPT_PARAMS 0X000B -#define L2CAP_CR_LE_INVALID_PARAMS 0X000C +#define L2CAP_CR_LE_SCID_IN_USE 0x000A +#define L2CAP_CR_LE_UNACCEPT_PARAMS 0x000B +#define L2CAP_CR_LE_INVALID_PARAMS 0x000C /* connect/create channel status */ #define L2CAP_CS_NO_INFO 0x0000 @@ -493,6 +491,8 @@ struct l2cap_ecred_reconf_req { #define L2CAP_RECONF_SUCCESS 0x0000 #define L2CAP_RECONF_INVALID_MTU 0x0001 #define L2CAP_RECONF_INVALID_MPS 0x0002 +#define L2CAP_RECONF_INVALID_CID 0x0003 +#define L2CAP_RECONF_INVALID_PARAMS 0x0004 struct l2cap_ecred_reconf_rsp { __le16 result; @@ -617,7 +617,8 @@ struct l2cap_chan { struct l2cap_ops { char *name; - struct l2cap_chan *(*new_connection) (struct l2cap_chan *chan); + int (*new_connection)(struct l2cap_chan *chan, + struct l2cap_chan *new_chan); int (*recv) (struct l2cap_chan * chan, struct sk_buff *skb); void (*teardown) (struct l2cap_chan *chan, int err); @@ -655,8 +656,8 @@ struct l2cap_conn { struct sk_buff *rx_skb; __u32 rx_len; + struct ida tx_ida; __u8 tx_ident; - struct mutex ident_lock; struct sk_buff_head pending_rx; struct work_struct pending_rx_work; @@ -698,7 +699,12 @@ struct l2cap_rx_busy { struct l2cap_pinfo { struct bt_sock bt; + + /* With owning sk_socket chan may be read without lock, other access + * should hold lock_sock. + */ struct l2cap_chan *chan; + struct list_head rx_busy; }; @@ -745,6 +751,7 @@ enum { FLAG_ECRED_CONN_REQ_SENT, FLAG_PENDING_SECURITY, FLAG_HOLD_HCI_CONN, + FLAG_DEL, }; /* Lock nesting levels for L2CAP channels. We need these because lockdep @@ -881,9 +888,10 @@ static inline __u16 __next_seq(struct l2cap_chan *chan, __u16 seq) return (seq + 1) % (chan->tx_win_max + 1); } -static inline struct l2cap_chan *l2cap_chan_no_new_connection(struct l2cap_chan *chan) +static inline int l2cap_chan_no_new_connection(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - return NULL; + return -EOPNOTSUPP; } static inline int l2cap_chan_no_recv(struct l2cap_chan *chan, struct sk_buff *skb) @@ -960,7 +968,7 @@ int l2cap_chan_send(struct l2cap_chan *chan, struct msghdr *msg, size_t len, void l2cap_chan_busy(struct l2cap_chan *chan, int busy); void l2cap_chan_rx_avail(struct l2cap_chan *chan, ssize_t rx_avail); int l2cap_chan_check_security(struct l2cap_chan *chan, bool initiator); -void l2cap_chan_set_defaults(struct l2cap_chan *chan); +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan); int l2cap_ertm_init(struct l2cap_chan *chan); void l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); void __l2cap_chan_add(struct l2cap_conn *conn, struct l2cap_chan *chan); diff --git a/include/net/bluetooth/mgmt.h b/include/net/bluetooth/mgmt.h index 8234915854b6..1e22eab1081c 100644 --- a/include/net/bluetooth/mgmt.h +++ b/include/net/bluetooth/mgmt.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* BlueZ - Bluetooth protocol stack for Linux Copyright (C) 2010 Nokia Corporation Copyright (C) 2011-2012 Intel Corporation - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 as - published by the Free Software Foundation; - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. @@ -121,6 +118,7 @@ struct mgmt_rp_read_index_list { #define MGMT_SETTING_LL_PRIVACY BIT(22) #define MGMT_SETTING_PAST_SENDER BIT(23) #define MGMT_SETTING_PAST_RECEIVER BIT(24) +#define MGMT_SETTING_SCI BIT(25) #define MGMT_OP_READ_INFO 0x0004 #define MGMT_READ_INFO_SIZE 0 @@ -896,6 +894,23 @@ struct mgmt_cp_hci_cmd_sync { } __packed; #define MGMT_HCI_CMD_SYNC_SIZE 6 +#define MGMT_OP_LOAD_CONN_SUBRATE 0x005C +struct mgmt_conn_subrate { + struct mgmt_addr_info addr; + __le16 min_interval; + __le16 max_interval; + __le16 subrate_min; + __le16 subrate_max; + __le16 max_latency; + __le16 cont_num; + __le16 supv_timeout; +} __packed; +struct mgmt_cp_load_conn_subrate { + __le16 param_count; + struct mgmt_conn_subrate params[] __counted_by_le(param_count); +} __packed; +#define MGMT_LOAD_CONN_SUBRATE_SIZE 2 + #define MGMT_EV_CMD_COMPLETE 0x0001 struct mgmt_ev_cmd_complete { __le16 opcode; @@ -1195,3 +1210,14 @@ struct mgmt_ev_mesh_device_found { struct mgmt_ev_mesh_pkt_cmplt { __u8 handle; } __packed; + +#define MGMT_EV_CONN_SUBRATE 0x0033 +struct mgmt_ev_conn_subrate { + struct mgmt_addr_info addr; + __u8 status; + __le16 interval; + __le16 subrate; + __le16 latency; + __le16 cont_num; + __le16 supv_timeout; +} __packed; diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h index c05882476900..102c278e3584 100644 --- a/include/net/bluetooth/rfcomm.h +++ b/include/net/bluetooth/rfcomm.h @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* RFCOMM implementation for Linux Bluetooth stack (BlueZ) Copyright (C) 2002 Maxim Krasnyansky <maxk@qualcomm.com> Copyright (C) 2002 Marcel Holtmann <marcel@holtmann.org> - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 as - published by the Free Software Foundation; - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. @@ -229,6 +226,9 @@ int rfcomm_send_rpn(struct rfcomm_session *s, int cr, u8 dlci, u8 bit_rate, u8 data_bits, u8 stop_bits, u8 parity, u8 flow_ctrl_settings, u8 xon_char, u8 xoff_char, u16 param_mask); +int rfcomm_dlc_send_rpn(struct rfcomm_dlc *d, u8 bit_rate, u8 data_bits, + u8 stop_bits, u8 parity, u8 flow_ctrl_settings, + u8 xon_char, u8 xoff_char, u16 param_mask); /* ---- RFCOMM DLCs (channels) ---- */ struct rfcomm_dlc *rfcomm_dlc_alloc(gfp_t prio); diff --git a/include/net/bluetooth/sco.h b/include/net/bluetooth/sco.h index f40ddb4264fc..214409527b35 100644 --- a/include/net/bluetooth/sco.h +++ b/include/net/bluetooth/sco.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* BlueZ - Bluetooth protocol stack for Linux Copyright (C) 2000-2001 Qualcomm Incorporated Written 2000,2001 by Maxim Krasnyansky <maxk@qualcomm.com> - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2 as - published by the Free Software Foundation; - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. diff --git a/include/net/bond_3ad.h b/include/net/bond_3ad.h index c92d4a976246..ef667dff2972 100644 --- a/include/net/bond_3ad.h +++ b/include/net/bond_3ad.h @@ -243,7 +243,7 @@ typedef struct port { churn_state_t sm_churn_actor_state; churn_state_t sm_churn_partner_state; struct slave *slave; /* pointer to the bond slave that this port belongs to */ - struct aggregator *aggregator; /* pointer to an aggregator that this port related to */ + struct aggregator __rcu *aggregator; /* pointer to an aggregator that this port related to */ struct port *next_port_in_aggregator; /* Next port on the linked list of the parent aggregator */ u32 transaction_id; /* continuous number for identification of Marker PDU's; */ struct lacpdu lacpdu; /* the lacpdu that will be sent for this port */ @@ -302,8 +302,8 @@ void bond_3ad_state_machine_handler(struct work_struct *); void bond_3ad_initiate_agg_selection(struct bonding *bond, int timeout); void bond_3ad_adapter_speed_duplex_changed(struct slave *slave); void bond_3ad_handle_link_change(struct slave *slave, char link); -int bond_3ad_get_active_agg_info(struct bonding *bond, struct ad_info *ad_info); -int __bond_3ad_get_active_agg_info(struct bonding *bond, +int bond_3ad_get_active_agg_info(const struct bonding *bond, struct ad_info *ad_info); +int __bond_3ad_get_active_agg_info(const struct bonding *bond, struct ad_info *ad_info); int bond_3ad_lacpdu_recv(const struct sk_buff *skb, struct bonding *bond, struct slave *slave); diff --git a/include/net/bond_options.h b/include/net/bond_options.h index e6eedf23aea1..52b966e92793 100644 --- a/include/net/bond_options.h +++ b/include/net/bond_options.h @@ -79,6 +79,7 @@ enum { BOND_OPT_COUPLED_CONTROL, BOND_OPT_BROADCAST_NEIGH, BOND_OPT_ACTOR_PORT_PRIO, + BOND_OPT_LACP_STRICT, BOND_OPT_LAST }; diff --git a/include/net/bonding.h b/include/net/bonding.h index 462078403557..598d56b1bc97 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -69,9 +69,6 @@ #define bond_first_slave_rcu(bond) \ netdev_lower_get_first_private_rcu(bond->dev) -#define bond_is_first_slave(bond, pos) (pos == bond_first_slave(bond)) -#define bond_is_last_slave(bond, pos) (pos == bond_last_slave(bond)) - /** * bond_for_each_slave - iterate over all slaves * @bond: the bond holding this list @@ -91,22 +88,22 @@ NETIF_F_GSO_ESP) #ifdef CONFIG_NET_POLL_CONTROLLER -extern atomic_t netpoll_block_tx; +DECLARE_STATIC_KEY_FALSE(netpoll_block_tx); static inline void block_netpoll_tx(void) { - atomic_inc(&netpoll_block_tx); + static_branch_inc(&netpoll_block_tx); } static inline void unblock_netpoll_tx(void) { - atomic_dec(&netpoll_block_tx); + static_branch_dec(&netpoll_block_tx); } static inline int is_netpoll_tx_blocked(struct net_device *dev) { - if (unlikely(netpoll_tx_running(dev))) - return atomic_read(&netpoll_block_tx); + if (static_branch_unlikely(&netpoll_block_tx)) + return netpoll_tx_running(dev); return 0; } #else @@ -132,6 +129,7 @@ struct bond_params { int peer_notif_delay; int lacp_active; int lacp_fast; + int lacp_strict; unsigned int min_links; int ad_select; char primary[IFNAMSIZ]; @@ -254,6 +252,7 @@ struct bonding { struct delayed_work ad_work; struct delayed_work mcast_work; struct delayed_work slave_arr_work; + struct delayed_work peer_notify_work; #ifdef CONFIG_DEBUG_FS /* debugging support via debugfs */ struct dentry *debug_dir; @@ -346,14 +345,14 @@ static inline bool bond_mode_uses_primary(int mode) mode == BOND_MODE_ALB; } -static inline bool bond_uses_primary(struct bonding *bond) +static inline bool bond_uses_primary(const struct bonding *bond) { return bond_mode_uses_primary(BOND_MODE(bond)); } -static inline struct net_device *bond_option_active_slave_get_rcu(struct bonding *bond) +static inline struct net_device *bond_option_active_slave_get_rcu(const struct bonding *bond) { - struct slave *slave = rcu_dereference_rtnl(bond->curr_active_slave); + const struct slave *slave = rcu_dereference_rtnl(bond->curr_active_slave); return bond_uses_primary(bond) && slave ? slave->dev : NULL; } @@ -698,18 +697,20 @@ void bond_debug_register(struct bonding *bond); void bond_debug_unregister(struct bonding *bond); void bond_debug_reregister(struct bonding *bond); const char *bond_mode_name(int mode); +bool __bond_xdp_check(int mode, int xmit_policy); bool bond_xdp_check(struct bonding *bond, int mode); void bond_setup(struct net_device *bond_dev); unsigned int bond_get_num_tx_queues(void); int bond_netlink_init(void); void bond_netlink_fini(void); -struct net_device *bond_option_active_slave_get_rcu(struct bonding *bond); +struct net_device *bond_option_active_slave_get_rcu(const struct bonding *bond); const char *bond_slave_link_status(s8 link); struct bond_vlan_tag *bond_verify_device_path(struct net_device *start_dev, struct net_device *end_dev, int level); int bond_update_slave_arr(struct bonding *bond, struct slave *skipslave); void bond_slave_arr_work_rearm(struct bonding *bond, unsigned long delay); +void bond_peer_notify_work_rearm(struct bonding *bond, unsigned long delay); void bond_work_init_all(struct bonding *bond); void bond_work_cancel_all(struct bonding *bond); diff --git a/include/net/caif/caif_dev.h b/include/net/caif/caif_dev.h deleted file mode 100644 index b655d8666f55..000000000000 --- a/include/net/caif/caif_dev.h +++ /dev/null @@ -1,128 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CAIF_DEV_H_ -#define CAIF_DEV_H_ - -#include <net/caif/caif_layer.h> -#include <net/caif/cfcnfg.h> -#include <net/caif/caif_device.h> -#include <linux/caif/caif_socket.h> -#include <linux/if.h> -#include <linux/net.h> - -/** - * struct caif_param - CAIF parameters. - * @size: Length of data - * @data: Binary Data Blob - */ -struct caif_param { - u16 size; - u8 data[256]; -}; - -/** - * struct caif_connect_request - Request data for CAIF channel setup. - * @protocol: Type of CAIF protocol to use (at, datagram etc) - * @sockaddr: Socket address to connect. - * @priority: Priority of the connection. - * @link_selector: Link selector (high bandwidth or low latency) - * @ifindex: kernel index of the interface. - * @param: Connect Request parameters (CAIF_SO_REQ_PARAM). - * - * This struct is used when connecting a CAIF channel. - * It contains all CAIF channel configuration options. - */ -struct caif_connect_request { - enum caif_protocol_type protocol; - struct sockaddr_caif sockaddr; - enum caif_channel_priority priority; - enum caif_link_selector link_selector; - int ifindex; - struct caif_param param; -}; - -/** - * caif_connect_client - Connect a client to CAIF Core Stack. - * @config: Channel setup parameters, specifying what address - * to connect on the Modem. - * @client_layer: User implementation of client layer. This layer - * MUST have receive and control callback functions - * implemented. - * @ifindex: Link layer interface index used for this connection. - * @headroom: Head room needed by CAIF protocol. - * @tailroom: Tail room needed by CAIF protocol. - * - * This function connects a CAIF channel. The Client must implement - * the struct cflayer. This layer represents the Client layer and holds - * receive functions and control callback functions. Control callback - * function will receive information about connect/disconnect responses, - * flow control etc (see enum caif_control). - * E.g. CAIF Socket will call this function for each socket it connects - * and have one client_layer instance for each socket. - */ -int caif_connect_client(struct net *net, - struct caif_connect_request *conn_req, - struct cflayer *client_layer, int *ifindex, - int *headroom, int *tailroom); - -/** - * caif_disconnect_client - Disconnects a client from the CAIF stack. - * - * @client_layer: Client layer to be disconnected. - */ -int caif_disconnect_client(struct net *net, struct cflayer *client_layer); - - -/** - * caif_client_register_refcnt - register ref-count functions provided by client. - * - * @adapt_layer: Client layer using CAIF Stack. - * @hold: Function provided by client layer increasing ref-count - * @put: Function provided by client layer decreasing ref-count - * - * Client of the CAIF Stack must register functions for reference counting. - * These functions are called by the CAIF Stack for every upstream packet, - * and must therefore be implemented efficiently. - * - * Client should call caif_free_client when reference count degrease to zero. - */ - -void caif_client_register_refcnt(struct cflayer *adapt_layer, - void (*hold)(struct cflayer *lyr), - void (*put)(struct cflayer *lyr)); -/** - * caif_free_client - Free memory used to manage the client in the CAIF Stack. - * - * @client_layer: Client layer to be removed. - * - * This function must be called from client layer in order to free memory. - * Caller must guarantee that no packets are in flight upstream when calling - * this function. - */ -void caif_free_client(struct cflayer *adap_layer); - -/** - * struct caif_enroll_dev - Enroll a net-device as a CAIF Link layer - * @dev: Network device to enroll. - * @caifdev: Configuration information from CAIF Link Layer - * @link_support: Link layer support layer - * @head_room: Head room needed by link support layer - * @layer: Lowest layer in CAIF stack - * @rcv_fun: Receive function for CAIF stack. - * - * This function enroll a CAIF link layer into CAIF Stack and - * expects the interface to be able to handle CAIF payload. - * The link_support layer is used to add any Link Layer specific - * framing. - */ -int caif_enroll_dev(struct net_device *dev, struct caif_dev_common *caifdev, - struct cflayer *link_support, int head_room, - struct cflayer **layer, int (**rcv_func)( - struct sk_buff *, struct net_device *, - struct packet_type *, struct net_device *)); - -#endif /* CAIF_DEV_H_ */ diff --git a/include/net/caif/caif_device.h b/include/net/caif/caif_device.h deleted file mode 100644 index 91d1fd5b44a4..000000000000 --- a/include/net/caif/caif_device.h +++ /dev/null @@ -1,55 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CAIF_DEVICE_H_ -#define CAIF_DEVICE_H_ -#include <linux/kernel.h> -#include <linux/net.h> -#include <linux/netdevice.h> -#include <linux/caif/caif_socket.h> -#include <net/caif/caif_device.h> - -/** - * struct caif_dev_common - data shared between CAIF drivers and stack. - * @flowctrl: Flow Control callback function. This function is - * supplied by CAIF Core Stack and is used by CAIF - * Link Layer to send flow-stop to CAIF Core. - * The flow information will be distributed to all - * clients of CAIF. - * - * @link_select: Profile of device, either high-bandwidth or - * low-latency. This member is set by CAIF Link - * Layer Device in order to indicate if this device - * is a high bandwidth or low latency device. - * - * @use_frag: CAIF Frames may be fragmented. - * Is set by CAIF Link Layer in order to indicate if the - * interface receives fragmented frames that must be - * assembled by CAIF Core Layer. - * - * @use_fcs: Indicate if Frame CheckSum (fcs) is used. - * Is set if the physical interface is - * using Frame Checksum on the CAIF Frames. - * - * @use_stx: Indicate STart of frame eXtension (stx) in use. - * Is set if the CAIF Link Layer expects - * CAIF Frames to start with the STX byte. - * - * This structure is shared between the CAIF drivers and the CAIF stack. - * It is used by the device to register its behavior. - * CAIF Core layer must set the member flowctrl in order to supply - * CAIF Link Layer with the flow control function. - * - */ - struct caif_dev_common { - void (*flowctrl)(struct net_device *net, int on); - enum caif_link_selector link_select; - int use_frag; - int use_fcs; - int use_stx; -}; - -#endif /* CAIF_DEVICE_H_ */ diff --git a/include/net/caif/caif_layer.h b/include/net/caif/caif_layer.h deleted file mode 100644 index 053e7c6a6a66..000000000000 --- a/include/net/caif/caif_layer.h +++ /dev/null @@ -1,277 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CAIF_LAYER_H_ -#define CAIF_LAYER_H_ - -#include <linux/list.h> - -struct cflayer; -struct cfpkt; -struct caif_payload_info; - -#define CAIF_LAYER_NAME_SZ 16 - -/** - * caif_assert() - Assert function for CAIF. - * @assert: expression to evaluate. - * - * This function will print a error message and a do WARN_ON if the - * assertion fails. Normally this will do a stack up at the current location. - */ -#define caif_assert(assert) \ -do { \ - if (!(assert)) { \ - pr_err("caif:Assert detected:'%s'\n", #assert); \ - WARN_ON(!(assert)); \ - } \ -} while (0) - -/** - * enum caif_ctrlcmd - CAIF Stack Control Signaling sent in layer.ctrlcmd(). - * - * @CAIF_CTRLCMD_FLOW_OFF_IND: Flow Control is OFF, transmit function - * should stop sending data - * - * @CAIF_CTRLCMD_FLOW_ON_IND: Flow Control is ON, transmit function - * can start sending data - * - * @CAIF_CTRLCMD_REMOTE_SHUTDOWN_IND: Remote end modem has decided to close - * down channel - * - * @CAIF_CTRLCMD_INIT_RSP: Called initially when the layer below - * has finished initialization - * - * @CAIF_CTRLCMD_DEINIT_RSP: Called when de-initialization is - * complete - * - * @CAIF_CTRLCMD_INIT_FAIL_RSP: Called if initialization fails - * - * @_CAIF_CTRLCMD_PHYIF_FLOW_OFF_IND: CAIF Link layer temporarily cannot - * send more packets. - * @_CAIF_CTRLCMD_PHYIF_FLOW_ON_IND: Called if CAIF Link layer is able - * to send packets again. - * @_CAIF_CTRLCMD_PHYIF_DOWN_IND: Called if CAIF Link layer is going - * down. - * - * These commands are sent upwards in the CAIF stack to the CAIF Client. - * They are used for signaling originating from the modem or CAIF Link Layer. - * These are either responses (*_RSP) or events (*_IND). - */ -enum caif_ctrlcmd { - CAIF_CTRLCMD_FLOW_OFF_IND, - CAIF_CTRLCMD_FLOW_ON_IND, - CAIF_CTRLCMD_REMOTE_SHUTDOWN_IND, - CAIF_CTRLCMD_INIT_RSP, - CAIF_CTRLCMD_DEINIT_RSP, - CAIF_CTRLCMD_INIT_FAIL_RSP, - _CAIF_CTRLCMD_PHYIF_FLOW_OFF_IND, - _CAIF_CTRLCMD_PHYIF_FLOW_ON_IND, - _CAIF_CTRLCMD_PHYIF_DOWN_IND, -}; - -/** - * enum caif_modemcmd - Modem Control Signaling, sent from CAIF Client - * to the CAIF Link Layer or modem. - * - * @CAIF_MODEMCMD_FLOW_ON_REQ: Flow Control is ON, transmit function - * can start sending data. - * - * @CAIF_MODEMCMD_FLOW_OFF_REQ: Flow Control is OFF, transmit function - * should stop sending data. - * - * @_CAIF_MODEMCMD_PHYIF_USEFULL: Notify physical layer that it is in use - * - * @_CAIF_MODEMCMD_PHYIF_USELESS: Notify physical layer that it is - * no longer in use. - * - * These are requests sent 'downwards' in the stack. - * Flow ON, OFF can be indicated to the modem. - */ -enum caif_modemcmd { - CAIF_MODEMCMD_FLOW_ON_REQ = 0, - CAIF_MODEMCMD_FLOW_OFF_REQ = 1, - _CAIF_MODEMCMD_PHYIF_USEFULL = 3, - _CAIF_MODEMCMD_PHYIF_USELESS = 4 -}; - -/** - * enum caif_direction - CAIF Packet Direction. - * Indicate if a packet is to be sent out or to be received in. - * @CAIF_DIR_IN: Incoming packet received. - * @CAIF_DIR_OUT: Outgoing packet to be transmitted. - */ -enum caif_direction { - CAIF_DIR_IN = 0, - CAIF_DIR_OUT = 1 -}; - -/** - * struct cflayer - CAIF Stack layer. - * Defines the framework for the CAIF Core Stack. - * @up: Pointer up to the layer above. - * @dn: Pointer down to the layer below. - * @node: List node used when layer participate in a list. - * @receive: Packet receive function. - * @transmit: Packet transmit function. - * @ctrlcmd: Used for control signalling upwards in the stack. - * @modemcmd: Used for control signaling downwards in the stack. - * @id: The identity of this layer - * @name: Name of the layer. - * - * This structure defines the layered structure in CAIF. - * - * It defines CAIF layering structure, used by all CAIF Layers and the - * layers interfacing CAIF. - * - * In order to integrate with CAIF an adaptation layer on top of the CAIF stack - * and PHY layer below the CAIF stack - * must be implemented. These layer must follow the design principles below. - * - * Principles for layering of protocol layers: - * - All layers must use this structure. If embedding it, then place this - * structure first in the layer specific structure. - * - * - Each layer should not depend on any others layer's private data. - * - * - In order to send data upwards do - * layer->up->receive(layer->up, packet); - * - * - In order to send data downwards do - * layer->dn->transmit(layer->dn, info, packet); - */ -struct cflayer { - struct cflayer *up; - struct cflayer *dn; - struct list_head node; - - /* - * receive() - Receive Function (non-blocking). - * Contract: Each layer must implement a receive function passing the - * CAIF packets upwards in the stack. - * Packet handling rules: - * - The CAIF packet (cfpkt) ownership is passed to the - * called receive function. This means that the - * packet cannot be accessed after passing it to the - * above layer using up->receive(). - * - * - If parsing of the packet fails, the packet must be - * destroyed and negative error code returned - * from the function. - * EXCEPTION: If the framing layer (cffrml) returns - * -EILSEQ, the packet is not freed. - * - * - If parsing succeeds (and above layers return OK) then - * the function must return a value >= 0. - * - * Returns result < 0 indicates an error, 0 or positive value - * indicates success. - * - * @layr: Pointer to the current layer the receive function is - * implemented for (this pointer). - * @cfpkt: Pointer to CaifPacket to be handled. - */ - int (*receive)(struct cflayer *layr, struct cfpkt *cfpkt); - - /* - * transmit() - Transmit Function (non-blocking). - * Contract: Each layer must implement a transmit function passing the - * CAIF packet downwards in the stack. - * Packet handling rules: - * - The CAIF packet (cfpkt) ownership is passed to the - * transmit function. This means that the packet - * cannot be accessed after passing it to the below - * layer using dn->transmit(). - * - * - Upon error the packet ownership is still passed on, - * so the packet shall be freed where error is detected. - * Callers of the transmit function shall not free packets, - * but errors shall be returned. - * - * - Return value less than zero means error, zero or - * greater than zero means OK. - * - * Returns result < 0 indicates an error, 0 or positive value - * indicates success. - * - * @layr: Pointer to the current layer the receive function - * isimplemented for (this pointer). - * @cfpkt: Pointer to CaifPacket to be handled. - */ - int (*transmit) (struct cflayer *layr, struct cfpkt *cfpkt); - - /* - * cttrlcmd() - Control Function upwards in CAIF Stack (non-blocking). - * Used for signaling responses (CAIF_CTRLCMD_*_RSP) - * and asynchronous events from the modem (CAIF_CTRLCMD_*_IND) - * - * @layr: Pointer to the current layer the receive function - * is implemented for (this pointer). - * @ctrl: Control Command. - */ - void (*ctrlcmd) (struct cflayer *layr, enum caif_ctrlcmd ctrl, - int phyid); - - /* - * modemctrl() - Control Function used for controlling the modem. - * Used to signal down-wards in the CAIF stack. - * Returns 0 on success, < 0 upon failure. - * - * @layr: Pointer to the current layer the receive function - * is implemented for (this pointer). - * @ctrl: Control Command. - */ - int (*modemcmd) (struct cflayer *layr, enum caif_modemcmd ctrl); - - unsigned int id; - char name[CAIF_LAYER_NAME_SZ]; -}; - -/** - * layer_set_up() - Set the up pointer for a specified layer. - * @layr: Layer where up pointer shall be set. - * @above: Layer above. - */ -#define layer_set_up(layr, above) ((layr)->up = (struct cflayer *)(above)) - -/** - * layer_set_dn() - Set the down pointer for a specified layer. - * @layr: Layer where down pointer shall be set. - * @below: Layer below. - */ -#define layer_set_dn(layr, below) ((layr)->dn = (struct cflayer *)(below)) - -/** - * struct dev_info - Physical Device info information about physical layer. - * @dev: Pointer to native physical device. - * @id: Physical ID of the physical connection used by the - * logical CAIF connection. Used by service layers to - * identify their physical id to Caif MUX (CFMUXL)so - * that the MUX can add the correct physical ID to the - * packet. - */ -struct dev_info { - void *dev; - unsigned int id; -}; - -/** - * struct caif_payload_info - Payload information embedded in packet (sk_buff). - * - * @dev_info: Information about the receiving device. - * - * @hdr_len: Header length, used to align pay load on 32bit boundary. - * - * @channel_id: Channel ID of the logical CAIF connection. - * Used by mux to insert channel id into the caif packet. - */ -struct caif_payload_info { - struct dev_info *dev_info; - unsigned short hdr_len; - unsigned short channel_id; -}; - -#endif /* CAIF_LAYER_H_ */ diff --git a/include/net/caif/cfcnfg.h b/include/net/caif/cfcnfg.h deleted file mode 100644 index 8819ff4db35a..000000000000 --- a/include/net/caif/cfcnfg.h +++ /dev/null @@ -1,90 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CFCNFG_H_ -#define CFCNFG_H_ -#include <linux/spinlock.h> -#include <linux/netdevice.h> -#include <net/caif/caif_layer.h> -#include <net/caif/cfctrl.h> - -struct cfcnfg; - -/** - * enum cfcnfg_phy_preference - Physical preference HW Abstraction - * - * @CFPHYPREF_UNSPECIFIED: Default physical interface - * - * @CFPHYPREF_LOW_LAT: Default physical interface for low-latency - * traffic - * @CFPHYPREF_HIGH_BW: Default physical interface for high-bandwidth - * traffic - * @CFPHYPREF_LOOP: TEST only Loopback interface simulating modem - * responses. - * - */ -enum cfcnfg_phy_preference { - CFPHYPREF_UNSPECIFIED, - CFPHYPREF_LOW_LAT, - CFPHYPREF_HIGH_BW, - CFPHYPREF_LOOP -}; - -/** - * cfcnfg_create() - Get the CAIF configuration object given network. - * @net: Network for the CAIF configuration object. - */ -struct cfcnfg *get_cfcnfg(struct net *net); - -/** - * cfcnfg_create() - Create the CAIF configuration object. - */ -struct cfcnfg *cfcnfg_create(void); - -/** - * cfcnfg_remove() - Remove the CFCNFG object - * @cfg: config object - */ -void cfcnfg_remove(struct cfcnfg *cfg); - -/** - * cfcnfg_add_phy_layer() - Adds a physical layer to the CAIF stack. - * @cnfg: Pointer to a CAIF configuration object, created by - * cfcnfg_create(). - * @dev: Pointer to link layer device - * @phy_layer: Specify the physical layer. The transmit function - * MUST be set in the structure. - * @pref: The phy (link layer) preference. - * @link_support: Protocol implementation for link layer specific protocol. - * @fcs: Specify if checksum is used in CAIF Framing Layer. - * @head_room: Head space needed by link specific protocol. - */ -int -cfcnfg_add_phy_layer(struct cfcnfg *cnfg, - struct net_device *dev, struct cflayer *phy_layer, - enum cfcnfg_phy_preference pref, - struct cflayer *link_support, - bool fcs, int head_room); - -/** - * cfcnfg_del_phy_layer - Deletes an phy layer from the CAIF stack. - * - * @cnfg: Pointer to a CAIF configuration object, created by - * cfcnfg_create(). - * @phy_layer: Adaptation layer to be removed. - */ -int cfcnfg_del_phy_layer(struct cfcnfg *cnfg, struct cflayer *phy_layer); - -/** - * cfcnfg_set_phy_state() - Set the state of the physical interface device. - * @cnfg: Configuration object - * @phy_layer: Physical Layer representation - * @up: State of device - */ -int cfcnfg_set_phy_state(struct cfcnfg *cnfg, struct cflayer *phy_layer, - bool up); - -#endif /* CFCNFG_H_ */ diff --git a/include/net/caif/cfctrl.h b/include/net/caif/cfctrl.h deleted file mode 100644 index 86d17315c8a1..000000000000 --- a/include/net/caif/cfctrl.h +++ /dev/null @@ -1,130 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CFCTRL_H_ -#define CFCTRL_H_ -#include <net/caif/caif_layer.h> -#include <net/caif/cfsrvl.h> - -/* CAIF Control packet commands */ -enum cfctrl_cmd { - CFCTRL_CMD_LINK_SETUP = 0, - CFCTRL_CMD_LINK_DESTROY = 1, - CFCTRL_CMD_LINK_ERR = 2, - CFCTRL_CMD_ENUM = 3, - CFCTRL_CMD_SLEEP = 4, - CFCTRL_CMD_WAKE = 5, - CFCTRL_CMD_LINK_RECONF = 6, - CFCTRL_CMD_START_REASON = 7, - CFCTRL_CMD_RADIO_SET = 8, - CFCTRL_CMD_MODEM_SET = 9, - CFCTRL_CMD_MASK = 0xf -}; - -/* Channel types */ -enum cfctrl_srv { - CFCTRL_SRV_DECM = 0, - CFCTRL_SRV_VEI = 1, - CFCTRL_SRV_VIDEO = 2, - CFCTRL_SRV_DBG = 3, - CFCTRL_SRV_DATAGRAM = 4, - CFCTRL_SRV_RFM = 5, - CFCTRL_SRV_UTIL = 6, - CFCTRL_SRV_MASK = 0xf -}; - -#define CFCTRL_RSP_BIT 0x20 -#define CFCTRL_ERR_BIT 0x10 - -struct cfctrl_rsp { - void (*linksetup_rsp)(struct cflayer *layer, u8 linkid, - enum cfctrl_srv serv, u8 phyid, - struct cflayer *adapt_layer); - void (*linkdestroy_rsp)(struct cflayer *layer, u8 linkid); - void (*linkerror_ind)(void); - void (*enum_rsp)(void); - void (*sleep_rsp)(void); - void (*wake_rsp)(void); - void (*restart_rsp)(void); - void (*radioset_rsp)(void); - void (*reject_rsp)(struct cflayer *layer, u8 linkid, - struct cflayer *client_layer); -}; - -/* Link Setup Parameters for CAIF-Links. */ -struct cfctrl_link_param { - enum cfctrl_srv linktype;/* (T3,T0) Type of Channel */ - u8 priority; /* (P4,P0) Priority of the channel */ - u8 phyid; /* (U2-U0) Physical interface to connect */ - u8 endpoint; /* (E1,E0) Endpoint for data channels */ - u8 chtype; /* (H1,H0) Channel-Type, applies to - * VEI, DEBUG */ - union { - struct { - u8 connid; /* (D7,D0) Video LinkId */ - } video; - - struct { - u32 connid; /* (N31,Ngit0) Connection ID used - * for Datagram */ - } datagram; - - struct { - u32 connid; /* Connection ID used for RFM */ - char volume[20]; /* Volume to mount for RFM */ - } rfm; /* Configuration for RFM */ - - struct { - u16 fifosize_kb; /* Psock FIFO size in KB */ - u16 fifosize_bufs; /* Psock # signal buffers */ - char name[16]; /* Name of the PSOCK service */ - u8 params[255]; /* Link setup Parameters> */ - u16 paramlen; /* Length of Link Setup - * Parameters */ - } utility; /* Configuration for Utility Links (Psock) */ - } u; -}; - -/* This structure is used internally in CFCTRL */ -struct cfctrl_request_info { - int sequence_no; - enum cfctrl_cmd cmd; - u8 channel_id; - struct cfctrl_link_param param; - struct cflayer *client_layer; - struct list_head list; -}; - -struct cfctrl { - struct cfsrvl serv; - struct cfctrl_rsp res; - atomic_t req_seq_no; - atomic_t rsp_seq_no; - struct list_head list; - /* Protects from simultaneous access to first_req list */ - spinlock_t info_list_lock; -#ifndef CAIF_NO_LOOP - u8 loop_linkid; - int loop_linkused[256]; - /* Protects simultaneous access to loop_linkid and loop_linkused */ - spinlock_t loop_linkid_lock; -#endif - -}; - -void cfctrl_enum_req(struct cflayer *cfctrl, u8 physlinkid); -int cfctrl_linkup_request(struct cflayer *cfctrl, - struct cfctrl_link_param *param, - struct cflayer *user_layer); -int cfctrl_linkdown_req(struct cflayer *cfctrl, u8 linkid, - struct cflayer *client); - -struct cflayer *cfctrl_create(void); -struct cfctrl_rsp *cfctrl_get_respfuncs(struct cflayer *layer); -int cfctrl_cancel_req(struct cflayer *layr, struct cflayer *adap_layer); -void cfctrl_remove(struct cflayer *layr); - -#endif /* CFCTRL_H_ */ diff --git a/include/net/caif/cffrml.h b/include/net/caif/cffrml.h deleted file mode 100644 index 1ab8a80ede4d..000000000000 --- a/include/net/caif/cffrml.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CFFRML_H_ -#define CFFRML_H_ -#include <net/caif/caif_layer.h> -#include <linux/netdevice.h> - -struct cffrml; -struct cflayer *cffrml_create(u16 phyid, bool use_fcs); -void cffrml_free(struct cflayer *layr); -void cffrml_set_uplayer(struct cflayer *this, struct cflayer *up); -void cffrml_set_dnlayer(struct cflayer *this, struct cflayer *dn); -void cffrml_put(struct cflayer *layr); -void cffrml_hold(struct cflayer *layr); -int cffrml_refcnt_read(struct cflayer *layr); - -#endif /* CFFRML_H_ */ diff --git a/include/net/caif/cfmuxl.h b/include/net/caif/cfmuxl.h deleted file mode 100644 index 92ccb2648309..000000000000 --- a/include/net/caif/cfmuxl.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CFMUXL_H_ -#define CFMUXL_H_ -#include <net/caif/caif_layer.h> - -struct cfsrvl; -struct cffrml; - -struct cflayer *cfmuxl_create(void); -int cfmuxl_set_uplayer(struct cflayer *layr, struct cflayer *up, u8 linkid); -struct cflayer *cfmuxl_remove_dnlayer(struct cflayer *layr, u8 phyid); -int cfmuxl_set_dnlayer(struct cflayer *layr, struct cflayer *up, u8 phyid); -struct cflayer *cfmuxl_remove_uplayer(struct cflayer *layr, u8 linkid); - -#endif /* CFMUXL_H_ */ diff --git a/include/net/caif/cfpkt.h b/include/net/caif/cfpkt.h deleted file mode 100644 index acf664227d96..000000000000 --- a/include/net/caif/cfpkt.h +++ /dev/null @@ -1,232 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CFPKT_H_ -#define CFPKT_H_ -#include <net/caif/caif_layer.h> -#include <linux/types.h> -struct cfpkt; - -/* Create a CAIF packet. - * len: Length of packet to be created - * @return New packet. - */ -struct cfpkt *cfpkt_create(u16 len); - -/* - * Destroy a CAIF Packet. - * pkt Packet to be destroyed. - */ -void cfpkt_destroy(struct cfpkt *pkt); - -/* - * Extract header from packet. - * - * pkt Packet to extract header data from. - * data Pointer to copy the header data into. - * len Length of head data to copy. - * @return zero on success and error code upon failure - */ -int cfpkt_extr_head(struct cfpkt *pkt, void *data, u16 len); - -static inline u8 cfpkt_extr_head_u8(struct cfpkt *pkt) -{ - u8 tmp; - - cfpkt_extr_head(pkt, &tmp, 1); - - return tmp; -} - -static inline u16 cfpkt_extr_head_u16(struct cfpkt *pkt) -{ - __le16 tmp; - - cfpkt_extr_head(pkt, &tmp, 2); - - return le16_to_cpu(tmp); -} - -static inline u32 cfpkt_extr_head_u32(struct cfpkt *pkt) -{ - __le32 tmp; - - cfpkt_extr_head(pkt, &tmp, 4); - - return le32_to_cpu(tmp); -} - -/* - * Peek header from packet. - * Reads data from packet without changing packet. - * - * pkt Packet to extract header data from. - * data Pointer to copy the header data into. - * len Length of head data to copy. - * @return zero on success and error code upon failure - */ -int cfpkt_peek_head(struct cfpkt *pkt, void *data, u16 len); - -/* - * Extract header from trailer (end of packet). - * - * pkt Packet to extract header data from. - * data Pointer to copy the trailer data into. - * len Length of header data to copy. - * @return zero on success and error code upon failure - */ -int cfpkt_extr_trail(struct cfpkt *pkt, void *data, u16 len); - -/* - * Add header to packet. - * - * - * pkt Packet to add header data to. - * data Pointer to data to copy into the header. - * len Length of header data to copy. - * @return zero on success and error code upon failure - */ -int cfpkt_add_head(struct cfpkt *pkt, const void *data, u16 len); - -/* - * Add trailer to packet. - * - * - * pkt Packet to add trailer data to. - * data Pointer to data to copy into the trailer. - * len Length of trailer data to copy. - * @return zero on success and error code upon failure - */ -int cfpkt_add_trail(struct cfpkt *pkt, const void *data, u16 len); - -/* - * Pad trailer on packet. - * Moves data pointer in packet, no content copied. - * - * pkt Packet in which to pad trailer. - * len Length of padding to add. - * @return zero on success and error code upon failure - */ -int cfpkt_pad_trail(struct cfpkt *pkt, u16 len); - -/* - * Add a single byte to packet body (tail). - * - * pkt Packet in which to add byte. - * data Byte to add. - * @return zero on success and error code upon failure - */ -int cfpkt_addbdy(struct cfpkt *pkt, const u8 data); - -/* - * Add a data to packet body (tail). - * - * pkt Packet in which to add data. - * data Pointer to data to copy into the packet body. - * len Length of data to add. - * @return zero on success and error code upon failure - */ -int cfpkt_add_body(struct cfpkt *pkt, const void *data, u16 len); - -/* - * Checks whether there are more data to process in packet. - * pkt Packet to check. - * @return true if more data are available in packet false otherwise - */ -bool cfpkt_more(struct cfpkt *pkt); - -/* - * Checks whether the packet is erroneous, - * i.e. if it has been attempted to extract more data than available in packet - * or writing more data than has been allocated in cfpkt_create(). - * pkt Packet to check. - * @return true on error false otherwise - */ -bool cfpkt_erroneous(struct cfpkt *pkt); - -/* - * Get the packet length. - * pkt Packet to get length from. - * @return Number of bytes in packet. - */ -u16 cfpkt_getlen(struct cfpkt *pkt); - -/* - * Set the packet length, by adjusting the trailer pointer according to length. - * pkt Packet to set length. - * len Packet length. - * @return Number of bytes in packet. - */ -int cfpkt_setlen(struct cfpkt *pkt, u16 len); - -/* - * cfpkt_append - Appends a packet's data to another packet. - * dstpkt: Packet to append data into, WILL BE FREED BY THIS FUNCTION - * addpkt: Packet to be appended and automatically released, - * WILL BE FREED BY THIS FUNCTION. - * expectlen: Packet's expected total length. This should be considered - * as a hint. - * NB: Input packets will be destroyed after appending and cannot be used - * after calling this function. - * @return The new appended packet. - */ -struct cfpkt *cfpkt_append(struct cfpkt *dstpkt, struct cfpkt *addpkt, - u16 expectlen); - -/* - * cfpkt_split - Split a packet into two packets at the specified split point. - * pkt: Packet to be split (will contain the first part of the data on exit) - * pos: Position to split packet in two parts. - * @return The new packet, containing the second part of the data. - */ -struct cfpkt *cfpkt_split(struct cfpkt *pkt, u16 pos); - -/* - * Iteration function, iterates the packet buffers from start to end. - * - * Checksum iteration function used to iterate buffers - * (we may have packets consisting of a chain of buffers) - * pkt: Packet to calculate checksum for - * iter_func: Function pointer to iteration function - * chks: Checksum calculated so far. - * buf: Pointer to the buffer to checksum - * len: Length of buf. - * data: Initial checksum value. - * @return Checksum of buffer. - */ - -int cfpkt_iterate(struct cfpkt *pkt, - u16 (*iter_func)(u16 chks, void *buf, u16 len), - u16 data); - -/* Map from a "native" packet (e.g. Linux Socket Buffer) to a CAIF packet. - * dir - Direction indicating whether this packet is to be sent or received. - * nativepkt - The native packet to be transformed to a CAIF packet - * @return The mapped CAIF Packet CFPKT. - */ -struct cfpkt *cfpkt_fromnative(enum caif_direction dir, void *nativepkt); - -/* Map from a CAIF packet to a "native" packet (e.g. Linux Socket Buffer). - * pkt - The CAIF packet to be transformed into a "native" packet. - * @return The native packet transformed from a CAIF packet. - */ -void *cfpkt_tonative(struct cfpkt *pkt); - -/* - * Returns packet information for a packet. - * pkt Packet to get info from; - * @return Packet information - */ -struct caif_payload_info *cfpkt_info(struct cfpkt *pkt); - -/** cfpkt_set_prio - set priority for a CAIF packet. - * - * @pkt: The CAIF packet to be adjusted. - * @prio: one of TC_PRIO_ constants. - */ -void cfpkt_set_prio(struct cfpkt *pkt, int prio); - -#endif /* CFPKT_H_ */ diff --git a/include/net/caif/cfserl.h b/include/net/caif/cfserl.h deleted file mode 100644 index 67cce8757175..000000000000 --- a/include/net/caif/cfserl.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CFSERL_H_ -#define CFSERL_H_ -#include <net/caif/caif_layer.h> - -struct cflayer *cfserl_create(int instance, bool use_stx); -void cfserl_release(struct cflayer *layer); -#endif diff --git a/include/net/caif/cfsrvl.h b/include/net/caif/cfsrvl.h deleted file mode 100644 index a000dc45f966..000000000000 --- a/include/net/caif/cfsrvl.h +++ /dev/null @@ -1,61 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson AB 2010 - * Author: Sjur Brendeland - */ - -#ifndef CFSRVL_H_ -#define CFSRVL_H_ -#include <linux/list.h> -#include <linux/stddef.h> -#include <linux/types.h> -#include <linux/kref.h> -#include <linux/rculist.h> - -struct cfsrvl { - struct cflayer layer; - bool open; - bool phy_flow_on; - bool modem_flow_on; - bool supports_flowctrl; - void (*release)(struct cflayer *layer); - struct dev_info dev_info; - void (*hold)(struct cflayer *lyr); - void (*put)(struct cflayer *lyr); - struct rcu_head rcu; -}; - -struct cflayer *cfvei_create(u8 linkid, struct dev_info *dev_info); -struct cflayer *cfdgml_create(u8 linkid, struct dev_info *dev_info); -struct cflayer *cfutill_create(u8 linkid, struct dev_info *dev_info); -struct cflayer *cfvidl_create(u8 linkid, struct dev_info *dev_info); -struct cflayer *cfrfml_create(u8 linkid, struct dev_info *dev_info, - int mtu_size); -struct cflayer *cfdbgl_create(u8 linkid, struct dev_info *dev_info); - -bool cfsrvl_phyid_match(struct cflayer *layer, int phyid); - -void cfsrvl_init(struct cfsrvl *service, - u8 channel_id, - struct dev_info *dev_info, - bool supports_flowctrl); -bool cfsrvl_ready(struct cfsrvl *service, int *err); - -static inline void cfsrvl_get(struct cflayer *layr) -{ - struct cfsrvl *s = container_of(layr, struct cfsrvl, layer); - if (layr == NULL || layr->up == NULL || s->hold == NULL) - return; - - s->hold(layr->up); -} - -static inline void cfsrvl_put(struct cflayer *layr) -{ - struct cfsrvl *s = container_of(layr, struct cfsrvl, layer); - if (layr == NULL || layr->up == NULL || s->hold == NULL) - return; - - s->put(layr->up); -} -#endif /* CFSRVL_H_ */ diff --git a/include/net/can.h b/include/net/can.h new file mode 100644 index 000000000000..6db9e826f0e0 --- /dev/null +++ b/include/net/can.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ +/* + * net/can.h + * + * Definitions for the CAN network socket buffer extensions + * + * Copyright (C) 2026 Oliver Hartkopp <socketcan@hartkopp.net> + * + */ + +#ifndef _NET_CAN_H +#define _NET_CAN_H + +/** + * struct can_skb_ext - skb extensions for CAN specific content + * @can_iif: ifindex of the first interface the CAN frame appeared on + * @can_framelen: cached echo CAN frame length for bql + * @can_gw_hops: can-gw CAN frame time-to-live counter + * @can_ext_flags: CAN skb extensions flags + */ +struct can_skb_ext { + int can_iif; + u16 can_framelen; + u8 can_gw_hops; + u8 can_ext_flags; +}; + +#endif /* _NET_CAN_H */ diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 2900202588a5..97c16d4ff127 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -7,7 +7,7 @@ * Copyright 2006-2010 Johannes Berg <johannes@sipsolutions.net> * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2015-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2025 Intel Corporation + * Copyright (C) 2018-2026 Intel Corporation */ #include <linux/ethtool.h> @@ -126,6 +126,7 @@ struct wiphy; * @IEEE80211_CHAN_NO_4MHZ: 4 MHz bandwidth is not permitted on this channel. * @IEEE80211_CHAN_NO_8MHZ: 8 MHz bandwidth is not permitted on this channel. * @IEEE80211_CHAN_NO_16MHZ: 16 MHz bandwidth is not permitted on this channel. + * @IEEE80211_CHAN_NO_UHR: UHR operation is not permitted on this channel. */ enum ieee80211_channel_flags { IEEE80211_CHAN_DISABLED = BIT(0), @@ -143,6 +144,7 @@ enum ieee80211_channel_flags { IEEE80211_CHAN_NO_10MHZ = BIT(12), IEEE80211_CHAN_NO_HE = BIT(13), /* can use free bits here */ + IEEE80211_CHAN_NO_UHR = BIT(18), IEEE80211_CHAN_NO_320MHZ = BIT(19), IEEE80211_CHAN_NO_EHT = BIT(20), IEEE80211_CHAN_DFS_CONCURRENT = BIT(21), @@ -188,6 +190,8 @@ enum ieee80211_channel_flags { * on this channel. * @dfs_state_entered: timestamp (jiffies) when the dfs state was entered. * @dfs_cac_ms: DFS CAC time in milliseconds, this is valid for DFS channels. + * @cac_start_time: timestamp (CLOCK_BOOTTIME, nanoseconds) when CAC was + * started on this channel. Zero when CAC is not in progress. * @psd: power spectral density (in dBm) */ struct ieee80211_channel { @@ -205,6 +209,7 @@ struct ieee80211_channel { enum nl80211_dfs_state dfs_state; unsigned long dfs_state_entered; unsigned int dfs_cac_ms; + u64 cac_start_time; s8 psd; }; @@ -429,6 +434,18 @@ struct ieee80211_sta_eht_cap { u8 eht_ppe_thres[IEEE80211_EHT_PPE_THRES_MAX_LEN]; }; +/** + * struct ieee80211_sta_uhr_cap - STA's UHR capabilities + * @has_uhr: true iff UHR is supported and data is valid + * @mac: fixed MAC capabilities + * @phy: fixed PHY capabilities + */ +struct ieee80211_sta_uhr_cap { + bool has_uhr; + struct ieee80211_uhr_cap_mac mac; + struct ieee80211_uhr_cap_phy phy; +}; + /* sparse defines __CHECKER__; see Documentation/dev-tools/sparse.rst */ #ifdef __CHECKER__ /* @@ -454,6 +471,7 @@ struct ieee80211_sta_eht_cap { * @he_6ghz_capa: HE 6 GHz capabilities, must be filled in for a * 6 GHz band channel (and 0 may be valid value). * @eht_cap: STA's EHT capabilities + * @uhr_cap: STA's UHR capabilities * @vendor_elems: vendor element(s) to advertise * @vendor_elems.data: vendor element(s) data * @vendor_elems.len: vendor element(s) length @@ -463,6 +481,7 @@ struct ieee80211_sband_iftype_data { struct ieee80211_sta_he_cap he_cap; struct ieee80211_he_6ghz_capa he_6ghz_capa; struct ieee80211_sta_eht_cap eht_cap; + struct ieee80211_sta_uhr_cap uhr_cap; struct { const u8 *data; unsigned int len; @@ -705,6 +724,26 @@ ieee80211_get_eht_iftype_cap(const struct ieee80211_supported_band *sband, } /** + * ieee80211_get_uhr_iftype_cap - return UHR capabilities for an sband's iftype + * @sband: the sband to search for the iftype on + * @iftype: enum nl80211_iftype + * + * Return: pointer to the struct ieee80211_sta_uhr_cap, or NULL is none found + */ +static inline const struct ieee80211_sta_uhr_cap * +ieee80211_get_uhr_iftype_cap(const struct ieee80211_supported_band *sband, + enum nl80211_iftype iftype) +{ + const struct ieee80211_sband_iftype_data *data = + ieee80211_get_sband_iftype_data(sband, iftype); + + if (data && data->uhr_cap.has_uhr) + return &data->uhr_cap; + + return NULL; +} + +/** * wiphy_read_of_freq_limits - read frequency limits from device tree * * @wiphy: the wireless device to get extra limits for @@ -791,6 +830,8 @@ struct vif_params { * @seq_len: length of @seq. * @vlan_id: vlan_id for VLAN group key (if nonzero) * @mode: key install mode (RX_TX, NO_TX or SET_TX) + * @ltf_keyseed: LTF key seed material + * @ltf_keyseed_len: length of LTF key seed material */ struct key_params { const u8 *key; @@ -800,11 +841,18 @@ struct key_params { u16 vlan_id; u32 cipher; enum nl80211_key_mode mode; + const u8 *ltf_keyseed; + size_t ltf_keyseed_len; }; /** * struct cfg80211_chan_def - channel definition * @chan: the (control) channel + * @npca_chan: the NPCA primary channel + * Note that if DBE is in use, this channel may appear to be + * inside the primary half of the chandef. Implementations + * can use the position of this channel to understand how + * NPCA is used. * @width: channel width * @center_freq1: center frequency of first segment * @center_freq2: center frequency of second segment @@ -817,18 +865,24 @@ struct key_params { * @punctured: mask of the punctured 20 MHz subchannels, with * bits turned on being disabled (punctured); numbered * from lower to higher frequency (like in the spec) + * @npca_punctured: NPCA puncturing bitmap, like @punctured but for + * NPCA transmissions. If NPCA is used (@npca_chan is not %NULL) + * this will be a superset of the @punctured bimap. + * Note that if DBE is used, this bitmap is also shifted to be in + * accordance with the overall chandef bandwidth. * @s1g_primary_2mhz: Indicates if the control channel pointed to * by 'chan' exists as a 1MHz primary subchannel within an * S1G 2MHz primary channel. */ struct cfg80211_chan_def { struct ieee80211_channel *chan; + struct ieee80211_channel *npca_chan; enum nl80211_chan_width width; u32 center_freq1; u32 center_freq2; struct ieee80211_edmg edmg; u16 freq1_offset; - u16 punctured; + u16 punctured, npca_punctured; bool s1g_primary_2mhz; }; @@ -975,7 +1029,9 @@ cfg80211_chandef_identical(const struct cfg80211_chan_def *chandef1, chandef1->freq1_offset == chandef2->freq1_offset && chandef1->center_freq2 == chandef2->center_freq2 && chandef1->punctured == chandef2->punctured && - chandef1->s1g_primary_2mhz == chandef2->s1g_primary_2mhz); + chandef1->s1g_primary_2mhz == chandef2->s1g_primary_2mhz && + chandef1->npca_chan == chandef2->npca_chan && + chandef1->npca_punctured == chandef2->npca_punctured); } /** @@ -1105,6 +1161,46 @@ int cfg80211_chandef_primary(const struct cfg80211_chan_def *chandef, u16 *punctured); /** + * cfg80211_chandef_npca_valid - check that NPCA information is valid + * @wiphy: the wiphy to check for, for channel pointer lookup + * @chandef: the BSS channel chandef to check against + * @npca: NPCA information, can be %NULL in which case this + * always returns %true + * + * Note that DBE must not have been configured into the chandef yet + * before checking NPCA, i.e. @chandef must represent the BSS channel. + * + * Returns: %true if the NPCA channel and puncturing bitmap are valid + * according to the chandef, %false otherwise + */ +bool cfg80211_chandef_npca_valid(struct wiphy *wiphy, + const struct cfg80211_chan_def *chandef, + const struct ieee80211_uhr_npca_info *npca); + +/** + * cfg80211_chandef_add_npca - parse and add NPCA information to chandef + * @wiphy: the wiphy this will be used for, for channel pointer lookup + * @chandef: the chandef to modify, must be a valid chandef without NPCA + * @npca: the NPCA information, can be %NULL + * + * Returns: 0 if the NPCA information was added and the resulting + * chandef is valid, a negative error code on errors + */ +int cfg80211_chandef_add_npca(struct wiphy *wiphy, + struct cfg80211_chan_def *chandef, + const struct ieee80211_uhr_npca_info *npca); + +/** + * cfg80211_chandef_add_dbe - parse and add DBE information to chandef + * @chandef: the chandef to expand + * @dbe: the DBE information, must be size-checked if not %NULL + * + * Returns: 0 for success, a negative error code otherwise + */ +int cfg80211_chandef_add_dbe(struct cfg80211_chan_def *chandef, + const struct ieee80211_uhr_dbe_info *dbe); + +/** * nl80211_send_chandef - sends the channel definition. * @msg: the msg to send channel definition * @chandef: the channel definition to check @@ -1141,6 +1237,26 @@ ieee80211_chandef_max_power(struct cfg80211_chan_def *chandef) } /** + * cfg80211_chandef_s1g_pri_width - return S1G primary width in MHz + * + * An S1G interface may have a primary channel width of either 1 + * or 2MHz depending on whether chandef::s1g_primary_2mhz is set. + * + * Note: There is _always_ a 1MHz primary subchannel, regardless + * of the primary width. So chandef::chan always points to this + * 1MHz primary channel. + * + * @chandef: the chandef to use + * + * Returns: width in MHz of the S1G primary channel in use + */ +static inline int +cfg80211_chandef_s1g_pri_width(struct cfg80211_chan_def *chandef) +{ + return chandef->s1g_primary_2mhz ? 2 : 1; +} + +/** * cfg80211_any_usable_channels - check for usable channels * @wiphy: the wiphy to check for * @band_mask: which bands to check on @@ -1357,6 +1473,13 @@ struct cfg80211_rnr_elems { * @he_bss_color: BSS Color settings * @he_bss_color_valid: indicates whether bss color * attribute is present in beacon data or not. + * @ht_required: stations must support HT + * @vht_required: stations must support VHT + * @ht_oper: HT operation element (or %NULL if HT isn't enabled) + * @vht_oper: VHT operation element (or %NULL if VHT isn't enabled) + * @he_oper: HE operation IE (or %NULL if HE isn't enabled) + * @eht_oper: EHT operation IE (or %NULL if EHT isn't enabled) + * @uhr_oper: UHR operation (or %NULL if UHR isn't enabled) */ struct cfg80211_beacon_data { unsigned int link_id; @@ -1381,6 +1504,13 @@ struct cfg80211_beacon_data { size_t civicloc_len; struct cfg80211_he_bss_color he_bss_color; bool he_bss_color_valid; + + bool ht_required, vht_required; + const struct ieee80211_ht_operation *ht_oper; + const struct ieee80211_vht_operation *vht_oper; + const struct ieee80211_he_operation *he_oper; + const struct ieee80211_eht_operation *eht_oper; + const struct ieee80211_uhr_operation *uhr_oper; }; struct mac_address { @@ -1485,15 +1615,9 @@ struct cfg80211_s1g_short_beacon { * @vht_cap: VHT capabilities (or %NULL if VHT isn't enabled) * @he_cap: HE capabilities (or %NULL if HE isn't enabled) * @eht_cap: EHT capabilities (or %NULL if EHT isn't enabled) - * @eht_oper: EHT operation IE (or %NULL if EHT isn't enabled) - * @ht_required: stations must support HT - * @vht_required: stations must support VHT * @twt_responder: Enable Target Wait Time - * @he_required: stations must support HE - * @sae_h2e_required: stations must support direct H2E technique in SAE * @flags: flags, as defined in &enum nl80211_ap_settings_flags * @he_obss_pd: OBSS Packet Detection settings - * @he_oper: HE operation IE (or %NULL if HE isn't enabled) * @fils_discovery: FILS discovery transmission parameters * @unsol_bcast_probe_resp: Unsolicited broadcast probe response parameters * @mbssid_config: AP settings for multiple bssid @@ -1522,10 +1646,7 @@ struct cfg80211_ap_settings { const struct ieee80211_ht_cap *ht_cap; const struct ieee80211_vht_cap *vht_cap; const struct ieee80211_he_cap_elem *he_cap; - const struct ieee80211_he_operation *he_oper; const struct ieee80211_eht_cap_elem *eht_cap; - const struct ieee80211_eht_operation *eht_oper; - bool ht_required, vht_required, he_required, sae_h2e_required; bool twt_responder; u32 flags; struct ieee80211_he_obss_pd he_obss_pd; @@ -1698,6 +1819,8 @@ struct sta_txpwr { * @eht_capa: EHT capabilities of station * @eht_capa_len: the length of the EHT capabilities * @s1g_capa: S1G capabilities of station + * @uhr_capa: UHR capabilities of the station + * @uhr_capa_len: the length of the UHR capabilities */ struct link_station_parameters { const u8 *mld_mac; @@ -1717,6 +1840,8 @@ struct link_station_parameters { const struct ieee80211_eht_cap_elem *eht_capa; u8 eht_capa_len; const struct ieee80211_s1g_cap *s1g_capa; + const struct ieee80211_uhr_cap *uhr_capa; + u8 uhr_capa_len; }; /** @@ -1785,6 +1910,8 @@ struct cfg80211_ttlm_params { * present/updated * @eml_cap: EML capabilities of this station * @link_sta_params: link related params. + * @epp_peer: EPP peer indication + * @nmi_mac: MAC address of the NMI station of the NAN peer */ struct station_parameters { struct net_device *vlan; @@ -1811,6 +1938,8 @@ struct station_parameters { bool eml_cap_present; u16 eml_cap; struct link_station_parameters link_sta_params; + bool epp_peer; + const u8 *nmi_mac; }; /** @@ -1850,6 +1979,8 @@ struct station_del_parameters { * entry that is operating, has been marked authorized by userspace) * @CFG80211_STA_MESH_PEER_KERNEL: peer on mesh interface (kernel managed) * @CFG80211_STA_MESH_PEER_USER: peer on mesh interface (user managed) + * @CFG80211_STA_NAN_MGMT: NAN management interface station + * @CFG80211_STA_NAN_DATA: NAN data path station */ enum cfg80211_station_type { CFG80211_STA_AP_CLIENT, @@ -1861,6 +1992,8 @@ enum cfg80211_station_type { CFG80211_STA_TDLS_PEER_ACTIVE, CFG80211_STA_MESH_PEER_KERNEL, CFG80211_STA_MESH_PEER_USER, + CFG80211_STA_NAN_MGMT, + CFG80211_STA_NAN_DATA, }; /** @@ -1896,6 +2029,11 @@ int cfg80211_check_station_change(struct wiphy *wiphy, * @RATE_INFO_FLAGS_EXTENDED_SC_DMG: 60GHz extended SC MCS * @RATE_INFO_FLAGS_EHT_MCS: EHT MCS information * @RATE_INFO_FLAGS_S1G_MCS: MCS field filled with S1G MCS + * @RATE_INFO_FLAGS_UHR_MCS: UHR MCS information + * @RATE_INFO_FLAGS_UHR_ELR_MCS: UHR ELR MCS was used + * (set together with @RATE_INFO_FLAGS_UHR_MCS) + * @RATE_INFO_FLAGS_UHR_IM: UHR Interference Mitigation + * was used */ enum rate_info_flags { RATE_INFO_FLAGS_MCS = BIT(0), @@ -1907,6 +2045,9 @@ enum rate_info_flags { RATE_INFO_FLAGS_EXTENDED_SC_DMG = BIT(6), RATE_INFO_FLAGS_EHT_MCS = BIT(7), RATE_INFO_FLAGS_S1G_MCS = BIT(8), + RATE_INFO_FLAGS_UHR_MCS = BIT(9), + RATE_INFO_FLAGS_UHR_ELR_MCS = BIT(10), + RATE_INFO_FLAGS_UHR_IM = BIT(11), }; /** @@ -1922,7 +2063,7 @@ enum rate_info_flags { * @RATE_INFO_BW_160: 160 MHz bandwidth * @RATE_INFO_BW_HE_RU: bandwidth determined by HE RU allocation * @RATE_INFO_BW_320: 320 MHz bandwidth - * @RATE_INFO_BW_EHT_RU: bandwidth determined by EHT RU allocation + * @RATE_INFO_BW_EHT_RU: bandwidth determined by EHT/UHR RU allocation * @RATE_INFO_BW_1: 1 MHz bandwidth * @RATE_INFO_BW_2: 2 MHz bandwidth * @RATE_INFO_BW_4: 4 MHz bandwidth @@ -1953,7 +2094,7 @@ enum rate_info_bw { * * @flags: bitflag of flags from &enum rate_info_flags * @legacy: bitrate in 100kbit/s for 802.11abg - * @mcs: mcs index if struct describes an HT/VHT/HE/EHT/S1G rate + * @mcs: mcs index if struct describes an HT/VHT/HE/EHT/S1G/UHR rate * @nss: number of streams (VHT & HE only) * @bw: bandwidth (from &enum rate_info_bw) * @he_gi: HE guard interval (from &enum nl80211_he_gi) @@ -2682,7 +2823,7 @@ struct mesh_setup { u8 path_metric; u8 auth_id; const u8 *ie; - u8 ie_len; + size_t ie_len; bool is_authenticated; bool is_secure; bool user_mpm; @@ -3260,6 +3401,7 @@ struct cfg80211_ml_reconf_req { * Drivers shall disable MLO features for the current association if this * flag is not set. * @ASSOC_REQ_SPP_AMSDU: SPP A-MSDUs will be used on this connection (if any) + * @ASSOC_REQ_DISABLE_UHR: Disable UHR */ enum cfg80211_assoc_req_flags { ASSOC_REQ_DISABLE_HT = BIT(0), @@ -3270,6 +3412,7 @@ enum cfg80211_assoc_req_flags { ASSOC_REQ_DISABLE_EHT = BIT(5), CONNECT_REQ_MLO_SUPPORT = BIT(6), ASSOC_REQ_SPP_AMSDU = BIT(7), + ASSOC_REQ_DISABLE_UHR = BIT(8), }; /** @@ -3599,6 +3742,7 @@ enum wiphy_params_flags { /* The per TXQ device queue limit in airtime */ #define IEEE80211_DEFAULT_AQL_TXQ_LIMIT_L 5000 #define IEEE80211_DEFAULT_AQL_TXQ_LIMIT_H 12000 +#define IEEE80211_DEFAULT_AQL_TXQ_LIMIT_MC 50000 /* The per interface airtime threshold to switch to lower queue limit */ #define IEEE80211_AQL_THRESHOLD 24000 @@ -3924,6 +4068,77 @@ struct cfg80211_qos_map { }; /** + * DOC: Neighbor Awareness Networking (NAN) + * + * NAN uses two interface types: + * + * - %NL80211_IFTYPE_NAN: a non-netdev interface. This has two roles: (1) holds + * the configuration of all NAN activities (DE parameters, synchronisation + * parameters, local schedule, etc.), and (2) uses as the NAN Management + * Interface (NMI), which is used for NAN management communication. + * + * - %NL80211_IFTYPE_NAN_DATA: The NAN Data Interface (NDI), used for data + * communication with NAN peers. + * + * An NDI interface can only be started (IFF_UP) if the NMI one is running and + * NAN is started. Before NAN is stopped, all associated NDI interfaces + * must be stopped first. + * + * The local schedule specifies which channels the device is available on and + * when. Must be cancelled before NAN is stopped. + * + * NAN Stations + * ~~~~~~~~~~~~ + * + * There are two types of stations corresponding to the two interface types: + * + * - NMI station: Represents the NAN peer. Peer-specific data such as the peer's + * schedule and the HT, VHT and HE capabilities belongs to the NMI station. + * Also used for Tx/Rx of NAN management frames to/from the peer. + * Added on the %NL80211_IFTYPE_NAN interface. + * + * - NDI station: Used for Tx/Rx of data frames (and non-NAN management frames) + * for a specific NDP established with the NAN peer. Added on the + * %NL80211_IFTYPE_NAN_DATA interface. + * + * A peer may reuse its NMI address as the NDI address. In that case, two + * separate stations should be added even though they share the same MAC + * address. + * + * HT, VHT and HE capabilities should not changes after it was set. It is the + * driver's responsibility to check that. + * + * An NDI station can only be added if the corresponding NMI station has already + * been configured with HT (and possibly VHT and HE) capabilities. It is the + * driver's responsibility to check that. + * + * All NDI stations must be removed before corresponding NMI station is removed. + * Therefore, removing a NMI station implies that the associated NDI station(s) + * (if any) will be removed first. + * + * NAN Dependencies + * ~~~~~~~~~~~~~~~~ + * + * The following diagram shows the dependencies between NAN components. + * An arrow from A to B means A must be started/added before B, and B must be + * stopped/removed before A: + * + * +-------------+ + * | NMI iface |---(local schedule) + * +------+------+ + * / \ + * v v + * +-----------+ +-------------+ + * | NDI iface | | NMI sta |---(peer schedule) + * +-----+-----+ +------+------+ + * \ / + * v v + * +----------+ + * | NDI sta | + * +----------+ + */ + +/** * struct cfg80211_nan_band_config - NAN band specific configuration * * @chan: Pointer to the IEEE 802.11 channel structure. The channel to be used @@ -3966,7 +4181,6 @@ struct cfg80211_nan_band_config { * (i.e. BIT(NL80211_BAND_2GHZ)). * @cluster_id: cluster ID used for NAN synchronization. This is a MAC address * that can take a value from 50-6F-9A-01-00-00 to 50-6F-9A-01-FF-FF. - * If NULL, the device will pick a random Cluster ID. * @scan_period: period (in seconds) between NAN scans. * @scan_dwell_time: dwell time (in milliseconds) for NAN scans. * @discovery_beacon_interval: interval (in TUs) for discovery beacons. @@ -3982,7 +4196,7 @@ struct cfg80211_nan_band_config { struct cfg80211_nan_conf { u8 master_pref; u8 bands; - const u8 *cluster_id; + u8 cluster_id[ETH_ALEN] __aligned(2); u16 scan_period; u16 scan_dwell_time; u8 discovery_beacon_interval; @@ -3994,6 +4208,102 @@ struct cfg80211_nan_conf { u16 vendor_elems_len; }; +#define CFG80211_NAN_SCHED_NUM_TIME_SLOTS 32 + +/** + * struct cfg80211_nan_channel - NAN channel configuration + * + * This struct defines a NAN channel configuration + * + * @chandef: the channel definition + * @channel_entry: pointer to the Channel Entry blob as defined in Wi-Fi Aware + * (TM) 4.0 specification Table 100 (Channel Entry format for the NAN + * Availability attribute). + * @rx_nss: number of spatial streams supported on this channel + */ +struct cfg80211_nan_channel { + struct cfg80211_chan_def chandef; + const u8 *channel_entry; + u8 rx_nss; +}; + +/** + * struct cfg80211_nan_local_sched - NAN local schedule + * + * This struct defines NAN local schedule parameters + * + * @schedule: a mapping of time slots to chandef indexes in %nan_channels. + * An unscheduled slot will be set to %NL80211_NAN_SCHED_NOT_AVAIL_SLOT. + * @n_channels: number of channel definitions in %nan_channels. + * @nan_avail_blob: pointer to NAN Availability attribute blob. + * See %NL80211_ATTR_NAN_AVAIL_BLOB for more details. + * @nan_avail_blob_len: length of the @nan_avail_blob in bytes. + * @deferred: if true, the command containing this schedule configuration is a + * request from the device to perform an announced schedule update. This + * means that it needs to send the updated NAN availability to the peers, + * and do the actual switch on the right time (i.e. at the end of the slot + * after the slot in which the updated NAN Availability was sent). + * See %NL80211_ATTR_NAN_SCHED_DEFERRED for more details. + * If false, the schedule is applied immediately. + * @nan_channels: array of NAN channel definitions that can be scheduled. + */ +struct cfg80211_nan_local_sched { + u8 schedule[CFG80211_NAN_SCHED_NUM_TIME_SLOTS]; + u8 n_channels; + const u8 *nan_avail_blob; + u16 nan_avail_blob_len; + bool deferred; + struct cfg80211_nan_channel nan_channels[] __counted_by(n_channels); +}; + +/** + * struct cfg80211_nan_peer_map - NAN peer schedule map + * + * This struct defines a single NAN peer schedule map + * + * @map_id: map ID of this schedule map + * @schedule: a mapping of time slots to chandef indexes in the schedule's + * @nan_channels. Each slot lasts 16TUs. An unscheduled slot will be + * set to %NL80211_NAN_SCHED_NOT_AVAIL_SLOT. + */ +struct cfg80211_nan_peer_map { + u8 map_id; + u8 schedule[CFG80211_NAN_SCHED_NUM_TIME_SLOTS]; +}; + +#define CFG80211_NAN_MAX_PEER_MAPS 2 +#define CFG80211_NAN_INVALID_MAP_ID 0xff + +/** + * struct cfg80211_nan_peer_sched - NAN peer schedule + * + * This struct defines NAN peer schedule parameters for a peer. + * + * @peer_addr: MAC address of the peer (NMI address) + * @seq_id: sequence ID of the peer schedule. + * @committed_dw: committed DW as published by the peer. + * See %NL80211_ATTR_NAN_COMMITTED_DW + * @max_chan_switch: maximum channel switch time in microseconds as published + * by the peer. See %NL80211_ATTR_NAN_MAX_CHAN_SWITCH_TIME. + * @init_ulw: initial ULWs as published by the peer. + * @ulw_size: number of bytes in @init_ulw. + * @n_channels: number of channel definitions in @nan_channels. + * @nan_channels: array of NAN channel definitions for this schedule. + * @maps: array of peer schedule maps. Unused entries have + * map_id = %CFG80211_NAN_INVALID_MAP_ID. + */ +struct cfg80211_nan_peer_sched { + const u8 *peer_addr; + u8 seq_id; + u16 committed_dw; + u16 max_chan_switch; + const u8 *init_ulw; + u16 ulw_size; + u8 n_channels; + struct cfg80211_nan_channel *nan_channels; + struct cfg80211_nan_peer_map maps[CFG80211_NAN_MAX_PEER_MAPS]; +}; + /** * enum cfg80211_nan_conf_changes - indicates changed fields in NAN * configuration @@ -4187,6 +4497,7 @@ struct cfg80211_ftm_responder_stats { * @num_bursts_exp: actual number of bursts exponent negotiated * @burst_duration: actual burst duration negotiated * @ftms_per_burst: actual FTMs per burst negotiated + * @burst_period: actual burst period negotiated in units of 100ms * @lci_len: length of LCI information (if present) * @civicloc_len: length of civic location information (if present) * @lci: LCI data (may be %NULL) @@ -4203,6 +4514,25 @@ struct cfg80211_ftm_responder_stats { * (must have either this or @rtt_avg) * @dist_variance: variance of distances measured (see also @rtt_variance) * @dist_spread: spread of distances measured (see also @rtt_spread) + * @tx_ltf_repetition_count: negotiated value of number of tx ltf repetitions + * in NDP frames + * @rx_ltf_repetition_count: negotiated value of number of rx ltf repetitions + * in NDP frames + * @max_time_between_measurements: the negotiated maximum interval (in units of + * 10 ms) by which the ISTA must complete the next measurement cycle. + * @min_time_between_measurements: the negotiated minimum interval (in units of + * 100 us) between two consecutive range measurements initiated by the + * ISTA. + * @num_tx_spatial_streams: number of Tx space-time streams used in the NDP + * frame during the measurement sounding phase. + * @num_rx_spatial_streams: number of Rx space-time streams used in the NDP + * frame during the measurement sounding phase. + * @nominal_time: negotiated nominal duration between adjacent availability + * windows in units of milliseconds (u32). + * @availability_window: negotiated availability window time used in this + * session in units of milliseconds (u8). + * @chan_width: band width used for measurement. + * @preamble: preamble used for measurement. * @num_ftmr_attempts_valid: @num_ftmr_attempts is valid * @num_ftmr_successes_valid: @num_ftmr_successes is valid * @rssi_avg_valid: @rssi_avg is valid @@ -4215,6 +4545,18 @@ struct cfg80211_ftm_responder_stats { * @dist_avg_valid: @dist_avg is valid * @dist_variance_valid: @dist_variance is valid * @dist_spread_valid: @dist_spread is valid + * @tx_ltf_repetition_count_valid: @tx_ltf_repetition_count is valid + * @rx_ltf_repetition_count_valid: @rx_ltf_repetition_count is valid + * @max_time_between_measurements_valid: @max_time_between_measurements is valid + * @min_time_between_measurements_valid: @min_time_between_measurements is valid + * @num_tx_spatial_streams_valid: @num_tx_spatial_streams is valid + * @num_rx_spatial_streams_valid: @num_rx_spatial_streams is valid + * @nominal_time_valid: @nominal_time is valid + * @availability_window_valid: @availability_window is valid + * @chan_width_valid: @chan_width is valid. + * @preamble_valid: @preamble is valid. + * @is_delayed_lmr: indicates if the reported LMR is of the current burst or the + * previous burst, flag. */ struct cfg80211_pmsr_ftm_result { const u8 *lci; @@ -4228,6 +4570,7 @@ struct cfg80211_pmsr_ftm_result { u8 num_bursts_exp; u8 burst_duration; u8 ftms_per_burst; + u16 burst_period; s32 rssi_avg; s32 rssi_spread; struct rate_info tx_rate, rx_rate; @@ -4237,8 +4580,18 @@ struct cfg80211_pmsr_ftm_result { s64 dist_avg; s64 dist_variance; s64 dist_spread; + u32 tx_ltf_repetition_count; + u32 rx_ltf_repetition_count; + u32 max_time_between_measurements; + u32 min_time_between_measurements; + u8 num_tx_spatial_streams; + u8 num_rx_spatial_streams; + u32 nominal_time; + u8 availability_window; + enum nl80211_chan_width chan_width; + enum nl80211_preamble preamble; - u16 num_ftmr_attempts_valid:1, + u32 num_ftmr_attempts_valid:1, num_ftmr_successes_valid:1, rssi_avg_valid:1, rssi_spread_valid:1, @@ -4249,7 +4602,18 @@ struct cfg80211_pmsr_ftm_result { rtt_spread_valid:1, dist_avg_valid:1, dist_variance_valid:1, - dist_spread_valid:1; + dist_spread_valid:1, + tx_ltf_repetition_count_valid:1, + rx_ltf_repetition_count_valid:1, + max_time_between_measurements_valid:1, + min_time_between_measurements_valid:1, + num_tx_spatial_streams_valid:1, + num_rx_spatial_streams_valid:1, + nominal_time_valid:1, + availability_window_valid:1, + chan_width_valid:1, + preamble_valid:1, + is_delayed_lmr:1; }; /** @@ -4290,8 +4654,11 @@ struct cfg80211_pmsr_result { * @burst_period: burst period to use * @asap: indicates to use ASAP mode * @num_bursts_exp: number of bursts exponent - * @burst_duration: burst duration - * @ftms_per_burst: number of FTMs per burst + * @burst_duration: burst duration. If @trigger_based or @non_trigger_based is + * set, this is the burst duration in milliseconds, and zero means the + * device should pick an appropriate value based on @ftms_per_burst. + * @ftms_per_burst: number of FTMs per burst. If set to 0, the firmware or + * driver can automatically select an appropriate value. * @ftmr_retries: number of retries for FTM request * @request_lci: request LCI information * @request_civicloc: request civic location information @@ -4303,9 +4670,53 @@ struct cfg80211_pmsr_result { * EDCA based ranging will be used. * @lmr_feedback: negotiate for I2R LMR feedback. Only valid if either * @trigger_based or @non_trigger_based is set. + * @rsta: Operate as the RSTA in the measurement. Only valid if @lmr_feedback + * and either @trigger_based or @non_trigger_based is set. * @bss_color: the bss color of the responder. Optional. Set to zero to * indicate the driver should set the BSS color. Only valid if * @non_trigger_based or @trigger_based is set. + * @request_type: ranging request type, one of + * &enum nl80211_peer_measurement_ftm_req_type. Defaults to + * %NL80211_PMSR_FTM_REQ_TYPE_INFRA if not specified. + * @min_time_between_measurements: minimum time between two consecutive range + * measurements in units of 100 microseconds, for non-trigger based + * ranging. Should be set as short as possible to minimize turnaround + * time, since two-way ranging with delayed LMR requires two measurements. + * Only valid if @non_trigger_based is set. + * @max_time_between_measurements: maximum time between two consecutive range + * measurements in units of 10 milliseconds, for non-trigger based + * ranging. Acts as a session timeout; if exceeded, the ranging session + * should be terminated. Only valid if @non_trigger_based is set. + * @availability_window: duration of the availability window (AW) in units of + * 1 millisecond (0-255 ms). Only valid if @non_trigger_based is set. + * If set to 0, the firmware or driver can automatically select an + * appropriate value. + * @nominal_time: Nominal duration between adjacent availability windows + * in units of milli seconds. Only valid if @non_trigger_based is set. + * If set to 0, the firmware or driver can automatically select an + * appropriate value. + * @num_measurements: number of Availability Windows (AWs) to schedule + * for non-trigger-based ranging. Each AW may contain multiple FTM + * exchanges as configured by @ftms_per_burst. Only valid if + * @non_trigger_based is set. If set to 0, the firmware or driver + * can automatically select an appropriate value. + * @ingress_distance: optional ingress threshold in units of mm. When set, + * the measurement result of the peer needs to be indicated if the device + * moves into this range. Measurement results need to be sent on a burst + * index basis in this case. + * @egress_distance: optional egress threshold in units of mm. When set, + * the measurement result of the peer needs to be indicated if the device + * moves out of this range. Measurement results need to be sent on a burst + * index basis in this case. + * If neither or only one of @ingress_distance and @egress_distance + * is set, only the specified threshold is used. If both are set, both + * thresholds are applied. If neither is set, results are reported without + * threshold filtering. + * @pd_suppress_range_results: flag to suppress ranging results for PD + * requests. When set, the device performs ranging measurements to + * provide ranging services to a peer (e.g. in RSTA role) but does + * not report the measurement results to userspace. Only valid when + * @request_type is %NL80211_PMSR_FTM_REQ_TYPE_PD. * * See also nl80211 for the respective attribute documentation. */ @@ -4318,12 +4729,23 @@ struct cfg80211_pmsr_ftm_request_peer { request_civicloc:1, trigger_based:1, non_trigger_based:1, - lmr_feedback:1; + lmr_feedback:1, + rsta:1; u8 num_bursts_exp; u8 burst_duration; u8 ftms_per_burst; u8 ftmr_retries; u8 bss_color; + + u32 request_type; + u32 min_time_between_measurements; + u32 max_time_between_measurements; + u8 availability_window; + u32 nominal_time; + u32 num_measurements; + u64 ingress_distance; + u64 egress_distance; + u8 pd_suppress_range_results:1; }; /** @@ -4685,8 +5107,9 @@ struct mgmt_frame_regs { * @tdls_mgmt: Transmit a TDLS management frame. * @tdls_oper: Perform a high-level TDLS operation (e.g. TDLS link setup). * - * @probe_client: probe an associated client, must return a cookie that it - * later passes to cfg80211_probe_status(). + * @probe_peer: probe a connected peer (AP: STA MAC required; STA: no MAC), + * must use the @cookie as provided which is later passed to + * cfg80211_probe_status(). * * @set_noack_map: Set the NoAck Map for the TIDs. * @@ -4767,6 +5190,19 @@ struct mgmt_frame_regs { * @nan_change_conf: changes NAN configuration. The changed parameters must * be specified in @changes (using &enum cfg80211_nan_conf_changes); * All other parameters must be ignored. + * @nan_set_local_sched: configure the local schedule for NAN. The schedule + * consists of an array of %cfg80211_nan_channel and the schedule itself, + * in which each entry maps each time slot to the channel on which the + * radio should operate on. If the chandef of a NAN channel is not + * changed, the channel entry must also remain unchanged. It is the + * driver's responsibility to verify this. + * @nan_set_peer_sched: configure the peer schedule for NAN. The schedule + * consists of an array of %cfg80211_nan_channel and the schedule itself, + * in which each entry maps each time slot to a channel on which the + * radio should operate on. In addition, it contains more peer's schedule + * information such as committed DW, etc. When updating an existing peer + * schedule, the full new schedule is provided - partial updates are not + * supported, and the new schedule completely replaces the previous one. * * @set_multicast_to_unicast: configure multicast to unicast conversion for BSS * @@ -4784,7 +5220,8 @@ struct mgmt_frame_regs { * user space * * @tx_control_port: TX a control port frame (EAPoL). The noencrypt parameter - * tells the driver that the frame should not be encrypted. + * tells the driver that the frame should not be encrypted. A @cookie + * value of 0 means the caller does not want TX status reporting. * * @get_ftm_responder_stats: Retrieve FTM responder statistics, if available. * Statistics should be cumulative, currently no way to reset is provided. @@ -4836,6 +5273,9 @@ struct mgmt_frame_regs { * links by calling cfg80211_mlo_reconf_add_done(). When calling * cfg80211_mlo_reconf_add_done() the bss pointer must be given for each * link for which MLO reconfiguration 'add' operation was requested. + * + * @start_pd: Start the PD interface. + * @stop_pd: Stop the PD interface. */ struct cfg80211_ops { int (*suspend)(struct wiphy *wiphy, struct cfg80211_wowlan *wow); @@ -4861,24 +5301,24 @@ struct cfg80211_ops { struct wireless_dev *wdev, unsigned int link_id); - int (*add_key)(struct wiphy *wiphy, struct net_device *netdev, + int (*add_key)(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id, u8 key_index, bool pairwise, const u8 *mac_addr, struct key_params *params); - int (*get_key)(struct wiphy *wiphy, struct net_device *netdev, + int (*get_key)(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id, u8 key_index, bool pairwise, const u8 *mac_addr, void *cookie, void (*callback)(void *cookie, struct key_params*)); - int (*del_key)(struct wiphy *wiphy, struct net_device *netdev, + int (*del_key)(struct wiphy *wiphy, struct wireless_dev *wdev, int link_id, u8 key_index, bool pairwise, const u8 *mac_addr); int (*set_default_key)(struct wiphy *wiphy, struct net_device *netdev, int link_id, u8 key_index, bool unicast, bool multicast); int (*set_default_mgmt_key)(struct wiphy *wiphy, - struct net_device *netdev, int link_id, + struct wireless_dev *wdev, int link_id, u8 key_index); int (*set_default_beacon_key)(struct wiphy *wiphy, - struct net_device *netdev, + struct wireless_dev *wdev, int link_id, u8 key_index); @@ -4890,17 +5330,17 @@ struct cfg80211_ops { unsigned int link_id); - int (*add_station)(struct wiphy *wiphy, struct net_device *dev, + int (*add_station)(struct wiphy *wiphy, struct wireless_dev *wdev, const u8 *mac, struct station_parameters *params); - int (*del_station)(struct wiphy *wiphy, struct net_device *dev, + int (*del_station)(struct wiphy *wiphy, struct wireless_dev *wdev, struct station_del_parameters *params); - int (*change_station)(struct wiphy *wiphy, struct net_device *dev, + int (*change_station)(struct wiphy *wiphy, struct wireless_dev *wdev, const u8 *mac, struct station_parameters *params); - int (*get_station)(struct wiphy *wiphy, struct net_device *dev, + int (*get_station)(struct wiphy *wiphy, struct wireless_dev *wdev, const u8 *mac, struct station_info *sinfo); - int (*dump_station)(struct wiphy *wiphy, struct net_device *dev, + int (*dump_station)(struct wiphy *wiphy, struct wireless_dev *wdev, int idx, u8 *mac, struct station_info *sinfo); int (*add_mpath)(struct wiphy *wiphy, struct net_device *dev, @@ -5018,14 +5458,14 @@ struct cfg80211_ops { struct wireless_dev *wdev, struct ieee80211_channel *chan, unsigned int duration, - u64 *cookie); + u64 cookie, const u8 *rx_addr); int (*cancel_remain_on_channel)(struct wiphy *wiphy, struct wireless_dev *wdev, u64 cookie); int (*mgmt_tx)(struct wiphy *wiphy, struct wireless_dev *wdev, struct cfg80211_mgmt_tx_params *params, - u64 *cookie); + u64 cookie); int (*mgmt_tx_cancel_wait)(struct wiphy *wiphy, struct wireless_dev *wdev, u64 cookie); @@ -5071,8 +5511,8 @@ struct cfg80211_ops { int (*tdls_oper)(struct wiphy *wiphy, struct net_device *dev, const u8 *peer, enum nl80211_tdls_operation oper); - int (*probe_client)(struct wiphy *wiphy, struct net_device *dev, - const u8 *peer, u64 *cookie); + int (*probe_peer)(struct wiphy *wiphy, struct net_device *dev, + const u8 *peer, u64 cookie); int (*set_noack_map)(struct wiphy *wiphy, struct net_device *dev, @@ -5144,7 +5584,12 @@ struct cfg80211_ops { struct wireless_dev *wdev, struct cfg80211_nan_conf *conf, u32 changes); - + int (*nan_set_local_sched)(struct wiphy *wiphy, + struct wireless_dev *wdev, + struct cfg80211_nan_local_sched *sched); + int (*nan_set_peer_sched)(struct wiphy *wiphy, + struct wireless_dev *wdev, + struct cfg80211_nan_peer_sched *sched); int (*set_multicast_to_unicast)(struct wiphy *wiphy, struct net_device *dev, const bool enabled); @@ -5165,7 +5610,7 @@ struct cfg80211_ops { const u8 *buf, size_t len, const u8 *dest, const __be16 proto, const bool noencrypt, int link_id, - u64 *cookie); + u64 cookie); int (*get_ftm_responder_stats)(struct wiphy *wiphy, struct net_device *dev, @@ -5207,6 +5652,8 @@ struct cfg80211_ops { struct cfg80211_ml_reconf_req *req); int (*set_epcs)(struct wiphy *wiphy, struct net_device *dev, bool val); + int (*start_pd)(struct wiphy *wiphy, struct wireless_dev *wdev); + void (*stop_pd)(struct wiphy *wiphy, struct wireless_dev *wdev); }; /* @@ -5252,7 +5699,6 @@ struct cfg80211_ops { * responds to probe-requests in hardware. * @WIPHY_FLAG_OFFCHAN_TX: Device supports direct off-channel TX. * @WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL: Device supports remain-on-channel call. - * @WIPHY_FLAG_SUPPORTS_5_10_MHZ: Device supports 5 MHz and 10 MHz channels. * @WIPHY_FLAG_HAS_CHANNEL_SWITCH: Device supports channel switch in * beaconing mode (AP, IBSS, Mesh, ...). * @WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK: The device supports bigger kek and kck keys @@ -5267,7 +5713,6 @@ struct cfg80211_ops { * set this flag to update channels on beacon hints. * @WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY: support connection to non-primary link * of an NSTR mobile AP MLD. - * @WIPHY_FLAG_DISABLE_WEXT: disable wireless extensions for this device */ enum wiphy_flags { WIPHY_FLAG_SUPPORTS_EXT_KEK_KCK = BIT(0), @@ -5279,7 +5724,7 @@ enum wiphy_flags { WIPHY_FLAG_4ADDR_STATION = BIT(6), WIPHY_FLAG_CONTROL_PORT_PROTOCOL = BIT(7), WIPHY_FLAG_IBSS_RSN = BIT(8), - WIPHY_FLAG_DISABLE_WEXT = BIT(9), + /* reuse bit 9 */ WIPHY_FLAG_MESH_AUTH = BIT(10), WIPHY_FLAG_SUPPORTS_EXT_KCK_32 = BIT(11), WIPHY_FLAG_SUPPORTS_NSTR_NONPRIMARY = BIT(12), @@ -5292,7 +5737,6 @@ enum wiphy_flags { WIPHY_FLAG_AP_PROBE_RESP_OFFLOAD = BIT(19), WIPHY_FLAG_OFFCHAN_TX = BIT(20), WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL = BIT(21), - WIPHY_FLAG_SUPPORTS_5_10_MHZ = BIT(22), WIPHY_FLAG_HAS_CHANNEL_SWITCH = BIT(23), WIPHY_FLAG_NOTIFY_REGDOM_BY_DRIVER = BIT(24), WIPHY_FLAG_CHANNEL_CHANGE_ON_BEACON = BIT(25), @@ -5598,6 +6042,7 @@ struct wiphy_vendor_command { * @extended_capabilities_len: length of the extended capabilities * @eml_capabilities: EML capabilities (for MLO) * @mld_capa_and_ops: MLD capabilities and operations (for MLO) + * @ext_mld_capa_and_ops: Extended MLD capabilities and operations (for MLO) */ struct wiphy_iftype_ext_capab { enum nl80211_iftype iftype; @@ -5606,6 +6051,7 @@ struct wiphy_iftype_ext_capab { u8 extended_capabilities_len; u16 eml_capabilities; u16 mld_capa_and_ops; + u16 ext_mld_capa_and_ops; }; /** @@ -5638,6 +6084,64 @@ cfg80211_get_iftype_ext_capa(struct wiphy *wiphy, enum nl80211_iftype type); * not limited) * @ftm.trigger_based: trigger based ranging measurement is supported * @ftm.non_trigger_based: non trigger based ranging measurement is supported + * @ftm.support_6ghz: supports ranging in 6 GHz band + * @ftm.max_tx_ltf_rep: maximum number of TX LTF repetitions supported (0 means + * only one LTF, no repetitions) + * @ftm.max_rx_ltf_rep: maximum number of RX LTF repetitions supported (0 means + * only one LTF, no repetitions) + * @ftm.max_tx_sts: maximum number of TX STS supported (zero based) + * @ftm.max_rx_sts: maximum number of RX STS supported (zero based) + * @ftm.max_total_ltf_tx: maximum total number of LTFs that can be transmitted + * (0 means unknown) + * @ftm.max_total_ltf_rx: maximum total number of LTFs that can be received + * (0 means unknown) + * @ftm.ista: initiator role capabilities + * @ftm.ista.support_ntb: supports operating as ISTA in PMSR FTM request for + * NTB ranging. + * @ftm.ista.support_tb: supports operating as ISTA in PMSR FTM request for + * TB ranging. + * @ftm.ista.support_edca: supports operating as ISTA in PMSR FTM request for + * EDCA based ranging. + * @ftm.ista.max_peers: maximum number of peers supported in the ISTA role. + * If zero, no role-specific peer limit applies. + * @ftm.rsta: responder role capabilities + * @ftm.rsta.support_ntb: supports operating as RSTA in PMSR FTM request for + * NTB ranging. + * @ftm.rsta.support_tb: supports operating as RSTA in PMSR FTM request for + * TB ranging. + * @ftm.rsta.support_edca: supports operating as RSTA in PMSR FTM request for + * EDCA based ranging. + * @ftm.rsta.max_peers: maximum number of peers supported in the RSTA role. + * If zero, no role-specific peer limit applies. + * @ftm.max_no_of_tx_antennas: maximum number of transmit antennas supported for + * EDCA based ranging (0 means unknown) + * @ftm.max_no_of_rx_antennas: maximum number of receive antennas supported for + * EDCA based ranging (0 means unknown) + * @ftm.min_allowed_ranging_interval_edca: Minimum EDCA ranging + * interval supported by the device in milli seconds. (0 means unknown). + * Applications can use this value to estimate the burst period to be + * given in the FTM request for the EDCA based ranging case. If + * non-zero, this value will be used to validate the burst period in + * the FTM request. + * @ftm.min_allowed_ranging_interval_ntb: Minimum NTB ranging + * interval supported by the device in milli seconds. (0 means unknown). + * Applications can use this value to estimate the burst period to be + * given in the FTM request for the NTB ranging case. If non-zero, + * this value will be used to validate the nominal time in the FTM + * request. + * @ftm.type: ranging type capabilities + * @ftm.type.infra_support: supports infrastructure ranging (STA-to-AP or + * AP-to-STA) as part of Proximity Detection + * @ftm.type.pd_support: supports peer-to-peer ranging as mentioned in the + * specification "PR Implementation Consideration Draft 1.9 rev 1" where + * PD stands for proximity detection + * @ftm.concurrent_ista_rsta_support: indicates if the device can + * simultaneously act as initiator and responder in a multi-peer + * measurement request. Only valid if @ftm.rsta_support is set. + * @ftm.pd_preambles: bitmap of preambles supported (&enum nl80211_preamble) + * for PD ranging requests. Ignored if @ftm.type.pd_support is not set. + * @ftm.pd_bandwidths: bitmap of bandwidths supported (&enum nl80211_chan_width) + * for PD ranging requests. Ignored if @ftm.type.pd_support is not set. */ struct cfg80211_pmsr_capabilities { unsigned int max_peers; @@ -5655,7 +6159,37 @@ struct cfg80211_pmsr_capabilities { request_lci:1, request_civicloc:1, trigger_based:1, - non_trigger_based:1; + non_trigger_based:1, + support_6ghz:1; + u8 max_tx_ltf_rep; + u8 max_rx_ltf_rep; + u8 max_tx_sts; + u8 max_rx_sts; + u8 max_total_ltf_tx; + u8 max_total_ltf_rx; + struct { + u8 support_ntb:1, + support_tb:1, + support_edca:1; + u32 max_peers; + } ista; + struct { + u8 support_ntb:1, + support_tb:1, + support_edca:1; + u32 max_peers; + } rsta; + u8 max_no_of_tx_antennas; + u8 max_no_of_rx_antennas; + u32 min_allowed_ranging_interval_edca; + u32 min_allowed_ranging_interval_ntb; + struct { + u8 infra_support:1, + pd_support:1; + } type; + u8 concurrent_ista_rsta_support:1; + u32 pd_preambles; + u32 pd_bandwidths; } ftm; }; @@ -5750,9 +6284,15 @@ enum wiphy_nan_flags { * nibble indicates the number of TX antennas and upper nibble indicates the * number of RX antennas. Value 0 indicates the information is not * available. - * @max_channel_switch_time: maximum channel switch time in milliseconds. + * @max_channel_switch_time: maximum channel switch time in microseconds. * @dev_capabilities: NAN device capabilities as defined in Wi-Fi Aware (TM) * specification Table 79 (Capabilities field). + * @phy: Band-agnostic capabilities for NAN data interfaces. Since NAN + * operates on multiple channels simultaneously, these capabilities apply + * across all bands. Valid only if NL80211_IFTYPE_NAN_DATA is supported. + * @phy.ht: HT capabilities (mandatory for NAN data) + * @phy.vht: VHT capabilities (optional) + * @phy.he: HE capabilities (optional) */ struct wiphy_nan_capa { u32 flags; @@ -5760,6 +6300,11 @@ struct wiphy_nan_capa { u8 n_antennas; u16 max_channel_switch_time; u8 dev_capabilities; + struct { + struct ieee80211_sta_ht_cap ht; + struct ieee80211_sta_vht_cap vht; + struct ieee80211_sta_he_cap he; + } phy; }; #define CFG80211_HW_TIMESTAMP_ALL_PEERS 0xffff @@ -6653,8 +7198,8 @@ enum ieee80211_ap_reg_power { * the P2P Device. * @ps: powersave mode is enabled * @ps_timeout: dynamic powersave timeout - * @ap_unexpected_nlportid: (private) netlink port ID of application - * registered for unexpected class 3 frames (AP mode) + * @unexpected_nlportid: (private) netlink port ID of application + * registered for unexpected frames (AP mode or NAN_DATA mode) * @conn: (private) cfg80211 software SME connection state machine data * @connect_keys: (private) keys to set after connection is established * @conn_bss_type: connecting/connected BSS type @@ -6705,7 +7250,7 @@ struct wireless_dev { enum ieee80211_bss_type conn_bss_type; u32 conn_owner_nlportid; - struct work_struct disconnect_wk; + struct wiphy_work disconnect_wk; u8 disconnect_bssid[ETH_ALEN]; struct list_head event_list; @@ -6716,7 +7261,7 @@ struct wireless_dev { bool ps; int ps_timeout; - u32 ap_unexpected_nlportid; + u32 unexpected_nlportid; u32 owner_nlportid; bool nl_owner_dead; @@ -6742,7 +7287,7 @@ struct wireless_dev { struct list_head pmsr_list; spinlock_t pmsr_lock; - struct work_struct pmsr_free_wk; + struct wiphy_work pmsr_free_wk; unsigned long unprot_beacon_reported; @@ -6776,6 +7321,9 @@ struct wireless_dev { } ocb; struct { u8 cluster_id[ETH_ALEN] __aligned(2); + u8 n_channels; + struct cfg80211_chan_def *chandefs; + bool sched_update_pending; } nan; } u; @@ -7879,6 +8427,8 @@ cfg80211_inform_bss(struct wiphy *wiphy, * @bss_type: type of BSS, see &enum ieee80211_bss_type * @privacy: privacy filter, see &enum ieee80211_privacy * @use_for: indicates which use is intended + * @extack: (optional) extack that is filled with the reason when no + * usable entry was found; may be %NULL * * Return: Reference-counted BSS on success. %NULL on error. */ @@ -7888,7 +8438,8 @@ struct cfg80211_bss *__cfg80211_get_bss(struct wiphy *wiphy, const u8 *ssid, size_t ssid_len, enum ieee80211_bss_type bss_type, enum ieee80211_privacy privacy, - u32 use_for); + u32 use_for, + struct netlink_ext_ack *extack); /** * cfg80211_get_bss - get a BSS reference @@ -7912,7 +8463,7 @@ cfg80211_get_bss(struct wiphy *wiphy, struct ieee80211_channel *channel, { return __cfg80211_get_bss(wiphy, channel, bssid, ssid, ssid_len, bss_type, privacy, - NL80211_BSS_USE_FOR_NORMAL); + NL80211_BSS_USE_FOR_NORMAL, NULL); } static inline struct cfg80211_bss * @@ -8014,6 +8565,7 @@ void cfg80211_auth_timeout(struct net_device *dev, const u8 *addr); * as the AC bitmap in the QoS info field * @req_ies: information elements from the (Re)Association Request frame * @req_ies_len: length of req_ies data + * @assoc_encrypted: indicate if the (re)association exchange is encrypted. * @ap_mld_addr: AP MLD address (in case of MLO) * @links: per-link information indexed by link ID, use links[0] for * non-MLO connections @@ -8028,6 +8580,7 @@ struct cfg80211_rx_assoc_resp_data { const u8 *req_ies; size_t req_ies_len; int uapsd_queues; + bool assoc_encrypted; const u8 *ap_mld_addr; struct { u8 addr[ETH_ALEN] __aligned(2); @@ -8157,7 +8710,7 @@ void cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, * cfg80211 then sends a notification to userspace. */ void cfg80211_notify_new_peer_candidate(struct net_device *dev, - const u8 *macaddr, const u8 *ie, u8 ie_len, + const u8 *macaddr, const u8 *ie, size_t ie_len, int sig_dbm, gfp_t gfp); /** @@ -8547,6 +9100,9 @@ struct cfg80211_fils_resp_params { * @links.status: per-link status code, to report a status code that's not * %WLAN_STATUS_SUCCESS for a given link, it must also be in the * @valid_links bitmap and may have a BSS pointer (which is then released) + * @assoc_encrypted: The driver should set this flag to indicate that the + * (Re)Association Request/Response frames are transmitted encrypted over + * the air. */ struct cfg80211_connect_resp_params { int status; @@ -8556,6 +9112,7 @@ struct cfg80211_connect_resp_params { size_t resp_ie_len; struct cfg80211_fils_resp_params fils; enum nl80211_timeout_reason timeout_reason; + bool assoc_encrypted; const u8 *ap_mld_addr; u16 valid_links; @@ -8881,35 +9438,35 @@ static inline void cfg80211_sinfo_release_content(struct station_info *sinfo) /** * cfg80211_new_sta - notify userspace about station * - * @dev: the netdev + * @wdev: the wireless device * @mac_addr: the station's address * @sinfo: the station information * @gfp: allocation flags */ -void cfg80211_new_sta(struct net_device *dev, const u8 *mac_addr, +void cfg80211_new_sta(struct wireless_dev *wdev, const u8 *mac_addr, struct station_info *sinfo, gfp_t gfp); /** * cfg80211_del_sta_sinfo - notify userspace about deletion of a station - * @dev: the netdev + * @wdev: the wireless device * @mac_addr: the station's address. For MLD station, MLD address is used. * @sinfo: the station information/statistics * @gfp: allocation flags */ -void cfg80211_del_sta_sinfo(struct net_device *dev, const u8 *mac_addr, +void cfg80211_del_sta_sinfo(struct wireless_dev *wdev, const u8 *mac_addr, struct station_info *sinfo, gfp_t gfp); /** * cfg80211_del_sta - notify userspace about deletion of a station * - * @dev: the netdev + * @wdev: the wireless device * @mac_addr: the station's address. For MLD station, MLD address is used. * @gfp: allocation flags */ -static inline void cfg80211_del_sta(struct net_device *dev, +static inline void cfg80211_del_sta(struct wireless_dev *wdev, const u8 *mac_addr, gfp_t gfp) { - cfg80211_del_sta_sinfo(dev, mac_addr, NULL, gfp); + cfg80211_del_sta_sinfo(wdev, mac_addr, NULL, gfp); } /** @@ -9284,9 +9841,10 @@ void cfg80211_pmksa_candidate_notify(struct net_device *dev, int index, * @addr: the transmitter address * @gfp: context flags * - * This function is used in AP mode (only!) to inform userspace that - * a spurious class 3 frame was received, to be able to deauth the - * sender. + * This function is used in AP mode to inform userspace that a spurious + * class 3 frame was received, to be able to deauth the sender. + * It is also used in NAN_DATA mode to report frames from unknown peers + * (A2 not assigned to any active NDP), per Wi-Fi Aware (TM) 4.0 specification 6.2.5. * Return: %true if the frame was passed to userspace (or this failed * for a reason other than not having a subscription.) */ @@ -9313,15 +9871,17 @@ bool cfg80211_rx_unexpected_4addr_frame(struct net_device *dev, const u8 *addr, /** * cfg80211_probe_status - notify userspace about probe status * @dev: the device the probe was sent on - * @addr: the address of the peer - * @cookie: the cookie filled in @probe_client previously + * @peer: The peer MAC address (or MLD address for MLO) or %NULL if not + * applicable (e.g. for STA/P2P-client) + * @cookie: the cookie filled in @probe_peer previously + * @link_id: The link ID on which the probe was sent (or -1 for non-MLO) * @acked: indicates whether probe was acked or not * @ack_signal: signal strength (in dBm) of the ACK frame. * @is_valid_ack_signal: indicates the ack_signal is valid or not. * @gfp: allocation flags */ -void cfg80211_probe_status(struct net_device *dev, const u8 *addr, - u64 cookie, bool acked, s32 ack_signal, +void cfg80211_probe_status(struct net_device *dev, const u8 *peer, u64 cookie, + int link_id, bool acked, s32 ack_signal, bool is_valid_ack_signal, gfp_t gfp); /** @@ -9785,6 +10345,21 @@ int cfg80211_iter_combinations(struct wiphy *wiphy, int cfg80211_get_radio_idx_by_chan(struct wiphy *wiphy, const struct ieee80211_channel *chan); +/** + * cfg80211_stop_link - stop AP/P2P_GO link if link_id is non-negative or stops + * all links on the interface. + * + * @wiphy: the wiphy + * @wdev: wireless device + * @link_id: valid link ID in case of MLO AP/P2P_GO Operation or else -1 + * @gfp: context flags + * + * If link_id is set during MLO operation, stops only the specified AP/P2P_GO + * link and if link_id is set to -1 or last link is stopped, the entire + * interface is stopped as if AP was stopped, IBSS/mesh left, STA disconnected. + */ +void cfg80211_stop_link(struct wiphy *wiphy, struct wireless_dev *wdev, + int link_id, gfp_t gfp); /** * cfg80211_stop_iface - trigger interface disconnection @@ -9798,8 +10373,11 @@ int cfg80211_get_radio_idx_by_chan(struct wiphy *wiphy, * * Note: This doesn't need any locks and is asynchronous. */ -void cfg80211_stop_iface(struct wiphy *wiphy, struct wireless_dev *wdev, - gfp_t gfp); +static inline void +cfg80211_stop_iface(struct wiphy *wiphy, struct wireless_dev *wdev, gfp_t gfp) +{ + cfg80211_stop_link(wiphy, wdev, -1, gfp); +} /** * cfg80211_shutdown_all_interfaces - shut down all interfaces for a wiphy @@ -9915,6 +10493,18 @@ void cfg80211_nan_func_terminated(struct wireless_dev *wdev, enum nl80211_nan_func_term_reason reason, u64 cookie, gfp_t gfp); +/** + * cfg80211_nan_sched_update_done - notify deferred schedule update completion + * @wdev: the wireless device reporting the event + * @success: whether or not the schedule update was successful + * @gfp: allocation flags + * + * This function notifies user space that a deferred local NAN schedule update + * (requested with %NL80211_ATTR_NAN_SCHED_DEFERRED) has been completed. + */ +void cfg80211_nan_sched_update_done(struct wireless_dev *wdev, bool success, + gfp_t gfp); + /* ethtool helper */ void cfg80211_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info); @@ -10147,9 +10737,9 @@ cfg80211_6ghz_power_type(u8 control, u32 client_flags) case IEEE80211_6GHZ_CTRL_REG_LPI_AP: case IEEE80211_6GHZ_CTRL_REG_INDOOR_LPI_AP: case IEEE80211_6GHZ_CTRL_REG_AP_ROLE_NOT_RELEVANT: + case IEEE80211_6GHZ_CTRL_REG_INDOOR_SP_AP_OLD: return IEEE80211_REG_LPI_AP; case IEEE80211_6GHZ_CTRL_REG_SP_AP: - case IEEE80211_6GHZ_CTRL_REG_INDOOR_SP_AP_OLD: return IEEE80211_REG_SP_AP; case IEEE80211_6GHZ_CTRL_REG_VLP_AP: return IEEE80211_REG_VLP_AP; @@ -10255,6 +10845,39 @@ void cfg80211_nan_cluster_joined(struct wireless_dev *wdev, const u8 *cluster_id, bool new_cluster, gfp_t gfp); +/** + * cfg80211_nan_ulw_update - Notify user space about ULW update + * @wdev: Pointer to the wireless device structure + * @ulw: Pointer to the ULW blob data + * @ulw_len: Length of the ULW blob in bytes + * @gfp: Memory allocation flags + * + * This function is used by drivers to notify user space when the device's + * ULW (Unaligned Schedule) blob has been updated. User space can use this + * blob to attach to frames sent to peers. + */ +void cfg80211_nan_ulw_update(struct wireless_dev *wdev, + const u8 *ulw, size_t ulw_len, gfp_t gfp); + +/** + * cfg80211_nan_channel_evac - Notify user space about NAN channel evacuation + * @wdev: Pointer to the wireless device structure + * @chandef: Pointer to the channel definition of the NAN channel that was + * evacuated + * @gfp: Memory allocation flags + * + * This function is used by drivers to notify user space when a NAN + * channel has been evacuated (i.e. ULWed) due to channel resource conflicts + * with other interfaces. + * This can happen when another interface sharing the channel resource with NAN + * needs to move to a different channel (e.g. due to channel switch or link + * switch). User space may reconfigure the local schedule to exclude the + * evacuated channel. + */ +void cfg80211_nan_channel_evac(struct wireless_dev *wdev, + const struct cfg80211_chan_def *chandef, + gfp_t gfp); + #ifdef CONFIG_CFG80211_DEBUGFS /** * wiphy_locked_debugfs_read - do a locked read in debugfs @@ -10373,4 +10996,27 @@ cfg80211_s1g_get_primary_sibling(struct wiphy *wiphy, return ieee80211_get_channel_khz(wiphy, sibling_1mhz_khz); } + +/** + * cfg80211_incumbent_signal_notify - Notify userspace of incumbent signal detection + * @wiphy: the wiphy to use + * @chandef: channel definition in which the interference was detected + * @signal_interference_bitmap: bitmap indicating interference across 20 MHz segments + * @gfp: allocation context for message creation and multicast; pass GFP_ATOMIC + * if called from atomic context (e.g. firmware event handler), otherwise + * GFP_KERNEL + * + * Use this function to notify userspace when an incumbent signal is detected on + * the operating channel in the 6 GHz band. The notification includes the + * current channel definition and a bitmap representing interference across + * the operating bandwidth. Each bit in the bitmap corresponds to a 20 MHz + * segment, with the lowest bit representing the lowest frequency segment. + * Punctured sub-channels are included in the bitmap structure but are always + * set to zero since interference detection is not performed on them. + */ +void cfg80211_incumbent_signal_notify(struct wiphy *wiphy, + const struct cfg80211_chan_def *chandef, + u32 signal_interference_bitmap, + gfp_t gfp); + #endif /* __NET_CFG80211_H */ diff --git a/include/net/cfg802154.h b/include/net/cfg802154.h index 76d2cd2e2b30..2e960441ea49 100644 --- a/include/net/cfg802154.h +++ b/include/net/cfg802154.h @@ -376,6 +376,7 @@ struct cfg802154_mac_pkt { struct list_head node; struct sk_buff *skb; struct ieee802154_sub_if_data *sdata; + netdevice_tracker dev_tracker; u8 page; u8 channel; }; diff --git a/include/net/codel.h b/include/net/codel.h index aa80f744826c..183d43c2bd43 100644 --- a/include/net/codel.h +++ b/include/net/codel.h @@ -140,6 +140,11 @@ struct codel_vars { /* needed shift to get a Q0.32 number from rec_inv_sqrt */ #define REC_INV_SQRT_SHIFT (32 - REC_INV_SQRT_BITS) +/* Cap on drops per codel_dequeue() call: the loop's work depends on the + * idle gap and backlog, both outside our control; resync when exceeded. + */ +#define CODEL_MAX_DROPS_PER_DEQUEUE 256 + /** * struct codel_stats - contains codel shared variables and stats * @maxpacket: largest packet we've seen so far diff --git a/include/net/codel_impl.h b/include/net/codel_impl.h index 78a27ac73070..8f26132d45b7 100644 --- a/include/net/codel_impl.h +++ b/include/net/codel_impl.h @@ -93,12 +93,17 @@ static void codel_Newton_step(struct codel_vars *vars) * CoDel control_law is t + interval/sqrt(count) * We maintain in rec_inv_sqrt the reciprocal value of sqrt(count) to avoid * both sqrt() and divide operation. + * + * Clamp the increment to at least 1 tick: a very small interval (or a + * large count) can truncate it to zero, stalling the dropping loop. */ static codel_time_t codel_control_law(codel_time_t t, codel_time_t interval, u32 rec_inv_sqrt) { - return t + reciprocal_scale(interval, rec_inv_sqrt << REC_INV_SQRT_SHIFT); + return t + max_t(u32, 1, + reciprocal_scale(interval, + rec_inv_sqrt << REC_INV_SQRT_SHIFT)); } static bool codel_should_drop(const struct sk_buff *skb, @@ -120,10 +125,10 @@ static bool codel_should_drop(const struct sk_buff *skb, } skb_len = skb_len_func(skb); - vars->ldelay = now - skb_time_func(skb); + WRITE_ONCE(vars->ldelay, now - skb_time_func(skb)); if (unlikely(skb_len > stats->maxpacket)) - stats->maxpacket = skb_len; + WRITE_ONCE(stats->maxpacket, skb_len); if (codel_time_before(vars->ldelay, params->target) || *backlog <= params->mtu) { @@ -154,11 +159,13 @@ static struct sk_buff *codel_dequeue(void *ctx, codel_skb_dequeue_t dequeue_func) { struct sk_buff *skb = dequeue_func(vars, ctx); + unsigned int drops = 0; codel_time_t now; bool drop; if (!skb) { - vars->dropping = false; + vars->first_above_time = 0; + WRITE_ONCE(vars->dropping, false); return skb; } now = codel_get_time(); @@ -167,7 +174,7 @@ static struct sk_buff *codel_dequeue(void *ctx, if (vars->dropping) { if (!drop) { /* sojourn time below target - leave dropping state */ - vars->dropping = false; + WRITE_ONCE(vars->dropping, false); } else if (codel_time_after_eq(now, vars->drop_next)) { /* It's time for the next drop. Drop the current * packet and dequeue the next. The dequeue might @@ -179,16 +186,26 @@ static struct sk_buff *codel_dequeue(void *ctx, */ while (vars->dropping && codel_time_after_eq(now, vars->drop_next)) { - vars->count++; /* dont care of possible wrap - * since there is no more divide - */ + if (++drops > CODEL_MAX_DROPS_PER_DEQUEUE) { + /* fell far behind the schedule */ + WRITE_ONCE(vars->drop_next, + codel_control_law(now, + params->interval, + vars->rec_inv_sqrt)); + break; + } + /* dont care of possible wrap + * since there is no more divide. + */ + WRITE_ONCE(vars->count, vars->count + 1); codel_Newton_step(vars); if (params->ecn && INET_ECN_set_ce(skb)) { - stats->ecn_mark++; - vars->drop_next = + WRITE_ONCE(stats->ecn_mark, + stats->ecn_mark + 1); + WRITE_ONCE(vars->drop_next, codel_control_law(vars->drop_next, params->interval, - vars->rec_inv_sqrt); + vars->rec_inv_sqrt)); goto end; } stats->drop_len += skb_len_func(skb); @@ -201,13 +218,13 @@ static struct sk_buff *codel_dequeue(void *ctx, skb_time_func, backlog, now)) { /* leave dropping state */ - vars->dropping = false; + WRITE_ONCE(vars->dropping, false); } else { /* and schedule the next drop */ - vars->drop_next = + WRITE_ONCE(vars->drop_next, codel_control_law(vars->drop_next, params->interval, - vars->rec_inv_sqrt); + vars->rec_inv_sqrt)); } } } @@ -215,7 +232,7 @@ static struct sk_buff *codel_dequeue(void *ctx, u32 delta; if (params->ecn && INET_ECN_set_ce(skb)) { - stats->ecn_mark++; + WRITE_ONCE(stats->ecn_mark, stats->ecn_mark + 1); } else { stats->drop_len += skb_len_func(skb); drop_func(skb, ctx); @@ -226,7 +243,7 @@ static struct sk_buff *codel_dequeue(void *ctx, stats, skb_len_func, skb_time_func, backlog, now); } - vars->dropping = true; + WRITE_ONCE(vars->dropping, true); /* if min went above target close to when we last went below it * assume that the drop rate that controlled the queue on the * last cycle is a good starting point to control it now. @@ -235,19 +252,20 @@ static struct sk_buff *codel_dequeue(void *ctx, if (delta > 1 && codel_time_before(now - vars->drop_next, 16 * params->interval)) { - vars->count = delta; + WRITE_ONCE(vars->count, delta); /* we dont care if rec_inv_sqrt approximation * is not very precise : * Next Newton steps will correct it quadratically. */ codel_Newton_step(vars); } else { - vars->count = 1; + WRITE_ONCE(vars->count, 1); vars->rec_inv_sqrt = ~0U >> REC_INV_SQRT_SHIFT; } - vars->lastcount = vars->count; - vars->drop_next = codel_control_law(now, params->interval, - vars->rec_inv_sqrt); + WRITE_ONCE(vars->lastcount, vars->count); + WRITE_ONCE(vars->drop_next, + codel_control_law(now, params->interval, + vars->rec_inv_sqrt)); } end: if (skb && codel_time_after(vars->ldelay, params->ce_threshold)) { @@ -261,7 +279,7 @@ end: params->ce_threshold_selector)); } if (set_ce && INET_ECN_set_ce(skb)) - stats->ce_mark++; + WRITE_ONCE(stats->ce_mark, stats->ce_mark + 1); } return skb; } diff --git a/include/net/devlink.h b/include/net/devlink.h index cb839e0435a1..7abd23376319 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -36,7 +36,7 @@ struct devlink_port_phys_attrs { * struct devlink_port_pci_pf_attrs - devlink port's PCI PF attributes * @controller: Associated controller number * @pf: associated PCI function number for the devlink port instance - * @external: when set, indicates if a port is for an external controller + * @external: when set, indicates if a port is for an external host controller. */ struct devlink_port_pci_pf_attrs { u32 controller; @@ -50,7 +50,7 @@ struct devlink_port_pci_pf_attrs { * @pf: associated PCI function number for the devlink port instance * @vf: associated PCI VF number of a PF for the devlink port instance; * VF number starts from 0 for the first PCI virtual function - * @external: when set, indicates if a port is for an external controller + * @external: when set, indicates if a port is for an external host controller. */ struct devlink_port_pci_vf_attrs { u32 controller; @@ -64,7 +64,7 @@ struct devlink_port_pci_vf_attrs { * @controller: Associated controller number * @sf: associated SF number of a PF for the devlink port instance * @pf: associated PCI function number for the devlink port instance - * @external: when set, indicates if a port is for an external controller + * @external: when set, indicates if a port is for an external host controller. */ struct devlink_port_pci_sf_attrs { u32 controller; @@ -129,6 +129,7 @@ struct devlink_rate { struct devlink_port { struct list_head list; struct list_head region_list; + struct list_head resource_list; struct devlink *devlink; const struct devlink_port_ops *ops; unsigned int index; @@ -432,6 +433,13 @@ enum devlink_param_type { DEVLINK_PARAM_TYPE_U64 = DEVLINK_VAR_ATTR_TYPE_U64, DEVLINK_PARAM_TYPE_STRING = DEVLINK_VAR_ATTR_TYPE_STRING, DEVLINK_PARAM_TYPE_BOOL = DEVLINK_VAR_ATTR_TYPE_FLAG, + DEVLINK_PARAM_TYPE_U64_ARRAY = DEVLINK_VAR_ATTR_TYPE_U64_ARRAY, +}; + +#define __DEVLINK_PARAM_MAX_ARRAY_SIZE 32 +struct devlink_param_u64_array { + u64 size; + u64 val[__DEVLINK_PARAM_MAX_ARRAY_SIZE]; }; union devlink_param_value { @@ -441,6 +449,7 @@ union devlink_param_value { u64 vu64; char vstr[__DEVLINK_PARAM_MAX_STRING_VALUE]; bool vbool; + struct devlink_param_u64_array u64arr; }; struct devlink_param_gset_ctx { @@ -500,7 +509,7 @@ struct devlink_param { struct devlink_param_gset_ctx *ctx, struct netlink_ext_ack *extack); int (*validate)(struct devlink *devlink, u32 id, - union devlink_param_value val, + union devlink_param_value *val, struct netlink_ext_ack *extack); int (*get_default)(struct devlink *devlink, u32 id, struct devlink_param_gset_ctx *ctx, @@ -545,6 +554,7 @@ enum devlink_param_generic_id { DEVLINK_PARAM_GENERIC_ID_TOTAL_VFS, DEVLINK_PARAM_GENERIC_ID_NUM_DOORBELLS, DEVLINK_PARAM_GENERIC_ID_MAX_MAC_PER_VF, + DEVLINK_PARAM_GENERIC_ID_MAX_SFS, /* add new param generic ids above here*/ __DEVLINK_PARAM_GENERIC_ID_MAX, @@ -618,6 +628,9 @@ enum devlink_param_generic_id { #define DEVLINK_PARAM_GENERIC_MAX_MAC_PER_VF_NAME "max_mac_per_vf" #define DEVLINK_PARAM_GENERIC_MAX_MAC_PER_VF_TYPE DEVLINK_PARAM_TYPE_U32 +#define DEVLINK_PARAM_GENERIC_MAX_SFS_NAME "max_sfs" +#define DEVLINK_PARAM_GENERIC_MAX_SFS_TYPE DEVLINK_PARAM_TYPE_U32 + #define DEVLINK_PARAM_GENERIC(_id, _cmodes, _get, _set, _validate) \ { \ .id = DEVLINK_PARAM_GENERIC_ID_##_id, \ @@ -1585,6 +1598,15 @@ struct devlink_ops { struct devlink_rate *parent, void *priv_child, void *priv_parent, struct netlink_ext_ack *extack); + /* Indicates if cross-device rate nodes are supported. + * This also requires a shared common ancestor object all devices that + * could share rate nodes are nested in. + * If enabled, rate operations may be called on an instance with only + * the common ancestor lock held and *without that instance lock held*. + * It is the driver's responsibility to ensure proper serialization + * with other operations. + */ + bool supported_cross_device_rate_nodes; /** * selftests_check() - queries if selftest is supported * @devlink: devlink instance @@ -1611,6 +1633,9 @@ struct devlink_ops { void *devlink_priv(struct devlink *devlink); struct devlink *priv_to_devlink(void *priv); struct device *devlink_to_dev(const struct devlink *devlink); +const char *devlink_bus_name(const struct devlink *devlink); +const char *devlink_dev_name(const struct devlink *devlink); +const char *devlink_dev_driver_name(const struct devlink *devlink); /* Devlink instance explicit locking */ void devl_lock(struct devlink *devlink); @@ -1644,6 +1669,13 @@ void devlink_register(struct devlink *devlink); void devlink_unregister(struct devlink *devlink); void devlink_free(struct devlink *devlink); +struct devlink *devlink_shd_get(const char *id, + const struct devlink_ops *ops, + size_t priv_size, + const struct device_driver *driver); +void devlink_shd_put(struct devlink *devlink); +void *devlink_shd_get_priv(struct devlink *devlink); + /** * struct devlink_port_ops - Port operations * @port_split: Callback used to split the port into multiple ones. @@ -1875,12 +1907,19 @@ int devl_resource_register(struct devlink *devlink, u64 resource_size, u64 resource_id, u64 parent_resource_id, - const struct devlink_resource_size_params *size_params); + const struct devlink_resource_size_params *params); void devl_resources_unregister(struct devlink *devlink); void devlink_resources_unregister(struct devlink *devlink); int devl_resource_size_get(struct devlink *devlink, u64 resource_id, u64 *p_resource_size); +int +devl_port_resource_register(struct devlink_port *devlink_port, + const char *resource_name, + u64 resource_size, u64 resource_id, + u64 parent_resource_id, + const struct devlink_resource_size_params *params); +void devl_port_resources_unregister(struct devlink_port *devlink_port); int devl_dpipe_table_resource_set(struct devlink *devlink, const char *table_name, u64 resource_id, u64 resource_units); @@ -1905,7 +1944,7 @@ void devlink_params_unregister(struct devlink *devlink, int devl_param_driverinit_value_get(struct devlink *devlink, u32 param_id, union devlink_param_value *val); void devl_param_driverinit_value_set(struct devlink *devlink, u32 param_id, - union devlink_param_value init_val); + union devlink_param_value *init_val); void devl_param_value_changed(struct devlink *devlink, u32 param_id); struct devlink_region *devl_region_create(struct devlink *devlink, const struct devlink_region_ops *ops, diff --git a/include/net/dropreason-core.h b/include/net/dropreason-core.h index a7b7abd66e21..2f312d1f67d6 100644 --- a/include/net/dropreason-core.h +++ b/include/net/dropreason-core.h @@ -68,18 +68,15 @@ FN(SECURITY_HOOK) \ FN(QDISC_DROP) \ FN(QDISC_BURST_DROP) \ - FN(QDISC_OVERLIMIT) \ - FN(QDISC_CONGESTED) \ - FN(CAKE_FLOOD) \ - FN(FQ_BAND_LIMIT) \ - FN(FQ_HORIZON_LIMIT) \ - FN(FQ_FLOW_LIMIT) \ FN(CPU_BACKLOG) \ + FN(MACVLAN_BROADCAST_BACKLOG) \ + FN(IPVLAN_MULTICAST_BACKLOG) \ FN(XDP) \ FN(TC_INGRESS) \ FN(UNHANDLED_PROTO) \ FN(SKB_CSUM) \ FN(SKB_GSO_SEG) \ + FN(SKB_BAD_GSO) \ FN(SKB_UCOPY_FAULT) \ FN(DEV_HDR) \ FN(DEV_READY) \ @@ -102,6 +99,7 @@ FN(FRAG_TOO_FAR) \ FN(TCP_MINTTL) \ FN(IPV6_BAD_EXTHDR) \ + FN(IPV6_TOO_MANY_EXTHDRS) \ FN(IPV6_NDISC_FRAG) \ FN(IPV6_NDISC_HOP_LIMIT) \ FN(IPV6_NDISC_BAD_CODE) \ @@ -127,9 +125,9 @@ FN(CANFD_RX_INVALID_FRAME) \ FN(CANXL_RX_INVALID_FRAME) \ FN(PFMEMALLOC) \ - FN(DUALPI2_STEP_DROP) \ FN(PSP_INPUT) \ FN(PSP_OUTPUT) \ + FN(RECURSION_LIMIT) \ FNe(MAX) /** @@ -371,8 +369,10 @@ enum skb_drop_reason { /** @SKB_DROP_REASON_SECURITY_HOOK: dropped due to security HOOK */ SKB_DROP_REASON_SECURITY_HOOK, /** - * @SKB_DROP_REASON_QDISC_DROP: dropped by qdisc when packet outputting ( - * failed to enqueue to current qdisc) + * @SKB_DROP_REASON_QDISC_DROP: dropped by qdisc during enqueue or + * dequeue. More specific drop reasons are available via the + * qdisc:qdisc_drop tracepoint, which also provides qdisc handle + * and name for identifying the source. */ SKB_DROP_REASON_QDISC_DROP, /** @@ -381,41 +381,21 @@ enum skb_drop_reason { */ SKB_DROP_REASON_QDISC_BURST_DROP, /** - * @SKB_DROP_REASON_QDISC_OVERLIMIT: dropped by qdisc when a qdisc - * instance exceeds its total buffer size limit. - */ - SKB_DROP_REASON_QDISC_OVERLIMIT, - /** - * @SKB_DROP_REASON_QDISC_CONGESTED: dropped by a qdisc AQM algorithm - * due to congestion. - */ - SKB_DROP_REASON_QDISC_CONGESTED, - /** - * @SKB_DROP_REASON_CAKE_FLOOD: dropped by the flood protection part of - * CAKE qdisc AQM algorithm (BLUE). - */ - SKB_DROP_REASON_CAKE_FLOOD, - /** - * @SKB_DROP_REASON_FQ_BAND_LIMIT: dropped by fq qdisc when per band - * limit is reached. - */ - SKB_DROP_REASON_FQ_BAND_LIMIT, - /** - * @SKB_DROP_REASON_FQ_HORIZON_LIMIT: dropped by fq qdisc when packet - * timestamp is too far in the future. - */ - SKB_DROP_REASON_FQ_HORIZON_LIMIT, - /** - * @SKB_DROP_REASON_FQ_FLOW_LIMIT: dropped by fq qdisc when a flow - * exceeds its limits. - */ - SKB_DROP_REASON_FQ_FLOW_LIMIT, - /** * @SKB_DROP_REASON_CPU_BACKLOG: failed to enqueue the skb to the per CPU * backlog queue. This can be caused by backlog queue full (see * netdev_max_backlog in net.rst) or RPS flow limit */ SKB_DROP_REASON_CPU_BACKLOG, + /** + * @SKB_DROP_REASON_MACVLAN_BROADCAST_BACKLOG: failed to enqueue the skb + * to macvlan broadcast queue. + */ + SKB_DROP_REASON_MACVLAN_BROADCAST_BACKLOG, + /** + * @SKB_DROP_REASON_IPVLAN_MULTICAST_BACKLOG: failed to enqueue the skb + * to ipvlan multicast queue. + */ + SKB_DROP_REASON_IPVLAN_MULTICAST_BACKLOG, /** @SKB_DROP_REASON_XDP: dropped by XDP in input path */ SKB_DROP_REASON_XDP, /** @SKB_DROP_REASON_TC_INGRESS: dropped in TC ingress HOOK */ @@ -426,6 +406,8 @@ enum skb_drop_reason { SKB_DROP_REASON_SKB_CSUM, /** @SKB_DROP_REASON_SKB_GSO_SEG: gso segmentation error */ SKB_DROP_REASON_SKB_GSO_SEG, + /** @SKB_DROP_REASON_SKB_BAD_GSO: malicious gso packet. */ + SKB_DROP_REASON_SKB_BAD_GSO, /** * @SKB_DROP_REASON_SKB_UCOPY_FAULT: failed to copy data from user space, * e.g., via zerocopy_sg_from_iter() or skb_orphan_frags_rx() @@ -513,6 +495,11 @@ enum skb_drop_reason { SKB_DROP_REASON_TCP_MINTTL, /** @SKB_DROP_REASON_IPV6_BAD_EXTHDR: Bad IPv6 extension header. */ SKB_DROP_REASON_IPV6_BAD_EXTHDR, + /** + * @SKB_DROP_REASON_IPV6_TOO_MANY_EXTHDRS: Number of IPv6 extension + * headers in the packet exceeds IP6_MAX_EXT_HDRS_CNT. + */ + SKB_DROP_REASON_IPV6_TOO_MANY_EXTHDRS, /** @SKB_DROP_REASON_IPV6_NDISC_FRAG: invalid frag (suppress_frag_ndisc). */ SKB_DROP_REASON_IPV6_NDISC_FRAG, /** @SKB_DROP_REASON_IPV6_NDISC_HOP_LIMIT: invalid hop limit. */ @@ -613,15 +600,12 @@ enum skb_drop_reason { * reached a path or socket not eligible for use of memory reserves */ SKB_DROP_REASON_PFMEMALLOC, - /** - * @SKB_DROP_REASON_DUALPI2_STEP_DROP: dropped by the step drop - * threshold of DualPI2 qdisc. - */ - SKB_DROP_REASON_DUALPI2_STEP_DROP, /** @SKB_DROP_REASON_PSP_INPUT: PSP input checks failed */ SKB_DROP_REASON_PSP_INPUT, /** @SKB_DROP_REASON_PSP_OUTPUT: PSP output checks failed */ SKB_DROP_REASON_PSP_OUTPUT, + /** @SKB_DROP_REASON_RECURSION_LIMIT: Dead loop on virtual device. */ + SKB_DROP_REASON_RECURSION_LIMIT, /** * @SKB_DROP_REASON_MAX: the maximum of core drop reasons, which * shouldn't be used as a real 'reason' - only for tracing code gen diff --git a/include/net/dropreason-qdisc.h b/include/net/dropreason-qdisc.h new file mode 100644 index 000000000000..fb151cd31751 --- /dev/null +++ b/include/net/dropreason-qdisc.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef _LINUX_DROPREASON_QDISC_H +#define _LINUX_DROPREASON_QDISC_H +#include <net/dropreason.h> + +#define DEFINE_QDISC_DROP_REASON(FN, FNe) \ + FN(UNSPEC) \ + FN(GENERIC) \ + FN(OVERLIMIT) \ + FN(CONGESTED) \ + FN(MAXFLOWS) \ + FN(FLOOD_PROTECTION) \ + FN(BAND_LIMIT) \ + FN(HORIZON_LIMIT) \ + FN(FLOW_LIMIT) \ + FN(L4S_STEP_NON_ECN) \ + FNe(MAX) + +#undef FN +#undef FNe +#define FN(reason) QDISC_DROP_##reason, +#define FNe(reason) QDISC_DROP_##reason + +/** + * enum qdisc_drop_reason - reason why a qdisc dropped a packet + * + * Qdisc-specific drop reasons for packet drops that occur within the + * traffic control (TC) queueing discipline layer. These reasons provide + * detailed diagnostics about why packets were dropped by various qdisc + * algorithms, enabling fine-grained monitoring and troubleshooting of + * queue behavior. + */ +enum qdisc_drop_reason { + /** + * @QDISC_DROP_UNSPEC: unspecified/invalid qdisc drop reason. + * Value 0 serves as analogous to SKB_NOT_DROPPED_YET for enum skb_drop_reason. + * Used for catching zero-initialized drop_reason fields. + */ + QDISC_DROP_UNSPEC = 0, + /** + * @__QDISC_DROP_REASON: subsystem base value for qdisc drop reasons + */ + __QDISC_DROP_REASON = SKB_DROP_REASON_SUBSYS_QDISC << + SKB_DROP_REASON_SUBSYS_SHIFT, + /** + * @QDISC_DROP_GENERIC: generic/default qdisc drop, used when no + * more specific reason applies + */ + QDISC_DROP_GENERIC, + /** + * @QDISC_DROP_OVERLIMIT: packet dropped because the qdisc queue + * length exceeded its configured limit (sch->limit). This typically + * indicates the queue is full and cannot accept more packets. + */ + QDISC_DROP_OVERLIMIT, + /** + * @QDISC_DROP_CONGESTED: packet dropped due to active congestion + * control algorithms (e.g., CoDel, PIE, RED) detecting network + * congestion. The qdisc proactively dropped the packet to signal + * congestion to the sender and prevent bufferbloat. + */ + QDISC_DROP_CONGESTED, + /** + * @QDISC_DROP_MAXFLOWS: packet dropped because the qdisc's flow + * tracking table is full and no free slots are available to allocate + * for a new flow. This indicates flow table exhaustion in flow-based + * qdiscs that maintain per-flow state (e.g., SFQ). + */ + QDISC_DROP_MAXFLOWS, + /** + * @QDISC_DROP_FLOOD_PROTECTION: packet dropped by flood protection + * mechanism detecting unresponsive flows (potential DoS/flood). + * Used by qdiscs implementing probabilistic drop algorithms like + * BLUE (e.g., CAKE's Cobalt AQM). + */ + QDISC_DROP_FLOOD_PROTECTION, + /** + * @QDISC_DROP_BAND_LIMIT: packet dropped because the priority band's + * limit was reached. Used by qdiscs with priority bands that have + * per-band packet limits (e.g., FQ). + */ + QDISC_DROP_BAND_LIMIT, + /** + * @QDISC_DROP_HORIZON_LIMIT: packet dropped because its timestamp + * is too far in the future (beyond the configured horizon). + * Used by qdiscs with time-based scheduling (e.g., FQ). + */ + QDISC_DROP_HORIZON_LIMIT, + /** + * @QDISC_DROP_FLOW_LIMIT: packet dropped because an individual flow + * exceeded its per-flow packet/depth limit. Used by FQ and SFQ qdiscs + * to enforce per-flow fairness and prevent a single flow from + * monopolizing queue resources. + */ + QDISC_DROP_FLOW_LIMIT, + /** + * @QDISC_DROP_L4S_STEP_NON_ECN: DualPI2 qdisc dropped a non-ECN-capable + * packet because the L4S queue delay exceeded the step threshold. + * Since the packet cannot be ECN-marked, it must be dropped to signal + * congestion. See RFC 9332 for the DualQ Coupled AQM step mechanism. + */ + QDISC_DROP_L4S_STEP_NON_ECN, + /** + * @QDISC_DROP_MAX: the maximum of qdisc drop reasons, which + * shouldn't be used as a real 'reason' - only for tracing code gen + */ + QDISC_DROP_MAX, +}; + +#undef FN +#undef FNe + +#endif diff --git a/include/net/dropreason.h b/include/net/dropreason.h index 7d3b1a2a6fec..1df60645fb27 100644 --- a/include/net/dropreason.h +++ b/include/net/dropreason.h @@ -23,6 +23,12 @@ enum skb_drop_reason_subsys { */ SKB_DROP_REASON_SUBSYS_OPENVSWITCH, + /** + * @SKB_DROP_REASON_SUBSYS_QDISC: TC qdisc drop reasons, + * see include/net/dropreason-qdisc.h + */ + SKB_DROP_REASON_SUBSYS_QDISC, + /** @SKB_DROP_REASON_SUBSYS_NUM: number of subsystems defined */ SKB_DROP_REASON_SUBSYS_NUM }; diff --git a/include/net/dsa.h b/include/net/dsa.h index 6b2b5ed64ea4..7507d632e7c6 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -57,6 +57,10 @@ struct tc_action; #define DSA_TAG_PROTO_BRCM_LEGACY_FCS_VALUE 29 #define DSA_TAG_PROTO_YT921X_VALUE 30 #define DSA_TAG_PROTO_MXL_GSW1XX_VALUE 31 +#define DSA_TAG_PROTO_MXL862_VALUE 32 +#define DSA_TAG_PROTO_NETC_VALUE 33 +#define DSA_TAG_PROTO_KSZ8463_VALUE 34 +#define DSA_TAG_PROTO_MT7628_VALUE 35 enum dsa_tag_protocol { DSA_TAG_PROTO_NONE = DSA_TAG_PROTO_NONE_VALUE, @@ -91,6 +95,10 @@ enum dsa_tag_protocol { DSA_TAG_PROTO_VSC73XX_8021Q = DSA_TAG_PROTO_VSC73XX_8021Q_VALUE, DSA_TAG_PROTO_YT921X = DSA_TAG_PROTO_YT921X_VALUE, DSA_TAG_PROTO_MXL_GSW1XX = DSA_TAG_PROTO_MXL_GSW1XX_VALUE, + DSA_TAG_PROTO_MXL862 = DSA_TAG_PROTO_MXL862_VALUE, + DSA_TAG_PROTO_NETC = DSA_TAG_PROTO_NETC_VALUE, + DSA_TAG_PROTO_KSZ8463 = DSA_TAG_PROTO_KSZ8463_VALUE, + DSA_TAG_PROTO_MT7628 = DSA_TAG_PROTO_MT7628_VALUE, }; struct dsa_switch; @@ -216,12 +224,6 @@ struct dsa_mall_mirror_tc_entry { bool ingress; }; -/* TC port policer entry */ -struct dsa_mall_policer_tc_entry { - u32 burst; - u64 rate_bytes_per_sec; -}; - /* TC matchall entry */ struct dsa_mall_tc_entry { struct list_head list; @@ -229,7 +231,7 @@ struct dsa_mall_tc_entry { enum dsa_port_mall_action_type type; union { struct dsa_mall_mirror_tc_entry mirror; - struct dsa_mall_policer_tc_entry policer; + struct flow_action_police policer; }; }; @@ -835,6 +837,22 @@ dsa_tree_offloads_bridge_dev(struct dsa_switch_tree *dst, return false; } +#define dsa_switch_for_each_bridge_member(_dp, _ds, _bdev) \ + dsa_switch_for_each_user_port(_dp, _ds) \ + if (dsa_port_offloads_bridge_dev(_dp, _bdev)) + +static inline u32 +dsa_bridge_ports(struct dsa_switch *ds, const struct net_device *bdev) +{ + struct dsa_port *dp; + u32 mask = 0; + + dsa_switch_for_each_bridge_member(dp, ds, bdev) + mask |= BIT(dp->index); + + return mask; +} + static inline bool dsa_port_tree_same(const struct dsa_port *a, const struct dsa_port *b) { @@ -1110,7 +1128,8 @@ struct dsa_switch_ops { void (*port_mirror_del)(struct dsa_switch *ds, int port, struct dsa_mall_mirror_tc_entry *mirror); int (*port_policer_add)(struct dsa_switch *ds, int port, - struct dsa_mall_policer_tc_entry *policer); + const struct flow_action_police *policer, + struct netlink_ext_ack *extack); void (*port_policer_del)(struct dsa_switch *ds, int port); int (*port_setup_tc)(struct dsa_switch *ds, int port, enum tc_setup_type type, void *type_data); diff --git a/include/net/dst.h b/include/net/dst.h index f8aa1239b4db..dbedfe72e1fd 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -219,6 +219,12 @@ static inline u32 dst_mtu(const struct dst_entry *dst) return INDIRECT_CALL_INET(dst->ops->mtu, ip6_mtu, ipv4_mtu, dst); } +/* Variant of dst_mtu() for IPv4 users. */ +static inline u32 dst4_mtu(const struct dst_entry *dst) +{ + return INDIRECT_CALL_1(dst->ops->mtu, ipv4_mtu, dst); +} + /* RTT metrics are stored in milliseconds for user ABI, but used as jiffies */ static inline unsigned long dst_metric_rtt(const struct dst_entry *dst, int metric) { @@ -449,7 +455,8 @@ static inline unsigned int dst_dev_overhead(struct dst_entry *dst, struct sk_buff *skb) { if (likely(dst)) - return LL_RESERVED_SPACE(dst->dev); + return max_t(unsigned int, skb->mac_len, + LL_RESERVED_SPACE(dst->dev)); return skb->mac_len; } diff --git a/include/net/dst_metadata.h b/include/net/dst_metadata.h index 1fc2fb03ce3f..f45d1e3163f0 100644 --- a/include/net/dst_metadata.h +++ b/include/net/dst_metadata.h @@ -164,8 +164,11 @@ static inline struct metadata_dst *tun_dst_unclone(struct sk_buff *skb) if (!new_md) return ERR_PTR(-ENOMEM); - memcpy(&new_md->u.tun_info, &md_dst->u.tun_info, - sizeof(struct ip_tunnel_info) + md_size); + /* Copy in two stages to keep the __counted_by happy. */ + new_md->u.tun_info = md_dst->u.tun_info; + memcpy(ip_tunnel_info_opts(&new_md->u.tun_info), + ip_tunnel_info_opts(&md_dst->u.tun_info), md_size); + #ifdef CONFIG_DST_CACHE /* Unclone the dst cache if there is one */ if (new_md->u.tun_info.dst_cache.cache) { diff --git a/include/net/fib_rules.h b/include/net/fib_rules.h index 6e68e359ad18..c6b94790fa81 100644 --- a/include/net/fib_rules.h +++ b/include/net/fib_rules.h @@ -82,7 +82,7 @@ struct fib_rules_ops { struct fib_rule_hdr *, struct nlattr **, struct netlink_ext_ack *); - int (*delete)(struct fib_rule *); + void (*delete)(struct fib_rule *); int (*compare)(struct fib_rule *, struct fib_rule_hdr *, struct nlattr **); @@ -93,11 +93,13 @@ struct fib_rules_ops { /* Called after modifications to the rules set, must flush * the route cache if one exists. */ void (*flush_cache)(struct fib_rules_ops *ops); + bool (*need_rtnl)(struct net *net); int nlgroup; struct list_head rules_list; struct module *owner; struct net *fro_net; + struct mutex lock; struct rcu_head rcu; }; @@ -111,6 +113,11 @@ static inline void fib_rule_get(struct fib_rule *rule) refcount_inc(&rule->refcnt); } +static inline bool fib_rule_get_safe(struct fib_rule *rule) +{ + return refcount_inc_not_zero(&rule->refcnt); +} + static inline void fib_rule_put(struct fib_rule *rule) { if (refcount_dec_and_test(&rule->refcnt)) diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h index 596ab9791e4d..70a02ee14308 100644 --- a/include/net/flow_offload.h +++ b/include/net/flow_offload.h @@ -231,6 +231,21 @@ struct flow_action_cookie *flow_action_cookie_create(void *data, gfp_t gfp); void flow_action_cookie_destroy(struct flow_action_cookie *cookie); +struct flow_action_police { + u32 burst; + u64 rate_bytes_ps; + u64 peakrate_bytes_ps; + u32 avrate; + u16 overhead; + u64 burst_pkt; + u64 rate_pkt_ps; + u32 mtu; + struct { + enum flow_action_id act_id; + u32 extval; + } exceed, notexceed; +}; + struct flow_action_entry { enum flow_action_id id; u32 hw_index; @@ -275,20 +290,7 @@ struct flow_action_entry { u32 trunc_size; bool truncate; } sample; - struct { /* FLOW_ACTION_POLICE */ - u32 burst; - u64 rate_bytes_ps; - u64 peakrate_bytes_ps; - u32 avrate; - u16 overhead; - u64 burst_pkt; - u64 rate_pkt_ps; - u32 mtu; - struct { - enum flow_action_id act_id; - u32 extval; - } exceed, notexceed; - } police; + struct flow_action_police police; /* FLOW_ACTION_POLICE */ struct { /* FLOW_ACTION_CT */ int action; u16 zone; @@ -526,7 +528,7 @@ static inline bool flow_rule_has_enc_control_flags(const u32 enc_ctrl_flags, * * Return: true if control flags are set, false otherwise. */ -static inline bool flow_rule_match_has_control_flags(struct flow_rule *rule, +static inline bool flow_rule_match_has_control_flags(const struct flow_rule *rule, struct netlink_ext_ack *extack) { struct flow_match_control match; @@ -718,7 +720,7 @@ struct flow_offload_action { struct flow_offload_action *offload_action_alloc(unsigned int num_actions); static inline struct flow_rule * -flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) +flow_cls_offload_flow_rule(const struct flow_cls_offload *flow_cmd) { return flow_cmd->rule; } diff --git a/include/net/fq_impl.h b/include/net/fq_impl.h index 9467e33dfb36..8aca881937da 100644 --- a/include/net/fq_impl.h +++ b/include/net/fq_impl.h @@ -358,7 +358,7 @@ static int fq_init(struct fq *fq, int flows_cnt) fq->limit = 8192; fq->memory_limit = 16 << 20; /* 16 MBytes */ - fq->flows = kvcalloc(fq->flows_cnt, sizeof(fq->flows[0]), GFP_KERNEL); + fq->flows = kvzalloc_objs(fq->flows[0], fq->flows_cnt); if (!fq->flows) return -ENOMEM; diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h index 7aa2b8e1fb29..856b5e90fbdb 100644 --- a/include/net/gen_stats.h +++ b/include/net/gen_stats.h @@ -47,19 +47,19 @@ int gnet_stats_start_copy_compat(struct sk_buff *skb, int type, int padattr); int gnet_stats_copy_basic(struct gnet_dump *d, - struct gnet_stats_basic_sync __percpu *cpu, - struct gnet_stats_basic_sync *b, bool running); + const struct gnet_stats_basic_sync __percpu *cpu, + const struct gnet_stats_basic_sync *b, bool running); void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats, - struct gnet_stats_basic_sync __percpu *cpu, - struct gnet_stats_basic_sync *b, bool running); + const struct gnet_stats_basic_sync __percpu *cpu, + const struct gnet_stats_basic_sync *b, bool running); int gnet_stats_copy_basic_hw(struct gnet_dump *d, - struct gnet_stats_basic_sync __percpu *cpu, - struct gnet_stats_basic_sync *b, bool running); + const struct gnet_stats_basic_sync __percpu *cpu, + const struct gnet_stats_basic_sync *b, bool running); int gnet_stats_copy_rate_est(struct gnet_dump *d, struct net_rate_estimator __rcu **ptr); int gnet_stats_copy_queue(struct gnet_dump *d, - struct gnet_stats_queue __percpu *cpu_q, - struct gnet_stats_queue *q, __u32 qlen); + const struct gnet_stats_queue __percpu *cpu_q, + const struct gnet_stats_queue *q, __u32 qlen); void gnet_stats_add_queue(struct gnet_stats_queue *qstats, const struct gnet_stats_queue __percpu *cpu_q, const struct gnet_stats_queue *q); diff --git a/include/net/genetlink.h b/include/net/genetlink.h index 7b84f2cef8b1..d70510ac31ab 100644 --- a/include/net/genetlink.h +++ b/include/net/genetlink.h @@ -489,8 +489,10 @@ genlmsg_multicast_netns_filtered(const struct genl_family *family, netlink_filter_fn filter, void *filter_data) { - if (WARN_ON_ONCE(group >= family->n_mcgrps)) + if (WARN_ON_ONCE(group >= family->n_mcgrps)) { + nlmsg_free(skb); return -EINVAL; + } group = family->mcgrp_offset + group; return nlmsg_multicast_filtered(net->genl_sock, skb, portid, group, flags, filter, filter_data); diff --git a/include/net/geneve.h b/include/net/geneve.h index 5c96827a487e..ba2c14d61e90 100644 --- a/include/net/geneve.h +++ b/include/net/geneve.h @@ -68,9 +68,4 @@ static inline bool netif_is_geneve(const struct net_device *dev) !strcmp(dev->rtnl_link_ops->kind, "geneve"); } -#ifdef CONFIG_INET -struct net_device *geneve_dev_create_fb(struct net *net, const char *name, - u8 name_assign_type, u16 dst_port); -#endif /*ifdef CONFIG_INET */ - #endif /*ifdef__NET_GENEVE_H */ diff --git a/include/net/gre.h b/include/net/gre.h index ccd293203284..b55f67ecd2fc 100644 --- a/include/net/gre.h +++ b/include/net/gre.h @@ -32,8 +32,6 @@ struct gre_protocol { int gre_add_protocol(const struct gre_protocol *proto, u8 version); int gre_del_protocol(const struct gre_protocol *proto, u8 version); -struct net_device *gretap_fb_dev_create(struct net *net, const char *name, - u8 name_assign_type); int gre_parse_header(struct sk_buff *skb, struct tnl_ptk_info *tpi, bool *csum_err, __be16 proto, int nhs); diff --git a/include/net/gro.h b/include/net/gro.h index b65f631c521d..2300b6da05b2 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -405,9 +405,8 @@ INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, struct sk_buff *)); INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int)); -INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *, - struct sk_buff *)); -INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int)); +struct sk_buff *udp6_gro_receive(struct list_head *, struct sk_buff *); +int udp6_gro_complete(struct sk_buff *, int); #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \ ({ \ diff --git a/include/net/gue.h b/include/net/gue.h index dfca298bec9c..d377155fd0b3 100644 --- a/include/net/gue.h +++ b/include/net/gue.h @@ -80,12 +80,13 @@ static inline size_t guehdr_flags_len(__be16 flags) static inline size_t guehdr_priv_flags_len(__be32 flags) { - return 0; + return (flags & GUE_PFLAG_REMCSUM) ? GUE_PLEN_REMCSUM : 0; } /* Validate standard and private flags. Returns non-zero (meaning invalid) - * if there is an unknown standard or private flags, or the options length for - * the flags exceeds the options length specific in hlen of the GUE header. + * if there is an unknown standard or private flags, if the options length for + * the flags exceeds the options length specified in hlen of the GUE header, or + * if a private option contains invalid data. */ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen) { @@ -103,8 +104,8 @@ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen) /* Private flags are last four bytes accounted in * guehdr_flags_len */ - __be32 pflags = *(__be32 *)((void *)&guehdr[1] + - len - GUE_LEN_PRIV); + void *data = (void *)&guehdr[1] + len; + __be32 pflags = *(__be32 *)(data - GUE_LEN_PRIV); if (pflags & ~GUE_PFLAGS_ALL) return 1; @@ -112,6 +113,16 @@ static inline int validate_gue_flags(struct guehdr *guehdr, size_t optlen) len += guehdr_priv_flags_len(pflags); if (len > optlen) return 1; + + if (pflags & GUE_PFLAG_REMCSUM) { + __be16 *pd = data; + + /* The field offset pd[1] must not be less + * than the start pd[0]. + */ + if (ntohs(pd[1]) < ntohs(pd[0])) + return 1; + } } return 0; diff --git a/include/net/hotdata.h b/include/net/hotdata.h index 6632b1aa7584..62534d1f3c70 100644 --- a/include/net/hotdata.h +++ b/include/net/hotdata.h @@ -6,6 +6,9 @@ #include <linux/types.h> #include <linux/netdevice.h> #include <net/protocol.h> +#ifdef CONFIG_RPS +#include <net/rps-types.h> +#endif struct skb_defer_node { struct llist_head defer_list; @@ -33,7 +36,7 @@ struct net_hotdata { struct kmem_cache *skbuff_fclone_cache; struct kmem_cache *skb_small_head_cache; #ifdef CONFIG_RPS - struct rps_sock_flow_table __rcu *rps_sock_flow_table; + rps_tag_ptr rps_sock_flow_table; u32 rps_cpu_mask; #endif struct skb_defer_node __percpu *skb_defer_nodes; diff --git a/include/net/ieee80211_radiotap.h b/include/net/ieee80211_radiotap.h index c60867e7e43c..8bbaf77da7cf 100644 --- a/include/net/ieee80211_radiotap.h +++ b/include/net/ieee80211_radiotap.h @@ -95,6 +95,8 @@ enum ieee80211_radiotap_presence { IEEE80211_RADIOTAP_EXT = 31, IEEE80211_RADIOTAP_EHT_USIG = 33, IEEE80211_RADIOTAP_EHT = 34, + IEEE80211_RADIOTAP_UHR_ELR = 37, + IEEE80211_RADIOTAP_UHR = 38, }; /* for IEEE80211_RADIOTAP_FLAGS */ @@ -602,6 +604,194 @@ enum ieee80211_radiotap_eht_usig_tb { IEEE80211_RADIOTAP_EHT_USIG2_TB_B20_B25_TAIL = 0xfc000000, }; +/* + * ieee80211_radiotap_uhr_elr - content of UHR-ELR TLV (type 37) + * see https://www.radiotap.org/fields/UHR-ELR for details + */ +struct ieee80211_radiotap_uhr_elr { + __le32 known; + __le32 sig1, sig2, mark; +} __packed; + +enum ieee80211_radiotap_uhr_elr_known { + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_VERSION_ID = 0x00000001, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_UL_DL = 0x00000002, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_MCS = 0x00000004, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_CODING = 0x00000008, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_LENGTH = 0x00000010, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_LDPC_EXTRA_OFDM_SYM = 0x00000020, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_1_CRC = 0x00000040, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_1_TAIL = 0x00000080, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_STA_ID = 0x00000100, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_DISREGARD = 0x00000200, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_2_CRC = 0x00000400, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_2_TAIL = 0x00000800, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_1_CRC_CHECKED = 0x00001000, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_SIG_2_CRC_CHECKED = 0x00002000, + IEEE80211_RADIOTAP_UHR_ELR_KNOWN_MARK_BSS_COLOR = 0x00010000, +}; + +enum ieee80211_radiotap_uhr_elr_sig1 { + IEEE80211_RADIOTAP_UHR_ELR_SIG1_VERSION_ID = 0x00000001, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_UL_DL = 0x00000002, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_MCS = 0x00000004, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_CODING = 0x00000008, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_LENGTH = 0x00001FF0, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_LDPC_EXTRA_OFDM_SYM = 0x00002000, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_CRC = 0x0003C000, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_TAIL = 0x00FC0000, + IEEE80211_RADIOTAP_UHR_ELR_SIG1_CRC_VALID = 0x80000000, +}; + +enum ieee80211_radiotap_uhr_elr_sig2 { + IEEE80211_RADIOTAP_UHR_ELR_SIG2_STA_ID = 0x000007FF, + IEEE80211_RADIOTAP_UHR_ELR_SIG2_DISREGARD = 0x00003800, + IEEE80211_RADIOTAP_UHR_ELR_SIG2_CRC = 0x0003C000, + IEEE80211_RADIOTAP_UHR_ELR_SIG2_TAIL = 0x00FC0000, + IEEE80211_RADIOTAP_UHR_ELR_SIG2_CRC_VALID = 0x80000000, +}; + +enum ieee80211_radiotap_uhr_elr_mark { + IEEE80211_RADIOTAP_UHR_ELR_MARK_BSS_COLOR = 0x0000003F, +}; + +/* + * ieee80211_radiotap_uhr - content of UHR TLV (type 38) + * see https://www.radiotap.org/fields/UHR for details + */ +struct ieee80211_radiotap_uhr { + __le32 known; + __le32 data[9]; + struct { + __le32 known, info; + } user[]; +} __packed; + +enum ieee80211_radiotap_uhr_known { + IEEE80211_RADIOTAP_UHR_KNOWN_SPATIAL_REUSE = 0x00000001, + IEEE80211_RADIOTAP_UHR_KNOWN_GI_LTF_SIZE = 0x00000002, + IEEE80211_RADIOTAP_UHR_KNOWN_NUMBER_OF_UHR_LTF_SYMBOLS = 0x00000004, + IEEE80211_RADIOTAP_UHR_KNOWN_LDPC_EXTRA_SYMBOL_SEGMENT = 0x00000008, + IEEE80211_RADIOTAP_UHR_KNOWN_PRE_FEC_PADDING_FACTOR = 0x00000010, + IEEE80211_RADIOTAP_UHR_KNOWN_PE_DISAMBIGUITY = 0x00000020, + IEEE80211_RADIOTAP_UHR_KNOWN_DISREGARD_OFDMA = 0x00000040, + IEEE80211_RADIOTAP_UHR_KNOWN_CRC1 = 0x00000080, + IEEE80211_RADIOTAP_UHR_KNOWN_TAIL1 = 0x00000100, + IEEE80211_RADIOTAP_UHR_KNOWN_CRC2 = 0x00000200, + IEEE80211_RADIOTAP_UHR_KNOWN_TAIL2 = 0x00000400, + IEEE80211_RADIOTAP_UHR_KNOWN_INTERFERENCE_MITIGATION = 0x00000800, + IEEE80211_RADIOTAP_UHR_KNOWN_DISREGARD_NON_OFDMA = 0x00001000, + IEEE80211_RADIOTAP_UHR_KNOWN_NUMBER_OF_NON_OFDMA_USERS = 0x00002000, + IEEE80211_RADIOTAP_UHR_KNOWN_COMMON_ENCODING_BLOCK_CRC = 0x00004000, + IEEE80211_RADIOTAP_UHR_KNOWN_COMMON_ENCODING_BLOCK_TAIL = 0x00008000, + IEEE80211_RADIOTAP_UHR_KNOWN_RU_MRU_DRU_SIZE = 0x00010000, + IEEE80211_RADIOTAP_UHR_KNOWN_RU_MRU_INDEX = 0x00020000, + IEEE80211_RADIOTAP_UHR_KNOWN_DRU_RRU_ALLOC_TB_FMT = 0x00040000, + IEEE80211_RADIOTAP_UHR_KNOWN_PRI80_CHAN_POS = 0x00080000, +}; + +enum ieee80211_radiotap_uhr_data { + /* data[0] */ + IEEE80211_RADIOTAP_UHR_DATA0_SPATIAL_REUSE = 0x0000000F, + IEEE80211_RADIOTAP_UHR_DATA0_GI_LTF_SIZE = 0x00000030, + IEEE80211_RADIOTAP_UHR_DATA0_NUMBER_OF_LTF_SYMBOLS = 0x00000700, + IEEE80211_RADIOTAP_UHR_DATA0_LDPC_EXTRA_SYMBOL_SEGMENT = 0x00000800, + IEEE80211_RADIOTAP_UHR_DATA0_PRE_FEC_PADDING_FACTOR = 0x00003000, + IEEE80211_RADIOTAP_UHR_DATA0_PE_DISAMBIGUITY = 0x00004000, + IEEE80211_RADIOTAP_UHR_DATA0_DISREGARD_OFDMA = 0x00078000, + IEEE80211_RADIOTAP_UHR_DATA0_CRC1 = 0x00780000, + IEEE80211_RADIOTAP_UHR_DATA0_TAIL1 = 0x1f800000, + /* data[1] */ + IEEE80211_RADIOTAP_UHR_DATA1_RU_MRU_DRU_SIZE = 0x0000001f, + IEEE80211_RADIOTAP_UHR_DATA1_RU_MRU_INDEX = 0x00001fe0, + IEEE80211_RADIOTAP_UHR_DATA1_RU_ALLOC_CC_1_1_1 = 0x003fe000, + IEEE80211_RADIOTAP_UHR_DATA1_RU_ALLOC_CC_1_1_1_KNOWN = 0x00400000, + IEEE80211_RADIOTAP_UHR_DATA1_PRI80_CHAN_POS = 0xc0000000, + /* data[2] */ + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_1 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_1_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_1_1_2 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_1_1_2_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_2 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA2_RU_ALLOC_CC_2_1_2_KNOWN = 0x20000000, + /* data[3] */ + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_1 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_1_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_2_2_1 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_2_2_1_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_2 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA3_RU_ALLOC_CC_1_2_2_KNOWN = 0x20000000, + /* data[4] */ + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_2 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_2_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_1_2_3 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_1_2_3_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_3 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA4_RU_ALLOC_CC_2_2_3_KNOWN = 0x20000000, + /* data[5] */ + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_4 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_4_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_2_2_4 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_2_2_4_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_5 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA5_RU_ALLOC_CC_1_2_5_KNOWN = 0x20000000, + /* data[6] */ + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_5 = 0x000001ff, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_5_KNOWN = 0x00000200, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_1_2_6 = 0x0007fc00, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_1_2_6_KNOWN = 0x00080000, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_6 = 0x1ff00000, + IEEE80211_RADIOTAP_UHR_DATA6_RU_ALLOC_CC_2_2_6_KNOWN = 0x20000000, + /* data[7] */ + IEEE80211_RADIOTAP_UHR_DATA7_CRC2 = 0x0000000f, + IEEE80211_RADIOTAP_UHR_DATA7_TAIL2 = 0x000003f0, + IEEE80211_RADIOTAP_UHR_DATA7_INTERFERENCE_MITIGATION = 0x00000400, + IEEE80211_RADIOTAP_UHR_DATA7_DISREGARD_NON_OFDMA = 0x00001800, + IEEE80211_RADIOTAP_UHR_DATA7_NUMBER_OF_NON_OFDMA_USERS = 0x0000e000, + IEEE80211_RADIOTAP_UHR_DATA7_COMMON_ENCODING_BLOCK_CRC = 0x000f0000, + IEEE80211_RADIOTAP_UHR_DATA7_COMMON_ENCODING_BLOCK_TAIL = 0x03f00000, + /* data[8] */ + IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_ALLOC_TB_FMT_PS_160= 0x00000001, + IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_ALLOC_TB_FMT_B0 = 0x00000002, + IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_ALLOC_TB_FMT_B7_B1 = 0x000001fc, + IEEE80211_RADIOTAP_UHR_DATA8_DRU_RRU_INDICATION = 0x00000200, +}; + +enum ieee80211_radiotap_uhr_user_known { + IEEE80211_RADIOTAP_UHR_USER_KNOWN_STA_ID = 0x00000001, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_MCS = 0x00000002, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_NSS = 0x00000004, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_UEQM = 0x00000008, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_BF = 0x00000010, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_CODING = 0x00000020, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_UEQM_PATTERN = 0x00000040, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_2X_LDPC = 0x00000080, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_SPATIAL_CONFIG = 0x00000100, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_DISREGARD = 0x00000200, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_BSS_COLOR_INDICATION = 0x00000400, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_USR_ENC_BLK_CRC = 0x00000800, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_USR_ENC_BLK_TAIL = 0x00001000, + /* really 'known' but actual data */ + IEEE80211_RADIOTAP_UHR_USER_KNOWN_DATA_USR_ENC_BLK_CRC = 0x000f0000, + IEEE80211_RADIOTAP_UHR_USER_KNOWN_DATA_USR_ENC_BLK_TAIL = 0x03f00000, + /* indicates this user was captured */ + IEEE80211_RADIOTAP_UHR_USER_KNOWN_USER_CAPTURED = 0x80000000, +}; + +enum ieee80211_radiotap_uhr_user_info { + IEEE80211_RADIOTAP_UHR_USER_INFO_STA_ID = 0x000007ff, + IEEE80211_RADIOTAP_UHR_USER_INFO_MCS = 0x0000f800, + IEEE80211_RADIOTAP_UHR_USER_INFO_NSS = 0x00070000, + IEEE80211_RADIOTAP_UHR_USER_INFO_SPATIAL_CONFIG = 0x000f0000, + IEEE80211_RADIOTAP_UHR_USER_INFO_UEQM = 0x00100000, + IEEE80211_RADIOTAP_UHR_USER_INFO_DISREGARD = 0x00100000, + IEEE80211_RADIOTAP_UHR_USER_INFO_BF = 0x00200000, + IEEE80211_RADIOTAP_UHR_USER_INFO_BSS_COLOR_INDICATION = 0x00200000, + IEEE80211_RADIOTAP_UHR_USER_INFO_UEQM_PATTERN = 0x00c00000, + IEEE80211_RADIOTAP_UHR_USER_INFO_CODING = 0x01000000, + IEEE80211_RADIOTAP_UHR_USER_INFO_2X_LDPC = 0x02000000, +}; + /** * ieee80211_get_radiotap_len - get radiotap header length * @data: pointer to the header diff --git a/include/net/if_inet6.h b/include/net/if_inet6.h index 238ad3349456..795fb41b45f5 100644 --- a/include/net/if_inet6.h +++ b/include/net/if_inet6.h @@ -88,8 +88,6 @@ struct ip6_sf_socklist { struct in6_addr sl_addr[] __counted_by(sl_max); }; -#define IP6_SFBLOCK 10 /* allocate this many at once */ - struct ipv6_mc_socklist { struct in6_addr addr; int ifindex; diff --git a/include/net/inet6_connection_sock.h b/include/net/inet6_connection_sock.h index 745891d2e113..b814e1acc512 100644 --- a/include/net/inet6_connection_sock.h +++ b/include/net/inet6_connection_sock.h @@ -18,10 +18,14 @@ struct sk_buff; struct sock; struct sockaddr; -struct dst_entry *inet6_csk_route_req(const struct sock *sk, struct flowi6 *fl6, +struct dst_entry *inet6_csk_route_socket(struct sock *sk, + struct flowi6 *fl6); + +struct dst_entry *inet6_csk_route_req(const struct sock *sk, + struct dst_entry *dst, + struct flowi6 *fl6, const struct request_sock *req, u8 proto); int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl); -struct dst_entry *inet6_csk_update_pmtu(struct sock *sk, u32 mtu); #endif /* _INET6_CONNECTION_SOCK_H */ diff --git a/include/net/inet6_hashtables.h b/include/net/inet6_hashtables.h index 282e29237d93..2cc5d416bbb5 100644 --- a/include/net/inet6_hashtables.h +++ b/include/net/inet6_hashtables.h @@ -24,6 +24,8 @@ struct inet_hashinfo; +void inet6_init_ehash_secret(void); + static inline unsigned int __inet6_ehashfn(const u32 lhash, const u16 lport, const u32 fhash, @@ -175,7 +177,7 @@ static inline bool inet6_match(const struct net *net, const struct sock *sk, { if (!net_eq(sock_net(sk), net) || sk->sk_family != AF_INET6 || - sk->sk_portpair != ports || + READ_ONCE(sk->sk_portpair) != ports || !ipv6_addr_equal(&sk->sk_v6_daddr, saddr) || !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return false; diff --git a/include/net/inet_common.h b/include/net/inet_common.h index 5dd2bf24449e..3d747896be30 100644 --- a/include/net/inet_common.h +++ b/include/net/inet_common.h @@ -59,8 +59,7 @@ int inet_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg); int inet_ctl_sock_create(struct sock **sk, unsigned short family, unsigned short type, unsigned char protocol, struct net *net); -int inet_recv_error(struct sock *sk, struct msghdr *msg, int len, - int *addr_len); +int inet_recv_error(struct sock *sk, struct msghdr *msg, int len); struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb); int inet_gro_complete(struct sk_buff *skb, int nhoff); diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index ecb362025c4e..433c2df23076 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -34,7 +34,7 @@ struct tcp_congestion_ops; */ struct inet_connection_sock_af_ops { int (*queue_xmit)(struct sock *sk, struct sk_buff *skb, struct flowi *fl); - void (*send_check)(struct sock *sk, struct sk_buff *skb); + u16 net_header_len; int (*rebuild_header)(struct sock *sk); void (*sk_rx_dst_set)(struct sock *sk, const struct sk_buff *skb); int (*conn_request)(struct sock *sk, struct sk_buff *skb); @@ -42,8 +42,9 @@ struct inet_connection_sock_af_ops { struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req); - u16 net_header_len; + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)); int (*setsockopt)(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); int (*getsockopt)(struct sock *sk, int level, int optname, diff --git a/include/net/inet_ecn.h b/include/net/inet_ecn.h index ea32393464a2..827b87a95dab 100644 --- a/include/net/inet_ecn.h +++ b/include/net/inet_ecn.h @@ -51,11 +51,25 @@ static inline __u8 INET_ECN_encapsulate(__u8 outer, __u8 inner) return outer; } +/* Apply either ECT(0) or ECT(1) */ +static inline void __INET_ECN_xmit(struct sock *sk, bool use_ect_1) +{ + __u8 ect = use_ect_1 ? INET_ECN_ECT_1 : INET_ECN_ECT_0; + + /* Mask the complete byte in case the connection alternates between + * ECT(0) and ECT(1). + */ + inet_sk(sk)->tos &= ~INET_ECN_MASK; + inet_sk(sk)->tos |= ect; + if (inet6_sk(sk)) { + inet6_sk(sk)->tclass &= ~INET_ECN_MASK; + inet6_sk(sk)->tclass |= ect; + } +} + static inline void INET_ECN_xmit(struct sock *sk) { - inet_sk(sk)->tos |= INET_ECN_ECT_0; - if (inet6_sk(sk) != NULL) - inet6_sk(sk)->tclass |= INET_ECN_ECT_0; + __INET_ECN_xmit(sk, false); } static inline void INET_ECN_dontxmit(struct sock *sk) diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h index ac05a52d9e13..6e2fe186d0dc 100644 --- a/include/net/inet_hashtables.h +++ b/include/net/inet_hashtables.h @@ -264,6 +264,20 @@ inet_bhashfn_portaddr(const struct inet_hashinfo *hinfo, const struct sock *sk, return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)]; } +static inline bool inet_use_hash2_on_bind(const struct sock *sk) +{ +#if IS_ENABLED(CONFIG_IPV6) + if (sk->sk_family == AF_INET6) { + if (ipv6_addr_any(&sk->sk_v6_rcv_saddr)) + return false; + + if (!ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) + return true; + } +#endif + return sk->sk_rcv_saddr != htonl(INADDR_ANY); +} + struct inet_bind_hashbucket * inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port); @@ -286,7 +300,6 @@ void inet_hashinfo2_init(struct inet_hashinfo *h, const char *name, unsigned long numentries, int scale, unsigned long low_limit, unsigned long high_limit); -int inet_hashinfo2_init_mod(struct inet_hashinfo *h); bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk); bool inet_ehash_nolisten(struct sock *sk, struct sock *osk, @@ -345,7 +358,7 @@ static inline bool inet_match(const struct net *net, const struct sock *sk, int dif, int sdif) { if (!net_eq(sock_net(sk), net) || - sk->sk_portpair != ports || + READ_ONCE(sk->sk_portpair) != ports || sk->sk_addrpair != cookie) return false; diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index ac1c75975908..7cdcbed3e5cb 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -26,6 +26,8 @@ #include <net/tcp_states.h> #include <net/l3mdev.h> +#define IP_OPTIONS_DATA_FIXED_SIZE 40 + /** struct ip_options - IP Options * * @faddr - Saved first hop address @@ -58,12 +60,9 @@ struct ip_options { struct ip_options_rcu { struct rcu_head rcu; - struct ip_options opt; -}; -struct ip_options_data { - struct ip_options_rcu opt; - char data[40]; + /* Must be last as it ends in a flexible-array member. */ + struct ip_options opt; }; struct inet_request_sock { @@ -101,10 +100,7 @@ struct inet_request_sock { }; }; -static inline struct inet_request_sock *inet_rsk(const struct request_sock *sk) -{ - return (struct inet_request_sock *)sk; -} +#define inet_rsk(ptr) container_of_const(ptr, struct inet_request_sock, req) static inline u32 inet_request_mark(const struct sock *sk, struct sk_buff *skb) { @@ -163,6 +159,13 @@ static inline bool inet_sk_bound_dev_eq(const struct net *net, #endif } +struct inet6_cork { + struct ipv6_txoptions *opt; + u8 hop_limit; + u8 tclass; + u8 dontfrag:1; +}; + struct inet_cork { unsigned int flags; __be32 addr; @@ -183,6 +186,9 @@ struct inet_cork { struct inet_cork_full { struct inet_cork base; struct flowi fl; +#if IS_ENABLED(CONFIG_IPV6) + struct inet6_cork base6; +#endif }; struct ip_mc_socklist; diff --git a/include/net/inetpeer.h b/include/net/inetpeer.h index f475757daafb..414e9adf4c51 100644 --- a/include/net/inetpeer.h +++ b/include/net/inetpeer.h @@ -35,6 +35,7 @@ struct inetpeer_addr { struct inet_peer { struct rb_node rb_node; + u64 hash; struct inetpeer_addr daddr; u32 metrics[RTAX_MAX]; @@ -125,6 +126,9 @@ static inline int inetpeer_addr_cmp(const struct inetpeer_addr *a, { int i, n; + if (a->family != b->family) + return a->family < b->family ? -1 : 1; + if (a->family == AF_INET) n = sizeof(a->a4) / sizeof(u32); else diff --git a/include/net/ioam6.h b/include/net/ioam6.h index 2cbbee6e806a..a75912fe247e 100644 --- a/include/net/ioam6.h +++ b/include/net/ioam6.h @@ -60,6 +60,8 @@ void ioam6_fill_trace_data(struct sk_buff *skb, struct ioam6_trace_hdr *trace, bool is_input); +u8 ioam6_trace_compute_nodelen(u32 trace_type); + int ioam6_init(void); void ioam6_exit(void); diff --git a/include/net/ip.h b/include/net/ip.h index 69d5cef46004..6f602df72ee6 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -101,7 +101,7 @@ static inline void ipcm_init_sk(struct ipcm_cookie *ipcm, ipcm->oif = READ_ONCE(inet->sk.sk_bound_dev_if); ipcm->addr = inet->inet_saddr; - ipcm->protocol = inet->inet_num; + ipcm->protocol = READ_ONCE(inet->inet_num); } #define IPCB(skb) ((struct inet_skb_parm*)((skb)->cb)) @@ -506,6 +506,31 @@ out: return res; } +/* Configured/administrative MTU of a route, for advertising the TCP MSS. + * + * Unlike ip_dst_mtu_maybe_forward(), this deliberately ignores the + * ICMP-learned path MTU (rt->rt_pmtu). The advertised MSS bounds what the + * peer may send to us and must reflect our receive capability (the device or + * route-configured MTU), not a path MTU learned on the reverse (send) + * direction, which may not apply to the peer->us path and outlives the fnhe + * for the whole connection. See RFC 2923 section 2.3 and the comment above + * tcp_advertise_mss(). + */ +static inline unsigned int ip_dst_mtu_configured(const struct dst_entry *dst) +{ + unsigned int mtu, res; + + rcu_read_lock(); + mtu = dst_metric_raw(dst, RTAX_MTU); + if (!mtu) + mtu = READ_ONCE(dst_dev_rcu(dst)->mtu); + mtu = min_t(unsigned int, mtu, IP_MAX_MTU); + res = mtu - lwtunnel_headroom(dst->lwtstate, mtu); + rcu_read_unlock(); + + return res; +} + static inline unsigned int ip_skb_dst_mtu(struct sock *sk, const struct sk_buff *skb) { @@ -679,7 +704,8 @@ static inline void ip_ipgre_mc_map(__be32 naddr, const unsigned char *broadcast, static __inline__ void inet_reset_saddr(struct sock *sk) { - inet_sk(sk)->inet_rcv_saddr = inet_sk(sk)->inet_saddr = 0; + inet_sk(sk)->inet_saddr = 0; + WRITE_ONCE(inet_sk(sk)->inet_rcv_saddr, 0); #if IS_ENABLED(CONFIG_IPV6) if (sk->sk_family == PF_INET6) { struct ipv6_pinfo *np = inet6_sk(sk); @@ -692,14 +718,6 @@ static __inline__ void inet_reset_saddr(struct sock *sk) #endif -#if IS_MODULE(CONFIG_IPV6) -#define EXPORT_IPV6_MOD(X) EXPORT_SYMBOL(X) -#define EXPORT_IPV6_MOD_GPL(X) EXPORT_SYMBOL_GPL(X) -#else -#define EXPORT_IPV6_MOD(X) -#define EXPORT_IPV6_MOD_GPL(X) -#endif - static inline unsigned int ipv4_addr_hash(__be32 ip) { return (__force unsigned int) ip; @@ -812,7 +830,7 @@ int ip_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int ip_ra_control(struct sock *sk, unsigned char on, void (*destructor)(struct sock *)); -int ip_recv_error(struct sock *sk, struct msghdr *msg, int len, int *addr_len); +int ip_recv_error(struct sock *sk, struct msghdr *msg, int len); void ip_icmp_error(struct sock *sk, struct sk_buff *skb, int err, __be16 port, u32 info, u8 *payload); void ip_local_error(struct sock *sk, int err, __be32 daddr, __be16 dport, diff --git a/include/net/ip6_checksum.h b/include/net/ip6_checksum.h index c8a96b888277..6677b3cc3972 100644 --- a/include/net/ip6_checksum.h +++ b/include/net/ip6_checksum.h @@ -82,6 +82,4 @@ static inline __sum16 udp_v6_check(int len, void udp6_set_csum(bool nocheck, struct sk_buff *skb, const struct in6_addr *saddr, const struct in6_addr *daddr, int len); - -int udp6_csum_init(struct sk_buff *skb, struct udphdr *uh, int proto); #endif diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h index 88b0dd4d8e09..232289b439c3 100644 --- a/include/net/ip6_fib.h +++ b/include/net/ip6_fib.h @@ -468,7 +468,8 @@ void fib6_clean_all_skip_notify(struct net *net, int fib6_add(struct fib6_node *root, struct fib6_info *rt, struct nl_info *info, struct netlink_ext_ack *extack); -int fib6_del(struct fib6_info *rt, struct nl_info *info); +int fib6_del(struct fib6_info *rt, struct nl_info *info, + enum rt_del_reason del_reason); static inline void rt6_get_prefsrc(const struct rt6_info *rt, struct in6_addr *addr) @@ -486,11 +487,30 @@ void rt6_get_prefsrc(const struct rt6_info *rt, struct in6_addr *addr) rcu_read_unlock(); } +#if IS_ENABLED(CONFIG_IPV6) int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, struct fib6_config *cfg, gfp_t gfp_flags, struct netlink_ext_ack *extack); void fib6_nh_release(struct fib6_nh *fib6_nh); void fib6_nh_release_dsts(struct fib6_nh *fib6_nh); +#else +static inline int fib6_nh_init(struct net *net, struct fib6_nh *fib6_nh, + struct fib6_config *cfg, gfp_t gfp_flags, + struct netlink_ext_ack *extack) +{ + NL_SET_ERR_MSG(extack, "IPv6 support not enabled in kernel"); + return -EAFNOSUPPORT; +} + +static inline void fib6_nh_release(struct fib6_nh *fib6_nh) +{ +} + +static inline void fib6_nh_release_dsts(struct fib6_nh *fib6_nh) +{ +} +#endif + int call_fib6_entry_notifiers(struct net *net, enum fib_event_type event_type, @@ -502,17 +522,28 @@ int call_fib6_multipath_entry_notifiers(struct net *net, unsigned int nsiblings, struct netlink_ext_ack *extack); int call_fib6_entry_notifiers_replace(struct net *net, struct fib6_info *rt); +#if IS_ENABLED(CONFIG_IPV6) void fib6_rt_update(struct net *net, struct fib6_info *rt, struct nl_info *info); +#else +static inline void fib6_rt_update(struct net *net, struct fib6_info *rt, + struct nl_info *info) +{ +} +#endif void inet6_rt_notify(int event, struct fib6_info *rt, struct nl_info *info, unsigned int flags); +void inet6_rt_del_notify(struct fib6_info *rt, struct nl_info *info, + enum rt_del_reason del_reason); +void fib6_age_exceptions(struct fib6_info *rt, struct fib6_gc_args *gc_args, + unsigned long now); void fib6_run_gc(unsigned long expires, struct net *net, bool force); - void fib6_gc_cleanup(void); int fib6_init(void); +#if IS_ENABLED(CONFIG_IPV6) /* Add the route to the gc list if it is not already there * * The callers should hold f6i->fib6_table->tb6_lock. @@ -545,6 +576,23 @@ static inline void fib6_remove_gc_list(struct fib6_info *f6i) hlist_del_init(&f6i->gc_link); } +static inline void fib6_may_remove_gc_list(struct net *net, + struct fib6_info *f6i) +{ + struct fib6_gc_args gc_args; + + if (hlist_unhashed(&f6i->gc_link)) + return; + + gc_args.timeout = READ_ONCE(net->ipv6.sysctl.ip6_rt_gc_interval); + gc_args.more = 0; + + rcu_read_lock(); + fib6_age_exceptions(f6i, &gc_args, jiffies); + rcu_read_unlock(); +} +#endif + struct ipv6_route_iter { struct seq_net_private p; struct fib6_walker w; @@ -569,8 +617,13 @@ int fib6_tables_dump(struct net *net, struct notifier_block *nb, struct netlink_ext_ack *extack); void fib6_update_sernum(struct net *net, struct fib6_info *rt); +#if IS_ENABLED(CONFIG_IPV6) void fib6_update_sernum_upto_root(struct net *net, struct fib6_info *rt); -void fib6_update_sernum_stub(struct net *net, struct fib6_info *f6i); +#else +static inline void fib6_update_sernum_upto_root(struct net *net, struct fib6_info *rt) +{ +} +#endif void fib6_metric_set(struct fib6_info *f6i, int metric, u32 val); static inline bool fib6_metric_locked(struct fib6_info *f6i, int metric) @@ -580,7 +633,7 @@ static inline bool fib6_metric_locked(struct fib6_info *f6i, int metric) void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, bool offload, bool trap, bool offload_failed); -#if IS_BUILTIN(CONFIG_IPV6) && defined(CONFIG_BPF_SYSCALL) +#if IS_ENABLED(CONFIG_IPV6) && defined(CONFIG_BPF_SYSCALL) struct bpf_iter__ipv6_route { __bpf_md_ptr(struct bpf_iter_meta *, meta); __bpf_md_ptr(struct fib6_info *, rt); diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index 7c5512baa4b2..0f9b7a260d25 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -77,7 +77,14 @@ static inline bool rt6_qualify_for_ecmp(const struct fib6_info *f6i) f6i->fib6_nh->fib_nh_gw_family; } +#if IS_ENABLED(CONFIG_IPV6) void ip6_route_input(struct sk_buff *skb); +#else +static inline void ip6_route_input(struct sk_buff *skb) +{ +} +#endif + struct dst_entry *ip6_route_input_lookup(struct net *net, struct net_device *dev, struct flowi6 *fl6, @@ -94,12 +101,12 @@ static inline struct dst_entry *ip6_route_output(struct net *net, } /* Only conditionally release dst if flags indicates - * !RT6_LOOKUP_F_DST_NOREF or dst is in uncached_list. + * !RT6_LOOKUP_F_DST_NOREF or dst is uncached. */ static inline void ip6_rt_put_flags(struct rt6_info *rt, int flags) { if (!(flags & RT6_LOOKUP_F_DST_NOREF) || - !list_empty(&rt->dst.rt_uncached)) + rt->dst.rt_uncached_list) ip6_rt_put(rt); } @@ -119,7 +126,17 @@ int ipv6_route_ioctl(struct net *net, unsigned int cmd, int ip6_route_add(struct fib6_config *cfg, gfp_t gfp_flags, struct netlink_ext_ack *extack); int ip6_ins_rt(struct net *net, struct fib6_info *f6i); +int ip6_del_rt_reason(struct net *net, struct fib6_info *f6i, + enum rt_del_reason del_reason); +#if IS_ENABLED(CONFIG_IPV6) int ip6_del_rt(struct net *net, struct fib6_info *f6i, bool skip_notify); +#else +static inline int ip6_del_rt(struct net *net, struct fib6_info *f6i, + bool skip_notify) +{ + return -EAFNOSUPPORT; +} +#endif void rt6_flush_exceptions(struct fib6_info *f6i); void rt6_age_exceptions(struct fib6_info *f6i, struct fib6_gc_args *gc_args, @@ -252,19 +269,43 @@ static inline bool ipv6_unicast_destination(const struct sk_buff *skb) return rt->rt6i_flags & RTF_LOCAL; } +static inline bool __ipv6_anycast_destination(const struct rt6key *rt6i_dst, + u32 rt6i_flags, + const struct in6_addr *daddr) +{ + return rt6i_flags & RTF_ANYCAST || + (rt6i_dst->plen < 127 && + !(rt6i_flags & (RTF_GATEWAY | RTF_NONEXTHOP)) && + ipv6_addr_equal(&rt6i_dst->addr, daddr)); +} + static inline bool ipv6_anycast_destination(const struct dst_entry *dst, const struct in6_addr *daddr) { const struct rt6_info *rt = dst_rt6_info(dst); - return rt->rt6i_flags & RTF_ANYCAST || - (rt->rt6i_dst.plen < 127 && - !(rt->rt6i_flags & (RTF_GATEWAY | RTF_NONEXTHOP)) && - ipv6_addr_equal(&rt->rt6i_dst.addr, daddr)); + return __ipv6_anycast_destination(&rt->rt6i_dst, rt->rt6i_flags, daddr); } +#if IS_ENABLED(CONFIG_IPV6) int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, int (*output)(struct net *, struct sock *, struct sk_buff *)); +#else +static inline int ip6_fragment(struct net *net, struct sock *sk, + struct sk_buff *skb, + int (*output)(struct net *, struct sock *, + struct sk_buff *)) +{ + kfree_skb(skb); + return -EAFNOSUPPORT; +} +#endif + +/* Variant of dst_mtu() for IPv6 users */ +static inline u32 dst6_mtu(const struct dst_entry *dst) +{ + return INDIRECT_CALL_1(dst->ops->mtu, ip6_mtu, dst); +} static inline unsigned int ip6_skb_dst_mtu(const struct sk_buff *skb) { @@ -343,6 +384,45 @@ static inline unsigned int ip6_dst_mtu_maybe_forward(const struct dst_entry *dst rcu_read_unlock(); out: + mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); + + return mtu - lwtunnel_headroom(dst->lwtstate, mtu); +} + +/* Configured/administrative MTU of a route, for advertising the TCP MSS. + * + * Unlike ip6_dst_mtu_maybe_forward(), this ignores any ICMPv6-learned path + * MTU (which is kept on the RTF_CACHE exception route) and returns the MTU of + * the underlying route (fib6_pmtu) or the egress device. The advertised MSS + * bounds what the peer may send to us and must reflect our receive + * capability, not a path MTU learned on the reverse (send) direction. See + * RFC 2923 section 2.3 and the comment above tcp_advertise_mss(). + */ +static inline unsigned int ip6_dst_mtu_configured(const struct dst_entry *dst) +{ + const struct rt6_info *rt = dst_rt6_info(dst); + const struct fib6_info *from; + struct inet6_dev *idev; + unsigned int mtu = 0; + + rcu_read_lock(); + /* IPv6 keeps the learned PMTU and the configured MTU in the same + * RTAX_MTU slot: the learned value sits on this (possibly RTF_CACHE) + * dst, the configured one on the underlying route. Reach the latter + * via ->from (fib6_pmtu), populated by ip6_route_info_create(). + */ + from = rcu_dereference(rt->from); + if (from) + mtu = from->fib6_pmtu; + if (!mtu) { + mtu = IPV6_MIN_MTU; + idev = __in6_dev_get(dst_dev_rcu(dst)); + if (idev) + mtu = max_t(unsigned int, mtu, READ_ONCE(idev->cnf.mtu6)); + } + rcu_read_unlock(); + + mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); return mtu - lwtunnel_headroom(dst->lwtstate, mtu); } diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h index 120db2865811..b99805ee2fd1 100644 --- a/include/net/ip6_tunnel.h +++ b/include/net/ip6_tunnel.h @@ -156,6 +156,18 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, { int pkt_len, err; + if (unlikely(dev_recursion_level() > IP_TUNNEL_RECURSION_LIMIT)) { + if (dev) { + net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", + dev->name); + DEV_STATS_INC(dev, tx_errors); + } + kfree_skb_reason(skb, SKB_DROP_REASON_RECURSION_LIMIT); + return; + } + + dev_xmit_recursion_inc(); + memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); IP6CB(skb)->flags = ip6cb_flags; pkt_len = skb->len - skb_inner_network_offset(skb); @@ -166,6 +178,8 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, pkt_len = -1; iptunnel_xmit_stats(dev, pkt_len); } + + dev_xmit_recursion_dec(); } #endif #endif diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index b4495c38e0a0..0a35355fb0f3 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h @@ -269,7 +269,6 @@ struct fib_dump_filter { bool filter_set; bool dump_routes; bool dump_exceptions; - bool rtnl_held; unsigned char protocol; unsigned char rt_type; unsigned int flags; @@ -303,7 +302,8 @@ static inline struct fib_table *fib_get_table(struct net *net, u32 id) &net->ipv4.fib_table_hash[TABLE_LOCAL_INDEX] : &net->ipv4.fib_table_hash[TABLE_MAIN_INDEX]; - tb_hlist = rcu_dereference_rtnl(hlist_first_rcu(ptr)); + /* Only fib4_rules_init() adds fib_table. */ + tb_hlist = rcu_dereference_protected(hlist_first_rcu(ptr), true); return hlist_entry(tb_hlist, struct fib_table, tb_hlist); } @@ -375,7 +375,7 @@ static inline int fib_lookup(struct net *net, struct flowi4 *flp, struct fib_result *res, unsigned int flags) { struct fib_table *tb; - int err = -ENETUNREACH; + int err = -EAGAIN; flags |= FIB_LOOKUP_NOREF; if (net->ipv4.fib_has_custom_rules) @@ -389,17 +389,16 @@ static inline int fib_lookup(struct net *net, struct flowi4 *flp, if (tb) err = fib_table_lookup(tb, flp, res, flags); - if (!err) + if (err != -EAGAIN) goto out; tb = rcu_dereference_rtnl(net->ipv4.fib_default); if (tb) err = fib_table_lookup(tb, flp, res, flags); -out: if (err == -EAGAIN) err = -ENETUNREACH; - +out: rcu_read_unlock(); return err; @@ -559,7 +558,7 @@ static inline u32 fib_multipath_hash_from_keys(const struct net *net, siphash_aligned_key_t hash_key; u32 mp_seed; - mp_seed = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed).mp_seed; + mp_seed = READ_ONCE(net->ipv4.sysctl_fib_multipath_hash_seed.mp_seed); fib_multipath_hash_construct_key(&hash_key, mp_seed); return flow_hash_from_keys_seed(keys, &hash_key); @@ -629,6 +628,11 @@ static inline void fib_info_hold(struct fib_info *fi) refcount_inc(&fi->fib_clntref); } +static inline bool fib_info_hold_safe(struct fib_info *fi) +{ + return refcount_inc_not_zero(&fi->fib_clntref); +} + static inline void fib_info_put(struct fib_info *fi) { if (refcount_dec_and_test(&fi->fib_clntref)) diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index 4021e6a73e32..7c9aadfe8fe3 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -27,6 +27,13 @@ #include <net/ip6_route.h> #endif +/* Recursion limit for tunnel xmit to detect routing loops. + * Unlike XMIT_RECURSION_LIMIT (8) used in the no-qdisc path, tunnel + * recursion involves route lookups and full IP output, consuming much + * more stack per level, so a lower limit is needed. + */ +#define IP_TUNNEL_RECURSION_LIMIT 5 + /* Keep error state on tunnel for 30 sec */ #define IPTUNNEL_ERR_TIMEO (30*HZ) @@ -405,7 +412,6 @@ bool ip_tunnel_parm_from_user(struct ip_tunnel_parm_kern *kp, bool ip_tunnel_parm_to_user(void __user *data, struct ip_tunnel_parm_kern *kp); int ip_tunnel_siocdevprivate(struct net_device *dev, struct ifreq *ifr, void __user *data, int cmd); -int __ip_tunnel_change_mtu(struct net_device *dev, int new_mtu, bool strict); int ip_tunnel_change_mtu(struct net_device *dev, int new_mtu); struct ip_tunnel *ip_tunnel_lookup(struct ip_tunnel_net *itn, @@ -622,8 +628,7 @@ struct metadata_dst *iptunnel_metadata_reply(struct metadata_dst *md, int skb_tunnel_check_pmtu(struct sk_buff *skb, struct dst_entry *encap_dst, int headroom, bool reply); -static inline void ip_tunnel_adj_headroom(struct net_device *dev, - unsigned int headroom) +static inline unsigned int ip_tunnel_limit_headroom(unsigned int headroom) { /* we must cap headroom to some upperlimit, else pskb_expand_head * will overflow header offsets in skb_headers_offset_update(). @@ -633,6 +638,14 @@ static inline void ip_tunnel_adj_headroom(struct net_device *dev, if (headroom > max_allowed) headroom = max_allowed; + return headroom; +} + +static inline void ip_tunnel_adj_headroom(struct net_device *dev, + unsigned int headroom) +{ + headroom = ip_tunnel_limit_headroom(headroom); + if (headroom > READ_ONCE(dev->needed_headroom)) WRITE_ONCE(dev->needed_headroom, headroom); } @@ -658,13 +671,29 @@ static inline int iptunnel_pull_offloads(struct sk_buff *skb) static inline void iptunnel_xmit_stats(struct net_device *dev, int pkt_len) { if (pkt_len > 0) { - struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats); - - u64_stats_update_begin(&tstats->syncp); - u64_stats_add(&tstats->tx_bytes, pkt_len); - u64_stats_inc(&tstats->tx_packets); - u64_stats_update_end(&tstats->syncp); - put_cpu_ptr(tstats); + if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { + struct pcpu_dstats *dstats = get_cpu_ptr(dev->dstats); + + u64_stats_update_begin(&dstats->syncp); + u64_stats_add(&dstats->tx_bytes, pkt_len); + u64_stats_inc(&dstats->tx_packets); + u64_stats_update_end(&dstats->syncp); + put_cpu_ptr(dstats); + return; + } + if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { + struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats); + + u64_stats_update_begin(&tstats->syncp); + u64_stats_add(&tstats->tx_bytes, pkt_len); + u64_stats_inc(&tstats->tx_packets); + u64_stats_update_end(&tstats->syncp); + put_cpu_ptr(tstats); + return; + } + pr_err_once("iptunnel_xmit_stats pcpu_stat_type=%d\n", + dev->pcpu_stat_type); + WARN_ON_ONCE(1); return; } diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index 29a36709e7f3..32fde731bceb 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -11,6 +11,7 @@ #include <asm/types.h> /* for __uXX types */ #include <linux/list.h> /* for struct list_head */ +#include <linux/rculist_bl.h> /* for struct hlist_bl_head */ #include <linux/spinlock.h> /* for struct rwlock_t */ #include <linux/atomic.h> /* for struct atomic_t */ #include <linux/refcount.h> /* for struct refcount_t */ @@ -30,10 +31,37 @@ #endif #include <net/net_namespace.h> /* Netw namespace */ #include <linux/sched/isolation.h> +#include <linux/siphash.h> #define IP_VS_HDR_INVERSE 1 #define IP_VS_HDR_ICMP 2 +/* Destination Server Flags */ +#define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */ + +/* Destination Server Config Flags */ +#define IP_VS_DEST_CF_AVAILABLE 0x0001 /* server is available */ + +/* conn_tab limits (as per Kconfig) */ +#define IP_VS_CONN_TAB_MIN_BITS 8 +#if BITS_PER_LONG > 32 +#define IP_VS_CONN_TAB_MAX_BITS 27 +#else +#define IP_VS_CONN_TAB_MAX_BITS 20 +#endif + +/* conn_max limits */ +#if BITS_PER_LONG > 32 +/* Limit of atomic_t but restricted by roundup_pow_of_two() in ip_vs_core.c */ +#define IP_VS_CONN_MAX (1 << 30) +#else +#define IP_VS_CONN_MAX (1 << 24) +#endif + +/* svc_table limits */ +#define IP_VS_SVC_TAB_MIN_BITS 4 +#define IP_VS_SVC_TAB_MAX_BITS 20 + /* Generic access of ipvs struct */ static inline struct netns_ipvs *net_ipvs(struct net* net) { @@ -43,8 +71,6 @@ static inline struct netns_ipvs *net_ipvs(struct net* net) /* Connections' size value needed by ip_vs_ctl.c */ extern int ip_vs_conn_tab_size; -extern struct mutex __ip_vs_mutex; - struct ip_vs_iphdr { int hdr_flags; /* ipvs flags */ __u32 off; /* Where IP or IPv4 header starts */ @@ -265,6 +291,29 @@ static inline const char *ip_vs_dbg_addr(int af, char *buf, size_t buf_len, pr_err(msg, ##__VA_ARGS__); \ } while (0) +struct ip_vs_aligned_lock { + spinlock_t l; /* Protect buckets */ +} ____cacheline_aligned_in_smp; + +/* For arrays per family */ +enum { + IP_VS_AF_INET, + IP_VS_AF_INET6, + IP_VS_AF_MAX +}; + +static inline int ip_vs_af_index(int af) +{ + return af == AF_INET6 ? IP_VS_AF_INET6 : IP_VS_AF_INET; +} + +/* work_flags */ +enum { + IP_VS_WORK_SVC_RESIZE, /* Schedule svc_resize_work */ + IP_VS_WORK_SVC_NORESIZE, /* Stopping svc_resize_work */ + IP_VS_WORK_CONN_RESIZE, /* Schedule conn_resize_work */ +}; + /* The port number of FTP service (in network order). */ #define FTPPORT cpu_to_be16(21) #define FTPDATA cpu_to_be16(20) @@ -456,6 +505,7 @@ struct ip_vs_est_kt_data { DECLARE_BITMAP(avail, IPVS_EST_NTICKS); /* tick has space for ests */ unsigned long est_timer; /* estimation timer (jiffies) */ struct ip_vs_stats *calc_stats; /* Used for calculation */ + int needed; /* task is needed */ int tick_len[IPVS_EST_NTICKS]; /* est count */ int id; /* ktid per netns */ int chain_max; /* max ests per tick chain */ @@ -466,6 +516,198 @@ struct ip_vs_est_kt_data { int est_row; /* estimated row */ }; +/* IPVS resizable hash tables */ +struct ip_vs_rht { + struct hlist_bl_head *buckets; + struct ip_vs_rht __rcu *new_tbl; /* New/Same table */ + seqcount_t *seqc; /* Protects moves */ + struct ip_vs_aligned_lock *lock; /* Protect seqc */ + int mask; /* Buckets mask */ + int size; /* Buckets */ + int seqc_mask; /* seqc mask */ + int lock_mask; /* lock mask */ + u32 table_id; + int u_thresh; /* upper threshold */ + int l_thresh; /* lower threshold */ + int lfactor; /* Load Factor (shift)*/ + int bits; /* size = 1 << bits */ + siphash_key_t hash_key; + struct rcu_head rcu_head; +}; + +/** + * ip_vs_rht_for_each_table() - Walk the hash tables + * @table: struct ip_vs_rht __rcu *table + * @t: current table, used as cursor, struct ip_vs_rht *var + * @p: previous table, temp struct ip_vs_rht *var + * + * Walk tables assuming others can not change the installed tables + */ +#define ip_vs_rht_for_each_table(table, t, p) \ + for (p = NULL, t = rcu_dereference_protected(table, 1); \ + t != p; \ + p = t, t = rcu_dereference_protected(t->new_tbl, 1)) + +/** + * ip_vs_rht_for_each_table_rcu() - Walk the hash tables under RCU reader lock + * @table: struct ip_vs_rht __rcu *table + * @t: current table, used as cursor, struct ip_vs_rht *var + * @p: previous table, temp struct ip_vs_rht *var + * + * We usually search in one table and also in second table on resizing + */ +#define ip_vs_rht_for_each_table_rcu(table, t, p) \ + for (p = NULL, t = rcu_dereference(table); \ + t != p; \ + p = t, t = rcu_dereference(t->new_tbl)) + +/** + * ip_vs_rht_for_each_bucket() - Walk all table buckets + * @t: current table, used as cursor, struct ip_vs_rht *var + * @bucket: bucket index, used as cursor, u32 var + * @head: bucket address, used as cursor, struct hlist_bl_head *var + */ +#define ip_vs_rht_for_each_bucket(t, bucket, head) \ + for (bucket = 0, head = (t)->buckets; \ + bucket < t->size; bucket++, head++) + +/** + * ip_vs_rht_for_bucket_retry() - Retry bucket if entries are moved + * @t: current table, used as cursor, struct ip_vs_rht *var + * @bucket: index of current bucket or hash key + * @sc: temp seqcount_t *var + * @seq: temp unsigned int var for sequence count + * @retry: temp int var + */ +#define ip_vs_rht_for_bucket_retry(t, bucket, sc, seq, retry) \ + for (retry = 1, sc = &(t)->seqc[(bucket) & (t)->seqc_mask]; \ + retry && ({ seq = read_seqcount_begin(sc); 1; }); \ + retry = read_seqcount_retry(sc, seq)) + +/** + * DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() - Declare variables + * + * Variables for ip_vs_rht_walk_buckets_rcu + */ +#define DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() \ + struct ip_vs_rht *_t, *_p; \ + unsigned int _seq; \ + seqcount_t *_sc; \ + u32 _bucket; \ + int _retry +/** + * ip_vs_rht_walk_buckets_rcu() - Walk all buckets under RCU read lock + * @table: struct ip_vs_rht __rcu *table + * @head: bucket address, used as cursor, struct hlist_bl_head *var + * + * Can be used while others add/delete/move entries + * Not suitable if duplicates are not desired + * Possible cases for reader that uses cond_resched_rcu() in the loop: + * - new table can not be installed, no need to repeat + * - new table can be installed => check and repeat if new table is + * installed, needed for !PREEMPT_RCU + */ +#define ip_vs_rht_walk_buckets_rcu(table, head) \ + ip_vs_rht_for_each_table_rcu(table, _t, _p) \ + ip_vs_rht_for_each_bucket(_t, _bucket, head) \ + ip_vs_rht_for_bucket_retry(_t, _bucket, _sc, \ + _seq, _retry) + +/** + * DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() - Declare variables + * + * Variables for ip_vs_rht_walk_bucket_rcu + */ +#define DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() \ + unsigned int _seq; \ + seqcount_t *_sc; \ + int _retry +/** + * ip_vs_rht_walk_bucket_rcu() - Walk bucket under RCU read lock + * @t: current table, struct ip_vs_rht *var + * @bucket: index of current bucket or hash key + * @head: bucket address, used as cursor, struct hlist_bl_head *var + * + * Can be used while others add/delete/move entries + * Not suitable if duplicates are not desired + * Possible cases for reader that uses cond_resched_rcu() in the loop: + * - new table can not be installed, no need to repeat + * - new table can be installed => check and repeat if new table is + * installed, needed for !PREEMPT_RCU + */ +#define ip_vs_rht_walk_bucket_rcu(t, bucket, head) \ + if (({ head = (t)->buckets + ((bucket) & (t)->mask); 0; })) \ + {} \ + else \ + ip_vs_rht_for_bucket_retry(t, (bucket), _sc, _seq, _retry) + +/** + * DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() - Declare variables + * + * Variables for ip_vs_rht_walk_buckets_safe_rcu + */ +#define DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() \ + struct ip_vs_rht *_t, *_p; \ + u32 _bucket +/** + * ip_vs_rht_walk_buckets_safe_rcu() - Walk all buckets under RCU read lock + * @table: struct ip_vs_rht __rcu *table + * @head: bucket address, used as cursor, struct hlist_bl_head *var + * + * Can be used while others add/delete entries but moving is disabled + * Using cond_resched_rcu() should be safe if tables do not change + */ +#define ip_vs_rht_walk_buckets_safe_rcu(table, head) \ + ip_vs_rht_for_each_table_rcu(table, _t, _p) \ + ip_vs_rht_for_each_bucket(_t, _bucket, head) + +/** + * DECLARE_IP_VS_RHT_WALK_BUCKETS() - Declare variables + * + * Variables for ip_vs_rht_walk_buckets + */ +#define DECLARE_IP_VS_RHT_WALK_BUCKETS() \ + struct ip_vs_rht *_t, *_p; \ + u32 _bucket + +/** + * ip_vs_rht_walk_buckets() - Walk all buckets + * @table: struct ip_vs_rht __rcu *table + * @head: bucket address, used as cursor, struct hlist_bl_head *var + * + * Use if others can not add/delete/move entries + */ +#define ip_vs_rht_walk_buckets(table, head) \ + ip_vs_rht_for_each_table(table, _t, _p) \ + ip_vs_rht_for_each_bucket(_t, _bucket, head) + +/* Entries can be in one of two tables, so we flip bit when new table is + * created and store it as highest bit in hash keys + */ +#define IP_VS_RHT_TABLE_ID_MASK BIT(31) + +/* Check if hash key is from this table */ +static inline bool ip_vs_rht_same_table(struct ip_vs_rht *t, u32 hash_key) +{ + return !((t->table_id ^ hash_key) & IP_VS_RHT_TABLE_ID_MASK); +} + +/* Build per-table hash key from hash value */ +static inline u32 ip_vs_rht_build_hash_key(struct ip_vs_rht *t, u32 hash) +{ + return t->table_id | (hash & ~IP_VS_RHT_TABLE_ID_MASK); +} + +void ip_vs_rht_free(struct ip_vs_rht *t); +void ip_vs_rht_rcu_free(struct rcu_head *head); +struct ip_vs_rht *ip_vs_rht_alloc(int buckets, int scounts, int locks); +int ip_vs_rht_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, int n, + int lfactor, int min_bits, int max_bits); +void ip_vs_rht_set_thresholds(struct ip_vs_rht *t, int size, int lfactor, + int min_bits, int max_bits); +u32 ip_vs_rht_hash_linfo(struct ip_vs_rht *t, int af, + const union nf_inet_addr *addr, u32 v1, u32 v2); + struct dst_entry; struct iphdr; struct ip_vs_conn; @@ -516,7 +758,8 @@ struct ip_vs_protocol { void (*state_transition)(struct ip_vs_conn *cp, int direction, const struct sk_buff *skb, - struct ip_vs_proto_data *pd); + struct ip_vs_proto_data *pd, + unsigned int iph_len); int (*register_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); @@ -559,50 +802,48 @@ struct ip_vs_conn_param { __u8 pe_data_len; }; +/* Hash node in conn_tab */ +struct ip_vs_conn_hnode { + struct hlist_bl_node node; /* node in conn_tab */ + u32 hash_key; /* Key for the hash table */ + u8 dir; /* 0=out->in, 1=in->out */ +} __packed; + /* IP_VS structure allocated for each dynamically scheduled connection */ struct ip_vs_conn { - struct hlist_node c_list; /* hashed list heads */ - /* Protocol, addresses and port numbers */ + /* Cacheline for hash table nodes - rarely modified */ + + struct ip_vs_conn_hnode hn0; /* Original direction */ + u8 af; /* address family */ __be16 cport; + struct ip_vs_conn_hnode hn1; /* Reply direction */ + u8 daf; /* Address family of the dest */ __be16 dport; - __be16 vport; - u16 af; /* address family */ - union nf_inet_addr caddr; /* client address */ - union nf_inet_addr vaddr; /* virtual address */ - union nf_inet_addr daddr; /* destination address */ + struct ip_vs_dest *dest; /* real server */ + atomic_t n_control; /* Number of controlled ones */ volatile __u32 flags; /* status flags */ - __u16 protocol; /* Which protocol (TCP/UDP) */ - __u16 daf; /* Address family of the dest */ - struct netns_ipvs *ipvs; + /* 44/64 */ - /* counter and timer */ - refcount_t refcnt; /* reference count */ - struct timer_list timer; /* Expiration timer */ - volatile unsigned long timeout; /* timeout */ - - /* Flags and state transition */ - spinlock_t lock; /* lock for state transition */ + struct ip_vs_conn *control; /* Master control connection */ + const struct ip_vs_pe *pe; + char *pe_data; + __u8 pe_data_len; volatile __u16 state; /* state info */ volatile __u16 old_state; /* old state, to be used for * state transition triggered * synchronization */ - __u32 fwmark; /* Fire wall mark from skb */ - unsigned long sync_endtime; /* jiffies + sent_retries */ + /* 2-byte hole */ + /* 64/96 */ - /* Control members */ - struct ip_vs_conn *control; /* Master control connection */ - atomic_t n_control; /* Number of controlled ones */ - struct ip_vs_dest *dest; /* real server */ - atomic_t in_pkts; /* incoming packet counter */ + union nf_inet_addr caddr; /* client address */ + union nf_inet_addr vaddr; /* virtual address */ + /* 96/128 */ - /* Packet transmitter for different forwarding methods. If it - * mangles the packet, it must return NF_DROP or better NF_STOLEN, - * otherwise this must be changed to a sk_buff **. - * NF_ACCEPT can be returned when destination is local. - */ - int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp, - struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); + union nf_inet_addr daddr; /* destination address */ + __u32 fwmark; /* Fire wall mark from skb */ + __be16 vport; + __u16 protocol; /* Which protocol (TCP/UDP) */ /* Note: we can group the following members into a structure, * in order to save more space, and the following members are @@ -610,14 +851,31 @@ struct ip_vs_conn { */ struct ip_vs_app *app; /* bound ip_vs_app object */ void *app_data; /* Application private data */ + /* 128/168 */ struct_group(sync_conn_opt, struct ip_vs_seq in_seq; /* incoming seq. struct */ struct ip_vs_seq out_seq; /* outgoing seq. struct */ ); + /* 152/192 */ - const struct ip_vs_pe *pe; - char *pe_data; - __u8 pe_data_len; + struct timer_list timer; /* Expiration timer */ + volatile unsigned long timeout; /* timeout */ + spinlock_t lock; /* lock for state transition */ + refcount_t refcnt; /* reference count */ + atomic_t in_pkts; /* incoming packet counter */ + /* 64-bit: 4-byte gap */ + + /* 188/256 */ + unsigned long sync_endtime; /* jiffies + sent_retries */ + struct netns_ipvs *ipvs; + + /* Packet transmitter for different forwarding methods. If it + * mangles the packet, it must return NF_DROP or better NF_STOLEN, + * otherwise this must be changed to a sk_buff **. + * NF_ACCEPT can be returned when destination is local. + */ + int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp, + struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); struct rcu_head rcu_head; }; @@ -673,15 +931,15 @@ struct ip_vs_dest_user_kern { * forwarding entries. */ struct ip_vs_service { - struct hlist_node s_list; /* for normal service table */ - struct hlist_node f_list; /* for fwmark-based service table */ - atomic_t refcnt; /* reference counter */ - + struct hlist_bl_node s_list; /* node in service table */ + u32 hash_key; /* Key for the hash table */ u16 af; /* address family */ __u16 protocol; /* which protocol (TCP/UDP) */ + union nf_inet_addr addr; /* IP address for virtual service */ - __be16 port; /* port number for the service */ __u32 fwmark; /* firewall mark of the service */ + atomic_t refcnt; /* reference counter */ + __be16 port; /* port number for the service */ unsigned int flags; /* service status flags */ unsigned int timeout; /* persistent timeout in ticks */ __be32 netmask; /* grouping granularity, mask/plen */ @@ -724,6 +982,7 @@ struct ip_vs_dest { volatile unsigned int flags; /* dest status flags */ atomic_t conn_flags; /* flags to copy to conn */ atomic_t weight; /* server weight */ + unsigned long cflags; /* config flags */ atomic_t last_weight; /* server latest weight */ __u16 tun_type; /* tunnel type */ __be16 tun_port; /* tunnel port */ @@ -735,10 +994,11 @@ struct ip_vs_dest { /* connection counters and thresholds */ atomic_t activeconns; /* active connections */ - atomic_t inactconns; /* inactive connections */ + atomic_t totalconns; /* total connections */ atomic_t persistconns; /* persistent connections */ __u32 u_threshold; /* upper threshold */ __u32 l_threshold; /* lower threshold */ + __u32 l_threshold_val;/* used lower threshold */ /* for destination cache */ spinlock_t dst_lock; /* lock of dst_cache */ @@ -791,8 +1051,8 @@ struct ip_vs_pe { int (*fill_param)(struct ip_vs_conn_param *p, struct sk_buff *skb); bool (*ct_match)(const struct ip_vs_conn_param *p, struct ip_vs_conn *ct); - u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, u32 initval, - bool inverse); + u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, + struct ip_vs_rht *t, bool inverse); int (*show_pe_data)(const struct ip_vs_conn *cp, char *buf); /* create connections for real-server outgoing packets */ struct ip_vs_conn* (*conn_out)(struct ip_vs_service *svc, @@ -931,21 +1191,28 @@ struct netns_ipvs { #endif /* ip_vs_conn */ atomic_t conn_count; /* connection counter */ + atomic_t no_cport_conns[IP_VS_AF_MAX]; + struct delayed_work conn_resize_work;/* resize conn_tab */ /* ip_vs_ctl */ struct ip_vs_stats_rcu *tot_stats; /* Statistics & est. */ - int num_services; /* no of virtual services */ - int num_services6; /* IPv6 virtual services */ - /* Trash for destinations */ struct list_head dest_trash; spinlock_t dest_trash_lock; struct timer_list dest_trash_timer; /* expiration timer */ + struct mutex service_mutex; /* service reconfig */ + struct rw_semaphore svc_resize_sem; /* svc_table resizing */ + struct rw_semaphore svc_replace_sem; /* svc_table replace */ + struct delayed_work svc_resize_work; /* resize svc_table */ + atomic_t svc_table_changes;/* ++ on table changes */ /* Service counters */ - atomic_t ftpsvc_counter; - atomic_t nullsvc_counter; - atomic_t conn_out_counter; + atomic_t num_services[IP_VS_AF_MAX]; /* Services */ + atomic_t fwm_services[IP_VS_AF_MAX]; /* Services */ + atomic_t nonfwm_services[IP_VS_AF_MAX];/* Services */ + atomic_t ftpsvc_counter[IP_VS_AF_MAX]; /* FTPPORT */ + atomic_t nullsvc_counter[IP_VS_AF_MAX];/* Zero port */ + atomic_t conn_out_counter[IP_VS_AF_MAX];/* out conn */ #ifdef CONFIG_SYSCTL /* delayed work for expiring no dest connections */ @@ -956,6 +1223,7 @@ struct netns_ipvs { int drop_counter; int old_secure_tcp; atomic_t dropentry; + s8 dropentry_counters[8]; /* locks in ctl.c */ spinlock_t dropentry_lock; /* drop entry handling */ spinlock_t droppacket_lock; /* drop packet handling */ @@ -969,6 +1237,10 @@ struct netns_ipvs { /* sysctl variables */ int sysctl_amemthresh; int sysctl_am_droprate; +#ifdef CONFIG_SYSCTL + int sysctl_conn_max;/* soft limit for conns */ + int conn_max_limit; /* hard limit for conn_max */ +#endif int sysctl_drop_entry; int sysctl_drop_packet; int sysctl_secure_tcp; @@ -1002,6 +1274,8 @@ struct netns_ipvs { int sysctl_est_nice; /* kthread nice */ int est_stopped; /* stop tasks */ #endif + int sysctl_conn_lfactor; + int sysctl_svc_lfactor; /* ip_vs_lblc */ int sysctl_lblc_expiration; @@ -1011,6 +1285,7 @@ struct netns_ipvs { int sysctl_lblcr_expiration; struct ctl_table_header *lblcr_ctl_header; struct ctl_table *lblcr_ctl_table; + unsigned long work_flags; /* IP_VS_WORK_* flags */ /* ip_vs_est */ struct delayed_work est_reload_work;/* Reload kthread tasks */ struct mutex est_mutex; /* protect kthread tasks */ @@ -1041,6 +1316,10 @@ struct netns_ipvs { */ unsigned int mixed_address_family_dests; unsigned int hooks_afmask; /* &1=AF_INET, &2=AF_INET6 */ + + struct ip_vs_rht __rcu *svc_table; /* Services */ + struct ip_vs_rht __rcu *conn_tab; /* Connections */ + atomic_t conn_tab_changes;/* ++ on new table */ }; #define DEFAULT_SYNC_THRESHOLD 3 @@ -1059,6 +1338,11 @@ struct netns_ipvs { #ifdef CONFIG_SYSCTL +static inline int sysctl_conn_max(struct netns_ipvs *ipvs) +{ + return READ_ONCE(ipvs->sysctl_conn_max); +} + static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) { return ipvs->sysctl_sync_threshold[0]; @@ -1155,7 +1439,7 @@ static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) return ipvs->sysctl_run_estimation; } -static inline const struct cpumask *sysctl_est_cpulist(struct netns_ipvs *ipvs) +static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) { if (ipvs->est_cpulist_valid) return ipvs->sysctl_est_cpulist; @@ -1178,6 +1462,11 @@ static inline int sysctl_est_nice(struct netns_ipvs *ipvs) #else +static inline int sysctl_conn_max(struct netns_ipvs *ipvs) +{ + return IP_VS_CONN_MAX; +} + static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) { return DEFAULT_SYNC_THRESHOLD; @@ -1273,7 +1562,7 @@ static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) return 1; } -static inline const struct cpumask *sysctl_est_cpulist(struct netns_ipvs *ipvs) +static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) { return housekeeping_cpumask(HK_TYPE_KTHREAD); } @@ -1290,6 +1579,36 @@ static inline int sysctl_est_nice(struct netns_ipvs *ipvs) #endif +/* Get load factor to map conn_count/u_thresh to t->size */ +static inline int sysctl_conn_lfactor(struct netns_ipvs *ipvs) +{ + return READ_ONCE(ipvs->sysctl_conn_lfactor); +} + +/* Get load factor to map num_services/u_thresh to t->size + * Smaller value decreases u_thresh to reduce collisions but increases + * the table size + * Returns factor where: + * - <0: u_thresh = size >> -factor, eg. lfactor -2 = 25% load + * - >=0: u_thresh = size << factor, eg. lfactor 1 = 200% load + */ +static inline int sysctl_svc_lfactor(struct netns_ipvs *ipvs) +{ + return READ_ONCE(ipvs->sysctl_svc_lfactor); +} + +static inline bool sysctl_est_cpulist_empty(struct netns_ipvs *ipvs) +{ + guard(rcu)(); + return cpumask_empty(__sysctl_est_cpulist(ipvs)); +} + +static inline unsigned int sysctl_est_cpulist_weight(struct netns_ipvs *ipvs) +{ + guard(rcu)(); + return cpumask_weight(__sysctl_est_cpulist(ipvs)); +} + /* IPVS core functions * (from ip_vs_core.c) */ @@ -1363,6 +1682,23 @@ static inline void __ip_vs_conn_put(struct ip_vs_conn *cp) } void ip_vs_conn_put(struct ip_vs_conn *cp); void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport); +int ip_vs_conn_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, + int lfactor); +struct ip_vs_rht *ip_vs_conn_tab_alloc(struct netns_ipvs *ipvs, int buckets, + int lfactor); + +static inline struct ip_vs_conn * +ip_vs_hn0_to_conn(struct ip_vs_conn_hnode *hn) +{ + return container_of(hn, struct ip_vs_conn, hn0); +} + +static inline struct ip_vs_conn * +ip_vs_hn_to_conn(struct ip_vs_conn_hnode *hn) +{ + return hn->dir ? container_of(hn, struct ip_vs_conn, hn1) : + container_of(hn, struct ip_vs_conn, hn0); +} struct ip_vs_conn *ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, const union nf_inet_addr *daddr, @@ -1519,8 +1855,7 @@ int register_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); int unregister_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); int ip_vs_bind_scheduler(struct ip_vs_service *svc, struct ip_vs_scheduler *scheduler); -void ip_vs_unbind_scheduler(struct ip_vs_service *svc, - struct ip_vs_scheduler *sched); +void ip_vs_unbind_scheduler(struct ip_vs_service *svc); struct ip_vs_scheduler *ip_vs_scheduler_get(const char *sched_name); void ip_vs_scheduler_put(struct ip_vs_scheduler *scheduler); struct ip_vs_conn * @@ -1580,6 +1915,8 @@ static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) kfree(dest); } +void ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode); + /* IPVS sync daemon data and function prototypes * (from ip_vs_sync.c) */ @@ -1593,18 +1930,26 @@ int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); void ip_vs_zero_estimator(struct ip_vs_stats *stats); void ip_vs_read_estimator(struct ip_vs_kstats *dst, struct ip_vs_stats *stats); -void ip_vs_est_reload_start(struct netns_ipvs *ipvs); +void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart); int ip_vs_est_kthread_start(struct netns_ipvs *ipvs, struct ip_vs_est_kt_data *kd); void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd); +static inline void ip_vs_stop_estimator_tot_stats(struct netns_ipvs *ipvs) +{ +#ifdef CONFIG_SYSCTL + ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s); + ipvs->tot_stats->s.est.ktid = -2; +#endif +} + static inline void ip_vs_est_stopped_recalc(struct netns_ipvs *ipvs) { #ifdef CONFIG_SYSCTL /* Stop tasks while cpulist is empty or if disabled with flag */ ipvs->est_stopped = !sysctl_run_estimation(ipvs) || (ipvs->est_cpulist_valid && - cpumask_empty(sysctl_est_cpulist(ipvs))); + sysctl_est_cpulist_empty(ipvs)); #endif } @@ -1620,7 +1965,7 @@ static inline bool ip_vs_est_stopped(struct netns_ipvs *ipvs) static inline int ip_vs_est_max_threads(struct netns_ipvs *ipvs) { unsigned int limit = IPVS_EST_CPU_KTHREADS * - cpumask_weight(sysctl_est_cpulist(ipvs)); + sysctl_est_cpulist_weight(ipvs); return max(1U, limit); } @@ -1637,8 +1982,8 @@ int ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, int ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, - struct ip_vs_protocol *pp, int offset, - unsigned int hooknum, struct ip_vs_iphdr *iph); + struct ip_vs_protocol *pp, unsigned int toff, + unsigned int hooknum, struct ip_vs_iphdr *ciph); void ip_vs_dest_dst_rcu_free(struct rcu_head *head); #ifdef CONFIG_IP_VS_IPV6 @@ -1651,8 +1996,8 @@ int ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, int ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, - struct ip_vs_protocol *pp, int offset, - unsigned int hooknum, struct ip_vs_iphdr *iph); + struct ip_vs_protocol *pp, unsigned int toff, + unsigned int hooknum, struct ip_vs_iphdr *ciph); #endif #ifdef CONFIG_SYSCTL @@ -1716,16 +2061,23 @@ static inline char ip_vs_fwd_tag(struct ip_vs_conn *cp) return fwd; } -void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, - struct ip_vs_conn *cp, int dir); +/* Check if connection uses double hashing */ +static inline bool ip_vs_conn_use_hash2(struct ip_vs_conn *cp) +{ + return IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ && + !(cp->flags & IP_VS_CONN_F_TEMPLATE); +} + +bool ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, + struct ip_vs_conn *cp, int dir, unsigned int toff, + bool has_ports, struct ip_vs_iphdr *ciph); #ifdef CONFIG_IP_VS_IPV6 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, - struct ip_vs_conn *cp, int dir); + struct ip_vs_conn *cp, int dir, unsigned int toff, + bool has_ports, struct ip_vs_iphdr *ciph); #endif -__sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset); - static inline __wsum ip_vs_check_diff4(__be32 old, __be32 new, __wsum oldsum) { __be32 diff[2] = { ~old, new }; @@ -1751,6 +2103,26 @@ static inline __wsum ip_vs_check_diff2(__be16 old, __be16 new, __wsum oldsum) return csum_partial(diff, sizeof(diff), oldsum); } +static inline bool ip_vs_checksum_needed(struct sk_buff *skb) +{ + /* Checksum unnecessary or already validated? */ + if (skb_csum_unnecessary(skb)) + return false; + /* Locally generated ? */ + if (!skb->dev) + return false; + return true; +} + +static inline bool ip_vs_checksum_common_check(struct sk_buff *skb, + int offset, int proto, int af) +{ + if (!ip_vs_checksum_needed(skb)) + return true; + /* Validate csum even for FORWARD */ + return !nf_checksum(skb, NF_INET_LOCAL_IN, offset, proto, af); +} + /* Forget current conntrack (unconfirmed) and attach notrack entry */ static inline void ip_vs_notrack(struct sk_buff *skb) { @@ -1759,7 +2131,7 @@ static inline void ip_vs_notrack(struct sk_buff *skb) struct nf_conn *ct = nf_ct_get(skb, &ctinfo); if (ct) { - nf_conntrack_put(&ct->ct_general); + nf_reset_ct(skb); nf_ct_set(skb, NULL, IP_CT_UNTRACKED); } #endif @@ -1782,7 +2154,7 @@ void ip_vs_update_conntrack(struct sk_buff *skb, struct ip_vs_conn *cp, int outin); int ip_vs_confirm_conntrack(struct sk_buff *skb); void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct, - struct ip_vs_conn *cp, u_int8_t proto, + struct ip_vs_conn *cp, u8 proto, const __be16 port, int from_rs); void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp); @@ -1858,14 +2230,21 @@ void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af); static inline int ip_vs_dest_conn_overhead(struct ip_vs_dest *dest) { - /* We think the overhead of processing active connections is 256 + /* We think the overhead of processing active connections is 257 * times higher than that of inactive connections in average. (This - * 256 times might not be accurate, we will change it later) We + * 257 times might not be accurate, we will change it later) We * use the following formula to estimate the overhead now: - * dest->activeconns*256 + dest->inactconns + * dest->activeconns*256 + dest->totalconns */ return (atomic_read(&dest->activeconns) << 8) + - atomic_read(&dest->inactconns); + atomic_read(&dest->totalconns); +} + +static inline int +ip_vs_dest_inactconns(const struct ip_vs_dest *dest) +{ + return max(atomic_read(&dest->totalconns) - + atomic_read(&dest->activeconns), 0); } #ifdef CONFIG_IP_VS_PROTO_TCP diff --git a/include/net/ipv6.h b/include/net/ipv6.h index 74fbf1ad8065..3de07e738538 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -25,8 +25,6 @@ struct ip_tunnel_info; #define SIN6_LEN_RFC2133 24 -#define IPV6_MAXPLEN 65535 - /* * NextHeader field of IPv6 header */ @@ -92,6 +90,9 @@ struct ip_tunnel_info; #define IP6_DEFAULT_MAX_DST_OPTS_LEN INT_MAX /* No limit */ #define IP6_DEFAULT_MAX_HBH_OPTS_LEN INT_MAX /* No limit */ +/* Hard limit on traversed IPv6 extension headers */ +#define IP6_MAX_EXT_HDRS_CNT 12 + /* * Addr type * @@ -151,17 +152,6 @@ struct frag_hdr { __be32 identification; }; -/* - * Jumbo payload option, as described in RFC 2675 2. - */ -struct hop_jumbo_hdr { - u8 nexthdr; - u8 hdrlen; - u8 tlv_type; /* IPV6_TLV_JUMBO, 0xC2 */ - u8 tlv_len; /* 4 */ - __be32 jumbo_payload_len; -}; - #define IP6_MF 0x0001 #define IP6_OFFSET 0xFFF8 @@ -464,72 +454,6 @@ bool ipv6_opt_accepted(const struct sock *sk, const struct sk_buff *skb, struct ipv6_txoptions *ipv6_update_options(struct sock *sk, struct ipv6_txoptions *opt); -/* This helper is specialized for BIG TCP needs. - * It assumes the hop_jumbo_hdr will immediately follow the IPV6 header. - * It assumes headers are already in skb->head. - * Returns: 0, or IPPROTO_TCP if a BIG TCP packet is there. - */ -static inline int ipv6_has_hopopt_jumbo(const struct sk_buff *skb) -{ - const struct hop_jumbo_hdr *jhdr; - const struct ipv6hdr *nhdr; - - if (likely(skb->len <= GRO_LEGACY_MAX_SIZE)) - return 0; - - if (skb->protocol != htons(ETH_P_IPV6)) - return 0; - - if (skb_network_offset(skb) + - sizeof(struct ipv6hdr) + - sizeof(struct hop_jumbo_hdr) > skb_headlen(skb)) - return 0; - - nhdr = ipv6_hdr(skb); - - if (nhdr->nexthdr != NEXTHDR_HOP) - return 0; - - jhdr = (const struct hop_jumbo_hdr *) (nhdr + 1); - if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 || - jhdr->nexthdr != IPPROTO_TCP) - return 0; - return jhdr->nexthdr; -} - -/* Return 0 if HBH header is successfully removed - * Or if HBH removal is unnecessary (packet is not big TCP) - * Return error to indicate dropping the packet - */ -static inline int ipv6_hopopt_jumbo_remove(struct sk_buff *skb) -{ - const int hophdr_len = sizeof(struct hop_jumbo_hdr); - int nexthdr = ipv6_has_hopopt_jumbo(skb); - struct ipv6hdr *h6; - - if (!nexthdr) - return 0; - - if (skb_cow_head(skb, 0)) - return -1; - - /* Remove the HBH header. - * Layout: [Ethernet header][IPv6 header][HBH][L4 Header] - */ - memmove(skb_mac_header(skb) + hophdr_len, skb_mac_header(skb), - skb_network_header(skb) - skb_mac_header(skb) + - sizeof(struct ipv6hdr)); - - __skb_pull(skb, hophdr_len); - skb->network_header += hophdr_len; - skb->mac_header += hophdr_len; - - h6 = ipv6_hdr(skb); - h6->nexthdr = nexthdr; - - return 0; -} - static inline bool ipv6_accept_ra(const struct inet6_dev *idev) { s32 accept_ra = READ_ONCE(idev->cnf.accept_ra); @@ -931,10 +855,10 @@ static inline void iph_to_flow_copy_v6addrs(struct flow_keys *flow, #if IS_ENABLED(CONFIG_IPV6) -static inline bool ipv6_can_nonlocal_bind(struct net *net, - struct inet_sock *inet) +static inline bool ipv6_can_nonlocal_bind(const struct net *net, + const struct inet_sock *inet) { - return net->ipv6.sysctl.ip_nonlocal_bind || + return READ_ONCE(net->ipv6.sysctl.ip_nonlocal_bind) || test_bit(INET_FLAGS_FREEBIND, &inet->inet_flags) || test_bit(INET_FLAGS_TRANSPARENT, &inet->inet_flags); } @@ -949,10 +873,12 @@ static inline bool ipv6_can_nonlocal_bind(struct net *net, #define IP6_DEFAULT_AUTO_FLOW_LABELS IP6_AUTO_FLOW_LABEL_OPTOUT -static inline __be32 ip6_make_flowlabel(struct net *net, struct sk_buff *skb, +static inline __be32 ip6_make_flowlabel(const struct net *net, + struct sk_buff *skb, __be32 flowlabel, bool autolabel, struct flowi6 *fl6) { + u8 auto_flowlabels; u32 hash; /* @flowlabel may include more than a flow label, eg, the traffic class. @@ -960,10 +886,12 @@ static inline __be32 ip6_make_flowlabel(struct net *net, struct sk_buff *skb, */ flowlabel &= IPV6_FLOWLABEL_MASK; - if (flowlabel || - net->ipv6.sysctl.auto_flowlabels == IP6_AUTO_FLOW_LABEL_OFF || - (!autolabel && - net->ipv6.sysctl.auto_flowlabels != IP6_AUTO_FLOW_LABEL_FORCED)) + if (flowlabel) + return flowlabel; + + auto_flowlabels = READ_ONCE(net->ipv6.sysctl.auto_flowlabels); + if (auto_flowlabels == IP6_AUTO_FLOW_LABEL_OFF || + (!autolabel && auto_flowlabels != IP6_AUTO_FLOW_LABEL_FORCED)) return flowlabel; hash = skb_get_hash_flowi6(skb, fl6); @@ -976,15 +904,15 @@ static inline __be32 ip6_make_flowlabel(struct net *net, struct sk_buff *skb, flowlabel = (__force __be32)hash & IPV6_FLOWLABEL_MASK; - if (net->ipv6.sysctl.flowlabel_state_ranges) + if (READ_ONCE(net->ipv6.sysctl.flowlabel_state_ranges)) flowlabel |= IPV6_FLOWLABEL_STATELESS_FLAG; return flowlabel; } -static inline int ip6_default_np_autolabel(struct net *net) +static inline int ip6_default_np_autolabel(const struct net *net) { - switch (net->ipv6.sysctl.auto_flowlabels) { + switch (READ_ONCE(net->ipv6.sysctl.auto_flowlabels)) { case IP6_AUTO_FLOW_LABEL_OFF: case IP6_AUTO_FLOW_LABEL_OPTIN: default: @@ -995,13 +923,13 @@ static inline int ip6_default_np_autolabel(struct net *net) } } #else -static inline __be32 ip6_make_flowlabel(struct net *net, struct sk_buff *skb, +static inline __be32 ip6_make_flowlabel(const struct net *net, struct sk_buff *skb, __be32 flowlabel, bool autolabel, struct flowi6 *fl6) { return flowlabel; } -static inline int ip6_default_np_autolabel(struct net *net) +static inline int ip6_default_np_autolabel(const struct net *net) { return 0; } @@ -1010,11 +938,11 @@ static inline int ip6_default_np_autolabel(struct net *net) #if IS_ENABLED(CONFIG_IPV6) static inline int ip6_multipath_hash_policy(const struct net *net) { - return net->ipv6.sysctl.multipath_hash_policy; + return READ_ONCE(net->ipv6.sysctl.multipath_hash_policy); } static inline u32 ip6_multipath_hash_fields(const struct net *net) { - return net->ipv6.sysctl.multipath_hash_fields; + return READ_ONCE(net->ipv6.sysctl.multipath_hash_fields); } #else static inline int ip6_multipath_hash_policy(const struct net *net) @@ -1027,6 +955,18 @@ static inline u32 ip6_multipath_hash_fields(const struct net *net) } #endif +/* Derive the IPv6 ECMP hash from txhash so a rehash may pick a different path; + * policy 0 only, and only when txhash is set. >> 1 clears the top bit + * (fib6_select_path() uses mp_hash as a signed 31-bit value); ?: 1 keeps the + * result non-zero, since mp_hash 0 falls back to rt6_multipath_hash(). + */ +static inline void ip6_ecmp_set_mp_hash(const struct net *net, + struct flowi6 *fl6, u32 txhash) +{ + if (ip6_multipath_hash_policy(net) == 0 && txhash) + fl6->mp_hash = (txhash >> 1) ?: 1; +} + /* * Header manipulation */ @@ -1103,8 +1043,7 @@ void ip6_flush_pending_frames(struct sock *sk); int ip6_send_skb(struct sk_buff *skb); struct sk_buff *__ip6_make_skb(struct sock *sk, struct sk_buff_head *queue, - struct inet_cork_full *cork, - struct inet6_cork *v6_cork); + struct inet_cork_full *cork); struct sk_buff *ip6_make_skb(struct sock *sk, int getfrag(void *from, char *to, int offset, int len, int odd, struct sk_buff *skb), @@ -1115,14 +1054,23 @@ struct sk_buff *ip6_make_skb(struct sock *sk, static inline struct sk_buff *ip6_finish_skb(struct sock *sk) { - return __ip6_make_skb(sk, &sk->sk_write_queue, &inet_sk(sk)->cork, - &inet6_sk(sk)->cork); + return __ip6_make_skb(sk, &sk->sk_write_queue, &inet_sk(sk)->cork); } int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst, struct flowi6 *fl6); +#if IS_ENABLED(CONFIG_IPV6) struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, struct flowi6 *fl6, const struct in6_addr *final_dst); +#else +static inline struct dst_entry *ip6_dst_lookup_flow(struct net *net, const struct sock *sk, + struct flowi6 *fl6, + const struct in6_addr *final_dst) +{ + return ERR_PTR(-EAFNOSUPPORT); +} +#endif + struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6, const struct in6_addr *final_dst, bool connected); @@ -1147,11 +1095,11 @@ int ip6_local_out(struct net *net, struct sock *sk, struct sk_buff *skb); * Extension header (options) processing */ -void ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, - u8 *proto, struct in6_addr **daddr_p, - struct in6_addr *saddr); -void ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, - u8 *proto); +u8 ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, + u8 proto, struct in6_addr **daddr_p, + struct in6_addr *saddr); +u8 ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, + u8 proto); int ipv6_skip_exthdr(const struct sk_buff *, int start, u8 *nexthdrp, __be16 *frag_offp); @@ -1170,9 +1118,19 @@ int ipv6_find_hdr(const struct sk_buff *skb, unsigned int *offset, int target, int ipv6_find_tlv(const struct sk_buff *skb, int offset, int type); -struct in6_addr *fl6_update_dst(struct flowi6 *fl6, - const struct ipv6_txoptions *opt, - struct in6_addr *orig); +struct in6_addr *__fl6_update_dst(struct flowi6 *fl6, + const struct ipv6_txoptions *opt, + struct in6_addr *orig); + +static inline struct in6_addr * +fl6_update_dst(struct flowi6 *fl6, const struct ipv6_txoptions *opt, + struct in6_addr *orig) +{ + if (likely(!opt)) + return NULL; + + return __fl6_update_dst(fl6, opt, orig); +} /* * socket options (ipv6_sockglue.c) @@ -1196,10 +1154,8 @@ int ip6_datagram_connect_v6_only(struct sock *sk, struct sockaddr_unsized *addr, int ip6_datagram_dst_update(struct sock *sk, bool fix_sk_saddr); void ip6_datagram_release_cb(struct sock *sk); -int ipv6_recv_error(struct sock *sk, struct msghdr *msg, int len, - int *addr_len); -int ipv6_recv_rxpmtu(struct sock *sk, struct msghdr *msg, int len, - int *addr_len); +int ipv6_recv_error(struct sock *sk, struct msghdr *msg, int len); +int ipv6_recv_rxpmtu(struct sock *sk, struct msghdr *msg, int len); void ipv6_icmp_error(struct sock *sk, struct sk_buff *skb, int err, __be16 port, u32 info, u8 *payload); void ipv6_local_error(struct sock *sk, int err, struct flowi6 *fl6, u32 info); @@ -1208,6 +1164,8 @@ void ipv6_local_rxpmtu(struct sock *sk, struct flowi6 *fl6, u32 mtu); void inet6_cleanup_sock(struct sock *sk); void inet6_sock_destruct(struct sock *sk); int inet6_release(struct socket *sock); +int __inet6_bind(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len, + u32 flags); int inet6_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int addr_len); int inet6_bind_sk(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len); int inet6_getname(struct socket *sock, struct sockaddr *uaddr, @@ -1248,8 +1206,6 @@ int tcp6_proc_init(struct net *net); void tcp6_proc_exit(struct net *net); int udp6_proc_init(struct net *net); void udp6_proc_exit(struct net *net); -int udplite6_proc_init(void); -void udplite6_proc_exit(void); int ipv6_misc_proc_init(void); void ipv6_misc_proc_exit(void); int snmp6_register_dev(struct inet6_dev *idev); @@ -1280,12 +1236,15 @@ int ipv6_sock_mc_drop(struct sock *sk, int ifindex, static inline int ip6_sock_set_v6only(struct sock *sk) { - if (inet_sk(sk)->inet_num) - return -EINVAL; + int ret = 0; + lock_sock(sk); - sk->sk_ipv6only = true; + if (inet_sk(sk)->inet_num) + ret = -EINVAL; + else + sk->sk_ipv6only = true; release_sock(sk); - return 0; + return ret; } static inline void ip6_sock_set_recverr(struct sock *sk) diff --git a/include/net/ipv6_stubs.h b/include/net/ipv6_stubs.h deleted file mode 100644 index d3013e721b14..000000000000 --- a/include/net/ipv6_stubs.h +++ /dev/null @@ -1,102 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _IPV6_STUBS_H -#define _IPV6_STUBS_H - -#include <linux/in6.h> -#include <linux/netdevice.h> -#include <linux/skbuff.h> -#include <net/dst.h> -#include <net/flow.h> -#include <net/neighbour.h> -#include <net/sock.h> -#include <net/ipv6.h> - -/* structs from net/ip6_fib.h */ -struct fib6_info; -struct fib6_nh; -struct fib6_config; -struct fib6_result; - -/* This is ugly, ideally these symbols should be built - * into the core kernel. - */ -struct ipv6_stub { - int (*ipv6_sock_mc_join)(struct sock *sk, int ifindex, - const struct in6_addr *addr); - int (*ipv6_sock_mc_drop)(struct sock *sk, int ifindex, - const struct in6_addr *addr); - struct dst_entry *(*ipv6_dst_lookup_flow)(struct net *net, - const struct sock *sk, - struct flowi6 *fl6, - const struct in6_addr *final_dst); - int (*ipv6_route_input)(struct sk_buff *skb); - - struct fib6_table *(*fib6_get_table)(struct net *net, u32 id); - int (*fib6_lookup)(struct net *net, int oif, struct flowi6 *fl6, - struct fib6_result *res, int flags); - int (*fib6_table_lookup)(struct net *net, struct fib6_table *table, - int oif, struct flowi6 *fl6, - struct fib6_result *res, int flags); - void (*fib6_select_path)(const struct net *net, struct fib6_result *res, - struct flowi6 *fl6, int oif, bool oif_match, - const struct sk_buff *skb, int strict); - u32 (*ip6_mtu_from_fib6)(const struct fib6_result *res, - const struct in6_addr *daddr, - const struct in6_addr *saddr); - - int (*fib6_nh_init)(struct net *net, struct fib6_nh *fib6_nh, - struct fib6_config *cfg, gfp_t gfp_flags, - struct netlink_ext_ack *extack); - void (*fib6_nh_release)(struct fib6_nh *fib6_nh); - void (*fib6_nh_release_dsts)(struct fib6_nh *fib6_nh); - void (*fib6_update_sernum)(struct net *net, struct fib6_info *rt); - int (*ip6_del_rt)(struct net *net, struct fib6_info *rt, bool skip_notify); - void (*fib6_rt_update)(struct net *net, struct fib6_info *rt, - struct nl_info *info); - - void (*udpv6_encap_enable)(void); - void (*ndisc_send_na)(struct net_device *dev, const struct in6_addr *daddr, - const struct in6_addr *solicited_addr, - bool router, bool solicited, bool override, bool inc_opt); -#if IS_ENABLED(CONFIG_XFRM) - void (*xfrm6_local_rxpmtu)(struct sk_buff *skb, u32 mtu); - int (*xfrm6_udp_encap_rcv)(struct sock *sk, struct sk_buff *skb); - struct sk_buff *(*xfrm6_gro_udp_encap_rcv)(struct sock *sk, - struct list_head *head, - struct sk_buff *skb); - int (*xfrm6_rcv_encap)(struct sk_buff *skb, int nexthdr, __be32 spi, - int encap_type); -#endif - struct neigh_table *nd_tbl; - - int (*ipv6_fragment)(struct net *net, struct sock *sk, struct sk_buff *skb, - int (*output)(struct net *, struct sock *, struct sk_buff *)); - struct net_device *(*ipv6_dev_find)(struct net *net, const struct in6_addr *addr, - struct net_device *dev); - int (*ip6_xmit)(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, - __u32 mark, struct ipv6_txoptions *opt, int tclass, u32 priority); -}; -extern const struct ipv6_stub *ipv6_stub __read_mostly; - -/* A stub used by bpf helpers. Similarly ugly as ipv6_stub */ -struct ipv6_bpf_stub { - int (*inet6_bind)(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len, - u32 flags); - struct sock *(*udp6_lib_lookup)(const struct net *net, - const struct in6_addr *saddr, __be16 sport, - const struct in6_addr *daddr, __be16 dport, - int dif, int sdif, struct udp_table *tbl, - struct sk_buff *skb); - int (*ipv6_setsockopt)(struct sock *sk, int level, int optname, - sockptr_t optval, unsigned int optlen); - int (*ipv6_getsockopt)(struct sock *sk, int level, int optname, - sockptr_t optval, sockptr_t optlen); - int (*ipv6_dev_get_saddr)(struct net *net, - const struct net_device *dst_dev, - const struct in6_addr *daddr, - unsigned int prefs, - struct in6_addr *saddr); -}; -extern const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly; - -#endif diff --git a/include/net/iucv/iucv.h b/include/net/iucv/iucv.h index 9804fa5d9c67..18f511da9a09 100644 --- a/include/net/iucv/iucv.h +++ b/include/net/iucv/iucv.h @@ -195,35 +195,15 @@ struct iucv_handler { struct list_head paths; }; -/** - * iucv_register: - * @handler: address of iucv handler structure - * @smp: != 0 indicates that the handler can deal with out of order messages - * - * Registers a driver with IUCV. - * - * Returns: 0 on success, -ENOMEM if the memory allocation for the pathid - * table failed, or -EIO if IUCV_DECLARE_BUFFER failed on all cpus. - */ int iucv_register(struct iucv_handler *handler, int smp); +void iucv_unregister(struct iucv_handler *handler, int smp); /** - * iucv_unregister - * @handler: address of iucv handler structure - * @smp: != 0 indicates that the handler can deal with out of order messages - * - * Unregister driver from IUCV. - */ -void iucv_unregister(struct iucv_handler *handle, int smp); - -/** - * iucv_path_alloc + * iucv_path_alloc - Allocate a new path structure for use with iucv_connect. * @msglim: initial message limit * @flags: initial flags * @gfp: kmalloc allocation flag * - * Allocate a new path structure for use with iucv_connect. - * * Returns: NULL if the memory allocation failed or a pointer to the * path structure. */ @@ -231,7 +211,7 @@ static inline struct iucv_path *iucv_path_alloc(u16 msglim, u8 flags, gfp_t gfp) { struct iucv_path *path; - path = kzalloc(sizeof(struct iucv_path), gfp); + path = kzalloc_obj(struct iucv_path, gfp); if (path) { path->msglim = msglim; path->flags = flags; @@ -240,229 +220,48 @@ static inline struct iucv_path *iucv_path_alloc(u16 msglim, u8 flags, gfp_t gfp) } /** - * iucv_path_free + * iucv_path_free - Frees a path structure. * @path: address of iucv path structure - * - * Frees a path structure. */ static inline void iucv_path_free(struct iucv_path *path) { kfree(path); } -/** - * iucv_path_accept - * @path: address of iucv path structure - * @handler: address of iucv handler structure - * @userdata: 16 bytes of data reflected to the communication partner - * @private: private data passed to interrupt handlers for this path - * - * This function is issued after the user received a connection pending - * external interrupt and now wishes to complete the IUCV communication path. - * - * Returns: the result of the CP IUCV call. - */ int iucv_path_accept(struct iucv_path *path, struct iucv_handler *handler, u8 *userdata, void *private); -/** - * iucv_path_connect - * @path: address of iucv path structure - * @handler: address of iucv handler structure - * @userid: 8-byte user identification - * @system: 8-byte target system identification - * @userdata: 16 bytes of data reflected to the communication partner - * @private: private data passed to interrupt handlers for this path - * - * This function establishes an IUCV path. Although the connect may complete - * successfully, you are not able to use the path until you receive an IUCV - * Connection Complete external interrupt. - * - * Returns: the result of the CP IUCV call. - */ int iucv_path_connect(struct iucv_path *path, struct iucv_handler *handler, u8 *userid, u8 *system, u8 *userdata, void *private); -/** - * iucv_path_quiesce: - * @path: address of iucv path structure - * @userdata: 16 bytes of data reflected to the communication partner - * - * This function temporarily suspends incoming messages on an IUCV path. - * You can later reactivate the path by invoking the iucv_resume function. - * - * Returns: the result from the CP IUCV call. - */ int iucv_path_quiesce(struct iucv_path *path, u8 *userdata); -/** - * iucv_path_resume: - * @path: address of iucv path structure - * @userdata: 16 bytes of data reflected to the communication partner - * - * This function resumes incoming messages on an IUCV path that has - * been stopped with iucv_path_quiesce. - * - * Returns: the result from the CP IUCV call. - */ int iucv_path_resume(struct iucv_path *path, u8 *userdata); -/** - * iucv_path_sever - * @path: address of iucv path structure - * @userdata: 16 bytes of data reflected to the communication partner - * - * This function terminates an IUCV path. - * - * Returns: the result from the CP IUCV call. - */ int iucv_path_sever(struct iucv_path *path, u8 *userdata); -/** - * iucv_message_purge - * @path: address of iucv path structure - * @msg: address of iucv msg structure - * @srccls: source class of message - * - * Cancels a message you have sent. - * - * Returns: the result from the CP IUCV call. - */ int iucv_message_purge(struct iucv_path *path, struct iucv_message *msg, u32 srccls); -/** - * iucv_message_receive - * @path: address of iucv path structure - * @msg: address of iucv msg structure - * @flags: flags that affect how the message is received (IUCV_IPBUFLST) - * @buffer: address of data buffer or address of struct iucv_array - * @size: length of data buffer - * @residual: - * - * This function receives messages that are being sent to you over - * established paths. This function will deal with RMDATA messages - * embedded in struct iucv_message as well. - * - * Locking: local_bh_enable/local_bh_disable - * - * Returns: the result from the CP IUCV call. - */ int iucv_message_receive(struct iucv_path *path, struct iucv_message *msg, u8 flags, void *buffer, size_t size, size_t *residual); -/** - * __iucv_message_receive - * @path: address of iucv path structure - * @msg: address of iucv msg structure - * @flags: flags that affect how the message is received (IUCV_IPBUFLST) - * @buffer: address of data buffer or address of struct iucv_array - * @size: length of data buffer - * @residual: - * - * This function receives messages that are being sent to you over - * established paths. This function will deal with RMDATA messages - * embedded in struct iucv_message as well. - * - * Locking: no locking. - * - * Returns: the result from the CP IUCV call. - */ int __iucv_message_receive(struct iucv_path *path, struct iucv_message *msg, u8 flags, void *buffer, size_t size, size_t *residual); -/** - * iucv_message_reject - * @path: address of iucv path structure - * @msg: address of iucv msg structure - * - * The reject function refuses a specified message. Between the time you - * are notified of a message and the time that you complete the message, - * the message may be rejected. - * - * Returns: the result from the CP IUCV call. - */ int iucv_message_reject(struct iucv_path *path, struct iucv_message *msg); -/** - * iucv_message_reply - * @path: address of iucv path structure - * @msg: address of iucv msg structure - * @flags: how the reply is sent (IUCV_IPRMDATA, IUCV_IPPRTY, IUCV_IPBUFLST) - * @reply: address of data buffer or address of struct iucv_array - * @size: length of reply data buffer - * - * This function responds to the two-way messages that you receive. You - * must identify completely the message to which you wish to reply. ie, - * pathid, msgid, and trgcls. Prmmsg signifies the data is moved into - * the parameter list. - * - * Returns: the result from the CP IUCV call. - */ int iucv_message_reply(struct iucv_path *path, struct iucv_message *msg, u8 flags, void *reply, size_t size); -/** - * iucv_message_send - * @path: address of iucv path structure - * @msg: address of iucv msg structure - * @flags: how the message is sent (IUCV_IPRMDATA, IUCV_IPPRTY, IUCV_IPBUFLST) - * @srccls: source class of message - * @buffer: address of data buffer or address of struct iucv_array - * @size: length of send buffer - * - * This function transmits data to another application. Data to be - * transmitted is in a buffer and this is a one-way message and the - * receiver will not reply to the message. - * - * Locking: local_bh_enable/local_bh_disable - * - * Returns: the result from the CP IUCV call. - */ int iucv_message_send(struct iucv_path *path, struct iucv_message *msg, u8 flags, u32 srccls, void *buffer, size_t size); -/** - * __iucv_message_send - * @path: address of iucv path structure - * @msg: address of iucv msg structure - * @flags: how the message is sent (IUCV_IPRMDATA, IUCV_IPPRTY, IUCV_IPBUFLST) - * @srccls: source class of message - * @buffer: address of data buffer or address of struct iucv_array - * @size: length of send buffer - * - * This function transmits data to another application. Data to be - * transmitted is in a buffer and this is a one-way message and the - * receiver will not reply to the message. - * - * Locking: no locking. - * - * Returns: the result from the CP IUCV call. - */ int __iucv_message_send(struct iucv_path *path, struct iucv_message *msg, u8 flags, u32 srccls, void *buffer, size_t size); -/** - * iucv_message_send2way - * @path: address of iucv path structure - * @msg: address of iucv msg structure - * @flags: how the message is sent and the reply is received - * (IUCV_IPRMDATA, IUCV_IPBUFLST, IUCV_IPPRTY, IUCV_ANSLST) - * @srccls: source class of message - * @buffer: address of data buffer or address of struct iucv_array - * @size: length of send buffer - * @ansbuf: address of answer buffer or address of struct iucv_array - * @asize: size of reply buffer - * - * This function transmits data to another application. Data to be - * transmitted is in a buffer. The receiver of the send is expected to - * reply to the message and a buffer is provided into which IUCV moves - * the reply to this message. - * - * Returns: the result from the CP IUCV call. - */ int iucv_message_send2way(struct iucv_path *path, struct iucv_message *msg, u8 flags, u32 srccls, void *buffer, size_t size, void *answer, size_t asize, size_t *residual); diff --git a/include/net/l3mdev.h b/include/net/l3mdev.h index 1eb8dad18f7e..710e98665eb3 100644 --- a/include/net/l3mdev.h +++ b/include/net/l3mdev.h @@ -207,18 +207,19 @@ struct sk_buff *l3mdev_ip6_rcv(struct sk_buff *skb) static inline struct sk_buff *l3mdev_l3_out(struct sock *sk, struct sk_buff *skb, u16 proto) { - struct net_device *dev = skb_dst(skb)->dev; + struct net_device *dev; + rcu_read_lock(); + dev = skb_dst_dev_rcu(skb); if (netif_is_l3_slave(dev)) { struct net_device *master; - rcu_read_lock(); master = netdev_master_upper_dev_get_rcu(dev); if (master && master->l3mdev_ops->l3mdev_l3_out) skb = master->l3mdev_ops->l3mdev_l3_out(master, sk, skb, proto); - rcu_read_unlock(); } + rcu_read_unlock(); return skb; } diff --git a/include/net/libeth/rx.h b/include/net/libeth/rx.h index 5d991404845e..0e736846c5e8 100644 --- a/include/net/libeth/rx.h +++ b/include/net/libeth/rx.h @@ -69,6 +69,7 @@ enum libeth_fqe_type { * @type: type of the buffers this queue has * @hsplit: flag whether header split is enabled * @xdp: flag indicating whether XDP is enabled + * @no_napi: the queue is not a data queue and does not have NAPI * @buf_len: HW-writeable length per each buffer * @nid: ID of the closest NUMA node with memory */ @@ -85,12 +86,13 @@ struct libeth_fq { enum libeth_fqe_type type:2; bool hsplit:1; bool xdp:1; + bool no_napi:1; u32 buf_len; int nid; }; -int libeth_rx_fq_create(struct libeth_fq *fq, struct napi_struct *napi); +int libeth_rx_fq_create(struct libeth_fq *fq, void *napi_dev); void libeth_rx_fq_destroy(struct libeth_fq *fq); /** diff --git a/include/net/libeth/xsk.h b/include/net/libeth/xsk.h index 481a7b28e6f2..a452b7828ce4 100644 --- a/include/net/libeth/xsk.h +++ b/include/net/libeth/xsk.h @@ -196,7 +196,7 @@ __libeth_xsk_xmit_fill_buf_md(const struct xdp_desc *xdesc, struct libeth_xdp_tx_desc desc; struct xdp_desc_ctx ctx; - ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr); + ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr, xdesc->options); desc = (typeof(desc)){ .addr = ctx.dma, __libeth_xdp_tx_len(xdesc->len), @@ -205,7 +205,7 @@ __libeth_xsk_xmit_fill_buf_md(const struct xdp_desc *xdesc, BUILD_BUG_ON(!__builtin_constant_p(tmo == libeth_xsktmo)); tmo = tmo == libeth_xsktmo ? &__libeth_xsktmo : tmo; - xsk_tx_metadata_request(ctx.meta, tmo, &desc); + xsk_tx_metadata_request(sq->pool, &ctx.meta, tmo, &desc); return desc; } @@ -597,6 +597,7 @@ __libeth_xsk_run_pass(struct libeth_xdp_buff *xdp, * @pending: current number of XSkFQEs to refill * @thresh: threshold below which the queue is refilled * @buf_len: HW-writeable length per each buffer + * @truesize: step between consecutive buffers, 0 if none exists * @nid: ID of the closest NUMA node with memory */ struct libeth_xskfq { @@ -614,6 +615,8 @@ struct libeth_xskfq { u32 thresh; u32 buf_len; + u32 truesize; + int nid; }; diff --git a/include/net/llc.h b/include/net/llc.h index e250dca03963..029ba8a22319 100644 --- a/include/net/llc.h +++ b/include/net/llc.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_H #define LLC_H /* * Copyright (c) 1997 by Procom Technology, Inc. * 2001-2003 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ #include <linux/if.h> diff --git a/include/net/llc_c_ac.h b/include/net/llc_c_ac.h index 7620a9196922..af31a49bebb0 100644 --- a/include/net/llc_c_ac.h +++ b/include/net/llc_c_ac.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_C_AC_H #define LLC_C_AC_H /* * Copyright (c) 1997 by Procom Technology,Inc. * 2001 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ /* Connection component state transition actions */ /* diff --git a/include/net/llc_c_ev.h b/include/net/llc_c_ev.h index 241889955157..d45140d4289a 100644 --- a/include/net/llc_c_ev.h +++ b/include/net/llc_c_ev.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_C_EV_H #define LLC_C_EV_H /* * Copyright (c) 1997 by Procom Technology,Inc. * 2001 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ #include <net/sock.h> diff --git a/include/net/llc_c_st.h b/include/net/llc_c_st.h index a4bea0f33188..f52a4cc4880d 100644 --- a/include/net/llc_c_st.h +++ b/include/net/llc_c_st.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_C_ST_H #define LLC_C_ST_H /* * Copyright (c) 1997 by Procom Technology,Inc. * 2001 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ #include <net/llc_c_ac.h> diff --git a/include/net/llc_conn.h b/include/net/llc_conn.h index 374411b3066c..e1a302696723 100644 --- a/include/net/llc_conn.h +++ b/include/net/llc_conn.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_CONN_H #define LLC_CONN_H /* * Copyright (c) 1997 by Procom Technology, Inc. * 2001, 2002 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ #include <linux/timer.h> #include <net/llc_if.h> diff --git a/include/net/llc_if.h b/include/net/llc_if.h index c72570a21a4f..a333259613ab 100644 --- a/include/net/llc_if.h +++ b/include/net/llc_if.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_IF_H #define LLC_IF_H /* * Copyright (c) 1997 by Procom Technology,Inc. * 2001 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ /* Defines LLC interface to network layer */ /* Available primitives */ diff --git a/include/net/llc_pdu.h b/include/net/llc_pdu.h index 86681f29bda7..f165237a3723 100644 --- a/include/net/llc_pdu.h +++ b/include/net/llc_pdu.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_PDU_H #define LLC_PDU_H /* * Copyright (c) 1997 by Procom Technology,Inc. * 2001-2003 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ #include <linux/if_ether.h> diff --git a/include/net/llc_s_ac.h b/include/net/llc_s_ac.h index f71790305bc9..076adc5217fd 100644 --- a/include/net/llc_s_ac.h +++ b/include/net/llc_s_ac.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_S_AC_H #define LLC_S_AC_H /* * Copyright (c) 1997 by Procom Technology,Inc. * 2001 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ struct llc_sap; diff --git a/include/net/llc_s_ev.h b/include/net/llc_s_ev.h index fb7df1d70af3..7b9c94093336 100644 --- a/include/net/llc_s_ev.h +++ b/include/net/llc_s_ev.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_S_EV_H #define LLC_S_EV_H /* * Copyright (c) 1997 by Procom Technology,Inc. * 2001 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ #include <linux/skbuff.h> diff --git a/include/net/llc_s_st.h b/include/net/llc_s_st.h index fca49d483d20..85ce230fd946 100644 --- a/include/net/llc_s_st.h +++ b/include/net/llc_s_st.h @@ -1,15 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_S_ST_H #define LLC_S_ST_H /* * Copyright (c) 1997 by Procom Technology,Inc. * 2001 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ #include <linux/types.h> diff --git a/include/net/llc_sap.h b/include/net/llc_sap.h index 1e4df9fd9fb2..b259f5993c46 100644 --- a/include/net/llc_sap.h +++ b/include/net/llc_sap.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef LLC_SAP_H #define LLC_SAP_H @@ -6,13 +7,6 @@ /* * Copyright (c) 1997 by Procom Technology,Inc. * 2001-2003 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ struct llc_sap; struct net_device; diff --git a/include/net/mac80211.h b/include/net/mac80211.h index c2e49542626c..ed6a5874ff96 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -7,7 +7,7 @@ * Copyright 2007-2010 Johannes Berg <johannes@sipsolutions.net> * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright (C) 2015 - 2017 Intel Deutschland GmbH - * Copyright (C) 2018 - 2025 Intel Corporation + * Copyright (C) 2018 - 2026 Intel Corporation */ #ifndef MAC80211_H @@ -218,6 +218,8 @@ struct ieee80211_low_level_stats { * bandwidth) OFDMA settings need to be changed * @IEEE80211_CHANCTX_CHANGE_PUNCTURING: The punctured channel(s) bitmap * was changed. + * @IEEE80211_CHANCTX_CHANGE_NPCA: NPCA configuration changed + * @IEEE80211_CHANCTX_CHANGE_NPCA_PUNCT: NPCA puncturing changed */ enum ieee80211_chanctx_change { IEEE80211_CHANCTX_CHANGE_WIDTH = BIT(0), @@ -227,6 +229,8 @@ enum ieee80211_chanctx_change { IEEE80211_CHANCTX_CHANGE_MIN_DEF = BIT(4), IEEE80211_CHANCTX_CHANGE_AP = BIT(5), IEEE80211_CHANCTX_CHANGE_PUNCTURING = BIT(6), + IEEE80211_CHANCTX_CHANGE_NPCA = BIT(7), + IEEE80211_CHANCTX_CHANGE_NPCA_PUNCT = BIT(8), }; /** @@ -234,10 +238,13 @@ enum ieee80211_chanctx_change { * @oper: channel definition to use for operation * @ap: the channel definition of the AP, if any * (otherwise the chan member is %NULL) + * @require_npca: If NPCA is configured, require it to + * remain, this is used by AP interfaces */ struct ieee80211_chan_req { struct cfg80211_chan_def oper; struct cfg80211_chan_def ap; + bool require_npca; }; /** @@ -365,6 +372,8 @@ struct ieee80211_vif_chanctx_switch { * @BSS_CHANGED_MLD_VALID_LINKS: MLD valid links status changed. * @BSS_CHANGED_MLD_TTLM: negotiated TID to link mapping was changed * @BSS_CHANGED_TPE: transmit power envelope changed + * @BSS_CHANGED_NAN_LOCAL_SCHED: NAN local schedule changed (NAN mode only) + * @BSS_CHANGED_NPCA: NPCA parameters changed */ enum ieee80211_bss_change { BSS_CHANGED_ASSOC = 1<<0, @@ -402,6 +411,8 @@ enum ieee80211_bss_change { BSS_CHANGED_MLD_VALID_LINKS = BIT_ULL(33), BSS_CHANGED_MLD_TTLM = BIT_ULL(34), BSS_CHANGED_TPE = BIT_ULL(35), + BSS_CHANGED_NAN_LOCAL_SCHED = BIT_ULL(36), + BSS_CHANGED_NPCA = BIT_ULL(37), /* when adding here, make sure to change ieee80211_reconfig */ }; @@ -588,6 +599,26 @@ struct ieee80211_parsed_tpe { }; /** + * struct ieee80211_bss_npca_params - NPCA parameters + * @min_dur_thresh: NPCA minimum duration threshold (512 + 128*n usec) + * @switch_delay: NPCA switch delay (units of 4 usec) + * @switch_back_delay: NPCA switch back delay (units of 4 usec) + * @init_qsrc: initial QSRC value + * @moplen: indicates MOPLEN NPCA is permitted in the BSS + * @enabled: NPCA is enabled for this link + * + * Note: the individual values (except @enabled) are in spec representation. + */ +struct ieee80211_bss_npca_params { + u32 min_dur_thresh:4, + switch_delay:6, + switch_back_delay:6, + init_qsrc:2, + moplen:1, + enabled:1; +}; + +/** * struct ieee80211_bss_conf - holds the BSS's changing parameters * * This structure keeps information about a BSS (and an association @@ -706,6 +737,7 @@ struct ieee80211_parsed_tpe { * @pwr_reduction: power constraint of BSS. * @eht_support: does this BSS support EHT * @epcs_support: does this BSS support EPCS + * @uhr_support: does this BSS support UHR * @csa_active: marks whether a channel switch is going on. * @mu_mimo_owner: indicates interface owns MU-MIMO capability * @chanctx_conf: The channel context this interface is assigned to, or %NULL @@ -758,8 +790,13 @@ struct ieee80211_parsed_tpe { * be updated to 1, even if bss_param_ch_cnt didn't change. This allows * the link to know that it heard the latest value from its own beacon * (as opposed to hearing its value from another link's beacon). + * @enh_bss_param_ch_cnt: In BSS-mode, the enhanced BSS parameters change + * counter. See @bss_param_ch_cnt, it works the same way. + * @enh_bss_param_ch_cnt_link_id: In BSS-mode, the link_id for the enhanced + * BSS parameter change counter, see @bss_param_ch_cnt_link_id. * @s1g_long_beacon_period: number of beacon intervals between each long * beacon transmission. + * @npca: NPCA parameters */ struct ieee80211_bss_conf { struct ieee80211_vif *vif; @@ -832,6 +869,8 @@ struct ieee80211_bss_conf { u8 pwr_reduction; bool eht_support; bool epcs_support; + bool uhr_support; + bool csa_active; bool mu_mimo_owner; @@ -859,8 +898,80 @@ struct ieee80211_bss_conf { u8 bss_param_ch_cnt; u8 bss_param_ch_cnt_link_id; + u8 enh_bss_param_ch_cnt; + u8 enh_bss_param_ch_cnt_link_id; u8 s1g_long_beacon_period; + + struct ieee80211_bss_npca_params npca; +}; + +#define IEEE80211_NAN_MAX_CHANNELS 3 + +/** + * struct ieee80211_nan_channel - NAN channel information + * + * @chanreq: channel request for this NAN channel. Even though this chanreq::ap + * is irrelevant for NAN, still store it for convenience - some functions + * require it as an argument. + * @needed_rx_chains: number of RX chains needed for this NAN channel + * @chanctx_conf: chanctx_conf assigned to this NAN channel. + * If a local channel is being ULWed (because we needed this chanctx for + * something else), the local NAN channel that used this chanctx, + * will have this pointer set to %NULL. + * A peer NAN channel should never have this pointer set to %NULL. + * @channel_entry: the Channel Entry blob as defined in Wi-Fi Aware + * (TM) 4.0 specification Table 100 (Channel Entry format for the NAN + * Availability attribute). + */ +struct ieee80211_nan_channel { + struct ieee80211_chan_req chanreq; + u8 needed_rx_chains; + struct ieee80211_chanctx_conf *chanctx_conf; + u8 channel_entry[6]; +}; + +/** + * struct ieee80211_nan_peer_map - NAN peer schedule map + * + * This stores a single map from a peer's schedule. Each peer can have + * multiple maps. + * + * @map_id: the map ID from the peer schedule, %CFG80211_NAN_INVALID_MAP_ID + * if unused + * @slots: mapping of time slots to channel configurations in the schedule's + * channels array + */ +struct ieee80211_nan_peer_map { + u8 map_id; + struct ieee80211_nan_channel *slots[CFG80211_NAN_SCHED_NUM_TIME_SLOTS]; +}; + +/** + * struct ieee80211_nan_peer_sched - NAN peer schedule + * + * This stores the complete schedule from a peer. Contains peer-level + * parameters and an array of schedule maps. + * + * @seq_id: the sequence ID from the peer schedule + * @committed_dw: committed DW as published by the peer + * @max_chan_switch: maximum channel switch time in microseconds + * @init_ulw: initial ULWs as published by the peer (copied) + * @ulw_size: number of bytes in @init_ulw + * @maps: array of peer schedule maps. Invalid slots have map_id set to + * %CFG80211_NAN_INVALID_MAP_ID. + * @n_channels: number of valid channel entries in @channels + * @channels: flexible array of negotiated peer channels for this schedule + */ +struct ieee80211_nan_peer_sched { + u8 seq_id; + u16 committed_dw; + u16 max_chan_switch; + const u8 *init_ulw; + u16 ulw_size; + struct ieee80211_nan_peer_map maps[CFG80211_NAN_MAX_PEER_MAPS]; + u8 n_channels; + struct ieee80211_nan_channel channels[] __counted_by(n_channels); }; /** @@ -1212,6 +1323,7 @@ ieee80211_rate_get_vht_nss(const struct ieee80211_tx_rate *rate) * @status_data: internal data for TX status handling, assigned privately, * see also &enum ieee80211_status_data for the internal documentation * @status_data_idr: indicates status data is IDR allocated ID for ack frame + * @tx_time_mc: TX time estimate is for a multicast frame, used internally * @tx_time_est: TX time estimate in units of 4us, used internally * @control: union part for control data * @control.rates: TX rates array to try @@ -1237,6 +1349,11 @@ ieee80211_rate_get_vht_nss(const struct ieee80211_tx_rate *rate) * @status.tx_time: airtime consumed for transmission; note this is only * used for WMM AC, not for airtime fairness * @status.flags: status flags, see &enum mac80211_tx_status_flags + * @status.link_valid: if the link which is identified by @status.link_id is + * valid. This flag is set by the driver in the TX status callback when the + * connection is MLO and the driver knows which link was used for TX. + * @status.link_id: id of the link used to transmit the packet. This is used + * along with @status.link_valid. * @status.status_driver_data: driver use area * @ack: union part for pure ACK data * @ack.cookie: cookie for the ACK @@ -1249,8 +1366,8 @@ struct ieee80211_tx_info { status_data_idr:1, status_data:13, hw_queue:4, + tx_time_mc:1, tx_time_est:10; - /* 1 free bit */ union { struct { @@ -1291,7 +1408,7 @@ struct ieee80211_tx_info { u8 pad; u16 tx_time; u8 flags; - u8 pad2; + u8 link_valid:1, link_id:4; void *status_driver_data[16 / sizeof(void *)]; } status; struct { @@ -1598,6 +1715,8 @@ enum mac80211_rx_encoding { RX_ENC_VHT, RX_ENC_HE, RX_ENC_EHT, + RX_ENC_UHR, + RX_ENC_S1G, }; /** @@ -1631,7 +1750,7 @@ enum mac80211_rx_encoding { * @antenna: antenna used * @rate_idx: index of data rate into band's supported rates or MCS index if * HT or VHT is used (%RX_FLAG_HT/%RX_FLAG_VHT) - * @nss: number of streams (VHT, HE and EHT only) + * @nss: number of streams (VHT, HE, EHT and UHR only) * @flag: %RX_FLAG_\* * @encoding: &enum mac80211_rx_encoding * @bw: &enum rate_info_bw @@ -1642,6 +1761,11 @@ enum mac80211_rx_encoding { * @eht: EHT specific rate information * @eht.ru: EHT RU, from &enum nl80211_eht_ru_alloc * @eht.gi: EHT GI, from &enum nl80211_eht_gi + * @uhr: UHR specific rate information + * @uhr.ru: UHR RU, from &enum nl80211_eht_ru_alloc + * @uhr.gi: UHR GI, from &enum nl80211_eht_gi + * @uhr.elr: UHR ELR MCS was used + * @uhr.im: UHR interference mitigation was used * @rx_flags: internal RX flags for mac80211 * @ampdu_reference: A-MPDU reference number, must be a different value for * each A-MPDU but the same for each subframe within one A-MPDU @@ -1668,12 +1792,18 @@ struct ieee80211_rx_status { u8 he_ru:3; u8 he_gi:2; u8 he_dcm:1; - }; + } __packed; struct { u8 ru:4; u8 gi:2; - } eht; - }; + } __packed eht; + struct { + u8 ru:4; + u8 gi:2; + u8 elr:1; + u8 im:1; + } __packed uhr; + } __packed; u8 rate_idx; u8 nss; u8 rx_flags; @@ -1686,6 +1816,8 @@ struct ieee80211_rx_status { u8 link_valid:1, link_id:4; }; +static_assert(sizeof(struct ieee80211_rx_status) <= sizeof_field(struct sk_buff, cb)); + static inline u32 ieee80211_rx_status_to_khz(struct ieee80211_rx_status *rx_status) { @@ -1894,12 +2026,69 @@ enum ieee80211_vif_flags { * @IEEE80211_OFFLOAD_DECAP_ENABLED: rx encapsulation offload is enabled * The driver supports passing received 802.11 frames as 802.3 frames to * mac80211. + * @IEEE80211_OFFLOAD_ENCAP_MCAST: tx multicast encapsulation offload is enabled + * The driver supports sending multicast frames passed as 802.3 frames + * by mac80211. */ enum ieee80211_offload_flags { IEEE80211_OFFLOAD_ENCAP_ENABLED = BIT(0), IEEE80211_OFFLOAD_ENCAP_4ADDR = BIT(1), IEEE80211_OFFLOAD_DECAP_ENABLED = BIT(2), + IEEE80211_OFFLOAD_ENCAP_MCAST = BIT(3), +}; + +#define IEEE80211_NAN_AVAIL_BLOB_MAX_LEN 54 + +/** + * struct ieee80211_eml_params - EHT Operating mode notification parameters + * + * EML Operating mode notification parameters received in the Operating mode + * notification frame. This struct is used as a container to pass the info to + * the underlay driver. + * + * @link_id: the link ID where the Operating mode notification frame has been + * received. + * @control: EML control field defined in P802.11be section 9.4.1.76. + * @link_bitmap: eMLSR/eMLMR enabled links defined in P802.11be + * section 9.4.1.76. + * @emlmr_mcs_map_count: eMLMR number of valid mcs_map_bw fields according to + * P802.11be section 9.4.1.76 (valid if eMLMR mode control bit is set). + * @emlmr_mcs_map_bw: eMLMR supported MCS and NSS set subfileds defined in + * P802.11be section 9.4.1.76 (valid if eMLMR mode control bit is set). + */ +struct ieee80211_eml_params { + u8 link_id; + u8 control; + u16 link_bitmap; + u8 emlmr_mcs_map_count; + u8 emlmr_mcs_map_bw[9]; +}; + +/** + * struct ieee80211_nan_sched_cfg - NAN schedule configuration + * @channels: array of NAN channels. A channel entry is in use if + * channels[i].chanreq.oper.chan is not NULL. + * @schedule: NAN local schedule - mapping of each 16TU time slot to + * the NAN channel on which the radio will operate. NULL if unscheduled. + * @avail_blob: NAN Availability attribute blob. + * @avail_blob_len: length of the @avail_blob in bytes. + * @deferred: indicates that the driver should notify peers before applying the + * new NAN schedule, and apply the new schedule the second NAN Slot + * boundary after it notified the peers, as defined in Wi-Fi Aware (TM) 4.0 + * specification, section 5.2.2. + * The driver must call ieee80211_nan_sched_update_done() after the + * schedule has been applied. + * If a HW restart happened while a deferred schedule update was pending, + * mac80211 will reconfigure the deferred schedule (and wait for the driver + * to notify that the schedule has been applied). + */ +struct ieee80211_nan_sched_cfg { + struct ieee80211_nan_channel channels[IEEE80211_NAN_MAX_CHANNELS]; + struct ieee80211_nan_channel *schedule[CFG80211_NAN_SCHED_NUM_TIME_SLOTS]; + u8 avail_blob[IEEE80211_NAN_AVAIL_BLOB_MAX_LEN]; + u16 avail_blob_len; + bool deferred; }; /** @@ -1930,6 +2119,7 @@ enum ieee80211_offload_flags { * your driver/device needs to do. * @ap_addr: AP MLD address, or BSSID for non-MLO connections * (station mode only) + * @nan_sched: NAN schedule parameters. &struct ieee80211_nan_sched_cfg */ struct ieee80211_vif_cfg { /* association related data */ @@ -1948,6 +2138,8 @@ struct ieee80211_vif_cfg { bool s1g; bool idle; u8 ap_addr[ETH_ALEN] __aligned(2); + /* Protected by the wiphy mutex */ + struct ieee80211_nan_sched_cfg nan_sched; }; #define IEEE80211_TTLM_NUM_TIDS 8 @@ -2034,6 +2226,7 @@ enum ieee80211_neg_ttlm_res { * @drv_priv: data area for driver use, will always be aligned to * sizeof(void \*). * @txq: the multicast data TX queue + * @txq_mgmt: the mgmt frame TX queue, currently only exists for NAN devices * @offload_flags: 802.3 -> 802.11 enapsulation offload flags, see * &enum ieee80211_offload_flags. */ @@ -2052,6 +2245,7 @@ struct ieee80211_vif { u8 hw_queue[IEEE80211_NUM_ACS]; struct ieee80211_txq *txq; + struct ieee80211_txq *txq_mgmt; netdev_features_t netdev_features; u32 driver_flags; @@ -2434,13 +2628,18 @@ struct ieee80211_sta_aggregates { * @he_cap: HE capabilities of this STA * @he_6ghz_capa: on 6 GHz, holds the HE 6 GHz band capabilities * @eht_cap: EHT capabilities of this STA + * @uhr_cap: UHR capabilities of this STA * @s1g_cap: S1G capabilities of this STA * @agg: per-link data for multi-link aggregation - * @bandwidth: current bandwidth the station can receive with + * @bandwidth: current bandwidth the station can receive with. + * This is the minimum between the peer's capabilities and our own + * operating channel width; Invalid for NAN since that is operating on + * multiple channels. * @rx_nss: in HT/VHT, the maximum number of spatial streams the * station can receive at the moment, changed by operating mode * notifications and capabilities. The value is only valid after - * the station moves to associated state. + * the station moves to associated state. Invalid for NAN since it + * operates on multiple configurations of rx_nss. * @txpwr: the station tx power configuration * */ @@ -2457,6 +2656,7 @@ struct ieee80211_link_sta { struct ieee80211_sta_he_cap he_cap; struct ieee80211_he_6ghz_capa he_6ghz_capa; struct ieee80211_sta_eht_cap eht_cap; + struct ieee80211_sta_uhr_cap uhr_cap; struct ieee80211_sta_s1g_cap s1g_cap; struct ieee80211_sta_aggregates agg; @@ -2520,6 +2720,11 @@ struct ieee80211_link_sta { * by the AP. * @valid_links: bitmap of valid links, or 0 for non-MLO * @spp_amsdu: indicates whether the STA uses SPP A-MSDU or not. + * @epp_peer: indicates that the peer is an EPP peer. + * @nmi: For NDI stations, pointer to the NMI station of the peer. + * @nan_sched: NAN peer schedule for this station. Valid only for NMI stations. + * @ext_mld_capa_ops: the MLD's extended MLD capabilities and operations + * NOTE: currently only tracked for AP STAs */ struct ieee80211_sta { u8 addr[ETH_ALEN] __aligned(2); @@ -2544,9 +2749,16 @@ struct ieee80211_sta { struct ieee80211_txq *txq[IEEE80211_NUM_TIDS + 1]; u16 valid_links; + u16 ext_mld_capa_ops; + bool epp_peer; struct ieee80211_link_sta deflink; struct ieee80211_link_sta __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS]; + struct ieee80211_sta __rcu *nmi; + + /* should only be accessed with the wiphy mutex held */ + struct ieee80211_nan_peer_sched *nan_sched; + /* must be last */ u8 drv_priv[] __aligned(sizeof(void *)); }; @@ -2780,6 +2992,8 @@ struct ieee80211_txq { * station has a unique address, i.e. each station entry can be identified * by just its MAC address; this prevents, for example, the same station * from connecting to two virtual AP interfaces at the same time. + * Note that this doesn't apply for NAN, in which the peer's NMI address + * can be equal to its NDI address. * * @IEEE80211_HW_SUPPORTS_REORDERING_BUFFER: Hardware (or driver) manages the * reordering buffer internally, guaranteeing mac80211 receives frames in @@ -2869,6 +3083,9 @@ struct ieee80211_txq { * HW flag so drivers can opt in according to their own control, e.g. in * testing. * + * @IEEE80211_HW_SUPPORTS_NDP_BLOCKACK: HW can transmit/receive S1G NDP + * BlockAck frames. + * * @NUM_IEEE80211_HW_FLAGS: number of hardware flags, used for sizing arrays */ enum ieee80211_hw_flags { @@ -2929,6 +3146,7 @@ enum ieee80211_hw_flags { IEEE80211_HW_DISALLOW_PUNCTURING, IEEE80211_HW_HANDLES_QUIET_CSA, IEEE80211_HW_STRICT, + IEEE80211_HW_SUPPORTS_NDP_BLOCKACK, /* keep last, obviously */ NUM_IEEE80211_HW_FLAGS @@ -4442,6 +4660,12 @@ struct ieee80211_prep_tx_info { * @del_nan_func: Remove a NAN function. The driver must call * ieee80211_nan_func_terminated() with * NL80211_NAN_FUNC_TERM_REASON_USER_REQUEST reason code upon removal. + * @nan_peer_sched_changed: Notifies the driver that the peer NAN schedule + * has changed. The new schedule is available via sta->nan_sched. + * Note that the channel_entry blob might not match the actual chandef + * since the bandwidth of the chandef is the minimum of the local and peer + * bandwidth. It is the driver responsibility to remove the peer schedule + * when the NMI station is removed. * @can_aggregate_in_amsdu: Called in order to determine if HW supports * aggregating two specific frames in the same A-MSDU. The relation * between the skbs should be symmetric and transitive. Note that while @@ -4511,6 +4735,9 @@ struct ieee80211_prep_tx_info { * interface with the specified type would be added, and thus drivers that * implement this callback need to handle such cases. The type is the full * &enum nl80211_iftype. + * @set_eml_op_mode: Configure eMLSR/eMLMR operation mode in the underlay + * driver according to the parameter received in the EML Operating mode + * notification frame. */ struct ieee80211_ops { void (*tx)(struct ieee80211_hw *hw, @@ -4844,6 +5071,8 @@ struct ieee80211_ops { void (*del_nan_func)(struct ieee80211_hw *hw, struct ieee80211_vif *vif, u8 instance_id); + int (*nan_peer_sched_changed)(struct ieee80211_hw *hw, + struct ieee80211_sta *sta); bool (*can_aggregate_in_amsdu)(struct ieee80211_hw *hw, struct sk_buff *head, struct sk_buff *skb); @@ -4906,6 +5135,10 @@ struct ieee80211_ops { struct ieee80211_neg_ttlm *ttlm); void (*prep_add_interface)(struct ieee80211_hw *hw, enum nl80211_iftype type); + int (*set_eml_op_mode)(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct ieee80211_sta *sta, + struct ieee80211_eml_params *eml_params); }; /** @@ -6272,6 +6505,30 @@ void ieee80211_iterate_active_interfaces_atomic(struct ieee80211_hw *hw, struct ieee80211_vif *vif), void *data); +struct ieee80211_vif * +__ieee80211_iterate_interfaces(struct ieee80211_hw *hw, + struct ieee80211_vif *prev, + u32 iter_flags); + +/** + * for_each_interface - iterate interfaces under wiphy mutex + * @vif: the iterator variable + * @hw: the HW to iterate for + * @flags: the iteration flags, see &enum ieee80211_interface_iteration_flags + */ +#define for_each_interface(vif, hw, flags) \ + for (vif = __ieee80211_iterate_interfaces(hw, NULL, flags); \ + vif; \ + vif = __ieee80211_iterate_interfaces(hw, vif, flags)) + +/** + * for_each_active_interface - iterate active interfaces under wiphy mutex + * @vif: the iterator variable + * @hw: the HW to iterate for + */ +#define for_each_active_interface(vif, hw) \ + for_each_interface(vif, hw, IEEE80211_IFACE_ITER_ACTIVE) + /** * ieee80211_iterate_active_interfaces_mtx - iterate active interfaces * @@ -6284,12 +6541,18 @@ void ieee80211_iterate_active_interfaces_atomic(struct ieee80211_hw *hw, * @iterator: the iterator function to call, cannot sleep * @data: first argument of the iterator function */ -void ieee80211_iterate_active_interfaces_mtx(struct ieee80211_hw *hw, - u32 iter_flags, - void (*iterator)(void *data, - u8 *mac, - struct ieee80211_vif *vif), - void *data); +static inline void +ieee80211_iterate_active_interfaces_mtx(struct ieee80211_hw *hw, + u32 iter_flags, + void (*iterator)(void *data, u8 *mac, + struct ieee80211_vif *vif), + void *data) +{ + struct ieee80211_vif *vif; + + for_each_interface(vif, hw, iter_flags | IEEE80211_IFACE_ITER_ACTIVE) + iterator(data, vif->addr, vif); +} /** * ieee80211_iterate_stations_atomic - iterate stations @@ -6308,6 +6571,20 @@ void ieee80211_iterate_stations_atomic(struct ieee80211_hw *hw, struct ieee80211_sta *sta), void *data); +struct ieee80211_sta * +__ieee80211_iterate_stations(struct ieee80211_hw *hw, + struct ieee80211_sta *prev); + +/** + * for_each_station - iterate stations under wiphy mutex + * @sta: the iterator variable + * @hw: the HW to iterate for + */ +#define for_each_station(sta, hw) \ + for (sta = __ieee80211_iterate_stations(hw, NULL); \ + sta; \ + sta = __ieee80211_iterate_stations(hw, sta)) + /** * ieee80211_iterate_stations_mtx - iterate stations * @@ -6320,10 +6597,17 @@ void ieee80211_iterate_stations_atomic(struct ieee80211_hw *hw, * @iterator: the iterator function to call * @data: first argument of the iterator function */ -void ieee80211_iterate_stations_mtx(struct ieee80211_hw *hw, - void (*iterator)(void *data, - struct ieee80211_sta *sta), - void *data); +static inline void +ieee80211_iterate_stations_mtx(struct ieee80211_hw *hw, + void (*iterator)(void *data, + struct ieee80211_sta *sta), + void *data) +{ + struct ieee80211_sta *sta; + + for_each_station(sta, hw) + iterator(data, sta); +} /** * ieee80211_queue_work - add work onto the mac80211 workqueue @@ -6605,6 +6889,17 @@ bool ieee80211_txq_airtime_check(struct ieee80211_hw *hw, struct ieee80211_txq *txq); /** + * ieee80211_txq_aql_pending - get pending AQL airtime for a txq + * + * @hw: pointer obtained from ieee80211_alloc_hw() + * @txq: pointer obtained from station or virtual interface + * + * Return: pending airtime (in usec) for the given txq. + */ +u32 ieee80211_txq_aql_pending(struct ieee80211_hw *hw, + struct ieee80211_txq *txq); + +/** * ieee80211_iter_keys - iterate keys programmed into the device * @hw: pointer obtained from ieee80211_alloc_hw() * @vif: virtual interface to iterate, may be %NULL for all @@ -7237,6 +7532,20 @@ ieee80211_get_eht_iftype_cap_vif(const struct ieee80211_supported_band *sband, } /** + * ieee80211_get_uhr_iftype_cap_vif - return UHR capabilities for sband/vif + * @sband: the sband to search for the iftype on + * @vif: the vif to get the iftype from + * + * Return: pointer to the struct ieee80211_sta_uhr_cap, or %NULL is none found + */ +static inline const struct ieee80211_sta_uhr_cap * +ieee80211_get_uhr_iftype_cap_vif(const struct ieee80211_supported_band *sband, + struct ieee80211_vif *vif) +{ + return ieee80211_get_uhr_iftype_cap(sband, ieee80211_vif_type_p2p(vif)); +} + +/** * ieee80211_update_mu_groups - set the VHT MU-MIMO groud data * * @vif: the specified virtual interface @@ -7272,6 +7581,24 @@ void ieee80211_disable_rssi_reports(struct ieee80211_vif *vif); int ieee80211_ave_rssi(struct ieee80211_vif *vif, int link_id); /** + * ieee80211_calculate_rx_timestamp - calculate timestamp in frame + * @hw: pointer as obtained from ieee80211_alloc_hw() + * @status: RX status + * @mpdu_len: total MPDU length (including FCS) + * @mpdu_offset: offset into MPDU to calculate timestamp at + * + * This function calculates the RX timestamp at the given MPDU offset, taking + * into account what the RX timestamp was. An offset of 0 will just normalize + * the timestamp to TSF at beginning of MPDU reception. + * + * Returns: the calculated timestamp + */ +u64 ieee80211_calculate_rx_timestamp(struct ieee80211_hw *hw, + struct ieee80211_rx_status *status, + unsigned int mpdu_len, + unsigned int mpdu_offset); + +/** * ieee80211_report_wowlan_wakeup - report WoWLAN wakeup * @vif: virtual interface * @wakeup: wakeup reason(s) @@ -7291,7 +7618,9 @@ void ieee80211_report_wowlan_wakeup(struct ieee80211_vif *vif, * @band: the band to transmit on * @sta: optional pointer to get the station to send the frame to * - * Return: %true if the skb was prepared, %false otherwise + * Return: %true if the skb was prepared, %false otherwise. + * On failure, the skb is freed by this function; callers must not + * free it again. * * Note: must be called under RCU lock */ @@ -7309,11 +7638,14 @@ bool ieee80211_tx_prepare_skb(struct ieee80211_hw *hw, * * @skb: packet injected by userspace * @dev: the &struct device of this 802.11 device + * @chandef: the channel definition the frame will be transmitted on, or + * %NULL to skip the bandwidth checks * * Return: %true if the radiotap header was parsed, %false otherwise */ bool ieee80211_parse_tx_radiotap(struct sk_buff *skb, - struct net_device *dev); + struct net_device *dev, + const struct cfg80211_chan_def *chandef); /** * struct ieee80211_noa_data - holds temporary data for tracking P2P NoA state @@ -7615,6 +7947,46 @@ void ieee80211_nan_func_match(struct ieee80211_vif *vif, gfp_t gfp); /** + * ieee80211_nan_sched_update_done - notify that NAN schedule update is done + * + * This function is called by the driver to notify mac80211 that the NAN + * schedule update has been applied. + * Must be called with wiphy mutex held. May sleep. + * + * @vif: &struct ieee80211_vif pointer from the add_interface callback. + */ +void ieee80211_nan_sched_update_done(struct ieee80211_vif *vif); + +/** + * ieee80211_nan_cluster_joined - notify about NAN cluster join. + * + * This function is used to notify mac80211 about NAN cluster join. + * + * @vif: &struct ieee80211_vif pointer from the add_interface callback. + * @cluster_id: the cluster ID that was joined + * @new_cluster: true if this is a new cluster + * @gfp: allocation flags + */ +void ieee80211_nan_cluster_joined(struct ieee80211_vif *vif, + const u8 *cluster_id, bool new_cluster, + gfp_t gfp); + +/** + * ieee80211_nan_try_evacuate - try to evacuate a NAN channel + * + * This function tries to evacuate a NAN channel that is using the given + * channel context, to free up channel context resources. + * + * @hw: pointer as obtained from ieee80211_alloc_hw() + * @conf: the channel context configuration to try to evacuate. If %NULL, + * the NAN channel that has the fewest slots scheduled will be evacuated. + * + * Return: %true if a channel was evacuated, %false otherwise + */ +bool ieee80211_nan_try_evacuate(struct ieee80211_hw *hw, + struct ieee80211_chanctx_conf *conf); + +/** * ieee80211_calc_rx_airtime - calculate estimated transmission airtime for RX. * * This function calculates the estimated airtime usage of a frame based on the @@ -7650,19 +8022,22 @@ u32 ieee80211_calc_tx_airtime(struct ieee80211_hw *hw, * ieee80211_get_fils_discovery_tmpl - Get FILS discovery template. * @hw: pointer obtained from ieee80211_alloc_hw(). * @vif: &struct ieee80211_vif pointer from the add_interface callback. + * @link_id: valid link_id during MLO or 0 for non-MLO. * * The driver is responsible for freeing the returned skb. * * Return: FILS discovery template. %NULL on error. */ struct sk_buff *ieee80211_get_fils_discovery_tmpl(struct ieee80211_hw *hw, - struct ieee80211_vif *vif); + struct ieee80211_vif *vif, + unsigned int link_id); /** * ieee80211_get_unsol_bcast_probe_resp_tmpl - Get unsolicited broadcast * probe response template. * @hw: pointer obtained from ieee80211_alloc_hw(). * @vif: &struct ieee80211_vif pointer from the add_interface callback. + * @link_id: valid link_id during MLO or 0 for non-MLO. * * The driver is responsible for freeing the returned skb. * @@ -7670,7 +8045,8 @@ struct sk_buff *ieee80211_get_fils_discovery_tmpl(struct ieee80211_hw *hw, */ struct sk_buff * ieee80211_get_unsol_bcast_probe_resp_tmpl(struct ieee80211_hw *hw, - struct ieee80211_vif *vif); + struct ieee80211_vif *vif, + unsigned int link_id); /** * ieee80211_obss_color_collision_notify - notify userland about a BSS color @@ -7846,4 +8222,11 @@ int ieee80211_emulate_switch_vif_chanctx(struct ieee80211_hw *hw, * Return: %true iff the vif is a NAN interface and NAN is started */ bool ieee80211_vif_nan_started(struct ieee80211_vif *vif); + +/** + * ieee80211_encrypt_tx_skb - Encrypt the transmit skb + * @skb: the skb + * Return: 0 if success and non-zero on error + */ +int ieee80211_encrypt_tx_skb(struct sk_buff *skb); #endif /* MAC80211_H */ diff --git a/include/net/macsec.h b/include/net/macsec.h index bc7de5b53e54..d962093ee923 100644 --- a/include/net/macsec.h +++ b/include/net/macsec.h @@ -9,6 +9,7 @@ #include <linux/u64_stats_sync.h> #include <linux/if_vlan.h> +#include <linux/workqueue.h> #include <uapi/linux/if_link.h> #include <uapi/linux/if_macsec.h> @@ -123,6 +124,7 @@ struct macsec_dev_stats { * @key: key structure * @ssci: short secure channel identifier * @stats: per-SA stats + * @destroy_work: deferred work to free the SA in process context after RCU grace period */ struct macsec_rx_sa { struct macsec_key key; @@ -136,7 +138,7 @@ struct macsec_rx_sa { bool active; struct macsec_rx_sa_stats __percpu *stats; struct macsec_rx_sc *sc; - struct rcu_head rcu; + struct rcu_work destroy_work; }; struct pcpu_rx_sc_stats { @@ -174,6 +176,7 @@ struct macsec_rx_sc { * @key: key structure * @ssci: short secure channel identifier * @stats: per-SA stats + * @destroy_work: deferred work to free the SA in process context after RCU grace period */ struct macsec_tx_sa { struct macsec_key key; @@ -186,7 +189,7 @@ struct macsec_tx_sa { refcount_t refcnt; bool active; struct macsec_tx_sa_stats __percpu *stats; - struct rcu_head rcu; + struct rcu_work destroy_work; }; /** diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index eaa27483f99b..308950f9b54b 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -35,6 +35,8 @@ enum gdma_request_type { GDMA_CREATE_MR = 31, GDMA_DESTROY_MR = 32, GDMA_QUERY_HWC_TIMEOUT = 84, /* 0x54 */ + GDMA_ALLOC_DM = 96, /* 0x60 */ + GDMA_DESTROY_DM = 97, /* 0x61 */ }; #define GDMA_RESOURCE_DOORBELL_PAGE 27 @@ -45,6 +47,7 @@ enum gdma_queue_type { GDMA_RQ, GDMA_CQ, GDMA_EQ, + GDMA_DIM, }; enum gdma_work_request_flags { @@ -124,6 +127,17 @@ union gdma_doorbell_entry { u64 tail_ptr : 31; u64 arm : 1; } eq; + + struct { + u64 id : 24; + u64 reserved : 8; + u64 mod_usec : 10; + u64 reserve1 : 5; + u64 mod_usec_vld : 1; + u64 mod_comps : 8; + u64 reserve2 : 7; + u64 mod_comps_vld: 1; + } dim; }; /* HW DATA */ struct gdma_msg_hdr { @@ -168,6 +182,7 @@ struct gdma_general_req { #define GDMA_MESSAGE_V2 2 #define GDMA_MESSAGE_V3 3 #define GDMA_MESSAGE_V4 4 +#define GDMA_MESSAGE_V5 5 struct gdma_general_resp { struct gdma_resp_hdr hdr; @@ -213,6 +228,12 @@ enum gdma_page_type { #define GDMA_INVALID_DMA_REGION 0 +struct mana_serv_work { + struct work_struct serv_work; + struct pci_dev *pdev; + enum gdma_eqe_type type; +}; + struct gdma_mem_info { struct device *dev; @@ -220,6 +241,14 @@ struct gdma_mem_info { void *virt_addr; u64 length; + /* Scattered fallback: when @nr_pages > 0 the ring is that many + * PAGE_SIZE coherent allocations in @pages_va/@pages_dma, not + * @virt_addr/@dma_handle. + */ + void **pages_va; + dma_addr_t *pages_dma; + unsigned int nr_pages; + /* Allocated by the PF driver */ u64 dma_region_handle; }; @@ -334,6 +363,7 @@ struct gdma_queue { void *context; unsigned int msix_index; + unsigned int irq; u32 log2_throttle_limit; } eq; @@ -380,18 +410,26 @@ struct gdma_irq_context { spinlock_t lock; struct list_head eq_list; char name[MANA_IRQ_NAME_SZ]; + unsigned int msi; + unsigned int irq; + refcount_t refcount; + unsigned int bitmap_refs; + bool dyn_msix; }; enum gdma_context_flags { GC_PROBE_SUCCEEDED = 0, + GC_IN_SERVICE = 1, }; struct gdma_context { struct device *dev; struct dentry *mana_pci_debugfs; - /* Per-vPort max number of queues */ + /* Hardware max number of queues */ unsigned int max_num_queues; + /* Per-vPort max number of queues */ + unsigned int max_num_queues_vport; unsigned int max_num_msix; unsigned int num_msix_usable; struct xarray irq_contexts; @@ -399,6 +437,9 @@ struct gdma_context { /* L2 MTU */ u16 adapter_mtu; + /* NIC supports CQE x8 coalescing */ + bool cqe8_coalescing_sup; + /* This maps a CQ index to the queue structure. */ unsigned int max_num_cqs; struct gdma_queue **cq_table; @@ -409,14 +450,16 @@ struct gdma_context { u32 test_event_eq_id; bool is_pf; - bool in_service; + bool is_pf2; phys_addr_t bar0_pa; void __iomem *bar0_va; + resource_size_t bar0_size; void __iomem *shm_base; void __iomem *db_page_base; phys_addr_t phys_db_page_base; - u32 db_page_size; + u64 db_page_off; + u64 db_page_size; int numa_node; /* Shared memory chanenl (used to bootstrap HWC) */ @@ -432,10 +475,20 @@ struct gdma_context { struct gdma_dev mana_ib; u64 pf_cap_flags1; + u64 gdma_protocol_ver; struct workqueue_struct *service_wq; unsigned long flags; + + /* Protect access to GIC context */ + struct mutex gic_mutex; + + /* Indicate if this device is sharing MSI for EQs on MANA */ + bool msi_sharing; + + /* Bitmap tracks where MSI is allocated when it is not shared for EQs */ + unsigned long *msi_bitmap; }; static inline bool mana_gd_is_mana(struct gdma_dev *gd) @@ -471,6 +524,14 @@ int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe); void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit); +ssize_t mana_gd_read_ring(struct gdma_queue *q, char __user *buf, + size_t count, loff_t *pos); + +int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type); + +void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld, + u32 mod_comps, bool mod_comps_vld); + struct gdma_wqe { u32 reserved :24; u32 last_vbytes :8; @@ -558,6 +619,7 @@ struct gdma_eqe { #define GDMA_SRIOV_REG_CFG_BASE_OFF 0x108 #define MANA_PF_DEVICE_ID 0x00B9 +#define MANA_PF2_DEVICE_ID 0x00C1 #define MANA_VF_DEVICE_ID 0x00BA struct gdma_posted_wqe_info { @@ -586,6 +648,7 @@ enum { #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG BIT(3) #define GDMA_DRV_CAP_FLAG_1_GDMA_PAGES_4MB_1GB_2GB BIT(4) #define GDMA_DRV_CAP_FLAG_1_VARIABLE_INDIRECTION_TABLE_SUPPORT BIT(5) +#define GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE BIT(6) /* Driver can handle holes (zeros) in the device list */ #define GDMA_DRV_CAP_FLAG_1_DEV_LIST_HOLES_SUP BIT(11) @@ -598,7 +661,12 @@ enum { /* Driver can self reset on FPGA Reconfig EQE notification */ #define GDMA_DRV_CAP_FLAG_1_HANDLE_RECONFIG_EQE BIT(17) -#define GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE BIT(6) + +/* Driver detects stalled send queues and recovers them */ +#define GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY BIT(18) + +/* Driver supports separate EQ/MSIs for each vPort */ +#define GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT BIT(19) /* Driver supports linearizing the skb when num_sge exceeds hardware limit */ #define GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE BIT(20) @@ -609,6 +677,15 @@ enum { /* Driver can handle hardware recovery events during probe */ #define GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY BIT(22) +/* Driver supports self recovery on Hardware Channel timeouts */ +#define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY BIT(25) + +/* Driver supports dynamic interrupt moderation - DIM */ +#define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28) + +/* Driver supports non-contiguous queue buffers */ +#define GDMA_DRV_CAP_FLAG_1_NON_CONTIGUOUS_BUFFERS BIT(30) + #define GDMA_DRV_CAP_FLAGS1 \ (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \ GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \ @@ -621,7 +698,12 @@ enum { GDMA_DRV_CAP_FLAG_1_HW_VPORT_LINK_AWARE | \ GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY | \ GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE | \ - GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY) + GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY | \ + GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \ + GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY | \ + GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT | \ + GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION | \ + GDMA_DRV_CAP_FLAG_1_NON_CONTIGUOUS_BUFFERS) #define GDMA_DRV_CAP_FLAGS2 0 @@ -657,6 +739,9 @@ struct gdma_verify_ver_req { u8 os_ver_str4[128]; }; /* HW DATA */ +/* HW supports dynamic interrupt moderation - DIM */ +#define GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(15) + struct gdma_verify_ver_resp { struct gdma_resp_hdr hdr; u64 gdma_protocol_ver; @@ -771,6 +856,7 @@ enum gdma_mr_access_flags { GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2), GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3), GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4), + GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5), }; /* GDMA_CREATE_DMA_REGION */ @@ -822,8 +908,8 @@ struct gdma_destroy_dma_region_req { }; /* HW DATA */ enum gdma_pd_flags { - GDMA_PD_FLAG_INVALID = 0, - GDMA_PD_FLAG_ALLOW_GPA_MR = 1, + GDMA_PD_FLAG_ALLOW_GPA_MR = BIT(0), + GDMA_PD_FLAG_SHORT_PDN = BIT(2), }; struct gdma_create_pd_req { @@ -844,7 +930,7 @@ struct gdma_destroy_pd_req { u64 pd_handle; };/* HW DATA */ -struct gdma_destory_pd_resp { +struct gdma_destroy_pd_resp { struct gdma_resp_hdr hdr; };/* HW DATA */ @@ -861,6 +947,12 @@ enum gdma_mr_type { GDMA_MR_TYPE_GVA = 2, /* Guest zero-based address MRs */ GDMA_MR_TYPE_ZBVA = 4, + /* Device address MRs */ + GDMA_MR_TYPE_DM = 5, + /* Memory Window type 1 */ + GDMA_MR_TYPE_MW1 = 6, + /* Memory Window type 2 */ + GDMA_MR_TYPE_MW2 = 7, }; struct gdma_create_mr_params { @@ -876,6 +968,12 @@ struct gdma_create_mr_params { u64 dma_region_handle; enum gdma_mr_access_flags access_flags; } zbva; + struct { + u64 dm_handle; + u64 offset; + u64 length; + enum gdma_mr_access_flags access_flags; + } da; }; }; @@ -890,13 +988,23 @@ struct gdma_create_mr_request { u64 dma_region_handle; u64 virtual_address; enum gdma_mr_access_flags access_flags; - } gva; + } __packed gva; struct { u64 dma_region_handle; enum gdma_mr_access_flags access_flags; - } zbva; - }; + } __packed zbva; + struct { + u64 dm_handle; + u64 offset; + enum gdma_mr_access_flags access_flags; + } __packed da; + } __packed; u32 reserved_2; + union { + struct { + u64 length; + } da_ext; + }; };/* HW DATA */ struct gdma_create_mr_response { @@ -915,6 +1023,27 @@ struct gdma_destroy_mr_response { struct gdma_resp_hdr hdr; };/* HW DATA */ +struct gdma_alloc_dm_req { + struct gdma_req_hdr hdr; + u64 length; + u32 alignment; + u32 flags; +}; /* HW Data */ + +struct gdma_alloc_dm_resp { + struct gdma_resp_hdr hdr; + u64 dm_handle; +}; /* HW Data */ + +struct gdma_destroy_dm_req { + struct gdma_req_hdr hdr; + u64 dm_handle; +}; /* HW Data */ + +struct gdma_destroy_dm_resp { + struct gdma_resp_hdr hdr; +}; /* HW Data */ + int mana_gd_verify_vf_version(struct pci_dev *pdev); int mana_gd_register_device(struct gdma_dev *gd); @@ -935,7 +1064,7 @@ void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue); int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, - struct gdma_mem_info *gmi); + struct gdma_mem_info *gmi, bool allow_scatter); void mana_gd_free_memory(struct gdma_mem_info *gmi); @@ -953,4 +1082,11 @@ int mana_gd_resume(struct pci_dev *pdev); bool mana_need_log(struct gdma_context *gc, int err); +struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc, + bool use_msi_bitmap, + int *msi_requested); +void mana_gd_put_gic(struct gdma_context *gc, bool use_msi_bitmap, int msi); +int mana_gd_query_device_cfg(struct gdma_context *gc, u32 proto_major_ver, + u32 proto_minor_ver, u32 proto_micro_ver, + u16 *max_num_vports, u8 *bm_hostmode); #endif /* _GDMA_H */ diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h index d7e089c6b694..83b7eff4646e 100644 --- a/include/net/mana/mana.h +++ b/include/net/mana/mana.h @@ -4,6 +4,7 @@ #ifndef _MANA_H #define _MANA_H +#include <linux/dim.h> #include <net/xdp.h> #include <net/net_shaper.h> @@ -30,6 +31,12 @@ enum TRI_STATE { TRI_STATE_TRUE = 1 }; +/* MANA ethtool private flag bit positions */ +enum mana_priv_flag_bits { + MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF = 0, + MANA_PRIV_FLAG_MAX, +}; + /* Number of entries for hardware indirection table must be in power of 2 */ #define MANA_INDIRECT_TABLE_MAX_SIZE 512 #define MANA_INDIRECT_TABLE_DEF_SIZE 64 @@ -61,8 +68,27 @@ enum TRI_STATE { #define MAX_PORTS_IN_MANA_DEV 256 +/* Maximum number of PPIs per coalesced CQE */ +#define MANA_RXCOMP_OOB_NUM_PPI 4 + +/* 8-pkt mode packs up to 2 packets per PPI entry */ +#define MANA_CQE_COAL_PKTS_8 8 + +/* Default/max interrupt moderation settings */ +#define MANA_INTR_MODR_USEC_DEF 0 +#define MANA_INTR_MODR_COMP_DEF 0 + +#define MANA_ADAPTIVE_RX_DEF true +#define MANA_ADAPTIVE_TX_DEF true + +/* DIM doorbell value field layout */ +#define MANA_INTR_MODR_USEC_MAX GENMASK(9, 0) +#define MANA_INTR_MODR_USEC_VLD BIT(15) +#define MANA_INTR_MODR_COMP_MAX GENMASK(7, 0) +#define MANA_INTR_MODR_COMP_MASK GENMASK(23, 16) + /* Update this count whenever the respective structures are changed */ -#define MANA_STATS_RX_COUNT 5 +#define MANA_STATS_RX_COUNT (6 + MANA_CQE_COAL_PKTS_8 - 1) #define MANA_STATS_TX_COUNT 11 #define MANA_RX_FRAG_ALIGNMENT 64 @@ -73,6 +99,8 @@ struct mana_stats_rx { u64 xdp_drop; u64 xdp_tx; u64 xdp_redirect; + u64 pkt_len0_err; + u64 coalesced_cqe[MANA_CQE_COAL_PKTS_8 - 1]; struct u64_stats_sync syncp; }; @@ -183,6 +211,7 @@ enum mana_cqe_type { CQE_RX_COALESCED_4 = 2, CQE_RX_OBJECT_FENCE = 3, CQE_RX_TRUNCATED = 4, + CQE_RX_COALESCED_8 = 7, CQE_TX_OKAY = 32, CQE_TX_SA_DROP = 33, @@ -219,16 +248,28 @@ struct mana_cqe_header { #define MANA_HASH_L4 \ (NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 | \ NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX) +#define MANA_HASH_ENABLE_SUPPORTED \ + (NDIS_HASH_IPV4 | NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | \ + NDIS_HASH_IPV6 | NDIS_HASH_TCP_IPV6 | NDIS_HASH_UDP_IPV6) + +/* Read PPI in two different layouts based on cqe_type */ +union mana_rxcomp_perpkt_info { + struct { + u32 pkt_len : 16; + u32 reserved1 : 16; + u32 reserved2; + u32 pkt_hash; + }; -struct mana_rxcomp_perpkt_info { - u32 pkt_len : 16; - u32 reserved1 : 16; - u32 reserved2; - u32 pkt_hash; + /* Up to two pkts per PPI entry */ + struct { + u32 pkt_hash0; + u16 pkt_len0; + u16 pkt_len1; + u32 pkt_hash1; + }; }; /* HW DATA */ -#define MANA_RXCOMP_OOB_NUM_PPI 4 - /* Receive completion OOB */ struct mana_rxcomp_oob { struct mana_cqe_header cqe_hdr; @@ -247,7 +288,7 @@ struct mana_rxcomp_oob { u32 rx_udp_csum_fail : 1; u32 reserved2 : 1; - struct mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI]; + union mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI]; u32 rx_wqe_offset; }; /* HW DATA */ @@ -294,6 +335,17 @@ struct mana_cq { int work_done; int work_done_since_doorbell; int budget; + + /* DIM - Dynamic Interrupt Moderation */ + struct dim dim; + u16 dim_event_ctr; + + /* Cumulative TX completions fed to DIM. Updated and read only in + * NAPI context (mana_poll_tx_cq() / mana_update_tx_dim()), so they + * measure the hardware completion rate and need no u64_stats_sync. + */ + u64 tx_dim_pkts; + u64 tx_dim_bytes; }; struct mana_recv_buf_oob { @@ -302,6 +354,14 @@ struct mana_recv_buf_oob { void *buf_va; bool from_pool; /* allocated from a page pool */ + /* head page of the page_pool fragment; valid only when + * from_pool && frag_count > 1. + */ + struct page *pp_page; + /* Fragment offset plus rxq->headroom, passed to + * page_pool_dma_sync_for_cpu(). + */ + u32 dma_sync_offset; /* SGL of the buffer going to be sent as part of the work request. */ u32 num_sge; @@ -378,7 +438,6 @@ struct mana_ethtool_stats { u64 tx_cqe_err; u64 tx_cqe_unknown_type; u64 tx_linear_pkt_cnt; - u64 rx_coalesced_err; u64 rx_cqe_unknown_type; }; @@ -478,9 +537,7 @@ struct mana_context { u8 bm_hostmode; struct mana_ethtool_hc_stats hc_stats; - struct mana_eq *eqs; - struct dentry *mana_eqs_debugfs; - + struct workqueue_struct *per_port_queue_reset_wq; /* Workqueue for querying hardware stats */ struct delayed_work gf_stats_work; bool hwc_timeout_occurred; @@ -495,16 +552,23 @@ struct mana_context { struct mana_port_context { struct mana_context *ac; struct net_device *ndev; + struct work_struct queue_reset_work; + + /* Debug knob to log TX timeout but skip recovery reset */ + bool tx_timeout_skip_reset; u8 mac_addr[ETH_ALEN]; + struct mana_eq *eqs; + struct dentry *mana_eqs_debugfs; + enum TRI_STATE rss_state; mana_handle_t default_rxobj; bool tx_shortform_allowed; u16 tx_vp_offset; - struct mana_tx_qp *tx_qp; + struct mana_tx_qp **tx_qp; /* Indirection Table for RX & TX. The values are queue indexes */ u32 *indir_table; @@ -528,6 +592,8 @@ struct mana_port_context { u32 rxbpre_headroom; u32 rxbpre_frag_count; + u32 priv_flags; + struct bpf_prog *bpf_prog; /* Create num_queues EQs, SQs, SQ-CQs, RQs and RQ-CQs, respectively. */ @@ -544,6 +610,12 @@ struct mana_port_context { struct mutex vport_mutex; int vport_use_count; + /* Set by mana_set_channels() under vport_mutex to block RDMA + * from grabbing the vport during the detach/attach window. + * Checked by mana_cfg_vport() when called from the RDMA path. + */ + bool channel_changing; + /* Net shaper handle*/ struct net_shaper_handle handle; @@ -552,26 +624,54 @@ struct mana_port_context { u32 speed; /* Maximum speed supported by the SKU (mbps) */ u32 max_speed; + /* 1 = not queried, 0 = cached success, negative = permanent error. + * Protected by the netdev instance lock. + */ + int link_cfg_error; bool port_is_up; bool port_st_save; /* Saved port state */ + u8 cqe_coalescing_enable; + u8 cqe8_coalescing_enable; + u32 cqe_coalescing_timeout_ns; + + /* Interrupt moderation settings */ + u16 intr_modr_rx_usec; + u16 intr_modr_rx_comp; + u16 intr_modr_tx_usec; + u16 intr_modr_tx_comp; + + bool rx_dim_enabled; + bool tx_dim_enabled; + struct mana_ethtool_stats eth_stats; struct mana_ethtool_phy_stats phy_stats; /* Debugfs */ struct dentry *mana_port_debugfs; + + /* Cached vport/steering config for debugfs */ + u32 vport_max_sq; + u32 vport_max_rq; + u32 steer_rx; + u32 steer_rss; + bool steer_update_tab; + u32 steer_cqe_coalescing; }; netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev); int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx, bool update_hash, bool update_tab); +int mana_disable_vport_rx(struct mana_port_context *apc); int mana_alloc_queues(struct net_device *ndev); int mana_attach(struct net_device *ndev); int mana_detach(struct net_device *ndev, bool from_close); +void mana_dim_change(struct mana_cq *cq, bool enable); + int mana_probe(struct gdma_dev *gd, bool resuming); void mana_remove(struct gdma_dev *gd, bool suspending); @@ -607,6 +707,9 @@ struct mana_obj_spec { u32 queue_size; u32 attached_eq; u32 modr_ctx_id; + u8 req_cq_moderation; + u16 cq_moderation_comp; + u16 cq_moderation_usec; }; enum mana_command_code { @@ -675,6 +778,8 @@ struct mana_query_device_cfg_req { u32 reserved; }; /* HW DATA */ +#define MANA_PF_FLAG_1_CQE_8_COALESCING_SUPPORTED BIT(5) + struct mana_query_device_cfg_resp { struct gdma_resp_hdr hdr; @@ -738,6 +843,15 @@ struct mana_create_wqobj_req { u32 cq_size; u32 cq_moderation_ctx_id; u32 cq_parent_qid; + + /* V2 */ + u8 allow_rqwqe_chain; + + /* V3 */ + u8 req_cq_moderation; + u16 cq_moderation_comp; + u16 cq_moderation_usec; + u8 reserved2[2]; }; /* HW DATA */ struct mana_create_wqobj_resp { @@ -745,6 +859,12 @@ struct mana_create_wqobj_resp { u32 wq_id; u32 cq_id; mana_handle_t wq_obj; + + /* V2 */ + u16 cq_moderation_comp; + u16 cq_moderation_usec; + u8 cq_moderation_enabled; + u8 reserved1[3]; }; /* HW DATA */ /* Destroy WQ Object */ @@ -895,12 +1015,19 @@ struct mana_cfg_rx_steer_req_v2 { mana_handle_t default_rxobj; u8 hashkey[MANA_HASH_KEY_SIZE]; u8 cqe_coalescing_enable; - u8 reserved2[7]; + u8 reserved2[3]; + u16 rss_hash_types; + u8 cqe8_coalescing_enable; /* v5 message */ + u8 reserved3; mana_handle_t indir_tab[] __counted_by(num_indir_entries); }; /* HW DATA */ struct mana_cfg_rx_steer_resp { struct gdma_resp_hdr hdr; + + /* V2 */ + u32 cqe_coalescing_timeout_ns; + u32 reserved1; }; /* HW DATA */ /* Register HW vPort */ @@ -997,6 +1124,7 @@ struct mana_deregister_filter_resp { #define STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR 0x0000000004000000 #define MANA_MAX_NUM_QUEUES 64 +#define MANA_DEF_NUM_QUEUES 16 #define MANA_SHORT_VPORT_OFFSET_MAX ((1U << 8) - 1) @@ -1020,8 +1148,10 @@ void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, mana_handle_t wq_obj); int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, - u32 doorbell_pg_id); + u32 doorbell_pg_id, bool check_channel_changing); void mana_uncfg_vport(struct mana_port_context *apc); +int mana_create_eq(struct mana_port_context *apc); +void mana_destroy_eq(struct mana_port_context *apc); struct net_device *mana_get_primary_netdev(struct mana_context *ac, u32 port_index, diff --git a/include/net/mana/shm_channel.h b/include/net/mana/shm_channel.h index 5199b41497ff..dbabcfb95daf 100644 --- a/include/net/mana/shm_channel.h +++ b/include/net/mana/shm_channel.h @@ -4,6 +4,12 @@ #ifndef _SHM_CHANNEL_H #define _SHM_CHANNEL_H +#define SMC_APERTURE_BITS 256 +#define SMC_BASIC_UNIT (sizeof(u32)) +#define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8)) +#define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1) +#define SMC_APERTURE_SIZE (SMC_APERTURE_BITS / 8) + struct shm_channel { struct device *dev; void __iomem *base; diff --git a/include/net/mctp.h b/include/net/mctp.h index c3207ce98f07..d8bf9074110d 100644 --- a/include/net/mctp.h +++ b/include/net/mctp.h @@ -26,6 +26,9 @@ struct mctp_hdr { #define MCTP_VER_MIN 1 #define MCTP_VER_MAX 1 +/* Definitions for ver field */ +#define MCTP_HDR_VER_MASK GENMASK(3, 0) + /* Definitions for flags_seq_tag field */ #define MCTP_HDR_FLAG_SOM BIT(7) #define MCTP_HDR_FLAG_EOM BIT(6) @@ -270,6 +273,7 @@ struct mctp_dst { struct mctp_dev *dev; unsigned int mtu; mctp_eid_t nexthop; + mctp_eid_t saddr; /* set for direct addressing */ unsigned char halen; diff --git a/include/net/mld.h b/include/net/mld.h index c07359808493..23255cc47ced 100644 --- a/include/net/mld.h +++ b/include/net/mld.h @@ -89,29 +89,196 @@ struct mld2_query { #define MLDV2_QQIC_EXP(value) (((value) >> 4) & 0x07) #define MLDV2_QQIC_MAN(value) ((value) & 0x0f) -#define MLD_EXP_MIN_LIMIT 32768UL -#define MLDV1_MRD_MAX_COMPAT (MLD_EXP_MIN_LIMIT - 1) +/* MLDv2 QQIC floating-point exponential field min threshold */ +#define MLD_QQIC_MIN_THRESHOLD 128 +/* MLDv2 QQIC FP max threshold (mant = 0xF, exp = 7) -> 31744 */ +#define MLD_QQIC_MAX_THRESHOLD 31744 +/* MLDv2 MRC floating-point exponential field min threshold */ +#define MLD_MRC_MIN_THRESHOLD 32768UL +/* MLDv2 MRC FP max threshold (mant = 0xFFF, exp = 7) -> 8387584 */ +#define MLD_MRC_MAX_THRESHOLD 8387584 +#define MLDV1_MRD_MAX_COMPAT (MLD_MRC_MIN_THRESHOLD - 1) #define MLD_MAX_QUEUE 8 #define MLD_MAX_SKBS 32 -static inline unsigned long mldv2_mrc(const struct mld2_query *mlh2) +/* V2 exponential field encoding */ + +/* + * Calculate Maximum Response Code from Maximum Response Delay + * + * MRC represents the 16-bit encoded form of Maximum Response Delay (MRD); + * once decoded, the resulting value is in milliseconds. + * + * RFC3810, 5.1.3. defines only the decoding formula: + * Maximum Response Delay = (mant | 0x1000) << (exp + 3) + * + * but does NOT define the encoding procedure. To derive exponent: + * + * For the 16-bit MRC, the "hidden bit" (0x1000) is left shifted by 12 to + * sit above the 12-bit mantissa. The RFC then shifts this entire block + * left by (exp + 3) to reconstruct the value. So, 'hidden bit' is the + * MSB which is shifted by (12 + exp + 3). + * + * Total left shift of the hidden bit = 12 + (exp + 3) = exp + 15. + * This is the MSB at the 0-based bit position: (exp + 15). + * Since fls() is 1-based, fls(value) - 1 = exp + 15. + * + * Therefore: + * exp = fls(value) - 16 + * mant = (value >> (exp + 3)) & 0x0FFF + * + * Final encoding formula: + * 0x8000 | (exp << 12) | mant + * + * Example (value = 1311744): + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0| 1311744 + * | ^-^--------mant---------^ ^...(exp+3)...^| exp=5 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * Encoded: + * 0x8000 | (5 << 12) | 0x404 = 0xD404 + */ +static inline u16 mldv2_mrc(unsigned long mrd) +{ + u16 mc_man, mc_exp; + + /* MRC < 32768 is literal */ + if (mrd < MLD_MRC_MIN_THRESHOLD) + return mrd; + + /* Saturate at max representable (mant = 0xFFF, exp = 7) -> 8387584 */ + if (mrd >= MLD_MRC_MAX_THRESHOLD) + return 0xFFFF; + + mc_exp = fls(mrd) - 16; + mc_man = (mrd >> (mc_exp + 3)) & 0x0FFF; + + return 0x8000 | (mc_exp << 12) | mc_man; +} + +/* + * Calculate Querier's Query Interval Code from Querier's Query Interval + * + * QQIC represents the 8-bit encoded form of Querier's Query Interval (QQI); + * once decoded, the resulting value is in seconds. + * + * RFC3810, 5.1.9. defines only the decoding formula: + * QQI = (mant | 0x10) << (exp + 3) + * + * but does NOT define the encoding procedure. To derive exponent: + * + * For any value of mantissa and exponent, the decoding formula indicates + * that the "hidden bit" (0x10) is shifted 4 bits left to sit above the + * 4-bit mantissa. The RFC again shifts this entire block left by (exp + 3) + * to reconstruct the value. So, 'hidden bit' is the MSB which is shifted + * by (4 + exp + 3). + * + * Total left shift of the 'hidden bit' = 4 + (exp + 3) = exp + 7. + * This is the MSB at the 0-based bit position: (exp + 7). + * Since fls() is 1-based, fls(value) - 1 = exp + 7. + * + * Therefore: + * exp = fls(value) - 8 + * mant = (value >> (exp + 3)) & 0x0F + * + * Final encoding formula: + * 0x80 | (exp << 4) | mant + * + * Example (value = 3200): + * 0 1 + * 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |0 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0| (value = 3200) + * | ^-^-mant^ ^..(exp+3)..^| exp = 4, mant = 9 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * Encoded: + * 0x80 | (4 << 4) | 9 = 0xC9 + */ +static inline u8 mldv2_qqic(unsigned long value) +{ + u8 mc_man, mc_exp; + + /* QQIC < 128 is literal */ + if (value < MLD_QQIC_MIN_THRESHOLD) + return value; + + /* Saturate at max representable (mant = 0xF, exp = 7) -> 31744 */ + if (value >= MLD_QQIC_MAX_THRESHOLD) + return 0xFF; + + mc_exp = fls(value) - 8; + mc_man = (value >> (mc_exp + 3)) & 0x0F; + + return 0x80 | (mc_exp << 4) | mc_man; +} + +/* V2 exponential field decoding */ + +/* Calculate Maximum Response Delay from Maximum Response Code + * + * RFC3810, relevant sections: + * - 5.1.3. Maximum Response Code defines the decoding formula: + * 0 1 2 3 4 5 6 7 8 9 A B C D E F + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |1| exp | mant | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * Maximum Response Delay = (mant | 0x1000) << (exp+3) + * - 9.3. Query Response Interval + * + * After decode, MRC represents the Maximum Response Delay (MRD) in + * units of milliseconds. + */ +static inline unsigned long mldv2_mrd(const struct mld2_query *mlh2) { - /* RFC3810, 5.1.3. Maximum Response Code */ - unsigned long ret, mc_mrc = ntohs(mlh2->mld2q_mrc); + unsigned long mc_mrc = ntohs(mlh2->mld2q_mrc); - if (mc_mrc < MLD_EXP_MIN_LIMIT) { - ret = mc_mrc; + if (mc_mrc < MLD_MRC_MIN_THRESHOLD) { + return mc_mrc; } else { unsigned long mc_man, mc_exp; mc_exp = MLDV2_MRC_EXP(mc_mrc); mc_man = MLDV2_MRC_MAN(mc_mrc); - ret = (mc_man | 0x1000) << (mc_exp + 3); + return (mc_man | 0x1000) << (mc_exp + 3); } +} - return ret; +/* Calculate Querier's Query Interval from Querier's Query Interval Code + * + * RFC3810, relevant sections: + * - 5.1.9. QQIC (Querier's Query Interval Code) defines the decoding formula: + * 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+ + * |1| exp | mant | + * +-+-+-+-+-+-+-+-+ + * QQI = (mant | 0x10) << (exp + 3) + * - 9.2. Query Interval + * - 9.12. Older Version Querier Present Timeout + * (the [Query Interval] in the last Query received) + * + * After decode, QQIC represents the Querier's Query Interval in units + * of seconds. + */ +static inline unsigned long mldv2_qqi(const struct mld2_query *mlh2) +{ + unsigned long qqic = mlh2->mld2q_qqic; + + if (qqic < MLD_QQIC_MIN_THRESHOLD) { + return qqic; + } else { + unsigned long mc_man, mc_exp; + + mc_exp = MLDV2_QQIC_EXP(qqic); + mc_man = MLDV2_QQIC_MAN(qqic); + + return (mc_man | 0x10) << (mc_exp + 3); + } } #endif diff --git a/include/net/mptcp.h b/include/net/mptcp.h index f7263fe2a2e4..485d55b66ea6 100644 --- a/include/net/mptcp.h +++ b/include/net/mptcp.h @@ -19,15 +19,13 @@ struct seq_file; /* MPTCP sk_buff extension data */ struct mptcp_ext { - union { - u64 data_ack; - u32 data_ack32; - }; u64 data_seq; u32 subflow_seq; u16 data_len; __sum16 csum; - u8 use_map:1, + + struct_group(flags, + u8 use_map:1, dsn64:1, data_fin:1, use_ack:1, @@ -35,9 +33,10 @@ struct mptcp_ext { mpc_map:1, frozen:1, reset_transient:1; - u8 reset_reason:4, + u8 reset_reason:4, csum_reqd:1, infinite_map:1; + ); /* end of flags group */ }; #define MPTCPOPT_HMAC_LEN 20 @@ -69,7 +68,8 @@ struct mptcp_out_options { u8 reset_reason:4, reset_transient:1, csum_reqd:1, - allow_join_id0:1; + allow_join_id0:1, + drop_ts:1; union { struct { u64 sndr_key; @@ -149,9 +149,9 @@ bool mptcp_syn_options(struct sock *sk, const struct sk_buff *skb, unsigned int *size, struct mptcp_out_options *opts); bool mptcp_synack_options(const struct request_sock *req, unsigned int *size, struct mptcp_out_options *opts); -bool mptcp_established_options(struct sock *sk, struct sk_buff *skb, - unsigned int *size, unsigned int remaining, - struct mptcp_out_options *opts); +int mptcp_established_options(struct sock *sk, struct sk_buff *skb, + unsigned int remaining, bool has_ts, + struct mptcp_out_options *opts); bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb); void mptcp_write_options(struct tcphdr *th, __be32 *ptr, struct tcp_sock *tp, @@ -266,15 +266,6 @@ static inline bool mptcp_synack_options(const struct request_sock *req, return false; } -static inline bool mptcp_established_options(struct sock *sk, - struct sk_buff *skb, - unsigned int *size, - unsigned int remaining, - struct mptcp_out_options *opts) -{ - return false; -} - static inline bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb) { diff --git a/include/net/ncsi.h b/include/net/ncsi.h index 08a50d9acb0a..e886358207aa 100644 --- a/include/net/ncsi.h +++ b/include/net/ncsi.h @@ -60,7 +60,7 @@ static inline int ncsi_start_dev(struct ncsi_dev *nd) return -ENOTTY; } -static void ncsi_stop_dev(struct ncsi_dev *nd) +static inline void ncsi_stop_dev(struct ncsi_dev *nd) { } diff --git a/include/net/ndisc.h b/include/net/ndisc.h index d38783a2ce57..9e5379ad2d8e 100644 --- a/include/net/ndisc.h +++ b/include/net/ndisc.h @@ -2,8 +2,6 @@ #ifndef _NDISC_H #define _NDISC_H -#include <net/ipv6_stubs.h> - /* * ICMP codes for neighbour discovery messages */ @@ -359,14 +357,6 @@ static inline struct neighbour *__ipv6_neigh_lookup_noref(struct net_device *dev return ___neigh_lookup_noref(&nd_tbl, neigh_key_eq128, ndisc_hashfn, pkey, dev); } -static inline -struct neighbour *__ipv6_neigh_lookup_noref_stub(struct net_device *dev, - const void *pkey) -{ - return ___neigh_lookup_noref(ipv6_stub->nd_tbl, neigh_key_eq128, - ndisc_hashfn, pkey, dev); -} - static inline struct neighbour *__ipv6_neigh_lookup(struct net_device *dev, const void *pkey) { struct neighbour *n; @@ -391,28 +381,20 @@ static inline void __ipv6_confirm_neigh(struct net_device *dev, rcu_read_unlock(); } -static inline void __ipv6_confirm_neigh_stub(struct net_device *dev, - const void *pkey) -{ - struct neighbour *n; - - rcu_read_lock(); - n = __ipv6_neigh_lookup_noref_stub(dev, pkey); - neigh_confirm(n); - rcu_read_unlock(); -} - -/* uses ipv6_stub and is meant for use outside of IPv6 core */ static inline struct neighbour *ip_neigh_gw6(struct net_device *dev, const void *addr) { +#if IS_ENABLED(CONFIG_IPV6) struct neighbour *neigh; - neigh = __ipv6_neigh_lookup_noref_stub(dev, addr); + neigh = __ipv6_neigh_lookup_noref(dev, addr); if (unlikely(!neigh)) - neigh = __neigh_create(ipv6_stub->nd_tbl, addr, dev, false); + neigh = __neigh_create(&nd_tbl, addr, dev, false); return neigh; +#else + return ERR_PTR(-EAFNOSUPPORT); +#endif } int ndisc_init(void); @@ -434,6 +416,7 @@ void ndisc_send_skb(struct sk_buff *skb, const struct in6_addr *daddr, void ndisc_send_rs(struct net_device *dev, const struct in6_addr *saddr, const struct in6_addr *daddr); + void ndisc_send_na(struct net_device *dev, const struct in6_addr *daddr, const struct in6_addr *solicited_addr, bool router, bool solicited, bool override, bool inc_opt); @@ -447,6 +430,8 @@ void ndisc_update(const struct net_device *dev, struct neighbour *neigh, const u8 *lladdr, u8 new, u32 flags, u8 icmp6_type, struct ndisc_options *ndopts); +int ndisc_check_ns_na(struct sk_buff *skb); + /* * IGMP */ diff --git a/include/net/neighbour.h b/include/net/neighbour.h index 2dfee6d4258a..8860cc2175fc 100644 --- a/include/net/neighbour.h +++ b/include/net/neighbour.h @@ -489,11 +489,15 @@ static inline int neigh_event_send(struct neighbour *neigh, struct sk_buff *skb) #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) static inline int neigh_hh_bridge(struct hh_cache *hh, struct sk_buff *skb) { - unsigned int seq, hh_alen; + unsigned int seq, hh_alen = HH_DATA_ALIGN(ETH_HLEN); + int err; + + err = skb_cow_head(skb, hh_alen); + if (err) + return err; do { seq = read_seqbegin(&hh->hh_lock); - hh_alen = HH_DATA_ALIGN(ETH_HLEN); memcpy(skb->data - hh_alen, hh->hh_data, ETH_ALEN + hh_alen - ETH_HLEN); } while (read_seqretry(&hh->hh_lock, seq)); return 0; diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index cb664f6e3558..e5ee673b9fcf 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -37,6 +37,7 @@ #include <net/netns/smc.h> #include <net/netns/bpf.h> #include <net/netns/mctp.h> +#include <net/netns/vsock.h> #include <net/net_trackers.h> #include <linux/ns_common.h> #include <linux/idr.h> @@ -120,6 +121,7 @@ struct net { * it is critical that it is on a read_mostly cache line. */ u32 hash_mix; + bool is_dying; struct net_device *loopback_dev; /* The loopback */ @@ -195,6 +197,12 @@ struct net { #ifdef CONFIG_DEBUG_NET_SMALL_RTNL /* Move to a better place when the config guard is removed. */ struct mutex rtnl_mutex; + struct work_struct rtnl_work; + struct list_head dev_unreg_head; + spinlock_t dev_unreg_lock; +#endif +#if IS_ENABLED(CONFIG_VSOCKETS) + struct netns_vsock vsock; #endif } __randomize_layout; @@ -259,14 +267,14 @@ void ipx_unregister_sysctl(void); #define ipx_unregister_sysctl() #endif -#ifdef CONFIG_NET_NS -void __put_net(struct net *net); - static inline struct net *to_net_ns(struct ns_common *ns) { return container_of(ns, struct net, ns); } +#ifdef CONFIG_NET_NS +void __put_net(struct net *net); + /* Try using get_net_track() instead */ static inline struct net *get_net(struct net *net) { @@ -304,7 +312,7 @@ static inline int check_net(const struct net *net) return ns_ref_read(net) != 0; } -void net_drop_ns(void *); +void net_drop_ns(struct ns_common *); void net_passive_dec(struct net *net); #else @@ -517,12 +525,13 @@ struct ctl_table; #ifdef CONFIG_SYSCTL int net_sysctl_init(void); struct ctl_table_header *register_net_sysctl_sz(struct net *net, const char *path, - struct ctl_table *table, size_t table_size); + const struct ctl_table *table, + size_t table_size); void unregister_net_sysctl_table(struct ctl_table_header *header); #else static inline int net_sysctl_init(void) { return 0; } static inline struct ctl_table_header *register_net_sysctl_sz(struct net *net, - const char *path, struct ctl_table *table, size_t table_size) + const char *path, const struct ctl_table *table, size_t table_size) { return NULL; } diff --git a/include/net/net_shaper.h b/include/net/net_shaper.h index 5c3f49b52fe9..330517a1cb5c 100644 --- a/include/net/net_shaper.h +++ b/include/net/net_shaper.h @@ -53,6 +53,7 @@ struct net_shaper { /* private: */ u32 leaves; /* accounted only for NODE scope */ + bool valid; struct rcu_head rcu; }; @@ -67,21 +68,52 @@ struct net_shaper { * The operations are serialized via a per device lock. * * Device not supporting any kind of nesting should not provide the - * group operation. + * @group operation. * * Each shaper is uniquely identified within the device with a 'handle' * comprising the shaper scope and a scope-specific id. + * + * Driver ops vs uAPI + * ------------------ + * Members of the driver ops mirror the Netlink uAPI but driver calls do not + * map 1:1 to user calls. Drivers need to be careful when assuming that calls + * disallowed at the uAPI level will never be made at the driver level. + * The shaper core performs automatic reparenting and cleanup, generating + * additional calls. Notably: + * + * - @group calls in the driver facing API may have nodes as leaves (user is + * only allowed to construct groups with queues as leaves) + * - @group calls may update leaf's parent if the parent is about + * to be removed (re-parenting nodes explicitly is not supported in the uAPI) + * + * Implicit creation + * ----------------- + * Shapers are created implicitly, meaning that @set and @group operations + * are called both for existing and new shapers. The driver has to infer + * whether the operation is an update or a creation by tracking the handles. + * Removal of shapers is explicit and done with a @delete call. + * + * The @set operation implicitly creates NET_SHAPER_SCOPE_NETDEV and + * NET_SHAPER_SCOPE_QUEUE shapers. + * The @group operation implicitly creates NET_SHAPER_SCOPE_NETDEV and + * NET_SHAPER_SCOPE_NODE shapers (the group shaper itself), as well as + * NET_SHAPER_SCOPE_QUEUE shapers (leaves). */ struct net_shaper_ops { /** - * @group: create the specified shapers scheduling group + * @group: create a scheduling group or add leaves * - * Nest the @leaves shapers identified under the * @node shaper. + * Nest the @leaves shapers identified under the @node shaper. * All the shapers belong to the device specified by @binding. - * The @leaves arrays size is specified by @leaves_count. - * Create either the @leaves and the @node shaper; or if they already - * exists, links them together in the desired way. - * @leaves scope must be NET_SHAPER_SCOPE_QUEUE. + * The @leaves array's size is specified by @leaves_count. + * + * @node and @leaves may or may not already exist + * (see the "Implicit creation" note). If @node already exists, + * the @leaves should be *added* to its children. In this case, + * the @leaves array only holds new/modified leaves, not the full list. + * + * Re-parenting @leaves is implemented by a @group call on a new parent. + * There's no explicit call to remove the children from the old parent. */ int (*group)(struct net_shaper_binding *binding, int leaves_count, const struct net_shaper *leaves, @@ -102,6 +134,13 @@ struct net_shaper_ops { * * Removes the shaper configuration as identified by the given @handle * on the device specified by @binding, restoring the default behavior. + * + * Note that a @delete call on a NET_SHAPER_SCOPE_QUEUE shaper also + * implicitly removes the associated queue from the scheduling + * hierarchy. The driver must take care of that step. + * @delete calls on NET_SHAPER_SCOPE_NODE should not require any + * implicit re-parenting in the driver as core will re-parent the leaves + * first, before deleting the SCOPE_NODE shaper. */ int (*delete)(struct net_shaper_binding *binding, const struct net_shaper_handle *handle, diff --git a/include/net/netdev_lock.h b/include/net/netdev_lock.h index 3d3aef80beac..9fb3e93857c3 100644 --- a/include/net/netdev_lock.h +++ b/include/net/netdev_lock.h @@ -64,7 +64,7 @@ static inline void netdev_unlock_full_to_ops(struct net_device *dev) netdev_unlock(dev); } -static inline void netdev_ops_assert_locked(const struct net_device *dev) +static inline void netdev_assert_locked_ops_compat(const struct net_device *dev) { if (netdev_need_ops_lock(dev)) lockdep_assert_held(&dev->lock); @@ -73,11 +73,17 @@ static inline void netdev_ops_assert_locked(const struct net_device *dev) } static inline void -netdev_ops_assert_locked_or_invisible(const struct net_device *dev) +netdev_assert_locked_ops_compat_or_invisible(const struct net_device *dev) { if (dev->reg_state == NETREG_REGISTERED || dev->reg_state == NETREG_UNREGISTERING) - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); +} + +static inline void netdev_assert_locked_ops(const struct net_device *dev) +{ + if (netdev_need_ops_lock(dev)) + netdev_assert_locked(dev); } static inline void netdev_lock_ops_compat(struct net_device *dev) @@ -96,6 +102,14 @@ static inline void netdev_unlock_ops_compat(struct net_device *dev) rtnl_unlock(); } +/* Matching "ops protected" category from netdevice.h */ +static inline int netdev_is_locked_ops_compat(const struct net_device *dev) +{ + if (netdev_need_ops_lock(dev)) + return lockdep_is_held(&dev->lock); + return lockdep_rtnl_is_held(); +} + static inline int netdev_lock_cmp_fn(const struct lockdep_map *a, const struct lockdep_map *b) { @@ -132,6 +146,9 @@ static inline int netdev_lock_cmp_fn(const struct lockdep_map *a, #define netdev_lock_dereference(p, dev) \ rcu_dereference_protected(p, lockdep_is_held(&(dev)->lock)) +#define netdev_ops_lock_dereference(p, dev) \ + rcu_dereference_protected(p, netdev_is_locked_ops_compat(dev)) + int netdev_debug_event(struct notifier_block *nb, unsigned long event, void *ptr); diff --git a/include/net/netdev_queues.h b/include/net/netdev_queues.h index cd00e0406cf4..70c9fe9e83cc 100644 --- a/include/net/netdev_queues.h +++ b/include/net/netdev_queues.h @@ -14,6 +14,10 @@ struct netdev_config { u8 hds_config; }; +struct netdev_queue_config { + u32 rx_page_size; +}; + /* See the netdev.yaml spec for definition of each statistic */ struct netdev_queue_stats_rx { u64 bytes; @@ -111,6 +115,11 @@ void netdev_stat_queue_sum(struct net_device *netdev, int tx_start, int tx_end, struct netdev_queue_stats_tx *tx_sum); +enum { + /* The queue checks and honours the page size qcfg parameter */ + QCFG_RX_PAGE_SIZE = 0x1, +}; + /** * struct netdev_queue_mgmt_ops - netdev ops for queue management * @@ -130,28 +139,60 @@ void netdev_stat_queue_sum(struct net_device *netdev, * @ndo_queue_get_dma_dev: Get dma device for zero-copy operations to be used * for this queue. Return NULL on error. * + * @ndo_default_qcfg: (Optional) Populate queue config struct with defaults. + * Queue config structs are passed to this helper before + * the user-requested settings are applied. + * + * @ndo_validate_qcfg: (Optional) Check if queue config is supported. + * Called when configuration affecting a queue may be + * changing, either due to NIC-wide config, or config + * scoped to the queue at a specified index. + * When NIC-wide config is changed the callback will + * be invoked for all queues. + * + * @ndo_queue_create: Create a new RX queue on a virtual device that will + * be paired with a physical device's queue via leasing. + * Return the new queue id on success, negative error + * on failure. + * + * @supported_params: Bitmask of supported parameters, see QCFG_*. + * * Note that @ndo_queue_mem_alloc and @ndo_queue_mem_free may be called while * the interface is closed. @ndo_queue_start and @ndo_queue_stop will only * be called for an interface which is open. */ struct netdev_queue_mgmt_ops { - size_t ndo_queue_mem_size; - int (*ndo_queue_mem_alloc)(struct net_device *dev, - void *per_queue_mem, - int idx); - void (*ndo_queue_mem_free)(struct net_device *dev, - void *per_queue_mem); - int (*ndo_queue_start)(struct net_device *dev, - void *per_queue_mem, - int idx); - int (*ndo_queue_stop)(struct net_device *dev, - void *per_queue_mem, - int idx); - struct device * (*ndo_queue_get_dma_dev)(struct net_device *dev, - int idx); + size_t ndo_queue_mem_size; + int (*ndo_queue_mem_alloc)(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *per_queue_mem, + int idx); + void (*ndo_queue_mem_free)(struct net_device *dev, + void *per_queue_mem); + int (*ndo_queue_start)(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *per_queue_mem, + int idx); + int (*ndo_queue_stop)(struct net_device *dev, + void *per_queue_mem, + int idx); + void (*ndo_default_qcfg)(struct net_device *dev, + struct netdev_queue_config *qcfg); + int (*ndo_validate_qcfg)(struct net_device *dev, + struct netdev_queue_config *qcfg, + struct netlink_ext_ack *extack); + struct device * (*ndo_queue_get_dma_dev)(struct net_device *dev, + int idx); + int (*ndo_queue_create)(struct net_device *dev, + struct netlink_ext_ack *extack); + + unsigned int supported_params; }; -bool netif_rxq_has_unreadable_mp(struct net_device *dev, int idx); +void netdev_queue_config(struct net_device *dev, int rxq, + struct netdev_queue_config *qcfg); + +bool netif_rxq_has_unreadable_mp(struct net_device *dev, unsigned int rxq_idx); /** * DOC: Lockless queue stopping / waking helpers. @@ -310,6 +351,17 @@ static inline void netif_subqueue_sent(const struct net_device *dev, netdev_tx_sent_queue(txq, bytes); } +static inline unsigned int netif_xmit_timeout_ms(struct netdev_queue *txq) +{ + unsigned long trans_start = READ_ONCE(txq->trans_start); + + if (netif_xmit_stopped(txq) && + time_after(jiffies, trans_start + txq->dev->watchdog_timeo)) + return jiffies_to_msecs(jiffies - trans_start); + + return 0; +} + #define netif_subqueue_maybe_stop(dev, idx, get_desc, stop_thrs, start_thrs) \ ({ \ struct netdev_queue *_txq; \ @@ -328,6 +380,14 @@ static inline void netif_subqueue_sent(const struct net_device *dev, get_desc, start_thrs); \ }) -struct device *netdev_queue_get_dma_dev(struct net_device *dev, int idx); - -#endif +struct device *netdev_queue_get_dma_dev(struct net_device *dev, + unsigned int idx, + enum netdev_queue_type type); +bool netdev_can_create_queue(const struct net_device *dev, + struct netlink_ext_ack *extack); +bool netdev_can_lease_queue(const struct net_device *dev, + struct netlink_ext_ack *extack); +bool netdev_queue_busy(struct net_device *dev, unsigned int idx, + enum netdev_queue_type type, + struct netlink_ext_ack *extack); +#endif /* _LINUX_NET_QUEUES_H */ diff --git a/include/net/netdev_rx_queue.h b/include/net/netdev_rx_queue.h index 8cdcd138b33f..9415a94d333d 100644 --- a/include/net/netdev_rx_queue.h +++ b/include/net/netdev_rx_queue.h @@ -7,13 +7,15 @@ #include <linux/sysfs.h> #include <net/xdp.h> #include <net/page_pool/types.h> +#include <net/netdev_queues.h> +#include <net/rps-types.h> /* This structure contains an instance of an RX queue. */ struct netdev_rx_queue { struct xdp_rxq_info xdp_rxq; #ifdef CONFIG_RPS struct rps_map __rcu *rps_map; - struct rps_dev_flow_table __rcu *rps_flow_table; + rps_tag_ptr rps_flow_table; #endif struct kobject kobj; const struct attribute_group **groups; @@ -27,7 +29,16 @@ struct netdev_rx_queue { struct xsk_buff_pool *pool; #endif struct napi_struct *napi; + struct netdev_queue_config qcfg; struct pp_memory_provider_params mp_params; + + /* If a queue is leased, then the lease pointer is always + * valid. From the physical device it points to the virtual + * queue, and from the virtual device it points to the + * physical queue. + */ + struct netdev_rx_queue *lease; + netdevice_tracker lease_tracker; } ____cacheline_aligned_in_smp; /* @@ -56,6 +67,18 @@ get_netdev_rx_queue_index(struct netdev_rx_queue *queue) return index; } -int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq); +enum netif_lease_dir { + NETIF_VIRT_TO_PHYS, + NETIF_PHYS_TO_VIRT, +}; -#endif +struct netdev_rx_queue * +__netif_get_rx_queue_lease(struct net_device **dev, unsigned int *rxq, + enum netif_lease_dir dir); + +int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq); +void netdev_rx_queue_lease(struct netdev_rx_queue *rxq_dst, + struct netdev_rx_queue *rxq_src); +void netdev_rx_queue_unlease(struct netdev_rx_queue *rxq_dst, + struct netdev_rx_queue *rxq_src); +#endif /* _LINUX_NETDEV_RX_QUEUE_H */ diff --git a/include/net/netfilter/ipv4/nf_conntrack_ipv4.h b/include/net/netfilter/ipv4/nf_conntrack_ipv4.h index 8d65ffbf57de..0b07d5e69c15 100644 --- a/include/net/netfilter/ipv4/nf_conntrack_ipv4.h +++ b/include/net/netfilter/ipv4/nf_conntrack_ipv4.h @@ -16,11 +16,12 @@ extern const struct nf_conntrack_l4proto nf_conntrack_l4proto_icmp; #ifdef CONFIG_NF_CT_PROTO_SCTP extern const struct nf_conntrack_l4proto nf_conntrack_l4proto_sctp; #endif -#ifdef CONFIG_NF_CT_PROTO_UDPLITE -extern const struct nf_conntrack_l4proto nf_conntrack_l4proto_udplite; -#endif #ifdef CONFIG_NF_CT_PROTO_GRE extern const struct nf_conntrack_l4proto nf_conntrack_l4proto_gre; #endif +#if IS_ENABLED(CONFIG_NF_CONNTRACK_PPTP) +void gre_pptp_destroy_siblings(struct nf_conn *ct); +#endif + #endif /*_NF_CONNTRACK_IPV4_H*/ diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h index aa0a7c82199e..c39425e54d87 100644 --- a/include/net/netfilter/nf_conntrack.h +++ b/include/net/netfilter/nf_conntrack.h @@ -16,6 +16,7 @@ #include <linux/bitops.h> #include <linux/compiler.h> +#include <net/netns/generic.h> #include <linux/netfilter/nf_conntrack_common.h> #include <linux/netfilter/nf_conntrack_tcp.h> #include <linux/netfilter/nf_conntrack_sctp.h> @@ -184,6 +185,11 @@ static inline void nf_ct_put(struct nf_conn *ct) nf_ct_destroy(&ct->ct_general); } +static inline bool nf_ct_shared(const struct nf_conn *ct) +{ + return refcount_read(&ct->ct_general.use) > 1; +} + /* load module; enable/disable conntrack in this namespace */ int nf_ct_netns_get(struct net *net, u8 nfproto); void nf_ct_netns_put(struct net *net, u8 nfproto); diff --git a/include/net/netfilter/nf_conntrack_core.h b/include/net/netfilter/nf_conntrack_core.h index 3384859a8921..8883575adcc1 100644 --- a/include/net/netfilter/nf_conntrack_core.h +++ b/include/net/netfilter/nf_conntrack_core.h @@ -83,6 +83,11 @@ void nf_conntrack_lock(spinlock_t *lock); extern spinlock_t nf_conntrack_expect_lock; +static inline void lockdep_nfct_expect_lock_held(void) +{ + lockdep_assert_held(&nf_conntrack_expect_lock); +} + /* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */ static inline void __nf_ct_set_timeout(struct nf_conn *ct, u64 timeout) diff --git a/include/net/netfilter/nf_conntrack_count.h b/include/net/netfilter/nf_conntrack_count.h index 52a06de41aa0..cf0166520cf3 100644 --- a/include/net/netfilter/nf_conntrack_count.h +++ b/include/net/netfilter/nf_conntrack_count.h @@ -13,6 +13,7 @@ struct nf_conncount_list { u32 last_gc; /* jiffies at most recent gc */ struct list_head head; /* connections with the same filtering key */ unsigned int count; /* length of list */ + unsigned int last_gc_count; /* length of list at most recent gc */ }; struct nf_conncount_data *nf_conncount_init(struct net *net, unsigned int keylen); diff --git a/include/net/netfilter/nf_conntrack_expect.h b/include/net/netfilter/nf_conntrack_expect.h index 165e7a03b8e9..8c978b9a467d 100644 --- a/include/net/netfilter/nf_conntrack_expect.h +++ b/include/net/netfilter/nf_conntrack_expect.h @@ -22,10 +22,17 @@ struct nf_conntrack_expect { /* Hash member */ struct hlist_node hnode; + /* Network namespace */ + possible_net_t net; + /* We expect this tuple, with the following mask */ + struct nf_conntrack_tuple master_tuple; struct nf_conntrack_tuple tuple; struct nf_conntrack_tuple_mask mask; +#ifdef CONFIG_NF_CONNTRACK_ZONES + struct nf_conntrack_zone zone; +#endif /* Usage count. */ refcount_t use; @@ -35,18 +42,24 @@ struct nf_conntrack_expect { /* Expectation class */ unsigned int class; + /* Event filter mask */ + u16 event_mask; + /* Function to call after setup and insertion */ void (*expectfn)(struct nf_conn *new, struct nf_conntrack_expect *this); + /* Helper that created this expectation */ + struct nf_conntrack_helper __rcu *helper; + /* Helper to assign to new connection */ - struct nf_conntrack_helper *helper; + struct nf_conntrack_helper __rcu *assign_helper; /* The conntrack of the master connection */ struct nf_conn *master; - /* Timer function; deletes the expectation. */ - struct timer_list timeout; + /* jiffies32 when this expectation expires */ + u32 timeout; #if IS_ENABLED(CONFIG_NF_NAT) union nf_inet_addr saved_addr; @@ -60,9 +73,27 @@ struct nf_conntrack_expect { struct rcu_head rcu; }; +static inline bool nf_ct_exp_is_expired(const struct nf_conntrack_expect *exp) +{ + if (READ_ONCE(exp->flags) & NF_CT_EXPECT_DEAD) + return true; + + return (__s32)(READ_ONCE(exp->timeout) - nfct_time_stamp) <= 0; +} + static inline struct net *nf_ct_exp_net(struct nf_conntrack_expect *exp) { - return nf_ct_net(exp->master); + return read_pnet(&exp->net); +} + +static inline bool nf_ct_exp_zone_equal_any(const struct nf_conntrack_expect *a, + const struct nf_conntrack_zone *b) +{ +#ifdef CONFIG_NF_CONNTRACK_ZONES + return a->zone.id == b->id; +#else + return true; +#endif } #define NF_CT_EXP_POLICY_NAME_LEN 16 @@ -111,7 +142,6 @@ static inline void nf_ct_unlink_expect(struct nf_conntrack_expect *exp) void nf_ct_remove_expectations(struct nf_conn *ct); void nf_ct_unexpect_related(struct nf_conntrack_expect *exp); -bool nf_ct_remove_expect(struct nf_conntrack_expect *exp); void nf_ct_expect_iterate_destroy(bool (*iter)(struct nf_conntrack_expect *e, void *data), void *data); void nf_ct_expect_iterate_net(struct net *net, @@ -134,5 +164,11 @@ static inline int nf_ct_expect_related(struct nf_conntrack_expect *expect, return nf_ct_expect_related_report(expect, 0, 0, flags); } +int nf_ct_expect_related_pair(struct nf_conntrack_expect *expect[], + unsigned int flag); + +struct nf_conn_help; +void nf_ct_expectation_gc(struct nf_conn_help *master_help); + #endif /*_NF_CONNTRACK_EXPECT_H*/ diff --git a/include/net/netfilter/nf_conntrack_extend.h b/include/net/netfilter/nf_conntrack_extend.h index 0b247248b032..fd5c4dbf72ca 100644 --- a/include/net/netfilter/nf_conntrack_extend.h +++ b/include/net/netfilter/nf_conntrack_extend.h @@ -38,7 +38,6 @@ enum nf_ct_ext_id { struct nf_ct_ext { u8 offset[NF_CT_EXT_NUM]; u8 len; - unsigned int gen_id; char data[] __aligned(8); }; @@ -52,8 +51,6 @@ static inline bool nf_ct_ext_exist(const struct nf_conn *ct, u8 id) return (ct->ext && __nf_ct_ext_exist(ct->ext, id)); } -void *__nf_ct_ext_find(const struct nf_ct_ext *ext, u8 id); - static inline void *nf_ct_ext_find(const struct nf_conn *ct, u8 id) { struct nf_ct_ext *ext = ct->ext; @@ -61,19 +58,10 @@ static inline void *nf_ct_ext_find(const struct nf_conn *ct, u8 id) if (!ext || !__nf_ct_ext_exist(ext, id)) return NULL; - if (unlikely(ext->gen_id)) - return __nf_ct_ext_find(ext, id); - return (void *)ct->ext + ct->ext->offset[id]; } /* Add this type, returns pointer to data or NULL. */ void *nf_ct_ext_add(struct nf_conn *ct, enum nf_ct_ext_id id, gfp_t gfp); -/* ext genid. if ext->id != ext_genid, extensions cannot be used - * anymore unless conntrack has CONFIRMED bit set. - */ -extern atomic_t nf_conntrack_ext_genid; -void nf_ct_ext_bump_genid(void); - #endif /* _NF_CONNTRACK_EXTEND_H */ diff --git a/include/net/netfilter/nf_conntrack_helper.h b/include/net/netfilter/nf_conntrack_helper.h index de2f956abf34..335b8c43694f 100644 --- a/include/net/netfilter/nf_conntrack_helper.h +++ b/include/net/netfilter/nf_conntrack_helper.h @@ -29,23 +29,27 @@ enum nf_ct_helper_flags { #define NF_CT_HELPER_NAME_LEN 16 +/* Must be kept in sync with the classes defined by helpers */ +#define NF_CT_MAX_EXPECT_CLASSES 4 + struct nf_conntrack_helper { struct hlist_node hnode; /* Internal use. */ + struct rcu_head rcu; + char name[NF_CT_HELPER_NAME_LEN]; /* name of the module */ - refcount_t refcnt; struct module *me; /* pointer to self */ - const struct nf_conntrack_expect_policy *expect_policy; + struct nf_conntrack_expect_policy expect_policy[NF_CT_MAX_EXPECT_CLASSES]; - /* Tuple of things we will help (compared against server response) */ - struct nf_conntrack_tuple tuple; + refcount_t ct_refcnt; - /* Function to call when data passes; return verdict, or -1 to - invalidate. */ - int (*help)(struct sk_buff *skb, - unsigned int protoff, - struct nf_conn *ct, - enum ip_conntrack_info conntrackinfo); + u8 nfproto; /* NFPROTO_*, can be NFPROTO_UNSPEC */ + u8 l4proto; /* IPPROTO_UDP/TCP */ + + /* Function to call when data passes; return verdict */ + int __rcu (*help)(struct sk_buff *skb, unsigned int protoff, + struct nf_conn *ct, + enum ip_conntrack_info conntrackinfo); void (*destroy)(struct nf_conn *ct); @@ -63,9 +67,6 @@ struct nf_conntrack_helper { char nat_mod_name[NF_CT_HELPER_NAME_LEN]; }; -/* Must be kept in sync with the classes defined by helpers */ -#define NF_CT_MAX_EXPECT_CLASSES 4 - /* nf_conn feature for connections that have a helper */ struct nf_conn_help { /* Helper. if any */ @@ -92,8 +93,7 @@ struct nf_conntrack_helper *nf_conntrack_helper_try_module_get(const char *name, void nf_conntrack_helper_put(struct nf_conntrack_helper *helper); void nf_ct_helper_init(struct nf_conntrack_helper *helper, - u16 l3num, u16 protonum, const char *name, - u16 default_port, u16 spec_port, u32 id, + u8 l3num, u16 protonum, const char *name, const struct nf_conntrack_expect_policy *exp_pol, u32 expect_class_max, int (*help)(struct sk_buff *skb, unsigned int protoff, @@ -103,13 +103,20 @@ void nf_ct_helper_init(struct nf_conntrack_helper *helper, struct nf_conn *ct), struct module *module); -int nf_conntrack_helper_register(struct nf_conntrack_helper *); +int nf_conntrack_helper_register(struct nf_conntrack_helper *, struct nf_conntrack_helper **); +int __nf_conntrack_helper_register(struct nf_conntrack_helper *); void nf_conntrack_helper_unregister(struct nf_conntrack_helper *); +void nf_conntrack_helper_release(struct nf_conntrack_helper *); -int nf_conntrack_helpers_register(struct nf_conntrack_helper *, unsigned int); -void nf_conntrack_helpers_unregister(struct nf_conntrack_helper *, +int nf_conntrack_helpers_register(struct nf_conntrack_helper *, unsigned int, + struct nf_conntrack_helper **); +void nf_conntrack_helpers_unregister(struct nf_conntrack_helper **, unsigned int); +#define nf_conntrack_helper_deprecated(name) \ + pr_warn("The %s conntrack helper is scheduled for removal.\n" \ + "Please contact the netfilter-devel mailing list if you still need this.\n", name) + struct nf_conn_help *nf_ct_helper_ext_add(struct nf_conn *ct, gfp_t gfp); int __nf_ct_try_assign_helper(struct nf_conn *ct, struct nf_conn *tmpl, @@ -132,10 +139,26 @@ static inline void *nfct_help_data(const struct nf_conn *ct) struct nf_conn_help *help; help = nf_ct_ext_find(ct, NF_CT_EXT_HELPER); + if (!help) + return NULL; return (void *)help->data; } +static inline void nf_ct_help_put(const struct nf_conn *ct) +{ + struct nf_conntrack_helper *helper; + struct nf_conn_help *help; + + help = nfct_help(ct); + if (!help) + return; + + helper = rcu_dereference(help->helper); + if (helper && refcount_dec_and_test(&helper->ct_refcnt)) + kfree_rcu(helper, rcu); +} + int nf_conntrack_helper_init(void); void nf_conntrack_helper_fini(void); @@ -155,6 +178,7 @@ void nf_ct_helper_log(struct sk_buff *skb, const struct nf_conn *ct, void nf_ct_helper_expectfn_register(struct nf_ct_helper_expectfn *n); void nf_ct_helper_expectfn_unregister(struct nf_ct_helper_expectfn *n); +void nf_ct_helper_expectfn_destroy(const struct nf_ct_helper_expectfn *n); struct nf_ct_helper_expectfn * nf_ct_helper_expectfn_find_by_name(const char *name); struct nf_ct_helper_expectfn * diff --git a/include/net/netfilter/nf_conntrack_l4proto.h b/include/net/netfilter/nf_conntrack_l4proto.h index cd5020835a6d..fde2427ceb8f 100644 --- a/include/net/netfilter/nf_conntrack_l4proto.h +++ b/include/net/netfilter/nf_conntrack_l4proto.h @@ -107,11 +107,6 @@ int nf_conntrack_udp_packet(struct nf_conn *ct, unsigned int dataoff, enum ip_conntrack_info ctinfo, const struct nf_hook_state *state); -int nf_conntrack_udplite_packet(struct nf_conn *ct, - struct sk_buff *skb, - unsigned int dataoff, - enum ip_conntrack_info ctinfo, - const struct nf_hook_state *state); int nf_conntrack_tcp_packet(struct nf_conn *ct, struct sk_buff *skb, unsigned int dataoff, @@ -139,8 +134,6 @@ void nf_conntrack_icmpv6_init_net(struct net *net); /* Existing built-in generic protocol */ extern const struct nf_conntrack_l4proto nf_conntrack_l4proto_generic; -#define MAX_NF_CT_PROTO IPPROTO_UDPLITE - const struct nf_conntrack_l4proto *nf_ct_l4proto_find(u8 l4proto); /* Generic netlink helpers */ diff --git a/include/net/netfilter/nf_conntrack_timeout.h b/include/net/netfilter/nf_conntrack_timeout.h index 9fdaba911de6..d60aa86be019 100644 --- a/include/net/netfilter/nf_conntrack_timeout.h +++ b/include/net/netfilter/nf_conntrack_timeout.h @@ -12,8 +12,10 @@ #define CTNL_TIMEOUT_NAME_MAX 32 struct nf_ct_timeout { + refcount_t refcnt; __u16 l3num; const struct nf_conntrack_l4proto *l4proto; + struct rcu_head rcu; char data[]; }; @@ -21,6 +23,22 @@ struct nf_conn_timeout { struct nf_ct_timeout __rcu *timeout; }; +static inline void nf_ct_timeout_put(const struct nf_conn *ct) +{ +#ifdef CONFIG_NF_CONNTRACK_TIMEOUT + struct nf_conn_timeout *timeout_ext; + struct nf_ct_timeout *timeout; + + timeout_ext = nf_ct_ext_find(ct, NF_CT_EXT_TIMEOUT); + if (!timeout_ext) + return; + + timeout = rcu_dereference(timeout_ext->timeout); + if (timeout && refcount_dec_and_test(&timeout->refcnt)) + kfree_rcu(timeout, rcu); +#endif +} + static inline unsigned int * nf_ct_timeout_data(const struct nf_conn_timeout *t) { @@ -55,8 +73,14 @@ struct nf_conn_timeout *nf_ct_timeout_ext_add(struct nf_conn *ct, #ifdef CONFIG_NF_CONNTRACK_TIMEOUT struct nf_conn_timeout *timeout_ext; + if (!timeout) + return NULL; + timeout_ext = nf_ct_ext_add(ct, NF_CT_EXT_TIMEOUT, gfp); - if (timeout_ext == NULL) + if (!timeout_ext || timeout_ext->timeout) + return NULL; + + if (!refcount_inc_not_zero(&timeout->refcnt)) return NULL; rcu_assign_pointer(timeout_ext->timeout, timeout); @@ -74,7 +98,7 @@ static inline unsigned int *nf_ct_timeout_lookup(const struct nf_conn *ct) struct nf_conn_timeout *timeout_ext; timeout_ext = nf_ct_timeout_find(ct); - if (timeout_ext) + if (timeout_ext && rcu_access_pointer(timeout_ext->timeout)) timeouts = nf_ct_timeout_data(timeout_ext); #endif return timeouts; diff --git a/include/net/netfilter/nf_conntrack_tuple.h b/include/net/netfilter/nf_conntrack_tuple.h index f7dd950ff250..4d55b7325707 100644 --- a/include/net/netfilter/nf_conntrack_tuple.h +++ b/include/net/netfilter/nf_conntrack_tuple.h @@ -11,7 +11,7 @@ #ifndef _NF_CONNTRACK_TUPLE_H #define _NF_CONNTRACK_TUPLE_H -#include <linux/netfilter/x_tables.h> +#include <linux/netfilter.h> #include <linux/netfilter/nf_conntrack_tuple_common.h> #include <linux/list_nulls.h> diff --git a/include/net/netfilter/nf_dup_netdev.h b/include/net/netfilter/nf_dup_netdev.h index b175d271aec9..f6b05bd80c3f 100644 --- a/include/net/netfilter/nf_dup_netdev.h +++ b/include/net/netfilter/nf_dup_netdev.h @@ -3,10 +3,47 @@ #define _NF_DUP_NETDEV_H_ #include <net/netfilter/nf_tables.h> +#include <linux/netdevice.h> +#include <linux/sched.h> void nf_dup_netdev_egress(const struct nft_pktinfo *pkt, int oif); void nf_fwd_netdev_egress(const struct nft_pktinfo *pkt, int oif); +#define NF_RECURSION_LIMIT 2 + +#ifndef CONFIG_PREEMPT_RT +static inline bool nf_dev_xmit_recursion(void) +{ + return unlikely(__this_cpu_read(softnet_data.xmit.nf_dup_skb_recursion) > + NF_RECURSION_LIMIT); +} + +static inline void nf_dev_xmit_recursion_inc(void) +{ + __this_cpu_inc(softnet_data.xmit.nf_dup_skb_recursion); +} + +static inline void nf_dev_xmit_recursion_dec(void) +{ + __this_cpu_dec(softnet_data.xmit.nf_dup_skb_recursion); +} +#else +static inline bool nf_dev_xmit_recursion(void) +{ + return unlikely(current->net_xmit.nf_dup_skb_recursion > NF_RECURSION_LIMIT); +} + +static inline void nf_dev_xmit_recursion_inc(void) +{ + current->net_xmit.nf_dup_skb_recursion++; +} + +static inline void nf_dev_xmit_recursion_dec(void) +{ + current->net_xmit.nf_dup_skb_recursion--; +} +#endif + struct nft_offload_ctx; struct nft_flow_rule; diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h index b09c11c048d5..f2e2771f188f 100644 --- a/include/net/netfilter/nf_flow_table.h +++ b/include/net/netfilter/nf_flow_table.h @@ -117,7 +117,7 @@ struct flow_offload_tunnel { struct in6_addr dst_v6; }; - u8 l3_proto; + u8 inner_proto; }; struct flow_offload_tuple { @@ -148,17 +148,19 @@ struct flow_offload_tuple { /* All members above are keys for lookups, see flow_offload_hash(). */ struct { } __hash; - u8 dir:2, + u16 dir:2, xmit_type:3, encap_num:2, + needs_gso_segment:1, tun_num:2, in_vlan_ingress:2; u16 mtu; + u32 dst_cookie; + struct dst_entry *dst_cache; + union { struct { - struct dst_entry *dst_cache; u32 ifidx; - u32 dst_cookie; }; struct { u32 ifidx; @@ -232,6 +234,7 @@ struct nf_flow_route { u32 hw_ifindex; u8 h_source[ETH_ALEN]; u8 h_dest[ETH_ALEN]; + u8 needs_gso_segment:1; } out; enum flow_offload_xmit_type xmit_type; } tuple[FLOW_OFFLOAD_DIR_MAX]; @@ -307,6 +310,14 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow) void flow_offload_refresh(struct nf_flowtable *flow_table, struct flow_offload *flow, bool force); +static inline bool nf_flow_dst_check(struct flow_offload_tuple *tuple) +{ + if (!tuple->dst_cache) + return true; + + return dst_check(tuple->dst_cache, tuple->dst_cookie); +} + struct flow_offload_tuple_rhash *flow_offload_lookup(struct nf_flowtable *flow_table, struct flow_offload_tuple *tuple); void nf_flow_table_gc_run(struct nf_flowtable *flow_table); @@ -355,6 +366,8 @@ static inline int nf_flow_register_bpf(void) void nf_flow_offload_add(struct nf_flowtable *flowtable, struct flow_offload *flow); +void nf_flow_offload_refresh(struct nf_flowtable *flowtable, + struct flow_offload *flow); void nf_flow_offload_del(struct nf_flowtable *flowtable, struct flow_offload *flow); void nf_flow_offload_stats(struct nf_flowtable *flowtable, diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h index 4aeffddb7586..fc3e81c07364 100644 --- a/include/net/netfilter/nf_queue.h +++ b/include/net/netfilter/nf_queue.h @@ -6,19 +6,24 @@ #include <linux/ipv6.h> #include <linux/jhash.h> #include <linux/netfilter.h> +#include <linux/rhashtable-types.h> #include <linux/skbuff.h> /* Each queued (to userspace) skbuff has one of these. */ struct nf_queue_entry { struct list_head list; + struct rhash_head hash_node; struct sk_buff *skb; + struct net_device *skb_dev; unsigned int id; unsigned int hook_index; /* index in hook_entries->hook[] */ #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) + struct net_device *bridge_dev; struct net_device *physin; struct net_device *physout; #endif struct nf_hook_state state; + bool nf_ct_is_unconfirmed; u16 size; /* sizeof(entry) + saved route keys */ /* extra space to store route keys */ diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index 0e266c2d0e7f..9d597482363d 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -6,7 +6,6 @@ #include <linux/list.h> #include <linux/netfilter.h> #include <linux/netfilter/nfnetlink.h> -#include <linux/netfilter/x_tables.h> #include <linux/netfilter/nf_tables.h> #include <linux/u64_stats_sync.h> #include <linux/rhashtable.h> @@ -32,7 +31,9 @@ struct nft_pktinfo { const struct nf_hook_state *state; u8 flags; u8 tprot; + __be16 ethertype; u16 fragoff; + u16 nhoff; u16 thoff; u16 inneroff; }; @@ -84,6 +85,8 @@ static inline void nft_set_pktinfo_unspec(struct nft_pktinfo *pkt) { pkt->flags = 0; pkt->tprot = 0; + pkt->ethertype = pkt->skb->protocol; + pkt->nhoff = 0; pkt->thoff = 0; pkt->fragoff = 0; } @@ -123,17 +126,6 @@ struct nft_regs { }; }; -struct nft_regs_track { - struct { - const struct nft_expr *selector; - const struct nft_expr *bitwise; - u8 num_reg; - } regs[NFT_REG32_NUM]; - - const struct nft_expr *cur; - const struct nft_expr *last; -}; - /* Store/load an u8, u16 or u64 integer to/from the u32 data register. * * Note, when using concatenations, register allocation happens at 32-bit @@ -188,6 +180,13 @@ static inline u64 nft_reg_load64(const u32 *sreg) return get_unaligned((u64 *)sreg); } +static inline bool nft_reg_overlap(u8 src, u8 dst, u32 len) +{ + unsigned int n = DIV_ROUND_UP(len, sizeof(u32)); + + return src != dst && src < dst + n && dst < src + n; +} + static inline void nft_data_copy(u32 *dst, const struct nft_data *src, unsigned int len) { @@ -317,11 +316,13 @@ static inline void *nft_elem_priv_cast(const struct nft_elem_priv *priv) * @NFT_ITER_UNSPEC: unspecified, to catch errors * @NFT_ITER_READ: read-only iteration over set elements * @NFT_ITER_UPDATE: iteration under mutex to update set element state + * @NFT_ITER_UPDATE_CLONE: clone set before iteration under mutex to update element */ enum nft_iter_type { NFT_ITER_UNSPEC, NFT_ITER_READ, NFT_ITER_UPDATE, + NFT_ITER_UPDATE_CLONE, }; struct nft_set; @@ -424,8 +425,6 @@ int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src, gfp_t gfp); void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr); int nft_expr_dump(struct sk_buff *skb, unsigned int attr, const struct nft_expr *expr, bool reset); -bool nft_expr_reduce_bitwise(struct nft_regs_track *track, - const struct nft_expr *expr); struct nft_set_ext; @@ -452,6 +451,7 @@ struct nft_set_ext; * @init: initialize private data of new set instance * @destroy: destroy private data of set instance * @gc_init: initialize garbage collection + * @abort_skip_removal: skip removal of elements from abort path * @elemsize: element private size * * Operations lookup, update and delete have simpler interfaces, are faster @@ -509,6 +509,7 @@ struct nft_set_ops { const struct nft_set *set); void (*gc_init)(const struct nft_set *set); + bool abort_skip_removal; unsigned int elemsize; }; @@ -869,8 +870,8 @@ struct nft_elem_priv *nft_set_elem_init(const struct nft_set *set, const u32 *key, const u32 *key_end, const u32 *data, u64 timeout, u64 expiration, gfp_t gfp); -int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set, - struct nft_expr *expr_array[]); +void nft_set_elem_expr_destroy(const struct nft_ctx *ctx, + struct nft_set_elem_expr *elem_expr); void nft_set_elem_destroy(const struct nft_set *set, const struct nft_elem_priv *elem_priv, bool destroy_expr); @@ -936,7 +937,6 @@ struct nft_offload_ctx; * @destroy_clone: destruction clone function * @dump: function to dump parameters * @validate: validate expression, called during loop detection - * @reduce: reduce expression * @gc: garbage collection expression * @offload: hardware offload expression * @offload_action: function to report true/false to allocate one slot or not in the flow @@ -970,8 +970,6 @@ struct nft_expr_ops { bool reset); int (*validate)(const struct nft_ctx *ctx, const struct nft_expr *expr); - bool (*reduce)(struct nft_regs_track *track, - const struct nft_expr *expr); bool (*gc)(struct net *net, const struct nft_expr *expr); int (*offload)(struct nft_offload_ctx *ctx, @@ -1211,12 +1209,15 @@ struct nft_stats { struct u64_stats_sync syncp; }; +#define NFT_HOOK_REMOVE (1 << 0) + struct nft_hook { struct list_head list; struct list_head ops_list; struct rcu_head rcu; char ifname[IFNAMSIZ]; u8 ifnamelen; + u8 flags; }; struct nf_hook_ops *nft_hook_find_ops(const struct nft_hook *hook, @@ -1291,6 +1292,7 @@ static inline void nft_use_inc_restore(u32 *use) * @sets: sets in the table * @objects: stateful objects in the table * @flowtables: flow tables in the table + * @objname_ht: hashtable for objects lookup by name * @hgenerator: handle generator state * @handle: table handle * @use: number of chain references to this table @@ -1310,6 +1312,7 @@ struct nft_table { struct list_head sets; struct list_head objects; struct list_head flowtables; + struct rhltable objname_ht; u64 hgenerator; u64 handle; u32 use; @@ -1397,7 +1400,7 @@ static inline void *nft_obj_data(const struct nft_object *obj) #define nft_expr_obj(expr) *((struct nft_object **)nft_expr_priv(expr)) struct nft_object *nft_obj_lookup(const struct net *net, - const struct nft_table *table, + struct nft_table *table, const struct nlattr *nla, u32 objtype, u8 genmask); @@ -1672,6 +1675,16 @@ struct nft_trans { }; /** + * struct nft_trans_hook - nf_tables hook update in transaction + * @list: used internally + * @hook: struct nft_hook with the device hook + */ +struct nft_trans_hook { + struct list_head list; + struct nft_hook *hook; +}; + +/** * struct nft_trans_binding - nf_tables object with binding support in transaction * @nft_trans: base structure, MUST be first member * @binding_list: list of objects with possible bindings @@ -1856,6 +1869,11 @@ struct nft_trans_gc { struct rcu_head rcu; }; +static inline int nft_trans_gc_space(const struct nft_trans_gc *trans) +{ + return NFT_TRANS_GC_BATCHCOUNT - trans->count; +} + static inline void nft_ctx_update(struct nft_ctx *ctx, const struct nft_trans *trans) { @@ -1929,6 +1947,7 @@ struct nftables_pernet { struct list_head binding_list; struct list_head module_list; struct list_head notify_list; + struct list_head set_update_list; struct mutex commit_mutex; u64 table_handle; u64 tstamp; @@ -1949,20 +1968,4 @@ static inline u64 nft_net_tstamp(const struct net *net) return nft_pernet(net)->tstamp; } -#define __NFT_REDUCE_READONLY 1UL -#define NFT_REDUCE_READONLY (void *)__NFT_REDUCE_READONLY - -void nft_reg_track_update(struct nft_regs_track *track, - const struct nft_expr *expr, u8 dreg, u8 len); -void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len); -void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg); - -static inline bool nft_reg_track_cmp(struct nft_regs_track *track, - const struct nft_expr *expr, u8 dreg) -{ - return track->regs[dreg].selector && - track->regs[dreg].selector->ops == expr->ops && - track->regs[dreg].num_reg == 0; -} - #endif /* _NET_NF_TABLES_H */ diff --git a/include/net/netfilter/nf_tables_ipv4.h b/include/net/netfilter/nf_tables_ipv4.h index fcf967286e37..e715405a73cb 100644 --- a/include/net/netfilter/nf_tables_ipv4.h +++ b/include/net/netfilter/nf_tables_ipv4.h @@ -12,16 +12,19 @@ static inline void nft_set_pktinfo_ipv4(struct nft_pktinfo *pkt) ip = ip_hdr(pkt->skb); pkt->flags = NFT_PKTINFO_L4PROTO; pkt->tprot = ip->protocol; + pkt->ethertype = pkt->skb->protocol; + pkt->nhoff = 0; pkt->thoff = ip_hdrlen(pkt->skb); pkt->fragoff = ntohs(ip->frag_off) & IP_OFFSET; } -static inline int __nft_set_pktinfo_ipv4_validate(struct nft_pktinfo *pkt) +static inline int __nft_set_pktinfo_ipv4_validate(struct nft_pktinfo *pkt, + int nhoff) { struct iphdr *iph, _iph; u32 len, thoff, skb_len; - iph = skb_header_pointer(pkt->skb, skb_network_offset(pkt->skb), + iph = skb_header_pointer(pkt->skb, skb_network_offset(pkt->skb) + nhoff, sizeof(*iph), &_iph); if (!iph) return -1; @@ -31,7 +34,7 @@ static inline int __nft_set_pktinfo_ipv4_validate(struct nft_pktinfo *pkt) len = iph_totlen(pkt->skb, iph); thoff = iph->ihl * 4; - skb_len = pkt->skb->len - skb_network_offset(pkt->skb); + skb_len = pkt->skb->len - skb_network_offset(pkt->skb) - nhoff; if (skb_len < len) return -1; @@ -42,7 +45,9 @@ static inline int __nft_set_pktinfo_ipv4_validate(struct nft_pktinfo *pkt) pkt->flags = NFT_PKTINFO_L4PROTO; pkt->tprot = iph->protocol; - pkt->thoff = skb_network_offset(pkt->skb) + thoff; + pkt->ethertype = pkt->skb->protocol; + pkt->nhoff = nhoff; + pkt->thoff = skb_network_offset(pkt->skb) + nhoff + thoff; pkt->fragoff = ntohs(iph->frag_off) & IP_OFFSET; return 0; @@ -50,7 +55,7 @@ static inline int __nft_set_pktinfo_ipv4_validate(struct nft_pktinfo *pkt) static inline void nft_set_pktinfo_ipv4_validate(struct nft_pktinfo *pkt) { - if (__nft_set_pktinfo_ipv4_validate(pkt) < 0) + if (__nft_set_pktinfo_ipv4_validate(pkt, 0) < 0) nft_set_pktinfo_unspec(pkt); } @@ -78,6 +83,8 @@ static inline int nft_set_pktinfo_ipv4_ingress(struct nft_pktinfo *pkt) } pkt->flags = NFT_PKTINFO_L4PROTO; + pkt->ethertype = pkt->skb->protocol; + pkt->nhoff = 0; pkt->tprot = iph->protocol; pkt->thoff = thoff; pkt->fragoff = ntohs(iph->frag_off) & IP_OFFSET; diff --git a/include/net/netfilter/nf_tables_ipv6.h b/include/net/netfilter/nf_tables_ipv6.h index a0633eeaec97..d7b8c559b795 100644 --- a/include/net/netfilter/nf_tables_ipv6.h +++ b/include/net/netfilter/nf_tables_ipv6.h @@ -20,21 +20,23 @@ static inline void nft_set_pktinfo_ipv6(struct nft_pktinfo *pkt) pkt->flags = NFT_PKTINFO_L4PROTO; pkt->tprot = protohdr; + pkt->ethertype = pkt->skb->protocol; + pkt->nhoff = 0; pkt->thoff = thoff; pkt->fragoff = frag_off; } -static inline int __nft_set_pktinfo_ipv6_validate(struct nft_pktinfo *pkt) +static inline int __nft_set_pktinfo_ipv6_validate(struct nft_pktinfo *pkt, int nhoff) { #if IS_ENABLED(CONFIG_IPV6) unsigned int flags = IP6_FH_F_AUTH; struct ipv6hdr *ip6h, _ip6h; - unsigned int thoff = 0; + unsigned int thoff = nhoff; unsigned short frag_off; u32 pkt_len, skb_len; int protohdr; - ip6h = skb_header_pointer(pkt->skb, skb_network_offset(pkt->skb), + ip6h = skb_header_pointer(pkt->skb, skb_network_offset(pkt->skb) + nhoff, sizeof(*ip6h), &_ip6h); if (!ip6h) return -1; @@ -42,8 +44,8 @@ static inline int __nft_set_pktinfo_ipv6_validate(struct nft_pktinfo *pkt) if (ip6h->version != 6) return -1; - pkt_len = ntohs(ip6h->payload_len); - skb_len = pkt->skb->len - skb_network_offset(pkt->skb); + pkt_len = ipv6_payload_len(pkt->skb, ip6h); + skb_len = pkt->skb->len - skb_network_offset(pkt->skb) - nhoff; if (pkt_len + sizeof(*ip6h) > skb_len) return -1; @@ -53,6 +55,8 @@ static inline int __nft_set_pktinfo_ipv6_validate(struct nft_pktinfo *pkt) pkt->flags = NFT_PKTINFO_L4PROTO; pkt->tprot = protohdr; + pkt->ethertype = pkt->skb->protocol; + pkt->nhoff = nhoff; pkt->thoff = thoff; pkt->fragoff = frag_off; @@ -64,7 +68,7 @@ static inline int __nft_set_pktinfo_ipv6_validate(struct nft_pktinfo *pkt) static inline void nft_set_pktinfo_ipv6_validate(struct nft_pktinfo *pkt) { - if (__nft_set_pktinfo_ipv6_validate(pkt) < 0) + if (__nft_set_pktinfo_ipv6_validate(pkt, 0) < 0) nft_set_pktinfo_unspec(pkt); } @@ -86,7 +90,7 @@ static inline int nft_set_pktinfo_ipv6_ingress(struct nft_pktinfo *pkt) if (ip6h->version != 6) goto inhdr_error; - pkt_len = ntohs(ip6h->payload_len); + pkt_len = ipv6_payload_len(pkt->skb, ip6h); if (pkt_len + sizeof(*ip6h) > pkt->skb->len) { idev = __in6_dev_get(nft_in(pkt)); __IP6_INC_STATS(nft_net(pkt), idev, IPSTATS_MIB_INTRUNCATEDPKTS); @@ -99,6 +103,8 @@ static inline int nft_set_pktinfo_ipv6_ingress(struct nft_pktinfo *pkt) pkt->flags = NFT_PKTINFO_L4PROTO; pkt->tprot = protohdr; + pkt->ethertype = pkt->skb->protocol; + pkt->nhoff = 0; pkt->thoff = thoff; pkt->fragoff = frag_off; diff --git a/include/net/netfilter/nf_tables_offload.h b/include/net/netfilter/nf_tables_offload.h index 3568b6a2f5f0..14c427891ee6 100644 --- a/include/net/netfilter/nf_tables_offload.h +++ b/include/net/netfilter/nf_tables_offload.h @@ -67,6 +67,16 @@ struct nft_flow_rule { struct flow_rule *rule; }; +static inline struct flow_action_entry * +nft_flow_action_entry_next(struct nft_offload_ctx *ctx, + struct nft_flow_rule *flow) +{ + if (unlikely(ctx->num_actions >= flow->rule->action.num_entries)) + return NULL; + + return &flow->rule->action.entries[ctx->num_actions++]; +} + void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow, enum flow_dissector_key_id addr_type); diff --git a/include/net/netfilter/nft_fib.h b/include/net/netfilter/nft_fib.h index 7370fba844ef..e0422456f27b 100644 --- a/include/net/netfilter/nft_fib.h +++ b/include/net/netfilter/nft_fib.h @@ -66,6 +66,4 @@ void nft_fib6_eval(const struct nft_expr *expr, struct nft_regs *regs, void nft_fib_store_result(void *reg, const struct nft_fib *priv, const struct net_device *dev); -bool nft_fib_reduce(struct nft_regs_track *track, - const struct nft_expr *expr); #endif diff --git a/include/net/netfilter/nft_meta.h b/include/net/netfilter/nft_meta.h index d602263590fe..6cf1d910bbf8 100644 --- a/include/net/netfilter/nft_meta.h +++ b/include/net/netfilter/nft_meta.h @@ -40,12 +40,11 @@ void nft_meta_set_eval(const struct nft_expr *expr, void nft_meta_set_destroy(const struct nft_ctx *ctx, const struct nft_expr *expr); +int nft_meta_get_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr); int nft_meta_set_validate(const struct nft_ctx *ctx, const struct nft_expr *expr); -bool nft_meta_get_reduce(struct nft_regs_track *track, - const struct nft_expr *expr); - struct nft_inner_tun_ctx; void nft_meta_inner_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt, diff --git a/include/net/netlink.h b/include/net/netlink.h index 1a8356ca4b78..b5048dd1d511 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -2260,11 +2260,33 @@ static inline struct nlattr *nla_nest_start(struct sk_buff *skb, int attrtype) */ static inline int nla_nest_end(struct sk_buff *skb, struct nlattr *start) { - start->nla_len = skb_tail_pointer(skb) - (unsigned char *)start; + unsigned long diff = skb_tail_pointer(skb) - (unsigned char *)start; + + DEBUG_NET_WARN_ON_ONCE(diff > U16_MAX); + start->nla_len = diff; return skb->len; } /** + * nla_nest_end_safe - Validate and finalize nesting of attributes + * @skb: socket buffer the attributes are stored in + * @start: container attribute + * + * Corrects the container attribute header to include all appended + * attributes. + * + * Returns: the total data length of the skb, or -EMSGSIZE if the + * nested attribute length exceeds U16_MAX. + */ +static inline int nla_nest_end_safe(struct sk_buff *skb, struct nlattr *start) +{ + if (skb_tail_pointer(skb) - (unsigned char *)start > U16_MAX) + return -EMSGSIZE; + + return nla_nest_end(skb, start); +} + +/** * nla_nest_cancel - Cancel nesting of attributes * @skb: socket buffer the message is stored in * @start: container attribute diff --git a/include/net/netmem.h b/include/net/netmem.h index 9e10f4ac50c3..bccacd21b6c3 100644 --- a/include/net/netmem.h +++ b/include/net/netmem.h @@ -93,25 +93,9 @@ enum net_iov_type { * supported. */ struct net_iov { - union { - struct netmem_desc desc; - - /* XXX: The following part should be removed once all - * the references to them are converted so as to be - * accessed via netmem_desc e.g. niov->desc.pp instead - * of niov->pp. - */ - struct { - unsigned long _flags; - unsigned long pp_magic; - struct page_pool *pp; - unsigned long _pp_mapping_pad; - unsigned long dma_addr; - atomic_long_t pp_ref_count; - }; - }; - struct net_iov_area *owner; + struct netmem_desc desc; enum net_iov_type type; + struct net_iov_area *owner; }; struct net_iov_area { @@ -123,26 +107,6 @@ struct net_iov_area { unsigned long base_virtual; }; -/* net_iov is union'ed with struct netmem_desc mirroring struct page, so - * the page_pool can access these fields without worrying whether the - * underlying fields are accessed via netmem_desc or directly via - * net_iov, until all the references to them are converted so as to be - * accessed via netmem_desc e.g. niov->desc.pp instead of niov->pp. - * - * The non-net stack fields of struct page are private to the mm stack - * and must never be mirrored to net_iov. - */ -#define NET_IOV_ASSERT_OFFSET(desc, iov) \ - static_assert(offsetof(struct netmem_desc, desc) == \ - offsetof(struct net_iov, iov)) -NET_IOV_ASSERT_OFFSET(_flags, _flags); -NET_IOV_ASSERT_OFFSET(pp_magic, pp_magic); -NET_IOV_ASSERT_OFFSET(pp, pp); -NET_IOV_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad); -NET_IOV_ASSERT_OFFSET(dma_addr, dma_addr); -NET_IOV_ASSERT_OFFSET(pp_ref_count, pp_ref_count); -#undef NET_IOV_ASSERT_OFFSET - static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov) { return niov->owner; @@ -153,6 +117,16 @@ static inline unsigned int net_iov_idx(const struct net_iov *niov) return niov - net_iov_owner(niov)->niovs; } +/* Initialize a niov: stamp the owning area, the memory provider type. + */ +static inline void net_iov_init(struct net_iov *niov, + struct net_iov_area *owner, + enum net_iov_type type) +{ + niov->owner = owner; + niov->type = type; +} + /* netmem */ /** @@ -389,8 +363,36 @@ static inline unsigned long netmem_get_dma_addr(netmem_ref netmem) return netmem_to_nmdesc(netmem)->dma_addr; } -void get_netmem(netmem_ref netmem); -void put_netmem(netmem_ref netmem); +#if defined(CONFIG_NET_DEVMEM) +static inline bool net_is_devmem_iov(const struct net_iov *niov) +{ + return niov->type == NET_IOV_DMABUF; +} +#else +static inline bool net_is_devmem_iov(const struct net_iov *niov) +{ + return false; +} +#endif + +void __get_netmem(netmem_ref netmem); +void __put_netmem(netmem_ref netmem); + +static __always_inline void get_netmem(netmem_ref netmem) +{ + if (netmem_is_net_iov(netmem)) + __get_netmem(netmem); + else + get_page(netmem_to_page(netmem)); +} + +static __always_inline void put_netmem(netmem_ref netmem) +{ + if (netmem_is_net_iov(netmem)) + __put_netmem(netmem); + else + put_page(netmem_to_page(netmem)); +} #define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \ do { \ diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index 2dbd46fc4734..cb7f8bf15671 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -74,6 +74,7 @@ struct netns_ipv4 { /* TXRX readonly hotpath cache lines */ __cacheline_group_begin(netns_ipv4_read_txrx); + u8 sysctl_tcp_shrink_window; __cacheline_group_end(netns_ipv4_read_txrx); /* RX readonly hotpath cache line */ @@ -88,6 +89,12 @@ struct netns_ipv4 { int sysctl_tcp_rcvbuf_low_rtt; __cacheline_group_end(netns_ipv4_read_rx); + /* ICMP rate limiter hot cache line. */ + __cacheline_group_begin_aligned(icmp); + atomic_t icmp_global_credit; + u32 icmp_global_stamp; + __cacheline_group_end_aligned(icmp); + struct inet_timewait_death_row tcp_death_row; struct udp_table *udp_table; @@ -111,12 +118,12 @@ struct netns_ipv4 { struct fib_rules_ops *rules_ops; struct fib_table __rcu *fib_main; struct fib_table __rcu *fib_default; + spinlock_t fib_table_hash_lock; unsigned int fib_rules_require_fldissect; bool fib_has_custom_rules; #endif bool fib_has_custom_local_routes; bool fib_offload_disabled; - u8 sysctl_tcp_shrink_window; #ifdef CONFIG_IP_ROUTE_CLASSID atomic_t fib_num_tclassid_users; #endif @@ -141,8 +148,7 @@ struct netns_ipv4 { int sysctl_icmp_ratemask; int sysctl_icmp_msgs_per_sec; int sysctl_icmp_msgs_burst; - atomic_t icmp_global_credit; - u32 icmp_global_stamp; + u32 ip_rt_min_pmtu; int ip_rt_mtu_expires; int ip_rt_min_advmss; @@ -161,6 +167,7 @@ struct netns_ipv4 { u8 sysctl_ip_autobind_reuse; /* Shall we try to damage output packets if routing dev changes? */ u8 sysctl_ip_dynaddr; + u32 sysctl_ip_local_port_step_width; #ifdef CONFIG_NET_L3_MASTER_DEV u8 sysctl_raw_l3mdev_accept; #endif @@ -269,11 +276,14 @@ struct netns_ipv4 { #ifdef CONFIG_IP_MROUTE #ifndef CONFIG_IP_MROUTE_MULTIPLE_TABLES - struct mr_table *mrt; + struct mr_table __rcu *mrt; #else struct list_head mr_tables; struct fib_rules_ops *mr_rules_ops; #endif + struct fib_notifier_ops *ipmr_notifier_ops; + atomic_t ipmr_seq; + struct mutex mfc_mutex; #endif #ifdef CONFIG_IP_ROUTE_MULTIPATH struct sysctl_fib_multipath_hash_seed sysctl_fib_multipath_hash_seed; @@ -285,9 +295,6 @@ struct netns_ipv4 { struct fib_notifier_ops *notifier_ops; unsigned int fib_seq; /* writes protected by rtnl_mutex */ - struct fib_notifier_ops *ipmr_notifier_ops; - unsigned int ipmr_seq; /* protected by rtnl_mutex */ - atomic_t rt_genid; siphash_key_t ip_id_key; struct hlist_head *inet_addr_lst; diff --git a/include/net/netns/ipv6.h b/include/net/netns/ipv6.h index 08d2ecc96e2b..ffda15318be5 100644 --- a/include/net/netns/ipv6.h +++ b/include/net/netns/ipv6.h @@ -30,19 +30,23 @@ struct netns_sysctl_ipv6 { int ip6_rt_min_advmss; u32 multipath_hash_fields; u8 multipath_hash_policy; - u8 bindv6only; + + __cacheline_group_begin(sysctl_ipv6_flowlabel); u8 flowlabel_consistency; u8 auto_flowlabels; - int icmpv6_time; + u8 flowlabel_state_ranges; + __cacheline_group_end(sysctl_ipv6_flowlabel); + u8 icmpv6_echo_ignore_all; u8 icmpv6_echo_ignore_multicast; u8 icmpv6_echo_ignore_anycast; + int icmpv6_time; DECLARE_BITMAP(icmpv6_ratemask, ICMPV6_MSG_MAX + 1); unsigned long *icmpv6_ratemask_ptr; u8 anycast_src_echo_reply; + u8 bindv6only; u8 ip_nonlocal_bind; u8 fwmark_reflect; - u8 flowlabel_state_ranges; int idgen_retries; int idgen_delay; int flowlabel_reflect; @@ -108,13 +112,15 @@ struct netns_ipv6 { struct list_head mr6_tables; struct fib_rules_ops *mr6_rules_ops; #endif + struct fib_notifier_ops *ip6mr_notifier_ops; + atomic_t ipmr_seq; + struct mutex mfc_mutex; #endif atomic_t dev_addr_genid; atomic_t fib6_sernum; struct seg6_pernet_data *seg6_data; struct fib_notifier_ops *notifier_ops; - struct fib_notifier_ops *ip6mr_notifier_ops; - unsigned int ipmr_seq; /* protected by rtnl_mutex */ + int flowlabel_count; struct { struct hlist_head head; spinlock_t lock; diff --git a/include/net/netns/mib.h b/include/net/netns/mib.h index 7e373664b1e7..dce05f8e6a33 100644 --- a/include/net/netns/mib.h +++ b/include/net/netns/mib.h @@ -28,11 +28,6 @@ struct netns_mib { DEFINE_SNMP_STAT(struct mptcp_mib, mptcp_statistics); #endif - DEFINE_SNMP_STAT(struct udp_mib, udplite_statistics); -#if IS_ENABLED(CONFIG_IPV6) - DEFINE_SNMP_STAT(struct udp_mib, udplite_stats_in6); -#endif - DEFINE_SNMP_STAT(struct icmp_mib, icmp_statistics); DEFINE_SNMP_STAT_ATOMIC(struct icmpmsg_mib, icmpmsg_statistics); #if IS_ENABLED(CONFIG_IPV6) diff --git a/include/net/netns/mpls.h b/include/net/netns/mpls.h index 6682e51513ef..2073cbac2afb 100644 --- a/include/net/netns/mpls.h +++ b/include/net/netns/mpls.h @@ -17,6 +17,7 @@ struct netns_mpls { size_t platform_labels; struct mpls_route __rcu * __rcu *platform_label; struct mutex platform_mutex; + seqcount_mutex_t platform_label_seq; struct ctl_table_header *ctl; }; diff --git a/include/net/netns/vsock.h b/include/net/netns/vsock.h new file mode 100644 index 000000000000..7f84aad92f57 --- /dev/null +++ b/include/net/netns/vsock.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NET_NET_NAMESPACE_VSOCK_H +#define __NET_NET_NAMESPACE_VSOCK_H + +#include <linux/types.h> + +enum vsock_net_mode { + VSOCK_NET_MODE_GLOBAL, + VSOCK_NET_MODE_LOCAL, +}; + +struct netns_vsock { + struct ctl_table_header *sysctl_hdr; + + /* protected by the vsock_table_lock in af_vsock.c */ + u32 port; + + enum vsock_net_mode mode; + enum vsock_net_mode child_ns_mode; + + /* 0 = unlocked, 1 = locked to global, 2 = locked to local */ + int child_ns_mode_locked; + + int g2h_fallback; +}; +#endif /* __NET_NET_NAMESPACE_VSOCK_H */ diff --git a/include/net/netns/xfrm.h b/include/net/netns/xfrm.h index 23dd647fe024..b73983a17e08 100644 --- a/include/net/netns/xfrm.h +++ b/include/net/netns/xfrm.h @@ -59,7 +59,7 @@ struct netns_xfrm { struct list_head inexact_bins; - struct sock *nlsk; + struct sock __rcu *nlsk; struct sock *nlsk_stash; u32 sysctl_aevent_etime; diff --git a/include/net/netrom.h b/include/net/netrom.h deleted file mode 100644 index f0565a5987d1..000000000000 --- a/include/net/netrom.h +++ /dev/null @@ -1,273 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Declarations of NET/ROM type objects. - * - * Jonathan Naylor G4KLX 9/4/95 - */ - -#ifndef _NETROM_H -#define _NETROM_H - -#include <linux/netrom.h> -#include <linux/list.h> -#include <linux/slab.h> -#include <net/sock.h> -#include <linux/refcount.h> -#include <linux/seq_file.h> -#include <net/ax25.h> - -#define NR_NETWORK_LEN 15 -#define NR_TRANSPORT_LEN 5 - -#define NR_PROTO_IP 0x0C - -#define NR_PROTOEXT 0x00 -#define NR_CONNREQ 0x01 -#define NR_CONNACK 0x02 -#define NR_DISCREQ 0x03 -#define NR_DISCACK 0x04 -#define NR_INFO 0x05 -#define NR_INFOACK 0x06 -#define NR_RESET 0x07 - -#define NR_CHOKE_FLAG 0x80 -#define NR_NAK_FLAG 0x40 -#define NR_MORE_FLAG 0x20 - -/* Define Link State constants. */ -enum { - NR_STATE_0, - NR_STATE_1, - NR_STATE_2, - NR_STATE_3 -}; - -#define NR_COND_ACK_PENDING 0x01 -#define NR_COND_REJECT 0x02 -#define NR_COND_PEER_RX_BUSY 0x04 -#define NR_COND_OWN_RX_BUSY 0x08 - -#define NR_DEFAULT_T1 120000 /* Outstanding frames - 120 seconds */ -#define NR_DEFAULT_T2 5000 /* Response delay - 5 seconds */ -#define NR_DEFAULT_N2 3 /* Number of Retries - 3 */ -#define NR_DEFAULT_T4 180000 /* Busy Delay - 180 seconds */ -#define NR_DEFAULT_IDLE 0 /* No Activity Timeout - none */ -#define NR_DEFAULT_WINDOW 4 /* Default Window Size - 4 */ -#define NR_DEFAULT_OBS 6 /* Default Obsolescence Count - 6 */ -#define NR_DEFAULT_QUAL 10 /* Default Neighbour Quality - 10 */ -#define NR_DEFAULT_TTL 16 /* Default Time To Live - 16 */ -#define NR_DEFAULT_ROUTING 1 /* Is routing enabled ? */ -#define NR_DEFAULT_FAILS 2 /* Link fails until route fails */ -#define NR_DEFAULT_RESET 0 /* Sent / accept reset cmds? */ - -#define NR_MODULUS 256 -#define NR_MAX_WINDOW_SIZE 127 /* Maximum Window Allowable - 127 */ -#define NR_MAX_PACKET_SIZE 236 /* Maximum Packet Length - 236 */ - -struct nr_sock { - struct sock sock; - ax25_address user_addr, source_addr, dest_addr; - struct net_device *device; - unsigned char my_index, my_id; - unsigned char your_index, your_id; - unsigned char state, condition, bpqext, window; - unsigned short vs, vr, va, vl; - unsigned char n2, n2count; - unsigned long t1, t2, t4, idle; - unsigned short fraglen; - struct timer_list t1timer; - struct timer_list t2timer; - struct timer_list t4timer; - struct timer_list idletimer; - struct sk_buff_head ack_queue; - struct sk_buff_head reseq_queue; - struct sk_buff_head frag_queue; -}; - -#define nr_sk(sk) ((struct nr_sock *)(sk)) - -struct nr_neigh { - struct hlist_node neigh_node; - ax25_address callsign; - ax25_digi *digipeat; - ax25_cb *ax25; - struct net_device *dev; - unsigned char quality; - unsigned char locked; - unsigned short count; - unsigned int number; - unsigned char failed; - refcount_t refcount; -}; - -struct nr_route { - unsigned char quality; - unsigned char obs_count; - struct nr_neigh *neighbour; -}; - -struct nr_node { - struct hlist_node node_node; - ax25_address callsign; - char mnemonic[7]; - unsigned char which; - unsigned char count; - struct nr_route routes[3]; - refcount_t refcount; - spinlock_t node_lock; -}; - -/********************************************************************* - * nr_node & nr_neigh lists, refcounting and locking - *********************************************************************/ - -#define nr_node_hold(__nr_node) \ - refcount_inc(&((__nr_node)->refcount)) - -static __inline__ void nr_node_put(struct nr_node *nr_node) -{ - if (refcount_dec_and_test(&nr_node->refcount)) { - kfree(nr_node); - } -} - -#define nr_neigh_hold(__nr_neigh) \ - refcount_inc(&((__nr_neigh)->refcount)) - -static __inline__ void nr_neigh_put(struct nr_neigh *nr_neigh) -{ - if (refcount_dec_and_test(&nr_neigh->refcount)) { - if (nr_neigh->ax25) - ax25_cb_put(nr_neigh->ax25); - kfree(nr_neigh->digipeat); - kfree(nr_neigh); - } -} - -/* nr_node_lock and nr_node_unlock also hold/put the node's refcounter. - */ -static __inline__ void nr_node_lock(struct nr_node *nr_node) -{ - nr_node_hold(nr_node); - spin_lock_bh(&nr_node->node_lock); -} - -static __inline__ void nr_node_unlock(struct nr_node *nr_node) -{ - spin_unlock_bh(&nr_node->node_lock); - nr_node_put(nr_node); -} - -#define nr_neigh_for_each(__nr_neigh, list) \ - hlist_for_each_entry(__nr_neigh, list, neigh_node) - -#define nr_neigh_for_each_safe(__nr_neigh, node2, list) \ - hlist_for_each_entry_safe(__nr_neigh, node2, list, neigh_node) - -#define nr_node_for_each(__nr_node, list) \ - hlist_for_each_entry(__nr_node, list, node_node) - -#define nr_node_for_each_safe(__nr_node, node2, list) \ - hlist_for_each_entry_safe(__nr_node, node2, list, node_node) - - -/*********************************************************************/ - -/* af_netrom.c */ -extern int sysctl_netrom_default_path_quality; -extern int sysctl_netrom_obsolescence_count_initialiser; -extern int sysctl_netrom_network_ttl_initialiser; -extern int sysctl_netrom_transport_timeout; -extern int sysctl_netrom_transport_maximum_tries; -extern int sysctl_netrom_transport_acknowledge_delay; -extern int sysctl_netrom_transport_busy_delay; -extern int sysctl_netrom_transport_requested_window_size; -extern int sysctl_netrom_transport_no_activity_timeout; -extern int sysctl_netrom_routing_control; -extern int sysctl_netrom_link_fails_count; -extern int sysctl_netrom_reset_circuit; - -int nr_rx_frame(struct sk_buff *, struct net_device *); -void nr_destroy_socket(struct sock *); - -/* nr_dev.c */ -int nr_rx_ip(struct sk_buff *, struct net_device *); -void nr_setup(struct net_device *); - -/* nr_in.c */ -int nr_process_rx_frame(struct sock *, struct sk_buff *); - -/* nr_loopback.c */ -void nr_loopback_init(void); -void nr_loopback_clear(void); -int nr_loopback_queue(struct sk_buff *); - -/* nr_out.c */ -void nr_output(struct sock *, struct sk_buff *); -void nr_send_nak_frame(struct sock *); -void nr_kick(struct sock *); -void nr_transmit_buffer(struct sock *, struct sk_buff *); -void nr_establish_data_link(struct sock *); -void nr_enquiry_response(struct sock *); -void nr_check_iframes_acked(struct sock *, unsigned short); - -/* nr_route.c */ -void nr_rt_device_down(struct net_device *); -struct net_device *nr_dev_first(void); -struct net_device *nr_dev_get(ax25_address *); -int nr_rt_ioctl(unsigned int, void __user *); -void nr_link_failed(ax25_cb *, int); -int nr_route_frame(struct sk_buff *, ax25_cb *); -extern const struct seq_operations nr_node_seqops; -extern const struct seq_operations nr_neigh_seqops; -void nr_rt_free(void); - -/* nr_subr.c */ -void nr_clear_queues(struct sock *); -void nr_frames_acked(struct sock *, unsigned short); -void nr_requeue_frames(struct sock *); -int nr_validate_nr(struct sock *, unsigned short); -int nr_in_rx_window(struct sock *, unsigned short); -void nr_write_internal(struct sock *, int); - -void __nr_transmit_reply(struct sk_buff *skb, int mine, unsigned char cmdflags); - -/* - * This routine is called when a Connect Acknowledge with the Choke Flag - * set is needed to refuse a connection. - */ -#define nr_transmit_refusal(skb, mine) \ -do { \ - __nr_transmit_reply((skb), (mine), NR_CONNACK | NR_CHOKE_FLAG); \ -} while (0) - -/* - * This routine is called when we don't have a circuit matching an incoming - * NET/ROM packet. This is an G8PZT Xrouter extension. - */ -#define nr_transmit_reset(skb, mine) \ -do { \ - __nr_transmit_reply((skb), (mine), NR_RESET); \ -} while (0) - -void nr_disconnect(struct sock *, int); - -/* nr_timer.c */ -void nr_init_timers(struct sock *sk); -void nr_start_heartbeat(struct sock *); -void nr_start_t1timer(struct sock *); -void nr_start_t2timer(struct sock *); -void nr_start_t4timer(struct sock *); -void nr_start_idletimer(struct sock *); -void nr_stop_heartbeat(struct sock *); -void nr_stop_t1timer(struct sock *); -void nr_stop_t2timer(struct sock *); -void nr_stop_t4timer(struct sock *); -void nr_stop_idletimer(struct sock *); -int nr_t1timer_running(struct sock *); - -/* sysctl_net_netrom.c */ -int nr_register_sysctl(void); -void nr_unregister_sysctl(void); - -#endif diff --git a/include/net/nexthop.h b/include/net/nexthop.h index 572e69cda476..f86c115074d7 100644 --- a/include/net/nexthop.h +++ b/include/net/nexthop.h @@ -28,6 +28,7 @@ struct nh_config { u8 nh_protocol; u8 nh_blackhole; u8 nh_fdb; + __be16 nh_dst_port; u32 nh_flags; int nh_ifindex; @@ -63,6 +64,7 @@ struct nh_info { u8 family; bool reject_nh; bool fdb_nh; + __be16 dst_port; union { struct fib_nh_common fib_nhc; @@ -574,7 +576,8 @@ struct fib_nh_common *nexthop_fdb_nhc(struct nexthop *nh) } static inline struct fib_nh_common *nexthop_path_fdb_result(struct nexthop *nh, - int hash) + int hash, + __be16 *dst_port) { struct nh_info *nhi; struct nexthop *nhp; @@ -583,6 +586,7 @@ static inline struct fib_nh_common *nexthop_path_fdb_result(struct nexthop *nh, if (unlikely(!nhp)) return NULL; nhi = rcu_dereference(nhp->nh_info); + *dst_port = nhi->dst_port; return &nhi->fib_nhc; } #endif diff --git a/include/net/nfc/hci.h b/include/net/nfc/hci.h index 756c11084f65..86ed63e5d533 100644 --- a/include/net/nfc/hci.h +++ b/include/net/nfc/hci.h @@ -144,7 +144,7 @@ struct nfc_hci_dev { data_exchange_cb_t async_cb; void *async_cb_context; - u8 *gb; + u8 gb[NFC_MAX_GT_LEN]; size_t gb_len; unsigned long quirks; diff --git a/include/net/nfc/nfc.h b/include/net/nfc/nfc.h index c54df042db6b..bcafab5c53e5 100644 --- a/include/net/nfc/nfc.h +++ b/include/net/nfc/nfc.h @@ -273,7 +273,8 @@ struct sk_buff *nfc_alloc_recv_skb(unsigned int size, gfp_t gfp); int nfc_set_remote_general_bytes(struct nfc_dev *dev, const u8 *gt, u8 gt_len); -u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, size_t *gb_len); +u8 *nfc_get_local_general_bytes(struct nfc_dev *dev, u8 *out_gb, + size_t gb_max_len, size_t *gb_len); int nfc_fw_download_done(struct nfc_dev *dev, const char *firmware_name, u32 result); diff --git a/include/net/nsh.h b/include/net/nsh.h index 16a751093896..15a26c590815 100644 --- a/include/net/nsh.h +++ b/include/net/nsh.h @@ -247,10 +247,10 @@ struct nshhdr { #define NSH_M_TYPE1_LEN 24 /* NSH header maximum Length. */ -#define NSH_HDR_MAX_LEN 256 +#define NSH_HDR_MAX_LEN ((NSH_LEN_MASK >> NSH_LEN_SHIFT) * 4) /* NSH context headers maximum Length. */ -#define NSH_CTX_HDRS_MAX_LEN 248 +#define NSH_CTX_HDRS_MAX_LEN (NSH_HDR_MAX_LEN - NSH_BASE_HDR_LEN) static inline struct nshhdr *nsh_hdr(struct sk_buff *skb) { diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h index 3247026e096a..e2730dd273b2 100644 --- a/include/net/page_pool/helpers.h +++ b/include/net/page_pool/helpers.h @@ -64,7 +64,7 @@ int page_pool_ethtool_stats_get_count(void); u8 *page_pool_ethtool_stats_get_strings(u8 *data); u64 *page_pool_ethtool_stats_get(u64 *data, const void *stats); -bool page_pool_get_stats(const struct page_pool *pool, +void page_pool_get_stats(const struct page_pool *pool, struct page_pool_stats *stats); #else static inline int page_pool_ethtool_stats_get_count(void) diff --git a/include/net/page_pool/memory_provider.h b/include/net/page_pool/memory_provider.h index ada4f968960a..255ce4cfd975 100644 --- a/include/net/page_pool/memory_provider.h +++ b/include/net/page_pool/memory_provider.h @@ -23,14 +23,10 @@ bool net_mp_niov_set_dma_addr(struct net_iov *niov, dma_addr_t addr); void net_mp_niov_set_page_pool(struct page_pool *pool, struct net_iov *niov); void net_mp_niov_clear_page_pool(struct net_iov *niov); -int net_mp_open_rxq(struct net_device *dev, unsigned ifq_idx, - struct pp_memory_provider_params *p); -int __net_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx, +int netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx, const struct pp_memory_provider_params *p, struct netlink_ext_ack *extack); -void net_mp_close_rxq(struct net_device *dev, unsigned ifq_idx, - struct pp_memory_provider_params *old_p); -void __net_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx, +void netif_mp_close_rxq(struct net_device *dev, unsigned int rxq_idx, const struct pp_memory_provider_params *old_p); /** diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index 1509a536cb85..03da138722f5 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -44,6 +44,8 @@ * use-case. The NAPI budget is 64 packets. After a NAPI poll the RX * ring is usually refilled and the max consumed elements will be 64, * thus a natural max size of objects needed in the cache. + * The refill watermark is set to 64 for 4KB pages, + * and scales to balance its size in bytes across page sizes. * * Keeping room for more objects, is due to XDP_DROP use-case. As * XDP_DROP allows the opportunity to recycle objects directly into @@ -51,8 +53,15 @@ * cache is already full (or partly full) then the XDP_DROP recycles * would have to take a slower code path. */ -#define PP_ALLOC_CACHE_SIZE 128 +#if PAGE_SIZE >= SZ_64K +#define PP_ALLOC_CACHE_REFILL 4 +#elif PAGE_SIZE >= SZ_16K +#define PP_ALLOC_CACHE_REFILL 16 +#else #define PP_ALLOC_CACHE_REFILL 64 +#endif + +#define PP_ALLOC_CACHE_SIZE (PP_ALLOC_CACHE_REFILL * 2) struct pp_alloc_cache { u32 count; netmem_ref cache[PP_ALLOC_CACHE_SIZE]; @@ -161,6 +170,7 @@ struct memory_provider_ops; struct pp_memory_provider_params { void *mp_priv; const struct memory_provider_ops *mp_ops; + u32 rx_page_size; }; struct page_pool { @@ -246,7 +256,7 @@ struct page_pool { /* User-facing fields, protected by page_pools_lock */ struct { struct hlist_node list; - u64 detach_time; + ktime_t detach_time; u32 id; } user; }; diff --git a/include/net/phy/realtek_phy.h b/include/net/phy/realtek_phy.h new file mode 100644 index 000000000000..d683bc1b0659 --- /dev/null +++ b/include/net/phy/realtek_phy.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _REALTEK_PHY_H +#define _REALTEK_PHY_H + +#define PHY_ID_RTL_DUMMY_SFP 0x001ccbff + +#endif /* _REALTEK_PHY_H */ diff --git a/include/net/pie.h b/include/net/pie.h index 01cbc66825a4..1f3db0c35514 100644 --- a/include/net/pie.h +++ b/include/net/pie.h @@ -104,7 +104,7 @@ static inline void pie_vars_init(struct pie_vars *vars) vars->dq_tstamp = DTIME_INVALID; vars->accu_prob = 0; vars->dq_count = DQCOUNT_INVALID; - vars->avg_dq_rate = 0; + WRITE_ONCE(vars->avg_dq_rate, 0); } static inline struct pie_skb_cb *get_pie_cb(const struct sk_buff *skb) diff --git a/include/net/ping.h b/include/net/ping.h index 05bfd594a64c..bcbdb5a136e3 100644 --- a/include/net/ping.h +++ b/include/net/ping.h @@ -20,8 +20,7 @@ /* Compatibility glue so we can support IPv6 when it's compiled as a module */ struct pingv6_ops { - int (*ipv6_recv_error)(struct sock *sk, struct msghdr *msg, int len, - int *addr_len); + int (*ipv6_recv_error)(struct sock *sk, struct msghdr *msg, int len); void (*ip6_datagram_recv_common_ctl)(struct sock *sk, struct msghdr *msg, struct sk_buff *skb); @@ -64,7 +63,7 @@ int ping_getfrag(void *from, char *to, int offset, int fraglen, int odd, struct sk_buff *); int ping_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, - int flags, int *addr_len); + int flags); int ping_common_sendmsg(int family, struct msghdr *msg, size_t len, void *user_icmph, size_t icmph_len); int ping_queue_rcv_skb(struct sock *sk, struct sk_buff *skb); diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h index 99ac747b7906..5f5cb36439fe 100644 --- a/include/net/pkt_cls.h +++ b/include/net/pkt_cls.h @@ -156,8 +156,20 @@ static inline int tcf_classify(struct sk_buff *skb, { return TC_ACT_UNSPEC; } - #endif +static inline int tcf_classify_qdisc(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res, bool compat_mode) +{ + int ret = tcf_classify(skb, NULL, tp, res, compat_mode); + + /* TC_ACT_REDIRECT from qdisc filter chains is not supported. + * Use BPF via tcx or mirred redirect instead. + */ + if (unlikely(ret == TC_ACT_REDIRECT)) + ret = TC_ACT_SHOT; + return ret; +} static inline unsigned long __cls_set_class(unsigned long *clp, unsigned long cl) @@ -1046,6 +1058,7 @@ struct tc_tbf_qopt_offload_replace_params { }; struct tc_tbf_qopt_offload { + struct netlink_ext_ack *extack; enum tc_tbf_command command; u32 handle; u32 parent; diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index e703c507d0da..90d3e7943b19 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -12,6 +12,7 @@ #define DEFAULT_TX_QUEUE_LEN 1000 #define STAB_SIZE_LOG_MAX 30 +#define QDISC_PKT_LEN_MAX (1 << 20) /* 1 MiB */ struct qdisc_walker { int stop; @@ -308,4 +309,28 @@ static inline unsigned int qdisc_peek_len(struct Qdisc *sch) return len; } +static inline void qdisc_lock_init(struct Qdisc *sch, + const struct Qdisc_ops *ops) +{ + spin_lock_init(&sch->q.lock); + + /* Skip dynamic keys if nesting is not possible */ + if (ops->static_flags & TCQ_F_INGRESS || + ops == &noqueue_qdisc_ops) + return; + + lockdep_register_key(&sch->root_lock_key); + lockdep_set_class(&sch->q.lock, &sch->root_lock_key); +} + +static inline void qdisc_lock_uninit(struct Qdisc *sch, + const struct Qdisc_ops *ops) +{ + if (ops->static_flags & TCQ_F_INGRESS || + ops == &noqueue_qdisc_ops) + return; + + lockdep_unregister_key(&sch->root_lock_key); +} + #endif diff --git a/include/net/psp/types.h b/include/net/psp/types.h index 25a9096d4e7d..87991a1ea02d 100644 --- a/include/net/psp/types.h +++ b/include/net/psp/types.h @@ -5,6 +5,7 @@ #include <linux/mutex.h> #include <linux/refcount.h> +#include <net/net_trackers.h> struct netlink_ext_ack; @@ -43,9 +44,29 @@ struct psp_dev_config { u32 versions; }; +/* Max number of devices that can be associated with a single PSP device. + * Each entry consumes ~24 bytes in the netlink dev-get response, and the + * response must fit in GENLMSG_DEFAULT_SIZE (~3.7KB). + */ +#define PSP_ASSOC_DEV_MAX 128 + +/** + * struct psp_assoc_dev - wrapper for associated net_device + * @dev_list: list node for psp_dev::assoc_dev_list + * @assoc_dev: the associated net_device + * @dev_tracker: tracker for the net_device reference + */ +struct psp_assoc_dev { + struct list_head dev_list; + struct net_device *assoc_dev; + netdevice_tracker dev_tracker; +}; + /** * struct psp_dev - PSP device struct * @main_netdev: original netdevice of this PSP device + * @assoc_dev_list: list of psp_assoc_dev entries associated with this PSP device + * @assoc_dev_cnt: number of entries in @assoc_dev_list * @ops: driver callbacks * @caps: device capabilities * @drv_priv: driver priv pointer @@ -67,6 +88,8 @@ struct psp_dev_config { */ struct psp_dev { struct net_device *main_netdev; + struct list_head assoc_dev_list; + int assoc_dev_cnt; struct psp_dev_ops *ops; struct psp_dev_caps *caps; diff --git a/include/net/request_sock.h b/include/net/request_sock.h index 9b9e04f6bb89..5a9c826a7092 100644 --- a/include/net/request_sock.h +++ b/include/net/request_sock.h @@ -123,14 +123,7 @@ static inline struct sock *skb_steal_sock(struct sk_buff *skb, return sk; } -static inline void __reqsk_free(struct request_sock *req) -{ - req->rsk_ops->destructor(req); - if (req->rsk_listener) - sock_put(req->rsk_listener); - kfree(req->saved_syn); - kmem_cache_free(req->rsk_ops->slab, req); -} +void __reqsk_free(struct request_sock *req); static inline void reqsk_free(struct request_sock *req) { @@ -196,8 +189,6 @@ struct request_sock_queue { */ }; -void reqsk_queue_alloc(struct request_sock_queue *queue); - void reqsk_fastopen_remove(struct sock *sk, struct request_sock *req, bool reset); diff --git a/include/net/rose.h b/include/net/rose.h index 2b5491bbf39a..41bfcb224f0b 100644 --- a/include/net/rose.h +++ b/include/net/rose.h @@ -1,266 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ -/* - * Declarations of Rose type objects. - * - * Jonathan Naylor G4KLX 25/8/96 - */ - #ifndef _ROSE_H -#define _ROSE_H - -#include <linux/refcount.h> -#include <linux/rose.h> -#include <net/ax25.h> -#include <net/sock.h> - -#define ROSE_ADDR_LEN 5 - -#define ROSE_MIN_LEN 3 - -#define ROSE_CALL_REQ_ADDR_LEN_OFF 3 -#define ROSE_CALL_REQ_ADDR_LEN_VAL 0xAA /* each address is 10 digits */ -#define ROSE_CALL_REQ_DEST_ADDR_OFF 4 -#define ROSE_CALL_REQ_SRC_ADDR_OFF 9 -#define ROSE_CALL_REQ_FACILITIES_OFF 14 - -#define ROSE_GFI 0x10 -#define ROSE_Q_BIT 0x80 -#define ROSE_D_BIT 0x40 -#define ROSE_M_BIT 0x10 - -#define ROSE_CALL_REQUEST 0x0B -#define ROSE_CALL_ACCEPTED 0x0F -#define ROSE_CLEAR_REQUEST 0x13 -#define ROSE_CLEAR_CONFIRMATION 0x17 -#define ROSE_DATA 0x00 -#define ROSE_INTERRUPT 0x23 -#define ROSE_INTERRUPT_CONFIRMATION 0x27 -#define ROSE_RR 0x01 -#define ROSE_RNR 0x05 -#define ROSE_REJ 0x09 -#define ROSE_RESET_REQUEST 0x1B -#define ROSE_RESET_CONFIRMATION 0x1F -#define ROSE_REGISTRATION_REQUEST 0xF3 -#define ROSE_REGISTRATION_CONFIRMATION 0xF7 -#define ROSE_RESTART_REQUEST 0xFB -#define ROSE_RESTART_CONFIRMATION 0xFF -#define ROSE_DIAGNOSTIC 0xF1 -#define ROSE_ILLEGAL 0xFD - -/* Define Link State constants. */ - -enum { - ROSE_STATE_0, /* Ready */ - ROSE_STATE_1, /* Awaiting Call Accepted */ - ROSE_STATE_2, /* Awaiting Clear Confirmation */ - ROSE_STATE_3, /* Data Transfer */ - ROSE_STATE_4, /* Awaiting Reset Confirmation */ - ROSE_STATE_5 /* Deferred Call Acceptance */ -}; - -#define ROSE_DEFAULT_T0 180000 /* Default T10 T20 value */ -#define ROSE_DEFAULT_T1 200000 /* Default T11 T21 value */ -#define ROSE_DEFAULT_T2 180000 /* Default T12 T22 value */ -#define ROSE_DEFAULT_T3 180000 /* Default T13 T23 value */ -#define ROSE_DEFAULT_HB 5000 /* Default Holdback value */ -#define ROSE_DEFAULT_IDLE 0 /* No Activity Timeout - none */ -#define ROSE_DEFAULT_ROUTING 1 /* Default routing flag */ -#define ROSE_DEFAULT_FAIL_TIMEOUT 120000 /* Time until link considered usable */ -#define ROSE_DEFAULT_MAXVC 50 /* Maximum number of VCs per neighbour */ -#define ROSE_DEFAULT_WINDOW_SIZE 7 /* Default window size */ - -#define ROSE_MODULUS 8 -#define ROSE_MAX_PACKET_SIZE 251 /* Maximum packet size */ - -#define ROSE_COND_ACK_PENDING 0x01 -#define ROSE_COND_PEER_RX_BUSY 0x02 -#define ROSE_COND_OWN_RX_BUSY 0x04 - -#define FAC_NATIONAL 0x00 -#define FAC_CCITT 0x0F - -#define FAC_NATIONAL_RAND 0x7F -#define FAC_NATIONAL_FLAGS 0x3F -#define FAC_NATIONAL_DEST_DIGI 0xE9 -#define FAC_NATIONAL_SRC_DIGI 0xEB -#define FAC_NATIONAL_FAIL_CALL 0xED -#define FAC_NATIONAL_FAIL_ADD 0xEE -#define FAC_NATIONAL_DIGIS 0xEF - -#define FAC_CCITT_DEST_NSAP 0xC9 -#define FAC_CCITT_SRC_NSAP 0xCB - -struct rose_neigh { - struct rose_neigh *next; - ax25_address callsign; - ax25_digi *digipeat; - ax25_cb *ax25; - struct net_device *dev; - unsigned short count; - refcount_t use; - unsigned int number; - char restarted; - char dce_mode; - char loopback; - struct sk_buff_head queue; - struct timer_list t0timer; - struct timer_list ftimer; -}; - -struct rose_node { - struct rose_node *next; - rose_address address; - unsigned short mask; - unsigned char count; - char loopback; - struct rose_neigh *neighbour[3]; -}; - -struct rose_route { - struct rose_route *next; - unsigned int lci1, lci2; - rose_address src_addr, dest_addr; - ax25_address src_call, dest_call; - struct rose_neigh *neigh1, *neigh2; - unsigned int rand; -}; - -struct rose_sock { - struct sock sock; - rose_address source_addr, dest_addr; - ax25_address source_call, dest_call; - unsigned char source_ndigis, dest_ndigis; - ax25_address source_digis[ROSE_MAX_DIGIS]; - ax25_address dest_digis[ROSE_MAX_DIGIS]; - struct rose_neigh *neighbour; - struct net_device *device; - netdevice_tracker dev_tracker; - unsigned int lci, rand; - unsigned char state, condition, qbitincl, defer; - unsigned char cause, diagnostic; - unsigned short vs, vr, va, vl; - unsigned long t1, t2, t3, hb, idle; -#ifdef M_BIT - unsigned short fraglen; - struct sk_buff_head frag_queue; -#endif - struct sk_buff_head ack_queue; - struct rose_facilities_struct facilities; - struct timer_list timer; - struct timer_list idletimer; -}; - -#define rose_sk(sk) ((struct rose_sock *)(sk)) - -static inline void rose_neigh_hold(struct rose_neigh *rose_neigh) -{ - refcount_inc(&rose_neigh->use); -} - -static inline void rose_neigh_put(struct rose_neigh *rose_neigh) -{ - if (refcount_dec_and_test(&rose_neigh->use)) { - if (rose_neigh->ax25) - ax25_cb_put(rose_neigh->ax25); - kfree(rose_neigh->digipeat); - kfree(rose_neigh); - } -} - -/* af_rose.c */ -extern ax25_address rose_callsign; -extern int sysctl_rose_restart_request_timeout; -extern int sysctl_rose_call_request_timeout; -extern int sysctl_rose_reset_request_timeout; -extern int sysctl_rose_clear_request_timeout; -extern int sysctl_rose_no_activity_timeout; -extern int sysctl_rose_ack_hold_back_timeout; -extern int sysctl_rose_routing_control; -extern int sysctl_rose_link_fail_timeout; -extern int sysctl_rose_maximum_vcs; -extern int sysctl_rose_window_size; - -int rosecmp(const rose_address *, const rose_address *); -int rosecmpm(const rose_address *, const rose_address *, unsigned short); -char *rose2asc(char *buf, const rose_address *); -struct sock *rose_find_socket(unsigned int, struct rose_neigh *); -void rose_kill_by_neigh(struct rose_neigh *); -unsigned int rose_new_lci(struct rose_neigh *); -int rose_rx_call_request(struct sk_buff *, struct net_device *, - struct rose_neigh *, unsigned int); -void rose_destroy_socket(struct sock *); - -/* rose_dev.c */ -void rose_setup(struct net_device *); - -/* rose_in.c */ -int rose_process_rx_frame(struct sock *, struct sk_buff *); - -/* rose_link.c */ -void rose_start_ftimer(struct rose_neigh *); -void rose_stop_ftimer(struct rose_neigh *); -void rose_stop_t0timer(struct rose_neigh *); -int rose_ftimer_running(struct rose_neigh *); -void rose_link_rx_restart(struct sk_buff *, struct rose_neigh *, - unsigned short); -void rose_transmit_clear_request(struct rose_neigh *, unsigned int, - unsigned char, unsigned char); -void rose_transmit_link(struct sk_buff *, struct rose_neigh *); - -/* rose_loopback.c */ -void rose_loopback_init(void); -void rose_loopback_clear(void); -int rose_loopback_queue(struct sk_buff *, struct rose_neigh *); - -/* rose_out.c */ -void rose_kick(struct sock *); -void rose_enquiry_response(struct sock *); - -/* rose_route.c */ -extern struct rose_neigh *rose_loopback_neigh; -extern const struct seq_operations rose_neigh_seqops; -extern const struct seq_operations rose_node_seqops; -extern struct seq_operations rose_route_seqops; - -void rose_add_loopback_neigh(void); -int __must_check rose_add_loopback_node(const rose_address *); -void rose_del_loopback_node(const rose_address *); -void rose_rt_device_down(struct net_device *); -void rose_link_device_down(struct net_device *); -struct net_device *rose_dev_first(void); -struct net_device *rose_dev_get(rose_address *); -struct rose_route *rose_route_free_lci(unsigned int, struct rose_neigh *); -struct rose_neigh *rose_get_neigh(rose_address *, unsigned char *, - unsigned char *, int); -int rose_rt_ioctl(unsigned int, void __user *); -void rose_link_failed(ax25_cb *, int); -int rose_route_frame(struct sk_buff *, ax25_cb *); -void rose_rt_free(void); - -/* rose_subr.c */ -void rose_clear_queues(struct sock *); -void rose_frames_acked(struct sock *, unsigned short); -void rose_requeue_frames(struct sock *); -int rose_validate_nr(struct sock *, unsigned short); -void rose_write_internal(struct sock *, int); -int rose_decode(struct sk_buff *, int *, int *, int *, int *, int *); -int rose_parse_facilities(unsigned char *, unsigned int, - struct rose_facilities_struct *); -void rose_disconnect(struct sock *, int, int, int); - -/* rose_timer.c */ -void rose_start_heartbeat(struct sock *); -void rose_start_t1timer(struct sock *); -void rose_start_t2timer(struct sock *); -void rose_start_t3timer(struct sock *); -void rose_start_hbtimer(struct sock *); -void rose_start_idletimer(struct sock *); -void rose_stop_heartbeat(struct sock *); -void rose_stop_timer(struct sock *); -void rose_stop_idletimer(struct sock *); +#define _ROSE_H -/* sysctl_net_rose.c */ -void rose_register_sysctl(void); -void rose_unregister_sysctl(void); +#define ROSE_ADDR_LEN 5 #endif diff --git a/include/net/route.h b/include/net/route.h index f90106f383c5..45290177a33c 100644 --- a/include/net/route.h +++ b/include/net/route.h @@ -276,6 +276,8 @@ int fib_dump_info_fnhe(struct sk_buff *skb, struct netlink_callback *cb, u32 table_id, struct fib_info *fi, int *fa_index, int fa_start, unsigned int flags); +void fnhe_update_pmtu(struct fib_nh_exception *fnhe, u32 new, u32 orig); + static inline void ip_rt_put(struct rtable *rt) { /* dst_release() accepts a NULL parameter. diff --git a/include/net/rps-types.h b/include/net/rps-types.h new file mode 100644 index 000000000000..6b90a66866c1 --- /dev/null +++ b/include/net/rps-types.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _NET_RPS_TYPES_H +#define _NET_RPS_TYPES_H + +/* Define a rps_tag_ptr: + * Low order 5 bits are used to store the ilog2(size) of an RPS table. + */ +typedef unsigned long rps_tag_ptr; + +static inline u8 rps_tag_to_log(rps_tag_ptr tag_ptr) +{ + return tag_ptr & 31U; +} + +static inline u32 rps_tag_to_mask(rps_tag_ptr tag_ptr) +{ + return (1U << rps_tag_to_log(tag_ptr)) - 1; +} + +static inline void *rps_tag_to_table(rps_tag_ptr tag_ptr) +{ + return (void *)(tag_ptr & ~31UL); +} +#endif /* _NET_RPS_TYPES_H */ diff --git a/include/net/rps.h b/include/net/rps.h index f1794cd2e7fb..e33c6a2fa8bb 100644 --- a/include/net/rps.h +++ b/include/net/rps.h @@ -8,6 +8,7 @@ #include <net/hotdata.h> #ifdef CONFIG_RPS +#include <net/rps-types.h> extern struct static_key_false rps_needed; extern struct static_key_false rfs_needed; @@ -39,17 +40,6 @@ struct rps_dev_flow { #define RPS_NO_FILTER 0xffff /* - * The rps_dev_flow_table structure contains a table of flow mappings. - */ -struct rps_dev_flow_table { - u8 log; - struct rcu_head rcu; - struct rps_dev_flow flows[]; -}; -#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \ - ((_num) * sizeof(struct rps_dev_flow))) - -/* * The rps_sock_flow_table contains mappings of flows to the last CPU * on which they were processed by the application (set in recvmsg). * Each entry is a 32bit value. Upper part is the high-order bits @@ -60,41 +50,38 @@ struct rps_dev_flow_table { * meaning we use 32-6=26 bits for the hash. */ struct rps_sock_flow_table { - struct rcu_head rcu; - u32 mask; - - u32 ents[] ____cacheline_aligned_in_smp; + u32 ent; }; -#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (offsetof(struct rps_sock_flow_table, ents[_num])) #define RPS_NO_CPU 0xffff -static inline void rps_record_sock_flow(struct rps_sock_flow_table *table, - u32 hash) +static inline void rps_record_sock_flow(rps_tag_ptr tag_ptr, u32 hash) { - unsigned int index = hash & table->mask; + unsigned int index = hash & rps_tag_to_mask(tag_ptr); u32 val = hash & ~net_hotdata.rps_cpu_mask; + struct rps_sock_flow_table *table; /* We only give a hint, preemption can change CPU under us */ val |= raw_smp_processor_id(); + table = rps_tag_to_table(tag_ptr); /* The following WRITE_ONCE() is paired with the READ_ONCE() * here, and another one in get_rps_cpu(). */ - if (READ_ONCE(table->ents[index]) != val) - WRITE_ONCE(table->ents[index], val); + if (READ_ONCE(table[index].ent) != val) + WRITE_ONCE(table[index].ent, val); } static inline void _sock_rps_record_flow_hash(__u32 hash) { - struct rps_sock_flow_table *sock_flow_table; + rps_tag_ptr tag_ptr; if (!hash) return; rcu_read_lock(); - sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table); - if (sock_flow_table) - rps_record_sock_flow(sock_flow_table, hash); + tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table); + if (tag_ptr) + rps_record_sock_flow(tag_ptr, hash); rcu_read_unlock(); } @@ -121,6 +108,7 @@ static inline void _sock_rps_record_flow(const struct sock *sk) static inline void _sock_rps_delete_flow(const struct sock *sk) { struct rps_sock_flow_table *table; + rps_tag_ptr tag_ptr; u32 hash, index; hash = READ_ONCE(sk->sk_rxhash); @@ -128,11 +116,12 @@ static inline void _sock_rps_delete_flow(const struct sock *sk) return; rcu_read_lock(); - table = rcu_dereference(net_hotdata.rps_sock_flow_table); - if (table) { - index = hash & table->mask; - if (READ_ONCE(table->ents[index]) != RPS_NO_CPU) - WRITE_ONCE(table->ents[index], RPS_NO_CPU); + tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table); + if (tag_ptr) { + index = hash & rps_tag_to_mask(tag_ptr); + table = rps_tag_to_table(tag_ptr); + if (READ_ONCE(table[index].ent) != RPS_NO_CPU) + WRITE_ONCE(table[index].ent, RPS_NO_CPU); } rcu_read_unlock(); } diff --git a/include/net/rtnetlink.h b/include/net/rtnetlink.h index ec65a8cebb99..2bff41aacc98 100644 --- a/include/net/rtnetlink.h +++ b/include/net/rtnetlink.h @@ -256,6 +256,8 @@ int rtnl_configure_link(struct net_device *dev, const struct ifinfomsg *ifm, int rtnl_nla_parse_ifinfomsg(struct nlattr **tb, const struct nlattr *nla_peer, struct netlink_ext_ack *exterr); struct net *rtnl_get_net_ns_capable(struct sock *sk, int netnsid); +bool rtnl_dev_link_net_capable(const struct net_device *dev, + const struct net *link_net); #define MODULE_ALIAS_RTNL_LINK(kind) MODULE_ALIAS("rtnl-link-" kind) diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index c3a7268b567e..f35bd06a6bad 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -20,12 +20,15 @@ #include <net/rtnetlink.h> #include <net/flow_offload.h> #include <linux/xarray.h> +#include <net/dropreason-qdisc.h> struct Qdisc_ops; struct qdisc_walker; struct tcf_walker; struct module; struct bpf_flow_keys; +struct Qdisc; +struct netdev_queue; struct qdisc_rate_table { struct tc_ratespec rate; @@ -83,7 +86,7 @@ struct Qdisc { #define TCQ_F_WARN_NONWC (1 << 16) #define TCQ_F_CPUSTATS 0x20 /* run using percpu statistics */ #define TCQ_F_NOPARENT 0x40 /* root of its hierarchy : - * qdisc_tree_decrease_qlen() should stop. + * qdisc_tree_reduce_backlog() should stop. */ #define TCQ_F_INVISIBLE 0x80 /* invisible by default in dump */ #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */ @@ -96,6 +99,7 @@ struct Qdisc { struct hlist_node hash; u32 handle; u32 parent; + int depth; struct netdev_queue *dev_queue; @@ -539,6 +543,21 @@ static inline int qdisc_qlen(const struct Qdisc *q) return q->q.qlen; } +static inline int qdisc_qlen_lockless(const struct Qdisc *q) +{ + return READ_ONCE(q->q.qlen); +} + +static inline void qdisc_qlen_inc(struct Qdisc *q) +{ + WRITE_ONCE(q->q.qlen, q->q.qlen + 1); +} + +static inline void qdisc_qlen_dec(struct Qdisc *q) +{ + WRITE_ONCE(q->q.qlen, q->q.qlen - 1); +} + static inline int qdisc_qlen_sum(const struct Qdisc *q) { __u32 qlen = q->qstats.qlen; @@ -546,9 +565,9 @@ static inline int qdisc_qlen_sum(const struct Qdisc *q) if (qdisc_is_percpu_stats(q)) { for_each_possible_cpu(i) - qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen; + qlen += READ_ONCE(per_cpu_ptr(q->cpu_qstats, i)->qlen); } else { - qlen += q->q.qlen; + qlen += qdisc_qlen_lockless(q); } return qlen; @@ -707,8 +726,8 @@ void dev_qdisc_change_real_num_tx(struct net_device *dev, void dev_init_scheduler(struct net_device *dev); void dev_shutdown(struct net_device *dev); void dev_activate(struct net_device *dev); -void dev_deactivate(struct net_device *dev); -void dev_deactivate_many(struct list_head *head); +void dev_deactivate(struct net_device *dev, bool reset_needed); +void dev_deactivate_many(struct list_head *head, bool reset_needed); struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue, struct Qdisc *qdisc); void qdisc_reset(struct Qdisc *qdisc); @@ -716,6 +735,34 @@ void qdisc_destroy(struct Qdisc *qdisc); void qdisc_put(struct Qdisc *qdisc); void qdisc_put_unlocked(struct Qdisc *qdisc); void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len); + +static inline void dev_reset_queue(struct net_device *dev, + struct netdev_queue *dev_queue, + void *_unused) +{ + struct Qdisc *qdisc; + bool nolock; + + qdisc = rtnl_dereference(dev_queue->qdisc_sleeping); + if (!qdisc) + return; + + nolock = qdisc->flags & TCQ_F_NOLOCK; + + if (nolock) + spin_lock_bh(&qdisc->seqlock); + spin_lock_bh(qdisc_lock(qdisc)); + + qdisc_reset(qdisc); + + spin_unlock_bh(qdisc_lock(qdisc)); + if (nolock) { + clear_bit(__QDISC_STATE_MISSED, &qdisc->state); + clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); + spin_unlock_bh(&qdisc->seqlock); + } +} + #ifdef CONFIG_NET_SCHED int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, void *type_data); @@ -746,7 +793,7 @@ void qdisc_offload_query_caps(struct net_device *dev, struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, struct netlink_ext_ack *extack); -void qdisc_free(struct Qdisc *qdisc); +void qdisc_free_rcu(struct Qdisc *qdisc); struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, u32 parentid, struct netlink_ext_ack *extack); @@ -778,13 +825,23 @@ static inline bool skb_skip_tc_classify(struct sk_buff *skb) static inline void qdisc_reset_all_tx_gt(struct net_device *dev, unsigned int i) { struct Qdisc *qdisc; + bool nolock; for (; i < dev->num_tx_queues; i++) { qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); if (qdisc) { + nolock = qdisc->flags & TCQ_F_NOLOCK; + + if (nolock) + spin_lock_bh(&qdisc->seqlock); spin_lock_bh(qdisc_lock(qdisc)); qdisc_reset(qdisc); spin_unlock_bh(qdisc_lock(qdisc)); + if (nolock) { + clear_bit(__QDISC_STATE_MISSED, &qdisc->state); + clear_bit(__QDISC_STATE_DRAINING, &qdisc->state); + spin_unlock_bh(&qdisc->seqlock); + } } } } @@ -896,6 +953,15 @@ static inline void _bstats_update(struct gnet_stats_basic_sync *bstats, u64_stats_update_end(&bstats->syncp); } +static inline void _bstats_set(struct gnet_stats_basic_sync *bstats, + u64 bytes, u64 packets) +{ + u64_stats_update_begin(&bstats->syncp); + u64_stats_set(&bstats->bytes, bytes); + u64_stats_set(&bstats->packets, packets); + u64_stats_update_end(&bstats->syncp); +} + static inline void bstats_update(struct gnet_stats_basic_sync *bstats, const struct sk_buff *skb) { @@ -914,10 +980,15 @@ static inline void qdisc_bstats_update(struct Qdisc *sch, bstats_update(&sch->bstats, skb); } +static inline void qstats_backlog_sub(struct Qdisc *sch, u32 val) +{ + WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog - val); +} + static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, const struct sk_buff *skb) { - sch->qstats.backlog -= qdisc_pkt_len(skb); + qstats_backlog_sub(sch, qdisc_pkt_len(skb)); } static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, @@ -926,10 +997,15 @@ static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb)); } +static inline void qstats_backlog_add(struct Qdisc *sch, u32 val) +{ + WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog + val); +} + static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, const struct sk_buff *skb) { - sch->qstats.backlog += qdisc_pkt_len(skb); + qstats_backlog_add(sch, qdisc_pkt_len(skb)); } static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, @@ -955,17 +1031,22 @@ static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) { - sch->qstats.drops += count; + WRITE_ONCE(sch->qstats.drops, sch->qstats.drops + count); } static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) { - qstats->drops++; + WRITE_ONCE(qstats->drops, qstats->drops + 1); +} + +static inline void qstats_cpu_drop_inc(struct gnet_stats_queue __percpu *qstats) +{ + this_cpu_inc(qstats->drops); } -static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) +static inline void qstats_cpu_overlimit_inc(struct gnet_stats_queue __percpu *qstats) { - qstats->overlimits++; + this_cpu_inc(qstats->overlimits); } static inline void qdisc_qstats_drop(struct Qdisc *sch) @@ -980,23 +1061,23 @@ static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) static inline void qdisc_qstats_overlimit(struct Qdisc *sch) { - sch->qstats.overlimits++; + WRITE_ONCE(sch->qstats.overlimits, sch->qstats.overlimits + 1); } -static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch) +static inline int qdisc_qstats_copy(struct gnet_dump *d, const struct Qdisc *sch) { __u32 qlen = qdisc_qlen_sum(sch); return gnet_stats_copy_queue(d, sch->cpu_qstats, &sch->qstats, qlen); } -static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch, __u32 *qlen, - __u32 *backlog) +static inline void qdisc_qstats_qlen_backlog(const struct Qdisc *sch, + u32 *qlen, u32 *backlog) { struct gnet_stats_queue qstats = { 0 }; gnet_stats_add_queue(&qstats, sch->cpu_qstats, &sch->qstats); - *qlen = qstats.qlen + qdisc_qlen(sch); + *qlen = qstats.qlen + qdisc_qlen_lockless(sch); *backlog = qstats.backlog; } @@ -1022,7 +1103,7 @@ static inline void __qdisc_enqueue_tail(struct sk_buff *skb, qh->tail = skb; qh->head = skb; } - qh->qlen++; + WRITE_ONCE(qh->qlen, qh->qlen + 1); } static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) @@ -1040,7 +1121,7 @@ static inline void __qdisc_enqueue_head(struct sk_buff *skb, if (!qh->head) qh->tail = skb; qh->head = skb; - qh->qlen++; + WRITE_ONCE(qh->qlen, qh->qlen + 1); } static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) @@ -1049,7 +1130,7 @@ static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh) if (likely(skb != NULL)) { qh->head = skb->next; - qh->qlen--; + WRITE_ONCE(qh->qlen, qh->qlen - 1); if (qh->head == NULL) qh->tail = NULL; skb->next = NULL; @@ -1064,7 +1145,7 @@ static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool dir skb = __skb_dequeue(&sch->gso_skb); if (skb) { - sch->q.qlen--; + qdisc_qlen_dec(sch); qdisc_qstats_backlog_dec(sch, skb); return skb; } @@ -1106,38 +1187,62 @@ static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb) return cb; } +/* TC classifier accessors - use enum skb_drop_reason */ static inline enum skb_drop_reason tcf_get_drop_reason(const struct sk_buff *skb) { - return tc_skb_cb(skb)->drop_reason; + return (enum skb_drop_reason)tc_skb_cb(skb)->drop_reason; } static inline void tcf_set_drop_reason(const struct sk_buff *skb, enum skb_drop_reason reason) { - tc_skb_cb(skb)->drop_reason = reason; + tc_skb_cb(skb)->drop_reason = (enum qdisc_drop_reason)reason; } -static inline void tcf_kfree_skb_list(struct sk_buff *skb) +/* Qdisc accessors - use enum qdisc_drop_reason */ +static inline enum qdisc_drop_reason +tcf_get_qdisc_drop_reason(const struct sk_buff *skb) { - while (unlikely(skb)) { - struct sk_buff *next = skb->next; + return tc_skb_cb(skb)->drop_reason; +} - prefetch(next); - kfree_skb_reason(skb, tcf_get_drop_reason(skb)); - skb = next; - } +static inline void tcf_set_qdisc_drop_reason(const struct sk_buff *skb, + enum qdisc_drop_reason reason) +{ + tc_skb_cb(skb)->drop_reason = reason; +} + +void __tcf_kfree_skb_list(struct sk_buff *skb, struct Qdisc *q, + struct netdev_queue *txq, struct net_device *dev); + +static inline void tcf_kfree_skb_list(struct sk_buff *skb, struct Qdisc *q, + struct netdev_queue *txq, + struct net_device *dev) +{ + if (unlikely(skb)) + __tcf_kfree_skb_list(skb, q, txq, dev); } static inline void qdisc_dequeue_drop(struct Qdisc *q, struct sk_buff *skb, - enum skb_drop_reason reason) + enum qdisc_drop_reason reason) { + struct Qdisc *root; + DEBUG_NET_WARN_ON_ONCE(!(q->flags & TCQ_F_DEQUEUE_DROPS)); DEBUG_NET_WARN_ON_ONCE(q->flags & TCQ_F_NOLOCK); - tcf_set_drop_reason(skb, reason); - skb->next = q->to_free; - q->to_free = skb; + rcu_read_lock(); + root = qdisc_root_sleeping(q); + + if (root->flags & TCQ_F_DEQUEUE_DROPS) { + tcf_set_qdisc_drop_reason(skb, reason); + skb->next = root->to_free; + root->to_free = skb; + } else { + kfree_skb_reason(skb, (enum skb_drop_reason)reason); + } + rcu_read_unlock(); } /* Instead of calling kfree_skb() while root qdisc lock is held, @@ -1196,7 +1301,7 @@ static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) __skb_queue_head(&sch->gso_skb, skb); /* it's still part of the queue */ qdisc_qstats_backlog_inc(sch, skb); - sch->q.qlen++; + qdisc_qlen_inc(sch); } } @@ -1213,7 +1318,7 @@ static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch, } else { qdisc_qstats_backlog_dec(sch, skb); qdisc_bstats_update(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } } @@ -1224,8 +1329,8 @@ static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch, qdisc_qstats_cpu_qlen_inc(sch); this_cpu_add(sch->cpu_qstats->backlog, pkt_len); } else { - sch->qstats.backlog += pkt_len; - sch->q.qlen++; + qstats_backlog_add(sch, pkt_len); + qdisc_qlen_inc(sch); } } @@ -1241,7 +1346,7 @@ static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) qdisc_qstats_cpu_qlen_dec(sch); } else { qdisc_qstats_backlog_dec(sch, skb); - sch->q.qlen--; + qdisc_qlen_dec(sch); } } else { skb = sch->dequeue(sch); @@ -1262,7 +1367,7 @@ static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh) qh->head = NULL; qh->tail = NULL; - qh->qlen = 0; + WRITE_ONCE(qh->qlen, 0); } } @@ -1312,9 +1417,9 @@ static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, static inline int qdisc_drop_reason(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free, - enum skb_drop_reason reason) + enum qdisc_drop_reason reason) { - tcf_set_drop_reason(skb, reason); + tcf_set_qdisc_drop_reason(skb, reason); return qdisc_drop(skb, sch, to_free); } @@ -1419,6 +1524,11 @@ void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc, void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp, struct tcf_block *block); +static inline bool mini_qdisc_pair_inited(struct mini_Qdisc_pair *miniqp) +{ + return !!miniqp->p_miniq; +} + void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx); int sch_frag_xmit_hook(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)); diff --git a/include/net/sch_priv.h b/include/net/sch_priv.h new file mode 100644 index 000000000000..4789f668ae87 --- /dev/null +++ b/include/net/sch_priv.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NET_SCHED_PRIV_H +#define __NET_SCHED_PRIV_H + +#include <net/sch_generic.h> + +struct mq_sched { + struct Qdisc **qdiscs; +}; + +int mq_init_common(struct Qdisc *sch, struct nlattr *opt, + struct netlink_ext_ack *extack, + const struct Qdisc_ops *qdisc_ops); +void mq_destroy_common(struct Qdisc *sch); +void mq_attach(struct Qdisc *sch); +void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb); +struct netdev_queue *mq_select_queue(struct Qdisc *sch, + struct tcmsg *tcm); +struct Qdisc *mq_leaf(struct Qdisc *sch, unsigned long cl); +unsigned long mq_find(struct Qdisc *sch, u32 classid); +int mq_dump_class(struct Qdisc *sch, unsigned long cl, + struct sk_buff *skb, struct tcmsg *tcm); +int mq_dump_class_stats(struct Qdisc *sch, unsigned long cl, + struct gnet_dump *d); +void mq_walk(struct Qdisc *sch, struct qdisc_walker *arg); + +#endif diff --git a/include/net/scm.h b/include/net/scm.h index c52519669349..86ae6bc109ec 100644 --- a/include/net/scm.h +++ b/include/net/scm.h @@ -50,8 +50,8 @@ struct scm_cookie { #endif }; -void scm_detach_fds(struct msghdr *msg, struct scm_cookie *scm); -void scm_detach_fds_compat(struct msghdr *msg, struct scm_cookie *scm); +void scm_detach_fds(struct msghdr *msg, struct scm_cookie *scm, bool notrunc); +void scm_detach_fds_compat(struct msghdr *msg, struct scm_cookie *scm, bool notrunc); int __scm_send(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm); void __scm_destroy(struct scm_cookie *scm); struct scm_fp_list *scm_fp_dup(struct scm_fp_list *fpl); @@ -107,13 +107,8 @@ void scm_recv(struct socket *sock, struct msghdr *msg, void scm_recv_unix(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm, int flags); -static inline int scm_recv_one_fd(struct file *f, int __user *ufd, - unsigned int flags) -{ - if (!ufd) - return -EFAULT; - return receive_fd(f, ufd, flags); -} +int scm_recv_one_fd(struct file *f, int __user *ufd, unsigned int flags, + bool notrunc); #endif /* __LINUX_NET_SCM_H */ diff --git a/include/net/sctp/auth.h b/include/net/sctp/auth.h index 6f2cd562b1de..85efa65d3912 100644 --- a/include/net/sctp/auth.h +++ b/include/net/sctp/auth.h @@ -22,6 +22,7 @@ struct sctp_endpoint; struct sctp_association; struct sctp_authkey; struct sctp_hmacalgo; +struct sctp_cookie; /* Defines an HMAC algorithm supported by SCTP chunk authentication */ struct sctp_hmac { @@ -72,6 +73,8 @@ struct sctp_shared_key *sctp_auth_get_shkey( int sctp_auth_asoc_copy_shkeys(const struct sctp_endpoint *ep, struct sctp_association *asoc, gfp_t gfp); +bool sctp_auth_verify_cookie_params(const struct sctp_endpoint *ep, + const struct sctp_cookie *cookie); const struct sctp_hmac *sctp_auth_get_hmac(__u16 hmac_id); const struct sctp_hmac * sctp_auth_asoc_get_hmac(const struct sctp_association *asoc); @@ -83,9 +86,9 @@ int sctp_auth_send_cid(enum sctp_cid chunk, const struct sctp_association *asoc); int sctp_auth_recv_cid(enum sctp_cid chunk, const struct sctp_association *asoc); -void sctp_auth_calculate_hmac(const struct sctp_association *asoc, - struct sk_buff *skb, struct sctp_auth_chunk *auth, - struct sctp_shared_key *ep_key, gfp_t gfp); +int sctp_auth_calculate_hmac(const struct sctp_association *asoc, + struct sk_buff *skb, struct sctp_auth_chunk *auth, + struct sctp_shared_key *ep_key, gfp_t gfp); void sctp_auth_shkey_release(struct sctp_shared_key *sh_key); void sctp_auth_shkey_hold(struct sctp_shared_key *sh_key); diff --git a/include/net/sctp/sctp.h b/include/net/sctp/sctp.h index 58242b37b47a..d50c27812504 100644 --- a/include/net/sctp/sctp.h +++ b/include/net/sctp/sctp.h @@ -111,7 +111,8 @@ int sctp_transport_lookup_process(sctp_callback_t cb, struct net *net, const union sctp_addr *paddr, void *p, int dif); int sctp_transport_traverse_process(sctp_callback_t cb, sctp_callback_t cb_done, struct net *net, int *pos, void *p); -int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), void *p); +int sctp_for_each_endpoint(int (*cb)(struct sctp_endpoint *, void *), + struct net *net, int *pos, void *p); int sctp_get_sctp_info(struct sock *sk, struct sctp_association *asoc, struct sctp_info *info); @@ -303,7 +304,7 @@ void sctp_dbg_objcnt_init(struct net *); static inline void sctp_dbg_objcnt_init(struct net *net) { return; } -#endif /* CONFIG_SCTP_DBG_OBJCOUNT */ +#endif /* CONFIG_SCTP_DBG_OBJCNT */ #if defined CONFIG_SYSCTL void sctp_sysctl_register(void); diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index affee44bd38e..b21f23b736fd 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -312,7 +312,8 @@ struct sctp_cookie { __u8 auth_random[sizeof(struct sctp_paramhdr) + SCTP_AUTH_RANDOM_LENGTH]; - __u8 auth_hmacs[SCTP_AUTH_NUM_HMACS * sizeof(__u16) + 2]; + __u8 auth_hmacs[sizeof(struct sctp_paramhdr) + + SCTP_AUTH_NUM_HMACS * sizeof(__u16)]; __u8 auth_chunks[sizeof(struct sctp_paramhdr) + SCTP_AUTH_MAX_CHUNKS]; /* This is a shim for my peer's INIT packet, followed by @@ -2056,7 +2057,7 @@ struct sctp_association { force_delay:1; __u8 strreset_enable; - __u8 strreset_outstanding; /* request param count on the fly */ + __u8 strreset_outstanding; /* request param bitmask on the fly */ __u32 strreset_outseq; /* Update after receiving response */ __u32 strreset_inseq; /* Update after receiving request */ diff --git a/include/net/secure_seq.h b/include/net/secure_seq.h index cddebafb9f77..6f996229167b 100644 --- a/include/net/secure_seq.h +++ b/include/net/secure_seq.h @@ -5,16 +5,47 @@ #include <linux/types.h> struct net; +extern struct net init_net; + +union tcp_seq_and_ts_off { + struct { + u32 seq; + u32 ts_off; + }; + u64 hash64; +}; u64 secure_ipv4_port_ephemeral(__be32 saddr, __be32 daddr, __be16 dport); u64 secure_ipv6_port_ephemeral(const __be32 *saddr, const __be32 *daddr, __be16 dport); -u32 secure_tcp_seq(__be32 saddr, __be32 daddr, - __be16 sport, __be16 dport); -u32 secure_tcp_ts_off(const struct net *net, __be32 saddr, __be32 daddr); -u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, - __be16 sport, __be16 dport); -u32 secure_tcpv6_ts_off(const struct net *net, - const __be32 *saddr, const __be32 *daddr); +union tcp_seq_and_ts_off +secure_tcp_seq_and_ts_off(const struct net *net, __be32 saddr, __be32 daddr, + __be16 sport, __be16 dport); + +static inline u32 secure_tcp_seq(__be32 saddr, __be32 daddr, + __be16 sport, __be16 dport) +{ + union tcp_seq_and_ts_off ts; + + ts = secure_tcp_seq_and_ts_off(&init_net, saddr, daddr, + sport, dport); + + return ts.seq; +} + +union tcp_seq_and_ts_off +secure_tcpv6_seq_and_ts_off(const struct net *net, const __be32 *saddr, + const __be32 *daddr, + __be16 sport, __be16 dport); + +static inline u32 secure_tcpv6_seq(const __be32 *saddr, const __be32 *daddr, + __be16 sport, __be16 dport) +{ + union tcp_seq_and_ts_off ts; + + ts = secure_tcpv6_seq_and_ts_off(&init_net, saddr, daddr, + sport, dport); + return ts.seq; +} #endif /* _NET_SECURE_SEQ */ diff --git a/include/net/smc.h b/include/net/smc.h index bfdc4c41f019..a2bc3ab88075 100644 --- a/include/net/smc.h +++ b/include/net/smc.h @@ -40,7 +40,6 @@ struct smcd_dev { struct dibs_dev *dibs; struct list_head list; spinlock_t lock; - struct smc_connection **conn; struct list_head vlan; struct workqueue_struct *event_wq; u8 pnetid[SMC_MAX_PNETID_LEN]; @@ -50,6 +49,7 @@ struct smcd_dev { atomic_t lgr_cnt; wait_queue_head_t lgrs_deleted; u8 going_away : 1; + struct smc_connection *conn[]; }; #define SMC_HS_CTRL_NAME_MAX 16 diff --git a/include/net/sock.h b/include/net/sock.h index aafe8bdb2c0f..60ea55dc1885 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -81,8 +81,13 @@ * mini-semaphore synchronizes multiple users amongst themselves. */ typedef struct { - spinlock_t slock; - int owned; + union { + struct slock_owned { + int owned; + spinlock_t slock; + }; + long combined; + }; wait_queue_head_t wq; /* * We express the mutex-alike socket_lock semantics @@ -121,14 +126,14 @@ typedef __u64 __bitwise __addrpair; * @skc_bypass_prot_mem: bypass the per-protocol memory accounting for skb * @skc_bound_dev_if: bound device index if != 0 * @skc_bind_node: bind hash linkage for various protocol lookup tables - * @skc_portaddr_node: second hash linkage for UDP/UDP-Lite protocol + * @skc_portaddr_node: second hash linkage for UDP * @skc_prot: protocol handlers inside a network family * @skc_net: reference to the network namespace of this socket * @skc_v6_daddr: IPV6 destination address * @skc_v6_rcv_saddr: IPV6 source address * @skc_cookie: socket's cookie value * @skc_node: main hash linkage for various protocol lookup tables - * @skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol + * @skc_nulls_node: main hash linkage for TCP * @skc_tx_queue_mapping: tx queue number for this connection * @skc_rx_queue_mapping: rx queue number for this connection * @skc_flags: place holder for sk_flags @@ -341,6 +346,7 @@ struct sk_filter; * @sk_reuseport_cb: reuseport group container * @sk_bpf_storage: ptr to cache and control for bpf_sk_storage * @sk_rcu: used during RCU grace period + * @sk_freeptr: used for SLAB_TYPESAFE_BY_RCU managed sockets * @sk_clockid: clockid used by time-based scheduling (SO_TXTIME) * @sk_txtime_deadline_mode: set deadline mode for SO_TXTIME * @sk_txtime_report_errors: set report errors mode for SO_TXTIME @@ -536,7 +542,7 @@ struct sock { rwlock_t sk_callback_lock; u32 sk_ack_backlog; u32 sk_max_ack_backlog; - unsigned long sk_ino; + u64 sk_ino; spinlock_t sk_peer_lock; int sk_bind_phc; struct pid *sk_peer_pid; @@ -582,7 +588,14 @@ struct sock { struct bpf_local_storage __rcu *sk_bpf_storage; #endif struct numa_drop_counters *sk_drop_counters; - struct rcu_head sk_rcu; + /* sockets using SLAB_TYPESAFE_BY_RCU can use sk_freeptr. + * By the time kfree() is called, sk_rcu can not be in + * use and can be mangled. + */ + union { + struct rcu_head sk_rcu; + freeptr_t sk_freeptr; + }; netns_tracker ns_tracker; struct xarray sk_user_frags; @@ -1308,7 +1321,7 @@ struct proto { int (*sendmsg)(struct sock *sk, struct msghdr *msg, size_t len); int (*recvmsg)(struct sock *sk, struct msghdr *msg, - size_t len, int flags, int *addr_len); + size_t len, int flags); void (*splice_eof)(struct socket *sock); int (*bind)(struct sock *sk, struct sockaddr_unsized *addr, int addr_len); @@ -1368,6 +1381,7 @@ struct proto { struct kmem_cache *slab; unsigned int obj_size; + unsigned int freeptr_offset; unsigned int ipv6_pinfo_offset; slab_flags_t slab_flags; unsigned int useroffset; /* Usercopy region offset */ @@ -1378,7 +1392,6 @@ struct proto { union { struct inet_hashinfo *hashinfo; - struct udp_table *udp_table; struct raw_hashinfo *raw_hash; struct smc_hashinfo *smc_hash; } h; @@ -1700,7 +1713,6 @@ static inline void lock_sock(struct sock *sk) lock_sock_nested(sk, 0); } -void __lock_sock(struct sock *sk); void __release_sock(struct sock *sk); void release_sock(struct sock *sk); @@ -1838,12 +1850,22 @@ static inline struct sock *sk_clone_lock(const struct sock *sk, const gfp_t prio struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force, gfp_t priority); -void __sock_wfree(struct sk_buff *skb); void sock_wfree(struct sk_buff *skb); +void __sock_wfree(struct sk_buff *skb); +void tcp_wfree(struct sk_buff *skb); + +static inline bool is_skb_wmem(const struct sk_buff *skb) +{ + return skb->destructor == sock_wfree || + (IS_ENABLED(CONFIG_INET) && skb->destructor == __sock_wfree) || + (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree); +} + struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size, gfp_t priority); void skb_orphan_partial(struct sk_buff *skb); void sock_rfree(struct sk_buff *skb); +void sock_rmem_free(struct sk_buff *skb); void sock_efree(struct sk_buff *skb); #ifdef CONFIG_INET void sock_edemux(struct sk_buff *skb); @@ -2089,7 +2111,7 @@ static inline int sk_rx_queue_get(const struct sock *sk) static inline void sk_set_socket(struct sock *sk, struct socket *sock) { - sk->sk_socket = sock; + WRITE_ONCE(sk->sk_socket, sock); if (sock) { WRITE_ONCE(sk->sk_uid, SOCK_INODE(sock)->i_uid); WRITE_ONCE(sk->sk_ino, SOCK_INODE(sock)->i_ino); @@ -2131,7 +2153,7 @@ static inline void sock_graft(struct sock *sk, struct socket *parent) write_unlock_bh(&sk->sk_callback_lock); } -static inline unsigned long sock_i_ino(const struct sock *sk) +static inline u64 sock_i_ino(const struct sock *sk) { /* Paired with WRITE_ONCE() in sock_graft() and sock_orphan() */ return READ_ONCE(sk->sk_ino); @@ -2240,6 +2262,20 @@ sk_dst_reset(struct sock *sk) sk_dst_set(sk, NULL); } +/* Re-roll the socket txhash. On a rehash, IPv6 also drops the cached route + * so the next transmit re-selects an ECMP path; IPv4 keeps its route, since + * IPv4 ECMP path selection does not use sk_txhash. + */ +static inline bool __sk_rethink_txhash_reset_dst(struct sock *sk) +{ + if (sk_rethink_txhash(sk)) { + if (sk->sk_family == AF_INET6) + __sk_dst_reset(sk); + return true; + } + return false; +} + struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie); struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie); @@ -2276,6 +2312,8 @@ static inline void sk_gso_disable(struct sock *sk) sk->sk_route_caps &= ~NETIF_F_GSO_MASK; } +bool sk_has_decrypt_user(const struct sock *sk); + static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb, struct iov_iter *from, char *to, int copy, int offset) @@ -2490,12 +2528,23 @@ int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue, struct sk_buff *skb)); int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb); -int sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb, - enum skb_drop_reason *reason); +enum skb_drop_reason +sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb); static inline int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) { - return sock_queue_rcv_skb_reason(sk, skb, NULL); + enum skb_drop_reason drop_reason = sock_queue_rcv_skb_reason(sk, skb); + + switch (drop_reason) { + case SKB_DROP_REASON_SOCKET_RCVBUFF: + return -ENOMEM; + case SKB_DROP_REASON_PROTO_MEM: + return -ENOBUFS; + case 0: + return 0; + default: + return -EPERM; + } } int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb); diff --git a/include/net/switchdev.h b/include/net/switchdev.h index 8346b0d29542..ee500706496b 100644 --- a/include/net/switchdev.h +++ b/include/net/switchdev.h @@ -15,6 +15,7 @@ #define SWITCHDEV_F_NO_RECURSE BIT(0) #define SWITCHDEV_F_SKIP_EOPNOTSUPP BIT(1) #define SWITCHDEV_F_DEFER BIT(2) +#define SWITCHDEV_F_NO_FOREIGN BIT(3) enum switchdev_attr_id { SWITCHDEV_ATTR_ID_UNDEFINED, diff --git a/include/net/tc_act/tc_gate.h b/include/net/tc_act/tc_gate.h index c1a67149c6b6..e0fded18e18c 100644 --- a/include/net/tc_act/tc_gate.h +++ b/include/net/tc_act/tc_gate.h @@ -32,6 +32,7 @@ struct tcf_gate_params { s32 tcfg_clockid; size_t num_entries; struct list_head entries; + struct rcu_head rcu; }; #define GATE_ACT_GATE_OPEN BIT(0) @@ -39,7 +40,7 @@ struct tcf_gate_params { struct tcf_gate { struct tc_action common; - struct tcf_gate_params param; + struct tcf_gate_params __rcu *param; u8 current_gate_status; ktime_t current_close_time; u32 current_entry_octets; @@ -51,47 +52,65 @@ struct tcf_gate { #define to_gate(a) ((struct tcf_gate *)a) +static inline struct tcf_gate_params *tcf_gate_params_locked(const struct tc_action *a) +{ + struct tcf_gate *gact = to_gate(a); + + return rcu_dereference_protected(gact->param, + lockdep_is_held(&gact->tcf_lock)); +} + static inline s32 tcf_gate_prio(const struct tc_action *a) { + struct tcf_gate_params *p; s32 tcfg_prio; - tcfg_prio = to_gate(a)->param.tcfg_priority; + p = tcf_gate_params_locked(a); + tcfg_prio = p->tcfg_priority; return tcfg_prio; } static inline u64 tcf_gate_basetime(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_basetime; - tcfg_basetime = to_gate(a)->param.tcfg_basetime; + p = tcf_gate_params_locked(a); + tcfg_basetime = p->tcfg_basetime; return tcfg_basetime; } static inline u64 tcf_gate_cycletime(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_cycletime; - tcfg_cycletime = to_gate(a)->param.tcfg_cycletime; + p = tcf_gate_params_locked(a); + tcfg_cycletime = p->tcfg_cycletime; return tcfg_cycletime; } static inline u64 tcf_gate_cycletimeext(const struct tc_action *a) { + struct tcf_gate_params *p; u64 tcfg_cycletimeext; - tcfg_cycletimeext = to_gate(a)->param.tcfg_cycletime_ext; + p = tcf_gate_params_locked(a); + tcfg_cycletimeext = p->tcfg_cycletime_ext; return tcfg_cycletimeext; } static inline u32 tcf_gate_num_entries(const struct tc_action *a) { + struct tcf_gate_params *p; u32 num_entries; - num_entries = to_gate(a)->param.num_entries; + p = tcf_gate_params_locked(a); + num_entries = p->num_entries; return num_entries; } @@ -105,7 +124,7 @@ static inline struct action_gate_entry u32 num_entries; int i = 0; - p = &to_gate(a)->param; + p = tcf_gate_params_locked(a); num_entries = p->num_entries; list_for_each_entry(entry, &p->entries, list) @@ -114,7 +133,7 @@ static inline struct action_gate_entry if (i != num_entries) return NULL; - oe = kcalloc(num_entries, sizeof(*oe), GFP_ATOMIC); + oe = kzalloc_objs(*oe, num_entries, GFP_ATOMIC); if (!oe) return NULL; diff --git a/include/net/tc_act/tc_ife.h b/include/net/tc_act/tc_ife.h index c7f24a2da1ca..24d4d5a62b3c 100644 --- a/include/net/tc_act/tc_ife.h +++ b/include/net/tc_act/tc_ife.h @@ -13,15 +13,13 @@ struct tcf_ife_params { u8 eth_src[ETH_ALEN]; u16 eth_type; u16 flags; - + struct list_head metalist; struct rcu_head rcu; }; struct tcf_ife_info { struct tc_action common; struct tcf_ife_params __rcu *params; - /* list of metaids allowed */ - struct list_head metalist; }; #define to_ife(a) ((struct tcf_ife_info *)a) diff --git a/include/net/tc_act/tc_pedit.h b/include/net/tc_act/tc_pedit.h index f58ee15cd858..97754ea0a827 100644 --- a/include/net/tc_act/tc_pedit.h +++ b/include/net/tc_act/tc_pedit.h @@ -15,7 +15,6 @@ struct tcf_pedit_parms { struct tc_pedit_key *tcfp_keys; struct tcf_pedit_key_ex *tcfp_keys_ex; int action; - u32 tcfp_off_max_hint; unsigned char tcfp_nkeys; unsigned char tcfp_flags; struct rcu_head rcu; @@ -38,17 +37,15 @@ static inline bool is_tcf_pedit(const struct tc_action *a) return false; } -static inline int tcf_pedit_nkeys(const struct tc_action *a) +/* Must be called with act->tcfa_lock held to ensure consistency of parallel + * reads of the same action's pedit keys (e.g. flow_offload count vs fill). + * Note, this is only used for pedit offload. + */ +static inline int tcf_pedit_nkeys_locked(const struct tc_action *a) { - struct tcf_pedit_parms *parms; - int nkeys; - - rcu_read_lock(); - parms = to_pedit_parms(a); - nkeys = parms->tcfp_nkeys; - rcu_read_unlock(); - - return nkeys; + lockdep_assert_held(&a->tcfa_lock); + return rcu_dereference_protected(to_pedit(a)->parms, + lockdep_is_held(&a->tcfa_lock))->tcfp_nkeys; } static inline u32 tcf_pedit_htype(const struct tc_action *a, int index) diff --git a/include/net/tc_wrapper.h b/include/net/tc_wrapper.h index ffe58a02537c..4ebb053bb0dd 100644 --- a/include/net/tc_wrapper.h +++ b/include/net/tc_wrapper.h @@ -12,7 +12,8 @@ #define TC_INDIRECT_SCOPE -extern struct static_key_false tc_skip_wrapper; +extern struct static_key_false tc_skip_wrapper_act; +extern struct static_key_false tc_skip_wrapper_cls; /* TC Actions */ #ifdef CONFIG_NET_CLS_ACT @@ -46,7 +47,7 @@ TC_INDIRECT_ACTION_DECLARE(tunnel_key_act); static inline int tc_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) { - if (static_branch_likely(&tc_skip_wrapper)) + if (static_branch_likely(&tc_skip_wrapper_act)) goto skip; #if IS_BUILTIN(CONFIG_NET_ACT_GACT) @@ -153,7 +154,7 @@ TC_INDIRECT_FILTER_DECLARE(u32_classify); static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp, struct tcf_result *res) { - if (static_branch_likely(&tc_skip_wrapper)) + if (static_branch_likely(&tc_skip_wrapper_cls)) goto skip; #if IS_BUILTIN(CONFIG_NET_CLS_BPF) @@ -202,8 +203,44 @@ skip: static inline void tc_wrapper_init(void) { #ifdef CONFIG_X86 - if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE)) - static_branch_enable(&tc_skip_wrapper); + int cnt_cls = IS_BUILTIN(CONFIG_NET_CLS_BPF) + + IS_BUILTIN(CONFIG_NET_CLS_U32) + + IS_BUILTIN(CONFIG_NET_CLS_FLOWER) + + IS_BUILTIN(CONFIG_NET_CLS_FW) + + IS_BUILTIN(CONFIG_NET_CLS_MATCHALL) + + IS_BUILTIN(CONFIG_NET_CLS_BASIC) + + IS_BUILTIN(CONFIG_NET_CLS_CGROUP) + + IS_BUILTIN(CONFIG_NET_CLS_FLOW) + + IS_BUILTIN(CONFIG_NET_CLS_ROUTE4); + + int cnt_act = IS_BUILTIN(CONFIG_NET_ACT_GACT) + + IS_BUILTIN(CONFIG_NET_ACT_MIRRED) + + IS_BUILTIN(CONFIG_NET_ACT_PEDIT) + + IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT) + + IS_BUILTIN(CONFIG_NET_ACT_SKBMOD) + + IS_BUILTIN(CONFIG_NET_ACT_POLICE) + + IS_BUILTIN(CONFIG_NET_ACT_BPF) + + IS_BUILTIN(CONFIG_NET_ACT_CONNMARK) + + IS_BUILTIN(CONFIG_NET_ACT_CSUM) + + IS_BUILTIN(CONFIG_NET_ACT_CT) + + IS_BUILTIN(CONFIG_NET_ACT_CTINFO) + + IS_BUILTIN(CONFIG_NET_ACT_GATE) + + IS_BUILTIN(CONFIG_NET_ACT_MPLS) + + IS_BUILTIN(CONFIG_NET_ACT_NAT) + + IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY) + + IS_BUILTIN(CONFIG_NET_ACT_VLAN) + + IS_BUILTIN(CONFIG_NET_ACT_IFE) + + IS_BUILTIN(CONFIG_NET_ACT_SIMP) + + IS_BUILTIN(CONFIG_NET_ACT_SAMPLE); + + if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) + return; + + if (cnt_cls > 1) + static_branch_enable(&tc_skip_wrapper_cls); + + if (cnt_act > 1) + static_branch_enable(&tc_skip_wrapper_act); #endif } diff --git a/include/net/tcp.h b/include/net/tcp.h index 0deb5e9dd911..4416cdf9bf30 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -43,6 +43,7 @@ #include <net/dst.h> #include <net/mptcp.h> #include <net/xfrm.h> +#include <net/secure_seq.h> #include <linux/seq_file.h> #include <linux/memcontrol.h> @@ -64,8 +65,6 @@ static inline void tcp_orphan_count_dec(void) this_cpu_dec(tcp_orphan_count); } -DECLARE_PER_CPU(u32, tcp_tw_isn); - void tcp_time_wait(struct sock *sk, int state, int timeo); #define MAX_TCP_HEADER L1_CACHE_ALIGN(128 + MAX_HEADER) @@ -347,13 +346,21 @@ extern struct proto tcp_prot; #define TCP_DEC_STATS(net, field) SNMP_DEC_STATS((net)->mib.tcp_statistics, field) #define TCP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val) +/* + * TCP splice context + */ +struct tcp_splice_state { + struct pipe_inode_info *pipe; + size_t len; + unsigned int flags; +}; + void tcp_tsq_work_init(void); int tcp_v4_err(struct sk_buff *skb, u32); void tcp_shutdown(struct sock *sk, int how); -int tcp_v4_early_demux(struct sk_buff *skb); int tcp_v4_rcv(struct sk_buff *skb); void tcp_remove_empty_skb(struct sock *sk); @@ -366,8 +373,21 @@ int tcp_send_mss(struct sock *sk, int *size_goal, int flags); int tcp_wmem_schedule(struct sock *sk, int copy); void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle, int size_goal); + void tcp_release_cb(struct sock *sk); -void tcp_wfree(struct sk_buff *skb); + +static inline bool tcp_release_cb_cond(struct sock *sk) +{ +#ifdef CONFIG_INET + if (likely(sk->sk_prot->release_cb == tcp_release_cb)) { + if (unlikely(smp_load_acquire(&sk->sk_tsq_flags) & TCP_DEFERRED_ALL)) + tcp_release_cb(sk); + return true; + } +#endif + return false; +} + void tcp_write_timer_handler(struct sock *sk); void tcp_delack_timer_handler(struct sock *sk); int tcp_ioctl(struct sock *sk, int cmd, int *karg); @@ -378,6 +398,8 @@ void tcp_rcv_space_adjust(struct sock *sk); int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp); void tcp_twsk_destructor(struct sock *sk); void tcp_twsk_purge(struct list_head *net_exit_list); +int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, + unsigned int offset, size_t len); ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags); @@ -489,11 +511,19 @@ void tcp_reset_keepalive_timer(struct sock *sk, unsigned long timeout); void tcp_set_keepalive(struct sock *sk, int val); void tcp_syn_ack_timeout(const struct request_sock *req); int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, - int flags, int *addr_len); + int flags); int tcp_set_rcvlowat(struct sock *sk, int val); +void tcp_set_rcvbuf(struct sock *sk, int val); int tcp_set_window_clamp(struct sock *sk, int val); -void tcp_update_recv_tstamps(struct sk_buff *skb, - struct scm_timestamping_internal *tss); + +static inline void +tcp_update_recv_tstamps(struct sk_buff *skb, + struct scm_timestamping_internal *tss) +{ + tss->ts[0] = skb->tstamp; + tss->ts[2] = skb_hwtstamps(skb)->hwtstamp; +} + void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk, struct scm_timestamping_internal *tss); void tcp_data_ready(struct sock *sk); @@ -520,7 +550,6 @@ u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops, * TCP v4 functions exported for the inet6 API */ -void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb); void tcp_v4_mtu_reduced(struct sock *sk); void tcp_req_err(struct sock *sk, u32 seq, bool abort); void tcp_ld_RTO_revert(struct sock *sk, u32 seq); @@ -533,7 +562,9 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, struct request_sock *req, struct dst_entry *dst, struct request_sock *req_unhash, - bool *own_req); + bool *own_req, + void (*opt_child_init)(struct sock *newsk, + const struct sock *sk)); int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb); int tcp_v4_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len); int tcp_connect(struct sock *sk); @@ -541,6 +572,7 @@ enum tcp_synack_type { TCP_SYNACK_NORMAL, TCP_SYNACK_FASTOPEN, TCP_SYNACK_COOKIE, + TCP_SYNACK_RETRANS, }; struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, struct request_sock *req, @@ -733,8 +765,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, void tcp_send_probe0(struct sock *); int tcp_write_wakeup(struct sock *, int mib); void tcp_send_fin(struct sock *sk); -void tcp_send_active_reset(struct sock *sk, gfp_t priority, - enum sk_rst_reason reason); +void tcp_send_active_reset(struct sock *sk, enum sk_rst_reason reason); int tcp_send_synack(struct sock *); void tcp_push_one(struct sock *, unsigned int mss_now); void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags); @@ -751,7 +782,15 @@ void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req); void tcp_done_with_error(struct sock *sk, int err); void tcp_reset(struct sock *sk, struct sk_buff *skb); void tcp_fin(struct sock *sk); -void tcp_check_space(struct sock *sk); +void __tcp_check_space(struct sock *sk); +static inline void tcp_check_space(struct sock *sk) +{ + /* pairs with tcp_poll() */ + smp_mb(); + + if (sk->sk_socket && test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) + __tcp_check_space(sk); +} void tcp_sack_compress_send_ack(struct sock *sk); static inline void tcp_cleanup_skb(struct sk_buff *skb) @@ -809,6 +848,7 @@ static inline int tcp_bound_to_half_wnd(struct tcp_sock *tp, int pktsize) /* tcp.c */ void tcp_get_info(struct sock *, struct tcp_info *); +void tcp_rate_check_app_limited(struct sock *sk); /* Read 'sendfile()'-style from a TCP socket */ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, @@ -891,6 +931,28 @@ static inline u32 tcp_receive_window(const struct tcp_sock *tp) return (u32) win; } +/* Compute the maximum receive window we ever advertised. + * Rcv_nxt can be after the window if our peer push more data + * than the offered window. + */ +static inline u32 tcp_max_receive_window(const struct tcp_sock *tp) +{ + s32 win = tp->rcv_mwnd_seq - tp->rcv_nxt; + + if (win < 0) + win = 0; + return (u32) win; +} + +/* Check if we need to update the maximum receive window sequence number */ +static inline void tcp_update_max_rcv_wnd_seq(struct tcp_sock *tp) +{ + u32 wre = tp->rcv_wup + tp->rcv_wnd; + + if (after(wre, tp->rcv_mwnd_seq)) + tp->rcv_mwnd_seq = wre; +} + /* Choose a new window, without checks for shrinking, and without * scaling applied to the result. The caller does these things * if necessary. This is a "raw" window selection. @@ -949,6 +1011,16 @@ static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp) return tcp_time_stamp_ms(tp); } +/* Refresh clocks of a TCP socket, + * ensuring monotically increasing values. + */ +static inline void tcp_mstamp_refresh_inline(struct tcp_sock *tp) +{ + u64 val = tcp_clock_ns(); + + tp->tcp_clock_cache = val; + tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC); +} void tcp_mstamp_refresh(struct tcp_sock *tp); static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0) @@ -1036,10 +1108,13 @@ struct tcp_skb_cb { __u32 seq; /* Starting sequence number */ __u32 end_seq; /* SEQ + FIN + SYN + datalen */ union { - /* Note : + /* Notes : + * tcp_tw_isn is used in input path only + * (isn chosen by tcp_timewait_state_process()) * tcp_gso_segs/size are used in write queue only, * cf tcp_skb_pcount()/tcp_skb_mss() */ + u32 tcp_tw_isn; struct { u16 tcp_gso_segs; u16 tcp_gso_size; @@ -1111,9 +1186,7 @@ static inline int tcp_v6_sdif(const struct sk_buff *skb) extern const struct inet_connection_sock_af_ops ipv6_specific; -INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb)); INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *skb)); -void tcp_v6_early_demux(struct sk_buff *skb); #endif @@ -1159,9 +1232,9 @@ static inline bool tcp_skb_can_collapse_to(const struct sk_buff *skb) static inline bool tcp_skb_can_collapse(const struct sk_buff *to, const struct sk_buff *from) { - /* skb_cmp_decrypted() not needed, use tcp_write_collapse_fence() */ return likely(tcp_skb_can_collapse_to(to) && mptcp_skb_can_collapse(to, from) && + !skb_cmp_decrypted(to, from) && skb_pure_zcopy_same(to, from) && skb_frags_readable(to) == skb_frags_readable(from)); } @@ -1203,7 +1276,15 @@ enum tcp_ca_ack_event_flags { #define TCP_CONG_NON_RESTRICTED BIT(0) /* Requires ECN/ECT set on all packets */ #define TCP_CONG_NEEDS_ECN BIT(1) -#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN) +/* Require successfully negotiated AccECN capability */ +#define TCP_CONG_NEEDS_ACCECN BIT(2) +/* Use ECT(1) instead of ECT(0) while the CA is uninitialized */ +#define TCP_CONG_ECT_1_NEGOTIATION BIT(3) +/* Cannot fallback to RFC3168 during AccECN negotiation */ +#define TCP_CONG_NO_FALLBACK_RFC3168 BIT(4) +#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN | \ + TCP_CONG_NEEDS_ACCECN | TCP_CONG_ECT_1_NEGOTIATION | \ + TCP_CONG_NO_FALLBACK_RFC3168) union tcp_cc_info; @@ -1243,33 +1324,45 @@ struct rate_sample { struct tcp_congestion_ops { /* fast path fields are put first to fill one cache line */ + /* A congestion control (CC) must provide one of either: + * + * (a) a cong_avoid function, if the CC wants to use the core TCP + * stack's default functionality to implement a "classic" + * (Reno/CUBIC-style) response to packet loss, RFC3168 ECN, + * idle periods, pacing rate computations, etc. + * + * (b) a cong_control function, if the CC wants custom behavior and + * complete control of all congestion control behaviors. + */ + /* (a) "classic" response: calculate new cwnd. + */ + void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); + /* (b) "custom" response: call when packets are delivered to update + * cwnd and pacing rate, after all the ca_state processing. + */ + void (*cong_control)(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs); + /* return slow start threshold (required) */ u32 (*ssthresh)(struct sock *sk); - /* do new cwnd calculation (required) */ - void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked); - /* call before changing ca_state (optional) */ void (*set_state)(struct sock *sk, u8 new_state); /* call when cwnd event occurs (optional) */ void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev); + /* call when CA_EVENT_TX_START cwnd event occurs (optional) */ + void (*cwnd_event_tx_start)(struct sock *sk); + /* call when ack arrives (optional) */ void (*in_ack_event)(struct sock *sk, u32 flags); /* hook for packet ack accounting (optional) */ void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); - /* override sysctl_tcp_min_tso_segs */ + /* override sysctl_tcp_min_tso_segs (optional) */ u32 (*min_tso_segs)(struct sock *sk); - /* call when packets are delivered to update cwnd and pacing rate, - * after all the ca_state processing. (optional) - */ - void (*cong_control)(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs); - - /* new value of cwnd after loss (required) */ u32 (*undo_cwnd)(struct sock *sk); /* returns the multiplier used in tcp_sndbuf_expand (optional) */ @@ -1335,10 +1428,36 @@ static inline bool tcp_ca_needs_ecn(const struct sock *sk) return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN; } +static inline bool tcp_ca_needs_accecn(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ACCECN; +} + +static inline bool tcp_ca_ect_1_negotiation(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_ECT_1_NEGOTIATION; +} + +static inline bool tcp_ca_no_fallback_rfc3168(const struct sock *sk) +{ + const struct inet_connection_sock *icsk = inet_csk(sk); + + return icsk->icsk_ca_ops->flags & TCP_CONG_NO_FALLBACK_RFC3168; +} + static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) { const struct inet_connection_sock *icsk = inet_csk(sk); + if (event == CA_EVENT_TX_START) { + if (icsk->icsk_ca_ops->cwnd_event_tx_start) + icsk->icsk_ca_ops->cwnd_event_tx_start(sk); + return; + } if (icsk->icsk_ca_ops->cwnd_event) icsk->icsk_ca_ops->cwnd_event(sk, event); } @@ -1346,13 +1465,6 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) /* From tcp_cong.c */ void tcp_set_ca_state(struct sock *sk, const u8 ca_state); -/* From tcp_rate.c */ -void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb); -void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, - struct rate_sample *rs); -void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, - bool is_sack_reneg, struct rate_sample *rs); -void tcp_rate_check_app_limited(struct sock *sk); static inline bool tcp_skb_sent_after(u64 t1, u64 t2, u32 seq1, u32 seq2) { @@ -1410,7 +1522,7 @@ static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp) static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val) { WARN_ON_ONCE((int)val <= 0); - tp->snd_cwnd = val; + WRITE_ONCE(tp->snd_cwnd, val); } static inline bool tcp_in_slow_start(const struct tcp_sock *tp) @@ -1578,11 +1690,16 @@ static inline bool tcp_checksum_complete(struct sk_buff *skb) __skb_checksum_complete(skb); } -bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb, - enum skb_drop_reason *reason); +enum skb_drop_reason tcp_add_backlog(struct sock *sk, struct sk_buff *skb); + +static inline enum skb_drop_reason +tcp_filter(struct sock *sk, struct sk_buff *skb) +{ + const struct tcphdr *th = (const struct tcphdr *)skb->data; + return sk_filter_trim_cap(sk, skb, __tcp_hdrlen(th)); +} -int tcp_filter(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason *reason); void tcp_set_state(struct sock *sk, int state); void tcp_done(struct sock *sk); int tcp_abort(struct sock *sk, int err); @@ -1664,6 +1781,11 @@ static inline int tcp_full_space(const struct sock *sk) return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); } +static inline u32 tcp_dst_advmss(const struct dst_entry *dst) +{ + return max_t(u32, dst_metric_advmss(dst), TCP_MIN_MSS); +} + static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh) { int unused_mem = sk_unused_reserved_mem(sk); @@ -1856,6 +1978,8 @@ static inline void tcp_fast_path_check(struct sock *sk) bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, int mib_idx, u32 *last_oow_ack_time); +void tcp_reqsk_send_challenge_ack(struct sock *sk, struct sk_buff *skb, + struct request_sock *req); static inline void tcp_mib_init(struct net *net) { @@ -1909,40 +2033,6 @@ struct tcp6_pseudohdr { __be32 protocol; /* including padding */ }; -/* - * struct tcp_sigpool - per-CPU pool of ahash_requests - * @scratch: per-CPU temporary area, that can be used between - * tcp_sigpool_start() and tcp_sigpool_end() to perform - * crypto request - * @req: pre-allocated ahash request - */ -struct tcp_sigpool { - void *scratch; - struct ahash_request *req; -}; - -int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size); -void tcp_sigpool_get(unsigned int id); -void tcp_sigpool_release(unsigned int id); -int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp, - const struct sk_buff *skb, - unsigned int header_len); - -/** - * tcp_sigpool_start - disable bh and start using tcp_sigpool_ahash - * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash() - * @c: returned tcp_sigpool for usage (uninitialized on failure) - * - * Returns: 0 on success, error otherwise. - */ -int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c); -/** - * tcp_sigpool_end - enable bh and stop using tcp_sigpool - * @c: tcp_sigpool context that was returned by tcp_sigpool_start() - */ -void tcp_sigpool_end(struct tcp_sigpool *c); -size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len); -/* - functions */ void tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key, const struct sock *sk, const struct sk_buff *skb); int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, @@ -2095,7 +2185,34 @@ enum tcp_chrono { __TCP_CHRONO_MAX, }; -void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); +static inline void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new) +{ + const u32 now = tcp_jiffies32; + enum tcp_chrono old = tp->chrono_type; + + /* Following WRITE_ONCE()s pair with READ_ONCE()s in + * tcp_get_info_chrono_stats(). + */ + if (old > TCP_CHRONO_UNSPEC) + WRITE_ONCE(tp->chrono_stat[old - 1], + tp->chrono_stat[old - 1] + now - tp->chrono_start); + WRITE_ONCE(tp->chrono_start, now); + WRITE_ONCE(tp->chrono_type, new); +} + +static inline void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type) +{ + struct tcp_sock *tp = tcp_sk(sk); + + /* If there are multiple conditions worthy of tracking in a + * chronograph then the highest priority enum takes precedence + * over the other conditions. So that if something "more interesting" + * starts happening, stop the previous chrono and start a new one. + */ + if (type > tp->chrono_type) + tcp_chrono_set(tp, type); +} + void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); /* This helper is needed, because skb->tcp_tsorted_anchor uses @@ -2210,7 +2327,7 @@ static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct soc static inline void tcp_write_collapse_fence(struct sock *sk) { - struct sk_buff *skb = tcp_write_queue_tail(sk); + struct sk_buff *skb = tcp_write_queue_tail(sk) ?: tcp_rtx_queue_tail(sk); if (skb) TCP_SKB_CB(skb)->eor = 1; @@ -2318,15 +2435,21 @@ struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb, struct tcphdr *th); INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff)); INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)); -INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff)); -INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)); #ifdef CONFIG_INET void tcp_gro_complete(struct sk_buff *skb); #else static inline void tcp_gro_complete(struct sk_buff *skb) { } #endif -void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr); +static inline void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, + __be32 daddr) +{ + struct tcphdr *th = tcp_hdr(skb); + + th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0); + skb->csum_start = skb_transport_header(skb) - skb->head; + skb->csum_offset = offsetof(struct tcphdr, check); +} static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp) { @@ -2369,9 +2492,9 @@ struct tcp_sock_af_ops { struct tcp_ao_key *(*ao_lookup)(const struct sock *sk, struct sock *addr_sk, int sndid, int rcvid); - int (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key, - const struct sock *sk, - __be32 sisn, __be32 disn, bool send); + void (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key, + const struct sock *sk, + __be32 sisn, __be32 disn, bool send); int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao, const struct sock *sk, const struct sk_buff *skb, const u8 *tkey, int hash_offset, u32 sne); @@ -2392,7 +2515,7 @@ struct tcp_request_sock_ops { struct tcp_ao_key *(*ao_lookup)(const struct sock *sk, struct request_sock *req, int sndid, int rcvid); - int (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk); + void (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk); int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt, struct request_sock *req, const struct sk_buff *skb, int hash_offset, u32 sne); @@ -2406,8 +2529,9 @@ struct tcp_request_sock_ops { struct flowi *fl, struct request_sock *req, u32 tw_isn); - u32 (*init_seq)(const struct sk_buff *skb); - u32 (*init_ts_off)(const struct net *net, const struct sk_buff *skb); + union tcp_seq_and_ts_off (*init_seq_and_ts_off)( + const struct net *net, + const struct sk_buff *skb); int (*send_synack)(const struct sock *sk, struct dst_entry *dst, struct flowi *fl, struct request_sock *req, struct tcp_fastopen_cookie *foc, @@ -2422,22 +2546,30 @@ extern const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops; #ifdef CONFIG_SYN_COOKIES static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, - const struct sock *sk, struct sk_buff *skb, - __u16 *mss) + struct sk_buff *skb, __u16 *mss) { - tcp_synq_overflow(sk); - __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); return ops->cookie_init_seq(skb, mss); } #else static inline __u32 cookie_init_sequence(const struct tcp_request_sock_ops *ops, - const struct sock *sk, struct sk_buff *skb, - __u16 *mss) + struct sk_buff *skb, __u16 *mss) { return 0; } #endif +#ifdef CONFIG_SYN_COOKIES +static inline void cookie_record_sent(const struct sock *sk) +{ + tcp_synq_overflow(sk); + __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); +} +#else +static inline void cookie_record_sent(const struct sock *sk) +{ +} +#endif + struct tcp_key { union { struct { @@ -2513,10 +2645,7 @@ void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced); extern s32 tcp_rack_skb_timeout(struct tcp_sock *tp, struct sk_buff *skb, u32 reo_wnd); extern bool tcp_rack_mark_lost(struct sock *sk); -extern void tcp_rack_advance(struct tcp_sock *tp, u8 sacked, u32 end_seq, - u64 xmit_time); extern void tcp_rack_reo_timeout(struct sock *sk); -extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs); /* tcp_plb.c */ @@ -2802,6 +2931,11 @@ static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, return tcp_call_bpf(sk, op, 3, args); } +static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) +{ + tcp_sk(sk)->bpf_sock_ops_cb_flags = 0; +} + #else static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) { @@ -2819,6 +2953,10 @@ static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, return -EPERM; } +static inline void tcp_clear_sock_ops_cb_flags(struct sock *sk) +{ +} + #endif static inline u32 tcp_timeout_init(struct sock *sk) @@ -2942,4 +3080,18 @@ enum skb_drop_reason tcp_inbound_hash(struct sock *sk, const void *saddr, const void *daddr, int family, int dif, int sdif); +static inline int tcp_recv_should_stop(struct sock *sk) +{ + return sk->sk_err || + sk->sk_state == TCP_CLOSE || + (sk->sk_shutdown & RCV_SHUTDOWN) || + signal_pending(current); +} + +INDIRECT_CALLABLE_DECLARE(union tcp_seq_and_ts_off + tcp_v4_init_seq_and_ts_off(const struct net *net, + const struct sk_buff *skb)); +INDIRECT_CALLABLE_DECLARE(union tcp_seq_and_ts_off + tcp_v6_init_seq_and_ts_off(const struct net *net, + const struct sk_buff *skb)); #endif /* _TCP_H */ diff --git a/include/net/tcp_ao.h b/include/net/tcp_ao.h index 1e9e27d6e06b..9a2333e62e99 100644 --- a/include/net/tcp_ao.h +++ b/include/net/tcp_ao.h @@ -2,8 +2,7 @@ #ifndef _TCP_AO_H #define _TCP_AO_H -#define TCP_AO_KEY_ALIGN 1 -#define __tcp_ao_key_align __aligned(TCP_AO_KEY_ALIGN) +#include <crypto/sha2.h> /* for SHA256_DIGEST_SIZE */ union tcp_ao_addr { struct in_addr a4; @@ -32,11 +31,27 @@ struct tcp_ao_counters { atomic64_t dropped_icmp; }; +enum tcp_ao_algo_id { + TCP_AO_ALGO_HMAC_SHA1 = 1, /* specified by RFC 5926 */ + TCP_AO_ALGO_HMAC_SHA256, /* Linux extension */ + TCP_AO_ALGO_AES_128_CMAC, /* specified by RFC 5926 */ +}; + +/* + * This is the maximum untruncated MAC length, in bytes. Note that the MACs + * actually get truncated to 20 or fewer bytes to fit in the TCP options space. + */ +#define TCP_AO_MAX_MAC_LEN SHA256_DIGEST_SIZE + +#define TCP_AO_MAX_TRAFFIC_KEY_LEN SHA256_DIGEST_SIZE + +struct tcp_ao_mac_ctx; + struct tcp_ao_key { struct hlist_node node; union tcp_ao_addr addr; - u8 key[TCP_AO_MAXKEYLEN] __tcp_ao_key_align; - unsigned int tcp_sigpool_id; + u8 key[TCP_AO_MAXKEYLEN]; + enum tcp_ao_algo_id algo; unsigned int digest_size; int l3index; u8 prefixlen; @@ -130,6 +145,7 @@ struct tcp_ao_info { u32 snd_sne; u32 rcv_sne; refcount_t refcnt; /* Protects twsk destruction */ + struct rcu_head rcu; }; #ifdef CONFIG_TCP_MD5SIG @@ -168,14 +184,15 @@ struct tcp6_ao_context { __be32 disn; }; -struct tcp_sigpool; /* Established states are fast-path and there always is current_key/rnext_key */ #define TCP_AO_ESTABLISHED (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | \ TCPF_CLOSE_WAIT | TCPF_LAST_ACK | TCPF_CLOSING) -int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb, - struct tcp_ao_key *key, struct tcphdr *th, - __u8 *hash_location); +void tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb, + struct tcp_ao_key *key, struct tcphdr *th, + __u8 *hash_location); +void tcp_ao_mac_update(struct tcp_ao_mac_ctx *mac_ctx, const void *data, + size_t data_len); int tcp_ao_hash_skb(unsigned short int family, char *ao_hash, struct tcp_ao_key *key, const struct sock *sk, const struct sk_buff *skb, @@ -188,8 +205,8 @@ struct tcp_ao_key *tcp_ao_established_key(const struct sock *sk, int tcp_ao_copy_all_matching(const struct sock *sk, struct sock *newsk, struct request_sock *req, struct sk_buff *skb, int family); -int tcp_ao_calc_traffic_key(struct tcp_ao_key *mkt, u8 *key, void *ctx, - unsigned int len, struct tcp_sigpool *hp); +void tcp_ao_calc_traffic_key(const struct tcp_ao_key *mkt, u8 *traffic_key, + const void *input, unsigned int input_len); void tcp_ao_destroy_sock(struct sock *sk, bool twsk); void tcp_ao_time_wait(struct tcp_timewait_sock *tcptw, struct tcp_sock *tp); bool tcp_ao_ignore_icmp(const struct sock *sk, int family, int type, int code); @@ -222,11 +239,11 @@ struct tcp_ao_key *tcp_v4_ao_lookup(const struct sock *sk, struct sock *addr_sk, int tcp_v4_ao_synack_hash(char *ao_hash, struct tcp_ao_key *mkt, struct request_sock *req, const struct sk_buff *skb, int hash_offset, u32 sne); -int tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key, - const struct sock *sk, - __be32 sisn, __be32 disn, bool send); -int tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key, - struct request_sock *req); +void tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key, + const struct sock *sk, + __be32 sisn, __be32 disn, bool send); +void tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key, + struct request_sock *req); struct tcp_ao_key *tcp_v4_ao_lookup_rsk(const struct sock *sk, struct request_sock *req, int sndid, int rcvid); @@ -234,16 +251,17 @@ int tcp_v4_ao_hash_skb(char *ao_hash, struct tcp_ao_key *key, const struct sock *sk, const struct sk_buff *skb, const u8 *tkey, int hash_offset, u32 sne); /* ipv6 specific functions */ -int tcp_v6_ao_hash_pseudoheader(struct tcp_sigpool *hp, - const struct in6_addr *daddr, - const struct in6_addr *saddr, int nbytes); -int tcp_v6_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key, - const struct sk_buff *skb, __be32 sisn, __be32 disn); -int tcp_v6_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key, - const struct sock *sk, __be32 sisn, - __be32 disn, bool send); -int tcp_v6_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key, - struct request_sock *req); +void tcp_v6_ao_hash_pseudoheader(struct tcp_ao_mac_ctx *mac_ctx, + const struct in6_addr *daddr, + const struct in6_addr *saddr, int nbytes); +void tcp_v6_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key, + const struct sk_buff *skb, __be32 sisn, + __be32 disn); +void tcp_v6_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key, + const struct sock *sk, __be32 sisn, + __be32 disn, bool send); +void tcp_v6_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key, + struct request_sock *req); struct tcp_ao_key *tcp_v6_ao_lookup(const struct sock *sk, struct sock *addr_sk, int sndid, int rcvid); struct tcp_ao_key *tcp_v6_ao_lookup_rsk(const struct sock *sk, @@ -263,11 +281,10 @@ void tcp_ao_syncookie(struct sock *sk, const struct sk_buff *skb, struct request_sock *req, unsigned short int family); #else /* CONFIG_TCP_AO */ -static inline int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb, - struct tcp_ao_key *key, struct tcphdr *th, - __u8 *hash_location) +static inline void tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb, + struct tcp_ao_key *key, + struct tcphdr *th, __u8 *hash_location) { - return 0; } static inline void tcp_ao_syncookie(struct sock *sk, const struct sk_buff *skb, diff --git a/include/net/tcp_ecn.h b/include/net/tcp_ecn.h index f13e5cd2b1ac..865d5c5a7718 100644 --- a/include/net/tcp_ecn.h +++ b/include/net/tcp_ecn.h @@ -29,8 +29,15 @@ enum tcp_accecn_option { TCP_ACCECN_OPTION_DISABLED = 0, TCP_ACCECN_OPTION_MINIMUM = 1, TCP_ACCECN_OPTION_FULL = 2, + TCP_ACCECN_OPTION_PERSIST = 3, }; +/* Apply either ECT(0) or ECT(1) based on TCP_CONG_ECT_1_NEGOTIATION flag */ +static inline void INET_ECN_xmit_ect_1_negotiation(struct sock *sk) +{ + __INET_ECN_xmit(sk, tcp_ca_ect_1_negotiation(sk)); +} + static inline void tcp_ecn_queue_cwr(struct tcp_sock *tp) { /* Do not set CWR if in AccECN mode! */ @@ -60,12 +67,6 @@ static inline void tcp_ecn_withdraw_cwr(struct tcp_sock *tp) tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR; } -/* tp->accecn_fail_mode */ -#define TCP_ACCECN_ACE_FAIL_SEND BIT(0) -#define TCP_ACCECN_ACE_FAIL_RECV BIT(1) -#define TCP_ACCECN_OPT_FAIL_SEND BIT(2) -#define TCP_ACCECN_OPT_FAIL_RECV BIT(3) - static inline bool tcp_accecn_ace_fail_send(const struct tcp_sock *tp) { return tp->accecn_fail_mode & TCP_ACCECN_ACE_FAIL_SEND; @@ -91,11 +92,6 @@ static inline void tcp_accecn_fail_mode_set(struct tcp_sock *tp, u8 mode) tp->accecn_fail_mode |= mode; } -#define TCP_ACCECN_OPT_NOT_SEEN 0x0 -#define TCP_ACCECN_OPT_EMPTY_SEEN 0x1 -#define TCP_ACCECN_OPT_COUNTER_SEEN 0x2 -#define TCP_ACCECN_OPT_FAIL_SEEN 0x3 - static inline u8 tcp_accecn_ace(const struct tcphdr *th) { return (th->ae << 2) | (th->cwr << 1) | th->ece; @@ -169,7 +165,9 @@ static inline void tcp_accecn_third_ack(struct sock *sk, switch (ace) { case 0x0: /* Invalid value */ - tcp_accecn_fail_mode_set(tp, TCP_ACCECN_ACE_FAIL_RECV); + if (!TCP_SKB_CB(skb)->sacked) + tcp_accecn_fail_mode_set(tp, TCP_ACCECN_ACE_FAIL_RECV | + TCP_ACCECN_OPT_FAIL_RECV); break; case 0x7: case 0x5: @@ -183,7 +181,7 @@ static inline void tcp_accecn_third_ack(struct sock *sk, tcp_accecn_validate_syn_feedback(sk, ace, sent_ect)) { if ((tcp_accecn_extract_syn_ect(ace) == INET_ECN_CE) && !tp->delivered_ce) - tp->delivered_ce++; + WRITE_ONCE(tp->delivered_ce, 1); } break; } @@ -398,6 +396,7 @@ static inline void tcp_accecn_init_counters(struct tcp_sock *tp) tp->received_ce_pending = 0; __tcp_accecn_init_bytes_counters(tp->received_ecn_bytes); __tcp_accecn_init_bytes_counters(tp->delivered_ecn_bytes); + tp->accecn_opt_sent_w_dsack = 0; tp->accecn_minlen = 0; tp->accecn_opt_demand = 0; tp->est_ecnfield = 0; @@ -467,6 +466,26 @@ static inline u8 tcp_accecn_option_init(const struct sk_buff *skb, return TCP_ACCECN_OPT_COUNTER_SEEN; } +static inline void tcp_ecn_rcv_synack_accecn(struct sock *sk, + const struct sk_buff *skb, u8 dsf) +{ + struct tcp_sock *tp = tcp_sk(sk); + + tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); + tp->syn_ect_rcv = dsf & INET_ECN_MASK; + /* Demand Accurate ECN option in response to the SYN on the SYN/ACK + * and the TCP server will try to send one more packet with an AccECN + * Option at a later point during the connection. + */ + if (tp->rx_opt.accecn && + tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { + u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); + + tcp_accecn_saw_opt_fail_recv(tp, saw_opt); + tp->accecn_opt_demand = 2; + } +} + /* See Table 2 of the AccECN draft */ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb, const struct tcphdr *th, u8 ip_dsfield) @@ -489,32 +508,32 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); break; case 0x1: - case 0x5: /* +========+========+============+=============+ * | A | B | SYN/ACK | Feedback | * | | | B->A | Mode of A | * | | | AE CWR ECE | | * +========+========+============+=============+ - * | AccECN | Nonce | 1 0 1 | (Reserved) | * | AccECN | ECN | 0 0 1 | Classic ECN | * | Nonce | AccECN | 0 0 1 | Classic ECN | * | ECN | AccECN | 0 0 1 | Classic ECN | * +========+========+============+=============+ */ - if (tcp_ecn_mode_pending(tp)) - /* Downgrade from AccECN, or requested initially */ + if (tcp_ca_no_fallback_rfc3168(sk)) + tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); + else tcp_ecn_mode_set(tp, TCP_ECN_MODE_RFC3168); break; - default: - tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); - tp->syn_ect_rcv = ip_dsfield & INET_ECN_MASK; - if (tp->rx_opt.accecn && - tp->saw_accecn_opt < TCP_ACCECN_OPT_COUNTER_SEEN) { - u8 saw_opt = tcp_accecn_option_init(skb, tp->rx_opt.accecn); - - tcp_accecn_saw_opt_fail_recv(tp, saw_opt); - tp->accecn_opt_demand = 2; + case 0x5: + if (tcp_ecn_mode_pending(tp)) { + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); + if (INET_ECN_is_ce(ip_dsfield)) { + tp->received_ce++; + tp->received_ce_pending++; + } } + break; + default: + tcp_ecn_rcv_synack_accecn(sk, skb, ip_dsfield); if (INET_ECN_is_ce(ip_dsfield) && tcp_accecn_validate_syn_feedback(sk, ace, tp->syn_ect_snt)) { @@ -525,9 +544,11 @@ static inline void tcp_ecn_rcv_synack(struct sock *sk, const struct sk_buff *skb } } -static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, +static inline void tcp_ecn_rcv_syn(struct sock *sk, const struct tcphdr *th, const struct sk_buff *skb) { + struct tcp_sock *tp = tcp_sk(sk); + if (tcp_ecn_mode_pending(tp)) { if (!tcp_accecn_syn_requested(th)) { /* Downgrade to classic ECN feedback */ @@ -539,7 +560,8 @@ static inline void tcp_ecn_rcv_syn(struct tcp_sock *tp, const struct tcphdr *th, tcp_ecn_mode_set(tp, TCP_ECN_MODE_ACCECN); } } - if (tcp_ecn_mode_rfc3168(tp) && (!th->ece || !th->cwr)) + if (tcp_ecn_mode_rfc3168(tp) && + (!th->ece || !th->cwr || tcp_ca_no_fallback_rfc3168(sk))) tcp_ecn_mode_set(tp, TCP_ECN_DISABLED); } @@ -561,7 +583,7 @@ static inline void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb) TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE; else if (tcp_ca_needs_ecn(sk) || tcp_bpf_ca_needs_ecn(sk)) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); if (tp->ecn_flags & TCP_ECN_MODE_ACCECN) { TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ACE; @@ -579,7 +601,8 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) bool use_ecn, use_accecn; u8 tcp_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn); - use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN; + use_accecn = tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ACCECN || + tcp_ca_needs_accecn(sk); use_ecn = tcp_ecn == TCP_ECN_IN_ECN_OUT_ECN || tcp_ecn == TCP_ECN_IN_ACCECN_OUT_ECN || tcp_ca_needs_ecn(sk) || bpf_needs_ecn || use_accecn; @@ -595,7 +618,7 @@ static inline void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb) if (use_ecn) { if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn) - INET_ECN_xmit(sk); + INET_ECN_xmit_ect_1_negotiation(sk); TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR; if (use_accecn) { @@ -619,12 +642,22 @@ static inline void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb) } static inline void -tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th) +tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th, + enum tcp_synack_type synack_type) { - if (tcp_rsk(req)->accecn_ok) - tcp_accecn_echo_syn_ect(th, tcp_rsk(req)->syn_ect_rcv); - else if (inet_rsk(req)->ecn_ok) - th->ece = 1; + /* Accurate ECN shall retransmit SYN/ACK with ACE=0 if the + * previously retransmitted SYN/ACK also times out. + */ + if (!req->num_timeout || synack_type != TCP_SYNACK_RETRANS) { + if (tcp_rsk(req)->accecn_ok) + tcp_accecn_echo_syn_ect(th, tcp_rsk(req)->syn_ect_rcv); + else if (inet_rsk(req)->ecn_ok) + th->ece = 1; + } else if (tcp_rsk(req)->accecn_ok) { + th->ae = 0; + th->cwr = 0; + th->ece = 0; + } } static inline bool tcp_accecn_option_beacon_check(const struct sock *sk) diff --git a/include/net/tls.h b/include/net/tls.h index ebd2550280ae..e57bef58851e 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -85,7 +85,6 @@ enum { TLS_BASE, TLS_SW, TLS_HW, - TLS_HW_RECORD, TLS_NUM_CONFIG, }; @@ -111,11 +110,16 @@ struct tls_sw_context_tx { struct tls_strparser { struct sock *sk; + /* Bitfield word and msg_ready are serialized by the lower + * socket lock; BH and worker contexts both acquire it. + */ u32 mark : 8; u32 stopped : 1; u32 copy_mode : 1; u32 mixed_decrypted : 1; + u32 msg_announced : 1; + bool msg_ready; struct strp_msg stm; diff --git a/include/net/tls_toe.h b/include/net/tls_toe.h deleted file mode 100644 index b3aa7593ce2c..000000000000 --- a/include/net/tls_toe.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. - * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include <linux/kref.h> -#include <linux/list.h> - -struct sock; - -#define TLS_TOE_DEVICE_NAME_MAX 32 - -/* - * This structure defines the routines for Inline TLS driver. - * The following routines are optional and filled with a - * null pointer if not defined. - * - * @name: Its the name of registered Inline tls device - * @dev_list: Inline tls device list - * int (*feature)(struct tls_toe_device *device); - * Called to return Inline TLS driver capability - * - * int (*hash)(struct tls_toe_device *device, struct sock *sk); - * This function sets Inline driver for listen and program - * device specific functioanlity as required - * - * void (*unhash)(struct tls_toe_device *device, struct sock *sk); - * This function cleans listen state set by Inline TLS driver - * - * void (*release)(struct kref *kref); - * Release the registered device and allocated resources - * @kref: Number of reference to tls_toe_device - */ -struct tls_toe_device { - char name[TLS_TOE_DEVICE_NAME_MAX]; - struct list_head dev_list; - int (*feature)(struct tls_toe_device *device); - int (*hash)(struct tls_toe_device *device, struct sock *sk); - void (*unhash)(struct tls_toe_device *device, struct sock *sk); - void (*release)(struct kref *kref); - struct kref kref; -}; - -int tls_toe_bypass(struct sock *sk); -int tls_toe_hash(struct sock *sk); -void tls_toe_unhash(struct sock *sk); - -void tls_toe_register_device(struct tls_toe_device *device); -void tls_toe_unregister_device(struct tls_toe_device *device); diff --git a/include/net/transp_v6.h b/include/net/transp_v6.h index 1a97e3f32029..c0a421fe0c2a 100644 --- a/include/net/transp_v6.h +++ b/include/net/transp_v6.h @@ -8,7 +8,6 @@ /* IPv6 transport protocols */ extern struct proto rawv6_prot; extern struct proto udpv6_prot; -extern struct proto udplitev6_prot; extern struct proto tcpv6_prot; extern struct proto pingv6_prot; @@ -28,8 +27,6 @@ int rawv6_init(void); void rawv6_exit(void); int udpv6_init(void); void udpv6_exit(void); -int udplitev6_init(void); -void udplitev6_exit(void); int tcpv6_init(void); void tcpv6_exit(void); diff --git a/include/net/tso.h b/include/net/tso.h index e7e157ae0526..da82aabd1d48 100644 --- a/include/net/tso.h +++ b/include/net/tso.h @@ -3,6 +3,7 @@ #define _TSO_H #include <linux/skbuff.h> +#include <linux/dma-mapping.h> #include <net/ip.h> #define TSO_HEADER_SIZE 256 @@ -28,4 +29,103 @@ void tso_build_hdr(const struct sk_buff *skb, char *hdr, struct tso_t *tso, void tso_build_data(const struct sk_buff *skb, struct tso_t *tso, int size); int tso_start(struct sk_buff *skb, struct tso_t *tso); +/** + * struct tso_dma_map - DMA mapping state for GSO payload + * @dev: device used for DMA mapping + * @skb: the GSO skb being mapped + * @hdr_len: per-segment header length + * @iova_state: DMA IOVA state (when IOMMU available) + * @iova_offset: global byte offset into IOVA range (IOVA path only) + * @total_len: total payload length + * @frag_idx: current region (-1 = linear, 0..nr_frags-1 = frag) + * @offset: byte offset within current region + * @linear_dma: DMA address of the linear payload + * @linear_len: length of the linear payload + * @nr_frags: number of frags successfully DMA-mapped + * @frags: per-frag DMA address and length + * + * DMA-maps the payload regions of a GSO skb (linear data + frags). + * Prefers the DMA IOVA API for a single contiguous mapping with one + * IOTLB sync; falls back to per-region dma_map_phys() otherwise. + */ +struct tso_dma_map { + struct device *dev; + const struct sk_buff *skb; + unsigned int hdr_len; + /* IOVA path */ + struct dma_iova_state iova_state; + size_t iova_offset; + size_t total_len; + /* Fallback path if IOVA path fails */ + int frag_idx; + unsigned int offset; + dma_addr_t linear_dma; + unsigned int linear_len; + unsigned int nr_frags; + struct { + dma_addr_t dma; + unsigned int len; + } frags[MAX_SKB_FRAGS]; +}; + +/** + * struct tso_dma_map_completion_state - Completion-time cleanup state + * @iova_state: DMA IOVA state (when IOMMU available) + * @total_len: total payload length of the IOVA mapping + * + * Drivers store this on their SW ring at xmit time via + * tso_dma_map_completion_save(), then call tso_dma_map_complete() at + * completion time. + */ +struct tso_dma_map_completion_state { + struct dma_iova_state iova_state; + size_t total_len; +}; + +int tso_dma_map_init(struct tso_dma_map *map, struct device *dev, + const struct sk_buff *skb, unsigned int hdr_len); +void tso_dma_map_cleanup(struct tso_dma_map *map); +unsigned int tso_dma_map_count(struct tso_dma_map *map, unsigned int len); +bool tso_dma_map_next(struct tso_dma_map *map, dma_addr_t *addr, + unsigned int *chunk_len, unsigned int *mapping_len, + unsigned int seg_remaining); + +/** + * tso_dma_map_completion_save - save state needed for completion-time cleanup + * @map: the xmit-time DMA map + * @cstate: driver-owned storage that persists until completion + * + * Should be called at xmit time to update the completion state and later passed + * to tso_dma_map_complete(). + */ +static inline void +tso_dma_map_completion_save(const struct tso_dma_map *map, + struct tso_dma_map_completion_state *cstate) +{ + cstate->iova_state = map->iova_state; + cstate->total_len = map->total_len; +} + +/** + * tso_dma_map_complete - tear down mapping at completion time + * @dev: the device that owns the mapping + * @cstate: state saved by tso_dma_map_completion_save() + * + * Return: true if the IOVA path was used and the mapping has been + * destroyed; false if the fallback per-region path was used and the + * driver must unmap via its normal completion path. + */ +static inline bool +tso_dma_map_complete(struct device *dev, + struct tso_dma_map_completion_state *cstate) +{ + if (dma_use_iova(&cstate->iova_state)) { + dma_iova_destroy(dev, &cstate->iova_state, cstate->total_len, + DMA_TO_DEVICE, 0); + return true; + } + + return false; +} + #endif /* _TSO_H */ diff --git a/include/net/udp.h b/include/net/udp.h index a061d1b22ddc..1fee17274745 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -29,13 +29,12 @@ #include <linux/seq_file.h> #include <linux/poll.h> #include <linux/indirect_call_wrapper.h> +#include <linux/math.h> /** - * struct udp_skb_cb - UDP(-Lite) private variables + * struct udp_skb_cb - UDP private variables * * @header: private variables used by IPv4/IPv6 - * @cscov: checksum coverage length (UDP-Lite only) - * @partial_cov: if set indicates partial csum coverage */ struct udp_skb_cb { union { @@ -44,8 +43,6 @@ struct udp_skb_cb { struct inet6_skb_parm h6; #endif } header; - __u16 cscov; - __u8 partial_cov; }; #define UDP_SKB_CB(__skb) ((struct udp_skb_cb *)((__skb)->cb)) @@ -104,7 +101,7 @@ struct udp_table { unsigned int log; }; extern struct udp_table udp_table; -void udp_table_init(struct udp_table *, const char *); + static inline struct udp_hslot *udp_hashslot(struct udp_table *table, const struct net *net, unsigned int num) @@ -215,13 +212,11 @@ extern int sysctl_udp_wmem_min; struct sk_buff; /* - * Generic checksumming routines for UDP(-Lite) v4 and v6 + * Generic checksumming routines for UDP v4 and v6 */ static inline __sum16 __udp_lib_checksum_complete(struct sk_buff *skb) { - return (UDP_SKB_CB(skb)->cscov == skb->len ? - __skb_checksum_complete(skb) : - __skb_checksum_complete_head(skb, UDP_SKB_CB(skb)->cscov)); + return __skb_checksum_complete(skb); } static inline int udp_lib_checksum_complete(struct sk_buff *skb) @@ -272,7 +267,6 @@ static inline void udp_csum_pull_header(struct sk_buff *skb) skb->csum = csum_partial(skb->data, sizeof(struct udphdr), skb->csum); skb_pull_rcsum(skb, sizeof(struct udphdr)); - UDP_SKB_CB(skb)->cscov -= sizeof(struct udphdr); } typedef struct sock *(*udp_lookup_t)(const struct sk_buff *skb, __be16 sport, @@ -281,6 +275,10 @@ typedef struct sock *(*udp_lookup_t)(const struct sk_buff *skb, __be16 sport, void udp_v6_early_demux(struct sk_buff *skb); INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *)); +int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len); +INDIRECT_CALLABLE_DECLARE(int udpv6_recvmsg(struct sock *sk, struct msghdr *msg, + size_t len, int flags)); + struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, netdev_features_t features, bool is_ipv6); @@ -294,8 +292,7 @@ static inline int udp_lib_init_sock(struct sock *sk) up->forward_threshold = sk->sk_rcvbuf >> 2; set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags); - up->udp_prod_queue = kcalloc(nr_node_ids, sizeof(*up->udp_prod_queue), - GFP_KERNEL); + up->udp_prod_queue = kzalloc_objs(*up->udp_prod_queue, nr_node_ids); if (!up->udp_prod_queue) return -ENOMEM; for (int i = 0; i < nr_node_ids; i++) @@ -308,7 +305,7 @@ static inline void udp_drops_inc(struct sock *sk) numa_drop_add(&udp_sk(sk)->drop_counters, 1); } -/* hash routines shared between UDPv4/6 and UDP-Litev4/6 */ +/* hash routines shared between UDPv4/6 */ static inline int udp_lib_hash(struct sock *sk) { BUG(); @@ -377,7 +374,7 @@ static inline __be16 udp_flow_src_port(struct net *net, struct sk_buff *skb, */ hash ^= hash << 16; - return htons((((u64) hash * (max - min)) >> 32) + min); + return htons(reciprocal_scale(hash, max - min + 1) + min); } static inline int udp_rqueue_get(struct sock *sk) @@ -416,6 +413,8 @@ bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst); int udp_err(struct sk_buff *, u32); int udp_abort(struct sock *sk, int err); int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len); +INDIRECT_CALLABLE_DECLARE(int udp_recvmsg(struct sock *sk, struct msghdr *msg, + size_t len, int flags)); void udp_splice_eof(struct socket *sock); int udp_push_pending_frames(struct sock *sk); void udp_flush_pending_frames(struct sock *sk); @@ -423,7 +422,6 @@ int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size); void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst); int udp_rcv(struct sk_buff *skb); int udp_ioctl(struct sock *sk, int cmd, int *karg); -int udp_init_sock(struct sock *sk); int udp_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int addr_len); int __udp_disconnect(struct sock *sk, int flags); int udp_disconnect(struct sock *sk, int flags); @@ -432,16 +430,15 @@ struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, netdev_features_t features, bool is_ipv6); int udp_lib_getsockopt(struct sock *sk, int level, int optname, - char __user *optval, int __user *optlen); + sockopt_t *opt); int udp_lib_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen, int (*push_pending_frames)(struct sock *)); struct sock *udp4_lib_lookup(const struct net *net, __be32 saddr, __be16 sport, __be32 daddr, __be16 dport, int dif); struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr, - __be16 sport, - __be32 daddr, __be16 dport, int dif, int sdif, - struct udp_table *tbl, struct sk_buff *skb); + __be16 sport, __be32 daddr, __be16 dport, + int dif, int sdif, struct sk_buff *skb); struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb, __be16 sport, __be16 dport); struct sock *udp6_lib_lookup(const struct net *net, @@ -451,8 +448,7 @@ struct sock *udp6_lib_lookup(const struct net *net, struct sock *__udp6_lib_lookup(const struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, __be16 dport, - int dif, int sdif, struct udp_table *tbl, - struct sk_buff *skb); + int dif, int sdif, struct sk_buff *skb); struct sock *udp6_lib_lookup_skb(const struct sk_buff *skb, __be16 sport, __be16 dport); int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor); @@ -524,38 +520,28 @@ static inline int copy_linear_skb(struct sk_buff *skb, int len, int off, } /* - * SNMP statistics for UDP and UDP-Lite + * SNMP statistics for UDP */ -#define UDP_INC_STATS(net, field, is_udplite) do { \ - if (is_udplite) SNMP_INC_STATS((net)->mib.udplite_statistics, field); \ - else SNMP_INC_STATS((net)->mib.udp_statistics, field); } while(0) -#define __UDP_INC_STATS(net, field, is_udplite) do { \ - if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_statistics, field); \ - else __SNMP_INC_STATS((net)->mib.udp_statistics, field); } while(0) - -#define __UDP6_INC_STATS(net, field, is_udplite) do { \ - if (is_udplite) __SNMP_INC_STATS((net)->mib.udplite_stats_in6, field);\ - else __SNMP_INC_STATS((net)->mib.udp_stats_in6, field); \ -} while(0) -#define UDP6_INC_STATS(net, field, __lite) do { \ - if (__lite) SNMP_INC_STATS((net)->mib.udplite_stats_in6, field); \ - else SNMP_INC_STATS((net)->mib.udp_stats_in6, field); \ -} while(0) +#define __UDP_INC_STATS(net, field) \ + __SNMP_INC_STATS((net)->mib.udp_statistics, field) +#define UDP_INC_STATS(net, field) \ + SNMP_INC_STATS((net)->mib.udp_statistics, field) +#define __UDP6_INC_STATS(net, field) \ + __SNMP_INC_STATS((net)->mib.udp_stats_in6, field) +#define UDP6_INC_STATS(net, field) \ + SNMP_INC_STATS((net)->mib.udp_stats_in6, field) #if IS_ENABLED(CONFIG_IPV6) -#define __UDPX_MIB(sk, ipv4) \ -({ \ - ipv4 ? (IS_UDPLITE(sk) ? sock_net(sk)->mib.udplite_statistics : \ - sock_net(sk)->mib.udp_statistics) : \ - (IS_UDPLITE(sk) ? sock_net(sk)->mib.udplite_stats_in6 : \ - sock_net(sk)->mib.udp_stats_in6); \ -}) +#define __UDPX_MIB(sk, ipv4) \ + ({ \ + ipv4 ? sock_net(sk)->mib.udp_statistics : \ + sock_net(sk)->mib.udp_stats_in6; \ + }) #else -#define __UDPX_MIB(sk, ipv4) \ -({ \ - IS_UDPLITE(sk) ? sock_net(sk)->mib.udplite_statistics : \ - sock_net(sk)->mib.udp_statistics; \ -}) +#define __UDPX_MIB(sk, ipv4) \ + ({ \ + sock_net(sk)->mib.udp_statistics; \ + }) #endif #define __UDPX_INC_STATS(sk, field) \ @@ -564,7 +550,6 @@ static inline int copy_linear_skb(struct sk_buff *skb, int len, int off, #ifdef CONFIG_PROC_FS struct udp_seq_afinfo { sa_family_t family; - struct udp_table *udp_table; }; struct udp_iter_state { @@ -576,9 +561,6 @@ void *udp_seq_start(struct seq_file *seq, loff_t *pos); void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos); void udp_seq_stop(struct seq_file *seq, void *v); -extern const struct seq_operations udp_seq_ops; -extern const struct seq_operations udp6_seq_ops; - int udp4_proc_init(void); void udp4_proc_exit(void); #endif /* CONFIG_PROC_FS */ @@ -649,9 +631,6 @@ drop: static inline void udp_post_segment_fix_csum(struct sk_buff *skb) { - /* UDP-lite can't land here - no GRO */ - WARN_ON_ONCE(UDP_SKB_CB(skb)->partial_cov); - /* UDP packets generated with UDP_SEGMENT and traversing: * * UDP tunnel(xmit) -> veth (segmentation) -> veth (gro) -> UDP tunnel (rx) @@ -665,7 +644,6 @@ static inline void udp_post_segment_fix_csum(struct sk_buff *skb) * a valid csum after the segmentation. * Additionally fixup the UDP CB. */ - UDP_SKB_CB(skb)->cscov = skb->len; if (skb->ip_summed == CHECKSUM_NONE && !skb->csum_valid) skb->csum_valid = 1; } diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h index 9acef2fbd2fd..498b7b262fa9 100644 --- a/include/net/udp_tunnel.h +++ b/include/net/udp_tunnel.h @@ -7,9 +7,13 @@ #if IS_ENABLED(CONFIG_IPV6) #include <net/ipv6.h> -#include <net/ipv6_stubs.h> #endif +#define UDP_TUNNEL_PARTIAL_FEATURES NETIF_F_GSO_ENCAP_ALL +#define UDP_TUNNEL_STRIPPED_GSO_TYPES ((UDP_TUNNEL_PARTIAL_FEATURES | \ + NETIF_F_GSO_PARTIAL) >> \ + NETIF_F_GSO_SHIFT) + struct udp_port_cfg { u8 family; @@ -47,7 +51,7 @@ int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, static inline int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, struct socket **sockp) { - return 0; + return -EPFNOSUPPORT; } #endif @@ -90,7 +94,7 @@ struct udp_tunnel_sock_cfg { }; /* Setup the given (UDP) sock to receive UDP encapsulated packets */ -void setup_udp_tunnel_sock(struct net *net, struct socket *sock, +void setup_udp_tunnel_sock(struct net *net, struct sock *sk, struct udp_tunnel_sock_cfg *sock_cfg); /* -- List of parsable UDP tunnel types -- @@ -123,12 +127,12 @@ struct udp_tunnel_info { }; /* Notify network devices of offloadable types */ -void udp_tunnel_push_rx_port(struct net_device *dev, struct socket *sock, +void udp_tunnel_push_rx_port(struct net_device *dev, struct sock *sk, unsigned short type); -void udp_tunnel_drop_rx_port(struct net_device *dev, struct socket *sock, +void udp_tunnel_drop_rx_port(struct net_device *dev, struct sock *sk, unsigned short type); -void udp_tunnel_notify_add_rx_port(struct socket *sock, unsigned short type); -void udp_tunnel_notify_del_rx_port(struct socket *sock, unsigned short type); +void udp_tunnel_notify_add_rx_port(struct sock *sk, unsigned short type); +void udp_tunnel_notify_del_rx_port(struct sock *sk, unsigned short type); /* Transmit the skb using UDP encapsulation. */ void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb, @@ -145,7 +149,34 @@ void udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk, __be16 src_port, __be16 dst_port, bool nocheck, u16 ip6cb_flags); -void udp_tunnel_sock_release(struct socket *sock); +static inline bool udp_tunnel_handle_partial(struct sk_buff *skb) +{ + bool double_encap = !!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL); + + /* + * If the skb went through partial segmentation, lower devices + * will not need to offload the related features - except for + * UDP_TUNNEL, that will be re-added by the later + * udp_tunnel_handle_offloads(). + */ + if (double_encap) + skb_shinfo(skb)->gso_type &= ~UDP_TUNNEL_STRIPPED_GSO_TYPES; + return double_encap; +} + +static inline void udp_tunnel_set_inner_protocol(struct sk_buff *skb, + bool double_encap, + __be16 inner_proto) +{ + /* + * The inner protocol has been set by the nested tunnel, don't + * overraid it. + */ + if (!double_encap) + skb_set_inner_protocol(skb, inner_proto); +} + +void udp_tunnel_sock_release(struct sock *sk); struct rtable *udp_tunnel_dst_lookup(struct sk_buff *skb, struct net_device *dev, @@ -157,7 +188,7 @@ struct rtable *udp_tunnel_dst_lookup(struct sk_buff *skb, struct dst_entry *udp_tunnel6_dst_lookup(struct sk_buff *skb, struct net_device *dev, struct net *net, - struct socket *sock, int oif, + struct sock *sk, int oif, struct in6_addr *saddr, const struct ip_tunnel_key *key, __be16 sport, __be16 dport, u8 dsfield, @@ -198,7 +229,7 @@ static inline void udp_tunnel_encap_enable(struct sock *sk) #if IS_ENABLED(CONFIG_IPV6) if (READ_ONCE(sk->sk_family) == PF_INET6) - ipv6_stub->udpv6_encap_enable(); + udpv6_encap_enable(); #endif udp_encap_enable(); } diff --git a/include/net/udplite.h b/include/net/udplite.h deleted file mode 100644 index 786919d29f8d..000000000000 --- a/include/net/udplite.h +++ /dev/null @@ -1,88 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Definitions for the UDP-Lite (RFC 3828) code. - */ -#ifndef _UDPLITE_H -#define _UDPLITE_H - -#include <net/ip6_checksum.h> -#include <net/udp.h> - -/* UDP-Lite socket options */ -#define UDPLITE_SEND_CSCOV 10 /* sender partial coverage (as sent) */ -#define UDPLITE_RECV_CSCOV 11 /* receiver partial coverage (threshold ) */ - -extern struct proto udplite_prot; -extern struct udp_table udplite_table; - -/* - * Checksum computation is all in software, hence simpler getfrag. - */ -static __inline__ int udplite_getfrag(void *from, char *to, int offset, - int len, int odd, struct sk_buff *skb) -{ - struct msghdr *msg = from; - return copy_from_iter_full(to, len, &msg->msg_iter) ? 0 : -EFAULT; -} - -/* - * Checksumming routines - */ -static inline int udplite_checksum_init(struct sk_buff *skb, struct udphdr *uh) -{ - u16 cscov; - - /* In UDPv4 a zero checksum means that the transmitter generated no - * checksum. UDP-Lite (like IPv6) mandates checksums, hence packets - * with a zero checksum field are illegal. */ - if (uh->check == 0) { - net_dbg_ratelimited("UDPLite: zeroed checksum field\n"); - return 1; - } - - cscov = ntohs(uh->len); - - if (cscov == 0) /* Indicates that full coverage is required. */ - ; - else if (cscov < 8 || cscov > skb->len) { - /* - * Coverage length violates RFC 3828: log and discard silently. - */ - net_dbg_ratelimited("UDPLite: bad csum coverage %d/%d\n", - cscov, skb->len); - return 1; - - } else if (cscov < skb->len) { - UDP_SKB_CB(skb)->partial_cov = 1; - UDP_SKB_CB(skb)->cscov = cscov; - if (skb->ip_summed == CHECKSUM_COMPLETE) - skb->ip_summed = CHECKSUM_NONE; - skb->csum_valid = 0; - } - - return 0; -} - -/* Fast-path computation of checksum. Socket may not be locked. */ -static inline __wsum udplite_csum(struct sk_buff *skb) -{ - const int off = skb_transport_offset(skb); - const struct sock *sk = skb->sk; - int len = skb->len - off; - - if (udp_test_bit(UDPLITE_SEND_CC, sk)) { - u16 pcslen = READ_ONCE(udp_sk(sk)->pcslen); - - if (pcslen < len) { - if (pcslen > 0) - len = pcslen; - udp_hdr(skb)->len = htons(pcslen); - } - } - skb->ip_summed = CHECKSUM_NONE; /* no HW support for checksumming */ - - return skb_checksum(skb, off, len, 0); -} - -void udplite4_register(void); -#endif /* _UDPLITE_H */ diff --git a/include/net/vxlan.h b/include/net/vxlan.h index 0ee50785f4f1..7b8207505523 100644 --- a/include/net/vxlan.h +++ b/include/net/vxlan.h @@ -185,7 +185,8 @@ struct vxlan_metadata { /* per UDP socket information */ struct vxlan_sock { struct hlist_node hlist; - struct socket *sock; + struct sock *sk; + struct rcu_head rcu; struct hlist_head vni_list[VNI_HASH_SIZE]; refcount_t refcnt; u32 flags; @@ -358,9 +359,6 @@ struct vxlan_dev { VXLAN_F_MC_ROUTE | \ 0) -struct net_device *vxlan_dev_create(struct net *net, const char *name, - u8 name_assign_type, struct vxlan_config *conf); - static inline netdev_features_t vxlan_features_check(struct sk_buff *skb, netdev_features_t features) { @@ -448,7 +446,7 @@ static inline __be32 vxlan_compute_rco(unsigned int start, unsigned int offset) static inline unsigned short vxlan_get_sk_family(struct vxlan_sock *vs) { - return vs->sock->sk->sk_family; + return vs->sk->sk_family; } #if IS_ENABLED(CONFIG_IPV6) @@ -566,8 +564,9 @@ static inline bool vxlan_fdb_nh_path_select(struct nexthop *nh, struct vxlan_rdst *rdst) { struct fib_nh_common *nhc; + __be16 dst_port = 0; - nhc = nexthop_path_fdb_result(nh, hash >> 1); + nhc = nexthop_path_fdb_result(nh, hash >> 1, &dst_port); if (unlikely(!nhc)) return false; @@ -582,6 +581,8 @@ static inline bool vxlan_fdb_nh_path_select(struct nexthop *nh, break; } + rdst->remote_port = dst_port; + return true; } diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index 23e8861e8b25..6e70b320b399 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -14,7 +14,7 @@ #include <linux/mm.h> #include <net/sock.h> -#define XDP_UMEM_SG_FLAG (1 << 1) +#define XDP_UMEM_SG_FLAG BIT(3) struct net_device; struct xsk_queue; @@ -80,6 +80,7 @@ struct xdp_sock { * call of __xsk_generic_xmit(). */ struct sk_buff *skb; + bool drain_cont; struct list_head map_list; /* Protects map_list */ @@ -140,45 +141,16 @@ INDIRECT_CALLABLE_DECLARE(void xsk_destruct_skb(struct sk_buff *)); static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, struct xsk_tx_metadata_compl *compl) { - if (!meta) - return; + compl->tx_timestamp = NULL; - if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) - compl->tx_timestamp = &meta->completion.tx_timestamp; - else - compl->tx_timestamp = NULL; -} - -/** - * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission - * and call appropriate xsk_tx_metadata_ops operation. - * @meta: pointer to AF_XDP metadata area - * @ops: pointer to struct xsk_tx_metadata_ops - * @priv: pointer to driver-private aread - * - * This function should be called by the networking device when - * it prepares AF_XDP egress packet. - */ -static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta, - const struct xsk_tx_metadata_ops *ops, - void *priv) -{ if (!meta) return; - if (ops->tmo_request_launch_time) - if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) - ops->tmo_request_launch_time(meta->request.launch_time, - priv); - - if (ops->tmo_request_timestamp) - if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) - ops->tmo_request_timestamp(priv); + /* we can only arrive here if the completion timestamp has been + * requested via XDP_TXMD_FLAGS_TIMESTAMP, see xsk_tx_metadata_request + */ - if (ops->tmo_request_checksum) - if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) - ops->tmo_request_checksum(meta->request.csum_start, - meta->request.csum_offset, priv); + compl->tx_timestamp = &meta->completion.tx_timestamp; } /** @@ -230,12 +202,6 @@ static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, { } -static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta, - const struct xsk_tx_metadata_ops *ops, - void *priv) -{ -} - static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, const struct xsk_tx_metadata_ops *ops, void *priv) diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index 242e34f771cc..d94aeb506379 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -41,16 +41,42 @@ static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool) return XDP_PACKET_HEADROOM + pool->headroom; } +static inline u32 xsk_pool_get_tailroom(bool mbuf) +{ + return mbuf ? SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : 0; +} + static inline u32 xsk_pool_get_chunk_size(struct xsk_buff_pool *pool) { return pool->chunk_size; } -static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) +static inline u32 __xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) { return xsk_pool_get_chunk_size(pool) - xsk_pool_get_headroom(pool); } +static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) +{ + u32 frame_size = __xsk_pool_get_rx_frame_size(pool); + struct xdp_umem *umem = pool->umem; + bool mbuf; + + /* Reserve tailroom only for zero-copy pools that opted into + * multi-buffer. The reserved area is used for skb_shared_info, + * matching the XDP core's xdp_data_hard_end() layout. + */ + mbuf = pool->dev && (umem->flags & XDP_UMEM_SG_FLAG); + frame_size -= xsk_pool_get_tailroom(mbuf); + + return ALIGN_DOWN(frame_size, 128); +} + +static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool) +{ + return pool->unaligned ? 0 : xsk_pool_get_chunk_size(pool); +} + static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) { @@ -122,7 +148,7 @@ static inline void xsk_buff_free(struct xdp_buff *xdp) goto out; list_for_each_entry_safe(pos, tmp, xskb_list, list_node) { - list_del(&pos->list_node); + list_del_init(&pos->list_node); xp_free(pos); } @@ -157,7 +183,7 @@ static inline struct xdp_buff *xsk_buff_get_frag(const struct xdp_buff *first) frag = list_first_entry_or_null(&xskb->pool->xskb_list, struct xdp_buff_xsk, list_node); if (frag) { - list_del(&frag->list_node); + list_del_init(&frag->list_node); ret = &frag->xdp; } @@ -168,7 +194,7 @@ static inline void xsk_buff_del_frag(struct xdp_buff *xdp) { struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); - list_del(&xskb->list_node); + list_del_init(&xskb->list_node); } static inline struct xdp_buff *xsk_buff_get_head(struct xdp_buff *first) @@ -214,17 +240,18 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) * xsk_buff_raw_get_ctx - get &xdp_desc context * @pool: XSk buff pool desc address belongs to * @addr: desc address (from userspace) + * @options: desc options (from userspace) * * Wrapper for xp_raw_get_ctx() to be used in drivers, see its kdoc for * details. * * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata - * pointer, if it is present and valid (initialized to %NULL otherwise). + * pointer, if it is present (initialized to %NULL otherwise). */ static inline struct xdp_desc_ctx -xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options) { - return xp_raw_get_ctx(pool, addr); + return xp_raw_get_ctx(pool, addr, options); } #define XDP_TXMD_FLAGS_VALID ( \ @@ -234,30 +261,78 @@ xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) 0) static inline bool -xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata *meta) +xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool, + const struct xsk_tx_metadata *meta, u64 *flags) { - return !(meta->flags & ~XDP_TXMD_FLAGS_VALID); + *flags = READ_ONCE(meta->flags); + if (*flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + if (pool->tx_metadata_len < + offsetofend(struct xsk_tx_metadata, request.launch_time)) + return false; + return !(*flags & ~XDP_TXMD_FLAGS_VALID); } -static inline struct xsk_tx_metadata * -__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) +/** + * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission + * and call appropriate xsk_tx_metadata_ops operation. + * @pool: pointer to AF_XDP buffer pool, used to validate the metadata + * @pmeta: pointer to pointer to AF_XDP metadata area + * @ops: pointer to struct xsk_tx_metadata_ops + * @priv: pointer to driver-private area + * + * This function should be called by the networking device when + * it prepares AF_XDP egress packet. + */ +static inline void +xsk_tx_metadata_request(const struct xsk_buff_pool *pool, + struct xsk_tx_metadata **pmeta, + const struct xsk_tx_metadata_ops *ops, void *priv) { - struct xsk_tx_metadata *meta; + const struct xsk_tx_metadata *meta = *pmeta; + u64 flags; - if (!pool->tx_metadata_len) - return NULL; + if (!meta) + return; - meta = data - pool->tx_metadata_len; - if (unlikely(!xsk_buff_valid_tx_metadata(meta))) - return NULL; /* no way to signal the error to the user */ + if (unlikely(!xsk_buff_valid_tx_metadata(pool, meta, &flags))) { + *pmeta = NULL; + return; /* no way to signal the error to the user */ + } + + if (ops->tmo_request_launch_time) + if (flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + ops->tmo_request_launch_time( + READ_ONCE(meta->request.launch_time), priv); + + if (ops->tmo_request_timestamp) + if (flags & XDP_TXMD_FLAGS_TIMESTAMP) + ops->tmo_request_timestamp(priv); + + if (ops->tmo_request_checksum) + if (flags & XDP_TXMD_FLAGS_CHECKSUM) + ops->tmo_request_checksum( + READ_ONCE(meta->request.csum_start), + READ_ONCE(meta->request.csum_offset), priv); + + if (!(flags & XDP_TXMD_FLAGS_TIMESTAMP)) + *pmeta = NULL; +} + +static inline struct xsk_tx_metadata * +__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data, + unsigned int options) +{ + if (!pool->tx_metadata_len || !(options & XDP_TX_METADATA)) + return NULL; - return meta; + return data - pool->tx_metadata_len; } static inline struct xsk_tx_metadata * -xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options) { - return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr)); + return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr), + options); } static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp) @@ -337,6 +412,11 @@ static inline u32 xsk_pool_get_rx_frame_size(struct xsk_buff_pool *pool) return 0; } +static inline u32 xsk_pool_get_rx_frag_step(struct xsk_buff_pool *pool) +{ + return 0; +} + static inline void xsk_pool_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq) { @@ -433,24 +513,34 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) } static inline struct xdp_desc_ctx -xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options) { return (struct xdp_desc_ctx){ }; } -static inline bool xsk_buff_valid_tx_metadata(struct xsk_tx_metadata *meta) +static inline bool +xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool, + const struct xsk_tx_metadata *meta, u64 *flags) { return false; } +static inline void +xsk_tx_metadata_request(const struct xsk_buff_pool *pool, + struct xsk_tx_metadata **pmeta, + const struct xsk_tx_metadata_ops *ops, void *priv) +{ +} + static inline struct xsk_tx_metadata * -__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) +__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data, + unsigned int options) { return NULL; } static inline struct xsk_tx_metadata * -xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options) { return NULL; } diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 0a14daaa5dd4..a6d69aaa6cd2 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -162,6 +162,8 @@ struct xfrm_dev_offload { */ struct net_device *real_dev; unsigned long offload_handle; + /* Snapshot the attached device index for dump paths. */ + int ifindex; u8 dir : 2; u8 type : 2; u8 flags : 2; @@ -682,12 +684,22 @@ struct xfrm_migrate { xfrm_address_t old_saddr; xfrm_address_t new_daddr; xfrm_address_t new_saddr; + struct xfrm_encap_tmpl *encap; + struct xfrm_user_offload *xuo; + struct xfrm_mark old_mark; + const struct xfrm_mark *new_mark; + struct xfrm_mark smark; u8 proto; u8 mode; - u16 reserved; - u32 reqid; + u16 msg_type; /* XFRM_MSG_MIGRATE or XFRM_MSG_MIGRATE_STATE */ + u32 flags; + u32 old_reqid; + u32 new_reqid; + u32 nat_keepalive_interval; + u32 mapping_maxage; u16 old_family; u16 new_family; + const struct xfrm_selector *new_sel; }; #define XFRM_KM_TIMEOUT 30 @@ -715,6 +727,7 @@ struct xfrm_mgr { const struct xfrm_migrate *m, int num_bundles, const struct xfrm_kmaddress *k, + struct net *net, const struct xfrm_encap_tmpl *encap); bool (*is_alive)(const struct km_event *c); }; @@ -942,6 +955,9 @@ static inline bool addr_match(const void *token1, const void *token2, unsigned int pdw; unsigned int pbi; + if (prefixlen > 128) + return false; + pdw = prefixlen >> 5; /* num of whole u32 in prefix */ pbi = prefixlen & 0x1f; /* num of bits in incomplete u32 in prefix */ @@ -966,6 +982,10 @@ static inline bool addr4_match(__be32 a1, __be32 a2, u8 prefixlen) /* C99 6.5.7 (3): u32 << 32 is undefined behaviour */ if (sizeof(long) == 4 && prefixlen == 0) return true; + + if (prefixlen > 32) + return false; + return !((a1 ^ a2) & htonl(~0UL << (32 - prefixlen))); } @@ -1156,19 +1176,19 @@ struct xfrm_offload { #define CRYPTO_INVALID_PROTOCOL 128 /* Used to keep whole l2 header for transport mode GRO */ - __u32 orig_mac_len; + __u16 orig_mac_len; __u8 proto; __u8 inner_ipproto; }; struct sec_path { - int len; - int olen; - int verified_cnt; - struct xfrm_state *xvec[XFRM_MAX_DEPTH]; struct xfrm_offload ovec[XFRM_MAX_OFFLOAD_DEPTH]; + + u8 len; + u8 olen; + u8 verified_cnt; }; struct sec_path *secpath_set(struct sk_buff *skb); @@ -1249,8 +1269,8 @@ int __xfrm_policy_check(struct sock *, int dir, struct sk_buff *skb, static inline bool __xfrm_check_nopolicy(struct net *net, struct sk_buff *skb, int dir) { - if (!net->xfrm.policy_count[dir] && !secpath_exists(skb)) - return net->xfrm.policy_default[dir] == XFRM_USERPOLICY_ACCEPT; + if (!READ_ONCE(net->xfrm.policy_count[dir]) && !secpath_exists(skb)) + return READ_ONCE(net->xfrm.policy_default[dir]) == XFRM_USERPOLICY_ACCEPT; return false; } @@ -1350,8 +1370,8 @@ static inline int xfrm_route_forward(struct sk_buff *skb, unsigned short family) { struct net *net = dev_net(skb->dev); - if (!net->xfrm.policy_count[XFRM_POLICY_OUT] && - net->xfrm.policy_default[XFRM_POLICY_OUT] == XFRM_USERPOLICY_ACCEPT) + if (!READ_ONCE(net->xfrm.policy_count[XFRM_POLICY_OUT]) && + READ_ONCE(net->xfrm.policy_default[XFRM_POLICY_OUT]) == XFRM_USERPOLICY_ACCEPT) return true; return (skb_dst(skb)->flags & DST_NOXFRM) || @@ -1774,7 +1794,7 @@ u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq); int xfrm_init_replay(struct xfrm_state *x, struct netlink_ext_ack *extack); u32 xfrm_state_mtu(struct xfrm_state *x, int mtu); int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack); -int xfrm_init_state(struct xfrm_state *x); +int xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack); int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type); int xfrm_input_resume(struct sk_buff *skb, int nexthdr); int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb, @@ -1891,15 +1911,21 @@ int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol); #ifdef CONFIG_XFRM_MIGRATE int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, const struct xfrm_migrate *m, int num_bundles, - const struct xfrm_kmaddress *k, + const struct xfrm_kmaddress *k, struct net *net, const struct xfrm_encap_tmpl *encap); struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net, u32 if_id); +struct xfrm_state *xfrm_state_migrate_create(struct xfrm_state *x, + const struct xfrm_migrate *m, + struct net *net, + struct netlink_ext_ack *extack); +int xfrm_state_migrate_install(const struct xfrm_state *x, + struct xfrm_state *xc, + const struct xfrm_migrate *m, + struct netlink_ext_ack *extack); struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x, struct xfrm_migrate *m, - struct xfrm_encap_tmpl *encap, struct net *net, - struct xfrm_user_offload *xuo, struct netlink_ext_ack *extack); int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type, struct xfrm_migrate *m, int num_bundles, @@ -2013,23 +2039,51 @@ static inline unsigned int xfrm_replay_state_esn_len(struct xfrm_replay_state_es #ifdef CONFIG_XFRM_MIGRATE static inline int xfrm_replay_clone(struct xfrm_state *x, - struct xfrm_state *orig) + const struct xfrm_state *orig) { + /* Counters synced later in xfrm_replay_sync() */ - x->replay_esn = kmemdup(orig->replay_esn, + x->replay = orig->replay; + x->preplay = orig->preplay; + + if (orig->replay_esn) { + x->replay_esn = kmemdup(orig->replay_esn, xfrm_replay_state_esn_len(orig->replay_esn), GFP_KERNEL); - if (!x->replay_esn) - return -ENOMEM; - x->preplay_esn = kmemdup(orig->preplay_esn, - xfrm_replay_state_esn_len(orig->preplay_esn), - GFP_KERNEL); - if (!x->preplay_esn) - return -ENOMEM; + if (!x->replay_esn) + return -ENOMEM; + x->preplay_esn = kmemdup(orig->preplay_esn, + xfrm_replay_state_esn_len(orig->preplay_esn), + GFP_KERNEL); + if (!x->preplay_esn) + return -ENOMEM; + } return 0; } +static inline void xfrm_replay_sync(struct xfrm_state *x, const struct xfrm_state *orig) +{ + x->replay = orig->replay; + x->preplay = orig->preplay; + + if (orig->replay_esn) { + memcpy(x->replay_esn, orig->replay_esn, + xfrm_replay_state_esn_len(orig->replay_esn)); + + memcpy(x->preplay_esn, orig->preplay_esn, + xfrm_replay_state_esn_len(orig->preplay_esn)); + } +} + +static inline void xfrm_migrate_sync(struct xfrm_state *x, + const struct xfrm_state *orig) +{ + /* called under lock so no race conditions or mallocs allowed */ + memcpy(&x->curlft, &orig->curlft, sizeof(x->curlft)); + xfrm_replay_sync(x, orig); +} + static inline struct xfrm_algo_aead *xfrm_algo_aead_clone(struct xfrm_algo_aead *orig) { return kmemdup(orig, aead_len(orig), GFP_KERNEL); @@ -2068,7 +2122,7 @@ void xfrm_dev_resume(struct sk_buff *skb); void xfrm_dev_backlog(struct softnet_data *sd); struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again); int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, - struct xfrm_user_offload *xuo, + const struct xfrm_user_offload *xuo, struct netlink_ext_ack *extack); int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp, struct xfrm_user_offload *xuo, u8 dir, @@ -2139,7 +2193,9 @@ static inline struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_fea return skb; } -static inline int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo, struct netlink_ext_ack *extack) +static inline int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, + const struct xfrm_user_offload *xuo, + struct netlink_ext_ack *extack) { return 0; } diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index 92a2358c6ce3..a7df573784fd 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -78,6 +78,9 @@ struct xsk_buff_pool { u32 chunk_size; u32 chunk_shift; u32 frame_len; + u32 tx_descs_nentries; + u32 reclaim_descs; + u32 tx_zc_pending_descs; u32 xdp_zc_max_segs; u8 tx_metadata_len; /* inherited from umem */ u8 cached_need_wakeup; @@ -90,11 +93,6 @@ struct xsk_buff_pool { * destructor callback. */ spinlock_t cq_prod_lock; - /* Mutual exclusion of the completion ring in the SKB mode. - * Protect: when sockets share a single cq when the same netdev - * and queue id is shared. - */ - spinlock_t cq_cached_prod_lock; struct xdp_buff_xsk *free_heads[]; }; @@ -107,12 +105,14 @@ struct xsk_buff_pool { /* AF_XDP core. */ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, - struct xdp_umem *umem); + struct xdp_umem *umem, + u32 max_segs); int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev, u16 queue_id, u16 flags); int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs, struct net_device *dev, u16 queue_id); -int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs); +int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs, + u32 max_segs); void xp_destroy(struct xsk_buff_pool *pool); void xp_get_pool(struct xsk_buff_pool *pool); bool xp_put_pool(struct xsk_buff_pool *pool); @@ -154,7 +154,8 @@ struct xdp_desc_ctx { struct xsk_tx_metadata *meta; }; -struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr); +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, + u32 options); static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb) { @@ -179,13 +180,6 @@ static inline void xp_dma_sync_for_device(struct xsk_buff_pool *pool, dma_sync_single_for_device(pool->dev, dma, size, DMA_BIDIRECTIONAL); } -/* Masks for xdp_umem_page flags. - * The low 12-bits of the addr will be 0 since this is the page address, so we - * can use them for flags. - */ -#define XSK_NEXT_PG_CONTIG_SHIFT 0 -#define XSK_NEXT_PG_CONTIG_MASK BIT_ULL(XSK_NEXT_PG_CONTIG_SHIFT) - static inline bool xp_desc_crosses_non_contig_pg(struct xsk_buff_pool *pool, u64 addr, u32 len) { |