diff options
Diffstat (limited to 'net/core')
37 files changed, 1087 insertions, 652 deletions
diff --git a/net/core/Makefile b/net/core/Makefile index dc17c5a61e9a..b3fdcb4e355f 100644 --- a/net/core/Makefile +++ b/net/core/Makefile @@ -13,7 +13,7 @@ obj-y += dev.o dev_api.o dev_addr_lists.o dst.o netevent.o \ neighbour.o rtnetlink.o utils.o link_watch.o filter.o \ sock_diag.o dev_ioctl.o tso.o sock_reuseport.o \ fib_notifier.o xdp.o flow_offload.o gro.o \ - netdev-genl.o netdev-genl-gen.o gso.o + netdev-genl.o netdev-genl-gen.o netdev_work.o gso.o obj-$(CONFIG_NETDEV_ADDR_LIST_TEST) += dev_addr_lists_test.o diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index 14eb7812bda4..ecd659f79fd4 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -172,7 +172,7 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) struct bpf_map *map; smap = rcu_dereference(SDATA(selem)->smap); - if (!(smap->map.map_flags & BPF_F_CLONE)) + if (!smap || !(smap->map.map_flags & BPF_F_CLONE)) continue; /* Note that for lockless listeners adding new element @@ -531,10 +531,10 @@ err_free: } EXPORT_SYMBOL_GPL(bpf_sk_storage_diag_alloc); -static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb) +static int diag_get(struct bpf_local_storage_map *smap, + struct bpf_local_storage_data *sdata, struct sk_buff *skb) { struct nlattr *nla_stg, *nla_value; - struct bpf_local_storage_map *smap; /* It cannot exceed max nlattr's payload */ BUILD_BUG_ON(U16_MAX - NLA_HDRLEN < BPF_LOCAL_STORAGE_MAX_VALUE_SIZE); @@ -543,7 +543,6 @@ static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb) if (!nla_stg) return -EMSGSIZE; - smap = rcu_dereference(sdata->smap); if (nla_put_u32(skb, SK_DIAG_BPF_STORAGE_MAP_ID, smap->map.id)) goto errout; @@ -558,6 +557,7 @@ static int diag_get(struct bpf_local_storage_data *sdata, struct sk_buff *skb) sdata->data, true); else copy_map_value(&smap->map, nla_data(nla_value), sdata->data); + check_and_init_map_value(&smap->map, nla_data(nla_value)); nla_nest_end(skb, nla_stg); return 0; @@ -596,9 +596,11 @@ static int bpf_sk_storage_diag_put_all(struct sock *sk, struct sk_buff *skb, saved_len = skb->len; hlist_for_each_entry_rcu(selem, &sk_storage->list, snode) { smap = rcu_dereference(SDATA(selem)->smap); + if (!smap) + continue; diag_size += nla_value_size(smap->map.value_size); - if (nla_stgs && diag_get(SDATA(selem), skb)) + if (nla_stgs && diag_get(smap, SDATA(selem), skb)) /* Continue to learn diag_size */ err = -EMSGSIZE; } @@ -665,7 +667,7 @@ int bpf_sk_storage_diag_put(struct bpf_sk_storage_diag *diag, diag_size += nla_value_size(diag->maps[i]->value_size); - if (nla_stgs && diag_get(sdata, skb)) + if (nla_stgs && diag_get((struct bpf_local_storage_map *)diag->maps[i], sdata, skb)) /* Continue to learn diag_size */ err = -EMSGSIZE; } diff --git a/net/core/dev.c b/net/core/dev.c index 06c195906231..4b3d5cfdf6e0 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -371,7 +371,7 @@ static void netdev_name_node_alt_free(struct rcu_head *head) static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node) { netdev_name_node_del(name_node); - list_del(&name_node->list); + list_del_rcu(&name_node->list); call_rcu(&name_node->rcu, netdev_name_node_alt_free); } @@ -1593,7 +1593,7 @@ EXPORT_SYMBOL(netdev_features_change); void netif_state_change(struct net_device *dev) { - netdev_ops_assert_locked_or_invisible(dev); + netdev_assert_locked_ops_compat_or_invisible(dev); if (dev->flags & IFF_UP) { struct netdev_notifier_change_info change_info = { @@ -1693,7 +1693,7 @@ static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack) set_bit(__LINK_STATE_START, &dev->state); - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); if (ops->ndo_validate_addr) ret = ops->ndo_validate_addr(dev); @@ -1770,11 +1770,12 @@ static void __dev_close_many(struct list_head *head) * event. */ - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); if (ops->ndo_stop) ops->ndo_stop(dev); + netif_rx_mode_cancel_retry(dev); netif_set_up(dev, false); netpoll_poll_enable(dev); } @@ -3198,7 +3199,7 @@ int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) if (dev->reg_state == NETREG_REGISTERED || dev->reg_state == NETREG_UNREGISTERING) { - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, txq); @@ -3247,7 +3248,7 @@ int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) return -EINVAL; if (dev->reg_state == NETREG_REGISTERED) { - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, rxq); @@ -3993,10 +3994,11 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, struct skb_shared_info *shinfo; struct net_iov *niov; - if (likely(skb_frags_readable(skb))) + if (likely(skb_frags_readable(skb) || + dev->netmem_tx == NETMEM_TX_NO_DMA)) goto out; - if (!dev->netmem_tx) + if (dev->netmem_tx == NETMEM_TX_NONE) goto out_free; shinfo = skb_shinfo(skb); @@ -5381,8 +5383,6 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu, u32 tail; reason = SKB_DROP_REASON_DEV_READY; - if (unlikely(!netif_running(skb->dev))) - goto bad_dev; sd = &per_cpu(softnet_data, cpu); @@ -5394,6 +5394,10 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu, backlog_lock_irq_save(sd, &flags); qlen = skb_queue_len(&sd->input_pkt_queue); if (likely(qlen <= max_backlog)) { + if (unlikely(!netif_running(skb->dev))) { + backlog_unlock_irq_restore(sd, flags); + goto bad_dev; + } if (!qlen) { /* Schedule NAPI for backlog device. We can use * non atomic operation as we own the queue lock. @@ -6862,20 +6866,6 @@ static void skb_defer_free_flush(void) #if defined(CONFIG_NET_RX_BUSY_POLL) -static void __busy_poll_stop(struct napi_struct *napi, bool skip_schedule) -{ - if (!skip_schedule) { - gro_normal_list(&napi->gro); - __napi_schedule(napi); - return; - } - - /* Flush too old packets. If HZ < 1000, flush all packets */ - gro_flush_normal(&napi->gro, HZ >= 1000); - - clear_bit(NAPI_STATE_SCHED, &napi->state); -} - enum { NAPI_F_PREFER_BUSY_POLL = 1, NAPI_F_END_ON_RESCHED = 2, @@ -6885,15 +6875,14 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, unsigned flags, u16 budget) { struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; - bool skip_schedule = false; - unsigned long timeout; + unsigned long timeout = 0; int rc; /* Busy polling means there is a high chance device driver hard irq * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was * set in napi_schedule_prep(). - * Since we are about to call napi->poll() once more, we can safely - * clear NAPI_STATE_MISSED. + * Since we either call napi->poll() once more or start the timer, + * we can safely clear NAPI_STATE_MISSED. * * Note: x86 could use a single "lock and ..." instruction * to perform these two clear_bit() @@ -6906,25 +6895,35 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, if (flags & NAPI_F_PREFER_BUSY_POLL) { napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi); - timeout = napi_get_gro_flush_timeout(napi); - if (napi->defer_hard_irqs_count && timeout) { - hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED); - skip_schedule = true; + if (napi->defer_hard_irqs_count) + timeout = napi_get_gro_flush_timeout(napi); + } + if (timeout) { + netpoll_poll_unlock(have_poll_lock); + + /* Drain aged GRO packets before clearing SCHED since the NAPI + * won't run again until after the timer fires. When HZ < 1000, + * GRO age comparison is too coarse, so flush everything. + */ + gro_flush_normal(&napi->gro, HZ >= 1000); + + clear_bit(NAPI_STATE_SCHED, &napi->state); + hrtimer_start(&napi->timer, ns_to_ktime(timeout), + HRTIMER_MODE_REL_PINNED); + } else { + /* Use driver poll to re-enable device interrupts. */ + rc = napi->poll(napi, budget); + /* Unless rc == budget we no longer own the NAPI instance, + * IRQ may fire on another CPU, poll this NAPI, and enter GRO. + */ + trace_napi_poll(napi, rc, budget); + netpoll_poll_unlock(have_poll_lock); + if (rc == budget) { + gro_normal_list(&napi->gro); + __napi_schedule(napi); } } - /* All we really want here is to re-enable device interrupts. - * Ideally, a new ndo_busy_poll_stop() could avoid another round. - */ - rc = napi->poll(napi, budget); - /* We can't gro_normal_list() here, because napi->poll() might have - * rearmed the napi (napi_complete_done()) in which case it could - * already be running on another CPU. - */ - trace_napi_poll(napi, rc, budget); - netpoll_poll_unlock(have_poll_lock); - if (rc == budget) - __busy_poll_stop(napi, skip_schedule); bpf_net_ctx_clear(bpf_net_ctx); local_bh_enable(); } @@ -7298,7 +7297,7 @@ void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index, if (WARN_ON_ONCE(napi && !napi->dev)) return; - netdev_ops_assert_locked_or_invisible(dev); + netdev_assert_locked_ops_compat_or_invisible(dev); switch (type) { case NETDEV_QUEUE_TYPE_RX: @@ -9593,7 +9592,7 @@ int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) kuid_t uid; kgid_t gid; - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); promiscuity = dev->promiscuity + inc; if (promiscuity == 0) { @@ -9652,7 +9651,7 @@ int netif_set_allmulti(struct net_device *dev, int inc, bool notify) unsigned int old_flags = dev->flags, old_gflags = dev->gflags; unsigned int allmulti, flags; - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); allmulti = dev->allmulti + inc; if (allmulti == 0) { @@ -9720,7 +9719,7 @@ int __dev_change_flags(struct net_device *dev, unsigned int flags, unsigned int old_flags = dev->flags; int ret; - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); /* * Set the flags on our device. @@ -9823,6 +9822,7 @@ int netif_change_flags(struct net_device *dev, unsigned int flags, __dev_notify_flags(dev, old_flags, changes, 0, NULL); return ret; } +EXPORT_SYMBOL(netif_change_flags); int __netif_set_mtu(struct net_device *dev, int new_mtu) { @@ -9868,7 +9868,7 @@ int netif_set_mtu_ext(struct net_device *dev, int new_mtu, { int err, orig_mtu; - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); if (new_mtu == dev->mtu) return 0; @@ -10141,17 +10141,36 @@ bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b) } EXPORT_SYMBOL(netdev_port_same_parent_id); +static struct net_device *dev_get_iflink_dev(struct net_device *dev) +{ + struct net *net; + + ASSERT_RTNL(); + + if (!dev->netdev_ops->ndo_get_iflink || !dev->rtnl_link_ops || + !dev->rtnl_link_ops->get_link_net) + return dev; + + net = dev->rtnl_link_ops->get_link_net(dev); + return __dev_get_by_index(net, dev_get_iflink(dev)); +} + int netif_change_proto_down(struct net_device *dev, bool proto_down) { + struct net_device *iflink_dev; + if (!dev->change_proto_down) return -EOPNOTSUPP; if (!netif_device_present(dev)) return -ENODEV; + iflink_dev = dev_get_iflink_dev(dev); + if (!iflink_dev) + return -ENODEV; + WRITE_ONCE(dev->proto_down, proto_down); if (proto_down) netif_carrier_off(dev); - else + else if (dev == iflink_dev || netif_carrier_ok(iflink_dev)) netif_carrier_on(dev); - WRITE_ONCE(dev->proto_down, proto_down); return 0; } @@ -10302,7 +10321,7 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode, struct netdev_bpf xdp; int err; - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && prog && !prog->aux->xdp_has_frags) { @@ -10754,7 +10773,7 @@ u32 dev_get_min_mp_channel_count(const struct net_device *dev) { int i; - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); for (i = dev->real_num_rx_queues - 1; i >= 0; i--) if (dev->_rx[i].mp_params.mp_priv) @@ -10982,7 +11001,7 @@ int __netdev_update_features(struct net_device *dev) int err = -1; ASSERT_RTNL(); - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); features = netdev_get_wanted_features(dev); @@ -11214,7 +11233,8 @@ static int netif_alloc_netdev_queues(struct net_device *dev) netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); spin_lock_init(&dev->tx_global_lock); - + spin_lock_init(&dev->watchdog_lock); + dev->watchdog_ref_held = false; return 0; } @@ -11715,6 +11735,8 @@ void netdev_run_todo(void) WARN_ON(rcu_access_pointer(dev->ip_ptr)); WARN_ON(rcu_access_pointer(dev->ip6_ptr)); + netdev_name_node_alt_flush(dev); + netdev_name_node_free(dev->name_node); netdev_do_free_pcpu_stats(dev); if (dev->priv_destructor) dev->priv_destructor(dev); @@ -12072,13 +12094,13 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, INIT_LIST_HEAD(&dev->ptype_all); INIT_LIST_HEAD(&dev->ptype_specific); INIT_LIST_HEAD(&dev->net_notifier_list); + INIT_LIST_HEAD(&dev->work_node); #ifdef CONFIG_NET_SCHED hash_init(dev->qdisc_hash); #endif mutex_init(&dev->lock); - INIT_LIST_HEAD(&dev->rx_mode_node); - __hw_addr_init(&dev->rx_mode_addr_cache); + netif_rx_mode_init(dev); dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; setup(dev); @@ -12428,8 +12450,6 @@ void unregister_netdevice_many_notify(struct list_head *head, dev_uc_flush(dev); dev_mc_flush(dev); - netdev_name_node_alt_flush(dev); - netdev_name_node_free(dev->name_node); netdev_rss_contexts_free(dev); diff --git a/net/core/dev.h b/net/core/dev.h index 0cf24b8f5008..5d0b0305d3ba 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -166,8 +166,19 @@ int dev_change_carrier(struct net_device *dev, bool new_carrier); void __dev_set_rx_mode(struct net_device *dev); int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify); -bool netif_rx_mode_clean(struct net_device *dev); +void netif_rx_mode_init(struct net_device *dev); +void netif_rx_mode_run(struct net_device *dev); void netif_rx_mode_sync(struct net_device *dev); +void netif_rx_mode_cancel_retry(struct net_device *dev); + +/* Events for the async netdev work, tracked in netdev->work_core_pending. */ +enum netdev_work_core { + NETDEV_WORK_RX_MODE = BIT(0), /* run the rx_mode update */ +}; + +void __netdev_work_core_sched(struct net_device *dev, unsigned long event); +unsigned long +__netdev_work_core_cancel(struct net_device *dev, unsigned long mask); void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, unsigned int gchanges, u32 portid, @@ -396,4 +407,10 @@ int dev_get_hwtstamp_phylib(struct net_device *dev, struct kernel_hwtstamp_config *cfg); int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg); +/* Caller holds RTNL, netdev->lock or RCU */ +static inline bool dev_isalive(const struct net_device *dev) +{ + return READ_ONCE(dev->reg_state) <= NETREG_REGISTERED; +} + #endif diff --git a/net/core/dev_addr_lists.c b/net/core/dev_addr_lists.c index d73fcb0c6785..08528ca0a8b3 100644 --- a/net/core/dev_addr_lists.c +++ b/net/core/dev_addr_lists.c @@ -12,17 +12,10 @@ #include <linux/export.h> #include <linux/list.h> #include <linux/spinlock.h> -#include <linux/workqueue.h> #include <kunit/visibility.h> #include "dev.h" -static void netdev_rx_mode_work(struct work_struct *work); - -static LIST_HEAD(rx_mode_list); -static DEFINE_SPINLOCK(rx_mode_lock); -static DECLARE_WORK(rx_mode_work, netdev_rx_mode_work); - /* * General list handling functions */ @@ -1252,7 +1245,36 @@ static int netif_uc_promisc_update(struct net_device *dev) return 0; } -static void netif_rx_mode_run(struct net_device *dev) +/* Total retry budget (4): 1+2+4+8 = 15 seconds */ +#define NETIF_RX_MODE_RETRY_MAX 4 + +void netif_rx_mode_schedule_retry(struct net_device *dev) +{ + unsigned long delay; + + netdev_assert_locked_ops_compat(dev); + + if (dev->rx_mode_retry_count >= NETIF_RX_MODE_RETRY_MAX) { + netdev_err(dev, "rx_mode retry limit reached, giving up\n"); + return; + } + + delay = HZ << dev->rx_mode_retry_count; + if (mod_timer(&dev->rx_mode_retry_timer, jiffies + delay)) + return; + if (!dev->rx_mode_retry_count) + netdev_info(dev, "rx_mode install failed, retrying with backoff\n"); + dev->rx_mode_retry_count++; +} +EXPORT_SYMBOL_GPL(netif_rx_mode_schedule_retry); + +void netif_rx_mode_cancel_retry(struct net_device *dev) +{ + timer_delete_sync(&dev->rx_mode_retry_timer); + dev->rx_mode_retry_count = 0; +} + +void netif_rx_mode_run(struct net_device *dev) { struct netdev_hw_addr_list uc_snap, mc_snap, uc_ref, mc_ref; const struct net_device_ops *ops = dev->netdev_ops; @@ -1260,7 +1282,7 @@ static void netif_rx_mode_run(struct net_device *dev) int err; might_sleep(); - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); __hw_addr_init(&uc_snap); __hw_addr_init(&mc_snap); @@ -1275,8 +1297,8 @@ static void netif_rx_mode_run(struct net_device *dev) err = netif_addr_lists_snapshot(dev, &uc_snap, &mc_snap, &uc_ref, &mc_ref); if (err) { - netdev_WARN(dev, "failed to sync uc/mc addresses\n"); netif_addr_unlock_bh(dev); + netif_rx_mode_schedule_retry(dev); return; } @@ -1292,12 +1314,17 @@ static void netif_rx_mode_run(struct net_device *dev) __dev_set_promiscuity(dev, promisc_inc, false); if (ops->ndo_set_rx_mode_async) { - ops->ndo_set_rx_mode_async(dev, &uc_snap, &mc_snap); + err = ops->ndo_set_rx_mode_async(dev, &uc_snap, &mc_snap); netif_addr_lock_bh(dev); netif_addr_lists_reconcile(dev, &uc_snap, &mc_snap, &uc_ref, &mc_ref); netif_addr_unlock_bh(dev); + + if (err) + netif_rx_mode_schedule_retry(dev); + else + dev->rx_mode_retry_count = 0; } else if (ops->ndo_set_rx_mode) { netif_addr_lock_bh(dev); ops->ndo_set_rx_mode(dev); @@ -1305,49 +1332,23 @@ static void netif_rx_mode_run(struct net_device *dev) } } -static void netdev_rx_mode_work(struct work_struct *work) +static void netif_rx_mode_queue(struct net_device *dev) { - struct net_device *dev; - - rtnl_lock(); - - while (true) { - spin_lock_bh(&rx_mode_lock); - if (list_empty(&rx_mode_list)) { - spin_unlock_bh(&rx_mode_lock); - break; - } - dev = list_first_entry(&rx_mode_list, struct net_device, - rx_mode_node); - list_del_init(&dev->rx_mode_node); - /* We must free netdev tracker under - * the spinlock protection. - */ - netdev_tracker_free(dev, &dev->rx_mode_tracker); - spin_unlock_bh(&rx_mode_lock); + __netdev_work_core_sched(dev, NETDEV_WORK_RX_MODE); +} - netdev_lock_ops(dev); - netif_rx_mode_run(dev); - netdev_unlock_ops(dev); - /* Use __dev_put() because netdev_tracker_free() was already - * called above. Must be after netdev_unlock_ops() to prevent - * netdev_run_todo() from freeing the device while still in use. - */ - __dev_put(dev); - } +static void netif_rx_mode_retry(struct timer_list *t) +{ + struct net_device *dev = + timer_container_of(dev, t, rx_mode_retry_timer); - rtnl_unlock(); + netif_rx_mode_queue(dev); } -static void netif_rx_mode_queue(struct net_device *dev) +void netif_rx_mode_init(struct net_device *dev) { - spin_lock_bh(&rx_mode_lock); - if (list_empty(&dev->rx_mode_node)) { - list_add_tail(&dev->rx_mode_node, &rx_mode_list); - netdev_hold(dev, &dev->rx_mode_tracker, GFP_ATOMIC); - } - spin_unlock_bh(&rx_mode_lock); - schedule_work(&rx_mode_work); + __hw_addr_init(&dev->rx_mode_addr_cache); + timer_setup(&dev->rx_mode_retry_timer, netif_rx_mode_retry, 0); } /** @@ -1393,24 +1394,6 @@ void dev_set_rx_mode(struct net_device *dev) netif_addr_unlock_bh(dev); } -bool netif_rx_mode_clean(struct net_device *dev) -{ - bool clean = false; - - spin_lock_bh(&rx_mode_lock); - if (!list_empty(&dev->rx_mode_node)) { - list_del_init(&dev->rx_mode_node); - clean = true; - /* We must release netdev tracker under - * the spinlock protection. - */ - netdev_tracker_free(dev, &dev->rx_mode_tracker); - } - spin_unlock_bh(&rx_mode_lock); - - return clean; -} - /** * netif_rx_mode_sync() - sync rx mode inline * @dev: network device @@ -1424,11 +1407,6 @@ bool netif_rx_mode_clean(struct net_device *dev) */ void netif_rx_mode_sync(struct net_device *dev) { - if (netif_rx_mode_clean(dev)) { + if (__netdev_work_core_cancel(dev, NETDEV_WORK_RX_MODE)) netif_rx_mode_run(dev); - /* Use __dev_put() because netdev_tracker_free() was already - * called inside netif_rx_mode_clean(). - */ - __dev_put(dev); - } } diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index f3979b276090..a320e264eaaf 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -260,7 +260,7 @@ int dev_get_hwtstamp_phylib(struct net_device *dev, { struct hwtstamp_provider *hwprov; - hwprov = rtnl_dereference(dev->hwprov); + hwprov = netdev_ops_lock_dereference(dev->hwprov, dev); if (hwprov) { cfg->qualifier = hwprov->desc.qualifier; if (hwprov->source == HWTSTAMP_SOURCE_PHYLIB && @@ -337,7 +337,7 @@ int dev_set_hwtstamp_phylib(struct net_device *dev, bool phy_ts; int err; - hwprov = rtnl_dereference(dev->hwprov); + hwprov = netdev_ops_lock_dereference(dev->hwprov, dev); if (hwprov) { if (hwprov->source == HWTSTAMP_SOURCE_PHYLIB && hwprov->phydev) { diff --git a/net/core/devmem.c b/net/core/devmem.c index cde4c89bc146..957d6b96216b 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -181,7 +181,7 @@ err_close_rxq: } struct net_devmem_dmabuf_binding * -net_devmem_bind_dmabuf(struct net_device *dev, +net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, struct netdev_nl_sock *priv, @@ -212,6 +212,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, } binding->dev = dev; + binding->vdev = vdev; xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC); err = percpu_ref_init(&binding->ref, @@ -241,6 +242,11 @@ net_devmem_bind_dmabuf(struct net_device *dev, } if (direction == DMA_TO_DEVICE) { + if (!IS_ALIGNED(dmabuf->size, PAGE_SIZE)) { + err = -EINVAL; + NL_SET_ERR_MSG(extack, "TX dma-buf size must be a multiple of PAGE_SIZE"); + goto err_unmap; + } binding->tx_vec = kvmalloc_objs(struct net_iov *, dmabuf->size / PAGE_SIZE); if (!binding->tx_vec) { @@ -267,6 +273,12 @@ net_devmem_bind_dmabuf(struct net_device *dev, size_t len = sg_dma_len(sg); struct net_iov *niov; + if (!IS_ALIGNED(len, PAGE_SIZE)) { + err = -EINVAL; + NL_SET_ERR_MSG(extack, "dma-buf SG length must be PAGE_SIZE aligned"); + goto err_free_chunks; + } + owner = kzalloc_node(sizeof(*owner), GFP_KERNEL, dev_to_node(&dev->dev)); if (!owner) { @@ -297,8 +309,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, for (i = 0; i < owner->area.num_niovs; i++) { niov = &owner->area.niovs[i]; - niov->type = NET_IOV_DMABUF; - niov->owner = &owner->area; + net_iov_init(niov, &owner->area, NET_IOV_DMABUF); page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov), net_devmem_get_dma_addr(niov)); if (direction == DMA_TO_DEVICE) @@ -397,7 +408,8 @@ struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, */ dst_dev = dst_dev_rcu(dst); if (unlikely(!dst_dev) || - unlikely(dst_dev != READ_ONCE(binding->dev))) { + unlikely(dst_dev != READ_ONCE(binding->dev) && + dst_dev != READ_ONCE(binding->vdev))) { err = -ENODEV; goto out_unlock; } diff --git a/net/core/devmem.h b/net/core/devmem.h index 1c5c18581fcb..3852a56036cb 100644 --- a/net/core/devmem.h +++ b/net/core/devmem.h @@ -19,7 +19,13 @@ struct net_devmem_dmabuf_binding { struct dma_buf *dmabuf; struct dma_buf_attachment *attachment; struct sg_table *sgt; + /* Physical NIC that does the actual DMA for this binding. */ struct net_device *dev; + /* Opaque cookie identifying the virtual device (e.g. netkit) the user + * called bind-tx on. Used only for pointer comparison. Never + * dereferenced. + */ + void *vdev; struct gen_pool *chunk_pool; /* Protect dev */ struct mutex lock; @@ -84,7 +90,7 @@ struct dmabuf_genpool_chunk_owner { void __net_devmem_dmabuf_binding_free(struct work_struct *wq); struct net_devmem_dmabuf_binding * -net_devmem_bind_dmabuf(struct net_device *dev, +net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, struct netdev_nl_sock *priv, @@ -165,7 +171,7 @@ static inline void net_devmem_put_net_iov(struct net_iov *niov) } static inline struct net_devmem_dmabuf_binding * -net_devmem_bind_dmabuf(struct net_device *dev, +net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, struct device *dma_dev, enum dma_data_direction direction, unsigned int dmabuf_fd, diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index f23cea9e1aaf..2bf3cab5e557 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -110,7 +110,7 @@ struct net_dm_alert_ops { void (*kfree_skb_probe)(void *ignore, struct sk_buff *skb, void *location, enum skb_drop_reason reason, - struct sock *rx_sk); + const struct sock *rx_sk); void (*napi_poll_probe)(void *ignore, struct napi_struct *napi, int work, int budget); void (*work_item_func)(struct work_struct *work); @@ -266,7 +266,7 @@ out: static void trace_kfree_skb_hit(void *ignore, struct sk_buff *skb, void *location, enum skb_drop_reason reason, - struct sock *rx_sk) + const struct sock *rx_sk) { trace_drop_common(skb, location); } @@ -493,7 +493,7 @@ static void net_dm_packet_trace_kfree_skb_hit(void *ignore, struct sk_buff *skb, void *location, enum skb_drop_reason reason, - struct sock *rx_sk) + const struct sock *rx_sk) { ktime_t tstamp = ktime_get_real(); struct per_cpu_dm_data *data; diff --git a/net/core/failover.c b/net/core/failover.c index 11bb183c7a1b..e43c59cd6868 100644 --- a/net/core/failover.c +++ b/net/core/failover.c @@ -12,6 +12,7 @@ #include <uapi/linux/if_arp.h> #include <linux/rtnetlink.h> #include <linux/if_vlan.h> +#include <net/netdev_lock.h> #include <net/failover.h> static LIST_HEAD(failover_list); @@ -221,8 +222,11 @@ failover_existing_slave_register(struct net_device *failover_dev) for_each_netdev(net, dev) { if (netif_is_failover(dev)) continue; - if (ether_addr_equal(failover_dev->perm_addr, dev->perm_addr)) + if (ether_addr_equal(failover_dev->perm_addr, dev->perm_addr)) { + netdev_lock_ops(dev); failover_slave_register(dev); + netdev_unlock_ops(dev); + } } rtnl_unlock(); } diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c index 8ca634964e36..cf374c208732 100644 --- a/net/core/fib_rules.c +++ b/net/core/fib_rules.c @@ -349,7 +349,7 @@ jumped: if (err != -EAGAIN) { if ((arg->flags & FIB_LOOKUP_NOREF) || - likely(refcount_inc_not_zero(&rule->refcnt))) { + likely(fib_rule_get_safe(rule))) { arg->rule = rule; goto out; } @@ -410,8 +410,12 @@ int fib_rules_dump(struct net *net, struct notifier_block *nb, int family, if (!ops) return -EAFNOSUPPORT; list_for_each_entry_rcu(rule, &ops->rules_list, list) { + if (!fib_rule_get_safe(rule)) + continue; + err = call_fib_rule_notifier(nb, FIB_EVENT_RULE_ADD, rule, family, extack); + fib_rule_put(rule); if (err) break; } diff --git a/net/core/filter.c b/net/core/filter.c index 80a3b702a2d4..b446aa8be5c3 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -1654,15 +1654,24 @@ err_prog_put: return err; } +static void sk_reuseport_prog_free_rcu(struct rcu_head *rcu) +{ + struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu); + struct bpf_prog *prog = aux->prog; + + bpf_release_orig_filter(prog); + bpf_prog_free(prog); +} + void sk_reuseport_prog_free(struct bpf_prog *prog) { if (!prog) return; - if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) - bpf_prog_put(prog); + if (bpf_prog_was_classic(prog)) + call_rcu(&prog->aux->rcu, sk_reuseport_prog_free_rcu); else - bpf_prog_destroy(prog); + bpf_prog_put(prog); } static inline int __bpf_try_make_writable(struct sk_buff *skb, @@ -2645,6 +2654,37 @@ static void sk_msg_reset_curr(struct sk_msg *msg) } } +static bool sk_msg_elem_is_copy(const struct sk_msg *msg, u32 i) +{ + return test_bit(i, msg->sg.copy); +} + +static void sk_msg_clear_elem_copy(struct sk_msg *msg, u32 i) +{ + __clear_bit(i, msg->sg.copy); +} + +static void sk_msg_set_elem_copy(struct sk_msg *msg, u32 i, bool sg_copy) +{ + __assign_bit(i, msg->sg.copy, sg_copy); +} + +static void sk_msg_clear_copy_range(struct sk_msg *msg, u32 start, u32 end) +{ + while (start != end) { + sk_msg_clear_elem_copy(msg, start); + sk_msg_iter_var_next(start); + } +} + +static void sk_msg_sg_move(struct sk_msg *msg, u32 dst, u32 src) +{ + msg->sg.data[dst] = msg->sg.data[src]; + + sk_msg_set_elem_copy(msg, dst, + sk_msg_elem_is_copy(msg, src)); +} + static const struct bpf_func_proto bpf_msg_cork_bytes_proto = { .func = bpf_msg_cork_bytes, .gpl_only = false, @@ -2683,7 +2723,7 @@ BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start, * account for the headroom. */ bytes_sg_total = start - offset + bytes; - if (!test_bit(i, msg->sg.copy) && bytes_sg_total <= len) + if (!sk_msg_elem_is_copy(msg, i) && bytes_sg_total <= len) goto out; /* At this point we need to linearize multiple scatterlist @@ -2724,11 +2764,13 @@ BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start, poffset += len; sge->length = 0; put_page(sg_page(sge)); + sk_msg_clear_elem_copy(msg, i); sk_msg_iter_var_next(i); } while (i != last_sge); sg_set_page(&msg->sg.data[first_sge], page, copy, 0); + sk_msg_clear_elem_copy(msg, first_sge); /* To repair sg ring we need to shift entries. If we only * had a single entry though we can just replace it and @@ -2738,8 +2780,14 @@ BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start, shift = last_sge > first_sge ? last_sge - first_sge - 1 : NR_MSG_FRAG_IDS - first_sge + last_sge - 1; - if (!shift) + if (!shift) { + sk_msg_clear_elem_copy(msg, msg->sg.end); goto out; + } + + i = first_sge; + sk_msg_iter_var_next(i); + sk_msg_clear_copy_range(msg, i, last_sge); i = first_sge; sk_msg_iter_var_next(i); @@ -2753,16 +2801,18 @@ BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start, if (move_from == msg->sg.end) break; - msg->sg.data[i] = msg->sg.data[move_from]; + sk_msg_sg_move(msg, i, move_from); msg->sg.data[move_from].length = 0; msg->sg.data[move_from].page_link = 0; msg->sg.data[move_from].offset = 0; + sk_msg_clear_elem_copy(msg, move_from); sk_msg_iter_var_next(i); } while (1); msg->sg.end = msg->sg.end - shift > msg->sg.end ? msg->sg.end - shift + NR_MSG_FRAG_IDS : msg->sg.end - shift; + sk_msg_clear_elem_copy(msg, msg->sg.end); out: sk_msg_reset_curr(msg); msg->data = sg_virt(&msg->sg.data[first_sge]) + start - offset; @@ -2783,8 +2833,10 @@ static const struct bpf_func_proto bpf_msg_pull_data_proto = { BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, u32, len, u64, flags) { + bool sge_copy = false, nsge_copy = false, nnsge_copy = false; struct scatterlist sge, nsge, nnsge, rsge = {0}, *psge; u32 new, i = 0, l = 0, space, copy = 0, offset = 0; + bool rsge_copy = false; u8 *raw, *to, *from; struct page *page; @@ -2820,6 +2872,9 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, if (!space || (space == 1 && start != offset)) copy = msg->sg.data[i].length; + if (unlikely(copy + len < copy)) + return -EINVAL; + page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP, get_order(copy + len)); if (unlikely(!page)) @@ -2857,10 +2912,11 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, sk_msg_iter_var_prev(i); psge = sk_msg_elem(msg, i); rsge = sk_msg_elem_cpy(msg, i); + rsge_copy = sk_msg_elem_is_copy(msg, i); psge->length = start - offset; rsge.length -= psge->length; - rsge.offset += start; + rsge.offset += start - offset; sk_msg_iter_var_next(i); sg_unmark_end(psge); @@ -2882,23 +2938,31 @@ BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start, /* Shift one or two slots as needed */ sge = sk_msg_elem_cpy(msg, new); sg_unmark_end(&sge); + sge_copy = sk_msg_elem_is_copy(msg, new); nsge = sk_msg_elem_cpy(msg, i); + nsge_copy = sk_msg_elem_is_copy(msg, i); if (rsge.length) { sk_msg_iter_var_next(i); nnsge = sk_msg_elem_cpy(msg, i); + nnsge_copy = sk_msg_elem_is_copy(msg, i); sk_msg_iter_next(msg, end); } while (i != msg->sg.end) { msg->sg.data[i] = sge; + sk_msg_set_elem_copy(msg, i, sge_copy); sge = nsge; + sge_copy = nsge_copy; sk_msg_iter_var_next(i); if (rsge.length) { nsge = nnsge; + nsge_copy = nnsge_copy; nnsge = sk_msg_elem_cpy(msg, i); + nnsge_copy = sk_msg_elem_is_copy(msg, i); } else { nsge = sk_msg_elem_cpy(msg, i); + nsge_copy = sk_msg_elem_is_copy(msg, i); } } @@ -2906,13 +2970,15 @@ place_new: /* Place newly allocated data buffer */ sk_mem_charge(msg->sk, len); msg->sg.size += len; - __clear_bit(new, msg->sg.copy); + sk_msg_clear_elem_copy(msg, new); sg_set_page(&msg->sg.data[new], page, len + copy, 0); if (rsge.length) { get_page(sg_page(&rsge)); sk_msg_iter_var_next(new); msg->sg.data[new] = rsge; + sk_msg_set_elem_copy(msg, new, rsge_copy); } + sk_msg_clear_elem_copy(msg, msg->sg.end); sk_msg_reset_curr(msg); sk_msg_compute_data_pointers(msg); @@ -2938,35 +3004,43 @@ static void sk_msg_shift_left(struct sk_msg *msg, int i) do { prev = i; sk_msg_iter_var_next(i); - msg->sg.data[prev] = msg->sg.data[i]; + sk_msg_sg_move(msg, prev, i); } while (i != msg->sg.end); sk_msg_iter_prev(msg, end); + sk_msg_clear_elem_copy(msg, msg->sg.end); } static void sk_msg_shift_right(struct sk_msg *msg, int i) { struct scatterlist tmp, sge; + bool tmp_copy, sge_copy; sk_msg_iter_next(msg, end); sge = sk_msg_elem_cpy(msg, i); + sge_copy = sk_msg_elem_is_copy(msg, i); sk_msg_iter_var_next(i); tmp = sk_msg_elem_cpy(msg, i); + tmp_copy = sk_msg_elem_is_copy(msg, i); while (i != msg->sg.end) { msg->sg.data[i] = sge; + sk_msg_set_elem_copy(msg, i, sge_copy); sk_msg_iter_var_next(i); sge = tmp; + sge_copy = tmp_copy; tmp = sk_msg_elem_cpy(msg, i); + tmp_copy = sk_msg_elem_is_copy(msg, i); } + sk_msg_clear_elem_copy(msg, msg->sg.end); } BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start, u32, len, u64, flags) { u32 i = 0, l = 0, space, offset = 0; - u64 last = start + len; - int pop; + u64 last = (u64)start + len; + u32 pop; if (unlikely(flags)) return -EINVAL; @@ -3015,8 +3089,10 @@ BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start, */ if (start != offset) { struct scatterlist *nsge, *sge = sk_msg_elem(msg, i); + bool sge_copy = sk_msg_elem_is_copy(msg, i); int a = start - offset; int b = sge->length - pop - a; + u32 sge_idx = i; sk_msg_iter_var_next(i); @@ -3029,6 +3105,7 @@ BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start, sg_set_page(nsge, sg_page(sge), b, sge->offset + pop + a); + sk_msg_set_elem_copy(msg, i, sge_copy); } else { struct page *page, *orig; u8 *to, *from; @@ -3045,6 +3122,7 @@ BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start, memcpy(to, from, a); memcpy(to + a, from + a + pop, b); sg_set_page(sge, page, a + b, 0); + sk_msg_clear_elem_copy(msg, sge_idx); put_page(orig); } pop = 0; @@ -4394,7 +4472,7 @@ u32 xdp_master_redirect(struct xdp_buff *xdp) struct net_device *master, *slave; master = netdev_master_upper_dev_get_rcu(xdp->rxq->dev); - if (unlikely(!(master->flags & IFF_UP))) + if (unlikely(!master || !(master->flags & IFF_UP))) return XDP_ABORTED; slave = master->netdev_ops->ndo_xdp_get_xmit_slave(master, xdp); if (slave && slave != xdp->rxq->dev) { @@ -5481,7 +5559,7 @@ static int sol_tcp_sockopt(struct sock *sk, int optname, char *optval, int *optlen, bool getopt) { - if (sk->sk_protocol != IPPROTO_TCP) + if (!sk_is_tcp(sk)) return -EINVAL; switch (optname) { @@ -5535,11 +5613,24 @@ static int sol_tcp_sockopt(struct sock *sk, int optname, KERNEL_SOCKPTR(optval), *optlen); } +static bool sk_allows_sol_ip_sockopt(struct sock *sk) +{ + switch (sk->sk_family) { + case AF_INET: + return true; + case AF_INET6: + /* Allow getting/setting sockopt for possible ipv4-mapped ipv6 socket. */ + return sk->sk_type != SOCK_RAW && !ipv6_only_sock(sk); + default: + return false; + } +} + static int sol_ip_sockopt(struct sock *sk, int optname, char *optval, int *optlen, bool getopt) { - if (sk->sk_family != AF_INET) + if (!sk_allows_sol_ip_sockopt(sk)) return -EINVAL; switch (optname) { @@ -5688,6 +5779,30 @@ const struct bpf_func_proto bpf_sk_getsockopt_proto = { .arg5_type = ARG_CONST_SIZE, }; +BPF_CALL_5(bpf_sk_setsockopt_nodelay, struct sock *, sk, int, level, + int, optname, char *, optval, int, optlen) +{ + /* + * TCP_NODELAY triggers tcp_push_pending_frames() and re-enters + * CA_EVENT_TX_START in bpf_tcp_cc. + */ + if (level == SOL_TCP && optname == TCP_NODELAY) + return -EOPNOTSUPP; + + return _bpf_setsockopt(sk, level, optname, optval, optlen); +} + +const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto = { + .func = bpf_sk_setsockopt_nodelay, + .gpl_only = false, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, + .arg4_type = ARG_PTR_TO_MEM | MEM_RDONLY, + .arg5_type = ARG_CONST_SIZE, +}; + BPF_CALL_5(bpf_unlocked_sk_setsockopt, struct sock *, sk, int, level, int, optname, char *, optval, int, optlen) { @@ -5833,6 +5948,12 @@ BPF_CALL_5(bpf_sock_ops_setsockopt, struct bpf_sock_ops_kern *, bpf_sock, if (!is_locked_tcp_sock_ops(bpf_sock)) return -EOPNOTSUPP; + /* TCP_NODELAY triggers tcp_push_pending_frames() and re-enters these callbacks. */ + if ((bpf_sock->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB || + bpf_sock->op == BPF_SOCK_OPS_WRITE_HDR_OPT_CB) && + level == SOL_TCP && optname == TCP_NODELAY) + return -EOPNOTSUPP; + return _bpf_setsockopt(bpf_sock->sk, level, optname, optval, optlen); } @@ -6100,7 +6221,7 @@ static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params, struct in_device *in_dev; struct net_device *dev; struct fib_result res; - struct flowi4 fl4; + struct flowi4 fl4 = {}; u32 mtu = 0; int err; @@ -6240,7 +6361,7 @@ static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params, struct neighbour *neigh; struct net_device *dev; struct inet6_dev *idev; - struct flowi6 fl6; + struct flowi6 fl6 = {}; int strict = 0; int oif, err; u32 mtu = 0; @@ -6443,6 +6564,8 @@ BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb, * against MTU of FIB lookup resulting net_device */ dev = dev_get_by_index_rcu(net, params->ifindex); + if (unlikely(!dev)) + return -ENODEV; if (!is_skb_forwardable(dev, skb)) rc = BPF_FIB_LKUP_RET_FRAG_NEEDED; @@ -7443,7 +7566,7 @@ u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, BPF_CALL_1(bpf_tcp_sock, struct sock *, sk) { - if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP) + if (sk_fullsock(sk) && sk_is_tcp(sk)) return (unsigned long)sk; return (unsigned long)NULL; @@ -11915,7 +12038,7 @@ BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk) */ BTF_TYPE_EMIT(struct tcp6_sock); if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP && - sk->sk_family == AF_INET6) + sk->sk_type == SOCK_STREAM && sk->sk_family == AF_INET6) return (unsigned long)sk; return (unsigned long)NULL; @@ -11931,7 +12054,7 @@ const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = { BPF_CALL_1(bpf_skc_to_tcp_sock, struct sock *, sk) { - if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP) + if (sk && sk_fullsock(sk) && sk_is_tcp(sk)) return (unsigned long)sk; return (unsigned long)NULL; diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 2a98f5fa74eb..8aa4f9b4df81 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -1173,13 +1173,21 @@ bool __skb_flow_dissect(const struct net *net, if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { - struct ethhdr *eth = eth_hdr(skb); struct flow_dissector_key_eth_addrs *key_eth_addrs; key_eth_addrs = skb_flow_dissector_target(flow_dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS, target_container); - memcpy(key_eth_addrs, eth, sizeof(*key_eth_addrs)); + /* TC filter blocks can be shared across devices with + * different link types, so we cannot validate this + * when the filter is installed -- check at dissect time. + */ + if (skb && skb->dev && + skb->dev->type == ARPHRD_ETHER && + skb_mac_header_was_set(skb)) + memcpy(key_eth_addrs, eth_hdr(skb), sizeof(*key_eth_addrs)); + else + memset(key_eth_addrs, 0, sizeof(*key_eth_addrs)); } if (dissector_uses_key(flow_dissector, diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c index b71ccaec0991..9468bc5ca3f4 100644 --- a/net/core/gen_stats.c +++ b/net/core/gen_stats.c @@ -124,13 +124,13 @@ void gnet_stats_basic_sync_init(struct gnet_stats_basic_sync *b) EXPORT_SYMBOL(gnet_stats_basic_sync_init); static void gnet_stats_add_basic_cpu(struct gnet_stats_basic_sync *bstats, - struct gnet_stats_basic_sync __percpu *cpu) + const struct gnet_stats_basic_sync __percpu *cpu) { u64 t_bytes = 0, t_packets = 0; int i; for_each_possible_cpu(i) { - struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i); + const struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i); unsigned int start; u64 bytes, packets; @@ -147,8 +147,8 @@ static void gnet_stats_add_basic_cpu(struct gnet_stats_basic_sync *bstats, } void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats, - struct gnet_stats_basic_sync __percpu *cpu, - struct gnet_stats_basic_sync *b, bool running) + const struct gnet_stats_basic_sync __percpu *cpu, + const struct gnet_stats_basic_sync *b, bool running) { unsigned int start; u64 bytes = 0; @@ -172,8 +172,8 @@ void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats, EXPORT_SYMBOL(gnet_stats_add_basic); static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets, - struct gnet_stats_basic_sync __percpu *cpu, - struct gnet_stats_basic_sync *b, bool running) + const struct gnet_stats_basic_sync __percpu *cpu, + const struct gnet_stats_basic_sync *b, bool running) { unsigned int start; @@ -182,8 +182,7 @@ static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets, int i; for_each_possible_cpu(i) { - struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i); - unsigned int start; + const struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i); u64 bytes, packets; do { @@ -209,8 +208,8 @@ static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets, static int ___gnet_stats_copy_basic(struct gnet_dump *d, - struct gnet_stats_basic_sync __percpu *cpu, - struct gnet_stats_basic_sync *b, + const struct gnet_stats_basic_sync __percpu *cpu, + const struct gnet_stats_basic_sync *b, int type, bool running) { u64 bstats_bytes, bstats_packets; @@ -258,8 +257,8 @@ ___gnet_stats_copy_basic(struct gnet_dump *d, */ int gnet_stats_copy_basic(struct gnet_dump *d, - struct gnet_stats_basic_sync __percpu *cpu, - struct gnet_stats_basic_sync *b, + const struct gnet_stats_basic_sync __percpu *cpu, + const struct gnet_stats_basic_sync *b, bool running) { return ___gnet_stats_copy_basic(d, cpu, b, TCA_STATS_BASIC, running); @@ -285,8 +284,8 @@ EXPORT_SYMBOL(gnet_stats_copy_basic); */ int gnet_stats_copy_basic_hw(struct gnet_dump *d, - struct gnet_stats_basic_sync __percpu *cpu, - struct gnet_stats_basic_sync *b, + const struct gnet_stats_basic_sync __percpu *cpu, + const struct gnet_stats_basic_sync *b, bool running) { return ___gnet_stats_copy_basic(d, cpu, b, TCA_STATS_BASIC_HW, running); @@ -345,11 +344,11 @@ static void gnet_stats_add_queue_cpu(struct gnet_stats_queue *qstats, for_each_possible_cpu(i) { const struct gnet_stats_queue *qcpu = per_cpu_ptr(q, i); - qstats->qlen += qcpu->qlen; - qstats->backlog += qcpu->backlog; - qstats->drops += qcpu->drops; - qstats->requeues += qcpu->requeues; - qstats->overlimits += qcpu->overlimits; + qstats->qlen += READ_ONCE(qcpu->qlen); + qstats->backlog += READ_ONCE(qcpu->backlog); + qstats->drops += READ_ONCE(qcpu->drops); + qstats->requeues += READ_ONCE(qcpu->requeues); + qstats->overlimits += READ_ONCE(qcpu->overlimits); } } @@ -360,11 +359,11 @@ void gnet_stats_add_queue(struct gnet_stats_queue *qstats, if (cpu) { gnet_stats_add_queue_cpu(qstats, cpu); } else { - qstats->qlen += q->qlen; - qstats->backlog += q->backlog; - qstats->drops += q->drops; - qstats->requeues += q->requeues; - qstats->overlimits += q->overlimits; + qstats->qlen += READ_ONCE(q->qlen); + qstats->backlog += READ_ONCE(q->backlog); + qstats->drops += READ_ONCE(q->drops); + qstats->requeues += READ_ONCE(q->requeues); + qstats->overlimits += READ_ONCE(q->overlimits); } } EXPORT_SYMBOL(gnet_stats_add_queue); @@ -385,8 +384,8 @@ EXPORT_SYMBOL(gnet_stats_add_queue); */ int gnet_stats_copy_queue(struct gnet_dump *d, - struct gnet_stats_queue __percpu *cpu_q, - struct gnet_stats_queue *q, __u32 qlen) + const struct gnet_stats_queue __percpu *cpu_q, + const struct gnet_stats_queue *q, __u32 qlen) { struct gnet_stats_queue qstats = {0}; diff --git a/net/core/gro.c b/net/core/gro.c index 31d21de5b15a..35f2f708f010 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -109,6 +109,9 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) if (p->pp_recycle != skb->pp_recycle) return -ETOOMANYREFS; + if (skb_zcopy(p) || skb_zcopy(skb)) + return -ETOOMANYREFS; + if (unlikely(p->len + len >= netif_get_gro_max_size(p->dev, p) || NAPI_GRO_CB(skb)->flush)) return -E2BIG; @@ -213,10 +216,12 @@ done: p->data_len += len; p->truesize += delta_truesize; p->len += len; + skb_shinfo(p)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG; if (lp != p) { lp->data_len += len; lp->truesize += delta_truesize; lp->len += len; + skb_shinfo(lp)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG; } NAPI_GRO_CB(skb)->same_flow = 1; return 0; @@ -227,6 +232,11 @@ int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) if (unlikely(p->len + skb->len >= 65536)) return -E2BIG; + if (!pskb_may_pull(skb, skb_gro_offset(skb))) { + NAPI_GRO_CB(skb)->flush = 1; + return -ENOMEM; + } + if (NAPI_GRO_CB(p)->last == p) skb_shinfo(p)->frag_list = skb; else @@ -244,6 +254,8 @@ int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) p->truesize += skb->truesize; p->len += skb->len; + skb_shinfo(p)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; + NAPI_GRO_CB(skb)->same_flow = 1; return 0; diff --git a/net/core/link_watch.c b/net/core/link_watch.c index ff2c1d4538ef..9c35aac8b2e9 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c @@ -277,7 +277,7 @@ static bool linkwatch_clean_dev(struct net_device *dev) void __linkwatch_sync_dev(struct net_device *dev) { - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); if (linkwatch_clean_dev(dev)) { linkwatch_do_dev(dev); diff --git a/net/core/lock_debug.c b/net/core/lock_debug.c index 9e9fb25314b9..8a81c5430705 100644 --- a/net/core/lock_debug.c +++ b/net/core/lock_debug.c @@ -24,7 +24,7 @@ int netdev_debug_event(struct notifier_block *nb, unsigned long event, case NETDEV_CHANGE: case NETDEV_REGISTER: case NETDEV_UP: - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); fallthrough; case NETDEV_DOWN: case NETDEV_REBOOT: @@ -65,6 +65,7 @@ int netdev_debug_event(struct notifier_block *nb, unsigned long event, break; case NETDEV_CHANGENAME: + netdev_assert_locked_ops(dev); ASSERT_RTNL_NET(net); break; } diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c index f71ef82a5f3d..bf588f508b79 100644 --- a/net/core/lwt_bpf.c +++ b/net/core/lwt_bpf.c @@ -599,6 +599,7 @@ static int handle_gso_encap(struct sk_buff *skb, bool ipv4, int encap_len) int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, bool ingress) { + bool is_udp_tunnel; struct iphdr *iph; bool ipv4; int err; @@ -612,10 +613,16 @@ int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, bool ingress) ipv4 = true; if (unlikely(len < iph->ihl * 4)) return -EINVAL; + is_udp_tunnel = iph->protocol == IPPROTO_UDP; + if (unlikely(is_udp_tunnel && len < iph->ihl * 4 + sizeof(struct udphdr))) + return -EINVAL; } else if (iph->version == 6) { ipv4 = false; if (unlikely(len < sizeof(struct ipv6hdr))) return -EINVAL; + is_udp_tunnel = ((struct ipv6hdr *)iph)->nexthdr == NEXTHDR_UDP; + if (unlikely(is_udp_tunnel && len < sizeof(struct ipv6hdr) + sizeof(struct udphdr))) + return -EINVAL; } else { return -EINVAL; } @@ -637,6 +644,11 @@ int bpf_lwt_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len, bool ingress) if (ingress) skb_postpush_rcsum(skb, iph, len); skb_reset_network_header(skb); + if (is_udp_tunnel) { + size_t iph_sz = ipv4 ? iph->ihl * 4 : sizeof(struct ipv6hdr); + + skb_set_transport_header(skb, skb_network_offset(skb) + iph_sz); + } memcpy(skb_network_header(skb), hdr, len); bpf_compute_data_pointers(skb); skb_clear_hash(skb); diff --git a/net/core/lwtunnel.c b/net/core/lwtunnel.c index f9d76d85d04f..b01a395d9a96 100644 --- a/net/core/lwtunnel.c +++ b/net/core/lwtunnel.c @@ -350,6 +350,8 @@ int lwtunnel_output(struct net *net, struct sock *sk, struct sk_buff *skb) rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[lwtstate->type]); if (likely(ops && ops->output)) { + /* Encap pushes outer headers over the metadata; drop it. */ + skb_metadata_clear(skb); dev_xmit_recursion_inc(); ret = ops->output(net, sk, skb); dev_xmit_recursion_dec(); @@ -404,6 +406,8 @@ int lwtunnel_xmit(struct sk_buff *skb) rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[lwtstate->type]); if (likely(ops && ops->xmit)) { + /* Encap pushes outer headers over the metadata; drop it. */ + skb_metadata_clear(skb); dev_xmit_recursion_inc(); ret = ops->xmit(skb); dev_xmit_recursion_dec(); @@ -455,6 +459,8 @@ int lwtunnel_input(struct sk_buff *skb) rcu_read_lock(); ops = rcu_dereference(lwtun_encaps[lwtstate->type]); if (likely(ops && ops->input)) { + /* Encap pushes outer headers over the metadata; drop it. */ + skb_metadata_clear(skb); dev_xmit_recursion_inc(); ret = ops->input(skb); dev_xmit_recursion_dec(); diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 9e12524b67fa..1349c0eedb64 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -457,7 +457,6 @@ void neigh_changeaddr(struct neigh_table *tbl, struct net_device *dev) neigh_flush_dev(tbl, dev, false); spin_unlock_bh(&tbl->lock); } -EXPORT_SYMBOL(neigh_changeaddr); static int __neigh_ifdown(struct neigh_table *tbl, struct net_device *dev, bool skip_perm) @@ -484,14 +483,12 @@ int neigh_carrier_down(struct neigh_table *tbl, struct net_device *dev) __neigh_ifdown(tbl, dev, true); return 0; } -EXPORT_SYMBOL(neigh_carrier_down); int neigh_ifdown(struct neigh_table *tbl, struct net_device *dev) { __neigh_ifdown(tbl, dev, false); return 0; } -EXPORT_SYMBOL(neigh_ifdown); static struct neighbour *neigh_alloc(struct neigh_table *tbl, struct net_device *dev, @@ -1652,13 +1649,11 @@ int neigh_connected_output(struct neighbour *neigh, struct sk_buff *skb) } return err; } -EXPORT_SYMBOL(neigh_connected_output); int neigh_direct_output(struct neighbour *neigh, struct sk_buff *skb) { return dev_queue_xmit(skb); } -EXPORT_SYMBOL(neigh_direct_output); static void neigh_managed_work(struct work_struct *work) { @@ -1880,7 +1875,6 @@ void neigh_table_init(int index, struct neigh_table *tbl) rcu_assign_pointer(neigh_tables[index], tbl); } -EXPORT_SYMBOL(neigh_table_init); /* * Only called from ndisc_cleanup(), which means this is dead code @@ -1914,7 +1908,6 @@ int neigh_table_clear(int index, struct neigh_table *tbl) return 0; } -EXPORT_SYMBOL(neigh_table_clear); static struct neigh_table *neigh_find_table(int family) { @@ -3210,8 +3203,10 @@ int neigh_xmit(int index, struct net_device *dev, rcu_read_lock(); tbl = rcu_dereference(neigh_tables[index]); - if (!tbl) - goto out_unlock; + if (!tbl) { + rcu_read_unlock(); + goto out_kfree_skb; + } if (index == NEIGH_ARP_TABLE) { u32 key = *((u32 *)addr); @@ -3227,7 +3222,6 @@ int neigh_xmit(int index, struct net_device *dev, goto out_kfree_skb; } err = READ_ONCE(neigh->output)(neigh, skb); -out_unlock: rcu_read_unlock(); } else if (index == NEIGH_LINK_TABLE) { @@ -3237,11 +3231,10 @@ out_unlock: goto out_kfree_skb; err = dev_queue_xmit(skb); } -out: return err; out_kfree_skb: kfree_skb(skb); - goto out; + return err; } EXPORT_SYMBOL(neigh_xmit); diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 3318b5666e43..0e71c9ed41e8 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -37,12 +37,6 @@ static const char fmt_uint[] = "%u\n"; static const char fmt_ulong[] = "%lu\n"; static const char fmt_u64[] = "%llu\n"; -/* Caller holds RTNL, netdev->lock or RCU */ -static inline int dev_isalive(const struct net_device *dev) -{ - return READ_ONCE(dev->reg_state) <= NETREG_REGISTERED; -} - /* There is a possible ABBA deadlock between rtnl_lock and kernfs_node->active, * when unregistering a net device and accessing associated sysfs files. The * potential deadlock is as follow: diff --git a/net/core/netdev-genl-gen.c b/net/core/netdev-genl-gen.c index 81aecb5d3bc5..d18c89b5a6c7 100644 --- a/net/core/netdev-genl-gen.c +++ b/net/core/netdev-genl-gen.c @@ -51,14 +51,28 @@ static const struct nla_policy netdev_dev_get_nl_policy[NETDEV_A_DEV_IFINDEX + 1 /* NETDEV_CMD_PAGE_POOL_GET - do */ #ifdef CONFIG_PAGE_POOL -static const struct nla_policy netdev_page_pool_get_nl_policy[NETDEV_A_PAGE_POOL_ID + 1] = { +static const struct nla_policy netdev_page_pool_get_do_nl_policy[NETDEV_A_PAGE_POOL_ID + 1] = { [NETDEV_A_PAGE_POOL_ID] = NLA_POLICY_FULL_RANGE(NLA_UINT, &netdev_a_page_pool_id_range), }; #endif /* CONFIG_PAGE_POOL */ +/* NETDEV_CMD_PAGE_POOL_GET - dump */ +#ifdef CONFIG_PAGE_POOL +static const struct nla_policy netdev_page_pool_get_dump_nl_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = { + [NETDEV_A_PAGE_POOL_IFINDEX] = NLA_POLICY_FULL_RANGE(NLA_U32, &netdev_a_page_pool_ifindex_range), +}; +#endif /* CONFIG_PAGE_POOL */ + /* NETDEV_CMD_PAGE_POOL_STATS_GET - do */ #ifdef CONFIG_PAGE_POOL_STATS -static const struct nla_policy netdev_page_pool_stats_get_nl_policy[NETDEV_A_PAGE_POOL_STATS_INFO + 1] = { +static const struct nla_policy netdev_page_pool_stats_get_do_nl_policy[NETDEV_A_PAGE_POOL_STATS_INFO + 1] = { + [NETDEV_A_PAGE_POOL_STATS_INFO] = NLA_POLICY_NESTED(netdev_page_pool_info_nl_policy), +}; +#endif /* CONFIG_PAGE_POOL_STATS */ + +/* NETDEV_CMD_PAGE_POOL_STATS_GET - dump */ +#ifdef CONFIG_PAGE_POOL_STATS +static const struct nla_policy netdev_page_pool_stats_get_dump_nl_policy[NETDEV_A_PAGE_POOL_STATS_INFO + 1] = { [NETDEV_A_PAGE_POOL_STATS_INFO] = NLA_POLICY_NESTED(netdev_page_pool_info_nl_policy), }; #endif /* CONFIG_PAGE_POOL_STATS */ @@ -138,28 +152,32 @@ static const struct genl_split_ops netdev_nl_ops[] = { { .cmd = NETDEV_CMD_PAGE_POOL_GET, .doit = netdev_nl_page_pool_get_doit, - .policy = netdev_page_pool_get_nl_policy, + .policy = netdev_page_pool_get_do_nl_policy, .maxattr = NETDEV_A_PAGE_POOL_ID, .flags = GENL_CMD_CAP_DO, }, { - .cmd = NETDEV_CMD_PAGE_POOL_GET, - .dumpit = netdev_nl_page_pool_get_dumpit, - .flags = GENL_CMD_CAP_DUMP, + .cmd = NETDEV_CMD_PAGE_POOL_GET, + .dumpit = netdev_nl_page_pool_get_dumpit, + .policy = netdev_page_pool_get_dump_nl_policy, + .maxattr = NETDEV_A_PAGE_POOL_IFINDEX, + .flags = GENL_CMD_CAP_DUMP, }, #endif /* CONFIG_PAGE_POOL */ #ifdef CONFIG_PAGE_POOL_STATS { .cmd = NETDEV_CMD_PAGE_POOL_STATS_GET, .doit = netdev_nl_page_pool_stats_get_doit, - .policy = netdev_page_pool_stats_get_nl_policy, + .policy = netdev_page_pool_stats_get_do_nl_policy, .maxattr = NETDEV_A_PAGE_POOL_STATS_INFO, .flags = GENL_CMD_CAP_DO, }, { - .cmd = NETDEV_CMD_PAGE_POOL_STATS_GET, - .dumpit = netdev_nl_page_pool_stats_get_dumpit, - .flags = GENL_CMD_CAP_DUMP, + .cmd = NETDEV_CMD_PAGE_POOL_STATS_GET, + .dumpit = netdev_nl_page_pool_stats_get_dumpit, + .policy = netdev_page_pool_stats_get_dump_nl_policy, + .maxattr = NETDEV_A_PAGE_POOL_STATS_INFO, + .flags = GENL_CMD_CAP_DUMP, }, #endif /* CONFIG_PAGE_POOL_STATS */ { @@ -202,7 +220,7 @@ static const struct genl_split_ops netdev_nl_ops[] = { .doit = netdev_nl_bind_rx_doit, .policy = netdev_bind_rx_nl_policy, .maxattr = NETDEV_A_DMABUF_FD, - .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + .flags = GENL_UNS_ADMIN_PERM | GENL_CMD_CAP_DO, }, { .cmd = NETDEV_CMD_NAPI_SET, diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c index b8f6076d8007..c15d8d4ca1f8 100644 --- a/net/core/netdev-genl.c +++ b/net/core/netdev-genl.c @@ -2,6 +2,7 @@ #include <linux/netdevice.h> #include <linux/notifier.h> +#include <linux/pid_namespace.h> #include <linux/rtnetlink.h> #include <net/busy_poll.h> #include <net/net_namespace.h> @@ -189,7 +190,8 @@ netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, goto nla_put_failure; if (napi->thread) { - pid = task_pid_nr(napi->thread); + pid = task_pid_nr_ns(napi->thread, + task_active_pid_ns(current)); if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) goto nla_put_failure; } @@ -1077,7 +1079,7 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) goto err_rxq_bitmap; } - binding = net_devmem_bind_dmabuf(netdev, dma_dev, DMA_FROM_DEVICE, + binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE, dmabuf_fd, priv, info->extack); if (IS_ERR(binding)) { err = PTR_ERR(binding); @@ -1095,8 +1097,6 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) genlmsg_end(rsp, hdr); err = genlmsg_reply(rsp, info); - if (err) - goto err_unbind; bitmap_free(rxq_bitmap); @@ -1104,7 +1104,7 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) mutex_unlock(&priv->lock); - return 0; + return err < 0 ? err : 0; err_unbind: net_devmem_unbind_dmabuf(binding); @@ -1119,9 +1119,43 @@ err_genlmsg_free: return err; } +/* Find the DMA-capable device for a netmem TX binding. + * + * For NETMEM_TX_DMA devices, return the device itself. + * For NETMEM_TX_NO_DMA devices, walk leased RX queues to find the underlying + * physical device and return it. + */ +static struct net_device * +netdev_find_netmem_tx_dev(struct net_device *dev) +{ + struct netdev_rx_queue *lease_rxq; + struct net_device *phys_dev; + int i; + + if (dev->netmem_tx == NETMEM_TX_DMA) + return dev; + + if (dev->netmem_tx != NETMEM_TX_NO_DMA) + return NULL; + + for (i = 0; i < dev->real_num_rx_queues; i++) { + lease_rxq = READ_ONCE(__netif_get_rx_queue(dev, i)->lease); + if (!lease_rxq) + continue; + + phys_dev = lease_rxq->dev; + if (netif_device_present(phys_dev) && + phys_dev->netmem_tx == NETMEM_TX_DMA) + return phys_dev; + } + + return NULL; +} + int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) { struct net_devmem_dmabuf_binding *binding; + struct net_device *bind_dev; struct netdev_nl_sock *priv; struct net_device *netdev; struct device *dma_dev; @@ -1164,29 +1198,48 @@ int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) goto err_unlock_netdev; } - if (!netdev->netmem_tx) { + if (netdev->netmem_tx == NETMEM_TX_NONE) { err = -EOPNOTSUPP; NL_SET_ERR_MSG(info->extack, "Driver does not support netmem TX"); goto err_unlock_netdev; } - dma_dev = netdev_queue_get_dma_dev(netdev, 0, NETDEV_QUEUE_TYPE_TX); - binding = net_devmem_bind_dmabuf(netdev, dma_dev, DMA_TO_DEVICE, - dmabuf_fd, priv, info->extack); + bind_dev = netdev_find_netmem_tx_dev(netdev); + if (!bind_dev) { + err = -EOPNOTSUPP; + NL_SET_ERR_MSG(info->extack, + "No DMA-capable device found for netmem TX"); + goto err_unlock_netdev; + } + + if (bind_dev != netdev) + netdev_lock(bind_dev); + + dma_dev = netdev_queue_get_dma_dev(bind_dev, 0, NETDEV_QUEUE_TYPE_TX); + + binding = net_devmem_bind_dmabuf(bind_dev, + bind_dev != netdev ? netdev : NULL, + dma_dev, DMA_TO_DEVICE, dmabuf_fd, + priv, info->extack); if (IS_ERR(binding)) { err = PTR_ERR(binding); - goto err_unlock_netdev; + goto err_unlock_bind_dev; } nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); genlmsg_end(rsp, hdr); + if (bind_dev != netdev) + netdev_unlock(bind_dev); netdev_unlock(netdev); mutex_unlock(&priv->lock); return genlmsg_reply(rsp, info); +err_unlock_bind_dev: + if (bind_dev != netdev) + netdev_unlock(bind_dev); err_unlock_netdev: netdev_unlock(netdev); err_unlock_sock: diff --git a/net/core/netdev_queues.c b/net/core/netdev_queues.c index 73fb28087a93..4d6864bb4f6b 100644 --- a/net/core/netdev_queues.c +++ b/net/core/netdev_queues.c @@ -40,7 +40,7 @@ struct device *netdev_queue_get_dma_dev(struct net_device *dev, struct netdev_rx_queue *hw_rxq; struct device *dma_dev; - netdev_ops_assert_locked(dev); + netdev_assert_locked_ops_compat(dev); /* Only RX side supports queue leasing today. */ if (type != NETDEV_QUEUE_TYPE_RX || !netif_rxq_is_leased(dev, idx)) diff --git a/net/core/netdev_rx_queue.c b/net/core/netdev_rx_queue.c index de4dac4c88b3..00a7011eb4d5 100644 --- a/net/core/netdev_rx_queue.c +++ b/net/core/netdev_rx_queue.c @@ -338,12 +338,12 @@ void __netif_mp_uninstall_rxq(struct netdev_rx_queue *rxq, void netif_rxq_cleanup_unlease(struct netdev_rx_queue *phys_rxq, struct netdev_rx_queue *virt_rxq) { - struct pp_memory_provider_params *p = &phys_rxq->mp_params; unsigned int rxq_idx = get_netdev_rx_queue_index(phys_rxq); + struct pp_memory_provider_params p = phys_rxq->mp_params; - if (!p->mp_ops) + if (!p.mp_ops) return; - __netif_mp_uninstall_rxq(virt_rxq, p); - __netif_mp_close_rxq(phys_rxq->dev, rxq_idx, p); + __netif_mp_close_rxq(phys_rxq->dev, rxq_idx, &p); + __netif_mp_uninstall_rxq(virt_rxq, &p); } diff --git a/net/core/netdev_work.c b/net/core/netdev_work.c new file mode 100644 index 000000000000..3109fae132ad --- /dev/null +++ b/net/core/netdev_work.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include <linux/export.h> +#include <linux/list.h> +#include <linux/netdevice.h> +#include <linux/rtnetlink.h> +#include <linux/spinlock.h> +#include <linux/workqueue.h> +#include <net/netdev_lock.h> + +#include "dev.h" + +static void netdev_work_proc(struct work_struct *work); + +/* @netdev_work_lock protects: + * - @netdev_work_list + * - within the list entries (struct net_device fields): + * - work_node + * - work_tracker + * - work_pending + * - work_core_pending + */ +static LIST_HEAD(netdev_work_list); +static DEFINE_SPINLOCK(netdev_work_lock); +static DECLARE_WORK(netdev_work, netdev_work_proc); + +static void netdev_work_enqueue(struct net_device *dev, unsigned long events, + unsigned long core) +{ + if (!events && !core) + return; + + spin_lock_bh(&netdev_work_lock); + if (list_empty(&dev->work_node)) { + list_add_tail(&dev->work_node, &netdev_work_list); + netdev_hold(dev, &dev->work_tracker, GFP_ATOMIC); + } + dev->work_pending |= events; + dev->work_core_pending |= core; + spin_unlock_bh(&netdev_work_lock); + + schedule_work(&netdev_work); +} + +static unsigned long +netdev_work_dequeue(struct net_device *dev, unsigned long *pending, + unsigned long mask) +{ + unsigned long events; + + spin_lock_bh(&netdev_work_lock); + events = *pending & mask; + *pending &= ~events; + if (!list_empty(&dev->work_node) && + !dev->work_pending && !dev->work_core_pending) { + list_del_init(&dev->work_node); + netdev_put(dev, &dev->work_tracker); + } + spin_unlock_bh(&netdev_work_lock); + + return events; +} + +void netdev_work_sched(struct net_device *dev, unsigned long events) +{ + netdev_work_enqueue(dev, events, 0); +} +EXPORT_SYMBOL(netdev_work_sched); + +/** + * netdev_work_cancel() - cancel selected work for a netdev + * @dev: net_device + * @mask: events to cancel + * + * Clear @mask from the device's work pending mask. If no work is left pending + * the device is dequeued and its ndo_work won't be called. + * + * No expectations on locking, but also no guarantees provided. If the caller + * wants to touch @dev afterwards (e.g. call the work that got canceled) + * they have to ensure @dev does not get freed. + * + * Returns: the subset of @mask that was actually pending, so the caller can run + * those events inline. + */ +unsigned long netdev_work_cancel(struct net_device *dev, unsigned long mask) +{ + return netdev_work_dequeue(dev, &dev->work_pending, mask); +} +EXPORT_SYMBOL(netdev_work_cancel); + +void __netdev_work_core_sched(struct net_device *dev, unsigned long events) +{ + netdev_work_enqueue(dev, 0, events); +} + +unsigned long +__netdev_work_core_cancel(struct net_device *dev, unsigned long mask) +{ + return netdev_work_dequeue(dev, &dev->work_core_pending, mask); +} + +static void netdev_work_run(struct net_device *dev, unsigned long events, + unsigned long core) +{ + if (!netif_device_present(dev)) + return; + + if (core & NETDEV_WORK_RX_MODE) + netif_rx_mode_run(dev); + if (events && dev->netdev_ops->ndo_work) + dev->netdev_ops->ndo_work(dev, events); +} + +static void netdev_work_proc(struct work_struct *work) +{ + rtnl_lock(); + + while (true) { + unsigned long events = 0, core = 0; + netdevice_tracker tracker; + struct net_device *dev; + + spin_lock_bh(&netdev_work_lock); + if (list_empty(&netdev_work_list)) { + spin_unlock_bh(&netdev_work_lock); + break; + } + dev = list_first_entry(&netdev_work_list, struct net_device, + work_node); + /* Take a temporary reference so @dev can't be freed while we + * drop the lock to grab its ops lock; the work reference is + * only released once we claim the work below. + * The re-locking dance is to ensure that ops lock is enough + * to ensure canceling work is not racy with dequeue. + */ + netdev_hold(dev, &tracker, GFP_ATOMIC); + spin_unlock_bh(&netdev_work_lock); + + netdev_lock_ops(dev); + spin_lock_bh(&netdev_work_lock); + if (!list_empty(&dev->work_node)) { + list_del_init(&dev->work_node); + core = dev->work_core_pending; + dev->work_core_pending = 0; + events = dev->work_pending; + dev->work_pending = 0; + /* We took another ref above */ + netdev_put(dev, &dev->work_tracker); + + if (!dev_isalive(dev)) + core = events = 0; + } + spin_unlock_bh(&netdev_work_lock); + + netdev_work_run(dev, events, core); + netdev_unlock_ops(dev); + + netdev_put(dev, &tracker); + } + + rtnl_unlock(); +} diff --git a/net/core/netmem_priv.h b/net/core/netmem_priv.h index 3e6fde8f1726..23175cb2bd86 100644 --- a/net/core/netmem_priv.h +++ b/net/core/netmem_priv.h @@ -8,18 +8,21 @@ static inline unsigned long netmem_get_pp_magic(netmem_ref netmem) return netmem_to_nmdesc(netmem)->pp_magic & ~PP_DMA_INDEX_MASK; } -static inline bool netmem_is_pp(netmem_ref netmem) +static inline void netmem_or_pp_magic(netmem_ref netmem, unsigned long pp_magic) +{ + netmem_to_nmdesc(netmem)->pp_magic |= pp_magic; +} + +static inline void netmem_clear_pp_magic(netmem_ref netmem) { - struct page *page; + WARN_ON_ONCE(netmem_to_nmdesc(netmem)->pp_magic & PP_DMA_INDEX_MASK); - /* XXX: Now that the offset of page_type is shared between - * struct page and net_iov, just cast the netmem to struct page - * unconditionally by clearing NET_IOV if any, no matter whether - * it comes from struct net_iov or struct page. This should be - * adjusted once the offset is no longer shared. - */ - page = (struct page *)((__force unsigned long)netmem & ~NET_IOV); - return PageNetpp(page); + netmem_to_nmdesc(netmem)->pp_magic = 0; +} + +static inline bool netmem_is_pp(netmem_ref netmem) +{ + return (netmem_get_pp_magic(netmem) & PP_MAGIC_MASK) == PP_SIGNATURE; } static inline void netmem_set_pp(netmem_ref netmem, struct page_pool *pool) diff --git a/net/core/netpoll.c b/net/core/netpoll.c index cd74beffd209..96d5945e6a30 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -29,12 +29,10 @@ #include <linux/slab.h> #include <linux/export.h> #include <linux/if_vlan.h> +#include <linux/udp.h> #include <net/tcp.h> -#include <net/udp.h> #include <net/addrconf.h> #include <net/ndisc.h> -#include <net/ip6_checksum.h> -#include <linux/unaligned.h> #include <trace/events/napi.h> #include <linux/kconfig.h> @@ -43,18 +41,9 @@ * message gets out even in extreme OOM situations. */ -#define MAX_UDP_CHUNK 1460 #define MAX_SKBS 32 #define USEC_PER_POLL 50 -#define MAX_SKB_SIZE \ - (sizeof(struct ethhdr) + \ - sizeof(struct iphdr) + \ - sizeof(struct udphdr) + \ - MAX_UDP_CHUNK) - -static void zap_completion_queue(void); - static unsigned int carrier_timeout = 4; module_param(carrier_timeout, uint, 0644); @@ -201,7 +190,7 @@ void netpoll_poll_dev(struct net_device *dev) up(&ni->dev_lock); - zap_completion_queue(); + netpoll_zap_completion_queue(); } EXPORT_SYMBOL(netpoll_poll_dev); @@ -240,7 +229,7 @@ static void refill_skbs(struct netpoll *np) } } -static void zap_completion_queue(void) +void netpoll_zap_completion_queue(void) { unsigned long flags; struct softnet_data *sd = &get_cpu_var(softnet_data); @@ -267,33 +256,7 @@ static void zap_completion_queue(void) put_cpu_var(softnet_data); } - -static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve) -{ - int count = 0; - struct sk_buff *skb; - - zap_completion_queue(); -repeat: - - skb = alloc_skb(len, GFP_ATOMIC); - if (!skb) { - skb = skb_dequeue(&np->skb_pool); - schedule_work(&np->refill_wq); - } - - if (!skb) { - if (++count < 10) { - netpoll_poll_dev(np->dev); - goto repeat; - } - return NULL; - } - - refcount_set(&skb->users, 1); - skb_reserve(skb, reserve); - return skb; -} +EXPORT_SYMBOL_NS_GPL(netpoll_zap_completion_queue, "NETDEV_INTERNAL"); static int netpoll_owner_active(struct net_device *dev) { @@ -319,6 +282,8 @@ static netdev_tx_t __netpoll_send_skb(struct netpoll *np, struct sk_buff *skb) lockdep_assert_irqs_disabled(); dev = np->dev; + /* npinfo->txq belongs to np->dev, so retries must stay bound to it. */ + skb->dev = dev; rcu_read_lock(); npinfo = rcu_dereference_bh(dev->npinfo); @@ -369,31 +334,6 @@ out: return ret; } -static void netpoll_udp_checksum(struct netpoll *np, struct sk_buff *skb, - int len) -{ - struct udphdr *udph; - int udp_len; - - udp_len = len + sizeof(struct udphdr); - udph = udp_hdr(skb); - - /* check needs to be set, since it will be consumed in csum_partial */ - udph->check = 0; - if (np->ipv6) - udph->check = csum_ipv6_magic(&np->local_ip.in6, - &np->remote_ip.in6, - udp_len, IPPROTO_UDP, - csum_partial(udph, udp_len, 0)); - else - udph->check = csum_tcpudp_magic(np->local_ip.ip, - np->remote_ip.ip, - udp_len, IPPROTO_UDP, - csum_partial(udph, udp_len, 0)); - if (udph->check == 0) - udph->check = CSUM_MANGLED_0; -} - netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb) { unsigned long flags; @@ -411,125 +351,6 @@ netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb) } EXPORT_SYMBOL(netpoll_send_skb); -static void push_ipv6(struct netpoll *np, struct sk_buff *skb, int len) -{ - struct ipv6hdr *ip6h; - - skb_push(skb, sizeof(struct ipv6hdr)); - skb_reset_network_header(skb); - ip6h = ipv6_hdr(skb); - - /* ip6h->version = 6; ip6h->priority = 0; */ - *(unsigned char *)ip6h = 0x60; - ip6h->flow_lbl[0] = 0; - ip6h->flow_lbl[1] = 0; - ip6h->flow_lbl[2] = 0; - - ip6h->payload_len = htons(sizeof(struct udphdr) + len); - ip6h->nexthdr = IPPROTO_UDP; - ip6h->hop_limit = 32; - ip6h->saddr = np->local_ip.in6; - ip6h->daddr = np->remote_ip.in6; - - skb->protocol = htons(ETH_P_IPV6); -} - -static void push_ipv4(struct netpoll *np, struct sk_buff *skb, int len) -{ - static atomic_t ip_ident; - struct iphdr *iph; - int ip_len; - - ip_len = len + sizeof(struct udphdr) + sizeof(struct iphdr); - - skb_push(skb, sizeof(struct iphdr)); - skb_reset_network_header(skb); - iph = ip_hdr(skb); - - /* iph->version = 4; iph->ihl = 5; */ - *(unsigned char *)iph = 0x45; - iph->tos = 0; - put_unaligned(htons(ip_len), &iph->tot_len); - iph->id = htons(atomic_inc_return(&ip_ident)); - iph->frag_off = 0; - iph->ttl = 64; - iph->protocol = IPPROTO_UDP; - iph->check = 0; - put_unaligned(np->local_ip.ip, &iph->saddr); - put_unaligned(np->remote_ip.ip, &iph->daddr); - iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); - skb->protocol = htons(ETH_P_IP); -} - -static void push_udp(struct netpoll *np, struct sk_buff *skb, int len) -{ - struct udphdr *udph; - int udp_len; - - udp_len = len + sizeof(struct udphdr); - - skb_push(skb, sizeof(struct udphdr)); - skb_reset_transport_header(skb); - - udph = udp_hdr(skb); - udph->source = htons(np->local_port); - udph->dest = htons(np->remote_port); - udph->len = htons(udp_len); - - netpoll_udp_checksum(np, skb, len); -} - -static void push_eth(struct netpoll *np, struct sk_buff *skb) -{ - struct ethhdr *eth; - - eth = skb_push(skb, ETH_HLEN); - skb_reset_mac_header(skb); - ether_addr_copy(eth->h_source, np->dev->dev_addr); - ether_addr_copy(eth->h_dest, np->remote_mac); - if (np->ipv6) - eth->h_proto = htons(ETH_P_IPV6); - else - eth->h_proto = htons(ETH_P_IP); -} - -int netpoll_send_udp(struct netpoll *np, const char *msg, int len) -{ - int total_len, ip_len, udp_len; - struct sk_buff *skb; - - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) - WARN_ON_ONCE(!irqs_disabled()); - - udp_len = len + sizeof(struct udphdr); - if (np->ipv6) - ip_len = udp_len + sizeof(struct ipv6hdr); - else - ip_len = udp_len + sizeof(struct iphdr); - - total_len = ip_len + LL_RESERVED_SPACE(np->dev); - - skb = find_skb(np, total_len + np->dev->needed_tailroom, - total_len - len); - if (!skb) - return -ENOMEM; - - skb_copy_to_linear_data(skb, msg, len); - skb_put(skb, len); - - push_udp(np, skb, len); - if (np->ipv6) - push_ipv6(np, skb, len); - else - push_ipv4(np, skb, len); - push_eth(np, skb); - skb->dev = np->dev; - - return (int)netpoll_send_skb(np, skb); -} -EXPORT_SYMBOL(netpoll_send_udp); - - static void skb_pool_flush(struct netpoll *np) { struct sk_buff_head *skb_pool; @@ -608,14 +429,16 @@ EXPORT_SYMBOL_GPL(__netpoll_setup); /* * Returns a pointer to a string representation of the identifier used * to select the egress interface for the given netpoll instance. buf - * must be a buffer of length at least MAC_ADDR_STR_LEN + 1. + * is used to format np->dev_mac when np->dev_name is empty; bufsz must + * be at least MAC_ADDR_STR_LEN + 1 to fit the formatted MAC address + * and its NUL terminator. */ -static char *egress_dev(struct netpoll *np, char *buf) +static char *egress_dev(struct netpoll *np, char *buf, size_t bufsz) { if (np->dev_name[0]) return np->dev_name; - snprintf(buf, MAC_ADDR_STR_LEN, "%pM", np->dev_mac); + snprintf(buf, bufsz, "%pM", np->dev_mac); return buf; } @@ -645,7 +468,7 @@ static int netpoll_take_ipv6(struct netpoll *np, struct net_device *ndev) if (!IS_ENABLED(CONFIG_IPV6)) { np_err(np, "IPv6 is not supported %s, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); return -EINVAL; } @@ -667,7 +490,7 @@ static int netpoll_take_ipv6(struct netpoll *np, struct net_device *ndev) } if (err) { np_err(np, "no IPv6 address for %s, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); return err; } @@ -687,14 +510,14 @@ static int netpoll_take_ipv4(struct netpoll *np, struct net_device *ndev) in_dev = __in_dev_get_rtnl(ndev); if (!in_dev) { np_err(np, "no IP address for %s, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); return -EDESTADDRREQ; } ifa = rtnl_dereference(in_dev->ifa_list); if (!ifa) { np_err(np, "no IP address for %s, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); return -EDESTADDRREQ; } @@ -704,6 +527,23 @@ static int netpoll_take_ipv4(struct netpoll *np, struct net_device *ndev) return 0; } +/* + * Test whether the caller left np->local_ip unset, so that + * netpoll_setup() should auto-populate it from the egress device. + * + * np->local_ip is a union of __be32 (IPv4) and struct in6_addr (IPv6), + * so an IPv6 address whose first 4 bytes are zero (e.g. ::1, ::2, + * IPv4-mapped ::ffff:a.b.c.d) must not be tested via the IPv4 arm — + * doing so would misclassify a caller-supplied address as unset and + * silently overwrite it with whatever address the device exposes. + */ +static bool netpoll_local_ip_unset(const struct netpoll *np) +{ + if (np->ipv6) + return ipv6_addr_any(&np->local_ip.in6); + return !np->local_ip.ip; +} + int netpoll_setup(struct netpoll *np) { struct net *net = current->nsproxy->net_ns; @@ -719,7 +559,8 @@ int netpoll_setup(struct netpoll *np) ndev = dev_getbyhwaddr(net, ARPHRD_ETHER, np->dev_mac); if (!ndev) { - np_err(np, "%s doesn't exist, aborting\n", egress_dev(np, buf)); + np_err(np, "%s doesn't exist, aborting\n", + egress_dev(np, buf, sizeof(buf))); err = -ENODEV; goto unlock; } @@ -727,14 +568,14 @@ int netpoll_setup(struct netpoll *np) if (netdev_master_upper_dev_get(ndev)) { np_err(np, "%s is a slave device, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); err = -EBUSY; goto put; } if (!netif_running(ndev)) { np_info(np, "device %s not up yet, forcing it\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); err = dev_open(ndev, NULL); if (err) { @@ -747,7 +588,7 @@ int netpoll_setup(struct netpoll *np) rtnl_lock(); } - if (!np->local_ip.ip) { + if (netpoll_local_ip_unset(np)) { if (!np->ipv6) { err = netpoll_take_ipv4(np, ndev); if (err) @@ -792,14 +633,6 @@ static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head) container_of(rcu_head, struct netpoll_info, rcu); skb_queue_purge(&npinfo->txq); - - /* we can't call cancel_delayed_work_sync here, as we are in softirq */ - cancel_delayed_work(&npinfo->tx_work); - - /* clean after last, unfinished work */ - __skb_queue_purge(&npinfo->txq); - /* now cancel it again */ - cancel_delayed_work(&npinfo->tx_work); kfree(npinfo); } @@ -823,6 +656,7 @@ static void __netpoll_cleanup(struct netpoll *np) ops->ndo_netpoll_cleanup(np->dev); RCU_INIT_POINTER(np->dev->npinfo, NULL); + disable_delayed_work_sync(&npinfo->tx_work); call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info); } diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 877bbf7a1938..21dc4a9c8714 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -76,20 +76,20 @@ static const char pp_stats[][ETH_GSTRING_LEN] = { * @pool: pool from which page was allocated * @stats: struct page_pool_stats to fill in * + * Deprecated driver API for querying stats. Page pool stats can be queried + * via netdev Netlink. + * * Retrieve statistics about the page_pool. This API is only available * if the kernel has been configured with ``CONFIG_PAGE_POOL_STATS=y``. * A pointer to a caller allocated struct page_pool_stats structure * is passed to this API which is filled in. The caller can then report * those stats to the user (perhaps via ethtool, debugfs, etc.). */ -bool page_pool_get_stats(const struct page_pool *pool, +void page_pool_get_stats(const struct page_pool *pool, struct page_pool_stats *stats) { int cpu = 0; - if (!stats) - return false; - /* The caller is responsible to initialize stats. */ stats->alloc_stats.fast += pool->alloc_stats.fast; stats->alloc_stats.slow += pool->alloc_stats.slow; @@ -108,8 +108,6 @@ bool page_pool_get_stats(const struct page_pool *pool, stats->recycle_stats.ring_full += pcpu->ring_full; stats->recycle_stats.released_refcnt += pcpu->released_refcnt; } - - return true; } EXPORT_SYMBOL(page_pool_get_stats); @@ -327,6 +325,11 @@ static void page_pool_uninit(struct page_pool *pool) if (!pool->system) free_percpu(pool->recycle_stats); #endif + + if (pool->mp_ops) { + pool->mp_ops->destroy(pool); + static_branch_dec(&page_pool_mem_providers); + } } /** @@ -702,18 +705,8 @@ s32 page_pool_inflight(const struct page_pool *pool, bool strict) void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) { - struct page *page; - netmem_set_pp(netmem, pool); - - /* XXX: Now that the offset of page_type is shared between - * struct page and net_iov, just cast the netmem to struct page - * unconditionally by clearing NET_IOV if any, no matter whether - * it comes from struct net_iov or struct page. This should be - * adjusted once the offset is no longer shared. - */ - page = (struct page *)((__force unsigned long)netmem & ~NET_IOV); - __SetPageNetpp(page); + netmem_or_pp_magic(netmem, PP_SIGNATURE); /* Ensuring all pages have been split into one fragment initially: * page_pool_set_pp_info() is only called once for every page when it @@ -728,17 +721,7 @@ void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) void page_pool_clear_pp_info(netmem_ref netmem) { - struct page *page; - - /* XXX: Now that the offset of page_type is shared between - * struct page and net_iov, just cast the netmem to struct page - * unconditionally by clearing NET_IOV if any, no matter whether - * it comes from struct net_iov or struct page. This should be - * adjusted once the offset is no longer shared. - */ - page = (struct page *)((__force unsigned long)netmem & ~NET_IOV); - __ClearPageNetpp(page); - + netmem_clear_pp_magic(netmem); netmem_set_pp(netmem, NULL); } @@ -1146,11 +1129,6 @@ static void __page_pool_destroy(struct page_pool *pool) page_pool_unlist(pool); page_pool_uninit(pool); - if (pool->mp_ops) { - pool->mp_ops->destroy(pool); - static_branch_dec(&page_pool_mem_providers); - } - kfree(pool); } diff --git a/net/core/page_pool_user.c b/net/core/page_pool_user.c index ee5060d8eec0..ef4261c0e8ea 100644 --- a/net/core/page_pool_user.c +++ b/net/core/page_pool_user.c @@ -79,7 +79,7 @@ struct page_pool_dump_cb { static int netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb, - pp_nl_fill_cb fill) + pp_nl_fill_cb fill, struct nlattr *ifindex_attr) { struct page_pool_dump_cb *state = (void *)cb->ctx; const struct genl_info *info = genl_info_dump(cb); @@ -88,9 +88,17 @@ netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb, struct page_pool *pool; int err = 0; + if (ifindex_attr) + state->ifindex = nla_get_u32(ifindex_attr); + rtnl_lock(); mutex_lock(&page_pools_lock); for_each_netdev_dump(net, netdev, state->ifindex) { + /* Either the provided ifindex doesn't exist or done dumping */ + if (ifindex_attr && + netdev->ifindex != nla_get_u32(ifindex_attr)) + break; + hlist_for_each_entry(pool, &netdev->page_pools, user.list) { if (state->pp_id && state->pp_id < pool->user.id) continue; @@ -119,14 +127,15 @@ page_pool_nl_stats_fill(struct sk_buff *rsp, const struct page_pool *pool, struct nlattr *nest; void *hdr; - if (!page_pool_get_stats(pool, &stats)) - return 0; + page_pool_get_stats(pool, &stats); hdr = genlmsg_iput(rsp, info); if (!hdr) return -EMSGSIZE; nest = nla_nest_start(rsp, NETDEV_A_PAGE_POOL_STATS_INFO); + if (!nest) + goto err_cancel_msg; if (nla_put_uint(rsp, NETDEV_A_PAGE_POOL_ID, pool->user.id) || (pool->slow.netdev->ifindex != LOOPBACK_IFINDEX && @@ -206,10 +215,40 @@ int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb, return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill); } +static const struct netlink_range_validation page_pool_ifindex_range = { + .min = 1ULL, + .max = S32_MAX, +}; + +static const struct nla_policy +page_pool_stat_info_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = { + [NETDEV_A_PAGE_POOL_IFINDEX] = + NLA_POLICY_FULL_RANGE(NLA_U32, &page_pool_ifindex_range), +}; + int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { - return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill); + struct nlattr *tb[ARRAY_SIZE(page_pool_stat_info_policy)]; + const struct genl_info *info = genl_info_dump(cb); + struct nlattr *ifindex_attr = NULL; + + if (info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]) { + struct nlattr *nest; + int err; + + nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]; + err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest, + page_pool_stat_info_policy, + info->extack); + if (err) + return err; + + ifindex_attr = tb[NETDEV_A_PAGE_POOL_IFINDEX]; + } + + return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill, + ifindex_attr); } static int @@ -305,7 +344,10 @@ int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info) int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { - return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill); + const struct genl_info *info = genl_info_dump(cb); + + return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill, + info->attrs[NETDEV_A_PAGE_POOL_IFINDEX]); } int page_pool_list(struct page_pool *pool) diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index b613bb6e07df..12aa3aa1688b 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -63,7 +63,7 @@ #include "dev.h" #define RTNL_MAX_TYPE 50 -#define RTNL_SLAVE_MAX_TYPE 44 +#define RTNL_SLAVE_MAX_TYPE 45 struct rtnl_link { rtnl_doit_func doit; @@ -750,13 +750,12 @@ static size_t rtnl_link_get_size(const struct net_device *dev) } static LIST_HEAD(rtnl_af_ops); +static DEFINE_SPINLOCK(rtnl_af_ops_lock); static struct rtnl_af_ops *rtnl_af_lookup(const int family, int *srcu_index) { struct rtnl_af_ops *ops; - ASSERT_RTNL(); - rcu_read_lock(); list_for_each_entry_rcu(ops, &rtnl_af_ops, list) { @@ -791,9 +790,9 @@ int rtnl_af_register(struct rtnl_af_ops *ops) if (err) return err; - rtnl_lock(); + spin_lock(&rtnl_af_ops_lock); list_add_tail_rcu(&ops->list, &rtnl_af_ops); - rtnl_unlock(); + spin_unlock(&rtnl_af_ops_lock); return 0; } @@ -805,9 +804,9 @@ EXPORT_SYMBOL_GPL(rtnl_af_register); */ void rtnl_af_unregister(struct rtnl_af_ops *ops) { - rtnl_lock(); + spin_lock(&rtnl_af_ops_lock); list_del_rcu(&ops->list); - rtnl_unlock(); + spin_unlock(&rtnl_af_ops_lock); synchronize_rcu(); synchronize_srcu(&ops->srcu); @@ -1264,11 +1263,11 @@ static size_t rtnl_devlink_port_size(const struct net_device *dev) return size; } -static size_t rtnl_dpll_pin_size(const struct net_device *dev) +static size_t rtnl_dpll_pin_size(void) { size_t size = nla_total_size(0); /* nest IFLA_DPLL_PIN */ - size += dpll_netdev_pin_handle_size(dev); + size += dpll_netdev_pin_handle_size(); return size; } @@ -1295,7 +1294,12 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev, size = NLMSG_ALIGN(sizeof(struct ifinfomsg)) + nla_total_size(IFNAMSIZ) /* IFLA_IFNAME */ - + nla_total_size(IFALIASZ) /* IFLA_IFALIAS */ + + rtnl_prop_list_size(dev); + + if (ext_filter_mask & RTEXT_FILTER_NAME_ONLY) + return size; + + size += nla_total_size(IFALIASZ) /* IFLA_IFALIAS */ + nla_total_size(IFNAMSIZ) /* IFLA_QDISC */ + nla_total_size_64bit(sizeof(struct rtnl_link_ifmap)) + nla_total_size(MAX_ADDR_LEN) /* IFLA_ADDRESS */ @@ -1342,10 +1346,9 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev, + nla_total_size(4) /* IFLA_CARRIER_DOWN_COUNT */ + nla_total_size(4) /* IFLA_MIN_MTU */ + nla_total_size(4) /* IFLA_MAX_MTU */ - + rtnl_prop_list_size(dev) + nla_total_size(MAX_ADDR_LEN) /* IFLA_PERM_ADDRESS */ + rtnl_devlink_port_size(dev) - + rtnl_dpll_pin_size(dev) + + rtnl_dpll_pin_size() + nla_total_size(8) /* IFLA_MAX_PACING_OFFLOAD_HORIZON */ + nla_total_size(2) /* IFLA_HEADROOM */ + nla_total_size(2) /* IFLA_TAILROOM */ @@ -1572,6 +1575,7 @@ static noinline_for_stack int rtnl_fill_vfinfo(struct sk_buff *skb, port_guid.vf = ivi.vf; memcpy(vf_mac.mac, ivi.mac, sizeof(ivi.mac)); + memset(&vf_broadcast, 0, sizeof(vf_broadcast)); memcpy(vf_broadcast.broadcast, dev->broadcast, dev->addr_len); vf_vlan.vlan = ivi.vlan; vf_vlan.qos = ivi.qos; @@ -1940,15 +1944,18 @@ static int rtnl_fill_alt_ifnames(struct sk_buff *skb, struct netdev_name_node *name_node; int count = 0; + rcu_read_lock(); list_for_each_entry_rcu(name_node, &dev->name_node->list, list) { - if (nla_put_string(skb, IFLA_ALT_IFNAME, name_node->name)) + if (nla_put_string(skb, IFLA_ALT_IFNAME, name_node->name)) { + rcu_read_unlock(); return -EMSGSIZE; + } count++; } + rcu_read_unlock(); return count; } -/* RCU protected. */ static int rtnl_fill_prop_list(struct sk_buff *skb, const struct net_device *dev) { @@ -1963,7 +1970,10 @@ static int rtnl_fill_prop_list(struct sk_buff *skb, if (ret <= 0) goto nest_cancel; - nla_nest_end(skb, prop_list); + ret = -EMSGSIZE; + if (nla_nest_end_safe(skb, prop_list) < 0) + goto nest_cancel; + return 0; nest_cancel: @@ -2058,7 +2068,6 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct nlmsghdr *nlh; struct Qdisc *qdisc; - ASSERT_RTNL(); nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ifm), flags); if (nlh == NULL) return -EMSGSIZE; @@ -2071,13 +2080,21 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, ifm->ifi_flags = netif_get_flags(dev); ifm->ifi_change = change; - if (tgt_netnsid >= 0 && nla_put_s32(skb, IFLA_TARGET_NETNSID, tgt_netnsid)) - goto nla_put_failure; - netdev_copy_name(dev, devname); if (nla_put_string(skb, IFLA_IFNAME, devname)) goto nla_put_failure; + if (rtnl_fill_prop_list(skb, dev)) + goto nla_put_failure; + + if (ext_filter_mask & RTEXT_FILTER_NAME_ONLY) + goto end; + + ASSERT_RTNL(); + if (tgt_netnsid >= 0 && + nla_put_s32(skb, IFLA_TARGET_NETNSID, tgt_netnsid)) + goto nla_put_failure; + if (nla_put_u32(skb, IFLA_TXQLEN, READ_ONCE(dev->tx_queue_len)) || nla_put_u8(skb, IFLA_OPERSTATE, netif_running(dev) ? READ_ONCE(dev->operstate) : @@ -2190,8 +2207,6 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, goto nla_put_failure_rcu; if (rtnl_fill_link_ifmap(skb, dev)) goto nla_put_failure_rcu; - if (rtnl_fill_prop_list(skb, dev)) - goto nla_put_failure_rcu; rcu_read_unlock(); if (dev->dev.parent && @@ -2210,6 +2225,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, if (rtnl_fill_dpll_pin(skb, dev)) goto nla_put_failure; +end: nlmsg_end(skb, nlh); return 0; @@ -2355,22 +2371,24 @@ static struct rtnl_link_ops *linkinfo_to_kind_ops(const struct nlattr *nla, static bool link_master_filtered(struct net_device *dev, int master_idx) { struct net_device *master; + bool res = false; if (!master_idx) return false; - master = netdev_master_upper_dev_get(dev); + rcu_read_lock(); + master = netdev_master_upper_dev_get_rcu(dev); /* 0 is already used to denote IFLA_MASTER wasn't passed, therefore need * another invalid value for ifindex to denote "no master". */ if (master_idx == -1) - return !!master; - - if (!master || master->ifindex != master_idx) - return true; + res = !!master; + else if (!master || master->ifindex != master_idx) + res = true; + rcu_read_unlock(); - return false; + return res; } static bool link_kind_filtered(const struct net_device *dev, @@ -2420,6 +2438,14 @@ struct net *rtnl_get_net_ns_capable(struct sock *sk, int netnsid) } EXPORT_SYMBOL_GPL(rtnl_get_net_ns_capable); +bool rtnl_dev_link_net_capable(const struct net_device *dev, + const struct net *link_net) +{ + return net_eq(link_net, dev_net(dev)) || + ns_capable(link_net->user_ns, CAP_NET_ADMIN); +} +EXPORT_SYMBOL_GPL(rtnl_dev_link_net_capable); + static int rtnl_valid_dump_ifinfo_req(const struct nlmsghdr *nlh, bool strict_check, struct nlattr **tb, struct netlink_ext_ack *extack) @@ -2481,6 +2507,7 @@ static int rtnl_dump_ifinfo(struct sk_buff *skb, struct netlink_callback *cb) int ops_srcu_index; int master_idx = 0; int netnsid = -1; + bool need_rtnl; int err, i; err = rtnl_valid_dump_ifinfo_req(nlh, cb->strict_check, tb, extack); @@ -2530,6 +2557,12 @@ static int rtnl_dump_ifinfo(struct sk_buff *skb, struct netlink_callback *cb) walk_entries: err = 0; + need_rtnl = !(ext_filter_mask & RTEXT_FILTER_NAME_ONLY); + if (need_rtnl) + rtnl_lock(); + else + rcu_read_lock(); + for_each_netdev_dump(tgt_net, dev, ctx->ifindex) { if (link_dump_filtered(dev, master_idx, kind_ops)) continue; @@ -2541,11 +2574,13 @@ walk_entries: if (err < 0) break; } - - - cb->seq = tgt_net->dev_base_seq; + cb->seq = READ_ONCE(tgt_net->dev_base_seq); nl_dump_check_consistent(cb, nlmsg_hdr(skb)); + if (need_rtnl) + rtnl_unlock(); + else + rcu_read_unlock(); out: if (kind_ops) @@ -3452,6 +3487,21 @@ static struct net_device *rtnl_dev_get(struct net *net, return __dev_get_by_name(net, ifname); } +static struct net_device *rtnl_dev_get_rcu(struct net *net, + struct nlattr *tb[]) +{ + char ifname[ALTIFNAMSIZ]; + + if (tb[IFLA_IFNAME]) + nla_strscpy(ifname, tb[IFLA_IFNAME], IFNAMSIZ); + else if (tb[IFLA_ALT_IFNAME]) + nla_strscpy(ifname, tb[IFLA_ALT_IFNAME], ALTIFNAMSIZ); + else + return NULL; + + return dev_get_by_name_rcu(net, ifname); +} + static int rtnl_setlink(struct sk_buff *skb, struct nlmsghdr *nlh, struct netlink_ext_ack *extack) { @@ -4171,14 +4221,16 @@ static int rtnl_getlink(struct sk_buff *skb, struct nlmsghdr *nlh, struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); + struct nlattr *tb[IFLA_MAX + 1]; + netdevice_tracker dev_tracker; + struct net_device *dev = NULL; struct net *tgt_net = net; + u32 ext_filter_mask = 0; struct ifinfomsg *ifm; - struct nlattr *tb[IFLA_MAX+1]; - struct net_device *dev = NULL; struct sk_buff *nskb; int netnsid = -1; + bool need_rtnl; int err; - u32 ext_filter_mask = 0; err = rtnl_valid_getlink_req(skb, nlh, tb, extack); if (err < 0) @@ -4198,43 +4250,71 @@ static int rtnl_getlink(struct sk_buff *skb, struct nlmsghdr *nlh, if (tb[IFLA_EXT_MASK]) ext_filter_mask = nla_get_u32(tb[IFLA_EXT_MASK]); - err = -EINVAL; ifm = nlmsg_data(nlh); - if (ifm->ifi_index > 0) - dev = __dev_get_by_index(tgt_net, ifm->ifi_index); - else if (tb[IFLA_IFNAME] || tb[IFLA_ALT_IFNAME]) - dev = rtnl_dev_get(tgt_net, tb); - else + rcu_read_lock(); + if (ifm->ifi_index > 0) { + dev = dev_get_by_index_rcu(tgt_net, ifm->ifi_index); + } else if (tb[IFLA_IFNAME] || tb[IFLA_ALT_IFNAME]) { + dev = rtnl_dev_get_rcu(tgt_net, tb); + } else { + rcu_read_unlock(); + err = -EINVAL; goto out; + } + netdev_hold(dev, &dev_tracker, GFP_ATOMIC); + rcu_read_unlock(); err = -ENODEV; if (dev == NULL) goto out; + need_rtnl = !(ext_filter_mask & RTEXT_FILTER_NAME_ONLY); + +retry: + if (need_rtnl) { + rtnl_lock(); + if (!dev_isalive(dev)) { + err = -ENODEV; + nskb = NULL; + goto unlock; + } + /* Synchronize the carrier state so we don't report a state + * that we're not actually going to honour immediately; if + * the driver just did a carrier off->on transition, we can + * only TX if link watch work has run, but without this we'd + * already report carrier on, even if it doesn't work yet. + */ + linkwatch_sync_dev(dev); + } + err = -ENOBUFS; nskb = nlmsg_new_large(if_nlmsg_size(dev, ext_filter_mask)); - if (nskb == NULL) - goto out; + if (nskb) + err = rtnl_fill_ifinfo(nskb, dev, net, + RTM_NEWLINK, NETLINK_CB(skb).portid, + nlh->nlmsg_seq, 0, 0, ext_filter_mask, + 0, NULL, 0, netnsid, GFP_KERNEL); - /* Synchronize the carrier state so we don't report a state - * that we're not actually going to honour immediately; if - * the driver just did a carrier off->on transition, we can - * only TX if link watch work has run, but without this we'd - * already report carrier on, even if it doesn't work yet. - */ - linkwatch_sync_dev(dev); +unlock: + if (need_rtnl) + rtnl_unlock(); - err = rtnl_fill_ifinfo(nskb, dev, net, - RTM_NEWLINK, NETLINK_CB(skb).portid, - nlh->nlmsg_seq, 0, 0, ext_filter_mask, - 0, NULL, 0, netnsid, GFP_KERNEL); if (err < 0) { - /* -EMSGSIZE implies BUG in if_nlmsg_size */ - WARN_ON(err == -EMSGSIZE); kfree_skb(nskb); - } else + if (err == -EMSGSIZE) { + if (!need_rtnl) { + /* Some altnames were added, retry with RTNL. */ + need_rtnl = true; + goto retry; + } + /* -EMSGSIZE implies BUG in if_nlmsg_size */ + WARN_ON_ONCE(1); + } + } else { err = rtnl_unicast(nskb, net, NETLINK_CB(skb).portid); + } out: + netdev_put(dev, &dev_tracker); if (netnsid >= 0) put_net(tgt_net); @@ -6327,8 +6407,9 @@ static int rtnl_stats_get(struct sk_buff *skb, struct nlmsghdr *nlh, NETLINK_CB(skb).portid, nlh->nlmsg_seq, 0, 0, &filters, &idxattr, &prividx, extack); if (err < 0) { - /* -EMSGSIZE implies BUG in if_nlmsg_stats_size */ - WARN_ON(err == -EMSGSIZE); + /* -EMSGSIZE implies BUG in if_nlmsg_stats_size + * or a too big nested attribute. + */ kfree_skb(nskb); } else { err = rtnl_unicast(nskb, net, NETLINK_CB(skb).portid); @@ -7100,7 +7181,10 @@ static const struct rtnl_msg_handler rtnetlink_rtnl_msg_handlers[] __initconst = {.msgtype = RTM_DELLINK, .doit = rtnl_dellink, .flags = RTNL_FLAG_DOIT_PERNET_WIP}, {.msgtype = RTM_GETLINK, .doit = rtnl_getlink, - .dumpit = rtnl_dump_ifinfo, .flags = RTNL_FLAG_DUMP_SPLIT_NLM_DONE}, + .dumpit = rtnl_dump_ifinfo, + .flags = RTNL_FLAG_DUMP_SPLIT_NLM_DONE | + RTNL_FLAG_DOIT_UNLOCKED | + RTNL_FLAG_DUMP_UNLOCKED}, {.msgtype = RTM_SETLINK, .doit = rtnl_setlink, .flags = RTNL_FLAG_DOIT_PERNET_WIP}, {.msgtype = RTM_GETADDR, .dumpit = rtnl_dump_all}, diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 7dad68e3b518..18dabb4e9cfa 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -78,6 +78,7 @@ #include <net/mpls.h> #include <net/mptcp.h> #include <net/mctp.h> +#include <net/tcp.h> #include <net/can.h> #include <net/page_pool/helpers.h> #include <net/psp/types.h> @@ -96,7 +97,6 @@ #include "devmem.h" #include "net-sysfs.h" #include "netmem_priv.h" -#include "sock_destructor.h" #ifdef CONFIG_SKB_EXTENSIONS static struct kmem_cache *skbuff_ext_cache __ro_after_init; @@ -288,11 +288,11 @@ static inline struct sk_buff *napi_skb_cache_get(bool alloc) local_lock_nested_bh(&napi_alloc_cache.bh_lock); if (unlikely(!nc->skb_count)) { - if (alloc) - nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, - GFP_ATOMIC | __GFP_NOWARN, - NAPI_SKB_CACHE_BULK, - nc->skb_cache); + if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, + GFP_ATOMIC | __GFP_NOWARN, + NAPI_SKB_CACHE_BULK, + nc->skb_cache)) + nc->skb_count = NAPI_SKB_CACHE_BULK; if (unlikely(!nc->skb_count)) { local_unlock_nested_bh(&napi_alloc_cache.bh_lock); return NULL; @@ -353,16 +353,18 @@ u32 napi_skb_cache_get_bulk(void **skbs, u32 n) /* No enough cached skbs. Try refilling the cache first */ bulk = min(NAPI_SKB_CACHE_SIZE - nc->skb_count, NAPI_SKB_CACHE_BULK); - nc->skb_count += kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, - GFP_ATOMIC | __GFP_NOWARN, bulk, - &nc->skb_cache[nc->skb_count]); + if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, + GFP_ATOMIC | __GFP_NOWARN, bulk, + &nc->skb_cache[nc->skb_count])) + nc->skb_count += bulk; if (likely(nc->skb_count >= n)) goto get; /* Still not enough. Bulk-allocate the missing part directly, zeroed */ - n -= kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, - GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN, - n - nc->skb_count, &skbs[nc->skb_count]); + if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, + GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN, + n - nc->skb_count, &skbs[nc->skb_count])) + n = nc->skb_count; if (likely(nc->skb_count >= n)) goto get; @@ -1206,7 +1208,7 @@ void __kfree_skb(struct sk_buff *skb) EXPORT_SYMBOL(__kfree_skb); static __always_inline -bool __sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb, +bool __sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb, enum skb_drop_reason reason) { if (unlikely(!skb_unref(skb))) @@ -1235,7 +1237,8 @@ bool __sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb, * 'kfree_skb' tracepoint. */ void __fix_address -sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason reason) +sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb, + enum skb_drop_reason reason) { if (__sk_skb_reason_drop(sk, skb, reason)) __kfree_skb(skb); @@ -2248,6 +2251,7 @@ struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, skb_frag_ref(skb, i); } skb_shinfo(n)->nr_frags = i; + skb_shinfo(n)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; } if (skb_has_frag_list(skb)) { @@ -2786,6 +2790,8 @@ done: skb->data_len = 0; skb_set_tail_pointer(skb, len); } + if (!skb_shinfo(skb)->nr_frags && !skb_has_frag_list(skb)) + skb->unreadable = 0; if (!skb->sk || skb->destructor == sock_edemux) skb_condense(skb); @@ -2793,16 +2799,37 @@ done: } EXPORT_SYMBOL(___pskb_trim); +static int pskb_trim_rcsum_complete(struct sk_buff *skb, unsigned int len) +{ + int delta = skb->len - len; + + if (skb_frags_readable(skb)) { + skb->csum = csum_block_sub(skb->csum, + skb_checksum(skb, len, delta, 0), + len); + return 0; + } + + if (len > skb_headlen(skb)) + return -EFAULT; + + /* The trimmed bytes are unreadable, but the remaining packet can be + * checksummed by software after trimming. + */ + skb->ip_summed = CHECKSUM_NONE; + return 0; +} + /* Note : use pskb_trim_rcsum() instead of calling this directly */ int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) { if (skb->ip_summed == CHECKSUM_COMPLETE) { - int delta = skb->len - len; + int err; - skb->csum = csum_block_sub(skb->csum, - skb_checksum(skb, len, delta, 0), - len); + err = pskb_trim_rcsum_complete(skb, len); + if (err) + return err; } else if (skb->ip_summed == CHECKSUM_PARTIAL) { int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; int offset = skb_checksum_start_offset(skb) + skb->csum_offset; @@ -4349,6 +4376,8 @@ onlymerged: tgt->ip_summed = CHECKSUM_PARTIAL; skb->ip_summed = CHECKSUM_PARTIAL; + skb_shinfo(tgt)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; + skb_len_add(skb, -shiftlen); skb_len_add(tgt, shiftlen); @@ -4959,7 +4988,8 @@ normal: skb_copy_from_linear_data_offset(head_skb, offset, skb_put(nskb, hsize), hsize); - skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags & + skb_shinfo(nskb)->flags |= (skb_shinfo(head_skb)->flags | + skb_shinfo(frag_skb)->flags) & SKBFL_SHARED_FRAG; if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) @@ -4976,6 +5006,9 @@ normal: nfrags = skb_shinfo(list_skb)->nr_frags; frag = skb_shinfo(list_skb)->frags; frag_skb = list_skb; + + skb_shinfo(nskb)->flags |= skb_shinfo(frag_skb)->flags & SKBFL_SHARED_FRAG; + if (!skb_headlen(list_skb)) { BUG_ON(!nfrags); } else { @@ -5420,7 +5453,7 @@ int skb_cow_data(struct sk_buff *skb, int tailbits, struct sk_buff **trailer) } EXPORT_SYMBOL_GPL(skb_cow_data); -static void sock_rmem_free(struct sk_buff *skb) +void sock_rmem_free(struct sk_buff *skb) { struct sock *sk = skb->sk; @@ -5429,8 +5462,8 @@ static void sock_rmem_free(struct sk_buff *skb) static void skb_set_err_queue(struct sk_buff *skb) { - /* pkt_type of skbs received on local sockets is never PACKET_OUTGOING. - * So, it is safe to (mis)use it to mark skbs on the error queue. + /* The error-queue test in skb_is_err_queue() matches this marker + * with the sock_rmem_free destructor installed by sock_queue_err_skb(). */ skb->pkt_type = PACKET_OUTGOING; BUILD_BUG_ON(PACKET_OUTGOING == 0); @@ -6200,6 +6233,8 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, from_shinfo->frags, from_shinfo->nr_frags * sizeof(skb_frag_t)); to_shinfo->nr_frags += from_shinfo->nr_frags; + if (from_shinfo->nr_frags) + to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG; if (!skb_cloned(from)) from_shinfo->nr_frags = 0; @@ -6791,6 +6826,11 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, skb_copy_from_linear_data_offset(skb, off, data, new_hlen); skb->len -= off; + /* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it + * while refcounting frags below. + */ + skb_zcopy_downgrade_managed(skb); + memcpy((struct skb_shared_info *)(data + size), skb_shinfo(skb), offsetof(struct skb_shared_info, @@ -6801,6 +6841,8 @@ static int pskb_carve_inside_header(struct sk_buff *skb, const u32 off, skb_kfree_head(data); return -ENOMEM; } + if (skb_zcopy(skb)) + net_zcopy_get(skb_zcopy(skb)); for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) skb_frag_ref(skb, i); if (skb_has_frag_list(skb)) @@ -6902,6 +6944,11 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, return -ENOMEM; size = SKB_WITH_OVERHEAD(size); + /* Remove SKBFL_MANAGED_FRAG_REFS instead of trying to honour it + * while refcounting frags below. + */ + skb_zcopy_downgrade_managed(skb); + memcpy((struct skb_shared_info *)(data + size), skb_shinfo(skb), offsetof(struct skb_shared_info, frags[0])); if (skb_orphan_frags(skb, gfp_mask)) { @@ -6944,6 +6991,8 @@ static int pskb_carve_inside_nonlinear(struct sk_buff *skb, const u32 off, skb_kfree_head(data); return -ENOMEM; } + if (skb_zcopy(skb)) + net_zcopy_get(skb_zcopy(skb)); skb_release_data(skb, SKB_CONSUMED); skb->head = data; diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 6187a83bd741..2521b643fa05 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -7,7 +7,6 @@ #include <net/sock.h> #include <net/tcp.h> -#include <net/tls.h> #include <trace/events/sock.h> static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce) @@ -66,6 +65,7 @@ int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, sge = &msg->sg.data[msg->sg.end]; sg_unmark_end(sge); sg_set_page(sge, pfrag->page, use, orig_offset); + __clear_bit(msg->sg.end, msg->sg.copy); get_page(pfrag->page); sk_msg_iter_next(msg, end); } @@ -186,6 +186,7 @@ static int sk_msg_free_elem(struct sock *sk, struct sk_msg *msg, u32 i, sk_mem_uncharge(sk, len); put_page(sg_page(sge)); } + __clear_bit(i, msg->sg.copy); memset(sge, 0, sizeof(*sge)); return len; } @@ -992,41 +993,6 @@ static int sk_psock_skb_redirect(struct sk_psock *from, struct sk_buff *skb) return 0; } -static void sk_psock_tls_verdict_apply(struct sk_buff *skb, - struct sk_psock *from, int verdict) -{ - switch (verdict) { - case __SK_REDIRECT: - sk_psock_skb_redirect(from, skb); - break; - case __SK_PASS: - case __SK_DROP: - default: - break; - } -} - -int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb) -{ - struct bpf_prog *prog; - int ret = __SK_PASS; - - rcu_read_lock(); - prog = READ_ONCE(psock->progs.stream_verdict); - if (likely(prog)) { - skb->sk = psock->sk; - skb_dst_drop(skb); - skb_bpf_redirect_clear(skb); - ret = bpf_prog_run_pin_on_cpu(prog, skb); - ret = sk_psock_map_verd(ret, skb_bpf_redirect_fetch(skb)); - skb->sk = NULL; - } - sk_psock_tls_verdict_apply(skb, psock, ret); - rcu_read_unlock(); - return ret; -} -EXPORT_SYMBOL_GPL(sk_psock_tls_strp_read); - static int sk_psock_verdict_apply(struct sk_psock *psock, struct sk_buff *skb, int verdict) { @@ -1167,13 +1133,9 @@ static void sk_psock_strp_data_ready(struct sock *sk) rcu_read_lock(); psock = sk_psock(sk); if (likely(psock)) { - if (tls_sw_has_ctx_rx(sk)) { - psock->saved_data_ready(sk); - } else { - read_lock_bh(&sk->sk_callback_lock); - strp_data_ready(&psock->strp); - read_unlock_bh(&sk->sk_callback_lock); - } + read_lock_bh(&sk->sk_callback_lock); + strp_data_ready(&psock->strp); + read_unlock_bh(&sk->sk_callback_lock); } rcu_read_unlock(); } @@ -1268,6 +1230,7 @@ out: static void sk_psock_verdict_data_ready(struct sock *sk) { const struct proto_ops *ops = NULL; + struct sk_psock *psock; struct socket *sock; int copied; @@ -1283,8 +1246,6 @@ static void sk_psock_verdict_data_ready(struct sock *sk) copied = ops->read_skb(sk, sk_psock_verdict_recv); if (copied >= 0) { - struct sk_psock *psock; - rcu_read_lock(); psock = sk_psock(sk); if (psock) diff --git a/net/core/sock.c b/net/core/sock.c index b37b664b6eb9..8a59bfaa8096 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -1465,6 +1465,11 @@ set_sndbuf: case SO_ATTACH_FILTER: { struct sock_fprog fprog; + if (sk_is_tcp(sk) && + !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { + ret = -EPERM; + break; + } ret = copy_bpf_fprog_from_user(&fprog, optval, optlen); if (!ret) ret = sk_attach_filter(&fprog, sk); @@ -2676,8 +2681,12 @@ void sock_wfree(struct sk_buff *skb) int old; if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) { + void (*sk_write_space)(struct sock *sk); + + sk_write_space = READ_ONCE(sk->sk_write_space); + if (sock_flag(sk, SOCK_RCU_FREE) && - sk->sk_write_space == sock_def_write_space) { + sk_write_space == sock_def_write_space) { rcu_read_lock(); free = __refcount_sub_and_test(len, &sk->sk_wmem_alloc, &old); @@ -2693,7 +2702,7 @@ void sock_wfree(struct sk_buff *skb) * after sk_write_space() call */ WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc)); - sk->sk_write_space(sk); + sk_write_space(sk); len = 1; } /* @@ -2708,6 +2717,7 @@ EXPORT_SYMBOL(sock_wfree); /* This variant of sock_wfree() is used by TCP, * since it sets SOCK_USE_WRITE_QUEUE. */ +#ifdef CONFIG_INET void __sock_wfree(struct sk_buff *skb) { struct sock *sk = skb->sk; @@ -2715,6 +2725,8 @@ void __sock_wfree(struct sk_buff *skb) if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc)) __sk_free(sk); } +EXPORT_SYMBOL_GPL(__sock_wfree); +#endif void skb_set_owner_w(struct sk_buff *skb, struct sock *sk) { @@ -3038,12 +3050,42 @@ int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg, sockc->tsflags |= tsflags; break; case SCM_TXTIME: + { + ktime_t tmin; + u64 txtime; + if (!sock_flag(sk, SOCK_TXTIME)) return -EINVAL; if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64))) return -EINVAL; - sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg)); + + txtime = get_unaligned((u64 *)CMSG_DATA(cmsg)); + + /* Allow sending without a delivery time: zero special case */ + if (!txtime) { + sockc->transmit_time = 0; + break; + } + + switch (sk->sk_clockid) { + case CLOCK_MONOTONIC: + tmin = 1; + break; + case CLOCK_REALTIME: + tmin = max(ktime_mono_to_real(0), 1); + break; + case CLOCK_TAI: + tmin = max(ktime_mono_to_any(0, TK_OFFS_TAI), 1); + break; + default: + tmin = 1; + WARN_ON_ONCE(1); + break; + } + + sockc->transmit_time = max_t(ktime_t, txtime, tmin); break; + } case SCM_TS_OPT_ID: if (sk_is_tcp(sk)) return -EINVAL; diff --git a/net/core/sock_destructor.h b/net/core/sock_destructor.h deleted file mode 100644 index 2f396e6bfba5..000000000000 --- a/net/core/sock_destructor.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -#ifndef _NET_CORE_SOCK_DESTRUCTOR_H -#define _NET_CORE_SOCK_DESTRUCTOR_H -#include <net/tcp.h> - -static inline bool is_skb_wmem(const struct sk_buff *skb) -{ - return skb->destructor == sock_wfree || - skb->destructor == __sock_wfree || - (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree); -} -#endif diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 02a68be3002a..c60ba6d292f9 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1515,6 +1515,17 @@ static int sock_map_prog_link_lookup(struct bpf_map *map, struct bpf_prog ***ppr return 0; } +static int sock_map_prog_attach_check(enum bpf_attach_type attach_type, + struct bpf_prog *prog) +{ + /* A stream parser must not modify the skb, only measure it. */ + if (prog && attach_type == BPF_SK_SKB_STREAM_PARSER && + prog->aux->changes_pkt_data) + return -EINVAL; + + return 0; +} + /* Handle the following four cases: * prog_attach: prog != NULL, old == NULL, link == NULL * prog_detach: prog == NULL, old != NULL, link == NULL @@ -1533,6 +1544,10 @@ static int sock_map_prog_update(struct bpf_map *map, struct bpf_prog *prog, if (ret) return ret; + ret = sock_map_prog_attach_check(which, prog); + if (ret) + return ret; + /* for prog_attach/prog_detach/link_attach, return error if a bpf_link * exists for that prog. */ @@ -1630,18 +1645,23 @@ void sock_map_unhash(struct sock *sk) void (*saved_unhash)(struct sock *sk); struct sk_psock *psock; +retry: rcu_read_lock(); psock = sk_psock(sk); if (unlikely(!psock)) { rcu_read_unlock(); saved_unhash = READ_ONCE(sk->sk_prot)->unhash; + if (unlikely(saved_unhash == sock_map_unhash)) + goto retry; } else { saved_unhash = psock->saved_unhash; sock_map_remove_links(sk, psock); rcu_read_unlock(); + + if (WARN_ON_ONCE(saved_unhash == sock_map_unhash)) + return; } - if (WARN_ON_ONCE(saved_unhash == sock_map_unhash)) - return; + if (saved_unhash) saved_unhash(sk); } @@ -1652,20 +1672,25 @@ void sock_map_destroy(struct sock *sk) void (*saved_destroy)(struct sock *sk); struct sk_psock *psock; +retry: rcu_read_lock(); psock = sk_psock_get(sk); if (unlikely(!psock)) { rcu_read_unlock(); saved_destroy = READ_ONCE(sk->sk_prot)->destroy; + if (unlikely(saved_destroy == sock_map_destroy)) + goto retry; } else { saved_destroy = psock->saved_destroy; sock_map_remove_links(sk, psock); rcu_read_unlock(); sk_psock_stop(psock); sk_psock_put(sk, psock); + + if (WARN_ON_ONCE(saved_destroy == sock_map_destroy)) + return; } - if (WARN_ON_ONCE(saved_destroy == sock_map_destroy)) - return; + if (saved_destroy) saved_destroy(sk); } @@ -1676,32 +1701,33 @@ void sock_map_close(struct sock *sk, long timeout) void (*saved_close)(struct sock *sk, long timeout); struct sk_psock *psock; +retry: lock_sock(sk); rcu_read_lock(); - psock = sk_psock(sk); + psock = sk_psock_get(sk); if (likely(psock)) { saved_close = psock->saved_close; sock_map_remove_links(sk, psock); - psock = sk_psock_get(sk); - if (unlikely(!psock)) - goto no_psock; rcu_read_unlock(); sk_psock_stop(psock); release_sock(sk); cancel_delayed_work_sync(&psock->work); sk_psock_put(sk, psock); + + /* Make sure we do not recurse. This is a bug. + * Leak the socket instead of crashing on a stack overflow. + */ + if (WARN_ON_ONCE(saved_close == sock_map_close)) + return; } else { saved_close = READ_ONCE(sk->sk_prot)->close; -no_psock: rcu_read_unlock(); release_sock(sk); + + if (unlikely(saved_close == sock_map_close)) + goto retry; } - /* Make sure we do not recurse. This is a bug. - * Leak the socket instead of crashing on a stack overflow. - */ - if (WARN_ON_ONCE(saved_close == sock_map_close)) - return; saved_close(sk, timeout); } EXPORT_SYMBOL_GPL(sock_map_close); @@ -1765,6 +1791,11 @@ static int sock_map_link_update_prog(struct bpf_link *link, ret = -EINVAL; goto out; } + + ret = sock_map_prog_attach_check(link->attach_type, prog); + if (ret) + goto out; + if (!sockmap_link->map) { ret = -ENOLINK; goto out; diff --git a/net/core/stream.c b/net/core/stream.c index 7a37e7dd2c43..2d748581862d 100644 --- a/net/core/stream.c +++ b/net/core/stream.c @@ -112,7 +112,6 @@ void sk_stream_wait_close(struct sock *sk, long timeout) remove_wait_queue(sk_sleep(sk), &wait); } } -EXPORT_SYMBOL(sk_stream_wait_close); /** * sk_stream_wait_memory - Wait for more memory for a socket |