diff options
Diffstat (limited to 'net')
276 files changed, 5881 insertions, 2403 deletions
diff --git a/net/atm/common.c b/net/atm/common.c index c7f92405daf0..81195727fa18 100644 --- a/net/atm/common.c +++ b/net/atm/common.c @@ -710,7 +710,7 @@ int vcc_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval, unsigned int optlen) { struct atm_vcc *vcc; - unsigned long value; + int value; int error; if (__SO_LEVEL_MATCH(optname, level) && optlen != __SO_SIZE(optname)) @@ -722,8 +722,10 @@ int vcc_setsockopt(struct socket *sock, int level, int optname, { struct atm_qos qos; - if (copy_from_sockptr(&qos, optval, sizeof(qos))) - return -EFAULT; + error = copy_safe_from_sockptr(&qos, sizeof(qos), optval, + optlen); + if (error) + return error; error = check_qos(&qos); if (error) return error; @@ -737,8 +739,10 @@ int vcc_setsockopt(struct socket *sock, int level, int optname, return 0; } case SO_SETCLP: - if (copy_from_sockptr(&value, optval, sizeof(value))) - return -EFAULT; + error = copy_safe_from_sockptr(&value, sizeof(value), optval, + optlen); + if (error) + return error; if (value) vcc->atm_options |= ATM_ATMOPT_CLP; else diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c index c40c9e02391b..a6fe4820f65b 100644 --- a/net/batman-adv/distributed-arp-table.c +++ b/net/batman-adv/distributed-arp-table.c @@ -546,7 +546,7 @@ static bool batadv_is_orig_node_eligible(struct batadv_dat_candidate *res, * the one with the lowest address */ if (tmp_max == max && max_orig_node && - batadv_compare_eth(candidate->orig, max_orig_node->orig)) + memcmp(candidate->orig, max_orig_node->orig, ETH_ALEN) >= 0) goto out; ret = true; diff --git a/net/batman-adv/fragmentation.c b/net/batman-adv/fragmentation.c index 8a006a0473a8..2e20a2cb64cb 100644 --- a/net/batman-adv/fragmentation.c +++ b/net/batman-adv/fragmentation.c @@ -518,8 +518,10 @@ int batadv_frag_send_packet(struct sk_buff *skb, mtu = min_t(unsigned int, mtu, BATADV_FRAG_MAX_FRAG_SIZE); max_fragment_size = mtu - header_size; - if (skb->len == 0 || max_fragment_size == 0) - return -EINVAL; + if (skb->len == 0 || max_fragment_size == 0) { + ret = -EINVAL; + goto free_skb; + } num_fragments = (skb->len - 1) / max_fragment_size + 1; max_fragment_size = (skb->len - 1) / num_fragments + 1; @@ -545,7 +547,7 @@ int batadv_frag_send_packet(struct sk_buff *skb, */ if (skb_has_frag_list(skb) && __skb_linearize(skb)) { ret = -ENOMEM; - goto free_skb; + goto put_primary_if; } /* Create one header to be copied to all fragments */ diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c index 4d3807a645b7..67bed3ee77e7 100644 --- a/net/batman-adv/main.c +++ b/net/batman-adv/main.c @@ -259,6 +259,7 @@ err_orig: void batadv_mesh_free(struct net_device *mesh_iface) { struct batadv_priv *bat_priv = netdev_priv(mesh_iface); + struct batadv_meshif_vlan *vlan; WRITE_ONCE(bat_priv->mesh_state, BATADV_MESH_DEACTIVATING); @@ -273,6 +274,13 @@ void batadv_mesh_free(struct net_device *mesh_iface) batadv_mcast_free(bat_priv); + /* destroy the "untagged" VLAN */ + vlan = batadv_meshif_vlan_get(bat_priv, BATADV_NO_FLAGS); + if (vlan) { + batadv_meshif_destroy_vlan(bat_priv, vlan); + batadv_meshif_vlan_put(vlan); + } + /* Free the TT and the originator tables only after having terminated * all the other depending components which may use these structures for * their purposes. @@ -368,7 +376,7 @@ void batadv_skb_set_priority(struct sk_buff *skb, int offset) switch (ethhdr->h_proto) { case htons(ETH_P_8021Q): - vhdr = skb_header_pointer(skb, offset + sizeof(*vhdr), + vhdr = skb_header_pointer(skb, offset, sizeof(*vhdr), &vhdr_tmp); if (!vhdr) return; diff --git a/net/batman-adv/mesh-interface.c b/net/batman-adv/mesh-interface.c index 511f70e0706a..fbfd99268de4 100644 --- a/net/batman-adv/mesh-interface.c +++ b/net/batman-adv/mesh-interface.c @@ -195,6 +195,9 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, if (READ_ONCE(bat_priv->mesh_state) != BATADV_MESH_ACTIVE) goto dropped; + if (!pskb_may_pull(skb, ETH_HLEN)) + goto dropped; + /* reset control block to avoid left overs from previous users */ memset(skb->cb, 0, sizeof(struct batadv_skb_cb)); @@ -592,8 +595,8 @@ int batadv_meshif_create_vlan(struct batadv_priv *bat_priv, unsigned short vid) * @bat_priv: the bat priv with all the mesh interface information * @vlan: the object to remove */ -static void batadv_meshif_destroy_vlan(struct batadv_priv *bat_priv, - struct batadv_meshif_vlan *vlan) +void batadv_meshif_destroy_vlan(struct batadv_priv *bat_priv, + struct batadv_meshif_vlan *vlan) { /* explicitly remove the associated TT local entry because it is marked * with the NOPURGE flag @@ -1088,22 +1091,13 @@ static int batadv_meshif_newlink(struct net_device *dev, static void batadv_meshif_destroy_netlink(struct net_device *mesh_iface, struct list_head *head) { - struct batadv_priv *bat_priv = netdev_priv(mesh_iface); struct batadv_hard_iface *hard_iface; - struct batadv_meshif_vlan *vlan; while (!list_empty(&mesh_iface->adj_list.lower)) { hard_iface = netdev_adjacent_get_private(mesh_iface->adj_list.lower.next); batadv_hardif_disable_interface(hard_iface); } - /* destroy the "untagged" VLAN */ - vlan = batadv_meshif_vlan_get(bat_priv, BATADV_NO_FLAGS); - if (vlan) { - batadv_meshif_destroy_vlan(bat_priv, vlan); - batadv_meshif_vlan_put(vlan); - } - unregister_netdevice_queue(mesh_iface, head); } diff --git a/net/batman-adv/mesh-interface.h b/net/batman-adv/mesh-interface.h index 53756c5a45e0..5e1e83e04ffb 100644 --- a/net/batman-adv/mesh-interface.h +++ b/net/batman-adv/mesh-interface.h @@ -21,6 +21,8 @@ void batadv_interface_rx(struct net_device *mesh_iface, bool batadv_meshif_is_valid(const struct net_device *net_dev); extern struct rtnl_link_ops batadv_link_ops; int batadv_meshif_create_vlan(struct batadv_priv *bat_priv, unsigned short vid); +void batadv_meshif_destroy_vlan(struct batadv_priv *bat_priv, + struct batadv_meshif_vlan *vlan); void batadv_meshif_vlan_release(struct kref *ref); struct batadv_meshif_vlan *batadv_meshif_vlan_get(struct batadv_priv *bat_priv, unsigned short vid); diff --git a/net/batman-adv/multicast_forw.c b/net/batman-adv/multicast_forw.c index b8668a80b94a..1404a3b7adfb 100644 --- a/net/batman-adv/multicast_forw.c +++ b/net/batman-adv/multicast_forw.c @@ -927,11 +927,11 @@ static int batadv_mcast_forw_packet(struct batadv_priv *bat_priv, { struct batadv_tvlv_mcast_tracker *mcast_tracker; struct batadv_neigh_node *neigh_node; - unsigned long offset, num_dests_off; struct sk_buff *nexthop_skb; unsigned char *skb_net_hdr; bool local_recv = false; unsigned int tvlv_len; + unsigned long offset; bool xmitted = false; u8 *dest, *next_dest; u16 num_dests; @@ -940,9 +940,8 @@ static int batadv_mcast_forw_packet(struct batadv_priv *bat_priv, /* (at least) TVLV part needs to be linearized */ SKB_LINEAR_ASSERT(skb); - /* check if num_dests is within skb length */ - num_dests_off = offsetof(struct batadv_tvlv_mcast_tracker, num_dests); - if (num_dests_off > skb_network_header_len(skb)) + /* check if batadv_tvlv_mcast_tracker header is within skb length */ + if (sizeof(*mcast_tracker) > skb_network_header_len(skb)) return -EINVAL; skb_net_hdr = skb_network_header(skb); diff --git a/net/batman-adv/translation-table.c b/net/batman-adv/translation-table.c index 4bfad36a4b70..dae5e1d8c038 100644 --- a/net/batman-adv/translation-table.c +++ b/net/batman-adv/translation-table.c @@ -2971,7 +2971,7 @@ static bool batadv_send_tt_request(struct batadv_priv *bat_priv, out: batadv_hardif_put(primary_if); - if (ret && tt_req_node) { + if (!ret && tt_req_node) { spin_lock_bh(&bat_priv->tt.req_list_lock); if (!hlist_unhashed(&tt_req_node->list)) { hlist_del_init(&tt_req_node->list); @@ -4033,7 +4033,8 @@ static int batadv_tt_tvlv_unicast_handler_v1(struct batadv_priv *bat_priv, u16 tvlv_value_len) { struct batadv_tvlv_tt_data *tt_data; - u16 tt_vlan_len, tt_num_entries; + u16 tt_num_entries; + size_t tt_vlan_len; char tt_flag; bool ret; diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c index cb1e329d66fd..d504a363a30f 100644 --- a/net/bluetooth/6lowpan.c +++ b/net/bluetooth/6lowpan.c @@ -632,7 +632,7 @@ static struct l2cap_chan *chan_create(void) if (!chan) return NULL; - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); chan->chan_type = L2CAP_CHAN_CONN_ORIENTED; chan->mode = L2CAP_MODE_LE_FLOWCTL; @@ -745,21 +745,6 @@ static inline void chan_ready_cb(struct l2cap_chan *chan) ifup(dev->netdev); } -static inline struct l2cap_chan *chan_new_conn_cb(struct l2cap_chan *pchan) -{ - struct l2cap_chan *chan; - - chan = chan_create(); - if (!chan) - return NULL; - - chan->ops = pchan->ops; - - BT_DBG("chan %p pchan %p", chan, pchan); - - return chan; -} - static void unregister_dev(struct lowpan_btle_dev *dev) { struct hci_dev *hdev = READ_ONCE(dev->hdev); @@ -797,20 +782,10 @@ static void chan_close_cb(struct l2cap_chan *chan) struct lowpan_btle_dev *dev = NULL; struct lowpan_peer *peer; int err = -ENOENT; - bool last = false, remove = true; + bool last = false; BT_DBG("chan %p conn %p", chan, chan->conn); - if (chan->conn && chan->conn->hcon) { - if (!is_bt_6lowpan(chan->conn->hcon)) - return; - - /* If conn is set, then the netdev is also there and we should - * not remove it. - */ - remove = false; - } - spin_lock(&devices_lock); list_for_each_entry_rcu(entry, &bt_6lowpan_devices, list) { @@ -837,10 +812,8 @@ static void chan_close_cb(struct l2cap_chan *chan) ifdown(dev->netdev); - if (remove) { - INIT_WORK(&entry->delete_netdev, delete_netdev); - schedule_work(&entry->delete_netdev); - } + INIT_WORK(&entry->delete_netdev, delete_netdev); + schedule_work(&entry->delete_netdev); } else { spin_unlock(&devices_lock); } @@ -901,7 +874,6 @@ static long chan_get_sndtimeo_cb(struct l2cap_chan *chan) static const struct l2cap_ops bt_6lowpan_chan_ops = { .name = "L2CAP 6LoWPAN channel", - .new_connection = chan_new_conn_cb, .recv = chan_recv_cb, .close = chan_close_cb, .state_change = chan_state_change_cb, @@ -1029,16 +1001,19 @@ static int get_l2cap_conn(char *buf, bdaddr_t *addr, u8 *addr_type, hci_dev_lock(hdev); hcon = hci_conn_hash_lookup_le(hdev, addr, le_addr_type); - hci_dev_unlock(hdev); - hci_dev_put(hdev); - - if (!hcon) + if (!hcon) { + hci_dev_unlock(hdev); + hci_dev_put(hdev); return -ENOENT; + } - *conn = (struct l2cap_conn *)hcon->l2cap_data; + *conn = l2cap_conn_hold_unless_zero(hcon->l2cap_data); BT_DBG("conn %p dst %pMR type %u", *conn, &hcon->dst, hcon->dst_type); + hci_dev_unlock(hdev); + hci_dev_put(hdev); + return 0; } @@ -1093,23 +1068,15 @@ done: } while (nchans); } -struct set_enable { - struct work_struct work; - bool flag; -}; - -static void do_enable_set(struct work_struct *work) +static void do_enable_set(bool flag) { - struct set_enable *set_enable = container_of(work, - struct set_enable, work); - - if (!set_enable->flag || enable_6lowpan != set_enable->flag) + if (!flag || enable_6lowpan != flag) /* Disconnect existing connections if 6lowpan is * disabled */ disconnect_all_peers(); - enable_6lowpan = set_enable->flag; + enable_6lowpan = flag; mutex_lock(&set_lock); if (listen_chan) { @@ -1121,22 +1088,11 @@ static void do_enable_set(struct work_struct *work) listen_chan = bt_6lowpan_listen(); mutex_unlock(&set_lock); - - kfree(set_enable); } static int lowpan_enable_set(void *data, u64 val) { - struct set_enable *set_enable; - - set_enable = kzalloc_obj(*set_enable); - if (!set_enable) - return -ENOMEM; - - set_enable->flag = !!val; - INIT_WORK(&set_enable->work, do_enable_set); - - schedule_work(&set_enable->work); + do_enable_set(!!val); return 0; } @@ -1185,18 +1141,22 @@ static ssize_t lowpan_control_write(struct file *fp, if (conn) { struct lowpan_peer *peer; - if (!is_bt_6lowpan(conn->hcon)) + if (!is_bt_6lowpan(conn->hcon)) { + l2cap_conn_put(conn); return -EINVAL; + } peer = lookup_peer(conn); if (peer) { BT_DBG("6LoWPAN connection already exists"); + l2cap_conn_put(conn); return -EALREADY; } BT_DBG("conn %p dst %pMR type %d user %u", conn, &conn->hcon->dst, conn->hcon->dst_type, addr_type); + l2cap_conn_put(conn); } ret = bt_6lowpan_connect(&addr, addr_type); @@ -1212,6 +1172,8 @@ static ssize_t lowpan_control_write(struct file *fp, return ret; ret = bt_6lowpan_disconnect(conn, addr_type); + if (conn) + l2cap_conn_put(conn); if (ret < 0) return ret; diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c index bcbc11c9cb15..a2290ffdc2c1 100644 --- a/net/bluetooth/af_bluetooth.c +++ b/net/bluetooth/af_bluetooth.c @@ -305,7 +305,7 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock) restart: for (sk = bt_accept_get(parent, NULL); sk; sk = next) { - /* Prevent early freeing of sk due to unlink and sock_kill */ + /* The reference from bt_accept_get() keeps sk alive. */ lock_sock(sk); /* Check sk has not already been unlinked via @@ -321,13 +321,11 @@ restart: next = bt_accept_get(parent, sk); - /* sk is safely in the parent list so reduce reference count */ - sock_put(sk); - /* FIXME: Is this check still needed */ if (sk->sk_state == BT_CLOSED) { bt_accept_unlink(sk); release_sock(sk); + sock_put(sk); continue; } @@ -337,16 +335,6 @@ restart: if (newsock) sock_graft(sk, newsock); - /* Hand the caller a reference taken while sk is - * still locked. bt_accept_unlink() just dropped - * the accept-queue reference; without this hold a - * concurrent teardown (e.g. l2cap_conn_del() -> - * l2cap_sock_kill()) could free sk between - * release_sock() and the caller using it. Every - * caller drops this with sock_put() when done. - */ - sock_hold(sk); - release_sock(sk); if (next) sock_put(next); @@ -354,6 +342,7 @@ restart: } release_sock(sk); + sock_put(sk); } return NULL; diff --git a/net/bluetooth/bnep/core.c b/net/bluetooth/bnep/core.c index add9a8f7535d..f7d88c33e23e 100644 --- a/net/bluetooth/bnep/core.c +++ b/net/bluetooth/bnep/core.c @@ -559,14 +559,18 @@ static int bnep_session(void *arg) return 0; } -static struct device *bnep_get_device(struct bnep_session *session) +static struct l2cap_conn *bnep_get_conn(struct bnep_session *session) { - struct l2cap_conn *conn = l2cap_pi(session->sock->sk)->chan->conn; + struct l2cap_chan *chan = l2cap_pi(session->sock->sk)->chan; + struct l2cap_conn *conn; - if (!conn || !conn->hcon) - return NULL; + l2cap_chan_lock(chan); + conn = chan->conn; + if (conn) + l2cap_conn_get(conn); + l2cap_chan_unlock(chan); - return &conn->hcon->dev; + return conn; } static const struct device_type bnep_type = { @@ -578,6 +582,7 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock) u32 valid_flags = BIT(BNEP_SETUP_RESPONSE); struct net_device *dev; struct bnep_session *s, *ss; + struct l2cap_conn *conn = NULL; u8 dst[ETH_ALEN], src[ETH_ALEN]; int err; @@ -637,10 +642,18 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock) bnep_set_default_proto_filter(s); #endif - SET_NETDEV_DEV(dev, bnep_get_device(s)); + conn = bnep_get_conn(s); + if (!conn) { + err = -ENOTCONN; + goto failed; + } + + SET_NETDEV_DEV(dev, &conn->hcon->dev); SET_NETDEV_DEVTYPE(dev, &bnep_type); err = register_netdev(dev); + l2cap_conn_put(conn); + conn = NULL; if (err) goto failed; @@ -662,6 +675,8 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock) return 0; failed: + if (conn) + l2cap_conn_put(conn); up_write(&bnep_session_sem); free_netdev(dev); return err; diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index c335372e4062..b1f911fd4ad6 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -1123,6 +1123,8 @@ static struct hci_conn *__hci_conn_add(struct hci_dev *hdev, int type, INIT_DELAYED_WORK(&conn->idle_work, hci_conn_idle); INIT_DELAYED_WORK(&conn->le_conn_timeout, le_conn_timeout); + spin_lock_init(&conn->proto_lock); + atomic_set(&conn->refcnt, 0); hci_dev_hold(hdev); @@ -3163,6 +3165,13 @@ static int abort_conn_sync(struct hci_dev *hdev, void *data) return hci_abort_conn_sync(hdev, conn, conn->abort_reason); } +static void abort_conn_destroy(struct hci_dev *hdev, void *data, int err) +{ + struct hci_conn *conn = data; + + hci_conn_put(conn); +} + int hci_abort_conn(struct hci_conn *conn, u8 reason) { struct hci_dev *hdev = conn->hdev; @@ -3178,32 +3187,20 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason) conn->abort_reason = reason; - /* If the connection is pending check the command opcode since that - * might be blocking on hci_cmd_sync_work while waiting its respective - * event so we need to hci_cmd_sync_cancel to cancel it. - * - * hci_connect_le serializes the connection attempts so only one - * connection can be in BT_CONNECT at time. + /* Cancel the connect attempt. A return of 0 means the create command + * was still queued and got dequeued, so there is nothing to disconnect. */ - if (conn->state == BT_CONNECT && READ_ONCE(hdev->req_status) == HCI_REQ_PEND) { - switch (hci_skb_event(hdev->sent_cmd)) { - case HCI_EV_CONN_COMPLETE: - case HCI_EV_LE_CONN_COMPLETE: - case HCI_EV_LE_ENHANCED_CONN_COMPLETE: - case HCI_EVT_LE_CIS_ESTABLISHED: - hci_cmd_sync_cancel(hdev, ECANCELED); - break; - } - /* Cancel connect attempt if still queued/pending */ - } else if (!hci_cancel_connect_sync(hdev, conn)) { + if (!hci_cancel_connect_sync(hdev, conn)) return 0; - } /* Run immediately if on cmd_sync_work since this may be called * as a result to MGMT_OP_DISCONNECT/MGMT_OP_UNPAIR which does * already queue its callback on cmd_sync_work. */ - err = hci_cmd_sync_run_once(hdev, abort_conn_sync, conn, NULL); + err = hci_cmd_sync_run_once(hdev, abort_conn_sync, hci_conn_get(conn), + abort_conn_destroy); + if (err) + hci_conn_put(conn); return (err == -EEXIST) ? 0 : err; } diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index b6d963ce26d0..741d658e9630 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -2763,7 +2763,7 @@ static void hci_cs_disconnect(struct hci_dev *hdev, u8 status) } mgmt_device_disconnected(hdev, &conn->dst, conn->type, conn->dst_type, - cp->reason, mgmt_conn); + hci_to_mgmt_reason(cp->reason), mgmt_conn); hci_disconn_cfm(conn, cp->reason); @@ -3381,22 +3381,6 @@ unlock: hci_dev_unlock(hdev); } -static u8 hci_to_mgmt_reason(u8 err) -{ - switch (err) { - case HCI_ERROR_CONNECTION_TIMEOUT: - return MGMT_DEV_DISCONN_TIMEOUT; - case HCI_ERROR_REMOTE_USER_TERM: - case HCI_ERROR_REMOTE_LOW_RESOURCES: - case HCI_ERROR_REMOTE_POWER_OFF: - return MGMT_DEV_DISCONN_REMOTE; - case HCI_ERROR_LOCAL_HOST_TERM: - return MGMT_DEV_DISCONN_LOCAL_HOST; - default: - return MGMT_DEV_DISCONN_UNKNOWN; - } -} - static void hci_disconn_complete_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 3be8c3581c6c..c8d14128c363 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -929,12 +929,16 @@ int hci_update_eir_sync(struct hci_dev *hdev) memset(&cp, 0, sizeof(cp)); + hci_dev_lock(hdev); eir_create(hdev, cp.data); - if (memcmp(cp.data, hdev->eir, sizeof(cp.data)) == 0) + if (memcmp(cp.data, hdev->eir, sizeof(cp.data)) == 0) { + hci_dev_unlock(hdev); return 0; + } memcpy(hdev->eir, cp.data, sizeof(cp.data)); + hci_dev_unlock(hdev); return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_EIR, sizeof(cp), &cp, HCI_CMD_TIMEOUT); @@ -966,6 +970,7 @@ int hci_update_class_sync(struct hci_dev *hdev) if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE)) return 0; + hci_dev_lock(hdev); cod[0] = hdev->minor_class; cod[1] = hdev->major_class; cod[2] = get_service_classes(hdev); @@ -973,8 +978,12 @@ int hci_update_class_sync(struct hci_dev *hdev) if (hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) cod[1] |= 0x20; - if (memcmp(cod, hdev->dev_class, 3) == 0) + if (memcmp(cod, hdev->dev_class, 3) == 0) { + hci_dev_unlock(hdev); return 0; + } + + hci_dev_unlock(hdev); return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_CLASS_OF_DEV, sizeof(cod), cod, HCI_CMD_TIMEOUT); @@ -1054,14 +1063,19 @@ static int hci_set_random_addr_sync(struct hci_dev *hdev, bdaddr_t *rpa) * In this kind of scenario skip the update and let the random * address be updated at the next cycle. */ + rcu_read_lock(); + if (bacmp(&hdev->random_addr, BDADDR_ANY) && (hci_dev_test_flag(hdev, HCI_LE_ADV) || hci_lookup_le_connect(hdev))) { bt_dev_dbg(hdev, "Deferring random address update"); hci_dev_set_flag(hdev, HCI_RPA_EXPIRED); + rcu_read_unlock(); return 0; } + rcu_read_unlock(); + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_RANDOM_ADDR, 6, rpa, HCI_CMD_TIMEOUT); } @@ -1219,10 +1233,11 @@ static int hci_set_adv_set_random_addr_sync(struct hci_dev *hdev, u8 instance, } static int -hci_set_ext_adv_params_sync(struct hci_dev *hdev, struct adv_info *adv, +hci_set_ext_adv_params_sync(struct hci_dev *hdev, u8 instance, const struct hci_cp_le_set_ext_adv_params *cp, struct hci_rp_le_set_ext_adv_params *rp) { + struct adv_info *adv; struct sk_buff *skb; skb = __hci_cmd_sync(hdev, HCI_OP_LE_SET_EXT_ADV_PARAMS, sizeof(*cp), @@ -1250,11 +1265,15 @@ hci_set_ext_adv_params_sync(struct hci_dev *hdev, struct adv_info *adv, if (!rp->status) { hdev->adv_addr_type = cp->own_addr_type; - if (!cp->handle) { + if (!instance) { /* Store in hdev for instance 0 */ hdev->adv_tx_power = rp->tx_power; - } else if (adv) { - adv->tx_power = rp->tx_power; + } else { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (adv) + adv->tx_power = rp->tx_power; + hci_dev_unlock(hdev); } } @@ -1270,9 +1289,13 @@ static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) int err; if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); - if (!adv || !adv->adv_data_changed) + if (!adv || !adv->adv_data_changed) { + hci_dev_unlock(hdev); return 0; + } } len = eir_create_adv_data(hdev, instance, pdu->data, @@ -1283,16 +1306,27 @@ static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) pdu->operation = LE_SET_ADV_DATA_OP_COMPLETE; pdu->frag_pref = LE_SET_ADV_DATA_NO_FRAG; + if (adv) { + adv->adv_data_changed = false; + hci_dev_unlock(hdev); + } + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_DATA, struct_size(pdu, data, len), pdu, HCI_CMD_TIMEOUT); - if (err) + if (err) { + if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (adv) + adv->adv_data_changed = true; + hci_dev_unlock(hdev); + } + return err; + } - /* Update data if the command succeed */ - if (adv) { - adv->adv_data_changed = false; - } else { + if (!instance) { memcpy(hdev->adv_data, pdu->data, len); hdev->adv_data_len = len; } @@ -1346,22 +1380,22 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) struct adv_info *adv; bool secondary_adv; - if (instance > 0) { - adv = hci_find_adv_instance(hdev, instance); - if (!adv) - return -EINVAL; - } else { - adv = NULL; - } - /* Updating parameters of an active instance will return a - * Command Disallowed error, so we must first disable the - * instance if it is active. + * Command Disallowed error, so disable it before taking a snapshot. */ - if (adv) { + if (instance > 0) { err = hci_disable_ext_adv_instance_sync(hdev, instance); if (err) return err; + + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (!adv) { + hci_dev_unlock(hdev); + return -EINVAL; + } + } else { + adv = NULL; } flags = hci_adv_instance_flags(hdev, instance); @@ -1372,8 +1406,11 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) || mgmt_get_connectable(hdev); - if (!is_advertising_allowed(hdev, connectable)) + if (!is_advertising_allowed(hdev, connectable)) { + if (instance) + hci_dev_unlock(hdev); return -EPERM; + } /* Set require_privacy to true only when non-connectable * advertising is used and it is not periodic. @@ -1384,8 +1421,11 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) err = hci_get_random_address(hdev, require_privacy, adv_use_rpa(hdev, flags), adv, &own_addr_type, &random_addr); - if (err < 0) + if (err < 0) { + if (instance) + hci_dev_unlock(hdev); return err; + } memset(&cp, 0, sizeof(cp)); @@ -1436,6 +1476,9 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) cp.channel_map = hdev->le_adv_channel_map; cp.handle = adv ? adv->handle : instance; + if (instance) + hci_dev_unlock(hdev); + if (flags & MGMT_ADV_FLAG_SEC_2M) { cp.primary_phy = HCI_ADV_PHY_1M; cp.secondary_phy = HCI_ADV_PHY_2M; @@ -1448,12 +1491,12 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) cp.secondary_phy = HCI_ADV_PHY_1M; } - err = hci_set_ext_adv_params_sync(hdev, adv, &cp, &rp); + err = hci_set_ext_adv_params_sync(hdev, instance, &cp, &rp); if (err) return err; /* Update adv data as tx power is known now */ - err = hci_set_ext_adv_data_sync(hdev, cp.handle); + err = hci_set_ext_adv_data_sync(hdev, instance); if (err) return err; @@ -1461,9 +1504,14 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) own_addr_type == ADDR_LE_DEV_RANDOM_RESOLVED) && bacmp(&random_addr, BDADDR_ANY)) { /* Check if random address need to be updated */ - if (adv) { - if (!bacmp(&random_addr, &adv->random_addr)) + if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (!adv || !bacmp(&random_addr, &adv->random_addr)) { + hci_dev_unlock(hdev); return 0; + } + hci_dev_unlock(hdev); } else { if (!bacmp(&random_addr, &hdev->random_addr)) return 0; @@ -1485,9 +1533,13 @@ static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) int err; if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); - if (!adv || !adv->scan_rsp_changed) + if (!adv || !adv->scan_rsp_changed) { + hci_dev_unlock(hdev); return 0; + } } len = eir_create_scan_rsp(hdev, instance, pdu->data); @@ -1497,15 +1549,27 @@ static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) pdu->operation = LE_SET_ADV_DATA_OP_COMPLETE; pdu->frag_pref = LE_SET_ADV_DATA_NO_FRAG; + if (adv) { + adv->scan_rsp_changed = false; + hci_dev_unlock(hdev); + } + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_RSP_DATA, struct_size(pdu, data, len), pdu, HCI_CMD_TIMEOUT); - if (err) + if (err) { + if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (adv) + adv->scan_rsp_changed = true; + hci_dev_unlock(hdev); + } + return err; + } - if (adv) { - adv->scan_rsp_changed = false; - } else { + if (!instance) { memcpy(hdev->scan_rsp_data, pdu->data, len); hdev->scan_rsp_data_len = len; } @@ -1520,8 +1584,14 @@ static int __hci_set_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) memset(&cp, 0, sizeof(cp)); + if (instance) + hci_dev_lock(hdev); + len = eir_create_scan_rsp(hdev, instance, cp.data); + if (instance) + hci_dev_unlock(hdev); + if (hdev->scan_rsp_data_len == len && !memcmp(cp.data, hdev->scan_rsp_data, len)) return 0; @@ -1656,9 +1726,13 @@ static int hci_set_per_adv_data_sync(struct hci_dev *hdev, u8 instance) struct adv_info *adv = NULL; if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); - if (!adv || !adv->periodic) + if (!adv || !adv->periodic) { + hci_dev_unlock(hdev); return 0; + } } len = eir_create_per_adv_data(hdev, instance, pdu->data); @@ -1667,6 +1741,9 @@ static int hci_set_per_adv_data_sync(struct hci_dev *hdev, u8 instance) pdu->handle = adv ? adv->handle : instance; pdu->operation = LE_SET_ADV_DATA_OP_COMPLETE; + if (adv) + hci_dev_unlock(hdev); + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_DATA, struct_size(pdu, data, len), pdu, HCI_CMD_TIMEOUT); @@ -2647,12 +2724,17 @@ static int hci_pause_addr_resolution(struct hci_dev *hdev) /* Cannot disable addr resolution if scanning is enabled or * when initiating an LE connection. */ + rcu_read_lock(); + if (hci_dev_test_flag(hdev, HCI_LE_SCAN) || hci_lookup_le_connect(hdev)) { + rcu_read_unlock(); bt_dev_err(hdev, "Command not allowed when scan/LE connect"); return -EPERM; } + rcu_read_unlock(); + /* Cannot disable addr resolution if advertising is enabled. */ err = hci_pause_advertising_sync(hdev); if (err) { @@ -2790,6 +2872,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) if (hci_dev_test_flag(hdev, HCI_PA_SYNC)) { struct hci_conn *conn; + rcu_read_lock(); + conn = hci_conn_hash_lookup_create_pa_sync(hdev); if (conn) { struct conn_params pa; @@ -2799,6 +2883,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) bacpy(&pa.addr, &conn->dst); pa.addr_type = conn->dst_type; + rcu_read_unlock(); + /* Clear first since there could be addresses left * behind. */ @@ -2808,6 +2894,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) err = hci_le_add_accept_list_sync(hdev, &pa, &num_entries); goto done; + } else { + rcu_read_unlock(); } } @@ -2818,10 +2906,13 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) * the controller. */ list_for_each_entry_safe(b, t, &hdev->le_accept_list, list) { - if (hci_conn_hash_lookup_le(hdev, &b->bdaddr, b->bdaddr_type)) + rcu_read_lock(); + + if (hci_conn_hash_lookup_le(hdev, &b->bdaddr, b->bdaddr_type)) { + rcu_read_unlock(); continue; + } - /* Pointers not dereferenced, no locks needed */ pend_conn = hci_pend_le_action_lookup(&hdev->pend_le_conns, &b->bdaddr, b->bdaddr_type); @@ -2829,6 +2920,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) &b->bdaddr, b->bdaddr_type); + rcu_read_unlock(); + /* If the device is not likely to connect or report, * remove it from the acceptlist. */ @@ -2955,6 +3048,8 @@ static int hci_le_set_ext_scan_param_sync(struct hci_dev *hdev, u8 type, if (sent) { struct hci_conn *conn; + rcu_read_lock(); + conn = hci_conn_hash_lookup_ba(hdev, PA_LINK, &sent->bdaddr); if (conn) { @@ -2979,8 +3074,12 @@ static int hci_le_set_ext_scan_param_sync(struct hci_dev *hdev, u8 type, phy++; } + rcu_read_unlock(); + if (num_phy) goto done; + } else { + rcu_read_unlock(); } } } @@ -3231,12 +3330,16 @@ int hci_update_passive_scan_sync(struct hci_dev *hdev) /* If there is at least one pending LE connection, we should * keep the background scan running. */ + bool exists; /* If controller is connecting, we should not start scanning * since some controllers are not able to scan and connect at * the same time. */ - if (hci_lookup_le_connect(hdev)) + rcu_read_lock(); + exists = hci_lookup_le_connect(hdev); + rcu_read_unlock(); + if (exists) return 0; bt_dev_dbg(hdev, "start background scanning"); @@ -3454,6 +3557,7 @@ int hci_write_fast_connectable_sync(struct hci_dev *hdev, bool enable) } static bool disconnected_accept_list_entries(struct hci_dev *hdev) + __must_hold(&hdev->lock) { struct bdaddr_list *b; @@ -3494,12 +3598,16 @@ int hci_update_scan_sync(struct hci_dev *hdev) if (hdev->scanning_paused) return 0; + hci_dev_lock(hdev); + if (hci_dev_test_flag(hdev, HCI_CONNECTABLE) || disconnected_accept_list_entries(hdev)) scan = SCAN_PAGE; else scan = SCAN_DISABLED; + hci_dev_unlock(hdev); + if (hci_dev_test_flag(hdev, HCI_DISCOVERABLE)) scan |= SCAN_INQUIRY; @@ -6478,7 +6586,7 @@ static int hci_le_ext_directed_advertising_sync(struct hci_dev *hdev, if (err) return err; - err = hci_set_ext_adv_params_sync(hdev, NULL, &cp, &rp); + err = hci_set_ext_adv_params_sync(hdev, 0, &cp, &rp); if (err) return err; @@ -6644,7 +6752,9 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) if (hci_dev_test_flag(hdev, HCI_LE_SCAN) && hdev->le_scan_type == LE_SCAN_ACTIVE && !hci_dev_test_flag(hdev, HCI_LE_SIMULTANEOUS_ROLES)) { - hci_conn_del(conn); + conn->state = BT_OPEN; + hci_abort_conn_sync(hdev, conn, + HCI_ERROR_REJ_LIMITED_RESOURCES); return -EBUSY; } @@ -6659,6 +6769,8 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) if (!hci_dev_test_flag(hdev, HCI_LE_SIMULTANEOUS_ROLES)) hci_pause_advertising_sync(hdev); + hci_dev_lock(hdev); + params = hci_conn_params_lookup(hdev, &conn->dst, conn->dst_type); if (params) { conn->le_conn_min_interval = params->conn_min_interval; @@ -6672,6 +6784,8 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) conn->le_supv_timeout = hdev->le_supv_timeout; } + hci_dev_unlock(hdev); + /* If controller is scanning, we stop it since some controllers are * not able to scan and connect at the same time. Also set the * HCI_LE_SCAN_INTERRUPTED flag so that the command complete @@ -6690,6 +6804,12 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) &own_addr_type); if (err) goto done; + + /* Mark create connection in flight so hci_cancel_connect_sync() can + * cancel it while blocking on the connection complete event. + */ + set_bit(HCI_CONN_CREATE, &conn->flags); + /* Send command LE Extended Create Connection if supported */ if (use_ext_conn(hdev)) { err = hci_le_ext_create_conn_sync(hdev, conn, own_addr_type); @@ -6725,6 +6845,8 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) conn->conn_timeout, NULL); done: + clear_bit(HCI_CONN_CREATE, &conn->flags); + if (err == -ETIMEDOUT) hci_le_connect_cancel_sync(hdev, conn, 0x00); @@ -7004,18 +7126,38 @@ static int hci_acl_create_conn_sync(struct hci_dev *hdev, void *data) else cp.role_switch = 0x00; - return __hci_cmd_sync_status_sk(hdev, HCI_OP_CREATE_CONN, - sizeof(cp), &cp, - HCI_EV_CONN_COMPLETE, - conn->conn_timeout, NULL); + /* Mark create connection in flight so hci_cancel_connect_sync() can + * cancel it while blocking on the connection complete event. + */ + set_bit(HCI_CONN_CREATE, &conn->flags); + + err = __hci_cmd_sync_status_sk(hdev, HCI_OP_CREATE_CONN, + sizeof(cp), &cp, + HCI_EV_CONN_COMPLETE, + conn->conn_timeout, NULL); + + clear_bit(HCI_CONN_CREATE, &conn->flags); + + return err; +} + +static void hci_acl_create_conn_sync_complete(struct hci_dev *hdev, void *data, + int err) +{ + struct hci_conn *conn = data; + + hci_conn_put(conn); } int hci_connect_acl_sync(struct hci_dev *hdev, struct hci_conn *conn) { int err; - err = hci_cmd_sync_queue_once(hdev, hci_acl_create_conn_sync, conn, - NULL); + err = hci_cmd_sync_queue_once(hdev, hci_acl_create_conn_sync, + hci_conn_get(conn), + hci_acl_create_conn_sync_complete); + if (err) + hci_conn_put(conn); return (err == -EEXIST) ? 0 : err; } @@ -7026,55 +7168,135 @@ static void create_le_conn_complete(struct hci_dev *hdev, void *data, int err) bt_dev_dbg(hdev, "err %d", err); if (err == -ECANCELED) - return; + goto done; hci_dev_lock(hdev); if (!hci_conn_valid(hdev, conn)) - goto done; + goto unlock; if (!err) { hci_connect_le_scan_cleanup(conn, 0x00); - goto done; + goto unlock; } /* Check if connection is still pending */ if (conn != hci_lookup_le_connect(hdev)) - goto done; + goto unlock; /* Flush to make sure we send create conn cancel command if needed */ flush_delayed_work(&conn->le_conn_timeout); hci_conn_failed(conn, bt_status(err)); -done: +unlock: hci_dev_unlock(hdev); +done: + hci_conn_put(conn); } int hci_connect_le_sync(struct hci_dev *hdev, struct hci_conn *conn) { int err; - err = hci_cmd_sync_queue_once(hdev, hci_le_create_conn_sync, conn, + err = hci_cmd_sync_queue_once(hdev, hci_le_create_conn_sync, + hci_conn_get(conn), create_le_conn_complete); + if (err) + hci_conn_put(conn); return (err == -EEXIST) ? 0 : err; } -int hci_cancel_connect_sync(struct hci_dev *hdev, struct hci_conn *conn) +static int hci_acl_cancel_create_conn_sync(struct hci_dev *hdev, + struct hci_conn *conn) { - if (conn->state != BT_OPEN) - return -EINVAL; + struct hci_cmd_sync_work_entry *entry; + int err = -EBUSY; + + /* cmd_sync_work_lock makes the HCI_CONN_CREATE test and the cancel + * atomic against the worker, which takes this lock to dequeue every + * entry: while it is held no other command can become pending, so + * hci_cmd_sync_cancel() cannot cancel an unrelated command. + */ + mutex_lock(&hdev->cmd_sync_work_lock); + + /* In flight: this connection owns the pending request, cancel it. */ + if (test_bit(HCI_CONN_CREATE, &conn->flags)) { + hci_cmd_sync_cancel(hdev, ECANCELED); + goto unlock; + } + + /* Still queued: a successful dequeue means it never started, so there + * is nothing to disconnect. + */ + entry = _hci_cmd_sync_lookup_entry(hdev, hci_acl_create_conn_sync, conn, + NULL); + if (entry) { + _hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED); + err = 0; + } + +unlock: + mutex_unlock(&hdev->cmd_sync_work_lock); + return err; +} + +static int hci_le_cancel_create_conn_sync(struct hci_dev *hdev, + struct hci_conn *conn) +{ + struct hci_cmd_sync_work_entry *entry; + int err = -EBUSY; + + /* cmd_sync_work_lock keeps the HCI_CONN_CREATE test and the cancel + * atomic against the cmd_sync worker. + */ + mutex_lock(&hdev->cmd_sync_work_lock); + + if (test_bit(HCI_CONN_CREATE, &conn->flags)) { + hci_cmd_sync_cancel(hdev, ECANCELED); + goto unlock; + } + + entry = _hci_cmd_sync_lookup_entry(hdev, hci_le_create_conn_sync, conn, + create_le_conn_complete); + if (entry) { + _hci_cmd_sync_cancel_entry(hdev, entry, -ECANCELED); + err = 0; + } + +unlock: + mutex_unlock(&hdev->cmd_sync_work_lock); + return err; +} + +static int hci_cis_cancel_create_conn_sync(struct hci_dev *hdev, + struct hci_conn *conn) +{ + /* LE Create CIS is shared by the whole CIG and cannot be dequeued + * per-connection, so only an in-flight command can be cancelled. + * cmd_sync_work_lock keeps the test and the cancel atomic against the + * cmd_sync worker. + */ + mutex_lock(&hdev->cmd_sync_work_lock); + if (test_bit(HCI_CONN_CREATE_CIS, &conn->flags)) + hci_cmd_sync_cancel(hdev, ECANCELED); + + mutex_unlock(&hdev->cmd_sync_work_lock); + return -EBUSY; +} + +int hci_cancel_connect_sync(struct hci_dev *hdev, struct hci_conn *conn) +{ switch (conn->type) { case ACL_LINK: - return !hci_cmd_sync_dequeue_once(hdev, - hci_acl_create_conn_sync, - conn, NULL); + return hci_acl_cancel_create_conn_sync(hdev, conn); case LE_LINK: - return !hci_cmd_sync_dequeue_once(hdev, hci_le_create_conn_sync, - conn, create_le_conn_complete); + return hci_le_cancel_create_conn_sync(hdev, conn); + case CIS_LINK: + return hci_cis_cancel_create_conn_sync(hdev, conn); + default: + return -ENOENT; } - - return -ENOENT; } int hci_le_conn_update_sync(struct hci_dev *hdev, struct hci_conn *conn, @@ -7103,7 +7325,7 @@ static void create_pa_complete(struct hci_dev *hdev, void *data, int err) bt_dev_dbg(hdev, "err %d", err); if (err == -ECANCELED) - return; + goto done; hci_dev_lock(hdev); @@ -7127,16 +7349,18 @@ static void create_pa_complete(struct hci_dev *hdev, void *data, int err) unlock: hci_dev_unlock(hdev); +done: + hci_conn_put(conn); } static int hci_le_past_params_sync(struct hci_dev *hdev, struct hci_conn *conn, - struct hci_conn *acl, struct bt_iso_qos *qos) + u16 acl_handle, struct bt_iso_qos *qos) { struct hci_cp_le_past_params cp; int err; memset(&cp, 0, sizeof(cp)); - cp.handle = cpu_to_le16(acl->handle); + cp.handle = cpu_to_le16(acl_handle); /* An HCI_LE_Periodic_Advertising_Sync_Transfer_Received event is sent * to the Host. HCI_LE_Periodic_Advertising_Report events will be * enabled with duplicate filtering enabled. @@ -7201,16 +7425,28 @@ static int hci_le_pa_create_sync(struct hci_dev *hdev, void *data) * 2. Check if that HCI_CONN_FLAG_PAST has been set which indicates that * user really intended to use PAST. */ + hci_dev_lock(hdev); + le = hci_conn_hash_lookup_le(hdev, &conn->dst, conn->dst_type); if (le) { struct hci_conn_params *params; + hci_conn_flags_t flags = 0; + u16 le_handle = le->handle; params = hci_conn_params_lookup(hdev, &le->dst, le->dst_type); - if (params && params->flags & HCI_CONN_FLAG_PAST) { - err = hci_le_past_params_sync(hdev, conn, le, qos); + if (params) + flags = params->flags; + + hci_dev_unlock(hdev); + + if (flags & HCI_CONN_FLAG_PAST) { + err = hci_le_past_params_sync(hdev, conn, le_handle, + qos); if (!err) goto done; } + } else { + hci_dev_unlock(hdev); } /* SID has not been set listen for HCI_EV_LE_EXT_ADV_REPORT to update @@ -7265,8 +7501,11 @@ int hci_connect_pa_sync(struct hci_dev *hdev, struct hci_conn *conn) { int err; - err = hci_cmd_sync_queue_once(hdev, hci_le_pa_create_sync, conn, + err = hci_cmd_sync_queue_once(hdev, hci_le_pa_create_sync, + hci_conn_get(conn), create_pa_complete); + if (err) + hci_conn_put(conn); return (err == -EEXIST) ? 0 : err; } @@ -7277,10 +7516,12 @@ static void create_big_complete(struct hci_dev *hdev, void *data, int err) bt_dev_dbg(hdev, "err %d", err); if (err == -ECANCELED) - return; + goto done; - if (hci_conn_valid(hdev, conn)) - clear_bit(HCI_CONN_CREATE_BIG_SYNC, &conn->flags); + clear_bit(HCI_CONN_CREATE_BIG_SYNC, &conn->flags); + +done: + hci_conn_put(conn); } static int hci_le_big_create_sync(struct hci_dev *hdev, void *data) @@ -7332,8 +7573,14 @@ int hci_connect_big_sync(struct hci_dev *hdev, struct hci_conn *conn) { int err; - err = hci_cmd_sync_queue_once(hdev, hci_le_big_create_sync, conn, + if (!conn) + return 0; + + err = hci_cmd_sync_queue_once(hdev, hci_le_big_create_sync, + hci_conn_get(conn), create_big_complete); + if (err) + hci_conn_put(conn); return (err == -EEXIST) ? 0 : err; } @@ -7348,6 +7595,8 @@ static void past_complete(struct hci_dev *hdev, void *data, int err) bt_dev_dbg(hdev, "err %d", err); + hci_conn_put(past->conn); + hci_conn_put(past->le); kfree(past); } @@ -7412,8 +7661,8 @@ int hci_past_sync(struct hci_conn *conn, struct hci_conn *le) if (!data) return -ENOMEM; - data->conn = conn; - data->le = le; + data->conn = hci_conn_get(conn); + data->le = hci_conn_get(le); if (conn->role == HCI_ROLE_MASTER) err = hci_cmd_sync_queue_once(conn->hdev, @@ -7423,8 +7672,11 @@ int hci_past_sync(struct hci_conn *conn, struct hci_conn *le) err = hci_cmd_sync_queue_once(conn->hdev, hci_le_past_sync, data, past_complete); - if (err) + if (err) { + hci_conn_put(data->conn); + hci_conn_put(data->le); kfree(data); + } return (err == -EEXIST) ? 0 : err; } diff --git a/net/bluetooth/hidp/core.c b/net/bluetooth/hidp/core.c index 0e24c5e2955e..f5bdf9f1ca63 100644 --- a/net/bluetooth/hidp/core.c +++ b/net/bluetooth/hidp/core.c @@ -543,9 +543,10 @@ static int hidp_process_data(struct hidp_session *session, struct sk_buff *skb, } if (test_bit(HIDP_WAITING_FOR_RETURN, &session->flags) && - param == session->waiting_report_type) { + param == session->waiting_report_type) { if (session->waiting_report_number < 0 || - session->waiting_report_number == skb->data[0]) { + (skb->len && + session->waiting_report_number == skb->data[0])) { /* hidp_get_raw_report() is waiting on this report. */ session->report_return = skb; done_with_skb = 0; @@ -560,16 +561,18 @@ static int hidp_process_data(struct hidp_session *session, struct sk_buff *skb, static void hidp_recv_ctrl_frame(struct hidp_session *session, struct sk_buff *skb) { - unsigned char hdr, type, param; + unsigned char type, param; + u8 *hdr; int free_skb = 1; BT_DBG("session %p skb %p len %u", session, skb, skb->len); - hdr = skb->data[0]; - skb_pull(skb, 1); + hdr = skb_pull_data(skb, 1); + if (!hdr) + goto free; - type = hdr & HIDP_HEADER_TRANS_MASK; - param = hdr & HIDP_HEADER_PARAM_MASK; + type = *hdr & HIDP_HEADER_TRANS_MASK; + param = *hdr & HIDP_HEADER_PARAM_MASK; switch (type) { case HIDP_TRANS_HANDSHAKE: @@ -590,6 +593,7 @@ static void hidp_recv_ctrl_frame(struct hidp_session *session, break; } +free: if (free_skb) kfree_skb(skb); } @@ -597,14 +601,15 @@ static void hidp_recv_ctrl_frame(struct hidp_session *session, static void hidp_recv_intr_frame(struct hidp_session *session, struct sk_buff *skb) { - unsigned char hdr; + u8 *hdr; BT_DBG("session %p skb %p len %u", session, skb, skb->len); - hdr = skb->data[0]; - skb_pull(skb, 1); + hdr = skb_pull_data(skb, 1); + if (!hdr) + goto free; - if (hdr == (HIDP_TRANS_DATA | HIDP_DATA_RTYPE_INPUT)) { + if (*hdr == (HIDP_TRANS_DATA | HIDP_DATA_RTYPE_INPUT)) { hidp_set_timer(session); if (session->input) @@ -616,9 +621,10 @@ static void hidp_recv_intr_frame(struct hidp_session *session, BT_DBG("report len %d", skb->len); } } else { - BT_DBG("Unsupported protocol header 0x%02x", hdr); + BT_DBG("Unsupported protocol header 0x%02x", *hdr); } +free: kfree_skb(skb); } diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index 793a481d7066..a461c8a4efed 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -24,15 +24,19 @@ static struct bt_sock_list iso_sk_list = { }; /* ---- ISO connections ---- */ +enum { + ISO_CONN_DROPPED, + __ISO_CONN_NUM_FLAGS +}; + struct iso_conn { struct hci_conn *hcon; + DECLARE_BITMAP(flags, __ISO_CONN_NUM_FLAGS); /* @lock: spinlock protecting changes to iso_conn fields */ spinlock_t lock; struct sock *sk; - struct delayed_work timeout_work; - struct sk_buff *rx_skb; __u32 rx_len; __u16 tx_sn; @@ -56,6 +60,7 @@ static void iso_sock_kill(struct sock *sk); enum { BT_SK_BIG_SYNC, BT_SK_PA_SYNC, + BT_SK_KILLED, }; struct iso_pinfo { @@ -74,6 +79,7 @@ struct iso_pinfo { __u8 base_len; __u8 base[BASE_MAX_LENGTH]; struct iso_conn *conn; + struct delayed_work timeout_work; }; static struct bt_iso_qos default_qos; @@ -102,16 +108,18 @@ static void iso_conn_free(struct kref *ref) BT_DBG("conn %p", conn); - if (conn->sk) - iso_pi(conn->sk)->conn = NULL; - if (conn->hcon) { - conn->hcon->iso_data = NULL; - hci_conn_drop(conn->hcon); - } + spin_lock(&conn->hcon->proto_lock); - /* Ensure no more work items will run since hci_conn has been dropped */ - disable_delayed_work_sync(&conn->timeout_work); + /* Check we are not racing with iso_conn_add */ + if (conn->hcon->iso_data == conn) { + conn->hcon->iso_data = NULL; + if (!test_and_set_bit(ISO_CONN_DROPPED, conn->flags)) + hci_conn_drop(conn->hcon); + } + + spin_unlock(&conn->hcon->proto_lock); + } kfree_skb(conn->rx_skb); @@ -125,7 +133,21 @@ static void iso_conn_put(struct iso_conn *conn) BT_DBG("conn %p refcnt %d", conn, kref_read(&conn->ref)); + /* The following race vs. iso_conn_del() is possible: + * + * 1. conn->hcon != NULL here + * 2. kref_put puts the last reference + * 3. concurrent iso_conn_del() gets iso_conn_hold_unless_zero() -> NULL + * and returns immediately, so conn->hcon is not cleared + * 4. iso_conn_free() dereferences conn->hcon + * + * To avoid UAF in step 4, take RCU before decrementing the refcount. + */ + rcu_read_lock(); + kref_put(&conn->ref, iso_conn_free); + + rcu_read_unlock(); } static struct iso_conn *iso_conn_hold_unless_zero(struct iso_conn *conn) @@ -141,6 +163,14 @@ static struct iso_conn *iso_conn_hold_unless_zero(struct iso_conn *conn) return conn; } +static struct iso_conn *iso_conn_hold(struct iso_conn *conn) +{ + BT_DBG("conn %p refcnt %u", conn, kref_read(&conn->ref)); + + kref_get(&conn->ref); + return conn; +} + static struct sock *iso_sock_hold(struct iso_conn *conn) { if (!conn || !bt_sock_linked(&iso_sk_list, conn->sk)) @@ -153,78 +183,81 @@ static struct sock *iso_sock_hold(struct iso_conn *conn) static void iso_sock_timeout(struct work_struct *work) { - struct iso_conn *conn = container_of(work, struct iso_conn, - timeout_work.work); - struct sock *sk; - - conn = iso_conn_hold_unless_zero(conn); - if (!conn) - return; - - iso_conn_lock(conn); - sk = iso_sock_hold(conn); - iso_conn_unlock(conn); - iso_conn_put(conn); - - if (!sk) - return; + struct iso_pinfo *pi = container_of(work, struct iso_pinfo, + timeout_work.work); + struct sock *sk = &pi->bt.sk; BT_DBG("sock %p state %d", sk, sk->sk_state); lock_sock(sk); - sk->sk_err = ETIMEDOUT; - sk->sk_state_change(sk); + if (!sock_flag(sk, SOCK_ZAPPED)) { + sk->sk_err = ETIMEDOUT; + sk->sk_state_change(sk); + } release_sock(sk); - sock_put(sk); } static void iso_sock_set_timer(struct sock *sk, long timeout) { + lockdep_assert(lockdep_sock_is_held(sk)); + + cancel_delayed_work(&iso_pi(sk)->timeout_work); + if (!iso_pi(sk)->conn) return; BT_DBG("sock %p state %d timeout %ld", sk, sk->sk_state, timeout); - cancel_delayed_work(&iso_pi(sk)->conn->timeout_work); - schedule_delayed_work(&iso_pi(sk)->conn->timeout_work, timeout); + schedule_delayed_work(&iso_pi(sk)->timeout_work, timeout); } static void iso_sock_clear_timer(struct sock *sk) { - if (!iso_pi(sk)->conn) - return; + BT_DBG("sock %p state %d", sk, sk->sk_state); + cancel_delayed_work(&iso_pi(sk)->timeout_work); +} + +static void iso_sock_disable_timer(struct sock *sk) +{ + lockdep_assert(!lockdep_sock_is_held(sk)); BT_DBG("sock %p state %d", sk, sk->sk_state); - cancel_delayed_work(&iso_pi(sk)->conn->timeout_work); + disable_delayed_work_sync(&iso_pi(sk)->timeout_work); } /* ---- ISO connections ---- */ static struct iso_conn *iso_conn_add(struct hci_conn *hcon) + __must_hold(&hcon->hdev->lock) { - struct iso_conn *conn = hcon->iso_data; + struct iso_conn *conn; + + spin_lock(&hcon->proto_lock); - conn = iso_conn_hold_unless_zero(conn); + conn = iso_conn_hold_unless_zero(hcon->iso_data); if (conn) { if (!conn->hcon) { iso_conn_lock(conn); conn->hcon = hcon; iso_conn_unlock(conn); } - iso_conn_put(conn); + spin_unlock(&hcon->proto_lock); return conn; } - conn = kzalloc_obj(*conn); - if (!conn) + conn = kzalloc_obj(*conn, GFP_ATOMIC); + if (!conn) { + spin_unlock(&hcon->proto_lock); return NULL; + } kref_init(&conn->ref); spin_lock_init(&conn->lock); - INIT_DELAYED_WORK(&conn->timeout_work, iso_sock_timeout); hcon->iso_data = conn; conn->hcon = hcon; conn->tx_sn = 0; + spin_unlock(&hcon->proto_lock); + BT_DBG("hcon %p conn %p", hcon, conn); return conn; @@ -263,11 +296,14 @@ static void iso_chan_del(struct sock *sk, int err) } static void iso_conn_del(struct hci_conn *hcon, int err) + __must_hold(&hcon->hdev->lock) { - struct iso_conn *conn = hcon->iso_data; + struct iso_conn *conn; struct sock *sk; - conn = iso_conn_hold_unless_zero(conn); + spin_lock(&hcon->proto_lock); + conn = iso_conn_hold_unless_zero(hcon->iso_data); + spin_unlock(&hcon->proto_lock); if (!conn) return; @@ -277,18 +313,28 @@ static void iso_conn_del(struct hci_conn *hcon, int err) iso_conn_lock(conn); sk = iso_sock_hold(conn); iso_conn_unlock(conn); - iso_conn_put(conn); - if (!sk) { - iso_conn_put(conn); - return; - } + if (!sk) + goto done; + + iso_sock_disable_timer(sk); lock_sock(sk); - iso_sock_clear_timer(sk); iso_chan_del(sk, err); release_sock(sk); + iso_sock_kill(sk); sock_put(sk); + +done: + /* No sk access to conn->hcon any more (lock_sock + hdev->lock) */ + spin_lock(&hcon->proto_lock); + iso_conn_lock(conn); + conn->hcon = NULL; + hcon->iso_data = NULL; + iso_conn_unlock(conn); + spin_unlock(&hcon->proto_lock); + + iso_conn_put(conn); } static int __iso_chan_add(struct iso_conn *conn, struct sock *sk, @@ -304,8 +350,14 @@ static int __iso_chan_add(struct iso_conn *conn, struct sock *sk, return -EBUSY; } - iso_pi(sk)->conn = conn; + if (!conn->hcon) { + BT_ERR("conn->hcon missing"); + return -EIO; + } + + iso_pi(sk)->conn = iso_conn_hold(conn); conn->sk = sk; + clear_bit(ISO_CONN_DROPPED, conn->flags); if (parent) bt_accept_enqueue(parent, sk, true); @@ -402,6 +454,8 @@ static int iso_connect_bis(struct sock *sk) iso_pi(sk)->bc_sid = hcon->sid; } + lockdep_assert_held(&hcon->hdev->lock); + conn = iso_conn_add(hcon); if (!conn) { hci_conn_drop(hcon); @@ -410,6 +464,7 @@ static int iso_connect_bis(struct sock *sk) } err = iso_chan_add(conn, sk, NULL); + iso_conn_put(conn); if (err) goto unlock; @@ -504,6 +559,8 @@ static int iso_connect_cis(struct sock *sk) } } + lockdep_assert_held(&hcon->hdev->lock); + conn = iso_conn_add(hcon); if (!conn) { hci_conn_drop(hcon); @@ -512,6 +569,7 @@ static int iso_connect_cis(struct sock *sk) } err = iso_chan_add(conn, sk, NULL); + iso_conn_put(conn); if (err) goto unlock; @@ -790,9 +848,15 @@ static void iso_sock_cleanup_listen(struct sock *parent) */ static void iso_sock_kill(struct sock *sk) { + iso_sock_disable_timer(sk); + + lock_sock(sk); + if (!sock_flag(sk, SOCK_ZAPPED) || sk->sk_socket || - sock_flag(sk, SOCK_DEAD)) + test_bit(BT_SK_KILLED, &iso_pi(sk)->flags)) { + release_sock(sk); return; + } BT_DBG("sk %p state %d", sk, sk->sk_state); @@ -806,6 +870,9 @@ static void iso_sock_kill(struct sock *sk) /* Kill poor orphan */ bt_sock_unlink(&iso_sk_list, sk); sock_set_flag(sk, SOCK_DEAD); + set_bit(BT_SK_KILLED, &iso_pi(sk)->flags); + + release_sock(sk); sock_put(sk); } @@ -825,8 +892,8 @@ static void iso_sock_disconn(struct sock *sk) */ if (bis_sk) { hcon->state = BT_OPEN; - hcon->iso_data = NULL; - iso_pi(sk)->conn->hcon = NULL; + set_bit(ISO_CONN_DROPPED, iso_pi(sk)->conn->flags); + iso_sock_clear_timer(sk); iso_chan_del(sk, bt_to_errno(hcon->abort_reason)); sock_put(bis_sk); @@ -835,10 +902,8 @@ static void iso_sock_disconn(struct sock *sk) } sk->sk_state = BT_DISCONN; - iso_conn_lock(iso_pi(sk)->conn); - hci_conn_drop(iso_pi(sk)->conn->hcon); - iso_pi(sk)->conn->hcon = NULL; - iso_conn_unlock(iso_pi(sk)->conn); + if (!test_and_set_bit(ISO_CONN_DROPPED, iso_pi(sk)->conn->flags)) + hci_conn_drop(iso_pi(sk)->conn->hcon); } static void __iso_sock_close(struct sock *sk) @@ -880,11 +945,11 @@ static void __iso_sock_close(struct sock *sk) /* Must be called on unlocked socket. */ static void iso_sock_close(struct sock *sk) { + iso_sock_disable_timer(sk); + lock_sock(sk); - iso_sock_clear_timer(sk); __iso_sock_close(sk); release_sock(sk); - iso_sock_kill(sk); } static void iso_sock_init(struct sock *sk, struct sock *parent) @@ -951,6 +1016,8 @@ static struct sock *iso_sock_alloc(struct net *net, struct socket *sock, iso_pi(sk)->qos = default_qos; iso_pi(sk)->sync_handle = -1; + INIT_DELAYED_WORK(&iso_pi(sk)->timeout_work, iso_sock_timeout); + bt_sock_link(&iso_sk_list, sk); return sk; } @@ -1033,15 +1100,15 @@ static int iso_sock_rebind_bis(struct sock *sk, struct sockaddr_iso *sa, goto done; } - iso_pi(sk)->bc_num_bis = sa->iso_bc->bc_num_bis; - - for (int i = 0; i < iso_pi(sk)->bc_num_bis; i++) + for (int i = 0; i < sa->iso_bc->bc_num_bis; i++) if (sa->iso_bc->bc_bis[i] < 0x01 || sa->iso_bc->bc_bis[i] > 0x1f) { err = -EINVAL; goto done; } + iso_pi(sk)->bc_num_bis = sa->iso_bc->bc_num_bis; + memcpy(iso_pi(sk)->bc_bis, sa->iso_bc->bc_bis, iso_pi(sk)->bc_num_bis); @@ -1276,6 +1343,8 @@ static int iso_listen_bis(struct sock *sk) goto unlock; } + lockdep_assert_held(&hcon->hdev->lock); + conn = iso_conn_add(hcon); if (!conn) { hci_conn_drop(hcon); @@ -1284,10 +1353,9 @@ static int iso_listen_bis(struct sock *sk) } err = iso_chan_add(conn, sk, NULL); - if (err) { - hci_conn_drop(hcon); + iso_conn_put(conn); + if (err) goto unlock; - } unlock: release_sock(sk); @@ -1466,6 +1534,8 @@ static int iso_sock_getname(struct socket *sock, struct sockaddr *addr, BT_DBG("sock %p, sk %p", sock, sk); + lock_sock(sk); + addr->sa_family = AF_BLUETOOTH; if (peer) { @@ -1487,6 +1557,8 @@ static int iso_sock_getname(struct socket *sock, struct sockaddr *addr, sa->iso_bdaddr_type = iso_pi(sk)->src_type; } + release_sock(sk); + return len; } @@ -1590,6 +1662,7 @@ static void iso_conn_big_sync(struct sock *sk) { int err; struct hci_dev *hdev; + struct iso_conn *conn; bdaddr_t src, dst; u8 src_type; @@ -1612,8 +1685,17 @@ static void iso_conn_big_sync(struct sock *sk) hci_dev_lock(hdev); lock_sock(sk); + /* The socket lock was dropped for hci_get_route(), so the connection + * may have been torn down meanwhile: iso_chan_del() clears conn and + * the broadcast teardown path can clear conn->hcon on its own. Check + * both before dereferencing conn->hcon. + */ + conn = iso_pi(sk)->conn; + if (!conn || !conn->hcon) + goto unlock; + if (!test_and_set_bit(BT_SK_BIG_SYNC, &iso_pi(sk)->flags)) { - err = hci_conn_big_create_sync(hdev, iso_pi(sk)->conn->hcon, + err = hci_conn_big_create_sync(hdev, conn->hcon, &iso_pi(sk)->qos, iso_pi(sk)->sync_handle, iso_pi(sk)->bc_num_bis, @@ -1622,6 +1704,7 @@ static void iso_conn_big_sync(struct sock *sk) bt_dev_err(hdev, "hci_big_create_sync: %d", err); } +unlock: release_sock(sk); hci_dev_unlock(hdev); hci_dev_put(hdev); @@ -1775,7 +1858,7 @@ static bool check_bcast_qos(struct bt_iso_qos *qos) return false; if (!qos->bcast.timeout) - qos->bcast.sync_timeout = BT_ISO_SYNC_TIMEOUT; + qos->bcast.timeout = BT_ISO_SYNC_TIMEOUT; if (qos->bcast.timeout < 0x000a || qos->bcast.timeout > 0x4000) return false; @@ -2017,8 +2100,16 @@ static int iso_sock_release(struct socket *sock) release_sock(sk); } + /* Make sure sk is valid even if iso_conn_del() is concurrent */ + sock_hold(sk); + + lock_sock(sk); sock_orphan(sk); + release_sock(sk); + iso_sock_kill(sk); + + sock_put(sk); return err; } @@ -2026,14 +2117,17 @@ static void iso_sock_ready(struct sock *sk) { BT_DBG("sk %p", sk); - if (!sk) + lockdep_assert(lockdep_sock_is_held(sk)); + + switch (sk->sk_state) { + case BT_DISCONN: + case BT_CLOSED: return; + } - lock_sock(sk); iso_sock_clear_timer(sk); sk->sk_state = BT_CONNECTED; sk->sk_state_change(sk); - release_sock(sk); } static bool iso_match_big(struct sock *sk, void *data) @@ -2063,7 +2157,7 @@ static bool iso_match_dst(struct sock *sk, void *data) static void iso_conn_ready(struct iso_conn *conn) { struct sock *parent = NULL; - struct sock *sk = conn->sk; + struct sock *sk; struct hci_ev_le_big_sync_established *ev = NULL; struct hci_ev_le_pa_sync_established *ev2 = NULL; struct hci_ev_le_per_adv_report *ev3 = NULL; @@ -2072,7 +2166,22 @@ static void iso_conn_ready(struct iso_conn *conn) BT_DBG("conn %p", conn); + iso_conn_lock(conn); + sk = iso_sock_hold(conn); + iso_conn_unlock(conn); + if (sk) { + lock_sock(sk); + + /* conn->sk may have become NULL if racing with sk close, but + * due to held hdev->lock, it can't become different sk. + */ + if (!conn->sk) { + release_sock(sk); + sock_put(sk); + return; + } + /* Attempt to update source address in case of BIS Sender if * the advertisement is using a random address. */ @@ -2085,14 +2194,15 @@ static void iso_conn_ready(struct iso_conn *conn) adv = hci_find_adv_instance(bis->hdev, bis->iso_qos.bcast.bis); if (adv && bacmp(&adv->random_addr, BDADDR_ANY)) { - lock_sock(sk); iso_pi(sk)->src_type = BDADDR_LE_RANDOM; bacpy(&iso_pi(sk)->src, &adv->random_addr); - release_sock(sk); } } - iso_sock_ready(conn->sk); + iso_sock_ready(sk); + + release_sock(sk); + sock_put(sk); } else { hcon = conn->hcon; if (!hcon) @@ -2339,7 +2449,7 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) lock_sock(sk); - hcon = iso_pi(sk)->conn->hcon; + hcon = iso_pi(sk)->conn ? iso_pi(sk)->conn->hcon : NULL; iso_pi(sk)->qos.bcast.encryption = ev2->encryption; if (ev2->num_bis < iso_pi(sk)->bc_num_bis) @@ -2379,9 +2489,11 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) if (!sk) goto done; - hcon = iso_pi(sk)->conn->hcon; + lock_sock(sk); + + hcon = iso_pi(sk)->conn ? iso_pi(sk)->conn->hcon : NULL; if (!hcon) - goto done; + goto release3; if (ev3->data_status == LE_PA_DATA_TRUNCATED) { /* The controller was unable to retrieve PA data. */ @@ -2389,12 +2501,12 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) HCI_MAX_PER_AD_TOT_LEN); hcon->le_per_adv_data_len = 0; hcon->le_per_adv_data_offset = 0; - goto done; + goto release3; } if (hcon->le_per_adv_data_offset + ev3->length > HCI_MAX_PER_AD_TOT_LEN) - goto done; + goto release3; memcpy(hcon->le_per_adv_data + hcon->le_per_adv_data_offset, ev3->data, ev3->length); @@ -2413,18 +2525,19 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) &base_len); if (!base || base_len > BASE_MAX_LENGTH) - goto done; + goto release3; - lock_sock(sk); memcpy(iso_pi(sk)->base, base, base_len); iso_pi(sk)->base_len = base_len; - release_sock(sk); } else { /* This is a PA data fragment. Keep pa_data_len set to 0 * until all data has been reassembled. */ hcon->le_per_adv_data_len = 0; } + +release3: + release_sock(sk); } else { sk = iso_get_sock(hdev, &hdev->bdaddr, BDADDR_ANY, BT_LISTEN, iso_match_dst, BDADDR_ANY); @@ -2443,6 +2556,7 @@ done: } static void iso_connect_cfm(struct hci_conn *hcon, __u8 status) + __must_hold(&hcon->hdev->lock) { if (hcon->type != CIS_LINK && hcon->type != BIS_LINK && hcon->type != PA_LINK) { @@ -2454,8 +2568,10 @@ static void iso_connect_cfm(struct hci_conn *hcon, __u8 status) struct hci_link *link, *t; list_for_each_entry_safe(link, t, &hcon->link_list, - list) + list) { + lockdep_assert_held(&link->conn->hdev->lock); iso_conn_del(link->conn, bt_to_errno(status)); + } return; } @@ -2477,14 +2593,17 @@ static void iso_connect_cfm(struct hci_conn *hcon, __u8 status) struct iso_conn *conn; conn = iso_conn_add(hcon); - if (conn) + if (conn) { iso_conn_ready(conn); + iso_conn_put(conn); + } } else { iso_conn_del(hcon, bt_to_errno(status)); } } static void iso_disconn_cfm(struct hci_conn *hcon, __u8 reason) + __must_hold(&hcon->hdev->lock) { if (hcon->type != CIS_LINK && hcon->type != BIS_LINK && hcon->type != PA_LINK) @@ -2511,7 +2630,10 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) return -ENOENT; } + spin_lock(&hcon->proto_lock); conn = iso_conn_hold_unless_zero(hcon->iso_data); + spin_unlock(&hcon->proto_lock); + hcon = NULL; hci_dev_unlock(hdev); @@ -2529,7 +2651,7 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) switch (pb) { case ISO_START: case ISO_SINGLE: - if (conn->rx_len) { + if (conn->rx_skb || conn->rx_len) { BT_ERR("Unexpected start frame (len %d)", skb->len); kfree_skb(conn->rx_skb); conn->rx_skb = NULL; @@ -2610,12 +2732,14 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) break; case ISO_CONT: - BT_DBG("Cont: frag len %d (expecting %d)", skb->len, + case ISO_END: + BT_DBG("%s: frag len %d (expecting %d)", + (pb == ISO_END) ? "End" : "Cont", skb->len, conn->rx_len); - if (!conn->rx_len) { - BT_ERR("Unexpected continuation frame (len %d)", - skb->len); + if (!conn->rx_skb) { + BT_ERR("Unexpected ISO %s frame (len %d)", + (pb == ISO_END) ? "End" : "Cont", skb->len); goto drop; } @@ -2631,17 +2755,9 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), skb->len); conn->rx_len -= skb->len; - break; - - case ISO_END: - if (!conn->rx_len) { - BT_ERR("Unexpected end frame (len %d)", skb->len); - goto drop; - } - skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), - skb->len); - conn->rx_len -= skb->len; + if (pb == ISO_CONT) + break; if (!conn->rx_len) { struct sk_buff *rx_skb = conn->rx_skb; @@ -2652,6 +2768,13 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) */ conn->rx_skb = NULL; iso_recv_frame(conn, rx_skb); + } else { + BT_ERR("ISO fragment incomplete (len %d, expected %d)", + skb->len, conn->rx_len); + kfree_skb(conn->rx_skb); + conn->rx_skb = NULL; + conn->rx_len = 0; + goto drop; } break; } diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 62133eef9d2f..1156aba4e83c 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -522,7 +522,10 @@ void l2cap_chan_put(struct l2cap_chan *c) } EXPORT_SYMBOL_GPL(l2cap_chan_put); -void l2cap_chan_set_defaults(struct l2cap_chan *chan) +/* Initialise @chan with default values, inheriting from the parent channel + * @pchan when it is given. + */ +void l2cap_chan_set_defaults(struct l2cap_chan *chan, struct l2cap_chan *pchan) { chan->fcs = L2CAP_FCS_CRC16; chan->max_tx = L2CAP_DEFAULT_MAX_TX; @@ -536,6 +539,31 @@ void l2cap_chan_set_defaults(struct l2cap_chan *chan) chan->retrans_timeout = L2CAP_DEFAULT_RETRANS_TO; chan->monitor_timeout = L2CAP_DEFAULT_MONITOR_TO; + if (pchan) { + BT_DBG("chan %p pchan %p", chan, pchan); + + chan->chan_type = pchan->chan_type; + chan->imtu = pchan->imtu; + chan->omtu = pchan->omtu; + chan->mode = pchan->mode; + chan->fcs = pchan->fcs; + chan->max_tx = pchan->max_tx; + chan->tx_win = pchan->tx_win; + chan->tx_win_max = pchan->tx_win_max; + chan->sec_level = pchan->sec_level; + chan->conf_state = pchan->conf_state; + chan->flags = pchan->flags; + chan->tx_credits = pchan->tx_credits; + chan->rx_credits = pchan->rx_credits; + + if (chan->chan_type == L2CAP_CHAN_FIXED) { + chan->scid = pchan->scid; + chan->dcid = pchan->scid; + } + + return; + } + chan->conf_state = 0; set_bit(CONF_NOT_COMPLETE, &chan->conf_state); @@ -1775,19 +1803,13 @@ static void l2cap_conn_del(struct hci_conn *hcon, int err) disable_delayed_work_sync(&conn->info_timer); disable_delayed_work_sync(&conn->id_addr_timer); + cancel_work_sync(&conn->pending_rx_work); + mutex_lock(&conn->lock); kfree_skb(conn->rx_skb); skb_queue_purge(&conn->pending_rx); - - /* We can not call flush_work(&conn->pending_rx_work) here since we - * might block if we are running on a worker from the same workqueue - * pending_rx_work is waiting on. - */ - if (work_pending(&conn->pending_rx_work)) - cancel_work_sync(&conn->pending_rx_work); - ida_destroy(&conn->tx_ida); l2cap_unregister_all_users(conn); @@ -3051,13 +3073,24 @@ fail: return NULL; } -static inline int l2cap_get_conf_opt(void **ptr, int *type, int *olen, - unsigned long *val) +static inline int l2cap_get_conf_opt(void **ptr, void *end, int *type, + int *olen, unsigned long *val) { struct l2cap_conf_opt *opt = *ptr; int len; + /* opt->len is attacker-controlled. Validate that the full option + * (header + value) actually fits in the buffer before touching + * opt->val, otherwise the switch below reads past the end of the + * caller's buffer. + */ + if (end - *ptr < L2CAP_CONF_OPT_SIZE) + return -EINVAL; + len = L2CAP_CONF_OPT_SIZE + opt->len; + if (end - *ptr < len) + return -EINVAL; + *ptr += len; *type = opt->type; @@ -3429,6 +3462,7 @@ static int l2cap_parse_conf_req(struct l2cap_chan *chan, void *data, size_t data void *ptr = rsp->data; void *endptr = data + data_size; void *req = chan->conf_req; + void *req_end = req + chan->conf_len; int len = chan->conf_len; int type, hint, olen; unsigned long val; @@ -3442,9 +3476,11 @@ static int l2cap_parse_conf_req(struct l2cap_chan *chan, void *data, size_t data BT_DBG("chan %p", chan); while (len >= L2CAP_CONF_OPT_SIZE) { - len -= l2cap_get_conf_opt(&req, &type, &olen, &val); - if (len < 0) + int ret = l2cap_get_conf_opt(&req, req_end, &type, &olen, &val); + + if (ret < 0) break; + len -= ret; hint = type & L2CAP_CONF_HINT; type &= L2CAP_CONF_MASK; @@ -3672,6 +3708,7 @@ static int l2cap_parse_conf_rsp(struct l2cap_chan *chan, void *rsp, int len, struct l2cap_conf_req *req = data; void *ptr = req->data; void *endptr = data + size; + void *rsp_end = rsp + len; int type, olen; unsigned long val; struct l2cap_conf_rfc rfc = { .mode = L2CAP_MODE_BASIC }; @@ -3680,9 +3717,11 @@ static int l2cap_parse_conf_rsp(struct l2cap_chan *chan, void *rsp, int len, BT_DBG("chan %p, rsp %p, len %d, req %p", chan, rsp, len, data); while (len >= L2CAP_CONF_OPT_SIZE) { - len -= l2cap_get_conf_opt(&rsp, &type, &olen, &val); - if (len < 0) + int ret = l2cap_get_conf_opt(&rsp, rsp_end, &type, &olen, &val); + + if (ret < 0) break; + len -= ret; switch (type) { case L2CAP_CONF_MTU: @@ -3933,6 +3972,7 @@ static void l2cap_conf_rfc_get(struct l2cap_chan *chan, void *rsp, int len) { int type, olen; unsigned long val; + void *rsp_end = rsp + len; /* Use sane default values in case a misbehaving remote device * did not send an RFC or extended window size option. */ @@ -3951,9 +3991,11 @@ static void l2cap_conf_rfc_get(struct l2cap_chan *chan, void *rsp, int len) return; while (len >= L2CAP_CONF_OPT_SIZE) { - len -= l2cap_get_conf_opt(&rsp, &type, &olen, &val); - if (len < 0) + int ret = l2cap_get_conf_opt(&rsp, rsp_end, &type, &olen, &val); + + if (ret < 0) break; + len -= ret; switch (type) { case L2CAP_CONF_RFC: @@ -4010,6 +4052,38 @@ static inline int l2cap_command_rej(struct l2cap_conn *conn, return 0; } +/* Allocate and initialise a channel for an incoming connection. + * + * The channel inherits its configuration from @pchan and is linked into @conn + * before ->new_connection() runs, so the conn list reference keeps it alive if + * the callback exposes it (e.g. via the socket accept queue) before this + * returns. The l2cap_chan_create() reference is taken over by the subsystem on + * success and dropped here on failure. + */ +static struct l2cap_chan *l2cap_new_connection(struct l2cap_conn *conn, + struct l2cap_chan *pchan) +{ + struct l2cap_chan *chan; + + chan = l2cap_chan_create(); + if (!chan) + return NULL; + + l2cap_chan_set_defaults(chan, pchan); + chan->ops = pchan->ops; + + __l2cap_chan_add(conn, chan); + + if (pchan->ops->new_connection && + pchan->ops->new_connection(pchan, chan) < 0) { + l2cap_chan_del(chan, 0); + l2cap_chan_put(chan); + return NULL; + } + + return chan; +} + static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, u8 *data, u8 rsp_code) { @@ -4056,7 +4130,7 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, goto response; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) goto response; @@ -4074,8 +4148,6 @@ static void l2cap_connect(struct l2cap_conn *conn, struct l2cap_cmd_hdr *cmd, chan->psm = psm; chan->dcid = scid; - __l2cap_chan_add(conn, chan); - dcid = chan->scid; __set_chan_timer(chan, chan->ops->get_sndtimeo(chan)); @@ -4748,6 +4820,10 @@ static int l2cap_le_connect_rsp(struct l2cap_conn *conn, if (!chan) return -EBADSLT; + chan = l2cap_chan_hold_unless_zero(chan); + if (!chan) + return -EBADSLT; + err = 0; l2cap_chan_lock(chan); @@ -4793,6 +4869,7 @@ static int l2cap_le_connect_rsp(struct l2cap_conn *conn, } l2cap_chan_unlock(chan); + l2cap_chan_put(chan); return err; } @@ -4807,6 +4884,7 @@ static void l2cap_put_ident(struct l2cap_conn *conn, u8 code, u8 id) case L2CAP_ECHO_RSP: case L2CAP_INFO_RSP: case L2CAP_CONN_PARAM_UPDATE_RSP: + case L2CAP_LE_CONN_RSP: case L2CAP_ECRED_CONN_RSP: case L2CAP_ECRED_RECONF_RSP: /* First do a lookup since the remote may send bogus ids that @@ -4958,7 +5036,7 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, goto response_unlock; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; goto response_unlock; @@ -4973,8 +5051,6 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_le_flowctl_init(chan, __le16_to_cpu(req->credits)); dcid = chan->scid; @@ -5182,7 +5258,7 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, continue; } - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (!chan) { result = L2CAP_CR_LE_NO_MEM; continue; @@ -5197,8 +5273,6 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, chan->omtu = mtu; chan->remote_mps = mps; - __l2cap_chan_add(conn, chan); - l2cap_ecred_init(chan, __le16_to_cpu(req->credits)); /* Init response */ @@ -6704,6 +6778,7 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) struct l2cap_conn *conn = chan->conn; struct l2cap_le_credits pkt; u16 return_credits = l2cap_le_rx_credits(chan); + int ident; if (chan->mode != L2CAP_MODE_LE_FLOWCTL && chan->mode != L2CAP_MODE_EXT_FLOWCTL) @@ -6721,9 +6796,18 @@ static void l2cap_chan_le_send_credits(struct l2cap_chan *chan) pkt.cid = cpu_to_le16(chan->scid); pkt.credits = cpu_to_le16(return_credits); - chan->ident = l2cap_get_ident(conn); + ident = l2cap_get_ident(conn); - l2cap_send_cmd(conn, chan->ident, L2CAP_LE_CREDITS, sizeof(pkt), &pkt); + l2cap_send_cmd(conn, ident, L2CAP_LE_CREDITS, sizeof(pkt), &pkt); + + /* L2CAP_LE_CREDITS has no response so the ident is never released by + * l2cap_put_ident() - release it right away, otherwise the tx_ida + * range is exhausted after 254 packets and from then on credits are + * sent with the invalid ident 0, which some remote stacks ignore, + * stalling the channel. + */ + if (ident > 0) + ida_free(&conn->tx_ida, ident); } void l2cap_chan_rx_avail(struct l2cap_chan *chan, ssize_t rx_avail) @@ -7478,14 +7562,12 @@ static void l2cap_connect_cfm(struct hci_conn *hcon, u8 status) goto next; l2cap_chan_lock(pchan); - chan = pchan->ops->new_connection(pchan); + chan = l2cap_new_connection(conn, pchan); if (chan) { bacpy(&chan->src, &hcon->src); bacpy(&chan->dst, &hcon->dst); chan->src_type = bdaddr_src_type(hcon); chan->dst_type = dst_type; - - __l2cap_chan_add(conn, chan); } l2cap_chan_unlock(pchan); @@ -7702,6 +7784,7 @@ struct l2cap_conn *l2cap_conn_hold_unless_zero(struct l2cap_conn *c) return c; } +EXPORT_SYMBOL(l2cap_conn_hold_unless_zero); int l2cap_recv_acldata(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 4853f1b33449..4058ff50cc27 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -43,7 +43,8 @@ static struct bt_sock_list l2cap_sk_list = { static const struct proto_ops l2cap_sock_ops; static void l2cap_sock_init(struct sock *sk, struct sock *parent); static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern); + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan); static void l2cap_sock_cleanup_listen(struct sock *parent); bool l2cap_is_socket(struct socket *sock) @@ -1284,6 +1285,23 @@ done: return err; } +/* Release the sock's ref on chan and clear the pointer so that the ref is + * dropped exactly once even if both l2cap_sock_kill() and + * l2cap_sock_destruct() run. Setting chan->data to NULL first stops any other + * task from dereferencing the now-dead sock pointer. + */ +static void l2cap_sock_put_chan(struct sock *sk) +{ + struct l2cap_chan *chan = l2cap_pi(sk)->chan; + + if (!chan) + return; + + chan->data = NULL; + l2cap_pi(sk)->chan = NULL; + l2cap_chan_put(chan); +} + /* Kill socket (only if zapped and orphan) * Must be called on unlocked socket, with l2cap channel lock. */ @@ -1294,13 +1312,9 @@ static void l2cap_sock_kill(struct sock *sk) BT_DBG("sk %p state %s", sk, state_to_string(sk->sk_state)); - /* Sock is dead, so set chan data to NULL, avoid other task use invalid - * sock pointer. - */ - l2cap_pi(sk)->chan->data = NULL; - /* Kill poor orphan */ + l2cap_sock_put_chan(sk); - l2cap_chan_put(l2cap_pi(sk)->chan); + /* Kill poor orphan */ sock_set_flag(sk, SOCK_DEAD); sock_put(sk); } @@ -1492,8 +1506,8 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) /* Close not yet accepted channels. * - * bt_accept_dequeue() now returns sk with an extra reference held - * (taken while sk was still locked) so a concurrent l2cap_conn_del() + * bt_accept_dequeue() returns sk with its temporary queue-walk + * reference held, so a concurrent l2cap_conn_del() * -> l2cap_sock_kill() cannot free sk under us. * * cleanup_listen() runs under the parent sk lock, so unlike @@ -1543,12 +1557,13 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) } } -static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) +static int l2cap_sock_new_connection_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { struct sock *sk, *parent = chan->data; if (!parent) - return NULL; + return -EINVAL; lock_sock(parent); @@ -1556,25 +1571,28 @@ static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) if (sk_acceptq_is_full(parent)) { BT_DBG("backlog full %d", parent->sk_ack_backlog); release_sock(parent); - return NULL; + return -ENOBUFS; } sk = l2cap_sock_alloc(sock_net(parent), NULL, BTPROTO_L2CAP, - GFP_ATOMIC, 0); + GFP_ATOMIC, 0, new_chan); if (!sk) { release_sock(parent); - return NULL; - } + return -ENOMEM; + } bt_sock_reclassify_lock(sk, BTPROTO_L2CAP); l2cap_sock_init(sk, parent); + /* The conn list reference taken by l2cap_new_connection() keeps new_chan + * alive once release_sock() lets another task free this socket. + */ bt_accept_enqueue(parent, sk, false); release_sock(parent); - return l2cap_pi(sk)->chan; + return 0; } static int l2cap_sock_recv_cb(struct l2cap_chan *chan, struct sk_buff *skb) @@ -1871,10 +1889,7 @@ static void l2cap_sock_destruct(struct sock *sk) BT_DBG("sk %p", sk); - if (l2cap_pi(sk)->chan) { - l2cap_pi(sk)->chan->data = NULL; - l2cap_chan_put(l2cap_pi(sk)->chan); - } + l2cap_sock_put_chan(sk); list_for_each_entry_safe(rx_busy, next, &l2cap_pi(sk)->rx_busy, list) { kfree_skb(rx_busy->skb); @@ -1907,30 +1922,12 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) BT_DBG("sk %p", sk); if (parent) { - struct l2cap_chan *pchan = l2cap_pi(parent)->chan; - sk->sk_type = parent->sk_type; bt_sk(sk)->flags = bt_sk(parent)->flags; - chan->chan_type = pchan->chan_type; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->conf_state = pchan->conf_state; - chan->mode = pchan->mode; - chan->fcs = pchan->fcs; - chan->max_tx = pchan->max_tx; - chan->tx_win = pchan->tx_win; - chan->tx_win_max = pchan->tx_win_max; - chan->sec_level = pchan->sec_level; - chan->flags = pchan->flags; - chan->tx_credits = pchan->tx_credits; - chan->rx_credits = pchan->rx_credits; - - if (chan->chan_type == L2CAP_CHAN_FIXED) { - chan->scid = pchan->scid; - chan->dcid = pchan->scid; - } - + /* Channel configuration is inherited from the parent by + * l2cap_new_connection(). + */ security_sk_clone(parent, sk); } else { switch (sk->sk_type) { @@ -1956,7 +1953,7 @@ static void l2cap_sock_init(struct sock *sk, struct sock *parent) chan->mode = L2CAP_MODE_BASIC; } - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); } /* Default config options */ @@ -1975,10 +1972,10 @@ static struct proto l2cap_proto = { }; static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, - int proto, gfp_t prio, int kern) + int proto, gfp_t prio, int kern, + struct l2cap_chan *chan) { struct sock *sk; - struct l2cap_chan *chan; sk = bt_sock_alloc(net, sock, &l2cap_proto, proto, prio, kern); if (!sk) @@ -1989,16 +1986,7 @@ static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, INIT_LIST_HEAD(&l2cap_pi(sk)->rx_busy); - chan = l2cap_chan_create(); - if (!chan) { - sk_free(sk); - if (sock) - sock->sk = NULL; - return NULL; - } - - l2cap_chan_hold(chan); - + /* The sock takes ownership of the caller's reference on chan. */ l2cap_pi(sk)->chan = chan; return sk; @@ -2008,6 +1996,7 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, int kern) { struct sock *sk; + struct l2cap_chan *chan; BT_DBG("sock %p", sock); @@ -2022,10 +2011,16 @@ static int l2cap_sock_create(struct net *net, struct socket *sock, int protocol, sock->ops = &l2cap_sock_ops; - sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern); - if (!sk) + chan = l2cap_chan_create(); + if (!chan) return -ENOMEM; + sk = l2cap_sock_alloc(net, sock, protocol, GFP_ATOMIC, kern, chan); + if (!sk) { + l2cap_chan_put(chan); + return -ENOMEM; + } + l2cap_sock_init(sk, NULL); bt_sock_link(&l2cap_sk_list, sk); return 0; diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index d23ca1dd0893..167d75e34526 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -2696,18 +2696,28 @@ static int mgmt_hci_cmd_sync(struct sock *sk, struct hci_dev *hdev, static bool pending_eir_or_class(struct hci_dev *hdev) { struct mgmt_pending_cmd *cmd; + bool pending = false; + + mutex_lock(&hdev->mgmt_pending_lock); list_for_each_entry(cmd, &hdev->mgmt_pending, list) { switch (cmd->opcode) { case MGMT_OP_ADD_UUID: case MGMT_OP_REMOVE_UUID: case MGMT_OP_SET_DEV_CLASS: + case MGMT_OP_SET_LOCAL_NAME: case MGMT_OP_SET_POWERED: - return true; + pending = true; + break; } + + if (pending) + break; } - return false; + mutex_unlock(&hdev->mgmt_pending_lock); + + return pending; } static const u8 bluetooth_base_uuid[] = { @@ -3091,6 +3101,8 @@ static int unpair_device_sync(struct hci_dev *hdev, void *data) struct mgmt_cp_unpair_device *cp = cmd->param; struct hci_conn *conn; + hci_dev_lock(hdev); + if (cp->addr.type == BDADDR_BREDR) conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &cp->addr.bdaddr); @@ -3098,6 +3110,11 @@ static int unpair_device_sync(struct hci_dev *hdev, void *data) conn = hci_conn_hash_lookup_le(hdev, &cp->addr.bdaddr, le_addr_type(cp->addr.type)); + if (conn) + hci_conn_get(conn); + + hci_dev_unlock(hdev); + if (!conn) return 0; @@ -3105,6 +3122,7 @@ static int unpair_device_sync(struct hci_dev *hdev, void *data) * will clean up the connection no matter the error. */ hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM); + hci_conn_put(conn); return 0; } @@ -3252,6 +3270,8 @@ static int disconnect_sync(struct hci_dev *hdev, void *data) struct mgmt_cp_disconnect *cp = cmd->param; struct hci_conn *conn; + hci_dev_lock(hdev); + if (cp->addr.type == BDADDR_BREDR) conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &cp->addr.bdaddr); @@ -3259,6 +3279,11 @@ static int disconnect_sync(struct hci_dev *hdev, void *data) conn = hci_conn_hash_lookup_le(hdev, &cp->addr.bdaddr, le_addr_type(cp->addr.type)); + if (conn) + hci_conn_get(conn); + + hci_dev_unlock(hdev); + if (!conn) return -ENOTCONN; @@ -3266,6 +3291,7 @@ static int disconnect_sync(struct hci_dev *hdev, void *data) * will clean up the connection no matter the error. */ hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM); + hci_conn_put(conn); return 0; } @@ -3498,11 +3524,13 @@ static int set_io_capability(struct sock *sk, struct hci_dev *hdev, void *data, NULL, 0); } -static struct mgmt_pending_cmd *find_pairing(struct hci_conn *conn) +static struct mgmt_pending_cmd *remove_pairing(struct hci_conn *conn) { struct hci_dev *hdev = conn->hdev; struct mgmt_pending_cmd *cmd; + mutex_lock(&hdev->mgmt_pending_lock); + list_for_each_entry(cmd, &hdev->mgmt_pending, list) { if (cmd->opcode != MGMT_OP_PAIR_DEVICE) continue; @@ -3510,9 +3538,39 @@ static struct mgmt_pending_cmd *find_pairing(struct hci_conn *conn) if (cmd->user_data != conn) continue; + list_del(&cmd->list); + mutex_unlock(&hdev->mgmt_pending_lock); return cmd; } + mutex_unlock(&hdev->mgmt_pending_lock); + + return NULL; +} + +static struct mgmt_pending_cmd *remove_pairing_by_addr(struct hci_dev *hdev, + bdaddr_t *bdaddr) +{ + struct mgmt_pending_cmd *cmd; + struct hci_conn *conn; + + mutex_lock(&hdev->mgmt_pending_lock); + + list_for_each_entry(cmd, &hdev->mgmt_pending, list) { + if (cmd->opcode != MGMT_OP_PAIR_DEVICE) + continue; + + conn = cmd->user_data; + if (bacmp(bdaddr, &conn->dst) != 0) + continue; + + list_del(&cmd->list); + mutex_unlock(&hdev->mgmt_pending_lock); + return cmd; + } + + mutex_unlock(&hdev->mgmt_pending_lock); + return NULL; } @@ -3550,10 +3608,10 @@ void mgmt_smp_complete(struct hci_conn *conn, bool complete) u8 status = complete ? MGMT_STATUS_SUCCESS : MGMT_STATUS_FAILED; struct mgmt_pending_cmd *cmd; - cmd = find_pairing(conn); + cmd = remove_pairing(conn); if (cmd) { cmd->cmd_complete(cmd, status); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); } } @@ -3563,14 +3621,14 @@ static void pairing_complete_cb(struct hci_conn *conn, u8 status) BT_DBG("status %u", status); - cmd = find_pairing(conn); + cmd = remove_pairing(conn); if (!cmd) { BT_DBG("Unable to find a pending command"); return; } cmd->cmd_complete(cmd, mgmt_status(status)); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); } static void le_pairing_complete_cb(struct hci_conn *conn, u8 status) @@ -3582,14 +3640,14 @@ static void le_pairing_complete_cb(struct hci_conn *conn, u8 status) if (!status) return; - cmd = find_pairing(conn); + cmd = remove_pairing(conn); if (!cmd) { BT_DBG("Unable to find a pending command"); return; } cmd->cmd_complete(cmd, mgmt_status(status)); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); } static int pair_device(struct sock *sk, struct hci_dev *hdev, void *data, @@ -3746,23 +3804,17 @@ static int cancel_pair_device(struct sock *sk, struct hci_dev *hdev, void *data, goto unlock; } - cmd = pending_find(MGMT_OP_PAIR_DEVICE, hdev); + cmd = remove_pairing_by_addr(hdev, &addr->bdaddr); if (!cmd) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_CANCEL_PAIR_DEVICE, MGMT_STATUS_INVALID_PARAMS); goto unlock; } - conn = cmd->user_data; - - if (bacmp(&addr->bdaddr, &conn->dst) != 0) { - err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_CANCEL_PAIR_DEVICE, - MGMT_STATUS_INVALID_PARAMS); - goto unlock; - } + conn = hci_conn_get(cmd->user_data); cmd->cmd_complete(cmd, MGMT_STATUS_CANCELLED); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_CANCEL_PAIR_DEVICE, 0, addr, sizeof(*addr)); @@ -3780,6 +3832,8 @@ static int cancel_pair_device(struct sock *sk, struct hci_dev *hdev, void *data, if (conn->conn_reason == CONN_REASON_PAIR_DEVICE) hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM); + hci_conn_put(conn); + unlock: hci_dev_unlock(hdev); return err; @@ -4027,6 +4081,12 @@ static int set_local_name(struct sock *sk, struct hci_dev *hdev, void *data, goto failed; } + if (hdev_is_powered(hdev) && pending_eir_or_class(hdev)) { + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_LOCAL_NAME, + MGMT_STATUS_BUSY); + goto failed; + } + memcpy(hdev->short_name, cp->short_name, sizeof(hdev->short_name)); if (!hdev_is_powered(hdev)) { @@ -5375,6 +5435,8 @@ static void mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, if (monitor->state == ADV_MONITOR_STATE_NOT_REGISTERED) monitor->state = ADV_MONITOR_STATE_REGISTERED; hci_update_passive_scan(hdev); + } else { + hci_free_adv_monitor(hdev, monitor); } mgmt_cmd_complete(cmd->sk, cmd->hdev->id, cmd->opcode, @@ -7386,6 +7448,9 @@ static void get_conn_info_complete(struct hci_dev *hdev, void *data, int err) rp.max_tx_power = HCI_TX_POWER_INVALID; } + if (conn) + hci_conn_put(conn); + mgmt_cmd_complete(cmd->sk, cmd->hdev->id, MGMT_OP_GET_CONN_INFO, status, &rp, sizeof(rp)); @@ -7400,6 +7465,8 @@ static int get_conn_info_sync(struct hci_dev *hdev, void *data) int err; __le16 handle; + hci_dev_lock(hdev); + /* Make sure we are still connected */ if (cp->addr.type == BDADDR_BREDR) conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, @@ -7407,12 +7474,16 @@ static int get_conn_info_sync(struct hci_dev *hdev, void *data) else conn = hci_conn_hash_lookup_ba(hdev, LE_LINK, &cp->addr.bdaddr); - if (!conn || conn->state != BT_CONNECTED) + if (!conn || conn->state != BT_CONNECTED) { + hci_dev_unlock(hdev); return MGMT_STATUS_NOT_CONNECTED; + } - cmd->user_data = conn; + cmd->user_data = hci_conn_get(conn); handle = cpu_to_le16(conn->handle); + hci_dev_unlock(hdev); + /* Refresh RSSI each time */ err = hci_read_rssi_sync(hdev, handle); @@ -7546,6 +7617,9 @@ static void get_clock_info_complete(struct hci_dev *hdev, void *data, int err) } complete: + if (conn) + hci_conn_put(conn); + mgmt_cmd_complete(cmd->sk, cmd->hdev->id, cmd->opcode, status, &rp, sizeof(rp)); @@ -7562,15 +7636,21 @@ static int get_clock_info_sync(struct hci_dev *hdev, void *data) memset(&hci_cp, 0, sizeof(hci_cp)); hci_read_clock_sync(hdev, &hci_cp); + hci_dev_lock(hdev); + /* Make sure connection still exists */ conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &cp->addr.bdaddr); - if (!conn || conn->state != BT_CONNECTED) + if (!conn || conn->state != BT_CONNECTED) { + hci_dev_unlock(hdev); return MGMT_STATUS_NOT_CONNECTED; + } - cmd->user_data = conn; + cmd->user_data = hci_conn_get(conn); hci_cp.handle = cpu_to_le16(conn->handle); hci_cp.which = 0x01; /* Piconet clock */ + hci_dev_unlock(hdev); + return hci_read_clock_sync(hdev, &hci_cp); } @@ -7658,6 +7738,8 @@ static void add_device_complete(struct hci_dev *hdev, void *data, int err) if (!err) { struct hci_conn_params *params; + hci_dev_lock(hdev); + params = hci_conn_params_lookup(hdev, &cp->addr.bdaddr, le_addr_type(cp->addr.type)); @@ -7666,6 +7748,7 @@ static void add_device_complete(struct hci_dev *hdev, void *data, int err) device_flags_changed(NULL, hdev, &cp->addr.bdaddr, cp->addr.type, hdev->conn_flags, params ? params->flags : 0); + hci_dev_unlock(hdev); } mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_ADD_DEVICE, @@ -7932,14 +8015,36 @@ unlock: static int conn_update_sync(struct hci_dev *hdev, void *data) { - struct hci_conn_params *params = data; - struct hci_conn *conn; + struct hci_conn *conn = data; + struct hci_conn_params *params; + struct hci_conn_params local = {}; - conn = hci_conn_hash_lookup_le(hdev, ¶ms->addr, params->addr_type); - if (!conn) - return -ECANCELED; + hci_dev_lock(hdev); + + if (!hci_conn_valid(hdev, conn) || conn->role != HCI_ROLE_MASTER) + goto cancel; + + params = hci_conn_params_lookup(hdev, &conn->dst, conn->dst_type); + if (!params) + goto cancel; + + local.conn_min_interval = params->conn_min_interval; + local.conn_max_interval = params->conn_max_interval; + local.conn_latency = params->conn_latency; + local.supervision_timeout = params->supervision_timeout; + + hci_dev_unlock(hdev); - return hci_le_conn_update_sync(hdev, conn, params); + return hci_le_conn_update_sync(hdev, conn, &local); + +cancel: + hci_dev_unlock(hdev); + return -ECANCELED; +} + +static void conn_update_sync_destroy(struct hci_dev *hdev, void *data, int err) +{ + hci_conn_put(data); } static int load_conn_param(struct sock *sk, struct hci_dev *hdev, void *data, @@ -8049,9 +8154,13 @@ static int load_conn_param(struct sock *sk, struct hci_dev *hdev, void *data, (conn->le_conn_min_interval != min || conn->le_conn_max_interval != max || conn->le_conn_latency != latency || - conn->le_supv_timeout != timeout)) - hci_cmd_sync_queue(hdev, conn_update_sync, - hci_param, NULL); + conn->le_supv_timeout != timeout)) { + hci_conn_get(conn); + if (hci_cmd_sync_queue(hdev, conn_update_sync, + conn, + conn_update_sync_destroy) < 0) + hci_conn_put(conn); + } } } @@ -9843,6 +9952,22 @@ bool mgmt_powering_down(struct hci_dev *hdev) return false; } +u8 hci_to_mgmt_reason(u8 err) +{ + switch (err) { + case HCI_ERROR_CONNECTION_TIMEOUT: + return MGMT_DEV_DISCONN_TIMEOUT; + case HCI_ERROR_REMOTE_USER_TERM: + case HCI_ERROR_REMOTE_LOW_RESOURCES: + case HCI_ERROR_REMOTE_POWER_OFF: + return MGMT_DEV_DISCONN_REMOTE; + case HCI_ERROR_LOCAL_HOST_TERM: + return MGMT_DEV_DISCONN_LOCAL_HOST; + default: + return MGMT_DEV_DISCONN_UNKNOWN; + } +} + void mgmt_device_disconnected(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type, u8 addr_type, u8 reason, bool mgmt_connected) @@ -9904,7 +10029,8 @@ void mgmt_connect_failed(struct hci_dev *hdev, struct hci_conn *conn, u8 status) if (test_and_clear_bit(HCI_CONN_MGMT_CONNECTED, &conn->flags)) { mgmt_device_disconnected(hdev, &conn->dst, conn->type, - conn->dst_type, status, true); + conn->dst_type, + hci_to_mgmt_reason(status), true); return; } @@ -10055,14 +10181,14 @@ void mgmt_auth_failed(struct hci_conn *conn, u8 hci_status) ev.addr.type = link_to_bdaddr(conn->type, conn->dst_type); ev.status = status; - cmd = find_pairing(conn); + cmd = remove_pairing(conn); mgmt_event(MGMT_EV_AUTH_FAILED, conn->hdev, &ev, sizeof(ev), cmd ? cmd->sk : NULL); if (cmd) { cmd->cmd_complete(cmd, status); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); } } diff --git a/net/bluetooth/msft.c b/net/bluetooth/msft.c index 2f008167cbaa..d7badce8746c 100644 --- a/net/bluetooth/msft.c +++ b/net/bluetooth/msft.c @@ -291,7 +291,7 @@ static int msft_le_monitor_advertisement_cb(struct hci_dev *hdev, u16 opcode, monitor->state = ADV_MONITOR_STATE_OFFLOADED; unlock: - if (status) + if (status && msft->resuming) hci_free_adv_monitor(hdev, monitor); hci_dev_unlock(hdev); diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c index ebeae17b71d1..2e8c080b4d9e 100644 --- a/net/bluetooth/rfcomm/core.c +++ b/net/bluetooth/rfcomm/core.c @@ -1028,6 +1028,23 @@ int rfcomm_send_rpn(struct rfcomm_session *s, int cr, u8 dlci, return rfcomm_send_frame(s, buf, ptr - buf); } +int rfcomm_dlc_send_rpn(struct rfcomm_dlc *d, u8 bit_rate, u8 data_bits, + u8 stop_bits, u8 parity, u8 flow_ctrl_settings, + u8 xon_char, u8 xoff_char, u16 param_mask) +{ + int err = -ENOTCONN; + + rfcomm_lock(); + if (d->session) + err = rfcomm_send_rpn(d->session, 1, d->dlci, bit_rate, + data_bits, stop_bits, parity, + flow_ctrl_settings, xon_char, xoff_char, + param_mask); + rfcomm_unlock(); + + return err; +} + static int rfcomm_send_rls(struct rfcomm_session *s, int cr, u8 dlci, u8 status) { struct rfcomm_hdr *hdr; @@ -1778,6 +1795,11 @@ static struct rfcomm_session *rfcomm_recv_frame(struct rfcomm_session *s, return s; } + if (skb->len < sizeof(*hdr) + 1) { + kfree_skb(skb); + return s; + } + dlci = __get_dlci(hdr->addr); type = __get_type(hdr->ctrl); diff --git a/net/bluetooth/rfcomm/tty.c b/net/bluetooth/rfcomm/tty.c index 4b9a699ec59b..b2c1060394e6 100644 --- a/net/bluetooth/rfcomm/tty.c +++ b/net/bluetooth/rfcomm/tty.c @@ -858,7 +858,7 @@ static void rfcomm_tty_set_termios(struct tty_struct *tty, BT_DBG("tty %p termios %p", tty, old); - if (!dev || !dev->dlc || !dev->dlc->session) + if (!dev || !dev->dlc) return; /* Handle turning off CRTSCTS */ @@ -979,9 +979,8 @@ static void rfcomm_tty_set_termios(struct tty_struct *tty, } if (changes) - rfcomm_send_rpn(dev->dlc->session, 1, dev->dlc->dlci, baud, - data_bits, stop_bits, parity, - RFCOMM_RPN_FLOW_NONE, x_on, x_off, changes); + rfcomm_dlc_send_rpn(dev->dlc, baud, data_bits, stop_bits, parity, + RFCOMM_RPN_FLOW_NONE, x_on, x_off, changes); } static void rfcomm_tty_throttle(struct tty_struct *tty) diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index fcc597be5bbd..3d4362a09df4 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -188,6 +188,9 @@ static void sco_sock_clear_timer(struct sock *sk) } /* ---- SCO connections ---- */ +/* Consumes a reference on @hcon, which the returned sco_conn owns until it is + * freed. On failure (NULL return) the reference is left for the caller to drop. + */ static struct sco_conn *sco_conn_add(struct hci_conn *hcon) { struct sco_conn *conn = hcon->sco_data; @@ -198,6 +201,9 @@ static struct sco_conn *sco_conn_add(struct hci_conn *hcon) sco_conn_lock(conn); conn->hcon = hcon; sco_conn_unlock(conn); + } else { + /* conn already owns a reference on hcon */ + hci_conn_drop(hcon); } return conn; } @@ -265,10 +271,8 @@ static void sco_conn_del(struct hci_conn *hcon, int err) sco_conn_unlock(conn); sco_conn_put(conn); - if (!sk) { - sco_conn_put(conn); + if (!sk) return; - } /* Kill socket */ lock_sock(sk); @@ -283,7 +287,7 @@ static void __sco_chan_add(struct sco_conn *conn, struct sock *sk, { BT_DBG("conn %p", conn); - sco_pi(sk)->conn = conn; + sco_pi(sk)->conn = sco_conn_hold(conn); conn->sk = sk; if (parent) @@ -366,15 +370,15 @@ static int sco_connect(struct sock *sk) */ if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) { release_sock(sk); - hci_conn_drop(hcon); + sco_conn_put(conn); err = -EBADFD; goto unlock; } err = sco_chan_add(conn, sk, NULL); + sco_conn_put(conn); if (err) { release_sock(sk); - hci_conn_drop(hcon); goto unlock; } @@ -570,10 +574,23 @@ static void __sco_sock_close(struct sock *sk) /* Must be called on unlocked socket. */ static void sco_sock_close(struct sock *sk) { + struct sco_conn *conn; + + lock_sock(sk); + conn = sco_pi(sk)->conn; + if (conn) + sco_conn_hold(conn); + release_sock(sk); + + if (conn) + disable_delayed_work_sync(&conn->timeout_work); + lock_sock(sk); - sco_sock_clear_timer(sk); __sco_sock_close(sk); release_sock(sk); + + if (conn) + sco_conn_put(conn); } static void sco_sock_init(struct sock *sk, struct sock *parent) @@ -1439,8 +1456,6 @@ static void sco_conn_ready(struct sco_conn *conn) bacpy(&sco_pi(sk)->src, &conn->hcon->src); bacpy(&sco_pi(sk)->dst, &conn->hcon->dst); - sco_conn_hold(conn); - hci_conn_hold(conn->hcon); __sco_chan_add(conn, sk, parent); if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(parent)->flags)) @@ -1496,10 +1511,12 @@ static void sco_connect_cfm(struct hci_conn *hcon, __u8 status) if (!status) { struct sco_conn *conn; - conn = sco_conn_add(hcon); + conn = sco_conn_add(hci_conn_hold(hcon)); if (conn) { sco_conn_ready(conn); sco_conn_put(conn); + } else { + hci_conn_drop(hcon); } } else sco_conn_del(hcon, bt_to_errno(status)); diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c index 031d3022cb1e..c4470958b0d5 100644 --- a/net/bluetooth/smp.c +++ b/net/bluetooth/smp.c @@ -3201,34 +3201,19 @@ static const struct l2cap_ops smp_chan_ops = { .get_sndtimeo = l2cap_chan_no_get_sndtimeo, }; -static inline struct l2cap_chan *smp_new_conn_cb(struct l2cap_chan *pchan) +static inline int smp_new_conn_cb(struct l2cap_chan *chan, + struct l2cap_chan *new_chan) { - struct l2cap_chan *chan; - - BT_DBG("pchan %p", pchan); - - chan = l2cap_chan_create(); - if (!chan) - return NULL; - - chan->chan_type = pchan->chan_type; - chan->ops = &smp_chan_ops; - chan->scid = pchan->scid; - chan->dcid = chan->scid; - chan->imtu = pchan->imtu; - chan->omtu = pchan->omtu; - chan->mode = pchan->mode; + new_chan->ops = &smp_chan_ops; /* Other L2CAP channels may request SMP routines in order to * change the security level. This means that the SMP channel * lock must be considered in its own category to avoid lockdep * warnings. */ - atomic_set(&chan->nesting, L2CAP_NESTING_SMP); - - BT_DBG("created chan %p", chan); + atomic_set(&new_chan->nesting, L2CAP_NESTING_SMP); - return chan; + return 0; } static const struct l2cap_ops smp_root_chan_ops = { @@ -3288,7 +3273,7 @@ create_chan: l2cap_add_scid(chan, cid); - l2cap_chan_set_defaults(chan); + l2cap_chan_set_defaults(chan, NULL); if (cid == L2CAP_CID_SMP) { u8 bdaddr_type; diff --git a/net/bridge/br_mrp.c b/net/bridge/br_mrp.c index 3f7126a7d720..ef16d0703924 100644 --- a/net/bridge/br_mrp.c +++ b/net/bridge/br_mrp.c @@ -215,7 +215,7 @@ static struct sk_buff *br_mrp_alloc_test_skb(struct br_mrp *mrp, struct br_mrp_oui_hdr *oui = NULL; u8 length; - length = sizeof(*sub_opt) + sizeof(*sub_tlv) + sizeof(oui) + + length = sizeof(*sub_opt) + sizeof(*sub_tlv) + sizeof(*oui) + MRP_OPT_PADDING; br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_OPTION, length); @@ -224,11 +224,9 @@ static struct sk_buff *br_mrp_alloc_test_skb(struct br_mrp *mrp, sub_opt = skb_put(skb, sizeof(*sub_opt)); memset(sub_opt, 0x0, sizeof(*sub_opt)); - sub_tlv = skb_put(skb, sizeof(*sub_tlv)); - sub_tlv->type = BR_MRP_SUB_TLV_HEADER_TEST_AUTO_MGR; - /* 32 bit alligment shall be ensured therefore add 2 bytes */ - skb_put(skb, MRP_OPT_PADDING); + sub_tlv = skb_put_zero(skb, sizeof(*sub_tlv) + MRP_OPT_PADDING); + sub_tlv->type = BR_MRP_SUB_TLV_HEADER_TEST_AUTO_MGR; } br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_END, 0x0); diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index 6b3ac473fd22..00aa9b2879d6 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -3687,6 +3687,7 @@ br_multicast_leave_group(struct net_bridge_mcast *brmctx, p->flags |= MDB_PG_FLAGS_FAST_LEAVE; br_multicast_del_pg(mp, p, pp); + break; } goto out; } diff --git a/net/bridge/br_netlink_tunnel.c b/net/bridge/br_netlink_tunnel.c index 71a12da30004..a713668ea34f 100644 --- a/net/bridge/br_netlink_tunnel.c +++ b/net/bridge/br_netlink_tunnel.c @@ -271,7 +271,8 @@ static void __vlan_tunnel_handle_range(const struct net_bridge_port *p, if (!*v_start) goto out_init; - if (v && curr_change && br_vlan_can_enter_range(v, *v_end)) { + if (v && curr_change && + br_vlan_can_enter_range(v, *v_end, br_get_pvid(vg))) { *v_end = v; return; } diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index d55ea9516e3e..d3880f31edc4 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -1627,7 +1627,8 @@ void br_vlan_notify(const struct net_bridge *br, u16 vid, u16 vid_range, int cmd); bool br_vlan_can_enter_range(const struct net_bridge_vlan *v_curr, - const struct net_bridge_vlan *range_end); + const struct net_bridge_vlan *range_end, + u16 pvid); void br_vlan_fill_forward_path_pvid(struct net_bridge *br, struct net_device_path_ctx *ctx, @@ -1874,7 +1875,8 @@ static inline void br_vlan_notify(const struct net_bridge *br, } static inline bool br_vlan_can_enter_range(const struct net_bridge_vlan *v_curr, - const struct net_bridge_vlan *range_end) + const struct net_bridge_vlan *range_end, + u16 pvid) { return true; } diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c index 5560afcaaca3..31c1b2cf75d9 100644 --- a/net/bridge/br_vlan.c +++ b/net/bridge/br_vlan.c @@ -1982,9 +1982,11 @@ out_kfree: /* check if v_curr can enter a range ending in range_end */ bool br_vlan_can_enter_range(const struct net_bridge_vlan *v_curr, - const struct net_bridge_vlan *range_end) + const struct net_bridge_vlan *range_end, + u16 pvid) { - return v_curr->vid - range_end->vid == 1 && + return v_curr->vid != pvid && range_end->vid != pvid && + v_curr->vid - range_end->vid == 1 && range_end->flags == v_curr->flags && br_vlan_opts_eq_range(v_curr, range_end); } @@ -2066,8 +2068,8 @@ static int br_vlan_dump_dev(const struct net_device *dev, idx += range_end->vid - range_start->vid + 1; range_start = v; - } else if (dump_stats || v->vid == pvid || - !br_vlan_can_enter_range(v, range_end)) { + } else if (dump_stats || + !br_vlan_can_enter_range(v, range_end, pvid)) { u16 vlan_flags = br_vlan_flags(range_start, pvid); if (!br_vlan_fill_vids(skb, range_start->vid, diff --git a/net/bridge/br_vlan_options.c b/net/bridge/br_vlan_options.c index fcc200c3e3da..cb0f556ff40d 100644 --- a/net/bridge/br_vlan_options.c +++ b/net/bridge/br_vlan_options.c @@ -350,8 +350,7 @@ int br_vlan_process_options(const struct net_bridge *br, continue; } - if (v->vid == pvid || - !br_vlan_can_enter_range(v, curr_end)) { + if (!br_vlan_can_enter_range(v, curr_end, pvid)) { br_vlan_notify(br, p, curr_start->vid, curr_end->vid, RTM_NEWVLAN); curr_start = v; diff --git a/net/bridge/netfilter/ebt_nflog.c b/net/bridge/netfilter/ebt_nflog.c index 61bf8f4465ab..426f8adc912c 100644 --- a/net/bridge/netfilter/ebt_nflog.c +++ b/net/bridge/netfilter/ebt_nflog.c @@ -41,11 +41,25 @@ ebt_nflog_tg(struct sk_buff *skb, const struct xt_action_param *par) static int ebt_nflog_tg_check(const struct xt_tgchk_param *par) { struct ebt_nflog_info *info = par->targinfo; + int ret; if (info->flags & ~EBT_NFLOG_MASK) return -EINVAL; info->prefix[EBT_NFLOG_PREFIX_SIZE - 1] = '\0'; - return 0; + + ret = nf_logger_find_get(par->family, NF_LOG_TYPE_ULOG); + if (ret != 0 && !par->nft_compat) { + request_module("%s", "nfnetlink_log"); + + ret = nf_logger_find_get(par->family, NF_LOG_TYPE_ULOG); + } + + return ret; +} + +static void ebt_nflog_tg_destroy(const struct xt_tgdtor_param *par) +{ + nf_logger_put(par->family, NF_LOG_TYPE_ULOG); } static struct xt_target ebt_nflog_tg_reg __read_mostly = { @@ -54,6 +68,7 @@ static struct xt_target ebt_nflog_tg_reg __read_mostly = { .family = NFPROTO_BRIDGE, .target = ebt_nflog_tg, .checkentry = ebt_nflog_tg_check, + .destroy = ebt_nflog_tg_destroy, .targetsize = sizeof(struct ebt_nflog_info), .me = THIS_MODULE, }; diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c index f20c039e44c8..96c9a8f57c87 100644 --- a/net/bridge/netfilter/ebtables.c +++ b/net/bridge/netfilter/ebtables.c @@ -403,6 +403,9 @@ ebt_check_match(struct ebt_entry_match *m, struct xt_mtchk_param *par, left - sizeof(struct ebt_entry_match) < m->match_size) return -EINVAL; + if (strnlen(m->u.name, XT_EXTENSION_MAXNAMELEN) == XT_EXTENSION_MAXNAMELEN) + return -EINVAL; + match = xt_find_match(NFPROTO_BRIDGE, m->u.name, m->u.revision); if (IS_ERR(match) || match->family != NFPROTO_BRIDGE) { if (!IS_ERR(match)) @@ -921,8 +924,7 @@ static int translate_table(struct net *net, const char *name, * if an error occurs */ newinfo->chainstack = - vmalloc_array(nr_cpu_ids, - sizeof(*(newinfo->chainstack))); + vcalloc(nr_cpu_ids, sizeof(*(newinfo->chainstack))); if (!newinfo->chainstack) return -ENOMEM; for_each_possible_cpu(i) { @@ -1434,6 +1436,8 @@ static int update_counters(struct net *net, sockptr_t arg, unsigned int len) if (copy_from_sockptr(&hlp, arg, sizeof(hlp))) return -EFAULT; + hlp.name[sizeof(hlp.name) - 1] = '\0'; + if (len != sizeof(hlp) + hlp.num_counters * sizeof(struct ebt_counter)) return -EINVAL; @@ -2273,6 +2277,8 @@ static int compat_copy_ebt_replace_from_user(struct ebt_replace *repl, memcpy(repl, &tmp, offsetof(struct ebt_replace, hook_entry)); + repl->name[sizeof(repl->name) - 1] = '\0'; + /* starting with hook_entry, 32 vs. 64 bit structures are different */ for (i = 0; i < NF_BR_NUMHOOKS; i++) repl->hook_entry[i] = compat_ptr(tmp.hook_entry[i]); @@ -2395,6 +2401,8 @@ static int compat_update_counters(struct net *net, sockptr_t arg, if (copy_from_sockptr(&hlp, arg, sizeof(hlp))) return -EFAULT; + hlp.name[sizeof(hlp.name) - 1] = '\0'; + /* try real handler in case userland supplied needed padding */ if (len != sizeof(hlp) + hlp.num_counters * sizeof(struct ebt_counter)) return update_counters(net, arg, len); diff --git a/net/bridge/netfilter/nf_conntrack_bridge.c b/net/bridge/netfilter/nf_conntrack_bridge.c index 58a33d0380b0..7ecb8a26bfa3 100644 --- a/net/bridge/netfilter/nf_conntrack_bridge.c +++ b/net/bridge/netfilter/nf_conntrack_bridge.c @@ -280,6 +280,7 @@ static unsigned int nf_ct_bridge_pre(void *priv, struct sk_buff *skb, ret = nf_ct_br_defrag6(skb, &bridge_state); break; default: + nf_reset_ct(skb); nf_ct_set(skb, NULL, IP_CT_UNTRACKED); return NF_ACCEPT; } diff --git a/net/can/bcm.c b/net/can/bcm.c index a4bef2c48a55..3d637a1e0ac1 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -58,6 +58,7 @@ #include <linux/can/skb.h> #include <linux/can/bcm.h> #include <linux/slab.h> +#include <linux/workqueue.h> #include <linux/spinlock.h> #include <net/can.h> #include <net/sock.h> @@ -92,6 +93,8 @@ MODULE_ALIAS("can-proto-2"); #define BCM_MIN_NAMELEN CAN_REQUIRED_SIZE(struct sockaddr_can, can_ifindex) +static struct workqueue_struct *bcm_wq; + /* * easy access to the first 64 bit of can(fd)_frame payload. cp->data is * 64 bit aligned so the offset has to be multiples of 8 which is ensured @@ -105,14 +108,16 @@ static inline u64 get_u64(const struct canfd_frame *cp, int offset) struct bcm_op { struct list_head list; struct rcu_head rcu; + struct work_struct work; int ifindex; canid_t can_id; u32 flags; - unsigned long frames_abs, frames_filtered; + atomic_long_t frames_abs, frames_filtered; struct bcm_timeval ival1, ival2; struct hrtimer timer, thrtimer; ktime_t rx_stamp, kt_ival1, kt_ival2, kt_lastmsg; int rx_ifindex; + int if_detected; /* first received ifindex in ANYDEV rx_op mode */ int cfsiz; u32 count; u32 nframes; @@ -124,7 +129,9 @@ struct bcm_op { struct canfd_frame last_sframe; struct sock *sk; struct net_device *rx_reg_dev; - spinlock_t bcm_tx_lock; /* protect currframe/count in runtime updates */ + netdevice_tracker rx_reg_dev_tracker; + spinlock_t bcm_tx_lock; /* protect tx data and timer updates */ + spinlock_t bcm_rx_update_lock; /* protect filter/timer data updates */ }; struct bcm_sock { @@ -224,10 +231,13 @@ static int bcm_proc_show(struct seq_file *m, void *v) list_for_each_entry_rcu(op, &bo->rx_ops, list) { - unsigned long reduction; + long reduction, frames_filtered, frames_abs; + + frames_filtered = atomic_long_read(&op->frames_filtered); + frames_abs = atomic_long_read(&op->frames_abs); /* print only active entries & prevent division by zero */ - if (!op->frames_abs) + if (!frames_abs) continue; seq_printf(m, "rx_op: %03X %-5s ", op->can_id, @@ -249,15 +259,15 @@ static int bcm_proc_show(struct seq_file *m, void *v) (long long)ktime_to_us(op->kt_ival2)); seq_printf(m, "# recv %ld (%ld) => reduction: ", - op->frames_filtered, op->frames_abs); + frames_filtered, frames_abs); - reduction = 100 - (op->frames_filtered * 100) / op->frames_abs; + reduction = 100 - (frames_filtered * 100) / frames_abs; seq_printf(m, "%s%ld%%\n", (reduction == 100) ? "near " : "", reduction); } - list_for_each_entry(op, &bo->tx_ops, list) { + list_for_each_entry_rcu(op, &bo->tx_ops, list) { seq_printf(m, "tx_op: %03X %s ", op->can_id, bcm_proc_getifname(net, ifname, op->ifindex)); @@ -275,7 +285,8 @@ static int bcm_proc_show(struct seq_file *m, void *v) seq_printf(m, "t2=%lld ", (long long)ktime_to_us(op->kt_ival2)); - seq_printf(m, "# sent %ld\n", op->frames_abs); + seq_printf(m, "# sent %ld\n", + atomic_long_read(&op->frames_abs)); } seq_putc(m, '\n'); @@ -285,26 +296,50 @@ static int bcm_proc_show(struct seq_file *m, void *v) } #endif /* CONFIG_PROC_FS */ +static void bcm_update_rx_stats(struct bcm_op *op) +{ + /* prevent overflow of the reduction% calculation in bcm_proc_show() */ + if (atomic_long_inc_return(&op->frames_abs) > LONG_MAX / 100) { + atomic_long_set(&op->frames_filtered, 0); + atomic_long_set(&op->frames_abs, 0); + } +} + +static void bcm_update_tx_stats(struct bcm_op *op) +{ + /* tx_op has no reduction% calculation - use the full range and + * just keep the displayed counter non-negative on overflow + */ + if (atomic_long_inc_return(&op->frames_abs) == LONG_MAX) + atomic_long_set(&op->frames_abs, 0); +} + /* * bcm_can_tx - send the (next) CAN frame to the appropriate CAN interface * of the given bcm tx op */ -static void bcm_can_tx(struct bcm_op *op) +static void bcm_can_tx(struct bcm_op *op, struct canfd_frame *cf) { struct sk_buff *skb; struct can_skb_ext *csx; struct net_device *dev; - struct canfd_frame *cf; + struct canfd_frame cframe; + bool cyclic = !cf; + unsigned int idx = 0; int err; /* no target device? => exit */ if (!op->ifindex) return; - /* read currframe under lock protection */ - spin_lock_bh(&op->bcm_tx_lock); - cf = op->frames + op->cfsiz * op->currframe; - spin_unlock_bh(&op->bcm_tx_lock); + if (cyclic) { + /* read currframe under lock protection */ + spin_lock_bh(&op->bcm_tx_lock); + idx = op->currframe; + memcpy(&cframe, op->frames + op->cfsiz * idx, op->cfsiz); + cf = &cframe; + spin_unlock_bh(&op->bcm_tx_lock); + } dev = dev_get_by_index(sock_net(op->sk), op->ifindex); if (!dev) { @@ -335,16 +370,22 @@ static void bcm_can_tx(struct bcm_op *op) spin_lock_bh(&op->bcm_tx_lock); if (!err) - op->frames_abs++; + bcm_update_tx_stats(op); - op->currframe++; + /* only advance the cyclic sequence if nothing reset currframe while + * we were sending - a concurrent TX_RESET_MULTI_IDX means this + * frame's bookkeeping belongs to a sequence that no longer exists + */ + if (!cyclic || op->currframe == idx) { + op->currframe++; - /* reached last frame? */ - if (op->currframe >= op->nframes) - op->currframe = 0; + /* reached last frame? */ + if (op->currframe >= op->nframes) + op->currframe = 0; - if (op->count > 0) - op->count--; + if (op->count > 0) + op->count--; + } spin_unlock_bh(&op->bcm_tx_lock); out: @@ -433,12 +474,18 @@ static bool bcm_tx_set_expiry(struct bcm_op *op, struct hrtimer *hrt) { ktime_t ival; + spin_lock_bh(&op->bcm_tx_lock); + if (op->kt_ival1 && op->count) ival = op->kt_ival1; - else if (op->kt_ival2) + else if (op->kt_ival2) { ival = op->kt_ival2; - else + } else { + spin_unlock_bh(&op->bcm_tx_lock); return false; + } + + spin_unlock_bh(&op->bcm_tx_lock); hrtimer_set_expires(hrt, ktime_add(ktime_get(), ival)); return true; @@ -455,26 +502,48 @@ static enum hrtimer_restart bcm_tx_timeout_handler(struct hrtimer *hrtimer) { struct bcm_op *op = container_of(hrtimer, struct bcm_op, timer); struct bcm_msg_head msg_head; + bool tx_ival1, tx_ival2; + + /* snapshot kt_ival1/kt_ival2/count under lock to avoid torn + * ktime_t reads racing with concurrent bcm_tx_setup() updates + */ + spin_lock_bh(&op->bcm_tx_lock); + tx_ival1 = op->kt_ival1 && (op->count > 0); + tx_ival2 = !!op->kt_ival2; + spin_unlock_bh(&op->bcm_tx_lock); - if (op->kt_ival1 && (op->count > 0)) { - bcm_can_tx(op); - if (!op->count && (op->flags & TX_COUNTEVT)) { + if (tx_ival1) { + u32 flags, count; + struct bcm_timeval ival1, ival2; + bcm_can_tx(op, NULL); + + /* snapshot variables under lock to avoid torn reads racing + * with concurrent bcm_tx_setup() updates + */ + spin_lock_bh(&op->bcm_tx_lock); + flags = op->flags; + count = op->count; + ival1 = op->ival1; + ival2 = op->ival2; + spin_unlock_bh(&op->bcm_tx_lock); + + if (!count && (flags & TX_COUNTEVT)) { /* create notification to user */ memset(&msg_head, 0, sizeof(msg_head)); msg_head.opcode = TX_EXPIRED; - msg_head.flags = op->flags; - msg_head.count = op->count; - msg_head.ival1 = op->ival1; - msg_head.ival2 = op->ival2; + msg_head.flags = flags; + msg_head.count = count; + msg_head.ival1 = ival1; + msg_head.ival2 = ival2; msg_head.can_id = op->can_id; msg_head.nframes = 0; bcm_send_to_user(op, &msg_head, NULL, 0); } - } else if (op->kt_ival2) { - bcm_can_tx(op); + } else if (tx_ival2) { + bcm_can_tx(op, NULL); } return bcm_tx_set_expiry(op, &op->timer) ? @@ -488,12 +557,9 @@ static void bcm_rx_changed(struct bcm_op *op, struct canfd_frame *data) { struct bcm_msg_head head; - /* update statistics */ - op->frames_filtered++; - - /* prevent statistics overflow */ - if (op->frames_filtered > ULONG_MAX/100) - op->frames_filtered = op->frames_abs = 0; + /* update statistics (frames_filtered <= frames_abs) */ + if (atomic_long_read(&op->frames_abs)) + atomic_long_inc(&op->frames_filtered); /* this element is not throttled anymore */ data->flags &= ~RX_THR; @@ -618,6 +684,8 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer) struct bcm_op *op = container_of(hrtimer, struct bcm_op, timer); struct bcm_msg_head msg_head; + spin_lock_bh(&op->bcm_rx_update_lock); + /* if user wants to be informed, when cyclic CAN-Messages come back */ if ((op->flags & RX_ANNOUNCE_RESUME) && op->last_frames) { /* clear received CAN frames to indicate 'nothing received' */ @@ -634,6 +702,8 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer) msg_head.can_id = op->can_id; msg_head.nframes = 0; + spin_unlock_bh(&op->bcm_rx_update_lock); + bcm_send_to_user(op, &msg_head, NULL, 0); return HRTIMER_NORESTART; @@ -682,15 +752,26 @@ static int bcm_rx_thr_flush(struct bcm_op *op) static enum hrtimer_restart bcm_rx_thr_handler(struct hrtimer *hrtimer) { struct bcm_op *op = container_of(hrtimer, struct bcm_op, thrtimer); + enum hrtimer_restart ret; + + spin_lock_bh(&op->bcm_rx_update_lock); - if (bcm_rx_thr_flush(op)) { + /* kt_ival2 may have been concurrently cleared by bcm_rx_setup() + * before it cancels this timer - never forward with a zero + * interval in that case. + */ + if (bcm_rx_thr_flush(op) && op->kt_ival2) { hrtimer_forward_now(hrtimer, op->kt_ival2); - return HRTIMER_RESTART; + ret = HRTIMER_RESTART; } else { /* rearm throttle handling */ op->kt_lastmsg = 0; - return HRTIMER_NORESTART; + ret = HRTIMER_NORESTART; } + + spin_unlock_bh(&op->bcm_rx_update_lock); + + return ret; } /* @@ -700,8 +781,10 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) { struct bcm_op *op = (struct bcm_op *)data; const struct canfd_frame *rxframe = (struct canfd_frame *)skb->data; + struct canfd_frame rtrframe; unsigned int i; unsigned char traffic_flags; + bool rtr_frame; if (op->can_id != rxframe->can_id) return; @@ -715,22 +798,59 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) return; } + /* An ANYDEV op with an active RX timeout and/or throttle timer + * tracks a single source interface: claim the first interface that + * delivers a matching frame and reject frames from any other one, + * before hrtimer_cancel() below can touch op->timer - this avoids + * racing bcm_rx_timeout_handler() across concurrent interfaces. + * RX_RTR_FRAME ops are excluded, as kt_ival1/kt_ival2 may briefly + * hold a stale value from an earlier non-RTR configuration. + */ + if (!op->ifindex) { + spin_lock_bh(&op->bcm_rx_update_lock); + + if (!(op->flags & RX_RTR_FRAME) && + (op->kt_ival1 || op->kt_ival2)) { + /* don't claim to vanishing interface */ + if (!op->if_detected && + READ_ONCE(skb->dev->reg_state) == NETREG_REGISTERED) + op->if_detected = skb->dev->ifindex; + + if (op->if_detected != skb->dev->ifindex) { + spin_unlock_bh(&op->bcm_rx_update_lock); + return; + } + } + + spin_unlock_bh(&op->bcm_rx_update_lock); + } + /* disable timeout */ hrtimer_cancel(&op->timer); - /* save rx timestamp */ - op->rx_stamp = skb->tstamp; - /* save originator for recvfrom() */ - op->rx_ifindex = skb->dev->ifindex; - /* update statistics */ - op->frames_abs++; + /* op->flags/op->frames may be updated concurrently by bcm_rx_setup() */ + spin_lock_bh(&op->bcm_rx_update_lock); + + rtr_frame = op->flags & RX_RTR_FRAME; + if (rtr_frame) { + bcm_update_rx_stats(op); + /* snapshot RTR content under lock */ + memcpy(&rtrframe, op->frames, op->cfsiz); + spin_unlock_bh(&op->bcm_rx_update_lock); - if (op->flags & RX_RTR_FRAME) { /* send reply for RTR-request (placed in op->frames[0]) */ - bcm_can_tx(op); + bcm_can_tx(op, &rtrframe); return; } + /* update statistics in the same critical section as bcm_rx_changed() + * below: frames_filtered must never be checked/incremented against a + * frames_abs snapshot from a concurrent bcm_rx_handler() call on + * another CPU for the same (wildcard) op, or frames_filtered can end + * up larger than frames_abs. + */ + bcm_update_rx_stats(op); + /* compute flags to distinguish between own/local/remote CAN traffic */ traffic_flags = 0; if (skb->sk) { @@ -739,6 +859,13 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) traffic_flags |= RX_OWN; } + /* save rx timestamp and originator for recvfrom() under lock: an + * ANYDEV op without an active timer can still run concurrently on + * different CPUs, so content and meta data must be bundled here. + */ + op->rx_stamp = skb->tstamp; + op->rx_ifindex = skb->dev->ifindex; + if (op->flags & RX_FILTER_ID) { /* the easiest case */ bcm_rx_update_and_send(op, op->last_frames, rxframe, @@ -774,6 +901,8 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) rx_starttimer: bcm_rx_starttimer(op); + + spin_unlock_bh(&op->bcm_rx_update_lock); } /* @@ -793,9 +922,12 @@ static struct bcm_op *bcm_find_op(struct list_head *ops, return NULL; } -static void bcm_free_op_rcu(struct rcu_head *rcu_head) +static void bcm_free_op_work(struct work_struct *work) { - struct bcm_op *op = container_of(rcu_head, struct bcm_op, rcu); + struct bcm_op *op = container_of(work, struct bcm_op, work); + + hrtimer_cancel(&op->timer); + hrtimer_cancel(&op->thrtimer); if ((op->frames) && (op->frames != &op->sframe)) kfree(op->frames); @@ -803,9 +935,23 @@ static void bcm_free_op_rcu(struct rcu_head *rcu_head) if ((op->last_frames) && (op->last_frames != &op->last_sframe)) kfree(op->last_frames); + /* the last possible access to op->timer/op->thrtimer has now + * happened above via hrtimer_cancel() - op->sk is no longer + * needed by any pending timer callback, so drop our reference + */ + sock_put(op->sk); + kfree(op); } +static void bcm_free_op_rcu(struct rcu_head *rcu_head) +{ + struct bcm_op *op = container_of(rcu_head, struct bcm_op, rcu); + + INIT_WORK(&op->work, bcm_free_op_work); + queue_work(bcm_wq, &op->work); +} + static void bcm_remove_op(struct bcm_op *op) { hrtimer_cancel(&op->timer); @@ -822,6 +968,7 @@ static void bcm_rx_unreg(struct net_device *dev, struct bcm_op *op) /* mark as removed subscription */ op->rx_reg_dev = NULL; + netdev_put(dev, &op->rx_reg_dev_tracker); } else printk(KERN_ERR "can-bcm: bcm_rx_unreg: registered device " "mismatch %p %p\n", op->rx_reg_dev, dev); @@ -852,17 +999,14 @@ static int bcm_delete_rx_op(struct list_head *ops, struct bcm_msg_head *mh, * Only remove subscriptions that had not * been removed due to NETDEV_UNREGISTER * in bcm_notifier() + * + * op->rx_reg_dev is a tracked reference taken + * when the subscription was registered, so it + * stays valid here even if a concurrent + * NETDEV_UNREGISTER already unlisted the dev. */ - if (op->rx_reg_dev) { - struct net_device *dev; - - dev = dev_get_by_index(sock_net(op->sk), - op->ifindex); - if (dev) { - bcm_rx_unreg(dev, op); - dev_put(dev); - } - } + if (op->rx_reg_dev) + bcm_rx_unreg(op->rx_reg_dev, op); } else can_rx_unregister(sock_net(op->sk), NULL, op->can_id, @@ -930,6 +1074,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, struct bcm_sock *bo = bcm_sk(sk); struct bcm_op *op; struct canfd_frame *cf; + bool add_op_to_list = false; unsigned int i; int err; @@ -948,6 +1093,8 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, /* check the given can_id */ op = bcm_find_op(&bo->tx_ops, msg_head, ifindex); if (op) { + void *new_frames; + /* update existing BCM operation */ /* @@ -958,11 +1105,23 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (msg_head->nframes > op->nframes) return -E2BIG; - /* update CAN frames content */ + /* get new CAN frames content into a staging buffer before + * locking: validate and normalize the frames there so that + * bcm_can_tx() / bcm_tx_timeout_handler() never observe a + * partially updated or unvalidated frame in op->frames + */ + new_frames = kmalloc(msg_head->nframes * op->cfsiz, GFP_KERNEL); + if (!new_frames) + return -ENOMEM; + for (i = 0; i < msg_head->nframes; i++) { - cf = op->frames + op->cfsiz * i; + cf = new_frames + op->cfsiz * i; err = memcpy_from_msg((u8 *)cf, msg, op->cfsiz); + if (err < 0) { + kfree(new_frames); + return err; + } if (op->flags & CAN_FD_FRAME) { if (cf->len > 64) @@ -972,36 +1131,38 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, err = -EINVAL; } - if (err < 0) + if (err < 0) { + kfree(new_frames); return err; + } if (msg_head->flags & TX_CP_CAN_ID) { /* copy can_id into frame */ cf->can_id = msg_head->can_id; } } + + spin_lock_bh(&op->bcm_tx_lock); + + /* update CAN frames content */ + memcpy(op->frames, new_frames, msg_head->nframes * op->cfsiz); + op->flags = msg_head->flags; - /* only lock for unlikely count/nframes/currframe changes */ if (op->nframes != msg_head->nframes || - op->flags & TX_RESET_MULTI_IDX || - op->flags & SETTIMER) { - - spin_lock_bh(&op->bcm_tx_lock); + op->flags & TX_RESET_MULTI_IDX) { + /* potentially update changed nframes */ + op->nframes = msg_head->nframes; + /* restart multiple frame transmission */ + op->currframe = 0; + } - if (op->nframes != msg_head->nframes || - op->flags & TX_RESET_MULTI_IDX) { - /* potentially update changed nframes */ - op->nframes = msg_head->nframes; - /* restart multiple frame transmission */ - op->currframe = 0; - } + if (op->flags & SETTIMER) + op->count = msg_head->count; - if (op->flags & SETTIMER) - op->count = msg_head->count; + spin_unlock_bh(&op->bcm_tx_lock); - spin_unlock_bh(&op->bcm_tx_lock); - } + kfree(new_frames); } else { /* insert new BCM operation for the given can_id */ @@ -1060,6 +1221,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, /* bcm_can_tx / bcm_tx_timeout_handler needs this */ op->sk = sk; + sock_hold(sk); op->ifindex = ifindex; /* initialize uninitialized (kzalloc) structure */ @@ -1070,17 +1232,18 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, hrtimer_setup(&op->thrtimer, hrtimer_dummy_timeout, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); - /* add this bcm_op to the list of the tx_ops */ - list_add(&op->list, &bo->tx_ops); + add_op_to_list = true; } /* if ((op = bcm_find_op(&bo->tx_ops, msg_head->can_id, ifindex))) */ if (op->flags & SETTIMER) { /* set timer values */ + spin_lock_bh(&op->bcm_tx_lock); op->ival1 = msg_head->ival1; op->ival2 = msg_head->ival2; op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + spin_unlock_bh(&op->bcm_tx_lock); /* disable an active timer due to zero values? */ if (!op->kt_ival1 && !op->kt_ival2) @@ -1093,8 +1256,12 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, op->flags |= TX_ANNOUNCE; } + /* add this bcm_op to the list of the tx_ops? */ + if (add_op_to_list) + list_add_rcu(&op->list, &bo->tx_ops); + if (op->flags & TX_ANNOUNCE) - bcm_can_tx(op); + bcm_can_tx(op, NULL); if (op->flags & STARTTIMER) bcm_tx_start_timer(op); @@ -1108,6 +1275,39 @@ free_op: return err; } +static int bcm_rx_setup_rtr_check(struct bcm_msg_head *msg_head, + struct bcm_op *op, void *new_frames) +{ + struct canfd_frame *frame0 = new_frames; + + if (!(msg_head->flags & RX_RTR_FRAME)) + return 0; + + /* this frame is sent out as-is by bcm_can_tx() whenever a matching + * remote request is received, so validate its length the same way + * bcm_tx_setup() validates TX_SETUP frames before installing it + */ + if (msg_head->flags & CAN_FD_FRAME) { + if (frame0->len > 64) + return -EINVAL; + } else { + if (frame0->len > 8) + return -EINVAL; + } + + /* funny feature in RX(!)_SETUP only for RTR-mode: + * copy can_id into frame BUT without RTR-flag to + * prevent a full-load-loopback-test ... ;-] + * normalize this on the staged buffer, before it is + * ever installed into op->frames. + */ + if ((msg_head->flags & TX_CP_CAN_ID) || + frame0->can_id == op->can_id) + frame0->can_id = op->can_id & ~CAN_RTR_FLAG; + + return 0; +} + /* * bcm_rx_setup - create or update a bcm rx op (for bcm_sendmsg) */ @@ -1117,6 +1317,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, struct bcm_sock *bo = bcm_sk(sk); struct bcm_op *op; int do_rx_register; + int new_op = 0; int err = 0; if ((msg_head->flags & RX_FILTER_ID) || (!(msg_head->nframes))) { @@ -1142,6 +1343,8 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, /* check the given can_id */ op = bcm_find_op(&bo->rx_ops, msg_head, ifindex); if (op) { + void *new_frames = NULL; + /* update existing BCM operation */ /* @@ -1153,21 +1356,62 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, return -E2BIG; if (msg_head->nframes) { - /* update CAN frames content */ - err = memcpy_from_msg(op->frames, msg, + /* get new CAN frames content before locking */ + new_frames = kmalloc(msg_head->nframes * op->cfsiz, + GFP_KERNEL); + if (!new_frames) + return -ENOMEM; + + err = memcpy_from_msg(new_frames, msg, msg_head->nframes * op->cfsiz); - if (err < 0) + if (err < 0) { + kfree(new_frames); return err; + } - /* clear last_frames to indicate 'nothing received' */ - memset(op->last_frames, 0, msg_head->nframes * op->cfsiz); + err = bcm_rx_setup_rtr_check(msg_head, op, new_frames); + if (err < 0) { + kfree(new_frames); + return err; + } } + spin_lock_bh(&op->bcm_rx_update_lock); op->nframes = msg_head->nframes; op->flags = msg_head->flags; - /* Only an update -> do not call can_rx_register() */ - do_rx_register = 0; + if (msg_head->nframes) { + /* update CAN frames content */ + memcpy(op->frames, new_frames, + msg_head->nframes * op->cfsiz); + + /* clear last_frames to indicate 'nothing received' */ + memset(op->last_frames, 0, + msg_head->nframes * op->cfsiz); + } + + if (msg_head->flags & SETTIMER) { + op->ival1 = msg_head->ival1; + op->ival2 = msg_head->ival2; + op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); + op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + op->kt_lastmsg = 0; + op->if_detected = 0; /* reclaim ifindex in ANYDEV mode */ + } + spin_unlock_bh(&op->bcm_rx_update_lock); + + /* free temporary frames / kfree(NULL) is safe */ + kfree(new_frames); + + /* Don't register a new CAN filter for the rx_op update unless + * a concurrent NETDEV_UNREGISTER notifier already tore down + * the previous registration. In this case the receiver needs + * to be re-registered here so that this update doesn't + * silently stop delivering frames for the given ifindex. + * Ops with ifindex = 0 (all CAN interfaces) never carry a + * tracked rx_reg_dev and stay registered as-is. + */ + do_rx_register = (ifindex && !op->rx_reg_dev) ? 1 : 0; } else { /* insert new BCM operation for the given can_id */ @@ -1176,6 +1420,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, return -ENOMEM; spin_lock_init(&op->bcm_tx_lock); + spin_lock_init(&op->bcm_rx_update_lock); op->can_id = msg_head->can_id; op->nframes = msg_head->nframes; op->cfsiz = CFSIZ(msg_head->flags); @@ -1209,18 +1454,17 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (msg_head->nframes) { err = memcpy_from_msg(op->frames, msg, msg_head->nframes * op->cfsiz); - if (err < 0) { - if (op->frames != &op->sframe) - kfree(op->frames); - if (op->last_frames != &op->last_sframe) - kfree(op->last_frames); - kfree(op); - return err; - } + if (err < 0) + goto free_op; + + err = bcm_rx_setup_rtr_check(msg_head, op, op->frames); + if (err < 0) + goto free_op; } /* bcm_can_tx / bcm_tx_timeout_handler needs this */ op->sk = sk; + sock_hold(sk); op->ifindex = ifindex; /* ifindex for timeout events w/o previous frame reception */ @@ -1232,40 +1476,31 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, hrtimer_setup(&op->thrtimer, bcm_rx_thr_handler, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); - /* add this bcm_op to the list of the rx_ops */ - list_add(&op->list, &bo->rx_ops); - /* call can_rx_register() */ do_rx_register = 1; + new_op = 1; } /* if ((op = bcm_find_op(&bo->rx_ops, msg_head->can_id, ifindex))) */ /* check flags */ if (op->flags & RX_RTR_FRAME) { - struct canfd_frame *frame0 = op->frames; - /* no timers in RTR-mode */ hrtimer_cancel(&op->thrtimer); hrtimer_cancel(&op->timer); - - /* - * funny feature in RX(!)_SETUP only for RTR-mode: - * copy can_id into frame BUT without RTR-flag to - * prevent a full-load-loopback-test ... ;-] - */ - if ((op->flags & TX_CP_CAN_ID) || - (frame0->can_id == op->can_id)) - frame0->can_id = op->can_id & ~CAN_RTR_FLAG; - } else { if (op->flags & SETTIMER) { - /* set timer value */ - op->ival1 = msg_head->ival1; - op->ival2 = msg_head->ival2; - op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); - op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + /* set timers (locked) for newly created op */ + if (new_op) { + spin_lock_bh(&op->bcm_rx_update_lock); + op->ival1 = msg_head->ival1; + op->ival2 = msg_head->ival2; + op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); + op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + op->kt_lastmsg = 0; + spin_unlock_bh(&op->bcm_rx_update_lock); + } /* disable an active timer due to zero value? */ if (!op->kt_ival1) @@ -1275,9 +1510,11 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, * In any case cancel the throttle timer, flush * potentially blocked msgs and reset throttle handling */ - op->kt_lastmsg = 0; hrtimer_cancel(&op->thrtimer); + + spin_lock_bh(&op->bcm_rx_update_lock); bcm_rx_thr_flush(op); + spin_unlock_bh(&op->bcm_rx_update_lock); } if ((op->flags & STARTTIMER) && op->kt_ival1) @@ -1285,7 +1522,10 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, HRTIMER_MODE_REL_SOFT); } - /* now we can register for can_ids, if we added a new bcm_op */ + /* now we can register for can_ids, if we added a new bcm_op + * or need to re-register after a NETDEV_UNREGISTER tore down + * the previous registration of an existing op + */ if (do_rx_register) { if (ifindex) { struct net_device *dev; @@ -1298,23 +1538,62 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, bcm_rx_handler, op, "bcm", sk); - op->rx_reg_dev = dev; + /* keep a tracked reference so that a later + * unregister can safely reach the device even + * if a concurrent NETDEV_UNREGISTER has + * already unlisted it by ifindex + */ + if (!err) { + op->rx_reg_dev = dev; + netdev_hold(dev, + &op->rx_reg_dev_tracker, + GFP_KERNEL); + } dev_put(dev); + } else { + /* the requested device is gone - do not + * silently succeed without registering + */ + err = -ENODEV; } - } else + } else { err = can_rx_register(sock_net(sk), NULL, op->can_id, REGMASK(op->can_id), bcm_rx_handler, op, "bcm", sk); + } + if (err) { - /* this bcm rx op is broken -> remove it */ - list_del_rcu(&op->list); - bcm_remove_op(op); + /* newly created bcm rx op is broken -> remove it */ + if (new_op) { + bcm_remove_op(op); + return err; + } + + /* an existing op just stays unregistered. + * Cancel op->timer and (defensively) op->thrtimer. + * Other settings can't be reached until the next + * successful RX_SETUP. + */ + hrtimer_cancel(&op->timer); + hrtimer_cancel(&op->thrtimer); return err; } + + /* add a new bcm_op to the list of the rx_ops */ + if (new_op) + list_add_rcu(&op->list, &bo->rx_ops); } return msg_head->nframes * op->cfsiz + MHSIZ; + +free_op: + if (op->frames != &op->sframe) + kfree(op->frames); + if (op->last_frames != &op->last_sframe) + kfree(op->last_frames); + kfree(op); + return err; } /* @@ -1373,12 +1652,13 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) { struct sock *sk = sock->sk; struct bcm_sock *bo = bcm_sk(sk); - int ifindex = bo->ifindex; /* default ifindex for this bcm_op */ + int ifindex; struct bcm_msg_head msg_head; int cfsiz; int ret; /* read bytes or error codes as return value */ - if (!bo->bound) + /* Lockless fast-path check for bound socket */ + if (!READ_ONCE(bo->bound)) return -ENOTCONN; /* check for valid message length from userspace */ @@ -1394,17 +1674,38 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) if ((size - MHSIZ) % cfsiz) return -EINVAL; + lock_sock(sk); + + /* Re-validate under the socket lock: a concurrent bcm_notify() + * may have unbound this socket (device removal) after the + * lockless fast-path check above. bo->ifindex is only ever + * mutated under lock_sock(), so reading it here - instead of + * before taking the lock - guarantees it can't be observed + * torn against bo->bound. + */ + if (!bo->bound) { + ret = -ENOTCONN; + goto out_release; + } + + /* default ifindex for this bcm_op */ + ifindex = bo->ifindex; + /* check for alternative ifindex for this bcm_op */ if (!ifindex && msg->msg_name) { /* no bound device as default => check msg_name */ DECLARE_SOCKADDR(struct sockaddr_can *, addr, msg->msg_name); - if (msg->msg_namelen < BCM_MIN_NAMELEN) - return -EINVAL; + if (msg->msg_namelen < BCM_MIN_NAMELEN) { + ret = -EINVAL; + goto out_release; + } - if (addr->can_family != AF_CAN) - return -EINVAL; + if (addr->can_family != AF_CAN) { + ret = -EINVAL; + goto out_release; + } /* ifindex from sendto() */ ifindex = addr->can_ifindex; @@ -1413,20 +1714,21 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) struct net_device *dev; dev = dev_get_by_index(sock_net(sk), ifindex); - if (!dev) - return -ENODEV; + if (!dev) { + ret = -ENODEV; + goto out_release; + } if (dev->type != ARPHRD_CAN) { dev_put(dev); - return -ENODEV; + ret = -ENODEV; + goto out_release; } dev_put(dev); } } - lock_sock(sk); - switch (msg_head.opcode) { case TX_SETUP: @@ -1476,6 +1778,7 @@ static int bcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) break; } +out_release: release_sock(sk); return ret; @@ -1499,11 +1802,30 @@ static void bcm_notify(struct bcm_sock *bo, unsigned long msg, case NETDEV_UNREGISTER: lock_sock(sk); - /* remove device specific receive entries */ - list_for_each_entry(op, &bo->rx_ops, list) + /* rx_ops: remove device specific receive entries */ + list_for_each_entry(op, &bo->rx_ops, list) { if (op->rx_reg_dev == dev) bcm_rx_unreg(dev, op); + /* release an ANYDEV op's claim (see bcm_rx_handler()) + * on this now confirmed-gone interface. + */ + if (!op->ifindex) { + spin_lock_bh(&op->bcm_rx_update_lock); + if (op->if_detected == dev->ifindex) + op->if_detected = 0; + spin_unlock_bh(&op->bcm_rx_update_lock); + } + } + + /* tx_ops: stop device specific cyclic transmissions on the + * vanishing ifindex. Cancelling the timer is enough to stop + * cyclic bcm_can_tx() calls as there is no re-arming. + */ + list_for_each_entry(op, &bo->tx_ops, list) + if (op->ifindex == dev->ifindex) + hrtimer_cancel(&op->timer); + /* remove device reference, if this is our bound device */ if (bo->bound && bo->ifindex == dev->ifindex) { #if IS_ENABLED(CONFIG_PROC_FS) @@ -1512,7 +1834,12 @@ static void bcm_notify(struct bcm_sock *bo, unsigned long msg, bo->bcm_proc_read = NULL; } #endif - bo->bound = 0; + /* Paired with the lockless fast-path check in + * bcm_sendmsg(); bo->ifindex itself is only ever + * accessed under lock_sock() so it needs no + * annotation. + */ + WRITE_ONCE(bo->bound, 0); bo->ifindex = 0; notify_enodev = 1; } @@ -1616,8 +1943,10 @@ static int bcm_release(struct socket *sock) remove_proc_entry(bo->procname, net->can.bcmproc_dir); #endif /* CONFIG_PROC_FS */ - list_for_each_entry_safe(op, next, &bo->tx_ops, list) + list_for_each_entry_safe(op, next, &bo->tx_ops, list) { + list_del_rcu(&op->list); bcm_remove_op(op); + } list_for_each_entry_safe(op, next, &bo->rx_ops, list) { /* @@ -1629,16 +1958,14 @@ static int bcm_release(struct socket *sock) * Only remove subscriptions that had not * been removed due to NETDEV_UNREGISTER * in bcm_notifier() + * + * op->rx_reg_dev is a tracked reference taken + * when the subscription was registered, so it + * stays valid here even if a concurrent + * NETDEV_UNREGISTER already unlisted the device. */ - if (op->rx_reg_dev) { - struct net_device *dev; - - dev = dev_get_by_index(net, op->ifindex); - if (dev) { - bcm_rx_unreg(dev, op); - dev_put(dev); - } - } + if (op->rx_reg_dev) + bcm_rx_unreg(op->rx_reg_dev, op); } else can_rx_unregister(net, NULL, op->can_id, REGMASK(op->can_id), @@ -1648,12 +1975,14 @@ static int bcm_release(struct socket *sock) synchronize_rcu(); - list_for_each_entry_safe(op, next, &bo->rx_ops, list) + list_for_each_entry_safe(op, next, &bo->rx_ops, list) { + list_del_rcu(&op->list); bcm_remove_op(op); + } /* remove device reference */ if (bo->bound) { - bo->bound = 0; + WRITE_ONCE(bo->bound, 0); bo->ifindex = 0; } @@ -1723,7 +2052,10 @@ static int bcm_connect(struct socket *sock, struct sockaddr_unsized *uaddr, int } #endif /* CONFIG_PROC_FS */ - bo->bound = 1; + /* bo->ifindex above is fully assigned before this point; pairs + * with the lockless fast-path check in bcm_sendmsg() + */ + WRITE_ONCE(bo->bound, 1); fail: release_sock(sk); @@ -1839,11 +2171,15 @@ static int __init bcm_module_init(void) { int err; + bcm_wq = alloc_workqueue("can-bcm-wq", WQ_UNBOUND, 0); + if (!bcm_wq) + return -ENOMEM; + pr_info("can: broadcast manager protocol\n"); err = register_pernet_subsys(&canbcm_pernet_ops); if (err) - return err; + goto register_pernet_failed; err = register_netdevice_notifier(&canbcm_notifier); if (err) @@ -1861,6 +2197,8 @@ register_proto_failed: unregister_netdevice_notifier(&canbcm_notifier); register_notifier_failed: unregister_pernet_subsys(&canbcm_pernet_ops); +register_pernet_failed: + destroy_workqueue(bcm_wq); return err; } @@ -1869,6 +2207,8 @@ static void __exit bcm_module_exit(void) can_proto_unregister(&bcm_can_proto); unregister_netdevice_notifier(&canbcm_notifier); unregister_pernet_subsys(&canbcm_pernet_ops); + rcu_barrier(); + destroy_workqueue(bcm_wq); } module_init(bcm_module_init); diff --git a/net/can/isotp.c b/net/can/isotp.c index c48b4a818297..155530aedce2 100644 --- a/net/can/isotp.c +++ b/net/can/isotp.c @@ -127,6 +127,15 @@ MODULE_PARM_DESC(max_pdu_size, "maximum isotp pdu size (default " #define ISOTP_FC_TIMEOUT 1 /* 1 sec */ #define ISOTP_ECHO_TIMEOUT 2 /* 2 secs */ +/* so->tx_result[so->tx_gen % ISOTP_TX_RESULT_SLOTS] holds the packed value + * (err << ISOTP_TX_RESULT_GEN_BITS | gen) for each tx generation slot, so it + * can be handled with a single READ_ONCE()/WRITE_ONCE() access. + */ +#define ISOTP_TX_RESULT_SLOTS 4 +#define ISOTP_TX_RESULT_GEN_BITS 24 +#define ISOTP_TX_RESULT_GEN_MASK ((1U << ISOTP_TX_RESULT_GEN_BITS) - 1) +#define ISOTP_TX_RESULT_ERR_MASK 0xFF + enum { ISOTP_IDLE = 0, ISOTP_WAIT_FIRST_FC, @@ -152,11 +161,13 @@ struct isotp_sock { struct sock sk; int bound; int ifindex; + struct net_device *dev; + netdevice_tracker dev_tracker; canid_t txid; canid_t rxid; ktime_t tx_gap; ktime_t lastrxcf_tstamp; - struct hrtimer rxtimer, txtimer, txfrtimer; + struct hrtimer rxtimer, txtimer, txfrtimer, echotimer; struct can_isotp_options opt; struct can_isotp_fc_options rxfc, txfc; struct can_isotp_ll_options ll; @@ -164,6 +175,8 @@ struct isotp_sock { u32 force_tx_stmin; u32 force_rx_stmin; u32 cfecho; /* consecutive frame echo tag */ + u32 tx_gen; /* transfer generation, increased per new tx transfer */ + u32 tx_result[ISOTP_TX_RESULT_SLOTS]; /* per-generation result slots */ struct tpcon rx, tx; struct list_head notifier; wait_queue_head_t wait; @@ -174,6 +187,65 @@ static LIST_HEAD(isotp_notifier_list); static DEFINE_SPINLOCK(isotp_notifier_lock); static struct isotp_sock *isotp_busy_notifier; +/* increase (24 bit) tx generation value */ +static u32 isotp_inc_tx_gen(u32 gen) +{ + return (gen + 1) & ISOTP_TX_RESULT_GEN_MASK; +} + +/* store 8 bit error and 24 bit tx generation values in packed u32 element */ +static u32 isotp_pack_tx_result(u32 gen, int err) +{ + return gen | ((u32)err << ISOTP_TX_RESULT_GEN_BITS); +} + +/* get the 24 bit tx generation value from the tx result */ +static u32 isotp_get_tx_gen(u32 gen_err) +{ + return gen_err & ISOTP_TX_RESULT_GEN_MASK; +} + +/* get the 8 bit error value from the tx result */ +static u32 isotp_get_tx_err(u32 gen_err) +{ + return (gen_err >> ISOTP_TX_RESULT_GEN_BITS) & ISOTP_TX_RESULT_ERR_MASK; +} + +/* store transfer result in per-generation%4 so->tx_result[] slot */ +static void isotp_set_tx_result(struct isotp_sock *so, u32 gen, int err) +{ + WRITE_ONCE(so->tx_result[gen % ISOTP_TX_RESULT_SLOTS], + isotp_pack_tx_result(gen, err)); +} + +/* fetch the result recorded for 'gen', as a (negative) errno (0 for success) */ +static int isotp_get_tx_result(struct isotp_sock *so, u32 gen) +{ + u32 result = READ_ONCE(so->tx_result[gen % ISOTP_TX_RESULT_SLOTS]); + + if (isotp_get_tx_gen(result) != gen) { + pr_notice_once("can-isotp: tx_result[] slot reused before read\n"); + + /* report failure rather than risk a false success */ + return -ECOMM; + } + + return -(isotp_get_tx_err(result)); +} + +/* true if done, shut down or superseded ('gen' is no longer the active + * transfer). Reads tx.state first (acquire) so tx_gen/tx_result reads + * below see at least what that state write published (common sequence). + */ +static bool isotp_tx_gen_done(struct isotp_sock *so, u32 gen) +{ + /* read tx.state first for the common sequence */ + u32 state = smp_load_acquire(&so->tx.state); + + return state == ISOTP_IDLE || state == ISOTP_SHUTDOWN || + READ_ONCE(so->tx_gen) != gen; +} + static inline struct isotp_sock *isotp_sk(const struct sock *sk) { return (struct isotp_sock *)sk; @@ -196,7 +268,7 @@ static enum hrtimer_restart isotp_rx_timer_handler(struct hrtimer *hrtimer) rxtimer); struct sock *sk = &so->sk; - if (so->rx.state == ISOTP_WAIT_DATA) { + if (READ_ONCE(so->rx.state) == ISOTP_WAIT_DATA) { /* we did not get new data frames in time */ /* report 'connection timed out' */ @@ -205,7 +277,7 @@ static enum hrtimer_restart isotp_rx_timer_handler(struct hrtimer *hrtimer) sk_error_report(sk); /* reset rx state */ - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); } return HRTIMER_NORESTART; @@ -369,13 +441,21 @@ static void isotp_send_cframe(struct isotp_sock *so); static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) { struct sock *sk = &so->sk; + int tx_err = EBADMSG; /* default for unknown FC status */ - if (so->tx.state != ISOTP_WAIT_FC && - so->tx.state != ISOTP_WAIT_FIRST_FC) + if (READ_ONCE(so->tx.state) != ISOTP_WAIT_FC && + READ_ONCE(so->tx.state) != ISOTP_WAIT_FIRST_FC) return 0; hrtimer_cancel(&so->txtimer); + /* isotp_tx_timeout() may have given up on this job while + * hrtimer_cancel() above waited for it to finish => recheck + */ + if (READ_ONCE(so->tx.state) != ISOTP_WAIT_FC && + READ_ONCE(so->tx.state) != ISOTP_WAIT_FIRST_FC) + return 1; + if ((cf->len < ae + FC_CONTENT_SZ) || ((so->opt.flags & ISOTP_CHECK_PADDING) && check_pad(so, cf, ae + FC_CONTENT_SZ, so->opt.rxpad_content))) { @@ -384,13 +464,15 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) if (!sock_flag(sk, SOCK_DEAD)) sk_error_report(sk); - so->tx.state = ISOTP_IDLE; + isotp_set_tx_result(so, so->tx_gen, EBADMSG); + /* set to IDLE after publishing tx_result */ + smp_store_release(&so->tx.state, ISOTP_IDLE); wake_up_interruptible(&so->wait); return 1; } /* get static/dynamic communication params from first/every FC frame */ - if (so->tx.state == ISOTP_WAIT_FIRST_FC || + if (READ_ONCE(so->tx.state) == ISOTP_WAIT_FIRST_FC || so->opt.flags & CAN_ISOTP_DYN_FC_PARMS) { so->txfc.bs = cf->data[ae + 1]; so->txfc.stmin = cf->data[ae + 2]; @@ -414,15 +496,15 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) so->tx_gap = ktime_add_ns(so->tx_gap, (so->txfc.stmin - 0xF0) * 100000); - so->tx.state = ISOTP_WAIT_FC; + WRITE_ONCE(so->tx.state, ISOTP_WAIT_FC); } switch (cf->data[ae] & 0x0F) { case ISOTP_FC_CTS: so->tx.bs = 0; - so->tx.state = ISOTP_SENDING; + WRITE_ONCE(so->tx.state, ISOTP_SENDING); /* send CF frame and enable echo timeout handling */ - hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), + hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); isotp_send_cframe(so); break; @@ -435,14 +517,19 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) case ISOTP_FC_OVFLW: /* overflow on receiver side - report 'message too long' */ - sk->sk_err = EMSGSIZE; - if (!sock_flag(sk, SOCK_DEAD)) - sk_error_report(sk); + tx_err = EMSGSIZE; fallthrough; default: - /* stop this tx job */ - so->tx.state = ISOTP_IDLE; + /* reserved/unknown flow status (tx_err defaults to EBADMSG) */ + + sk->sk_err = tx_err; + if (!sock_flag(sk, SOCK_DEAD)) + sk_error_report(sk); + + isotp_set_tx_result(so, so->tx_gen, tx_err); + /* set to IDLE after publishing tx_result */ + smp_store_release(&so->tx.state, ISOTP_IDLE); wake_up_interruptible(&so->wait); } return 0; @@ -455,7 +542,7 @@ static int isotp_rcv_sf(struct sock *sk, struct canfd_frame *cf, int pcilen, struct sk_buff *nskb; hrtimer_cancel(&so->rxtimer); - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); if (!len || len > cf->len - pcilen) return 1; @@ -489,7 +576,7 @@ static int isotp_rcv_ff(struct sock *sk, struct canfd_frame *cf, int ae) int ff_pci_sz; hrtimer_cancel(&so->rxtimer); - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); /* get the used sender LL_DL from the (first) CAN frame data length */ so->rx.ll_dl = padlen(cf->len); @@ -543,7 +630,7 @@ static int isotp_rcv_ff(struct sock *sk, struct canfd_frame *cf, int ae) /* initial setup for this pdu reception */ so->rx.sn = 1; - so->rx.state = ISOTP_WAIT_DATA; + WRITE_ONCE(so->rx.state, ISOTP_WAIT_DATA); /* no creation of flow control frames */ if (so->opt.flags & CAN_ISOTP_LISTEN_MODE) @@ -561,7 +648,7 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, struct sk_buff *nskb; int i; - if (so->rx.state != ISOTP_WAIT_DATA) + if (READ_ONCE(so->rx.state) != ISOTP_WAIT_DATA) return 0; /* drop if timestamp gap is less than force_rx_stmin nano secs */ @@ -575,6 +662,12 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, hrtimer_cancel(&so->rxtimer); + /* isotp_rx_timer_handler() may have raced us for so->rx.state + * while hrtimer_cancel() above waited for it to finish => recheck + */ + if (READ_ONCE(so->rx.state) != ISOTP_WAIT_DATA) + return 1; + /* CFs are never longer than the FF */ if (cf->len > so->rx.ll_dl) return 1; @@ -593,7 +686,7 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, sk_error_report(sk); /* reset rx state */ - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); return 1; } so->rx.sn++; @@ -607,7 +700,7 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, if (so->rx.idx >= so->rx.len) { /* we are done */ - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); if ((so->opt.flags & ISOTP_CHECK_PADDING) && check_pad(so, cf, i + 1, so->opt.rxpad_content)) { @@ -678,8 +771,10 @@ static void isotp_rcv(struct sk_buff *skb, void *data) if (so->opt.flags & CAN_ISOTP_HALF_DUPLEX) { /* check rx/tx path half duplex expectations */ - if ((so->tx.state != ISOTP_IDLE && n_pci_type != N_PCI_FC) || - (so->rx.state != ISOTP_IDLE && n_pci_type == N_PCI_FC)) + if ((READ_ONCE(so->tx.state) != ISOTP_IDLE && + n_pci_type != N_PCI_FC) || + (READ_ONCE(so->rx.state) != ISOTP_IDLE && + n_pci_type == N_PCI_FC)) goto out_unlock; } @@ -774,6 +869,7 @@ static void isotp_send_cframe(struct isotp_sock *so) struct canfd_frame *cf; int can_send_ret; int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0; + u32 old_cfecho; dev = dev_get_by_index(sock_net(sk), so->ifindex); if (!dev) @@ -794,6 +890,9 @@ static void isotp_send_cframe(struct isotp_sock *so) csx->can_iif = dev->ifindex; + /* set uid in tx skb to identify CF echo frames */ + can_set_skb_uid(skb); + cf = (struct canfd_frame *)skb->data; skb_put_zero(skb, so->ll.mtu); @@ -810,12 +909,15 @@ static void isotp_send_cframe(struct isotp_sock *so) skb->dev = dev; can_skb_set_owner(skb, sk); - /* cfecho should have been zero'ed by init/isotp_rcv_echo() */ - if (so->cfecho) - pr_notice_once("can-isotp: cfecho is %08X != 0\n", so->cfecho); + /* zero'ed by init/isotp_rcv_echo(); reached lock-free via + * isotp_txfr_timer_handler() too, so use READ_ONCE()/WRITE_ONCE() + */ + old_cfecho = READ_ONCE(so->cfecho); + if (old_cfecho) + pr_notice_once("can-isotp: cfecho is %08X != 0\n", old_cfecho); /* set consecutive frame echo tag */ - so->cfecho = *(u32 *)cf->data; + WRITE_ONCE(so->cfecho, skb->hash); /* send frame with local echo enabled */ can_send_ret = can_send(skb, 1); @@ -867,81 +969,133 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data) { struct sock *sk = (struct sock *)data; struct isotp_sock *so = isotp_sk(sk); - struct canfd_frame *cf = (struct canfd_frame *)skb->data; /* only handle my own local echo CF/SF skb's (no FF!) */ - if (skb->sk != sk || so->cfecho != *(u32 *)cf->data) + if (skb->sk != sk) return; + /* unlike isotp_rcv_fc()/isotp_rcv_cf(), not already under so->rx_lock + * (no isotp_rcv() caller here), so take it ourselves + */ + spin_lock(&so->rx_lock); + + /* so->cfecho may since belong to a new transfer; recheck under lock */ + if (READ_ONCE(so->cfecho) != skb->hash) + goto out_unlock; + /* cancel local echo timeout */ - hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->echotimer); /* local echo skb with consecutive frame has been consumed */ - so->cfecho = 0; + WRITE_ONCE(so->cfecho, 0); + + /* claiming a transfer also takes so->rx_lock, so a plain recheck + * is enough: so->tx.state can't have flipped to ISOTP_SENDING for + * a new claim while we're still in here + */ + if (READ_ONCE(so->tx.state) != ISOTP_SENDING) + goto out_unlock; if (so->tx.idx >= so->tx.len) { /* we are done */ - so->tx.state = ISOTP_IDLE; + + isotp_set_tx_result(so, so->tx_gen, 0); + /* set to IDLE after publishing tx_result */ + smp_store_release(&so->tx.state, ISOTP_IDLE); wake_up_interruptible(&so->wait); - return; + goto out_unlock; } if (so->txfc.bs && so->tx.bs >= so->txfc.bs) { /* stop and wait for FC with timeout */ - so->tx.state = ISOTP_WAIT_FC; + WRITE_ONCE(so->tx.state, ISOTP_WAIT_FC); hrtimer_start(&so->txtimer, ktime_set(ISOTP_FC_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); - return; + goto out_unlock; } /* no gap between data frames needed => use burst mode */ if (!so->tx_gap) { /* enable echo timeout handling */ - hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), + hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); isotp_send_cframe(so); - return; + goto out_unlock; } /* start timer to send next consecutive frame with correct delay */ hrtimer_start(&so->txfrtimer, so->tx_gap, HRTIMER_MODE_REL_SOFT); + +out_unlock: + spin_unlock(&so->rx_lock); } -static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer) +/* isotp_tx_timeout: we did not get any flow control or echo frame in time + * + * Shared by so->txtimer's and so->echotimer's callbacks. Both timers get + * cancelled under so->rx_lock elsewhere, so this must stay lock-free. + * + * tx.state is acquired before tx_gen. Common sequence in isotp_tx_gen_done(). + * cmpxchg() only orders itself, not the two preceding loads. + */ +static enum hrtimer_restart isotp_tx_timeout(struct isotp_sock *so) { - struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, - txtimer); struct sock *sk = &so->sk; + /* read tx.state first for the common sequence */ + u32 old_state = smp_load_acquire(&so->tx.state); + u32 gen = READ_ONCE(so->tx_gen); /* don't handle timeouts in IDLE or SHUTDOWN state */ - if (so->tx.state == ISOTP_IDLE || so->tx.state == ISOTP_SHUTDOWN) + if (old_state == ISOTP_IDLE || old_state == ISOTP_SHUTDOWN) return HRTIMER_NORESTART; - /* we did not get any flow control or echo frame in time */ + /* only claim the timeout if the state is still unchanged */ + if (cmpxchg(&so->tx.state, old_state, ISOTP_IDLE) != old_state) + return HRTIMER_NORESTART; + + /* detected timeout: report 'communication error on send' */ + + /* a stale read of this slot by a waiter still falls back to ECOMM */ + isotp_set_tx_result(so, gen, ECOMM); - /* report 'communication error on send' */ sk->sk_err = ECOMM; if (!sock_flag(sk, SOCK_DEAD)) sk_error_report(sk); - /* reset tx state */ - so->tx.state = ISOTP_IDLE; wake_up_interruptible(&so->wait); return HRTIMER_NORESTART; } +/* so->txtimer: fires when a Flow Control frame does not arrive in time */ +static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + txtimer); + + return isotp_tx_timeout(so); +} + +/* so->echotimer: fires when a sent CF/SF's local echo does not arrive */ +static enum hrtimer_restart isotp_echo_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + echotimer); + + return isotp_tx_timeout(so); +} + static enum hrtimer_restart isotp_txfr_timer_handler(struct hrtimer *hrtimer) { struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, txfrtimer); /* start echo timeout handling and cover below protocol error */ - hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), + hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); /* cfecho should be consumed by isotp_rcv_echo() here */ - if (so->tx.state == ISOTP_SENDING && !so->cfecho) + if (READ_ONCE(so->tx.state) == ISOTP_SENDING && !READ_ONCE(so->cfecho)) isotp_send_cframe(so); return HRTIMER_NORESTART; @@ -958,24 +1112,61 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0; int wait_tx_done = (so->opt.flags & CAN_ISOTP_WAIT_TX_DONE) ? 1 : 0; s64 hrtimer_sec = ISOTP_ECHO_TIMEOUT; + struct hrtimer *tx_hrt = &so->echotimer; + u32 new_state = ISOTP_SENDING; + u32 my_gen; + u32 old_cfecho; int off; int err; - if (!so->bound || so->tx.state == ISOTP_SHUTDOWN) + if (!so->bound || READ_ONCE(so->tx.state) == ISOTP_SHUTDOWN) return -EADDRNOTAVAIL; - while (cmpxchg(&so->tx.state, ISOTP_IDLE, ISOTP_SENDING) != ISOTP_IDLE) { + /* claim the socket under so->rx_lock: this serializes the claim + * with the RX path and with sendmsg()'s own error paths below, so + * none of them can ever see a transfer mid-claim + */ + for (;;) { + spin_lock_bh(&so->rx_lock); + if (READ_ONCE(so->tx.state) == ISOTP_IDLE) + break; + spin_unlock_bh(&so->rx_lock); + /* we do not support multiple buffers - for now */ if (msg->msg_flags & MSG_DONTWAIT) return -EAGAIN; - if (so->tx.state == ISOTP_SHUTDOWN) + if (READ_ONCE(so->tx.state) == ISOTP_SHUTDOWN) return -EADDRNOTAVAIL; /* wait for complete transmission of current pdu */ - err = wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + err = wait_event_interruptible(so->wait, + READ_ONCE(so->tx.state) == ISOTP_IDLE || + READ_ONCE(so->tx.state) == ISOTP_SHUTDOWN); if (err) - goto err_event_drop; + return err; + } + + /* txfrtimer's callback re-arms echotimer lock-free: drain it first */ + hrtimer_cancel(&so->txfrtimer); + hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->echotimer); + + /* new transfer: increment so->tx_gen and set tx.state after barrier */ + my_gen = isotp_inc_tx_gen(READ_ONCE(so->tx_gen)); + isotp_set_tx_result(so, my_gen, ECOMM); /* prevent stale slot matching */ + WRITE_ONCE(so->tx_gen, my_gen); + smp_wmb(); /* see smp_load_acquire() in isotp_tx_[timeout|gen_done] */ + WRITE_ONCE(so->tx.state, ISOTP_SENDING); + WRITE_ONCE(so->cfecho, 0); + spin_unlock_bh(&so->rx_lock); + + /* so->bound is only checked once above - a wakeup may have + * unbound/rebound the socket meanwhile => recheck + */ + if (!so->bound) { + err = -EADDRNOTAVAIL; + goto err_out_drop; } /* PDU size > default => try max_pdu_size */ @@ -1030,6 +1221,9 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) csx->can_iif = dev->ifindex; + /* set uid in tx skb to identify CF echo frames */ + can_set_skb_uid(skb); + so->tx.len = size; so->tx.idx = 0; @@ -1037,8 +1231,9 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) skb_put_zero(skb, so->ll.mtu); /* cfecho should have been zero'ed by init / former isotp_rcv_echo() */ - if (so->cfecho) - pr_notice_once("can-isotp: uninit cfecho %08X\n", so->cfecho); + old_cfecho = READ_ONCE(so->cfecho); + if (old_cfecho) + pr_notice_once("can-isotp: uninit cfecho %08X\n", old_cfecho); /* check for single frame transmission depending on TX_DL */ if (size <= so->tx.ll_dl - SF_PCI_SZ4 - ae - off) { @@ -1066,7 +1261,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) cf->data[ae] |= size; /* set CF echo tag for isotp_rcv_echo() (SF-mode) */ - so->cfecho = *(u32 *)cf->data; + WRITE_ONCE(so->cfecho, skb->hash); } else { /* send first frame */ @@ -1083,21 +1278,36 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) so->txfc.bs = 0; /* set CF echo tag for isotp_rcv_echo() (CF-mode) */ - so->cfecho = *(u32 *)cf->data; + WRITE_ONCE(so->cfecho, skb->hash); } else { /* standard flow control check */ - so->tx.state = ISOTP_WAIT_FIRST_FC; + new_state = ISOTP_WAIT_FIRST_FC; /* start timeout for FC */ hrtimer_sec = ISOTP_FC_TIMEOUT; + tx_hrt = &so->txtimer; /* no CF echo tag for isotp_rcv_echo() (FF-mode) */ - so->cfecho = 0; + WRITE_ONCE(so->cfecho, 0); } } - hrtimer_start(&so->txtimer, ktime_set(hrtimer_sec, 0), + spin_lock_bh(&so->rx_lock); + if (READ_ONCE(so->tx.state) == ISOTP_SHUTDOWN) { + /* isotp_release() has since taken over and already drained + * our timers - don't send into a socket that's going away + */ + spin_unlock_bh(&so->rx_lock); + kfree_skb(skb); + dev_put(dev); + wake_up_interruptible(&so->wait); + return -EADDRNOTAVAIL; + } + /* WAIT_FIRST_FC for standard FF, else stays ISOTP_SENDING */ + WRITE_ONCE(so->tx.state, new_state); + hrtimer_start(tx_hrt, ktime_set(hrtimer_sec, 0), HRTIMER_MODE_REL_SOFT); + spin_unlock_bh(&so->rx_lock); /* send the first or only CAN frame */ cf->flags = so->ll.tx_flags; @@ -1110,36 +1320,80 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) pr_notice_once("can-isotp: %s: can_send_ret %pe\n", __func__, ERR_PTR(err)); - /* no transmission -> no timeout monitoring */ - hrtimer_cancel(&so->txtimer); + spin_lock_bh(&so->rx_lock); - /* reset consecutive frame echo tag */ - so->cfecho = 0; + /* new transfer already claimed by a concurrent completion, + * timeout or sendmsg() while we were stuck in can_send()? + */ + if (READ_ONCE(so->tx_gen) != my_gen) { + /* don't touch timers and state of the new transfer */ + spin_unlock_bh(&so->rx_lock); + return err; + } - goto err_out_drop; + /* no transmission -> no timeout monitoring */ + hrtimer_cancel(tx_hrt); + goto err_out_drop_locked; } if (wait_tx_done) { - /* wait for complete transmission of current pdu */ - err = wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + /* wake up for: + * - concurrent sendmsg() claiming a new transfer + * - complete transmission of current PDU + * - shutdown state change in isotp_release() + * isotp_tx_gen_done() uses common tx.state/tx_gen read sequence + */ + err = wait_event_interruptible(so->wait, + isotp_tx_gen_done(so, my_gen)); if (err) goto err_event_drop; - err = sock_error(sk); - if (err) - return err; + /* still our claim, but isotp_release() force-shut it down */ + if (smp_load_acquire(&so->tx.state) == ISOTP_SHUTDOWN && + READ_ONCE(so->tx_gen) == my_gen) { + err = -EADDRNOTAVAIL; + goto err_event_drop; + } + + /* own completion, or tx_gen moved on - either way this is + * what isotp_get_tx_result() recorded for my_gen + */ + err = isotp_get_tx_result(so, my_gen); + + /* drain to avoid stale error for a later poll()/SO_ERROR */ + sock_error(sk); + + return err ? err : size; } return size; +err_out_drop: + /* claimed but nothing sent yet - no timer to cancel */ + spin_lock_bh(&so->rx_lock); + goto err_out_drop_locked; err_event_drop: - /* got signal: force tx state machine to be idle */ - so->tx.state = ISOTP_IDLE; + /* interrupted or shut down while waiting on our own transfer */ + spin_lock_bh(&so->rx_lock); + + /* new transfer already started by concurrent sendmsg()? */ + if (READ_ONCE(so->tx_gen) != my_gen) { + /* don't touch timers and states of the new transfer */ + spin_unlock_bh(&so->rx_lock); + return err; + } + hrtimer_cancel(&so->txfrtimer); hrtimer_cancel(&so->txtimer); -err_out_drop: - /* drop this PDU and unlock a potential wait queue */ - so->tx.state = ISOTP_IDLE; + hrtimer_cancel(&so->echotimer); +err_out_drop_locked: + /* release the claim; so->rx_lock still held from above */ + WRITE_ONCE(so->cfecho, 0); + + /* only claim to IDLE if isotp_release() has not taken over */ + if (READ_ONCE(so->tx.state) != ISOTP_SHUTDOWN) + WRITE_ONCE(so->tx.state, ISOTP_IDLE); + spin_unlock_bh(&so->rx_lock); wake_up_interruptible(&so->wait); return err; @@ -1201,14 +1455,25 @@ static int isotp_release(struct socket *sock) so = isotp_sk(sk); net = sock_net(sk); - /* wait for complete transmission of current pdu */ - while (wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE) == 0 && - cmpxchg(&so->tx.state, ISOTP_IDLE, ISOTP_SHUTDOWN) != ISOTP_IDLE) + /* best-effort: wait for a running pdu to finish, but don't block on + * it forever - give up after the first signal + */ + while (READ_ONCE(so->tx.state) != ISOTP_IDLE && + wait_event_interruptible(so->wait, + READ_ONCE(so->tx.state) == ISOTP_IDLE) == 0) ; - /* force state machines to be idle also when a signal occurred */ - so->tx.state = ISOTP_SHUTDOWN; - so->rx.state = ISOTP_IDLE; + /* claim the socket under so->rx_lock like sendmsg() does, so its + * claim can't race the forced ISOTP_SHUTDOWN below; force it + * unconditionally, even when a signal cut the wait above short + */ + spin_lock_bh(&so->rx_lock); + WRITE_ONCE(so->tx.state, ISOTP_SHUTDOWN); + spin_unlock_bh(&so->rx_lock); + WRITE_ONCE(so->rx.state, ISOTP_IDLE); + + /* forced SHUTDOWN may have skipped IDLE (gave up on a signal) */ + wake_up_interruptible(&so->wait); spin_lock(&isotp_notifier_lock); while (isotp_busy_notifier == so) { @@ -1219,36 +1484,43 @@ static int isotp_release(struct socket *sock) list_del(&so->notifier); spin_unlock(&isotp_notifier_lock); + rtnl_lock(); lock_sock(sk); - /* remove current filters & unregister */ - if (so->bound) { - if (so->ifindex) { - struct net_device *dev; - - dev = dev_get_by_index(net, so->ifindex); - if (dev) { - if (isotp_register_rxid(so)) - can_rx_unregister(net, dev, so->rxid, - SINGLE_MASK(so->rxid), - isotp_rcv, sk); - - can_rx_unregister(net, dev, so->txid, - SINGLE_MASK(so->txid), - isotp_rcv_echo, sk); - dev_put(dev); - synchronize_rcu(); - } - } + /* remove current filters & unregister + * tracked reference so->dev is taken at bind() time with rtnl_lock + */ + if (so->bound && so->dev) { + if (isotp_register_rxid(so)) + can_rx_unregister(net, so->dev, so->rxid, + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + + can_rx_unregister(net, so->dev, so->txid, + SINGLE_MASK(so->txid), + isotp_rcv_echo, sk); + netdev_put(so->dev, &so->dev_tracker); } + so->ifindex = 0; + so->bound = 0; + so->dev = NULL; + + rtnl_unlock(); + + /* Always wait for a grace period before touching the timers below. + * A concurrent NETDEV_UNREGISTER may have already unregistered our + * filters and cleared so->bound in isotp_notify() without waiting + * for in-flight isotp_rcv() callers to finish, so this call must not + * be skipped just because so->bound is already 0 here. + */ + synchronize_rcu(); + hrtimer_cancel(&so->txfrtimer); hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->echotimer); hrtimer_cancel(&so->rxtimer); - so->ifindex = 0; - so->bound = 0; - sock_orphan(sk); sock->sk = NULL; @@ -1303,6 +1575,7 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l if (!addr->can_ifindex) return -ENODEV; + rtnl_lock(); lock_sock(sk); if (so->bound) { @@ -1310,6 +1583,18 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l goto out; } + /* A transmission or reception that outlived a previous binding + * (unbound by NETDEV_UNREGISTER) may still be draining; the FC/echo + * and RX watchdog timers bound how long this takes. Checked together + * with so->bound in the same lock_sock() section above, so there is + * no window in which a concurrent isotp_notify() could be missed. + */ + if (READ_ONCE(so->tx.state) != ISOTP_IDLE || + READ_ONCE(so->rx.state) != ISOTP_IDLE) { + err = -EAGAIN; + goto out; + } + /* ensure different CAN IDs when the rx_id is to be registered */ if (isotp_register_rxid(so) && rx_id == tx_id) { err = -EADDRNOTAVAIL; @@ -1322,14 +1607,12 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l goto out; } if (dev->type != ARPHRD_CAN) { - dev_put(dev); err = -ENODEV; - goto out; + goto out_put_dev; } if (READ_ONCE(dev->mtu) < so->ll.mtu) { - dev_put(dev); err = -EINVAL; - goto out; + goto out_put_dev; } if (!(dev->flags & IFF_UP)) notify_enetdown = 1; @@ -1341,22 +1624,31 @@ static int isotp_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int l isotp_rcv, sk, "isotp", sk); /* no consecutive frame echo skb in flight */ - so->cfecho = 0; + WRITE_ONCE(so->cfecho, 0); /* register for echo skb's */ can_rx_register(net, dev, tx_id, SINGLE_MASK(tx_id), isotp_rcv_echo, sk, "isotpe", sk); - dev_put(dev); - /* switch to new settings */ so->ifindex = ifindex; so->rxid = rx_id; so->txid = tx_id; so->bound = 1; + /* bind() ok -> hold a reference for so->dev so that isotp_release() + * can safely reach the device later, even if a concurrent + * NETDEV_UNREGISTER has already unlisted it by ifindex. + */ + so->dev = dev; + netdev_hold(so->dev, &so->dev_tracker, GFP_KERNEL); + +out_put_dev: + /* remove potential reference from dev_get_by_index() */ + dev_put(dev); out: release_sock(sk); + rtnl_unlock(); if (notify_enetdown) { sk->sk_err = ENETDOWN; @@ -1559,7 +1851,7 @@ static void isotp_notify(struct isotp_sock *so, unsigned long msg, if (!net_eq(dev_net(dev), sock_net(sk))) return; - if (so->ifindex != dev->ifindex) + if (so->dev != dev) return; switch (msg) { @@ -1575,10 +1867,12 @@ static void isotp_notify(struct isotp_sock *so, unsigned long msg, can_rx_unregister(dev_net(dev), dev, so->txid, SINGLE_MASK(so->txid), isotp_rcv_echo, sk); + netdev_put(so->dev, &so->dev_tracker); } so->ifindex = 0; so->bound = 0; + so->dev = NULL; release_sock(sk); sk->sk_err = ENODEV; @@ -1638,6 +1932,7 @@ static int isotp_init(struct sock *sk) so->ifindex = 0; so->bound = 0; + so->dev = NULL; so->opt.flags = CAN_ISOTP_DEFAULT_FLAGS; so->opt.ext_address = CAN_ISOTP_DEFAULT_EXT_ADDRESS; @@ -1664,10 +1959,14 @@ static int isotp_init(struct sock *sk) so->rx.buflen = ARRAY_SIZE(so->rx.sbuf); so->tx.buflen = ARRAY_SIZE(so->tx.sbuf); - hrtimer_setup(&so->rxtimer, isotp_rx_timer_handler, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); - hrtimer_setup(&so->txtimer, isotp_tx_timer_handler, CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); - hrtimer_setup(&so->txfrtimer, isotp_txfr_timer_handler, CLOCK_MONOTONIC, - HRTIMER_MODE_REL_SOFT); + hrtimer_setup(&so->rxtimer, isotp_rx_timer_handler, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + hrtimer_setup(&so->txtimer, isotp_tx_timer_handler, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + hrtimer_setup(&so->echotimer, isotp_echo_timer_handler, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); + hrtimer_setup(&so->txfrtimer, isotp_txfr_timer_handler, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_SOFT); init_waitqueue_head(&so->wait); spin_lock_init(&so->rx_lock); @@ -1691,7 +1990,7 @@ static __poll_t isotp_poll(struct file *file, struct socket *sock, poll_table *w poll_wait(file, &so->wait, wait); /* Check for false positives due to TX state */ - if ((mask & EPOLLWRNORM) && (so->tx.state != ISOTP_IDLE)) + if ((mask & EPOLLWRNORM) && (READ_ONCE(so->tx.state) != ISOTP_IDLE)) mask &= ~(EPOLLOUT | EPOLLWRNORM); return mask; @@ -1751,13 +2050,18 @@ static __init int isotp_module_init(void) pr_info("can: isotp protocol (max_pdu_size %d)\n", max_pdu_size); + err = register_netdevice_notifier(&canisotp_notifier); + if (err) + return err; + err = can_proto_register(&isotp_can_proto); - if (err < 0) + if (err < 0) { pr_err("can: registration of isotp protocol failed %pe\n", ERR_PTR(err)); - else - register_netdevice_notifier(&canisotp_notifier); + unregister_netdevice_notifier(&canisotp_notifier); + return err; + } - return err; + return 0; } static __exit void isotp_module_exit(void) diff --git a/net/can/j1939/bus.c b/net/can/j1939/bus.c index dc374286eeb6..cdc3c0a71937 100644 --- a/net/can/j1939/bus.c +++ b/net/can/j1939/bus.c @@ -20,6 +20,7 @@ static void __j1939_ecu_release(struct kref *kref) struct j1939_priv *priv = ecu->priv; list_del(&ecu->list); + netdev_put(priv->ndev, &ecu->priv_dev_tracker); kfree(ecu); j1939_priv_put(priv); } @@ -155,6 +156,7 @@ struct j1939_ecu *j1939_ecu_create_locked(struct j1939_priv *priv, name_t name) if (!ecu) return ERR_PTR(-ENOMEM); kref_init(&ecu->kref); + netdev_hold(priv->ndev, &ecu->priv_dev_tracker, gfp_any()); ecu->addr = J1939_IDLE_ADDR; ecu->name = name; diff --git a/net/can/j1939/j1939-priv.h b/net/can/j1939/j1939-priv.h index 81f58924b4ac..cf26352d1d8c 100644 --- a/net/can/j1939/j1939-priv.h +++ b/net/can/j1939/j1939-priv.h @@ -38,6 +38,7 @@ struct j1939_ecu { struct hrtimer ac_timer; struct kref kref; struct j1939_priv *priv; + netdevice_tracker priv_dev_tracker; /* count users, to help transport protocol decide for interaction */ int nusers; @@ -60,6 +61,7 @@ struct j1939_priv { rwlock_t lock; struct net_device *ndev; + netdevice_tracker dev_tracker; /* list of 256 ecu ptrs, that cache the claimed addresses. * also protected by the above lock @@ -230,6 +232,7 @@ enum j1939_session_state { struct j1939_session { struct j1939_priv *priv; + netdevice_tracker priv_dev_tracker; struct list_head active_session_list_entry; struct list_head sk_session_queue_entry; struct kref kref; diff --git a/net/can/j1939/main.c b/net/can/j1939/main.c index 9937c04241bc..5e5e6c228f22 100644 --- a/net/can/j1939/main.c +++ b/net/can/j1939/main.c @@ -137,7 +137,7 @@ static struct j1939_priv *j1939_priv_create(struct net_device *ndev) priv->ndev = ndev; kref_init(&priv->kref); kref_init(&priv->rx_kref); - dev_hold(ndev); + netdev_hold(ndev, &priv->dev_tracker, GFP_KERNEL); netdev_dbg(priv->ndev, "%s : 0x%p\n", __func__, priv); @@ -163,7 +163,7 @@ static void __j1939_priv_release(struct kref *kref) WARN_ON_ONCE(!list_empty(&priv->ecus)); WARN_ON_ONCE(!list_empty(&priv->j1939_socks)); - dev_put(ndev); + netdev_put(ndev, &priv->dev_tracker); kfree(priv); } @@ -281,7 +281,7 @@ struct j1939_priv *j1939_netdev_start(struct net_device *ndev) */ kref_get(&priv_new->rx_kref); mutex_unlock(&j1939_netdev_lock); - dev_put(ndev); + netdev_put(ndev, &priv->dev_tracker); kfree(priv); return priv_new; } @@ -298,7 +298,7 @@ struct j1939_priv *j1939_netdev_start(struct net_device *ndev) j1939_priv_set(ndev, NULL); mutex_unlock(&j1939_netdev_lock); - dev_put(ndev); + netdev_put(ndev, &priv->dev_tracker); kfree(priv); return ERR_PTR(ret); diff --git a/net/can/j1939/transport.c b/net/can/j1939/transport.c index df93d57907da..8fcfd13e5e6f 100644 --- a/net/can/j1939/transport.c +++ b/net/can/j1939/transport.c @@ -283,6 +283,7 @@ static void j1939_session_destroy(struct j1939_session *session) kfree_skb(skb); } __j1939_session_drop(session); + netdev_put(session->priv->ndev, &session->priv_dev_tracker); j1939_priv_put(session->priv); kfree(session); } @@ -351,6 +352,18 @@ static void j1939_session_skb_drop_old(struct j1939_session *session) } } +static bool j1939_address_is_local(struct j1939_priv *priv, u8 addr) +{ + bool local = false; + + read_lock_bh(&priv->lock); + if (j1939_address_is_unicast(addr) && priv->ents[addr].nusers) + local = true; + read_unlock_bh(&priv->lock); + + return local; +} + void j1939_session_skb_queue(struct j1939_session *session, struct sk_buff *skb) { @@ -359,8 +372,7 @@ void j1939_session_skb_queue(struct j1939_session *session, j1939_ac_fixup(priv, skb); - if (j1939_address_is_unicast(skcb->addr.da) && - priv->ents[skcb->addr.da].nusers) + if (j1939_address_is_local(priv, skcb->addr.da)) skcb->flags |= J1939_ECU_LOCAL_DST; skcb->flags |= J1939_ECU_LOCAL_SRC; @@ -1515,6 +1527,7 @@ static struct j1939_session *j1939_session_new(struct j1939_priv *priv, INIT_LIST_HEAD(&session->active_session_list_entry); INIT_LIST_HEAD(&session->sk_session_queue_entry); kref_init(&session->kref); + netdev_hold(priv->ndev, &session->priv_dev_tracker, gfp_any()); j1939_priv_get(priv); session->priv = priv; @@ -1568,7 +1581,7 @@ j1939_session *j1939_session_fresh_new(struct j1939_priv *priv, } /* alloc data area */ - skb_put(skb, size); + skb_put_zero(skb, size); /* skb is recounted in j1939_session_new() */ return session; } @@ -2038,8 +2051,7 @@ struct j1939_session *j1939_tp_send(struct j1939_priv *priv, return ERR_PTR(ret); /* fix DST flags, it may be used there soon */ - if (j1939_address_is_unicast(skcb->addr.da) && - priv->ents[skcb->addr.da].nusers) + if (j1939_address_is_local(priv, skcb->addr.da)) skcb->flags |= J1939_ECU_LOCAL_DST; /* src is always local, I'm sending ... */ diff --git a/net/can/raw.c b/net/can/raw.c index a26942e78e68..82d9c0499c95 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -562,8 +562,8 @@ static int raw_getname(struct socket *sock, struct sockaddr *uaddr, return RAW_MIN_NAMELEN; } -static int raw_setsockopt(struct socket *sock, int level, int optname, - sockptr_t optval, unsigned int optlen) +static int raw_setsockopt_locked(struct socket *sock, int optname, + sockptr_t optval, unsigned int optlen) { struct sock *sk = sock->sk; struct raw_sock *ro = raw_sk(sk); @@ -575,9 +575,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, int flag; int err = 0; - if (level != SOL_CAN_RAW) - return -EINVAL; - switch (optname) { case CAN_RAW_FILTER: if (optlen % sizeof(struct can_filter) != 0) @@ -598,17 +595,11 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, return -EFAULT; } - rtnl_lock(); - lock_sock(sk); - dev = ro->dev; - if (ro->bound && dev) { - if (dev->reg_state != NETREG_REGISTERED) { - if (count > 1) - kfree(filter); - err = -ENODEV; - goto out_fil; - } + if (ro->bound && dev && dev->reg_state != NETREG_REGISTERED) { + if (count > 1) + kfree(filter); + return -ENODEV; } if (ro->bound) { @@ -622,7 +613,7 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, if (err) { if (count > 1) kfree(filter); - goto out_fil; + return err; } /* remove old filter registrations */ @@ -642,11 +633,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, } ro->filter = filter; ro->count = count; - - out_fil: - release_sock(sk); - rtnl_unlock(); - break; case CAN_RAW_ERR_FILTER: @@ -658,16 +644,9 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, err_mask &= CAN_ERR_MASK; - rtnl_lock(); - lock_sock(sk); - dev = ro->dev; - if (ro->bound && dev) { - if (dev->reg_state != NETREG_REGISTERED) { - err = -ENODEV; - goto out_err; - } - } + if (ro->bound && dev && dev->reg_state != NETREG_REGISTERED) + return -ENODEV; /* remove current error mask */ if (ro->bound) { @@ -676,7 +655,7 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, err_mask); if (err) - goto out_err; + return err; /* remove old err_mask registration */ raw_disable_errfilter(sock_net(sk), dev, sk, @@ -685,11 +664,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, /* link new err_mask to the socket */ ro->err_mask = err_mask; - - out_err: - release_sock(sk); - rtnl_unlock(); - break; case CAN_RAW_LOOPBACK: @@ -769,6 +743,26 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, return err; } +static int raw_setsockopt(struct socket *sock, int level, int optname, + sockptr_t optval, unsigned int optlen) +{ + struct sock *sk = sock->sk; + int err; + + if (level != SOL_CAN_RAW) + return -EINVAL; + + rtnl_lock(); + lock_sock(sk); + + err = raw_setsockopt_locked(sock, optname, optval, optlen); + + release_sock(sk); + rtnl_unlock(); + + return err; +} + static int raw_getsockopt(struct socket *sock, int level, int optname, sockopt_t *opt) { diff --git a/net/ceph/auth_x.c b/net/ceph/auth_x.c index 9e64e82d0b63..50a79e8aa656 100644 --- a/net/ceph/auth_x.c +++ b/net/ceph/auth_x.c @@ -849,9 +849,16 @@ static int ceph_x_update_authorizer( au = (struct ceph_x_authorizer *)auth->authorizer; if (au->secret_id < th->secret_id) { + int ret; + dout("ceph_x_update_authorizer service %u secret %llu < %llu\n", au->service, au->secret_id, th->secret_id); - return ceph_x_build_authorizer(ac, th, au); + ret = ceph_x_build_authorizer(ac, th, au); + if (ret) + return ret; + + auth->authorizer_buf = au->buf->vec.iov_base; + auth->authorizer_buf_len = au->buf->vec.iov_len; } return 0; } diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index 952121849180..a797c7360e3c 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -762,13 +762,13 @@ void ceph_destroy_client(struct ceph_client *client) atomic_set(&client->msgr.stopping, 1); + ceph_debugfs_client_cleanup(client); + /* unmount */ ceph_osdc_stop(&client->osdc); ceph_monc_stop(&client->monc); ceph_messenger_fini(&client->msgr); - ceph_debugfs_client_cleanup(client); - ceph_destroy_options(client->options); kfree(client); diff --git a/net/ceph/cls_lock_client.c b/net/ceph/cls_lock_client.c index c6956f1df333..2c8fefc1a721 100644 --- a/net/ceph/cls_lock_client.c +++ b/net/ceph/cls_lock_client.c @@ -259,7 +259,8 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker) if (ret) return ret; - ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name)); + ceph_decode_copy_safe(p, end, &locker->id.name, + sizeof(locker->id.name), bad); s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO); if (IS_ERR(s)) return PTR_ERR(s); @@ -270,19 +271,23 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker) if (ret) return ret; - *p += sizeof(struct ceph_timespec); /* skip expiration */ + /* skip expiration */ + ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), bad); ret = ceph_decode_entity_addr(p, end, &locker->info.addr); if (ret) return ret; - len = ceph_decode_32(p); - *p += len; /* skip description */ + /* skip description */ + ceph_decode_skip_string(p, end, bad); dout("%s %s%llu cookie %s addr %s\n", __func__, ENTITY_NAME(locker->id.name), locker->id.cookie, ceph_pr_addr(&locker->info.addr)); return 0; + +bad: + return -EINVAL; } static int decode_lockers(void **p, void *end, u8 *type, char **tag, @@ -299,7 +304,7 @@ static int decode_lockers(void **p, void *end, u8 *type, char **tag, if (ret) return ret; - *num_lockers = ceph_decode_32(p); + ceph_decode_32_safe(p, end, *num_lockers, err_inval); *lockers = kzalloc_objs(**lockers, *num_lockers, GFP_NOIO); if (!*lockers) return -ENOMEM; @@ -310,7 +315,8 @@ static int decode_lockers(void **p, void *end, u8 *type, char **tag, goto err_free_lockers; } - *type = ceph_decode_8(p); + ret = -EINVAL; + ceph_decode_8_safe(p, end, *type, err_free_lockers); s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO); if (IS_ERR(s)) { ret = PTR_ERR(s); @@ -320,6 +326,9 @@ static int decode_lockers(void **p, void *end, u8 *type, char **tag, *tag = s; return 0; +err_inval: + return -EINVAL; + err_free_lockers: ceph_free_lockers(*lockers, *num_lockers); return ret; diff --git a/net/ceph/decode.c b/net/ceph/decode.c index bc109a1a4616..2f21af38cd9a 100644 --- a/net/ceph/decode.c +++ b/net/ceph/decode.c @@ -87,8 +87,9 @@ bad: EXPORT_SYMBOL(ceph_decode_entity_addr); /* - * Return addr of desired type (MSGR2 or LEGACY) or error. - * Make sure there is only one match. + * Return addr of desired type (MSGR2 or LEGACY) or error. In case of + * multiple matches, use the first one for compatibility with userspace + * messenger. * * Assume encoding with MSG_ADDR2. */ @@ -121,14 +122,13 @@ int ceph_decode_entity_addrvec(void **p, void *end, bool msgr2, dout("%s i %d addr %s\n", __func__, i, ceph_pr_addr(&tmp_addr)); if (tmp_addr.type == my_type) { - if (found) { - pr_err("another match of type %d in addrvec\n", - le32_to_cpu(my_type)); - return -EINVAL; + if (!found) { + memcpy(addr, &tmp_addr, sizeof(*addr)); + found = true; + } else { + dout("%s skipping extra match of type %d in addrvec\n", + __func__, le32_to_cpu(my_type)); } - - memcpy(addr, &tmp_addr, sizeof(*addr)); - found = true; } } diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index d2cdc8ee3155..c56457378d00 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -114,7 +114,7 @@ static struct ceph_monmap *ceph_monmap_decode(void **p, void *end, bool msgr2) dout("%s fsid %pU epoch %u num_mon %u\n", __func__, &fsid, epoch, num_mon); - if (num_mon > CEPH_MAX_MON) + if (num_mon == 0 || num_mon > CEPH_MAX_MON) goto e_inval; monmap = kmalloc_flex(*monmap, mon_inst, num_mon, GFP_NOIO); @@ -821,7 +821,7 @@ static void handle_get_version_reply(struct ceph_mon_client *monc, struct ceph_mon_generic_request *req; u64 tid = le64_to_cpu(msg->hdr.tid); void *p = msg->front.iov_base; - void *end = p + msg->front_alloc_len; + void *const end = p + msg->front.iov_len; u64 handle; dout("%s msg %p tid %llu\n", __func__, msg, tid); diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 2ff00070c181..28d76c2f6b3e 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -5030,7 +5030,7 @@ static int decode_watchers(void **p, void *end, if (ret) return ret; - *num_watchers = ceph_decode_32(p); + ceph_decode_32_safe(p, end, *num_watchers, bad); *watchers = kzalloc_objs(**watchers, *num_watchers, GFP_NOIO); if (!*watchers) return -ENOMEM; @@ -5044,6 +5044,9 @@ static int decode_watchers(void **p, void *end, } return 0; + +bad: + return -EINVAL; } /* diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c index 8b5b0587a0cf..d6282f0bcff8 100644 --- a/net/ceph/osdmap.c +++ b/net/ceph/osdmap.c @@ -518,6 +518,8 @@ static struct crush_map *crush_decode(void *pbyval, void *end) ceph_decode_need(p, end, 4*sizeof(u32), bad); b->id = ceph_decode_32(p); b->type = ceph_decode_16(p); + if (b->type == 0) + goto bad; b->alg = ceph_decode_8(p); if (b->alg != alg) { b->alg = 0; @@ -1436,7 +1438,7 @@ static struct ceph_pg_mapping *__decode_pg_temp(void **p, void *end, ceph_decode_32_safe(p, end, len, e_inval); if (len == 0 && incremental) return NULL; /* new_pg_temp: [] to remove */ - if ((size_t)len > (SIZE_MAX - sizeof(*pg)) / sizeof(u32)) + if (len > CEPH_PG_MAX_SIZE) return ERR_PTR(-EINVAL); ceph_decode_need(p, end, len * sizeof(u32), e_inval); @@ -1617,7 +1619,7 @@ static struct ceph_pg_mapping *__decode_pg_upmap_items(void **p, void *end, u32 len, i; ceph_decode_32_safe(p, end, len, e_inval); - if ((size_t)len > (SIZE_MAX - sizeof(*pg)) / (2 * sizeof(u32))) + if (len > CEPH_PG_MAX_SIZE) return ERR_PTR(-EINVAL); ceph_decode_need(p, end, 2 * len * sizeof(u32), e_inval); @@ -1842,6 +1844,8 @@ static int decode_new_up_state_weight(void **p, void *end, u8 struct_v, void *new_up_client; void *new_state; void *new_weight_end; + const u32 new_state_item_size = + sizeof(u32) + (struct_v >= 5 ? sizeof(u32) : sizeof(u8)); u32 len; int ret; int i; @@ -1862,7 +1866,8 @@ static int decode_new_up_state_weight(void **p, void *end, u8 struct_v, new_state = *p; ceph_decode_32_safe(p, end, len, e_inval); - len *= sizeof(u32) + (struct_v >= 5 ? sizeof(u32) : sizeof(u8)); + if (check_mul_overflow(len, new_state_item_size, &len)) + goto e_inval; ceph_decode_need(p, end, len, e_inval); *p += len; @@ -2804,9 +2809,10 @@ static void get_temp_osds(struct ceph_osdmap *osdmap, } } - /* primary_temp? */ + /* primary_temp? (shouldn't ever be a nonexistent or down OSD) */ pg = lookup_pg_mapping(&osdmap->primary_temp, pgid); - if (pg) + if (pg && !WARN_ON_ONCE(ceph_osd_is_down(osdmap, + pg->primary_temp.osd))) temp->primary = pg->primary_temp.osd; } @@ -3055,8 +3061,11 @@ static int get_immediate_parent(struct crush_map *c, int id, if (b->items[j] != id) continue; - *parent_type_id = b->type; type_cn = lookup_crush_name(&c->type_names, b->type); + if (WARN_ON_ONCE(!type_cn)) + continue; + + *parent_type_id = b->type; parent_loc->cl_type_name = type_cn->cn_name; parent_loc->cl_name = cn->cn_name; return b->id; diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c index ecd659f79fd4..1d295a8769fa 100644 --- a/net/core/bpf_sk_storage.c +++ b/net/core/bpf_sk_storage.c @@ -158,8 +158,6 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) struct bpf_local_storage_elem *selem; int ret = 0; - RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL); - rcu_read_lock_dont_migrate(); sk_storage = rcu_dereference(sk->sk_bpf_storage); diff --git a/net/core/datagram.c b/net/core/datagram.c index c285c6465923..173b5d97bd40 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -712,6 +712,9 @@ zerocopy_fill_skb_from_devmem(struct sk_buff *skb, struct iov_iter *from, size_t virt_addr, size, off; struct net_iov *niov; + if (i && skb_frags_readable(skb)) + return -EFAULT; + /* Devmem filling works by taking an IOVEC from the user where the * iov_addrs are interpreted as an offset in bytes into the dma-buf to * send from. We do not support other iter types. diff --git a/net/core/dev.c b/net/core/dev.c index 4b3d5cfdf6e0..ece6700536d9 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4018,6 +4018,9 @@ out_free: return NULL; } +/* Returns the skb on success, NULL if dropped, or ERR_PTR(-EINPROGRESS) + * if stolen by async xfrm crypto (delivered via xfrm_dev_resume()). + */ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again) { netdev_features_t features; @@ -4089,7 +4092,7 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d skb->prev = skb; skb = validate_xmit_skb(skb, dev, again); - if (!skb) + if (IS_ERR_OR_NULL(skb)) continue; if (!head) @@ -4860,8 +4863,11 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) goto recursion_alert; skb = validate_xmit_skb(skb, dev, &again); - if (!skb) + if (IS_ERR_OR_NULL(skb)) { + if (PTR_ERR(skb) == -EINPROGRESS) + rc = NET_XMIT_SUCCESS; goto out; + } HARD_TX_LOCK(dev, txq, cpu); @@ -5511,12 +5517,16 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, } /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers - * (e.g. bpf_xdp_adjust_tail), we need to update data_len here. + * (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution + * from skb->len before updating data_len, then add the new one back. */ - if (xdp_buff_has_frags(xdp)) + skb->len -= skb->data_len; + if (xdp_buff_has_frags(xdp)) { skb->data_len = skb_shinfo(skb)->xdp_frags_size; - else + skb->len += skb->data_len; + } else { skb->data_len = 0; + } /* check if XDP changed eth hdr such SKB needs update */ eth = (struct ethhdr *)xdp->data; @@ -11488,6 +11498,7 @@ int register_netdevice(struct net_device *dev) * Prevent userspace races by waiting until the network * device is fully setup before sending notifications. */ + netdev_uevent_add(dev); if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing)) rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL); @@ -12429,6 +12440,7 @@ void unregister_netdevice_many_notify(struct list_head *head, dev_tcx_uninstall(dev); dev_xdp_uninstall(dev); dev_memory_provider_uninstall(dev); + netdev_work_cancel_all(dev); netdev_unlock_ops(dev); bpf_dev_bound_netdev_unregister(dev); diff --git a/net/core/dev.h b/net/core/dev.h index 5d0b0305d3ba..b757faead4d1 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -179,6 +179,7 @@ enum netdev_work_core { void __netdev_work_core_sched(struct net_device *dev, unsigned long event); unsigned long __netdev_work_core_cancel(struct net_device *dev, unsigned long mask); +void netdev_work_cancel_all(struct net_device *dev); void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, unsigned int gchanges, u32 portid, diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index 2bf3cab5e557..abaf108ac4db 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -530,10 +530,10 @@ static void net_dm_packet_trace_kfree_skb_hit(void *ignore, return; unlock_free: - spin_unlock_irqrestore(&data->drop_queue.lock, flags); u64_stats_update_begin(&data->stats.syncp); u64_stats_inc(&data->stats.dropped); u64_stats_update_end(&data->stats.syncp); + spin_unlock_irqrestore(&data->drop_queue.lock, flags); consume_skb(nskb); } @@ -566,13 +566,13 @@ static size_t net_dm_packet_report_size(size_t payload_len) /* NET_DM_ATTR_ORIGIN */ nla_total_size(sizeof(u16)) + /* NET_DM_ATTR_PC */ - nla_total_size(sizeof(u64)) + + nla_total_size_64bit(sizeof(u64)) + /* NET_DM_ATTR_SYMBOL */ nla_total_size(NET_DM_MAX_SYMBOL_LEN + 1) + /* NET_DM_ATTR_IN_PORT */ net_dm_in_port_size() + /* NET_DM_ATTR_TIMESTAMP */ - nla_total_size(sizeof(u64)) + + nla_total_size_64bit(sizeof(u64)) + /* NET_DM_ATTR_ORIG_LEN */ nla_total_size(sizeof(u32)) + /* NET_DM_ATTR_PROTO */ @@ -671,9 +671,7 @@ static int net_dm_packet_report_fill(struct sk_buff *msg, struct sk_buff *skb, if (nla_put_u16(msg, NET_DM_ATTR_PROTO, be16_to_cpu(skb->protocol))) goto nla_put_failure; - attr = skb_put(msg, nla_total_size(payload_len)); - attr->nla_type = NET_DM_ATTR_PAYLOAD; - attr->nla_len = nla_attr_size(payload_len); + attr = __nla_reserve(msg, NET_DM_ATTR_PAYLOAD, payload_len); if (skb_copy_bits(skb, 0, nla_data(attr), payload_len)) goto nla_put_failure; @@ -768,7 +766,7 @@ net_dm_hw_packet_report_size(size_t payload_len, /* NET_DM_ATTR_FLOW_ACTION_COOKIE */ net_dm_flow_action_cookie_size(hw_metadata) + /* NET_DM_ATTR_TIMESTAMP */ - nla_total_size(sizeof(u64)) + + nla_total_size_64bit(sizeof(u64)) + /* NET_DM_ATTR_ORIG_LEN */ nla_total_size(sizeof(u32)) + /* NET_DM_ATTR_PROTO */ @@ -831,9 +829,7 @@ static int net_dm_hw_packet_report_fill(struct sk_buff *msg, if (nla_put_u16(msg, NET_DM_ATTR_PROTO, be16_to_cpu(skb->protocol))) goto nla_put_failure; - attr = skb_put(msg, nla_total_size(payload_len)); - attr->nla_type = NET_DM_ATTR_PAYLOAD; - attr->nla_len = nla_attr_size(payload_len); + attr = __nla_reserve(msg, NET_DM_ATTR_PAYLOAD, payload_len); if (skb_copy_bits(skb, 0, nla_data(attr), payload_len)) goto nla_put_failure; @@ -1001,10 +997,10 @@ net_dm_hw_trap_packet_probe(void *ignore, const struct devlink *devlink, return; unlock_free: - spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags); u64_stats_update_begin(&hw_data->stats.syncp); u64_stats_inc(&hw_data->stats.dropped); u64_stats_update_end(&hw_data->stats.syncp); + spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags); net_dm_hw_metadata_free(n_hw_metadata); free: consume_skb(nskb); diff --git a/net/core/filter.c b/net/core/filter.c index b446aa8be5c3..16845987b244 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -2552,11 +2552,13 @@ out_drop: BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags) { - struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); + struct bpf_redirect_info *ri; - if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL))) + if (unlikely(!bpf_net_ctx_get() || + (flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))) return TC_ACT_SHOT; + ri = bpf_net_ctx_get_ri(); ri->flags = flags; ri->tgt_index = ifindex; @@ -2573,11 +2575,12 @@ static const struct bpf_func_proto bpf_redirect_proto = { BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags) { - struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); + struct bpf_redirect_info *ri; - if (unlikely(flags)) + if (unlikely(!bpf_net_ctx_get() || flags)) return TC_ACT_SHOT; + ri = bpf_net_ctx_get_ri(); ri->flags = BPF_F_PEER; ri->tgt_index = ifindex; @@ -2595,11 +2598,13 @@ static const struct bpf_func_proto bpf_redirect_peer_proto = { BPF_CALL_4(bpf_redirect_neigh, u32, ifindex, struct bpf_redir_neigh *, params, int, plen, u64, flags) { - struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); + struct bpf_redirect_info *ri; - if (unlikely((plen && plen < sizeof(*params)) || flags)) + if (unlikely((plen && plen < sizeof(*params)) || + !bpf_net_ctx_get() || flags)) return TC_ACT_SHOT; + ri = bpf_net_ctx_get_ri(); ri->flags = BPF_F_NEIGH | (plen ? BPF_F_NEXTHOP : 0); ri->tgt_index = ifindex; @@ -7679,7 +7684,7 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len return -EINVAL; /* sk_listener() allows TCP_NEW_SYN_RECV, which makes no sense here. */ - if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN) + if (sk->sk_state != TCP_LISTEN || sk->sk_protocol != IPPROTO_TCP) return -EINVAL; if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies)) @@ -7752,7 +7757,7 @@ BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len, if (unlikely(!sk || th_len < sizeof(*th) || th_len != th->doff * 4)) return -EINVAL; - if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN) + if (sk->sk_state != TCP_LISTEN || sk->sk_protocol != IPPROTO_TCP) return -EINVAL; if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies)) diff --git a/net/core/gro.c b/net/core/gro.c index 35f2f708f010..29b4d02bf519 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -229,7 +229,9 @@ done: int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) { - if (unlikely(p->len + skb->len >= 65536)) + /* make sure to check flush flag and to not merge */ + if (unlikely(p->len + skb->len >= 65536 || + NAPI_GRO_CB(skb)->flush)) return -E2BIG; if (!pskb_may_pull(skb, skb_gro_offset(skb))) { diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c index bf588f508b79..652952d416f2 100644 --- a/net/core/lwt_bpf.c +++ b/net/core/lwt_bpf.c @@ -255,8 +255,10 @@ static int bpf_lwt_xmit_reroute(struct sk_buff *skb) * if there is enough header space in skb. */ err = skb_cow_head(skb, LL_RESERVED_SPACE(dst->dev)); - if (unlikely(err)) + if (unlikely(err)) { + dst_release(dst); goto err; + } skb_dst_drop(skb); skb_dst_set(skb, dst); diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 0e71c9ed41e8..25546deacec8 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -2334,6 +2334,9 @@ int netdev_register_kobject(struct net_device *ndev) *groups++ = &wireless_group; #endif /* CONFIG_SYSFS */ + /* Hold back the KOBJ_ADD uevent until the device is listed. */ + dev_set_uevent_suppress(dev, 1); + error = device_add(dev); if (error) return error; @@ -2349,6 +2352,17 @@ int netdev_register_kobject(struct net_device *ndev) return error; } +/* Announce a fully registered device to userspace. This pairs with the uevent + * suppression from netdev_register_kobject(). + */ +void netdev_uevent_add(struct net_device *ndev) +{ + struct device *dev = &ndev->dev; + + dev_set_uevent_suppress(dev, 0); + kobject_uevent(&dev->kobj, KOBJ_ADD); +} + /* Change owner for sysfs entries when moving network devices across network * namespaces owned by different user namespaces. */ diff --git a/net/core/net-sysfs.h b/net/core/net-sysfs.h index 38e2e3ffd0bd..2f41a4dee866 100644 --- a/net/core/net-sysfs.h +++ b/net/core/net-sysfs.h @@ -4,6 +4,7 @@ int __init netdev_kobject_init(void); int netdev_register_kobject(struct net_device *); +void netdev_uevent_add(struct net_device *dev); void netdev_unregister_kobject(struct net_device *); int net_rx_queue_update_kobjects(struct net_device *, int old_num, int new_num); int netdev_queue_update_kobjects(struct net_device *net, diff --git a/net/core/netdev_queues.c b/net/core/netdev_queues.c index 4d6864bb4f6b..f5558b12877c 100644 --- a/net/core/netdev_queues.c +++ b/net/core/netdev_queues.c @@ -40,7 +40,7 @@ struct device *netdev_queue_get_dma_dev(struct net_device *dev, struct netdev_rx_queue *hw_rxq; struct device *dma_dev; - netdev_assert_locked_ops_compat(dev); + netdev_assert_locked(dev); /* Only RX side supports queue leasing today. */ if (type != NETDEV_QUEUE_TYPE_RX || !netif_rxq_is_leased(dev, idx)) diff --git a/net/core/netdev_work.c b/net/core/netdev_work.c index 3109fae132ad..e721a06d58df 100644 --- a/net/core/netdev_work.c +++ b/net/core/netdev_work.c @@ -31,6 +31,10 @@ static void netdev_work_enqueue(struct net_device *dev, unsigned long events, return; spin_lock_bh(&netdev_work_lock); + if (!dev_isalive(dev)) { + spin_unlock_bh(&netdev_work_lock); + return; + } if (list_empty(&dev->work_node)) { list_add_tail(&dev->work_node, &netdev_work_list); netdev_hold(dev, &dev->work_tracker, GFP_ATOMIC); @@ -61,6 +65,18 @@ netdev_work_dequeue(struct net_device *dev, unsigned long *pending, return events; } +void netdev_work_cancel_all(struct net_device *dev) +{ + spin_lock_bh(&netdev_work_lock); + dev->work_pending = 0; + dev->work_core_pending = 0; + if (!list_empty(&dev->work_node)) { + list_del_init(&dev->work_node); + netdev_put(dev, &dev->work_tracker); + } + spin_unlock_bh(&netdev_work_lock); +} + void netdev_work_sched(struct net_device *dev, unsigned long events) { netdev_work_enqueue(dev, events, 0); diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 8e185b318288..ee64f3012321 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -3972,6 +3972,7 @@ static void _rem_dev_from_if_list(struct pktgen_thread *t, struct pktgen_dev *p; if_lock(t); + proc_remove(pkt_dev->entry); list_for_each_safe(q, n, &t->if_list) { p = list_entry(q, struct pktgen_dev, list); if (p == pkt_dev) @@ -4001,9 +4002,6 @@ static int pktgen_remove_device(struct pktgen_thread *t, * list to determine if interface already exist, avoid race * with proc_create_data() */ - proc_remove(pkt_dev->entry); - - /* And update the thread if_list */ _rem_dev_from_if_list(t, pkt_dev); #ifdef CONFIG_XFRM diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 18dabb4e9cfa..ba3dbac80fb4 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -927,6 +927,18 @@ static void skb_clone_fraglist(struct sk_buff *skb) skb_get(list); } +/** + * skb_pp_cow_data() - copy skb data into page-pool backed storage + * @pool: page pool to allocate from + * @pskb: pointer to skb pointer, replaced with the copied skb on success + * @headroom: headroom to reserve in the copied skb + * + * skb_copy_bits() handles both frags[] and frag_list input. If the copied + * skb remains non-linear, it uses frags[], which is the representation used + * by XDP multi-buffer. + * + * Return: 0 on success or a negative errno on failure. + */ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, unsigned int headroom) { @@ -936,12 +948,6 @@ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, int err, i, head_off; void *data; - /* XDP does not support fraglist so we need to linearize - * the skb. - */ - if (skb_has_frag_list(skb)) - return -EOPNOTSUPP; - max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom); if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE) return -ENOMEM; diff --git a/net/core/sock.c b/net/core/sock.c index 8a59bfaa8096..1ad41904db25 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -779,7 +779,6 @@ bool sk_mc_loop(const struct sock *sk) return inet6_test_bit(MC6_LOOP, sk); #endif } - WARN_ON_ONCE(1); return true; } EXPORT_SYMBOL(sk_mc_loop); @@ -2492,6 +2491,9 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority, sock_copy(newsk, sk); newsk->sk_prot_creator = prot; +#ifdef CONFIG_BPF_SYSCALL + RCU_INIT_POINTER(newsk->sk_bpf_storage, NULL); +#endif /* SANITY */ if (likely(newsk->sk_net_refcnt)) { @@ -2544,6 +2546,11 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority, cgroup_sk_clone(&newsk->sk_cgrp_data); + RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); + + if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) + net_enable_timestamp(); + rcu_read_lock(); filter = rcu_dereference(sk->sk_filter); if (filter != NULL) @@ -2566,8 +2573,6 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority, goto free; } - RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); - if (bpf_sk_storage_clone(sk, newsk)) goto free; @@ -2595,9 +2600,6 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority, if (newsk->sk_prot->sockets_allocated) sk_sockets_allocated_inc(newsk); - - if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) - net_enable_timestamp(); out: return newsk; free: diff --git a/net/core/sock_map.c b/net/core/sock_map.c index c60ba6d292f9..9efbd8ca7db8 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -542,6 +542,8 @@ static bool sock_map_sk_state_allowed(const struct sock *sk) { if (sk_is_tcp(sk)) return (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_LISTEN); + if (sk_is_udp(sk)) + return sk_hashed(sk); if (sk_is_stream_unix(sk)) return (1 << READ_ONCE(sk->sk_state)) & TCPF_ESTABLISHED; if (sk_is_vsock(sk) && diff --git a/net/core/xdp.c b/net/core/xdp.c index 9890a30584ba..0194e69da339 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -871,7 +871,7 @@ struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf) headroom = xdpf->headroom + sizeof(*xdpf); totalsize = headroom + xdpf->len; - if (unlikely(totalsize > PAGE_SIZE)) + if (unlikely(totalsize > SKB_WITH_OVERHEAD(PAGE_SIZE))) return NULL; page = dev_alloc_page(); if (!page) diff --git a/net/devlink/dev.c b/net/devlink/dev.c index 57b2b8f03543..fd5633fa88ec 100644 --- a/net/devlink/dev.c +++ b/net/devlink/dev.c @@ -578,6 +578,7 @@ int devlink_nl_reload_doit(struct sk_buff *skb, struct genl_info *info) action != DEVLINK_RELOAD_ACTION_DRIVER_REINIT) { NL_SET_ERR_MSG_MOD(info->extack, "Changing namespace is only supported for reinit action"); + put_net(dest_net); return -EOPNOTSUPP; } } diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index 4ca2eca2e94b..3e969a070f9f 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -115,6 +115,34 @@ static inline void ethnl_update_u8(u8 *dst, const struct nlattr *attr, } /** + * ethnl_update_u8_u32() - update u8 value from an NLA_U32 attribute + * @dst: value to update + * @attr: netlink attribute with new value or null + * @mod: pointer to bool for modification tracking + * + * Some attributes are NLA_U32 on the wire but are stored in a u8. Read the + * full 32-bit value from NLA_U32 netlink attribute @attr and narrow it into + * the u8 pointed to by @dst; do nothing if @attr is null. + * Bool pointed to by @mod is set to true if this function changed the value + * of *dst, otherwise it is left as is. + */ +static inline void ethnl_update_u8_u32(u8 *dst, const struct nlattr *attr, + bool *mod) +{ + u32 val; + + if (!attr) + return; + val = nla_get_u32(attr); + DEBUG_NET_WARN_ON_ONCE(val > U8_MAX); + if (*dst == val) + return; + + *dst = val; + *mod = true; +} + +/** * ethnl_update_bool32() - update u32 used as bool from NLA_U8 attribute * @dst: value to update * @attr: netlink attribute with new value or null diff --git a/net/ethtool/rss.c b/net/ethtool/rss.c index d8adc78e3775..d4a1a4724b67 100644 --- a/net/ethtool/rss.c +++ b/net/ethtool/rss.c @@ -570,7 +570,7 @@ static const struct nla_policy ethnl_rss_flows_policy[] = { const struct nla_policy ethnl_rss_set_policy[ETHTOOL_A_RSS_FLOW_HASH + 1] = { [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), [ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32, }, - [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_MIN(NLA_U32, 1), + [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_RANGE(NLA_U32, 1, U8_MAX), [ETHTOOL_A_RSS_INDIR] = { .type = NLA_BINARY, }, [ETHTOOL_A_RSS_HKEY] = NLA_POLICY_MIN(NLA_BINARY, 1), [ETHTOOL_A_RSS_INPUT_XFRM] = @@ -851,7 +851,7 @@ ethnl_rss_set(struct ethnl_req_info *req_info, struct genl_info *info) indir_mod = !!tb[ETHTOOL_A_RSS_INDIR]; rxfh.hfunc = data.hfunc; - ethnl_update_u8(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod); + ethnl_update_u8_u32(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod); if (rxfh.hfunc == data.hfunc) rxfh.hfunc = ETH_RSS_HASH_NO_CHANGE; @@ -860,7 +860,8 @@ ethnl_rss_set(struct ethnl_req_info *req_info, struct genl_info *info) goto exit_free_indir; rxfh.input_xfrm = data.input_xfrm; - ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod); + ethnl_update_u8_u32(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], + &mod); xfrm_sym = rxfh.input_xfrm || data.input_xfrm; if (rxfh.input_xfrm == data.input_xfrm) rxfh.input_xfrm = RXH_XFRM_NO_CHANGE; @@ -934,7 +935,7 @@ const struct ethnl_request_ops ethnl_rss_request_ops = { const struct nla_policy ethnl_rss_create_policy[ETHTOOL_A_RSS_INPUT_XFRM + 1] = { [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), [ETHTOOL_A_RSS_CONTEXT] = NLA_POLICY_MIN(NLA_U32, 1), - [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_MIN(NLA_U32, 1), + [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_RANGE(NLA_U32, 1, U8_MAX), [ETHTOOL_A_RSS_INDIR] = NLA_POLICY_MIN(NLA_BINARY, 1), [ETHTOOL_A_RSS_HKEY] = NLA_POLICY_MIN(NLA_BINARY, 1), [ETHTOOL_A_RSS_INPUT_XFRM] = @@ -1048,14 +1049,15 @@ int ethnl_rss_create_doit(struct sk_buff *skb, struct genl_info *info) goto exit_clean_data; indir_user_size = ret; - ethnl_update_u8(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod); + ethnl_update_u8_u32(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod); ret = rss_set_prep_hkey(dev, info, &data, &rxfh, &mod); if (ret) goto exit_free_indir; rxfh.input_xfrm = RXH_XFRM_NO_CHANGE; - ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod); + ethnl_update_u8_u32(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], + &mod); ctx = ethtool_rxfh_ctx_alloc(ops, data.indir_size, data.hkey_size); if (!ctx) { diff --git a/net/hsr/hsr_slave.c b/net/hsr/hsr_slave.c index d9af9e65f72f..01c73b4b50dd 100644 --- a/net/hsr/hsr_slave.c +++ b/net/hsr/hsr_slave.c @@ -242,6 +242,8 @@ void hsr_del_port(struct hsr_port *port) netdev_rx_handler_unregister(port->dev); if (!port->hsr->fwd_offloaded) dev_set_promiscuity(port->dev, -1); + if (port->type == HSR_PT_SLAVE_A || port->type == HSR_PT_SLAVE_B) + vlan_vids_del_by_dev(port->dev, master->dev); netdev_upper_dev_unlink(port->dev, master->dev); if (hsr->prot_version == PRP_V1 && port->type == HSR_PT_SLAVE_B) { diff --git a/net/ipv4/datagram.c b/net/ipv4/datagram.c index 1614593b6d72..7d25519a6cdd 100644 --- a/net/ipv4/datagram.c +++ b/net/ipv4/datagram.c @@ -63,12 +63,12 @@ int __ip4_datagram_connect(struct sock *sk, struct sockaddr_unsized *uaddr, int } /* Update addresses before rehashing */ - inet->inet_daddr = fl4->daddr; + WRITE_ONCE(inet->inet_daddr, fl4->daddr); inet->inet_dport = usin->sin_port; if (!inet->inet_saddr) inet->inet_saddr = fl4->saddr; if (!inet->inet_rcv_saddr) { - inet->inet_rcv_saddr = fl4->saddr; + WRITE_ONCE(inet->inet_rcv_saddr, fl4->saddr); if (sk->sk_prot->rehash) sk->sk_prot->rehash(sk); } diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 4f3c0740dde9..0483519b7fb0 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -490,6 +490,34 @@ int ip_fib_check_default(__be32 gw, struct net_device *dev) return -1; } +static size_t fib_nexthop_nlmsg_size(const struct fib_nh_common *nhc, + bool skip_oif) +{ + size_t nhsize = 0; + + switch (nhc->nhc_gw_family) { + case AF_INET: + nhsize += nla_total_size(4); /* RTA_GATEWAY */ + break; + case AF_INET6: + nhsize += nla_total_size(sizeof(struct rtvia) + + sizeof(struct in6_addr)); + break; + } + + if (!skip_oif && nhc->nhc_dev) + nhsize += nla_total_size(4); /* RTA_OIF */ + + if (nhc->nhc_lwtstate) { + /* RTA_ENCAP */ + nhsize += lwtunnel_get_encap_size(nhc->nhc_lwtstate); + /* RTA_ENCAP_TYPE */ + nhsize += nla_total_size(2); + } + + return nhsize; +} + size_t fib_nlmsg_size(struct fib_info *fi) { size_t payload = NLMSG_ALIGN(sizeof(struct rtmsg)) @@ -507,32 +535,35 @@ size_t fib_nlmsg_size(struct fib_info *fi) payload += nla_total_size(4); /* RTA_NH_ID */ if (nhs) { - size_t nh_encapsize = 0; - /* Also handles the special case nhs == 1 */ - - /* each nexthop is packed in an attribute */ - size_t nhsize = nla_total_size(sizeof(struct rtnexthop)); + size_t mpsize = 0; unsigned int i; - /* may contain flow and gateway attribute */ - nhsize += 2 * nla_total_size(4); - - /* grab encap info */ for (i = 0; i < fib_info_num_path(fi); i++) { struct fib_nh_common *nhc = fib_info_nhc(fi, i); + size_t nhsize; + + nhsize = fib_nexthop_nlmsg_size(nhc, nhs != 1); + + if (nhs != 1) + nhsize += NLA_ALIGN(sizeof(struct rtnexthop)); + +#ifdef CONFIG_IP_ROUTE_CLASSID + if (nhc->nhc_family == AF_INET) { + struct fib_nh *nh; - if (nhc->nhc_lwtstate) { - /* RTA_ENCAP_TYPE */ - nh_encapsize += lwtunnel_get_encap_size( - nhc->nhc_lwtstate); - /* RTA_ENCAP */ - nh_encapsize += nla_total_size(2); + nh = container_of(nhc, struct fib_nh, nh_common); + if (nh->nh_tclassid) + nhsize += nla_total_size(4); } +#endif + if (nhs == 1) + payload += nhsize; + else + mpsize += nhsize; } - /* all nexthops are packed in a nested attribute */ - payload += nla_total_size((nhs * nhsize) + nh_encapsize); - + if (nhs != 1) + payload += nla_total_size(mpsize); } return payload; @@ -1864,42 +1895,30 @@ static int call_fib_nh_notifiers(struct fib_nh *nh, return NOTIFY_DONE; } -/* Update the PMTU of exceptions when: - * - the new MTU of the first hop becomes smaller than the PMTU - * - the old MTU was the same as the PMTU, and it limited discovery of - * larger MTUs on the path. With that limit raised, we can now - * discover larger MTUs - * A special case is locked exceptions, for which the PMTU is smaller - * than the minimal accepted PMTU: - * - if the new MTU is greater than the PMTU, don't make any change - * - otherwise, unlock and set PMTU +/* Walk the exceptions of a nexthop after its first hop MTU changed. The + * chain is RCU protected here, while fnhe_update_pmtu() takes fnhe_lock + * for the update of each entry. */ void fib_nhc_update_mtu(struct fib_nh_common *nhc, u32 new, u32 orig) { struct fnhe_hash_bucket *bucket; int i; - bucket = rcu_dereference_protected(nhc->nhc_exceptions, 1); + rcu_read_lock(); + bucket = rcu_dereference(nhc->nhc_exceptions); if (!bucket) - return; + goto out; for (i = 0; i < FNHE_HASH_SIZE; i++) { struct fib_nh_exception *fnhe; - for (fnhe = rcu_dereference_protected(bucket[i].chain, 1); + for (fnhe = rcu_dereference(bucket[i].chain); fnhe; - fnhe = rcu_dereference_protected(fnhe->fnhe_next, 1)) { - if (fnhe->fnhe_mtu_locked) { - if (new <= fnhe->fnhe_pmtu) { - fnhe->fnhe_pmtu = new; - fnhe->fnhe_mtu_locked = false; - } - } else if (new < fnhe->fnhe_pmtu || - orig == fnhe->fnhe_pmtu) { - fnhe->fnhe_pmtu = new; - } - } + fnhe = rcu_dereference(fnhe->fnhe_next)) + fnhe_update_pmtu(fnhe, new, orig); } +out: + rcu_read_unlock(); } void fib_sync_mtu(struct net_device *dev, u32 orig_mtu) diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index e11dc86ceda0..6badad29593b 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c @@ -1385,7 +1385,7 @@ succeeded: out_remove_new_fa: fib_remove_alias(t, tp, l, new_fa); out_free_new_fa: - kmem_cache_free(fn_alias_kmem, new_fa); + alias_free_mem_rcu(new_fa); out: fib_release_info(fi); err: diff --git a/net/ipv4/fou_core.c b/net/ipv4/fou_core.c index 865bd7205122..ab09dfcdecbd 100644 --- a/net/ipv4/fou_core.c +++ b/net/ipv4/fou_core.c @@ -629,9 +629,9 @@ static int fou_create(struct net *net, struct fou_cfg *cfg, return 0; error: - kfree(fou); if (sock) udp_tunnel_sock_release(sock->sk); + kfree_rcu(fou, rcu); return err; } diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 23e921d313b3..0caedfc7ca92 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -548,11 +548,23 @@ static struct rtable *icmp_route_lookup(struct net *net, struct flowi4 *fl4, if (IS_ERR(rt2)) err = PTR_ERR(rt2); } else { - struct flowi4 fl4_2 = {}; + struct flowi4 fl4_2 = fl4_dec; unsigned long orefdst; - fl4_2.daddr = fl4_dec.saddr; - rt2 = ip_route_output_key(net, &fl4_2); + swap(fl4_2.daddr, fl4_2.saddr); + switch (fl4_2.flowi4_proto) { + case IPPROTO_TCP: + case IPPROTO_UDP: + case IPPROTO_SCTP: + case IPPROTO_DCCP: + swap(fl4_2.fl4_sport, fl4_2.fl4_dport); + break; + } + + fl4_2.flowi4_oif = l3mdev_master_ifindex(route_lookup_dev); + fl4_2.flowi4_flags |= FLOWI_FLAG_ANYSRC; + + rt2 = __ip_route_output_key(net, &fl4_2); if (IS_ERR(rt2)) { err = PTR_ERR(rt2); goto relookup_failed; diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index b6337a47c141..bb2d4441a492 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -217,13 +217,18 @@ static void ip_sf_list_clear_all(struct ip_sf_list *psf) static void igmp_stop_timer(struct ip_mc_list *im) { + bool put = false; + spin_lock_bh(&im->lock); if (timer_delete(&im->timer)) - refcount_dec(&im->refcnt); + put = true; WRITE_ONCE(im->tm_running, 0); WRITE_ONCE(im->reporter, 0); im->unsolicit_count = 0; spin_unlock_bh(&im->lock); + + if (put) + ip_ma_put(im); } /* It must be called with locked im->lock */ @@ -248,20 +253,26 @@ static void igmp_gq_start_timer(struct in_device *in_dev) return; in_dev->mr_gq_running = 1; - if (!mod_timer(&in_dev->mr_gq_timer, exp)) - in_dev_hold(in_dev); + if (in_dev_hold_safe(in_dev)) { + if (mod_timer(&in_dev->mr_gq_timer, exp)) + in_dev_put(in_dev); + } } static void igmp_ifc_start_timer(struct in_device *in_dev, int delay) { - int tv = get_random_u32_below(delay); + if (in_dev_hold_safe(in_dev)) { + int tv = get_random_u32_below(delay); - if (!mod_timer(&in_dev->mr_ifc_timer, jiffies+tv+2)) - in_dev_hold(in_dev); + if (mod_timer(&in_dev->mr_ifc_timer, jiffies + tv + 2)) + in_dev_put(in_dev); + } } static void igmp_mod_timer(struct ip_mc_list *im, int max_delay) { + bool put = false; + spin_lock_bh(&im->lock); im->unsolicit_count = 0; if (timer_delete(&im->timer)) { @@ -271,10 +282,13 @@ static void igmp_mod_timer(struct ip_mc_list *im, int max_delay) spin_unlock_bh(&im->lock); return; } - refcount_dec(&im->refcnt); + put = true; } igmp_start_timer(im, max_delay); spin_unlock_bh(&im->lock); + + if (put) + ip_ma_put(im); } @@ -1922,6 +1936,7 @@ void ip_mc_destroy_dev(struct in_device *in_dev) #endif while ((i = rtnl_dereference(in_dev->mc_list)) != NULL) { + ip_mc_hash_remove(in_dev, i); in_dev->mc_list = i->next_rcu; WRITE_ONCE(in_dev->mc_count, in_dev->mc_count - 1); ip_mc_clear_src(i); diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 56902bba5483..6257459bcee2 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -943,11 +943,23 @@ static struct request_sock *inet_reqsk_clone(struct request_sock *req, nreq->rsk_listener = sk; - /* We need not acquire fastopenq->lock - * because the child socket is locked in inet_csk_listen_stop(). - */ - if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(nreq)->tfo_listener) + if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(nreq)->tfo_listener) { + struct fastopen_queue *fastopenq; + + /* reqsk_fastopen_remove() will uncharge nreq->rsk_listener, + * that is @sk, so charge it here. Unlike the listener + * being closed, @sk is live and needs its lock. + */ + fastopenq = &inet_csk(sk)->icsk_accept_queue.fastopenq; + spin_lock_bh(&fastopenq->lock); + fastopenq->qlen++; + spin_unlock_bh(&fastopenq->lock); + + /* We need not acquire fastopenq->lock + * because the child socket is locked in inet_csk_listen_stop(). + */ rcu_assign_pointer(tcp_sk(nreq->sk)->fastopen_rsk, nreq); + } return nreq; } diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c index 848c0f0c2ed9..fc0cb993959f 100644 --- a/net/ipv4/inet_fragment.c +++ b/net/ipv4/inet_fragment.c @@ -393,8 +393,8 @@ static struct inet_frag_queue *inet_frag_create(struct fqdir *fqdir, *prev = ERR_PTR(-ENOMEM); return NULL; } - mod_timer(&q->timer, jiffies + fqdir->timeout); + spin_lock_bh(&q->lock); *prev = rhashtable_lookup_get_insert_key(&fqdir->rhashtable, &q->key, &q->node, f->rhash_params); if (*prev) { @@ -402,13 +402,13 @@ static struct inet_frag_queue *inet_frag_create(struct fqdir *fqdir, * we need to cancel what inet_frag_alloc() * anticipated. */ - int refs = 1; - q->flags |= INET_FRAG_COMPLETE; - inet_frag_kill(q, &refs); - inet_frag_putn(q, refs); + spin_unlock_bh(&q->lock); + inet_frag_putn(q, 2); return NULL; } + mod_timer(&q->timer, jiffies + fqdir->timeout); + spin_unlock_bh(&q->lock); return q; } diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 3efdfb4ffa21..0ba1e94e9012 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -1018,6 +1018,8 @@ static void __gre_tunnel_init(struct net_device *dev) dev->features |= GRE_FEATURES; dev->hw_features |= GRE_FEATURES; + dev->lltx = true; + /* TCP offload with GRE SEQ is not supported, nor can we support 2 * levels of outer headers requiring an update. */ @@ -1029,8 +1031,6 @@ static void __gre_tunnel_init(struct net_device *dev) dev->features |= NETIF_F_GSO_SOFTWARE; dev->hw_features |= NETIF_F_GSO_SOFTWARE; - - dev->lltx = true; } static int ipgre_tunnel_init(struct net_device *dev) diff --git a/net/ipv4/netfilter/nf_nat_h323.c b/net/ipv4/netfilter/nf_nat_h323.c index 183e8a3ff2ba..6bcd6734769b 100644 --- a/net/ipv4/netfilter/nf_nat_h323.c +++ b/net/ipv4/netfilter/nf_nat_h323.c @@ -182,6 +182,7 @@ static int nat_rtp_rtcp(struct sk_buff *skb, struct nf_conn *ct, struct nf_conntrack_expect *rtp_exp, struct nf_conntrack_expect *rtcp_exp) { + struct nf_conntrack_expect *rtp_pair[2] = { rtp_exp, rtcp_exp }; struct nf_ct_h323_master *info = nfct_help_data(ct); int dir = CTINFO2DIR(ctinfo); int i; @@ -227,22 +228,13 @@ static int nat_rtp_rtcp(struct sk_buff *skb, struct nf_conn *ct, int ret; rtp_exp->tuple.dst.u.udp.port = htons(nated_port); - ret = nf_ct_expect_related(rtp_exp, 0); + rtcp_exp->tuple.dst.u.udp.port = htons(nated_port + 1); + ret = nf_ct_expect_related_pair(rtp_pair, 0); if (ret == 0) { - rtcp_exp->tuple.dst.u.udp.port = - htons(nated_port + 1); - ret = nf_ct_expect_related(rtcp_exp, 0); - if (ret == 0) - break; - else if (ret == -EBUSY) { - nf_ct_unexpect_related(rtp_exp); - continue; - } else if (ret < 0) { - nf_ct_unexpect_related(rtp_exp); - nated_port = 0; - break; - } - } else if (ret != -EBUSY) { + break; + } else if (ret == -EBUSY) { + continue; + } else if (ret < 0) { nated_port = 0; break; } diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 6205bd57aa85..0f1e21a5c812 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -1597,14 +1597,21 @@ static int fib6_check_nh_list(struct nexthop *old, struct nexthop *new, struct netlink_ext_ack *extack) { struct fib6_info *f6i; + int err = 0; if (list_empty(&old->f6i_list)) return 0; + spin_lock_bh(&old->lock); list_for_each_entry(f6i, &old->f6i_list, nh_list) { - if (check_src_addr(&f6i->fib6_src.addr, extack) < 0) - return -EINVAL; + err = check_src_addr(&f6i->fib6_src.addr, extack); + if (err) + break; } + spin_unlock_bh(&old->lock); + + if (err) + return err; return fib6_check_nexthop(new, NULL, extack); } @@ -1788,8 +1795,8 @@ static bool nh_res_bucket_migrate(struct nh_res_table *res_table, bool notify_nl, bool force) { struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index]; + struct netlink_ext_ack extack = {}; struct nh_grp_entry *new_nhge; - struct netlink_ext_ack extack; int err; new_nhge = list_first_entry_or_null(&res_table->uw_nh_entries, @@ -2233,18 +2240,18 @@ static void remove_one_nexthop(struct net *net, struct nexthop *nh, static void nh_rt_cache_flush(struct net *net, struct nexthop *nh, struct nexthop *replaced_nh) { - struct fib6_info *f6i; struct nh_group *nhg; + bool have_f6i; int i; if (!list_empty(&nh->fi_list)) rt_cache_flush(net); - list_for_each_entry(f6i, &nh->f6i_list, nh_list) { - spin_lock_bh(&f6i->fib6_table->tb6_lock); - fib6_update_sernum_upto_root(net, f6i); - spin_unlock_bh(&f6i->fib6_table->tb6_lock); - } + spin_lock_bh(&nh->lock); + have_f6i = !list_empty(&nh->f6i_list); + spin_unlock_bh(&nh->lock); + if (have_f6i) + rt_genid_bump_ipv6(net); /* if an IPv6 group was replaced, we have to release all old * dsts to make sure all refcounts are released @@ -2538,8 +2545,10 @@ static void __nexthop_replace_notify(struct net *net, struct nexthop *nh, fi->nh_updated = false; } + spin_lock_bh(&nh->lock); list_for_each_entry(f6i, &nh->f6i_list, nh_list) fib6_rt_update(net, f6i, info); + spin_unlock_bh(&nh->lock); } /* send RTM_NEWROUTE with REPLACE flag set for all FIB entries diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index e9fbab6ad914..7f74d8b95a37 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -118,13 +118,21 @@ bool raw_v4_match(struct net *net, const struct sock *sk, unsigned short num, __be32 raddr, __be32 laddr, int dif, int sdif) { const struct inet_sock *inet = inet_sk(sk); + __be32 daddr, rcv_saddr; - if (net_eq(sock_net(sk), net) && inet->inet_num == num && - !(inet->inet_daddr && inet->inet_daddr != raddr) && - !(inet->inet_rcv_saddr && inet->inet_rcv_saddr != laddr) && - raw_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) - return true; - return false; + if (!net_eq(sock_net(sk), net) || inet->inet_num != num) + return false; + + daddr = READ_ONCE(inet->inet_daddr); + if (daddr && daddr != raddr) + return false; + + rcv_saddr = READ_ONCE(inet->inet_rcv_saddr); + if (rcv_saddr && rcv_saddr != laddr) + return false; + + return raw_sk_bound_dev_eq(net, READ_ONCE(sk->sk_bound_dev_if), + dif, sdif); } EXPORT_SYMBOL_GPL(raw_v4_match); @@ -722,7 +730,8 @@ static int raw_bind(struct sock *sk, struct sockaddr_unsized *uaddr, chk_addr_ret)) goto out; - inet->inet_rcv_saddr = inet->inet_saddr = addr->sin_addr.s_addr; + inet->inet_saddr = addr->sin_addr.s_addr; + WRITE_ONCE(inet->inet_rcv_saddr, addr->sin_addr.s_addr); if (chk_addr_ret == RTN_MULTICAST || chk_addr_ret == RTN_BROADCAST) inet->inet_saddr = 0; /* Use device */ sk_dst_reset(sk); diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 3f3de5164d6e..b82401a6baed 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -741,6 +741,35 @@ out_unlock: spin_unlock_bh(&fnhe_lock); } +/* Update the PMTU of an exception when: + * - the new MTU of the first hop becomes smaller than the PMTU + * - the old MTU was the same as the PMTU, and it limited discovery of + * larger MTUs on the path. With that limit raised, we can now + * discover larger MTUs + * A special case is locked exceptions, for which the PMTU is smaller + * than the minimal accepted PMTU: + * - if the new MTU is greater than the PMTU, don't make any change + * - otherwise, unlock and set PMTU + * + * fnhe_lock keeps fnhe_pmtu and fnhe_mtu_locked consistent against + * update_or_create_fnhe(), which sets both under the same lock. + */ +void fnhe_update_pmtu(struct fib_nh_exception *fnhe, u32 new, u32 orig) +{ + spin_lock_bh(&fnhe_lock); + + if (fnhe->fnhe_mtu_locked) { + if (new <= fnhe->fnhe_pmtu) { + fnhe->fnhe_pmtu = new; + fnhe->fnhe_mtu_locked = false; + } + } else if (new < fnhe->fnhe_pmtu || orig == fnhe->fnhe_pmtu) { + fnhe->fnhe_pmtu = new; + } + + spin_unlock_bh(&fnhe_lock); +} + static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4, bool kill_route) { @@ -892,8 +921,6 @@ void ip_rt_send_redirect(struct sk_buff *skb) peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif); if (!peer) { rcu_read_unlock(); - icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, - rt_nexthop(rt, ip_hdr(skb)->daddr)); return; } diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index cc0bd73f36b6..2e234d155b5e 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -334,6 +334,7 @@ unlock: static int tcp_bpf_ioctl(struct sock *sk, int cmd, int *karg) { + struct sk_psock *psock; bool slow; if (cmd != SIOCINQ) @@ -344,7 +345,21 @@ static int tcp_bpf_ioctl(struct sock *sk, int cmd, int *karg) return -EINVAL; slow = lock_sock_fast(sk); - *karg = sk_psock_msg_inq(sk); + psock = sk_psock_get(sk); + if (unlikely(!psock)) { + unlock_sock_fast(sk, slow); + return tcp_ioctl(sk, cmd, karg); + } + *karg = sk_psock_get_msg_len_nolock(psock); + /* Without a verdict program, ingress data is never diverted to + * ingress_msg: it stays in sk_receive_queue and is read through + * the fallback to tcp_recvmsg(), so account for it like + * tcp_ioctl() does. + */ + if (!READ_ONCE(psock->progs.stream_verdict) && + !READ_ONCE(psock->progs.skb_verdict)) + *karg += tcp_inq(sk); + sk_psock_put(sk, psock); unlock_sock_fast(sk, slow); return 0; @@ -454,6 +469,7 @@ more_data: case __SK_REDIRECT: redir_ingress = psock->redir_ingress; sk_redir = psock->sk_redir; + sock_hold(sk_redir); sk_msg_apply_bytes(psock, tosend); if (!psock->apply_bytes) { /* Clean up before releasing the sock lock. */ @@ -474,6 +490,7 @@ more_data: if (eval == __SK_REDIRECT) sock_put(sk_redir); + sock_put(sk_redir); lock_sock(sk); sk_mem_uncharge(sk, sent); @@ -589,7 +606,7 @@ wait_for_sndbuf: wait_for_memory: err = sk_stream_wait_memory(sk, &timeo); if (err) { - if (msg_tx && msg_tx != psock->cork) + if (msg_tx == &tmp) sk_msg_free(sk, msg_tx); goto out_err; } diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 61045a8886e4..0f60a1dbf927 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -252,7 +252,7 @@ static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb) struct tcp_sock *tp = tcp_sk(sk); val = tcp_win_from_space(sk, sk->sk_rcvbuf); - tcp_set_window_clamp(sk, val); + WRITE_ONCE(tp->window_clamp, val); if (tp->window_clamp < tp->rcvq_space.space) tp->rcvq_space.space = tp->window_clamp; @@ -1039,9 +1039,9 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) /* The fastest case is the first. */ icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2; } else if (m < icsk->icsk_ack.ato) { - icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m; - if (icsk->icsk_ack.ato > icsk->icsk_rto) - icsk->icsk_ack.ato = icsk->icsk_rto; + icsk->icsk_ack.ato = min3((icsk->icsk_ack.ato >> 1) + (u32)m, + icsk->icsk_rto, + (u32)TCP_DELACK_MAX); } else if (m > icsk->icsk_rto) { /* Too long gap. Apparently sender failed to * restart window, so that we send ACKs quickly. @@ -4038,24 +4038,17 @@ static void tcp_send_ack_reflect_ect(struct sock *sk, bool accecn_reflector) __tcp_send_ack(sk, tp->rcv_nxt, flags); } -/* RFC 5961 7 [ACK Throttling] */ -static void tcp_send_challenge_ack(struct sock *sk, bool accecn_reflector) +/* Consume one slot from the per-netns RFC 5961 challenge ACK quota. + * Returns true if a challenge ACK may be sent. + */ +static bool tcp_challenge_ack_allowed(struct net *net) { - struct tcp_sock *tp = tcp_sk(sk); - struct net *net = sock_net(sk); u32 count, now, ack_limit; - /* First check our per-socket dupack rate limit. */ - if (__tcp_oow_rate_limited(net, - LINUX_MIB_TCPACKSKIPPEDCHALLENGE, - &tp->last_oow_ack_time)) - return; - ack_limit = READ_ONCE(net->ipv4.sysctl_tcp_challenge_ack_limit); if (ack_limit == INT_MAX) - goto send_ack; + return true; - /* Then check host-wide RFC 5961 rate limit. */ now = jiffies / HZ; if (now != READ_ONCE(net->ipv4.tcp_challenge_timestamp)) { u32 half = (ack_limit + 1) >> 1; @@ -4067,12 +4060,49 @@ static void tcp_send_challenge_ack(struct sock *sk, bool accecn_reflector) count = READ_ONCE(net->ipv4.tcp_challenge_count); if (count > 0) { WRITE_ONCE(net->ipv4.tcp_challenge_count, count - 1); -send_ack: + return true; + } + return false; +} + +/* RFC 5961 7 [ACK Throttling] */ +static void tcp_send_challenge_ack(struct sock *sk, bool accecn_reflector) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct net *net = sock_net(sk); + + /* First check our per-socket dupack rate limit. */ + if (__tcp_oow_rate_limited(net, + LINUX_MIB_TCPACKSKIPPEDCHALLENGE, + &tp->last_oow_ack_time)) + return; + + /* Then check the per-netns RFC 5961 rate limit. */ + if (tcp_challenge_ack_allowed(net)) { NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK); tcp_send_ack_reflect_ect(sk, accecn_reflector); } } +/* Send a challenge ACK from a SYN-RECEIVED request socket. Uses + * __tcp_oow_rate_limited() directly so that an RST carrying payload + * cannot bypass the per-request rate limit. + */ +void tcp_reqsk_send_challenge_ack(struct sock *sk, struct sk_buff *skb, + struct request_sock *req) +{ + struct net *net = sock_net(sk); + + if (__tcp_oow_rate_limited(net, LINUX_MIB_TCPACKSKIPPEDCHALLENGE, + &tcp_rsk(req)->last_oow_ack_time)) + return; + + if (tcp_challenge_ack_allowed(net)) { + NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK); + req->rsk_ops->send_ack(sk, skb, req); + } +} + static void tcp_store_ts_recent(struct tcp_sock *tp) { tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 209ef7522508..7f413f509d7d 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -971,6 +971,9 @@ static void tcp_v4_send_ack(const struct sock *sk, key->rcv_next); arg.iov[0].iov_len += tcp_ao_len_aligned(key->ao_key); rep.th.doff = arg.iov[0].iov_len / 4; + memset((u8 *)&rep.opt[offset] + tcp_ao_maclen(key->ao_key), + TCPOPT_NOP, tcp_ao_len_aligned(key->ao_key) - + tcp_ao_len(key->ao_key)); tcp_ao_hash_hdr(AF_INET, (char *)&rep.opt[offset], key->ao_key, key->traffic_key, @@ -2318,8 +2321,10 @@ do_time_wait: } drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb); - if (drop_reason) - break; + if (drop_reason) { + inet_twsk_put(inet_twsk(sk)); + goto discard_it; + } } /* to ACK */ fallthrough; @@ -3073,24 +3078,24 @@ static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq, { struct bpf_tcp_iter_state *iter = seq->private; struct hlist_nulls_node *node; - unsigned int expected = 1; - struct sock *sk; + struct sock *sk = *start_sk; + unsigned int expected = 0; - sock_hold(*start_sk); - iter->batch[iter->end_sk++].sk = *start_sk; - - sk = sk_nulls_next(*start_sk); *start_sk = NULL; sk_nulls_for_each_from(sk, node) { - if (seq_sk_match(seq, sk)) { - if (iter->end_sk < iter->max_sk) { - sock_hold(sk); - iter->batch[iter->end_sk++].sk = sk; - } else if (!*start_sk) { - /* Remember where we left off. */ - *start_sk = sk; - } - expected++; + if (!seq_sk_match(seq, sk)) + continue; + expected++; + if (iter->end_sk < iter->max_sk) { + /* reqsk_queue_hash_req() inserts with sk_refcnt == 0 + * and refcount_set()s it after the bucket lock drops. + */ + if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) + continue; + iter->batch[iter->end_sk++].sk = sk; + } else if (!*start_sk) { + /* Remember where we left off. */ + *start_sk = sk; } } @@ -3128,12 +3133,13 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) struct sock *sk; int err; +again: sk = bpf_iter_tcp_resume(seq); if (!sk) return NULL; /* Done */ expected = bpf_iter_fill_batch(seq, &sk); - if (likely(iter->end_sk == expected)) + if (likely(!sk)) goto done; /* Batch size was too small. */ @@ -3141,15 +3147,18 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) bpf_iter_tcp_put_batch(iter); err = bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2, GFP_USER); - if (err) + if (err) { + iter->cur_sk = 0; + iter->end_sk = 0; return ERR_PTR(err); + } sk = bpf_iter_tcp_resume(seq); if (!sk) return NULL; /* Done */ expected = bpf_iter_fill_batch(seq, &sk); - if (likely(iter->end_sk == expected)) + if (likely(!sk)) goto done; /* Batch size was still too small. Hold onto the lock while we try @@ -3162,10 +3171,14 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) return ERR_PTR(err); } - expected = bpf_iter_fill_batch(seq, &sk); - WARN_ON_ONCE(iter->end_sk != expected); + bpf_iter_fill_batch(seq, &sk); + WARN_ON_ONCE(sk); done: bpf_iter_tcp_unlock_bucket(seq); + if (unlikely(!iter->end_sk)) { + ++iter->state.bucket; + goto again; + } return iter->batch[0].sk; } diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index ddc4b17a826b..6ab3e3a0b431 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -833,7 +833,7 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, * elsewhere and is checked directly against the child socket rather * than req because user data may have been sent out. */ - if ((flg & TCP_FLAG_ACK) && !fastopen && + if ((flg & TCP_FLAG_ACK) && !(flg & TCP_FLAG_RST) && !fastopen && (TCP_SKB_CB(skb)->ack_seq != tcp_rsk(req)->snt_isn + 1)) return sk; @@ -872,6 +872,16 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, flg &= ~TCP_FLAG_SYN; } + /* RFC 5961 section 3.2, as clarified by RFC 9293 section + * 3.10.7.4, requires a challenge ACK for a non-exact + * in-window RST in SYN-RECEIVED. + */ + if ((flg & TCP_FLAG_RST) && + TCP_SKB_CB(skb)->seq != tcp_rsk(req)->rcv_nxt) { + tcp_reqsk_send_challenge_ack(sk, skb, req); + return NULL; + } + /* RFC793: "second check the RST bit" and * "fourth, check the SYN bit" */ diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index bf171b5e1eb3..4df1c9745c5d 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -334,7 +334,9 @@ void tcp_delack_timer_handler(struct sock *sk) if (inet_csk_ack_scheduled(sk)) { if (!inet_csk_in_pingpong_mode(sk)) { /* Delayed ACK missed: inflate ATO. */ - icsk->icsk_ack.ato = min_t(u32, icsk->icsk_ack.ato << 1, icsk->icsk_rto); + icsk->icsk_ack.ato = min3((u32)icsk->icsk_ack.ato << 1, + icsk->icsk_rto, + (u32)TCP_DELACK_MAX); } else { /* Delayed ACK missed: leave pingpong mode and * deflate ATO. diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 29651b1a0bc7..abcd3cc2d9a3 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -178,17 +178,19 @@ static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); bool remcsum, need_csum, offload_csum, gso_partial; struct sk_buff *segs = ERR_PTR(-EINVAL); - struct udphdr *uh = udp_hdr(skb); u16 mac_offset = skb->mac_header; __be16 protocol = skb->protocol; u16 mac_len = skb->mac_len; int udp_offset, outer_hlen; + struct udphdr *uh; __wsum partial; bool need_ipsec; if (unlikely(!pskb_may_pull(skb, tnl_hlen))) goto out; + uh = udp_hdr(skb); + /* Adjust partial header checksum to negate old length. * We cannot rely on the value contained in uh->len as it is * possible that the actual value exceeds the boundaries of the diff --git a/net/ipv4/udp_tunnel_nic.c b/net/ipv4/udp_tunnel_nic.c index 3b32a0afa979..53a1a9c1f8bf 100644 --- a/net/ipv4/udp_tunnel_nic.c +++ b/net/ipv4/udp_tunnel_nic.c @@ -32,13 +32,12 @@ struct udp_tunnel_nic_table_entry { * @lock: protects all fields * @need_sync: at least one port start changed * @need_replay: space was freed, we need a replay of all ports - * @work_pending: @work is currently scheduled * @n_tables: number of tables under @entries * @missed: bitmap of tables which overflown * @entries: table of tables of ports currently offloaded */ struct udp_tunnel_nic { - struct work_struct work; + struct delayed_work work; struct net_device *dev; @@ -46,7 +45,6 @@ struct udp_tunnel_nic { u8 need_sync:1; u8 need_replay:1; - u8 work_pending:1; unsigned int n_tables; unsigned long missed; @@ -301,11 +299,10 @@ __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) static void udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) { - if (!utn->need_sync || utn->work_pending) + if (!utn->need_sync) return; - queue_work(udp_tunnel_nic_workqueue, &utn->work); - utn->work_pending = 1; + queue_delayed_work(udp_tunnel_nic_workqueue, &utn->work, 0); } static bool @@ -731,12 +728,17 @@ udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn) static void udp_tunnel_nic_device_sync_work(struct work_struct *work) { struct udp_tunnel_nic *utn = - container_of(work, struct udp_tunnel_nic, work); + container_of(work, struct udp_tunnel_nic, work.work); - rtnl_lock(); + /* We cannot block on RTNL here, otherwise we would deadlock with + * udp_tunnel_nic_unregister() calling cancel_delayed_work_sync() + * while holding RTNL. Requeue with 1 jiffy delay if RTNL is contended. + */ + if (!rtnl_trylock()) { + queue_delayed_work(udp_tunnel_nic_workqueue, &utn->work, 1); + return; + } mutex_lock(&utn->lock); - - utn->work_pending = 0; __udp_tunnel_nic_device_sync(utn->dev, utn); if (utn->need_replay) @@ -757,7 +759,7 @@ udp_tunnel_nic_alloc(const struct udp_tunnel_nic_info *info, if (!utn) return NULL; utn->n_tables = n_tables; - INIT_WORK(&utn->work, udp_tunnel_nic_device_sync_work); + INIT_DELAYED_WORK(&utn->work, udp_tunnel_nic_device_sync_work); mutex_init(&utn->lock); for (i = 0; i < n_tables; i++) { @@ -901,11 +903,11 @@ udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn) udp_tunnel_nic_flush(dev, utn); udp_tunnel_nic_unlock(dev); - /* Wait for the work to be done using the state, netdev core will - * retry unregister until we give up our reference on this device. + /* Make sure no work is running or queued before freeing @utn. + * The work handler uses rtnl_trylock(), so it will not deadlock + * against the RTNL we are holding here. */ - if (utn->work_pending) - return; + cancel_delayed_work_sync(&utn->work); udp_tunnel_nic_free(utn); release_dev: diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c index e1b2b4fa6e18..89ee3c969ca7 100644 --- a/net/ipv6/fib6_rules.c +++ b/net/ipv6/fib6_rules.c @@ -308,6 +308,7 @@ INDIRECT_CALLABLE_SCOPE bool fib6_rule_suppress(struct fib_rule *rule, suppress_route: ip6_rt_put_flags(rt, flags); + res->rt6 = NULL; return true; } diff --git a/net/ipv6/ila/ila_common.c b/net/ipv6/ila/ila_common.c index e71571455c8a..b78179bfc4c7 100644 --- a/net/ipv6/ila/ila_common.c +++ b/net/ipv6/ila/ila_common.c @@ -85,6 +85,7 @@ static void ila_csum_adjust_transport(struct sk_buff *skb, struct tcphdr *th = (struct tcphdr *) (skb_network_header(skb) + nhoff); + ip6h = ipv6_hdr(skb); diff = get_csum_diff(ip6h, p); inet_proto_csum_replace_by_diff(&th->check, skb, diff, true, true); @@ -96,6 +97,7 @@ static void ila_csum_adjust_transport(struct sk_buff *skb, (skb_network_header(skb) + nhoff); if (uh->check || skb->ip_summed == CHECKSUM_PARTIAL) { + ip6h = ipv6_hdr(skb); diff = get_csum_diff(ip6h, p); inet_proto_csum_replace_by_diff(&uh->check, skb, diff, true, true); @@ -110,6 +112,7 @@ static void ila_csum_adjust_transport(struct sk_buff *skb, struct icmp6hdr *ih = (struct icmp6hdr *) (skb_network_header(skb) + nhoff); + ip6h = ipv6_hdr(skb); diff = get_csum_diff(ip6h, p); inet_proto_csum_replace_by_diff(&ih->icmp6_cksum, skb, diff, true, true); @@ -127,6 +130,15 @@ void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p, switch (p->csum_mode) { case ILA_CSUM_ADJUST_TRANSPORT: ila_csum_adjust_transport(skb, p); + /* + * ila_csum_adjust_transport() calls pskb_may_pull(), which can + * reallocate the skb head and leave ip6h (and the iaddr derived + * from it) dangling; reload both before the write below. The + * other csum modes do not pull, so their cached pointers stay + * valid. + */ + ip6h = ipv6_hdr(skb); + iaddr = ila_a2i(&ip6h->daddr); break; case ILA_CSUM_NEUTRAL_MAP: if (sir2ila) { diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index a130cdfaebfb..e9fc692d4f3b 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -1494,6 +1494,7 @@ int fib6_add(struct fib6_node *root, struct fib6_info *rt, root, and then (in failure) stale node in main tree. */ + fib6_info_release(info->nl_net->ipv6.fib6_null_entry); node_free_immediate(info->nl_net, sfn); err = PTR_ERR(sn); goto failure; diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 7c09a269b352..b843116e9b70 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -1455,6 +1455,8 @@ static void ip6gre_tnl_init_features(struct net_device *dev) dev->features |= GRE6_FEATURES; dev->hw_features |= GRE6_FEATURES; + dev->lltx = true; + /* TCP offload with GRE SEQ is not supported, nor can we support 2 * levels of outer headers requiring an update. */ @@ -1466,8 +1468,6 @@ static void ip6gre_tnl_init_features(struct net_device *dev) dev->features |= NETIF_F_GSO_SOFTWARE; dev->hw_features |= NETIF_F_GSO_SOFTWARE; - - dev->lltx = true; } static int ip6gre_tunnel_init_common(struct net_device *dev) diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 368e4fa3b43c..2c44e5ed6171 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -641,7 +641,7 @@ int ip6_forward(struct sk_buff *skb) /* Limit redirects both by destination (here) and by source (inside ndisc_send_redirect) */ - if (inet_peer_xrlim_allow(peer, 1*HZ)) + if (peer && inet_peer_xrlim_allow(peer, 1*HZ)) ndisc_send_redirect(skb, target); rcu_read_unlock(); } else { diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index bf8e40af60b0..ebf83f090376 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -684,6 +684,9 @@ ip6ip6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, if (!skb2) return 0; + /* Remove debris left by outer IPv6 stack. */ + memset(IP6CB(skb2), 0, sizeof(*IP6CB(skb2))); + skb_dst_drop(skb2); skb_pull(skb2, offset); skb_reset_network_header(skb2); diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index 04b811b3be97..4d2b9377ba2d 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -1083,8 +1083,10 @@ static void mld_gq_start_work(struct inet6_dev *idev) mc_assert_locked(idev); idev->mc_gq_running = 1; - if (!mod_delayed_work(mld_wq, &idev->mc_gq_work, tv + 2)) - in6_dev_hold(idev); + if (in6_dev_hold_safe(idev)) { + if (mod_delayed_work(mld_wq, &idev->mc_gq_work, tv + 2)) + in6_dev_put(idev); + } } static void mld_gq_stop_work(struct inet6_dev *idev) @@ -1102,8 +1104,10 @@ static void mld_ifc_start_work(struct inet6_dev *idev, unsigned long delay) mc_assert_locked(idev); - if (!mod_delayed_work(mld_wq, &idev->mc_ifc_work, tv + 2)) - in6_dev_hold(idev); + if (in6_dev_hold_safe(idev)) { + if (mod_delayed_work(mld_wq, &idev->mc_ifc_work, tv + 2)) + in6_dev_put(idev); + } } static void mld_ifc_stop_work(struct inet6_dev *idev) @@ -1121,8 +1125,10 @@ static void mld_dad_start_work(struct inet6_dev *idev, unsigned long delay) mc_assert_locked(idev); - if (!mod_delayed_work(mld_wq, &idev->mc_dad_work, tv + 2)) - in6_dev_hold(idev); + if (in6_dev_hold_safe(idev)) { + if (mod_delayed_work(mld_wq, &idev->mc_dad_work, tv + 2)) + in6_dev_put(idev); + } } static void mld_dad_stop_work(struct inet6_dev *idev) @@ -1395,18 +1401,23 @@ static void mld_process_v2(struct inet6_dev *idev, struct mld2_query *mld, void igmp6_event_query(struct sk_buff *skb) { struct inet6_dev *idev = __in6_dev_get(skb->dev); + bool put = false; if (!idev || idev->dead) goto out; spin_lock_bh(&idev->mc_query_lock); - if (skb_queue_len(&idev->mc_query_queue) < MLD_MAX_SKBS) { + if (skb_queue_len(&idev->mc_query_queue) < MLD_MAX_SKBS && + in6_dev_hold_safe(idev)) { __skb_queue_tail(&idev->mc_query_queue, skb); - if (!mod_delayed_work(mld_wq, &idev->mc_query_work, 0)) - in6_dev_hold(idev); + if (mod_delayed_work(mld_wq, &idev->mc_query_work, 0)) + put = true; skb = NULL; } spin_unlock_bh(&idev->mc_query_lock); + + if (put) + in6_dev_put(idev); out: kfree_skb(skb); } @@ -1570,18 +1581,23 @@ static void mld_query_work(struct work_struct *work) void igmp6_event_report(struct sk_buff *skb) { struct inet6_dev *idev = __in6_dev_get(skb->dev); + bool put = false; if (!idev || idev->dead) goto out; spin_lock_bh(&idev->mc_report_lock); - if (skb_queue_len(&idev->mc_report_queue) < MLD_MAX_SKBS) { + if (skb_queue_len(&idev->mc_report_queue) < MLD_MAX_SKBS && + in6_dev_hold_safe(idev)) { __skb_queue_tail(&idev->mc_report_queue, skb); - if (!mod_delayed_work(mld_wq, &idev->mc_report_work, 0)) - in6_dev_hold(idev); + if (mod_delayed_work(mld_wq, &idev->mc_report_work, 0)) + put = true; skb = NULL; } spin_unlock_bh(&idev->mc_report_lock); + + if (put) + in6_dev_put(idev); out: kfree_skb(skb); } diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index f867ec8d3d90..fe36b3f51285 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -1707,6 +1707,8 @@ void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target) } peer = inet_getpeer_v6(net->ipv6.peers, &ipv6_hdr(skb)->saddr); + if (!peer) + goto release; ret = inet_peer_xrlim_allow(peer, 1*HZ); if (!ret) diff --git a/net/ipv6/netfilter.c b/net/ipv6/netfilter.c index 6d80f85e55fa..a7025ec87035 100644 --- a/net/ipv6/netfilter.c +++ b/net/ipv6/netfilter.c @@ -120,7 +120,7 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, ktime_t tstamp = skb->tstamp; struct ip6_frag_state state; u8 *prevhdr, nexthdr = 0; - unsigned int mtu, hlen; + unsigned int mtu, hlen, nexthdr_offset; int hroom, err = 0; __be32 frag_id; @@ -129,6 +129,7 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, goto blackhole; hlen = err; nexthdr = *prevhdr; + nexthdr_offset = prevhdr - skb_network_header(skb); mtu = skb->dev->mtu; if (frag_max_size > mtu || @@ -147,6 +148,7 @@ int br_ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, (err = skb_checksum_help(skb))) goto blackhole; + prevhdr = skb_network_header(skb) + nexthdr_offset; hroom = LL_RESERVED_SPACE(skb->dev); if (skb_has_frag_list(skb)) { unsigned int first_len = skb_pagelen(skb); diff --git a/net/ipv6/netfilter/ip6t_ah.c b/net/ipv6/netfilter/ip6t_ah.c index 70da2f2ce064..1258783ed876 100644 --- a/net/ipv6/netfilter/ip6t_ah.c +++ b/net/ipv6/netfilter/ip6t_ah.c @@ -56,6 +56,11 @@ static bool ah_mt6(const struct sk_buff *skb, struct xt_action_param *par) } hdrlen = ipv6_authlen(ah); + if (skb->len - ptr < hdrlen) { + /* Packet smaller than its length field */ + par->hotdrop = true; + return false; + } pr_debug("IPv6 AH LEN %u %u ", hdrlen, ah->hdrlen); pr_debug("RES %04X ", ah->reserved); diff --git a/net/ipv6/netfilter/ip6t_hbh.c b/net/ipv6/netfilter/ip6t_hbh.c index 450dd53846a2..6d1a5d2026a6 100644 --- a/net/ipv6/netfilter/ip6t_hbh.c +++ b/net/ipv6/netfilter/ip6t_hbh.c @@ -75,6 +75,7 @@ hbh_mt6(const struct sk_buff *skb, struct xt_action_param *par) hdrlen = ipv6_optlen(oh); if (skb->len - ptr < hdrlen) { /* Packet smaller than it's length field */ + par->hotdrop = true; return false; } diff --git a/net/ipv6/netfilter/ip6t_rt.c b/net/ipv6/netfilter/ip6t_rt.c index 5561bd9cea81..278b52752f36 100644 --- a/net/ipv6/netfilter/ip6t_rt.c +++ b/net/ipv6/netfilter/ip6t_rt.c @@ -56,7 +56,8 @@ static bool rt_mt6(const struct sk_buff *skb, struct xt_action_param *par) hdrlen = ipv6_optlen(rh); if (skb->len - ptr < hdrlen) { - /* Pcket smaller than its length field */ + /* Packet smaller than its length field */ + par->hotdrop = true; return false; } diff --git a/net/ipv6/netfilter/nf_conntrack_reasm.c b/net/ipv6/netfilter/nf_conntrack_reasm.c index 64ab23ff559b..599c49bf0a0a 100644 --- a/net/ipv6/netfilter/nf_conntrack_reasm.c +++ b/net/ipv6/netfilter/nf_conntrack_reasm.c @@ -348,7 +348,8 @@ static int nf_ct_frag6_reasm(struct frag_queue *fq, struct sk_buff *skb, skb_network_header(skb)[fq->nhoffset] = skb_transport_header(skb)[0]; memmove(skb->head + sizeof(struct frag_hdr), skb->head, (skb->data - skb->head) - sizeof(struct frag_hdr)); - skb->mac_header += sizeof(struct frag_hdr); + if (skb_mac_header_was_set(skb)) + skb->mac_header += sizeof(struct frag_hdr); skb->network_header += sizeof(struct frag_hdr); skb_reset_transport_header(skb); @@ -418,7 +419,7 @@ find_prev_fhdr(struct sk_buff *skb, u8 *prevhdrp, int *prevhoff, int *fhoff) return -1; } if (skb_copy_bits(skb, start, &hdr, sizeof(hdr))) - BUG(); + return -1; if (nexthdr == NEXTHDR_AUTH) hdrlen = ipv6_authlen(&hdr); else diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index 3cc58698cbbd..b88d364e78aa 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -349,7 +349,7 @@ void raw6_icmp_error(struct sk_buff *skb, int nexthdr, const struct ipv6hdr *ip6h = (const struct ipv6hdr *)skb->data; if (!raw_v6_match(net, sk, nexthdr, &ip6h->saddr, &ip6h->daddr, - inet6_iif(skb), inet6_iif(skb))) + inet6_iif(skb), inet6_sdif(skb))) continue; rawv6_err(sk, skb, type, code, inner_offset, info); } diff --git a/net/ipv6/route.c b/net/ipv6/route.c index a1301334da48..5968ce5ad150 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -988,13 +988,13 @@ int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, } else if (rinfo->prefix_len > 128) { return -EINVAL; } else if (rinfo->prefix_len > 64) { - if (rinfo->length < 2) { + /* RFC 4191: Length MUST be 3 when Prefix Length > 64 */ + if (rinfo->length < 3) return -EINVAL; - } } else if (rinfo->prefix_len > 0) { - if (rinfo->length < 1) { + /* RFC 4191: Length MUST be 2 or 3 when Prefix Length > 0 */ + if (rinfo->length < 2) return -EINVAL; - } } pref = rinfo->route_pref; @@ -4022,7 +4022,7 @@ static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) struct fib6_node *fn; /* prefer to send a single notification with all hops */ - skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); + skb = nlmsg_new(rt6_nlmsg_size(rt), GFP_ATOMIC); if (skb) { u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; @@ -4078,7 +4078,7 @@ out_put: if (skb) { rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, - info->nlh, gfp_any()); + info->nlh, GFP_ATOMIC); } return err; } diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index ebe161d72fbd..9e9155b1b3aa 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -923,6 +923,8 @@ static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 (tcp_ao_len(key->ao_key) << 16) | (key->ao_key->sndid << 8) | (key->rcv_next)); + memset((u8 *)topt + tcp_ao_maclen(key->ao_key), TCPOPT_NOP, + tcp_ao_len_aligned(key->ao_key) - tcp_ao_len(key->ao_key)); tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key, key->traffic_key, @@ -1977,8 +1979,10 @@ do_time_wait: } drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb); - if (drop_reason) - break; + if (drop_reason) { + inet_twsk_put(inet_twsk(sk)); + goto discard_it; + } } /* to ACK */ fallthrough; diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index 125ea9a5b8a0..3b749475f6ed 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -88,6 +88,7 @@ static int xfrm6_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, xdst->u.rt6.rt6i_idev = in6_dev_get(dev); if (!xdst->u.rt6.rt6i_idev) { netdev_put(dev, &xdst->u.dst.dev_tracker); + xdst->u.dst.dev = NULL; return -ENODEV; } diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c index fed240b453bd..ea047bab65e7 100644 --- a/net/iucv/af_iucv.c +++ b/net/iucv/af_iucv.c @@ -337,6 +337,7 @@ static void iucv_sever_path(struct sock *sk, int with_user_data) unsigned char user_data[16]; struct iucv_sock *iucv = iucv_sk(sk); struct iucv_path *path = iucv->path; + struct sock_msg_q *p, *n; /* Whoever resets the path pointer, must sever and free it. */ if (xchg(&iucv->path, NULL)) { @@ -348,6 +349,19 @@ static void iucv_sever_path(struct sock *sk, int with_user_data) } else pr_iucv->path_sever(path, NULL); iucv_path_free(path); + + /* + * Message notifications queued on message_q still reference + * the now freed path; drop them, otherwise a later recvmsg() + * would pass the freed iucv_path to message_receive() via + * iucv_process_message_q(). + */ + spin_lock_bh(&iucv->message_q.lock); + list_for_each_entry_safe(p, n, &iucv->message_q.list, list) { + list_del(&p->list); + kfree(p); + } + spin_unlock_bh(&iucv->message_q.lock); } } @@ -1872,7 +1886,8 @@ static int afiucv_hs_callback_syn(struct sock *sk, struct sk_buff *skb) afiucv_swap_src_dest(skb); trans_hdr->flags = AF_IUCV_FLAG_SYN | AF_IUCV_FLAG_FIN; err = dev_queue_xmit(skb); - iucv_sock_kill(nsk); + if (nsk) + iucv_sock_kill(nsk); bh_unlock_sock(sk); goto out; } @@ -2089,6 +2104,8 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev, } } } + if (sk) + sock_hold(sk); read_unlock(&iucv_sk_list.lock); if (!iucv) sk = NULL; @@ -2138,6 +2155,8 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev, kfree_skb(skb); } + if (sk) + sock_put(sk); return err; } diff --git a/net/l2tp/l2tp_debugfs.c b/net/l2tp/l2tp_debugfs.c index b26986fda9d6..c12d5e1ee550 100644 --- a/net/l2tp/l2tp_debugfs.c +++ b/net/l2tp/l2tp_debugfs.c @@ -306,6 +306,10 @@ static int l2tp_dfs_seq_release(struct inode *inode, struct file *file) seq = file->private_data; pd = seq->private; + if (pd->session) + l2tp_session_put(pd->session); + if (pd->tunnel) + l2tp_tunnel_put(pd->tunnel); if (pd->net) put_net_track(pd->net, &pd->ns_tracker); kfree(pd); diff --git a/net/l2tp/l2tp_ppp.c b/net/l2tp/l2tp_ppp.c index f8881699e1ca..ef8fdfaf051d 100644 --- a/net/l2tp/l2tp_ppp.c +++ b/net/l2tp/l2tp_ppp.c @@ -1597,7 +1597,53 @@ static const struct seq_operations pppol2tp_seq_ops = { .stop = pppol2tp_seq_stop, .show = pppol2tp_seq_show, }; -#endif /* CONFIG_PROC_FS */ + +static int pppol2tp_proc_open(struct inode *inode, struct file *file) +{ + struct net *net = pde_data(inode); + struct pppol2tp_seq_data *pd; + + net = maybe_get_net(net); + if (!net) + return -ENXIO; + + pd = __seq_open_private(file, &pppol2tp_seq_ops, sizeof(*pd)); + if (!pd) { + put_net(net); + return -ENOMEM; + } + +#ifdef CONFIG_NET_NS + pd->p.net = net; + netns_tracker_alloc(net, &pd->p.ns_tracker, GFP_KERNEL); +#endif + return 0; +} + +static int pppol2tp_proc_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct pppol2tp_seq_data *pd = seq->private; + + if (pd->session) + l2tp_session_put(pd->session); + if (pd->tunnel) + l2tp_tunnel_put(pd->tunnel); + +#ifdef CONFIG_NET_NS + put_net_track(pd->p.net, &pd->p.ns_tracker); +#else + put_net(&init_net); +#endif + return seq_release_private(inode, file); +} + +static const struct proc_ops pppol2tp_proc_ops = { + .proc_open = pppol2tp_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = pppol2tp_proc_release, +}; /***************************************************************************** * Network namespace @@ -1608,8 +1654,8 @@ static __net_init int pppol2tp_init_net(struct net *net) struct proc_dir_entry *pde; int err = 0; - pde = proc_create_net("pppol2tp", 0444, net->proc_net, - &pppol2tp_seq_ops, sizeof(struct pppol2tp_seq_data)); + pde = proc_create_data("pppol2tp", 0444, net->proc_net, + &pppol2tp_proc_ops, net); if (!pde) { err = -ENOMEM; goto out; @@ -1624,9 +1670,13 @@ static __net_exit void pppol2tp_exit_net(struct net *net) remove_proc_entry("pppol2tp", net->proc_net); } +#endif /* CONFIG_PROC_FS */ + static struct pernet_operations pppol2tp_net_ops = { +#ifdef CONFIG_PROC_FS .init = pppol2tp_init_net, .exit = pppol2tp_exit_net, +#endif }; /***************************************************************************** diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index 8ed1be1ecccc..b0447c33dbf0 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c @@ -312,6 +312,7 @@ static int llc_ui_autobind(struct socket *sock, struct sockaddr_llc *addr) /* assign new connection to its SAP */ llc_sap_add_socket(sap, sk); sock_reset_flag(sk, SOCK_ZAPPED); + llc_sap_put(sap); rc = 0; out: dev_put(dev); diff --git a/net/llc/llc_conn.c b/net/llc/llc_conn.c index e8f427375c68..260460d50f54 100644 --- a/net/llc/llc_conn.c +++ b/net/llc/llc_conn.c @@ -767,7 +767,6 @@ static struct sock *llc_create_incoming_sock(struct sock *sk, newllc->dev = dev; dev_hold(dev); llc_sap_add_socket(llc->sap, newsk); - llc_sap_hold(llc->sap); out: return newsk; } diff --git a/net/mac80211/agg-tx.c b/net/mac80211/agg-tx.c index 4833b46770b6..0832213430f4 100644 --- a/net/mac80211/agg-tx.c +++ b/net/mac80211/agg-tx.c @@ -915,6 +915,7 @@ void ieee80211_stop_tx_ba_cb(struct sta_info *sta, int tid, struct tid_ampdu_tx *tid_tx) { struct ieee80211_sub_if_data *sdata = sta->sdata; + bool ndp = ndp = tid_tx->ndp; bool send_delba = false; bool start_txq = false; @@ -934,6 +935,7 @@ void ieee80211_stop_tx_ba_cb(struct sta_info *sta, int tid, send_delba = true; ieee80211_remove_tid_tx(sta, tid); + /* tid_tx is now invalid since ieee80211_remove_tid_tx() frees it */ start_txq = true; unlock_sta: @@ -946,7 +948,7 @@ void ieee80211_stop_tx_ba_cb(struct sta_info *sta, int tid, ieee80211_send_delba(sdata, sta->sta.addr, tid, WLAN_BACK_INITIATOR, WLAN_REASON_QSTA_NOT_USE, - tid_tx->ndp); + ndp); } void ieee80211_stop_tx_ba_cb_irqsafe(struct ieee80211_vif *vif, diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index 3b58af59f7e4..43f142624d33 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -1146,9 +1146,6 @@ static int ieee80211_set_fils_discovery(struct ieee80211_sub_if_data *sdata, fd->max_interval = params->max_interval; old = sdata_dereference(link->u.ap.fils_discovery, sdata); - if (old) - kfree_rcu(old, rcu_head); - if (params->tmpl && params->tmpl_len) { new = kzalloc(sizeof(*new) + params->tmpl_len, GFP_KERNEL); if (!new) @@ -1160,6 +1157,9 @@ static int ieee80211_set_fils_discovery(struct ieee80211_sub_if_data *sdata, RCU_INIT_POINTER(link->u.ap.fils_discovery, NULL); } + if (old) + kfree_rcu(old, rcu_head); + *changed |= BSS_CHANGED_FILS_DISCOVERY; return 0; } @@ -1179,8 +1179,6 @@ ieee80211_set_unsol_bcast_probe_resp(struct ieee80211_sub_if_data *sdata, link_conf->unsol_bcast_probe_resp_interval = params->interval; old = sdata_dereference(link->u.ap.unsol_bcast_probe_resp, sdata); - if (old) - kfree_rcu(old, rcu_head); if (params->tmpl && params->tmpl_len) { new = kzalloc(sizeof(*new) + params->tmpl_len, GFP_KERNEL); @@ -1193,6 +1191,9 @@ ieee80211_set_unsol_bcast_probe_resp(struct ieee80211_sub_if_data *sdata, RCU_INIT_POINTER(link->u.ap.unsol_bcast_probe_resp, NULL); } + if (old) + kfree_rcu(old, rcu_head); + *changed |= BSS_CHANGED_UNSOL_BCAST_PROBE_RESP; return 0; } @@ -2610,6 +2611,9 @@ static int sta_apply_parameters(struct ieee80211_local *local, memcpy(&sta->deflink.pub->supp_rates, &nmi_sta->deflink.pub->supp_rates, sizeof(sta->deflink.pub->supp_rates)); + + sta->deflink.pub->agg = nmi_sta->deflink.pub->agg; + __ieee80211_sta_recalc_aggregates(sta, 0); } /* set the STA state after all sta info from usermode has been set */ diff --git a/net/mac80211/ibss.c b/net/mac80211/ibss.c index d0fd6054f182..882f91abbb66 100644 --- a/net/mac80211/ibss.c +++ b/net/mac80211/ibss.c @@ -668,7 +668,9 @@ static void ieee80211_ibss_disconnect(struct ieee80211_sub_if_data *sdata) ifibss->state = IEEE80211_IBSS_MLME_SEARCH; - sta_info_flush(sdata, -1); + netif_carrier_off(sdata->dev); + if (!sta_info_flush(sdata, -1)) + synchronize_net(); spin_lock_bh(&ifibss->incomplete_lock); while (!list_empty(&ifibss->incomplete_stations)) { @@ -682,8 +684,6 @@ static void ieee80211_ibss_disconnect(struct ieee80211_sub_if_data *sdata) } spin_unlock_bh(&ifibss->incomplete_lock); - netif_carrier_off(sdata->dev); - sdata->vif.cfg.ibss_joined = false; sdata->vif.cfg.ibss_creator = false; sdata->vif.bss_conf.enable_beacon = false; @@ -710,7 +710,6 @@ static void ieee80211_csa_connection_drop_work(struct wiphy *wiphy, u.ibss.csa_connection_drop_work); ieee80211_ibss_disconnect(sdata); - synchronize_rcu(); skb_queue_purge(&sdata->skb_queue); /* trigger a scan to find another IBSS network to join */ @@ -1029,8 +1028,8 @@ static void ieee80211_update_sta_info(struct ieee80211_sub_if_data *sdata, u32 changed = IEEE80211_RC_SUPP_RATES_CHANGED; u8 rx_nss = sta->sta.deflink.rx_nss; - /* Force rx_nss recalculation */ - sta->sta.deflink.rx_nss = 0; + ieee80211_sta_init_nss_bw_capa(&sta->deflink, + &sdata->deflink.conf->chanreq.oper); rate_control_rate_init(&sta->deflink); if (sta->sta.deflink.rx_nss != rx_nss) changed |= IEEE80211_RC_NSS_CHANGED; @@ -1797,8 +1796,6 @@ int ieee80211_ibss_leave(struct ieee80211_sub_if_data *sdata) memset(&ifibss->ht_capa, 0, sizeof(ifibss->ht_capa)); memset(&ifibss->ht_capa_mask, 0, sizeof(ifibss->ht_capa_mask)); - synchronize_rcu(); - skb_queue_purge(&sdata->skb_queue); timer_delete_sync(&sdata->u.ibss.timer); diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 086272c3ec08..43460a705a6b 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -588,6 +588,7 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do WARN_ON(!list_empty(&sdata->u.ap.vlans)); } else if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) { /* remove all packets in parent bc_buf pointing to this dev */ + __skb_queue_head_init(&freeq); ps = &sdata->bss->ps; spin_lock_irqsave(&ps->bc_buf.lock, flags); @@ -595,10 +596,15 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do if (skb->dev == sdata->dev) { __skb_unlink(skb, &ps->bc_buf); local->total_ps_buffered--; - ieee80211_free_txskb(&local->hw, skb); + __skb_queue_tail(&freeq, skb); } } spin_unlock_irqrestore(&ps->bc_buf.lock, flags); + + skb_queue_walk_safe(&freeq, skb, tmp) { + __skb_unlink(skb, &freeq); + ieee80211_free_txskb(&local->hw, skb); + } } if (going_down) diff --git a/net/mac80211/link.c b/net/mac80211/link.c index d0535268962c..dc68144dc363 100644 --- a/net/mac80211/link.c +++ b/net/mac80211/link.c @@ -399,6 +399,10 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, memcpy(sdata->link, old_data, sizeof(old_data)); memcpy(sdata->vif.link_conf, old, sizeof(old)); ieee80211_set_vif_links_bitmaps(sdata, old_links, dormant_links); + for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) { + ieee80211_link_debugfs_remove(&links[link_id]->data); + ieee80211_link_stop(&links[link_id]->data); + } /* and free (only) the newly allocated links */ memset(to_free, 0, sizeof(links)); goto free; diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 90d295cc364f..eb1eaaf34612 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -1602,7 +1602,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) sband = kmemdup(sband, sizeof(*sband), GFP_KERNEL); if (!sband) { result = -ENOMEM; - goto fail_rate; + goto fail_band; } wiphy_dbg(hw->wiphy, "copying sband (band %d) due to VHT EXT NSS BW flag\n", @@ -1678,6 +1678,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) #endif wiphy_unregister(local->hw.wiphy); fail_wiphy_register: + fail_band: rtnl_lock(); rate_control_deinitialize(local); ieee80211_remove_interfaces(local); diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 9e92337bb6f9..fa773f3b0541 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -5641,13 +5641,15 @@ static void ieee80211_rx_mgmt_deauth(struct ieee80211_sub_if_data *sdata, struct ieee80211_mgmt *mgmt, size_t len) { struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; - u16 reason_code = le16_to_cpu(mgmt->u.deauth.reason_code); + u16 reason_code; lockdep_assert_wiphy(sdata->local->hw.wiphy); - if (len < 24 + 2) + if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code)) return; + reason_code = le16_to_cpu(mgmt->u.deauth.reason_code); + if (!ether_addr_equal(mgmt->bssid, mgmt->sa)) { ieee80211_tdls_handle_disconnect(sdata, mgmt->sa, reason_code); return; @@ -7138,7 +7140,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, { struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data; - u16 capab_info, status_code, aid; + u16 capab_info, status_code, aid = 0; struct ieee80211_elems_parse_params parse_params = { .bss = NULL, .link_id = -1, @@ -7217,8 +7219,10 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, if (elems->aid_resp) aid = le16_to_cpu(elems->aid_resp->aid); - else + else if (!assoc_data->s1g) aid = le16_to_cpu(mgmt->u.assoc_resp.aid); + else if (status_code == WLAN_STATUS_SUCCESS) + goto abandon_assoc; /* * The 5 MSB of the AID field are reserved for a non-S1G STA. For diff --git a/net/mac80211/nan.c b/net/mac80211/nan.c index 1800bb96dd29..19e08661be43 100644 --- a/net/mac80211/nan.c +++ b/net/mac80211/nan.c @@ -253,9 +253,12 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata, { struct ieee80211_nan_channel *sched_idx_to_chan[IEEE80211_NAN_MAX_CHANNELS] = {}; struct ieee80211_nan_sched_cfg *sched_cfg = &sdata->vif.cfg.nan_sched; - struct ieee80211_nan_sched_cfg backup_sched; + struct ieee80211_nan_sched_cfg *backup_sched __free(kfree) = kmalloc_obj(*backup_sched); int ret; + if (!backup_sched) + return -ENOMEM; + if (sched->n_channels > IEEE80211_NAN_MAX_CHANNELS) return -EOPNOTSUPP; @@ -275,13 +278,13 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata, bitmap_zero(sdata->u.nan.removed_channels, IEEE80211_NAN_MAX_CHANNELS); - memcpy(backup_sched.schedule, sched_cfg->schedule, - sizeof(backup_sched.schedule)); - memcpy(backup_sched.channels, sched_cfg->channels, - sizeof(backup_sched.channels)); - memcpy(backup_sched.avail_blob, sched_cfg->avail_blob, - sizeof(backup_sched.avail_blob)); - backup_sched.avail_blob_len = sched_cfg->avail_blob_len; + memcpy(backup_sched->schedule, sched_cfg->schedule, + sizeof(backup_sched->schedule)); + memcpy(backup_sched->channels, sched_cfg->channels, + sizeof(backup_sched->channels)); + memcpy(backup_sched->avail_blob, sched_cfg->avail_blob, + sizeof(backup_sched->avail_blob)); + backup_sched->avail_blob_len = sched_cfg->avail_blob_len; memcpy(sched_cfg->avail_blob, sched->nan_avail_blob, sched->nan_avail_blob_len); @@ -380,17 +383,17 @@ err: if (!chan_def->chan) continue; - if (!cfg80211_chandef_identical(&backup_sched.channels[i].chanreq.oper, + if (!cfg80211_chandef_identical(&backup_sched->channels[i].chanreq.oper, chan_def)) ieee80211_nan_remove_channel(sdata, &sched_cfg->channels[i]); } /* Re-add all backed up channels */ - for (int i = 0; i < ARRAY_SIZE(backup_sched.channels); i++) { + for (int i = 0; i < ARRAY_SIZE(backup_sched->channels); i++) { struct ieee80211_nan_channel *chan = &sched_cfg->channels[i]; - *chan = backup_sched.channels[i]; + *chan = backup_sched->channels[i]; /* * For deferred update, no channels were removed and the channel @@ -421,11 +424,11 @@ err: } } - memcpy(sched_cfg->schedule, backup_sched.schedule, - sizeof(backup_sched.schedule)); - memcpy(sched_cfg->avail_blob, backup_sched.avail_blob, - sizeof(backup_sched.avail_blob)); - sched_cfg->avail_blob_len = backup_sched.avail_blob_len; + memcpy(sched_cfg->schedule, backup_sched->schedule, + sizeof(backup_sched->schedule)); + memcpy(sched_cfg->avail_blob, backup_sched->avail_blob, + sizeof(backup_sched->avail_blob)); + sched_cfg->avail_blob_len = backup_sched->avail_blob_len; sched_cfg->deferred = false; bitmap_zero(sdata->u.nan.removed_channels, IEEE80211_NAN_MAX_CHANNELS); diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index fb9a3574afe9..5e26be8e27d8 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -1526,6 +1526,9 @@ ieee80211_rx_h_check_dup(struct ieee80211_rx_data *rx) if (status->flag & RX_FLAG_DUP_VALIDATED) return RX_CONTINUE; + if (ieee80211_is_ext(hdr->frame_control)) + return RX_CONTINUE; + /* * Drop duplicate 802.11 retransmissions * (IEEE 802.11-2012: 9.3.2.10 "Duplicate detection and recovery") @@ -1714,6 +1717,8 @@ static void sta_ps_start(struct sta_info *sta) else clear_bit(tid, &sta->txq_buffered_tids); } + + sta_info_recalc_tim(sta); } static void sta_ps_end(struct sta_info *sta) @@ -4510,12 +4515,16 @@ static bool ieee80211_accept_frame(struct ieee80211_rx_data *rx) struct ieee80211_hdr *hdr = (void *)skb->data; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); u8 *bssid = ieee80211_get_bssid(hdr, skb->len, sdata->vif.type); - bool multicast = is_multicast_ether_addr(hdr->addr1) || - ieee80211_is_s1g_beacon(hdr->frame_control); + bool multicast; static const u8 nan_network_id[ETH_ALEN] __aligned(2) = { 0x51, 0x6F, 0x9A, 0x01, 0x00, 0x00 }; + if (ieee80211_is_s1g_beacon(hdr->frame_control)) + return sdata->vif.type == NL80211_IFTYPE_STATION && bssid; + + multicast = is_multicast_ether_addr(hdr->addr1); + switch (sdata->vif.type) { case NL80211_IFTYPE_STATION: if (!bssid && !sdata->u.mgd.use_4addr) @@ -5212,6 +5221,11 @@ static bool ieee80211_prepare_and_rx_handle(struct ieee80211_rx_data *rx, hdr = (struct ieee80211_hdr *)rx->skb->data; } + if (ieee80211_is_s1g_beacon(hdr->frame_control)) { + ieee80211_invoke_rx_handlers(rx); + return true; + } + /* Store a copy of the pre-translated link addresses for SW crypto */ if (unlikely(is_unicast_ether_addr(hdr->addr1) && !ieee80211_is_data(hdr->frame_control))) @@ -5301,6 +5315,13 @@ static bool ieee80211_rx_for_interface(struct ieee80211_rx_data *rx, struct sta_info *sta; int link_id = -1; + if (ieee80211_is_s1g_beacon(hdr->frame_control)) { + if (!ieee80211_rx_data_set_sta(rx, NULL, -1)) + return false; + + return ieee80211_prepare_and_rx_handle(rx, skb, consume); + } + /* * Look up link station first, in case there's a * chance that they might have a link address that @@ -5376,6 +5397,17 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw, err = -ENOBUFS; else err = skb_linearize(skb); + } else if (ieee80211_is_s1g_beacon(fc)) { + size_t s1g_hdr_len = offsetof(struct ieee80211_ext, + u.s1g_beacon.variable) + + ieee80211_s1g_optional_len(fc); + + if (skb->len < s1g_hdr_len) + err = -ENOBUFS; + else + err = skb_linearize(skb); + } else if (ieee80211_is_ext(fc)) { + err = -EINVAL; } else { err = !pskb_may_pull(skb, ieee80211_hdrlen(fc)); } diff --git a/net/mac80211/s1g.c b/net/mac80211/s1g.c index 5af4a0c6c642..abc338e22e59 100644 --- a/net/mac80211/s1g.c +++ b/net/mac80211/s1g.c @@ -101,6 +101,10 @@ ieee80211_s1g_rx_twt_setup(struct ieee80211_sub_if_data *sdata, struct ieee80211_twt_setup *twt = (void *)mgmt->u.action.s1g.variable; struct ieee80211_twt_params *twt_agrt = (void *)twt->params; + if (!(twt->control & IEEE80211_TWT_CONTROL_NEG_TYPE_BROADCAST) && + twt->length < sizeof(twt->control) + sizeof(*twt_agrt)) + return; + twt_agrt->req_type &= cpu_to_le16(~IEEE80211_TWT_REQTYPE_REQUEST); /* broadcast TWT not supported yet */ diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index 02b587ff8504..22eba0e6e54c 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -355,6 +355,15 @@ static void sta_info_free_link(struct link_sta_info *link_sta) free_percpu(link_sta->pcpu_rx_stats); } +static void sta_link_free_rcu(struct rcu_head *head) +{ + struct sta_link_alloc *alloc = + container_of(head, struct sta_link_alloc, rcu_head); + + sta_info_free_link(&alloc->info); + kfree(alloc); +} + static void sta_accumulate_removed_link_stats(struct sta_info *sta, int link_id) { struct link_sta_info *link_sta = wiphy_dereference(sta->local->hw.wiphy, @@ -439,10 +448,8 @@ static void sta_remove_link(struct sta_info *sta, unsigned int link_id, RCU_INIT_POINTER(sta->link[link_id], NULL); RCU_INIT_POINTER(sta->sta.link[link_id], NULL); - if (alloc) { - sta_info_free_link(&alloc->info); - kfree_rcu(alloc, rcu_head); - } + if (alloc) + call_rcu(&alloc->rcu_head, sta_link_free_rcu); ieee80211_sta_recalc_aggregates(&sta->sta); } diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index c13b209fad47..91b14112e24f 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -2607,6 +2607,18 @@ static u16 ieee80211_store_ack_skb(struct ieee80211_local *local, return info_id; } +static void ieee80211_remove_ack_skb(struct ieee80211_local *local, u16 info_id) +{ + struct sk_buff *ack_skb; + unsigned long flags; + + spin_lock_irqsave(&local->ack_status_lock, flags); + ack_skb = idr_remove(&local->ack_status_frames, info_id); + spin_unlock_irqrestore(&local->ack_status_lock, flags); + + kfree_skb(ack_skb); +} + /** * ieee80211_build_hdr - build 802.11 header in the given frame * @sdata: virtual interface to build the header for @@ -2982,7 +2994,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, if (ieee80211_skb_resize(sdata, skb, head_need, ENCRYPT_DATA)) { ieee80211_free_txskb(&local->hw, skb); skb = NULL; - return ERR_PTR(-ENOMEM); + ret = -ENOMEM; + goto free; } } @@ -3050,6 +3063,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata, return skb; free: + if (info_id) + ieee80211_remove_ack_skb(local, info_id); kfree_skb(skb); return ERR_PTR(ret); } diff --git a/net/mac80211/util.c b/net/mac80211/util.c index f6d4ae4127c8..59f73dabe6e0 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -73,6 +73,9 @@ u8 *ieee80211_get_bssid(struct ieee80211_hdr *hdr, size_t len, if (ieee80211_is_s1g_beacon(fc)) { struct ieee80211_ext *ext = (void *) hdr; + if (len < offsetofend(struct ieee80211_ext, u.s1g_beacon.sa)) + return NULL; + return ext->u.s1g_beacon.sa; } diff --git a/net/mac802154/iface.c b/net/mac802154/iface.c index 000be60d9580..b823720630e7 100644 --- a/net/mac802154/iface.c +++ b/net/mac802154/iface.c @@ -703,7 +703,7 @@ void ieee802154_remove_interfaces(struct ieee802154_local *local) mutex_lock(&local->iflist_mtx); list_for_each_entry_safe(sdata, tmp, &local->interfaces, list) { - list_del(&sdata->list); + list_del_rcu(&sdata->list); unregister_netdevice(sdata->dev); } diff --git a/net/mac802154/llsec.c b/net/mac802154/llsec.c index 5e7cc11fab3a..85452ef9a58c 100644 --- a/net/mac802154/llsec.c +++ b/net/mac802154/llsec.c @@ -891,6 +891,11 @@ llsec_do_decrypt_auth(struct sk_buff *skb, const struct mac802154_llsec *sec, data = skb_mac_header(skb) + skb->mac_len; datalen = skb_tail_pointer(skb) - data; + if (datalen < authlen) { + kfree_sensitive(req); + return -EBADMSG; + } + sg_init_one(&sg, skb_mac_header(skb), assoclen + datalen); if (!(hdr->sec.level & IEEE802154_SCF_SECLEVEL_ENC)) { diff --git a/net/mac802154/scan.c b/net/mac802154/scan.c index 300d4584533e..005338f89b75 100644 --- a/net/mac802154/scan.c +++ b/net/mac802154/scan.c @@ -179,6 +179,7 @@ void mac802154_scan_worker(struct work_struct *work) enum nl802154_scan_types scan_req_type; struct ieee802154_sub_if_data *sdata; unsigned int scan_duration = 0; + netdevice_tracker dev_tracker; struct wpan_phy *wpan_phy; u8 scan_req_duration; u8 page, channel; @@ -209,6 +210,14 @@ void mac802154_scan_worker(struct work_struct *work) return; } + /* + * sdata->dev is dereferenced below after rcu_read_unlock() and outside + * the rtnl, and a concurrent DEL_INTERFACE / PHY teardown can free it + * asynchronously from netdev_run_todo(). Pin it with a reference taken + * while the RCU read lock is still held, and drop it at every exit. + */ + netdev_hold(sdata->dev, &dev_tracker, GFP_ATOMIC); + wpan_phy = scan_req->wpan_phy; scan_req_type = scan_req->type; scan_req_duration = scan_req->duration; @@ -262,12 +271,14 @@ void mac802154_scan_worker(struct work_struct *work) "Scan page %u channel %u for %ums\n", page, channel, jiffies_to_msecs(scan_duration)); queue_delayed_work(local->mac_wq, &local->scan_work, scan_duration); + netdev_put(sdata->dev, &dev_tracker); return; end_scan: rtnl_lock(); mac802154_scan_cleanup_locked(local, sdata, false); rtnl_unlock(); + netdev_put(sdata->dev, &dev_tracker); } int mac802154_trigger_scan_locked(struct ieee802154_sub_if_data *sdata, @@ -404,6 +415,7 @@ void mac802154_beacon_worker(struct work_struct *work) container_of(work, struct ieee802154_local, beacon_work.work); struct cfg802154_beacon_request *beacon_req; struct ieee802154_sub_if_data *sdata; + netdevice_tracker dev_tracker; struct wpan_dev *wpan_dev; u8 interval; int ret; @@ -416,12 +428,14 @@ void mac802154_beacon_worker(struct work_struct *work) } sdata = IEEE802154_WPAN_DEV_TO_SUB_IF(beacon_req->wpan_dev); + netdev_hold(sdata->dev, &dev_tracker, GFP_ATOMIC); /* Wait an arbitrary amount of time in case we cannot use the device */ if (local->suspended || !ieee802154_sdata_running(sdata)) { rcu_read_unlock(); queue_delayed_work(local->mac_wq, &local->beacon_work, msecs_to_jiffies(1000)); + netdev_put(sdata->dev, &dev_tracker); return; } @@ -439,6 +453,7 @@ void mac802154_beacon_worker(struct work_struct *work) if (interval < IEEE802154_ACTIVE_SCAN_DURATION) queue_delayed_work(local->mac_wq, &local->beacon_work, local->beacon_interval); + netdev_put(sdata->dev, &dev_tracker); } int mac802154_stop_beacons_locked(struct ieee802154_local *local, diff --git a/net/mctp/device.c b/net/mctp/device.c index 2c84df674669..822120e860c8 100644 --- a/net/mctp/device.c +++ b/net/mctp/device.c @@ -536,7 +536,9 @@ int __init mctp_device_init(void) { int err; - register_netdevice_notifier(&mctp_dev_nb); + err = register_netdevice_notifier(&mctp_dev_nb); + if (err) + return err; err = rtnl_af_register(&mctp_af_ops); if (err) diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index ca504d9626cf..961be5054a03 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -922,8 +922,7 @@ static int mpls_nh_build_multi(struct mpls_route_config *cfg, struct nlattr *nla_via, *nla_newdst; int remaining = cfg->rc_mp_len; int err = 0; - - rt->rt_nhn = 0; + u8 nhs = 0; change_nexthops(rt) { int attrlen; @@ -959,12 +958,15 @@ static int mpls_nh_build_multi(struct mpls_route_config *cfg, rt->rt_nhn_alive--; rtnh = rtnh_next(rtnh, &remaining); - rt->rt_nhn++; + nhs++; } endfor_nexthops(rt); + rt->rt_nhn = nhs; + return 0; errout: + rt->rt_nhn = nhs; return err; } @@ -2186,6 +2188,9 @@ static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh, int ifindex; if (i == RTA_OIF) { + if (!tb[i]) + continue; + ifindex = nla_get_u32(tb[i]); filter->dev = dev_get_by_index_rcu(net, ifindex); if (!filter->dev) @@ -2534,6 +2539,7 @@ static int mpls_getroute(struct sk_buff *in_skb, struct nlmsghdr *in_nlh, r->rtm_family = AF_MPLS; r->rtm_dst_len = 20; r->rtm_src_len = 0; + r->rtm_tos = 0; r->rtm_table = RT_TABLE_MAIN; r->rtm_type = RTN_UNICAST; r->rtm_scope = RT_SCOPE_UNIVERSE; diff --git a/net/mptcp/fastopen.c b/net/mptcp/fastopen.c index 082c46c0f50e..f717750906ff 100644 --- a/net/mptcp/fastopen.c +++ b/net/mptcp/fastopen.c @@ -24,12 +24,13 @@ void mptcp_fastopen_subflow_synack_set_params(struct mptcp_subflow_context *subf sk = subflow->conn; tp = tcp_sk(ssk); - subflow->is_mptfo = 1; - + /* A valid TFO cookie does not guarantee SYN data. */ skb = skb_peek(&ssk->sk_receive_queue); - if (WARN_ON_ONCE(!skb)) + if (!skb) return; + subflow->is_mptfo = 1; + /* dequeue the skb from sk receive queue */ __skb_unlink(skb, &ssk->sk_receive_queue); skb_ext_reset(skb); diff --git a/net/mptcp/options.c b/net/mptcp/options.c index dff3fd5d3b55..1057d500577b 100644 --- a/net/mptcp/options.c +++ b/net/mptcp/options.c @@ -50,6 +50,14 @@ static void mptcp_parse_option(const struct sk_buff *skb, } } + /* Only the MPC + ACK can be used with a RM_ADDR */ + if (subopt == OPTION_MPTCP_MPC_ACK) { + if ((mp_opt->suboptions & ~OPTION_MPTCP_RM_ADDR) != 0) + break; + } else if (mp_opt->suboptions != 0) { + break; + } + /* Cfr RFC 8684 Section 3.3.0: * If a checksum is present but its use had * not been negotiated in the MP_CAPABLE handshake, the receiver MUST @@ -122,6 +130,11 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_MP_JOIN: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTION_MPTCP_RM_ADDR | + OPTION_MPTCP_PRIO)) != 0) + break; + if (opsize == TCPOLEN_MPTCP_MPJ_SYN) { mp_opt->suboptions |= OPTION_MPTCP_MPJ_SYN; mp_opt->backup = *ptr++ & MPTCPOPT_BACKUP; @@ -153,21 +166,23 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_DSS: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTION_MPTCP_ADD_ADDR | + OPTION_MPTCP_RM_ADDR | + OPTION_MPTCP_PRIO | + OPTION_MPTCP_FASTCLOSE | + OPTION_MPTCP_FAIL)) != 0) + break; + pr_debug("DSS\n"); ptr++; flags = (*ptr++) & MPTCP_DSS_FLAG_MASK; - mp_opt->data_fin = (flags & MPTCP_DSS_DATA_FIN) != 0; mp_opt->dsn64 = (flags & MPTCP_DSS_DSN64) != 0; mp_opt->use_map = (flags & MPTCP_DSS_HAS_MAP) != 0; mp_opt->ack64 = (flags & MPTCP_DSS_ACK64) != 0; mp_opt->use_ack = (flags & MPTCP_DSS_HAS_ACK); - pr_debug("data_fin=%d dsn64=%d use_map=%d ack64=%d use_ack=%d\n", - mp_opt->data_fin, mp_opt->dsn64, - mp_opt->use_map, mp_opt->ack64, - mp_opt->use_ack); - expected_opsize = TCPOLEN_MPTCP_DSS_BASE; if (mp_opt->use_ack) { @@ -178,18 +193,30 @@ static void mptcp_parse_option(const struct sk_buff *skb, } if (mp_opt->use_map) { + mp_opt->data_fin = (flags & MPTCP_DSS_DATA_FIN) != 0; if (mp_opt->dsn64) expected_opsize += TCPOLEN_MPTCP_DSS_MAP64; else expected_opsize += TCPOLEN_MPTCP_DSS_MAP32; } + pr_debug("data_fin=%d dsn64=%d use_map=%d ack64=%d use_ack=%d\n", + mp_opt->data_fin, mp_opt->dsn64, + mp_opt->use_map, mp_opt->ack64, + mp_opt->use_ack); + /* Always parse any csum presence combination, we will enforce * RFC 8684 Section 3.3.0 checks later in subflow_data_ready */ if (opsize != expected_opsize && - opsize != expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM) + opsize != expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM) { + mp_opt->dsn64 = 0; + mp_opt->use_map = 0; + mp_opt->ack64 = 0; + mp_opt->use_ack = 0; + mp_opt->data_fin = 0; break; + } mp_opt->suboptions |= OPTION_MPTCP_DSS; if (mp_opt->use_ack) { @@ -234,6 +261,12 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_ADD_ADDR: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTIONS_MPTCP_DSS | + OPTION_MPTCP_RM_ADDR | + OPTION_MPTCP_PRIO)) != 0) + break; + mp_opt->echo = (*ptr++) & MPTCP_ADDR_ECHO; if (!mp_opt->echo) { if (opsize == TCPOLEN_MPTCP_ADD_ADDR || @@ -293,6 +326,14 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_RM_ADDR: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTION_MPTCP_MPC_ACK | + OPTIONS_MPTCP_MPJ | + OPTIONS_MPTCP_DSS | + OPTION_MPTCP_ADD_ADDR | + OPTION_MPTCP_PRIO)) != 0) + break; + if (opsize < TCPOLEN_MPTCP_RM_ADDR_BASE + 1 || opsize > TCPOLEN_MPTCP_RM_ADDR_BASE + MPTCP_RM_IDS_MAX) break; @@ -307,6 +348,13 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_MP_PRIO: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTIONS_MPTCP_MPJ | + OPTIONS_MPTCP_DSS | + OPTION_MPTCP_ADD_ADDR | + OPTION_MPTCP_RM_ADDR)) != 0) + break; + if (opsize != TCPOLEN_MPTCP_PRIO) break; @@ -316,6 +364,11 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_MP_FASTCLOSE: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTIONS_MPTCP_DSS | + OPTION_MPTCP_RST)) != 0) + break; + if (opsize != TCPOLEN_MPTCP_FASTCLOSE) break; @@ -327,6 +380,11 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_RST: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTION_MPTCP_FAIL | + OPTION_MPTCP_FASTCLOSE)) != 0) + break; + if (opsize != TCPOLEN_MPTCP_RST) break; @@ -342,6 +400,11 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_MP_FAIL: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTIONS_MPTCP_DSS | + OPTION_MPTCP_RST)) != 0) + break; + if (opsize != TCPOLEN_MPTCP_FAIL) break; @@ -571,10 +634,7 @@ static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb, bool ret = false; /* Zero `use_ack` and `use_map` flags with one shot. */ - BUILD_BUG_ON(sizeof_field(struct mptcp_ext, flags) != sizeof(u16)); - BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct mptcp_ext, flags), - sizeof(u16))); - *(u16 *)&opts->ext_copy.flags = 0; + memset(&opts->ext_copy.flags, 0, sizeof(opts->ext_copy.flags)); opts->csum_reqd = READ_ONCE(msk->csum_enabled); mpext = skb ? mptcp_get_ext(skb) : NULL; @@ -1403,7 +1463,7 @@ void mptcp_write_options(struct tcphdr *th, __be32 *ptr, struct tcp_sock *tp, * RM | C | C | C | P |------|------|------|------| * PRIO | X | C | C | C | C |------|------|------| * FAIL | X | X | C | X | X | X |------|------| - * FC | X | X | X | X | X | X | X |------| + * FC | X | X | P | X | X | X | X |------| * RST | X | X | X | X | X | X | O | O | * ------|------|------|------|------|------|------|------|------| * diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 6afd39aea110..64a1236aabee 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -380,6 +380,7 @@ static void mptcp_pm_add_addr_timer(struct timer_list *timer) struct mptcp_sock *msk = entry->sock; struct sock *sk = (struct sock *)msk; unsigned int timeout = 0; + bool retransmit; pr_debug("msk=%p\n", msk); @@ -412,14 +413,15 @@ static void mptcp_pm_add_addr_timer(struct timer_list *timer) entry->retrans_times++; } - if (entry->retrans_times < ADD_ADDR_RETRANS_MAX) + retransmit = entry->retrans_times < ADD_ADDR_RETRANS_MAX; + if (retransmit) timeout <<= entry->retrans_times; else timeout = 0; spin_unlock_bh(&msk->pm.lock); - if (entry->retrans_times == ADD_ADDR_RETRANS_MAX) + if (!retransmit) mptcp_pm_subflow_established(msk); out: @@ -441,6 +443,9 @@ bool mptcp_pm_announced_alloc(struct mptcp_sock *msk, lockdep_assert_held(&msk->pm.lock); + if (msk->pm.status & BIT(MPTCP_PM_DESTROYING)) + return false; + add_entry = mptcp_pm_announced_lookup(msk, addr); if (add_entry) { if (WARN_ON_ONCE(mptcp_pm_is_kernel(msk))) @@ -1143,10 +1148,16 @@ void mptcp_pm_worker(struct mptcp_sock *msk) void mptcp_pm_destroy(struct mptcp_sock *msk) { + spin_lock_bh(&msk->pm.lock); + msk->pm.status |= BIT(MPTCP_PM_DESTROYING); + spin_unlock_bh(&msk->pm.lock); + mptcp_pm_free_announced_list(msk); - if (mptcp_pm_is_userspace(msk)) - mptcp_userspace_pm_free_local_addr_list(msk); + /* Free the userspace local address list unconditionally: the socket + * can be reused (mptcp_disconnect()) and re-selected to a different PM + */ + mptcp_userspace_pm_free_local_addr_list(msk); } void mptcp_pm_data_reset(struct mptcp_sock *msk) diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c index d100867e9202..2203cc2d2748 100644 --- a/net/mptcp/pm_userspace.c +++ b/net/mptcp/pm_userspace.c @@ -54,6 +54,10 @@ static int mptcp_userspace_pm_append_new_local_addr(struct mptcp_sock *msk, bitmap_zero(id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); spin_lock_bh(&msk->pm.lock); + if (msk->pm.status & BIT(MPTCP_PM_DESTROYING)) { + ret = -EINVAL; + goto append_err; + } mptcp_for_each_userspace_pm_addr(msk, e) { addr_match = mptcp_addresses_equal(&e->addr, &entry->addr, true); if (addr_match && entry->addr.id == 0 && needs_id) @@ -132,12 +136,15 @@ int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk, __be16 msk_sport = ((struct inet_sock *) inet_sk((struct sock *)msk))->inet_sport; struct mptcp_pm_addr_entry *entry; + int id; spin_lock_bh(&msk->pm.lock); entry = mptcp_userspace_pm_lookup_addr(msk, &skc->addr); + id = entry ? entry->addr.id : -1; spin_unlock_bh(&msk->pm.lock); - if (entry) - return entry->addr.id; + + if (id != -1) + return id; if (skc->addr.port == msk_sport) skc->addr.port = 0; diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index cb9515f505aa..7c8180d8d5ef 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -149,6 +149,12 @@ struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk) static void mptcp_drop(struct sock *sk, struct sk_buff *skb) { + /* The skb forward memory was already transferred to sk by + * mptcp_borrow_fwdmem(), even before setting the destructor. + */ + if (!skb->destructor) + sk_mem_reclaim(sk); + sk_drops_skbadd(sk, skb); __kfree_skb(skb); } @@ -2545,6 +2551,22 @@ static void __mptcp_subflow_disconnect(struct sock *ssk, } } +static void mptcp_cleanup_ssk_backlog(struct sock *sk, struct sock *ssk) +{ + struct mptcp_sock *msk = mptcp_sk(sk); + struct sk_buff *skb; + + mptcp_data_lock(sk); + list_for_each_entry(skb, &msk->backlog_list, list) { + if (skb->sk != ssk) + continue; + + atomic_sub(skb->truesize, &skb->sk->sk_rmem_alloc); + skb->sk = NULL; + } + mptcp_data_unlock(sk); +} + /* subflow sockets can be either outgoing (connect) or incoming * (accept). * @@ -2568,6 +2590,9 @@ static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); subflow->closing = 1; + if (flags & MPTCP_CF_PUSH) + mptcp_cleanup_ssk_backlog(sk, ssk); + /* Borrow the fwd allocated page left-over; fwd memory for the subflow * could be negative at this point, but will be reach zero soon - when * the data allocated using such fragment will be freed. @@ -2659,9 +2684,6 @@ out: void mptcp_close_ssk(struct sock *sk, struct sock *ssk, struct mptcp_subflow_context *subflow) { - struct mptcp_sock *msk = mptcp_sk(sk); - struct sk_buff *skb; - /* The first subflow can already be closed or disconnected */ if (subflow->close_event_done || READ_ONCE(subflow->local_id) < 0) return; @@ -2671,17 +2693,6 @@ void mptcp_close_ssk(struct sock *sk, struct sock *ssk, if (sk->sk_state == TCP_ESTABLISHED) mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); - /* Remove any reference from the backlog to this ssk; backlog skbs consume - * space in the msk receive queue, no need to touch sk->sk_rmem_alloc - */ - list_for_each_entry(skb, &msk->backlog_list, list) { - if (skb->sk != ssk) - continue; - - atomic_sub(skb->truesize, &skb->sk->sk_rmem_alloc); - skb->sk = NULL; - } - /* subflow aborted before reaching the fully_established status * attempt the creation of the next subflow */ @@ -3907,6 +3918,7 @@ bool mptcp_finish_join(struct sock *ssk) mptcp_data_unlock(parent); if (!ret) { + mptcp_pm_close_subflow(msk); err_prohibited: subflow->reset_reason = MPTCP_RST_EPROHIBIT; return false; diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 4a2d40cd7b13..1b80f2d6ec5a 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -37,6 +37,7 @@ OPTION_MPTCP_MPC_ACK) #define OPTIONS_MPTCP_MPJ (OPTION_MPTCP_MPJ_SYN | OPTION_MPTCP_MPJ_SYNACK | \ OPTION_MPTCP_MPJ_ACK) +#define OPTIONS_MPTCP_DSS (OPTION_MPTCP_DSS | OPTION_MPTCP_CSUMREQD) /* MPTCP option subtypes */ #define MPTCPOPT_MP_CAPABLE 0 @@ -189,9 +190,10 @@ enum mptcp_pm_status { MPTCP_PM_ESTABLISHED, MPTCP_PM_SUBFLOW_ESTABLISHED, MPTCP_PM_ALREADY_ESTABLISHED, /* persistent status, set after ESTABLISHED event */ - MPTCP_PM_MPC_ENDPOINT_ACCOUNTED /* persistent status, set after MPC local address is - * accounted int id_avail_bitmap - */ + MPTCP_PM_MPC_ENDPOINT_ACCOUNTED, /* persistent status, set after MPC local address is + * accounted int id_avail_bitmap + */ + MPTCP_PM_DESTROYING, /* To fence out PM list allocs */ }; enum mptcp_pm_type { diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 8e386899ceb9..e1f20ff8fdb4 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -174,8 +174,6 @@ static int subflow_check_req(struct request_sock *req, if (unlikely(listener->pm_listener)) return subflow_reset_req_endp(req, skb); - if (opt_mp_join) - return 0; } else if (opt_mp_join) { SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINSYNRX); @@ -277,9 +275,6 @@ int mptcp_subflow_init_cookie_req(struct request_sock *req, opt_mp_capable = !!(mp_opt.suboptions & OPTION_MPTCP_MPC_ACK); opt_mp_join = !!(mp_opt.suboptions & OPTION_MPTCP_MPJ_ACK); - if (opt_mp_capable && opt_mp_join) - return -EINVAL; - if (opt_mp_capable && listener->request_mptcp) { if (mp_opt.sndr_key == 0) return -EINVAL; diff --git a/net/ncsi/ncsi-netlink.c b/net/ncsi/ncsi-netlink.c index 2f872d064396..8cc538358f6a 100644 --- a/net/ncsi/ncsi-netlink.c +++ b/net/ncsi/ncsi-netlink.c @@ -461,6 +461,10 @@ static int ncsi_send_cmd_nl(struct sk_buff *msg, struct genl_info *info) nca.req_flags = NCSI_REQ_FLAG_NETLINK_DRIVEN; nca.info = info; nca.payload = ntohs(hdr->length); + if (nca.payload > len - sizeof(*hdr)) { + ret = -EINVAL; + goto out_netlink; + } nca.data = data + sizeof(*hdr); ret = ncsi_xmit_cmd(&nca); diff --git a/net/netfilter/ipset/ip_set_bitmap_gen.h b/net/netfilter/ipset/ip_set_bitmap_gen.h index bb9b5bed10e1..d6a7e6604542 100644 --- a/net/netfilter/ipset/ip_set_bitmap_gen.h +++ b/net/netfilter/ipset/ip_set_bitmap_gen.h @@ -77,7 +77,7 @@ mtype_flush(struct ip_set *set) mtype_ext_cleanup(set); bitmap_zero(map->members, map->elements); set->elements = 0; - set->ext_size = 0; + DEBUG_NET_WARN_ON_ONCE(atomic64_read(&set->ext_size) > 0); } /* Calculate the actual memory size of the set data */ @@ -93,7 +93,7 @@ mtype_head(struct ip_set *set, struct sk_buff *skb) { const struct mtype *map = set->data; struct nlattr *nested; - size_t memsize = mtype_memsize(map, set->dsize) + set->ext_size; + size_t memsize = mtype_memsize(map, set->dsize) + atomic64_read(&set->ext_size); nested = nla_nest_start(skb, IPSET_ATTR_DATA); if (!nested) diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c index 6cfad152d7d1..0a86a170ba90 100644 --- a/net/netfilter/ipset/ip_set_core.c +++ b/net/netfilter/ipset/ip_set_core.c @@ -25,6 +25,7 @@ static LIST_HEAD(ip_set_type_list); /* all registered set types */ static DEFINE_MUTEX(ip_set_type_mutex); /* protects ip_set_type_list */ static DEFINE_RWLOCK(ip_set_ref_lock); /* protects the set refs */ +static struct workqueue_struct *ipset_destroy_wq; struct ip_set_net { struct ip_set * __rcu *ip_set_list; /* all individual sets */ @@ -350,7 +351,7 @@ ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment, size_t len = ext->comment ? strlen(ext->comment) : 0; if (unlikely(c)) { - set->ext_size -= sizeof(*c) + strlen(c->str) + 1; + atomic64_sub(sizeof(*c) + strlen(c->str) + 1, &set->ext_size); rcu_assign_pointer(comment->c, NULL); kfree_rcu(c, rcu); } @@ -362,7 +363,7 @@ ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment, if (unlikely(!c)) return; strscpy(c->str, ext->comment, len + 1); - set->ext_size += sizeof(*c) + strlen(c->str) + 1; + atomic64_add(sizeof(*c) + strlen(c->str) + 1, &set->ext_size); rcu_assign_pointer(comment->c, c); } EXPORT_SYMBOL_GPL(ip_set_init_comment); @@ -392,7 +393,7 @@ ip_set_comment_free(struct ip_set *set, void *ptr) c = rcu_dereference_protected(comment->c, 1); if (unlikely(!c)) return; - set->ext_size -= sizeof(*c) + strlen(c->str) + 1; + atomic64_sub(sizeof(*c) + strlen(c->str) + 1, &set->ext_size); rcu_assign_pointer(comment->c, NULL); kfree_rcu(c, rcu); } @@ -679,11 +680,18 @@ __ip_set_get(struct ip_set *set) } static void -__ip_set_put(struct ip_set *set) +__ip_set_put_locked(struct ip_set *set) { - write_lock_bh(&ip_set_ref_lock); + lockdep_assert_held(&ip_set_ref_lock); BUG_ON(set->ref == 0); set->ref--; +} + +static void +__ip_set_put(struct ip_set *set) +{ + write_lock_bh(&ip_set_ref_lock); + __ip_set_put_locked(set); write_unlock_bh(&ip_set_ref_lock); } @@ -854,11 +862,11 @@ __ip_set_put_byindex(struct ip_set_net *inst, ip_set_id_t index) { struct ip_set *set; - rcu_read_lock(); - set = rcu_dereference(inst->ip_set_list)[index]; + write_lock_bh(&ip_set_ref_lock); + set = ip_set(inst, index); if (set) - __ip_set_put(set); - rcu_read_unlock(); + __ip_set_put_locked(set); + write_unlock_bh(&ip_set_ref_lock); } void @@ -1178,22 +1186,26 @@ ip_set_setname_policy[IPSET_ATTR_CMD_MAX + 1] = { .len = IPSET_MAXNAMELEN - 1 }, }; -/* In order to return quickly when destroying a single set, it is split - * into two stages: - * - Cancel garbage collector - * - Destroy the set itself via call_rcu() - */ - static void -ip_set_destroy_set_rcu(struct rcu_head *head) +destroy_and_free_set(struct ip_set *set) { - struct ip_set *set = container_of(head, struct ip_set, rcu); - set->variant->destroy(set); module_put(set->type->me); kfree(set); } +/* In order to return quickly when destroying a single set, + * destruction is done asynchronously via work queues. + */ +static void +ip_set_destroy_set_work(struct work_struct *work) +{ + struct ip_set *set = container_of(to_rcu_work(work), + struct ip_set, rwork); + + destroy_and_free_set(set); +} + static void _destroy_all_sets(struct ip_set_net *inst) { @@ -1283,7 +1295,8 @@ static int ip_set_destroy(struct sk_buff *skb, const struct nfnl_info *info, /* Must wait for flush to be really finished */ rcu_barrier(); } - call_rcu(&s->rcu, ip_set_destroy_set_rcu); + INIT_RCU_WORK(&s->rwork, ip_set_destroy_set_work); + queue_rcu_work(ipset_destroy_wq, &s->rwork); } return 0; out: @@ -2421,18 +2434,23 @@ static struct pernet_operations ip_set_net_ops = { static int __init ip_set_init(void) { - int ret = register_pernet_subsys(&ip_set_net_ops); + int ret; + ipset_destroy_wq = alloc_ordered_workqueue("ipset_destroy_wq", 0); + if (!ipset_destroy_wq) + return -ENOMEM; + + ret = register_pernet_subsys(&ip_set_net_ops); if (ret) { pr_err("ip_set: cannot register pernet_subsys.\n"); - return ret; + goto out_wq; } ret = nfnetlink_subsys_register(&ip_set_netlink_subsys); if (ret != 0) { pr_err("ip_set: cannot register with nfnetlink.\n"); unregister_pernet_subsys(&ip_set_net_ops); - return ret; + goto out_wq; } ret = nf_register_sockopt(&so_set); @@ -2440,10 +2458,13 @@ ip_set_init(void) pr_err("SO_SET registry failed: %d\n", ret); nfnetlink_subsys_unregister(&ip_set_netlink_subsys); unregister_pernet_subsys(&ip_set_net_ops); - return ret; + goto out_wq; } return 0; +out_wq: + destroy_workqueue(ipset_destroy_wq); + return ret; } static void __exit @@ -2453,9 +2474,7 @@ ip_set_fini(void) nfnetlink_subsys_unregister(&ip_set_netlink_subsys); unregister_pernet_subsys(&ip_set_net_ops); - /* Wait for call_rcu() in destroy */ - rcu_barrier(); - + destroy_workqueue(ipset_destroy_wq); pr_debug("these are the famous last words\n"); } diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index dedf59b661dd..f00c82acd7f0 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -75,15 +75,17 @@ struct hbucket { struct htable_gc { struct delayed_work dwork; struct ip_set *set; /* Set the gc belongs to */ + spinlock_t lock; /* Lock to exclude gc and resize */ u32 region; /* Last gc run position */ }; /* The hash table: the table size stored here in order to make resizing easy */ struct htable { - atomic_t ref; /* References for resizing */ + bool resizing; /* Mark ongoing resize */ atomic_t uref; /* References for dumping and gc */ u8 htable_bits; /* size of hash table == 2^htable_bits */ u32 maxelem; /* Maxelem per region */ + struct list_head ad; /* Resize add|del backlist */ struct ip_set_region *hregion; /* Region locks and ext sizes */ struct hbucket __rcu *bucket[]; /* hashtable buckets */ }; @@ -97,9 +99,15 @@ struct htable { #endif /* Book-keeping of the prefixes added to the set */ +struct net_prefix { + u8 cidr; /* the cidr value */ + u32 count; /* number of elements of this cidr */ +}; + struct net_prefixes { - u32 nets[IPSET_NET_COUNT]; /* number of elements for this cidr */ - u8 cidr[IPSET_NET_COUNT]; /* the cidr value */ + struct rcu_head rcu; + u8 len; + struct net_prefix nets[] __counted_by(len); }; /* Compute the hash table size */ @@ -125,11 +133,6 @@ htable_size(u8 hbits) #else #define __CIDR(cidr, i) (cidr) #endif - -/* cidr + 1 is stored in net_prefixes to support /0 */ -#define NCIDR_PUT(cidr) ((cidr) + 1) -#define NCIDR_GET(cidr) ((cidr) - 1) - #ifdef IP_SET_HASH_WITH_NETS_PACKED /* When cidr is packed with nomatch, cidr - 1 is stored in the data entry */ #define DCIDR_PUT(cidr) ((cidr) - 1) @@ -139,21 +142,11 @@ htable_size(u8 hbits) #define DCIDR_GET(cidr, i) __CIDR(cidr, i) #endif -#define INIT_CIDR(cidr, host_mask) \ - DCIDR_PUT(((cidr) ? NCIDR_GET(cidr) : host_mask)) - -#ifdef IP_SET_HASH_WITH_NET0 -/* cidr from 0 to HOST_MASK value and c = cidr + 1 */ -#define NLEN (HOST_MASK + 1) -#define CIDR_POS(c) ((c) - 1) -#else -/* cidr from 1 to HOST_MASK value and c = cidr + 1 */ -#define NLEN HOST_MASK -#define CIDR_POS(c) ((c) - 2) -#endif +#define INIT_CIDR(n, host_mask) ({ \ + const struct net_prefixes *__n = rcu_dereference(n); \ + DCIDR_PUT((__n)->len ? (__n)->nets[0].cidr : host_mask);\ +}) -#else -#define NLEN 0 #endif /* IP_SET_HASH_WITH_NETS */ #define SET_ELEM_EXPIRED(set, d) \ @@ -202,12 +195,15 @@ static const union nf_inet_addr zeromask = {}; #undef mtype_ext_cleanup #undef mtype_add_cidr #undef mtype_del_cidr +#undef mtype_del_cidr_all #undef mtype_ahash_memsize #undef mtype_flush #undef mtype_destroy #undef mtype_same_set #undef mtype_kadt #undef mtype_uadt +#undef mtype_bucket_size +#undef mtype_hash_size #undef mtype_add #undef mtype_del @@ -247,12 +243,15 @@ static const union nf_inet_addr zeromask = {}; #define mtype_ext_cleanup IPSET_TOKEN(MTYPE, _ext_cleanup) #define mtype_add_cidr IPSET_TOKEN(MTYPE, _add_cidr) #define mtype_del_cidr IPSET_TOKEN(MTYPE, _del_cidr) +#define mtype_del_cidr_all IPSET_TOKEN(MTYPE, _del_cidr_all) #define mtype_ahash_memsize IPSET_TOKEN(MTYPE, _ahash_memsize) #define mtype_flush IPSET_TOKEN(MTYPE, _flush) #define mtype_destroy IPSET_TOKEN(MTYPE, _destroy) #define mtype_same_set IPSET_TOKEN(MTYPE, _same_set) #define mtype_kadt IPSET_TOKEN(MTYPE, _kadt) #define mtype_uadt IPSET_TOKEN(MTYPE, _uadt) +#define mtype_bucket_size IPSET_TOKEN(MTYPE, _bucket_size) +#define mtype_hash_size IPSET_TOKEN(MTYPE, _hash_size) #define mtype_add IPSET_TOKEN(MTYPE, _add) #define mtype_del IPSET_TOKEN(MTYPE, _del) @@ -290,6 +289,7 @@ static const union nf_inet_addr zeromask = {}; /* The generic hash structure */ struct htype { struct htable __rcu *table; /* the hash table */ + struct net_prefixes __rcu *rnets[IPSET_NET_COUNT]; /* cidr prefixes */ struct htable_gc gc; /* gc workqueue */ u32 maxelem; /* max elements in the hash */ u32 initval; /* random jhash init value */ @@ -301,11 +301,10 @@ struct htype { u8 netmask; /* netmask value for subnets to store */ union nf_inet_addr bitmask; /* stores bitmask */ #endif - struct list_head ad; /* Resize add|del backlist */ + /* Because 'next' is IPv4/IPv6 dependent, no elements of this + * structure and referred in create() may come after 'next'. + */ struct mtype_elem next; /* temporary storage for uadd */ -#ifdef IP_SET_HASH_WITH_NETS - struct net_prefixes nets[NLEN]; /* book-keeping of prefixes */ -#endif }; /* ADD|DEL entries saved during resize */ @@ -322,55 +321,108 @@ struct mtype_resize_ad { /* Network cidr size book keeping when the hash stores different * sized networks. cidr == real cidr + 1 to support /0. */ -static void +static int mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) { - int i, j; + struct net_prefixes *nets, *tmp; + int i, j, found, len = 0, ret = 0; spin_lock_bh(&set->lock); + nets = __ipset_dereference(h->rnets[n]); /* Add in increasing prefix order, so larger cidr first */ - for (i = 0, j = -1; i < NLEN && h->nets[i].cidr[n]; i++) { - if (j != -1) { + for (i = 0, found = -1; i < nets->len; i++) { + if (nets->nets[i].count) + len++; + if (found != -1) { continue; - } else if (h->nets[i].cidr[n] < cidr) { - j = i; - } else if (h->nets[i].cidr[n] == cidr) { - h->nets[CIDR_POS(cidr)].nets[n]++; + } else if (nets->nets[i].cidr < cidr) { + found = i; + } else if (nets->nets[i].cidr == cidr) { + nets->nets[i].count++; goto unlock; } } - if (j != -1) { - for (; i > j; i--) - h->nets[i].cidr[n] = h->nets[i - 1].cidr[n]; + len++; + tmp = kzalloc_flex(*tmp, nets, len, GFP_ATOMIC); + if (!tmp) { + ret = -ENOMEM; + goto unlock; + } + + tmp->len = len; + for (i = 0, j = 0; i < nets->len; i++) { + if (i == found) { + tmp->nets[j].cidr = cidr; + tmp->nets[j++].count = 1; + } + if (!nets->nets[i].count) + continue; + tmp->nets[j].cidr = nets->nets[i].cidr; + tmp->nets[j++].count = nets->nets[i].count; + } + if (found == -1) { + tmp->nets[j].cidr = cidr; + tmp->nets[j].count = 1; } - h->nets[i].cidr[n] = cidr; - h->nets[CIDR_POS(cidr)].nets[n] = 1; + rcu_assign_pointer(h->rnets[n], tmp); + kfree_rcu(nets, rcu); unlock: spin_unlock_bh(&set->lock); + return ret; } static void mtype_del_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) { - u8 i, j, net_end = NLEN - 1; + struct net_prefixes *nets, *tmp; + u8 i, j, len = 0; + int found; spin_lock_bh(&set->lock); - for (i = 0; i < NLEN; i++) { - if (h->nets[i].cidr[n] != cidr) - continue; - h->nets[CIDR_POS(cidr)].nets[n]--; - if (h->nets[CIDR_POS(cidr)].nets[n] > 0) - goto unlock; - for (j = i; j < net_end && h->nets[j].cidr[n]; j++) - h->nets[j].cidr[n] = h->nets[j + 1].cidr[n]; - h->nets[j].cidr[n] = 0; + nets = __ipset_dereference(h->rnets[n]); + for (i = 0, found = -1; i < nets->len; i++) { + if (nets->nets[i].count) + len++; + if (nets->nets[i].cidr == cidr) + found = i; + } + if (unlikely(found == -1)) + goto unlock; + + nets->nets[found].count--; + if (nets->nets[found].count) + goto unlock; + len--; + tmp = kzalloc_flex(*tmp, nets, len, GFP_ATOMIC); + if (!tmp) + /* Leave a hole */ goto unlock; + + tmp->len = len; + for (i = 0, j = 0; i < nets->len; i++) { + if (!nets->nets[i].count || i == found) + continue; + tmp->nets[j].cidr = nets->nets[i].cidr; + tmp->nets[j++].count = nets->nets[i].count; } + rcu_assign_pointer(h->rnets[n], tmp); + kfree_rcu(nets, rcu); unlock: spin_unlock_bh(&set->lock); } #endif +static void +mtype_del_cidr_all(struct ip_set *set, struct htype *h, const struct mtype_elem *data) +{ +#ifdef IP_SET_HASH_WITH_NETS + int k; + + for (k = 0; k < IPSET_NET_COUNT; k++) + mtype_del_cidr(set, h, DCIDR_GET(data->cidr, k), k); +#endif +} + /* Calculate the actual memory size of the set data */ static size_t mtype_ahash_memsize(const struct htype *h, const struct htable *t) @@ -398,6 +450,9 @@ static void mtype_flush(struct ip_set *set) { struct htype *h = set->data; +#ifdef IP_SET_HASH_WITH_NETS + struct net_prefixes *nets, *tmp; +#endif struct htable *t; struct hbucket *n; u32 r, i; @@ -421,7 +476,19 @@ mtype_flush(struct ip_set *set) spin_unlock_bh(&t->hregion[r].lock); } #ifdef IP_SET_HASH_WITH_NETS - memset(h->nets, 0, sizeof(h->nets)); + for (i = 0; i < IPSET_NET_COUNT; i++) { + nets = ipset_dereference_nfnl(h->rnets[i]); + tmp = kzalloc_obj(*tmp, GFP_ATOMIC); + if (!tmp) { + u8 j; + + for (j = 0; j < nets->len; j++) + nets->nets[j].count = 0; + } else { + rcu_assign_pointer(h->rnets[i], tmp); + kfree_rcu(nets, rcu); + } + } #endif } @@ -429,6 +496,9 @@ mtype_flush(struct ip_set *set) static void mtype_ahash_destroy(struct ip_set *set, struct htable *t, bool ext_destroy) { +#ifdef IP_SET_HASH_WITH_NETS + struct htype *h = set->data; +#endif struct hbucket *n; u32 i; @@ -442,6 +512,11 @@ mtype_ahash_destroy(struct ip_set *set, struct htable *t, bool ext_destroy) kfree(n); } +#ifdef IP_SET_HASH_WITH_NETS + if (ext_destroy) + for (i = 0; i < IPSET_NET_COUNT; i++) + kfree(rcu_dereference_raw(h->rnets[i])); +#endif ip_set_free(t->hregion); ip_set_free(t); } @@ -451,13 +526,14 @@ static void mtype_destroy(struct ip_set *set) { struct htype *h = set->data; + struct htable *t = (__force struct htable *)h->table; struct list_head *l, *lt; - mtype_ahash_destroy(set, (__force struct htable *)h->table, true); - list_for_each_safe(l, lt, &h->ad) { + list_for_each_safe(l, lt, &t->ad) { list_del(l); kfree(l); } + mtype_ahash_destroy(set, t, true); kfree(h); set->data = NULL; @@ -488,9 +564,6 @@ mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r) struct mtype_elem *data; u32 i, j, d; size_t dsize = set->dsize; -#ifdef IP_SET_HASH_WITH_NETS - u8 k; -#endif u8 pos, htable_bits = t->htable_bits; spin_lock_bh(&t->hregion[r].lock); @@ -511,12 +584,7 @@ mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r) pr_debug("expired %u/%u\n", i, j); clear_bit(j, n->used); smp_mb__after_atomic(); -#ifdef IP_SET_HASH_WITH_NETS - for (k = 0; k < IPSET_NET_COUNT; k++) - mtype_del_cidr(set, h, - NCIDR_PUT(DCIDR_GET(data->cidr, k)), - k); -#endif + mtype_del_cidr_all(set, h, data); t->hregion[r].elements--; ip_set_ext_destroy(set, data); d++; @@ -569,9 +637,10 @@ mtype_gc(struct work_struct *work) set = gc->set; h = set->data; - spin_lock_bh(&set->lock); - t = ipset_dereference_set(h->table, set); + rcu_read_lock_bh(); + t = rcu_dereference_bh(h->table); atomic_inc(&t->uref); + rcu_read_unlock_bh(); numof_locks = ahash_numof_locks(t->htable_bits); r = gc->region++; if (r >= numof_locks) { @@ -580,11 +649,13 @@ mtype_gc(struct work_struct *work) next_run = (IPSET_GC_PERIOD(set->timeout) * HZ) / numof_locks; if (next_run < HZ/10) next_run = HZ/10; - spin_unlock_bh(&set->lock); - mtype_gc_do(set, h, t, r); + spin_lock_bh(&gc->lock); + if (!t->resizing) + mtype_gc_do(set, h, t, r); + spin_unlock_bh(&gc->lock); - if (atomic_dec_and_test(&t->uref) && atomic_read(&t->ref)) { + if (atomic_dec_and_test(&t->uref) && t->resizing) { pr_debug("Table destroy after resize by expire: %p\n", t); mtype_ahash_destroy(set, t, false); } @@ -668,15 +739,18 @@ retry: } t->htable_bits = htable_bits; t->maxelem = h->maxelem / ahash_numof_locks(htable_bits); + INIT_LIST_HEAD(&t->ad); for (i = 0; i < ahash_numof_locks(htable_bits); i++) spin_lock_init(&t->hregion[i].lock); /* There can't be another parallel resizing, - * but dumping, gc, kernel side add/del are possible + * but dumping and kernel side add/del are possible */ orig = ipset_dereference_bh_nfnl(h->table); - atomic_set(&orig->ref, 1); atomic_inc(&orig->uref); + spin_lock_bh(&h->gc.lock); + orig->resizing = true; + spin_unlock_bh(&h->gc.lock); pr_debug("attempt to resize set %s from %u to %u, t %p\n", set->name, orig->htable_bits, htable_bits, orig); for (r = 0; r < ahash_numof_locks(orig->htable_bits); r++) { @@ -768,7 +842,7 @@ retry: * Kernel-side add cannot trigger a resize and userspace actions * are serialized by the mutex. */ - list_for_each_safe(l, lt, &h->ad) { + list_for_each_safe(l, lt, &orig->ad) { x = list_entry(l, struct mtype_resize_ad, list); if (x->ad == IPSET_ADD) { mtype_add(set, &x->d, &x->ext, &x->mext, x->flags); @@ -792,11 +866,24 @@ out: cleanup: rcu_read_unlock_bh(); - atomic_set(&orig->ref, 0); + spin_lock_bh(&h->gc.lock); + orig->resizing = false; + spin_unlock_bh(&h->gc.lock); + /* Make sure parallel readers see that orig->resizing is false + * before we decrement uref */ + synchronize_rcu(); atomic_dec(&orig->uref); mtype_ahash_destroy(set, t, false); if (ret == -EAGAIN) goto retry; + + /* Cleanup the backlog of ADD/DEL elements */ + spin_lock_bh(&set->lock); + list_for_each_safe(l, lt, &orig->ad) { + list_del(l); + kfree(l); + } + spin_unlock_bh(&set->lock); goto out; hbwarn: @@ -860,15 +947,13 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, key = HKEY(value, h->initval, t->htable_bits); r = ahash_region(key); atomic_inc(&t->uref); + rcu_read_unlock_bh(); elements = t->hregion[r].elements; maxelem = t->maxelem; if (elements >= maxelem) { u32 e; - if (SET_WITH_TIMEOUT(set)) { - rcu_read_unlock_bh(); + if (SET_WITH_TIMEOUT(set)) mtype_gc_do(set, h, t, r); - rcu_read_lock_bh(); - } maxelem = h->maxelem; elements = 0; for (e = 0; e < ahash_numof_locks(t->htable_bits); e++) @@ -876,7 +961,6 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, if (elements >= maxelem && SET_WITH_FORCEADD(set)) forceadd = true; } - rcu_read_unlock_bh(); spin_lock_bh(&t->hregion[r].lock); n = rcu_dereference_bh(hbucket(t, key)); @@ -926,12 +1010,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, j = 0; data = ahash_data(n, j, set->dsize); if (!deleted) { -#ifdef IP_SET_HASH_WITH_NETS - for (i = 0; i < IPSET_NET_COUNT; i++) - mtype_del_cidr(set, h, - NCIDR_PUT(DCIDR_GET(data->cidr, i)), - i); -#endif + mtype_del_cidr_all(set, h, data); ip_set_ext_destroy(set, data); t->hregion[r].elements--; } @@ -975,7 +1054,7 @@ copy_data: t->hregion[r].elements++; #ifdef IP_SET_HASH_WITH_NETS for (i = 0; i < IPSET_NET_COUNT; i++) - mtype_add_cidr(set, h, NCIDR_PUT(DCIDR_GET(d->cidr, i)), i); + mtype_add_cidr(set, h, DCIDR_GET(d->cidr, i), i); #endif memcpy(data, d, sizeof(struct mtype_elem)); overwrite_extensions: @@ -984,7 +1063,7 @@ overwrite_extensions: #endif if (SET_WITH_COUNTER(set)) ip_set_init_counter(ext_counter(data, set), ext); - if (SET_WITH_COMMENT(set)) + if (SET_WITH_COMMENT(set) && !ext->target) ip_set_init_comment(set, ext_comment(data, set), ext); if (SET_WITH_SKBINFO(set)) ip_set_init_skbinfo(ext_skbinfo(data, set), ext); @@ -1003,7 +1082,7 @@ overwrite_extensions: ret = 0; resize: spin_unlock_bh(&t->hregion[r].lock); - if (atomic_read(&t->ref) && ext->target) { + if (t->resizing && ext && ext->target) { /* Resize is in process and kernel side add, save values */ struct mtype_resize_ad *x; @@ -1017,7 +1096,7 @@ resize: memcpy(&x->mext, mext, sizeof(struct ip_set_ext)); x->flags = flags; spin_lock_bh(&set->lock); - list_add_tail(&x->list, &h->ad); + list_add_tail(&x->list, &t->ad); spin_unlock_bh(&set->lock); } goto out; @@ -1030,7 +1109,7 @@ set_full: unlock: spin_unlock_bh(&t->hregion[r].lock); out: - if (atomic_dec_and_test(&t->uref) && atomic_read(&t->ref)) { + if (atomic_dec_and_test(&t->uref) && t->resizing) { pr_debug("Table destroy after resize by add: %p\n", t); mtype_ahash_destroy(set, t, false); } @@ -1086,14 +1165,10 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext, if (i + 1 == pos) smp_store_release(&n->pos, --pos); t->hregion[r].elements--; -#ifdef IP_SET_HASH_WITH_NETS - for (j = 0; j < IPSET_NET_COUNT; j++) - mtype_del_cidr(set, h, - NCIDR_PUT(DCIDR_GET(d->cidr, j)), j); -#endif + mtype_del_cidr_all(set, h, d); ip_set_ext_destroy(set, data); - if (atomic_read(&t->ref) && ext->target) { + if (t->resizing && ext && ext->target) { /* Resize is in process and kernel side del, * save values */ @@ -1141,10 +1216,10 @@ out: spin_unlock_bh(&t->hregion[r].lock); if (x) { spin_lock_bh(&set->lock); - list_add(&x->list, &h->ad); + list_add(&x->list, &t->ad); spin_unlock_bh(&set->lock); } - if (atomic_dec_and_test(&t->uref) && atomic_read(&t->ref)) { + if (atomic_dec_and_test(&t->uref) && t->resizing) { pr_debug("Table destroy after resize by del: %p\n", t); mtype_ahash_destroy(set, t, false); } @@ -1172,28 +1247,37 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, { struct htype *h = set->data; struct htable *t = rcu_dereference_bh(h->table); + struct net_prefixes *nets0; struct hbucket *n; struct mtype_elem *data; #if IPSET_NET_COUNT == 2 + struct net_prefixes *nets1; struct mtype_elem orig = *d; - int ret, i, j = 0, k; + int ret, i, j, k; #else - int ret, i, j = 0; + int ret, i, j; #endif u32 key, multi = 0; u8 pos; pr_debug("test by nets\n"); - for (; j < NLEN && h->nets[j].cidr[0] && !multi; j++) { + rcu_read_lock_bh(); + nets0 = rcu_dereference_bh(h->rnets[0]); +#if IPSET_NET_COUNT == 2 + nets1 = rcu_dereference_bh(h->rnets[1]); +#endif + for (j = 0; j < nets0->len && !multi; j++) { + if (!nets0->nets[j].count) + continue; #if IPSET_NET_COUNT == 2 mtype_data_reset_elem(d, &orig); - mtype_data_netmask(d, NCIDR_GET(h->nets[j].cidr[0]), false); - for (k = 0; k < NLEN && h->nets[k].cidr[1] && !multi; - k++) { - mtype_data_netmask(d, NCIDR_GET(h->nets[k].cidr[1]), - true); + mtype_data_netmask(d, nets0->nets[j].cidr, false); + for (k = 0; k < nets1->len && !multi; k++) { + if (!nets1->nets[k].count) + continue; + mtype_data_netmask(d, nets1->nets[k].cidr, true); #else - mtype_data_netmask(d, NCIDR_GET(h->nets[j].cidr[0])); + mtype_data_netmask(d, nets0->nets[j].cidr); #endif key = HKEY(d, h->initval, t->htable_bits); n = rcu_dereference_bh(hbucket(t, key)); @@ -1208,7 +1292,7 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, continue; ret = mtype_data_match(data, ext, mext, set, flags); if (ret != 0) - return ret; + goto unlock; #ifdef IP_SET_HASH_WITH_MULTI /* No match, reset multiple match flag */ multi = 0; @@ -1218,7 +1302,10 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d, } #endif } - return 0; + ret = 0; +unlock: + rcu_read_unlock_bh(); + return ret; } #endif @@ -1273,6 +1360,24 @@ out: return ret; } +static u32 mtype_hash_size(const struct htype *h) +{ + const struct htable *t; + u8 htable_bits; + + rcu_read_lock(); + t = rcu_dereference(h->table); + htable_bits = t->htable_bits; + rcu_read_unlock(); + + return jhash_size(htable_bits); +} + +static u32 mtype_bucket_size(const struct htype *h) +{ + return h->bucketsize; +} + /* Reply a HEADER request: fill out the header part of the set */ static int mtype_head(struct ip_set *set, struct sk_buff *skb) @@ -1283,21 +1388,20 @@ mtype_head(struct ip_set *set, struct sk_buff *skb) size_t memsize; u32 elements = 0; size_t ext_size = 0; - u8 htable_bits; rcu_read_lock_bh(); t = rcu_dereference_bh(h->table); mtype_ext_size(set, &elements, &ext_size); - memsize = mtype_ahash_memsize(h, t) + ext_size + set->ext_size; - htable_bits = t->htable_bits; + memsize = mtype_ahash_memsize(h, t) + ext_size + atomic64_read(&set->ext_size); rcu_read_unlock_bh(); nested = nla_nest_start(skb, IPSET_ATTR_DATA); if (!nested) goto nla_put_failure; - if (nla_put_net32(skb, IPSET_ATTR_HASHSIZE, - htonl(jhash_size(htable_bits))) || - nla_put_net32(skb, IPSET_ATTR_MAXELEM, htonl(h->maxelem))) + + if (nla_put_net32(skb, IPSET_ATTR_HASHSIZE, htonl(mtype_hash_size(h)))) + goto nla_put_failure; + if (nla_put_net32(skb, IPSET_ATTR_MAXELEM, htonl(h->maxelem))) goto nla_put_failure; #ifdef IP_SET_HASH_WITH_BITMASK /* if netmask is set to anything other than HOST_MASK we know that the user supplied netmask @@ -1321,8 +1425,9 @@ mtype_head(struct ip_set *set, struct sk_buff *skb) goto nla_put_failure; #endif if (set->flags & IPSET_CREATE_FLAG_BUCKETSIZE) { - if (nla_put_u8(skb, IPSET_ATTR_BUCKETSIZE, h->bucketsize) || - nla_put_net32(skb, IPSET_ATTR_INITVAL, htonl(h->initval))) + if (nla_put_u8(skb, IPSET_ATTR_BUCKETSIZE, mtype_bucket_size(h))) + goto nla_put_failure; + if (nla_put_net32(skb, IPSET_ATTR_INITVAL, htonl(h->initval))) goto nla_put_failure; } if (nla_put_net32(skb, IPSET_ATTR_REFERENCES, htonl(set->ref)) || @@ -1353,7 +1458,7 @@ mtype_uref(struct ip_set *set, struct netlink_callback *cb, bool start) rcu_read_unlock_bh(); } else if (cb->args[IPSET_CB_PRIVATE]) { t = (struct htable *)cb->args[IPSET_CB_PRIVATE]; - if (atomic_dec_and_test(&t->uref) && atomic_read(&t->ref)) { + if (atomic_dec_and_test(&t->uref) && t->resizing) { pr_debug("Table destroy after resize " " by dump: %p\n", t); mtype_ahash_destroy(set, t, false); @@ -1484,6 +1589,9 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set, u8 netmask = set->family == NFPROTO_IPV4 ? 32 : 128; union nf_inet_addr bitmask = onesmask; #endif +#ifdef IP_SET_HASH_WITH_NETS + struct net_prefixes *nets; +#endif size_t hsize; struct htype *h; struct htable *t; @@ -1566,7 +1674,13 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set, if (tb[IPSET_ATTR_MAXELEM]) maxelem = ip_set_get_h32(tb[IPSET_ATTR_MAXELEM]); - hsize = sizeof(*h); +#ifdef IP_SET_PROTO_UNDEF + hsize = sizeof(struct htype); +#else + hsize = set->family == NFPROTO_IPV6 ? + sizeof(struct IPSET_TOKEN(HTYPE, 6)) : + sizeof(struct IPSET_TOKEN(HTYPE, 4)); +#endif h = kzalloc(hsize, GFP_KERNEL); if (!h) return -ENOMEM; @@ -1577,22 +1691,27 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set, */ hbits = fls(hashsize - 1); hsize = htable_size(hbits); - if (hsize == 0) { - kfree(h); - return -ENOMEM; - } + if (hsize == 0) + goto free_h; t = ip_set_alloc(hsize); - if (!t) { - kfree(h); - return -ENOMEM; - } + if (!t) + goto free_h; t->hregion = ip_set_alloc(ahash_sizeof_regions(hbits)); - if (!t->hregion) { - ip_set_free(t); - kfree(h); - return -ENOMEM; + if (!t->hregion) + goto free_t; +#ifdef IP_SET_HASH_WITH_NETS + for (i = 0; i < IPSET_NET_COUNT; i++) { + nets = kzalloc_obj(*nets); + if (!nets) { + while (i > 0) + kfree(rcu_dereference_raw(h->rnets[--i])); + goto free_hregion; + } + RCU_INIT_POINTER(h->rnets[i], nets); } +#endif h->gc.set = set; + spin_lock_init(&h->gc.lock); for (i = 0; i < ahash_numof_locks(hbits); i++) spin_lock_init(&t->hregion[i].lock); h->maxelem = maxelem; @@ -1619,10 +1738,10 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set, } t->htable_bits = hbits; t->maxelem = h->maxelem / ahash_numof_locks(hbits); + INIT_LIST_HEAD(&t->ad); RCU_INIT_POINTER(h->table, t); - - INIT_LIST_HEAD(&h->ad); set->data = h; + #ifndef IP_SET_PROTO_UNDEF if (set->family == NFPROTO_IPV4) { #endif @@ -1651,10 +1770,20 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set, #endif } pr_debug("create %s hashsize %u (%u) maxelem %u: %p(%p)\n", - set->name, jhash_size(t->htable_bits), + set->name, mtype_hash_size(h), t->htable_bits, h->maxelem, set->data, t); return 0; + +#ifdef IP_SET_HASH_WITH_NETS +free_hregion: + ip_set_free(t->hregion); +#endif +free_t: + ip_set_free(t); +free_h: + kfree(h); + return -ENOMEM; } #endif /* IP_SET_EMIT_CREATE */ diff --git a/net/netfilter/ipset/ip_set_hash_ipportnet.c b/net/netfilter/ipset/ip_set_hash_ipportnet.c index 2d6652d43199..195853a25b06 100644 --- a/net/netfilter/ipset/ip_set_hash_ipportnet.c +++ b/net/netfilter/ipset/ip_set_hash_ipportnet.c @@ -138,7 +138,7 @@ hash_ipportnet4_kadt(struct ip_set *set, const struct sk_buff *skb, const struct hash_ipportnet4 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_ipportnet4_elem e = { - .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK), + .cidr = INIT_CIDR(h->rnets[0], HOST_MASK), }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); @@ -398,7 +398,7 @@ hash_ipportnet6_kadt(struct ip_set *set, const struct sk_buff *skb, const struct hash_ipportnet6 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_ipportnet6_elem e = { - .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK), + .cidr = INIT_CIDR(h->rnets[0], HOST_MASK), }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); diff --git a/net/netfilter/ipset/ip_set_hash_net.c b/net/netfilter/ipset/ip_set_hash_net.c index ce0a9ce5a91f..092f3c9281b8 100644 --- a/net/netfilter/ipset/ip_set_hash_net.c +++ b/net/netfilter/ipset/ip_set_hash_net.c @@ -117,7 +117,7 @@ hash_net4_kadt(struct ip_set *set, const struct sk_buff *skb, const struct hash_net4 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_net4_elem e = { - .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK), + .cidr = INIT_CIDR(h->rnets[0], HOST_MASK), }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); @@ -291,7 +291,7 @@ hash_net6_kadt(struct ip_set *set, const struct sk_buff *skb, const struct hash_net6 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_net6_elem e = { - .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK), + .cidr = INIT_CIDR(h->rnets[0], HOST_MASK), }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); diff --git a/net/netfilter/ipset/ip_set_hash_netiface.c b/net/netfilter/ipset/ip_set_hash_netiface.c index 30a655e5c4fd..b44b95f766b7 100644 --- a/net/netfilter/ipset/ip_set_hash_netiface.c +++ b/net/netfilter/ipset/ip_set_hash_netiface.c @@ -161,7 +161,7 @@ hash_netiface4_kadt(struct ip_set *set, const struct sk_buff *skb, struct hash_netiface4 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_netiface4_elem e = { - .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK), + .cidr = INIT_CIDR(h->rnets[0], HOST_MASK), .elem = 1, }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); @@ -382,7 +382,7 @@ hash_netiface6_kadt(struct ip_set *set, const struct sk_buff *skb, struct hash_netiface6 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_netiface6_elem e = { - .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK), + .cidr = INIT_CIDR(h->rnets[0], HOST_MASK), .elem = 1, }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); diff --git a/net/netfilter/ipset/ip_set_hash_netnet.c b/net/netfilter/ipset/ip_set_hash_netnet.c index 8fbe649c9dd3..f7c8a1cc30fc 100644 --- a/net/netfilter/ipset/ip_set_hash_netnet.c +++ b/net/netfilter/ipset/ip_set_hash_netnet.c @@ -149,8 +149,10 @@ hash_netnet4_kadt(struct ip_set *set, const struct sk_buff *skb, struct hash_netnet4_elem e = { }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); - e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK); - e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK); + rcu_read_lock_bh(); + e.cidr[0] = INIT_CIDR(h->rnets[0], HOST_MASK); + e.cidr[1] = INIT_CIDR(h->rnets[1], HOST_MASK); + rcu_read_unlock_bh(); if (adt == IPSET_TEST) e.ccmp = (HOST_MASK << (sizeof(e.cidr[0]) * 8)) | HOST_MASK; @@ -388,8 +390,10 @@ hash_netnet6_kadt(struct ip_set *set, const struct sk_buff *skb, struct hash_netnet6_elem e = { }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); - e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK); - e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK); + rcu_read_lock_bh(); + e.cidr[0] = INIT_CIDR(h->rnets[0], HOST_MASK); + e.cidr[1] = INIT_CIDR(h->rnets[1], HOST_MASK); + rcu_read_unlock_bh(); if (adt == IPSET_TEST) e.ccmp = (HOST_MASK << (sizeof(u8) * 8)) | HOST_MASK; diff --git a/net/netfilter/ipset/ip_set_hash_netport.c b/net/netfilter/ipset/ip_set_hash_netport.c index d1a0628df4ef..5de4b511de76 100644 --- a/net/netfilter/ipset/ip_set_hash_netport.c +++ b/net/netfilter/ipset/ip_set_hash_netport.c @@ -133,7 +133,7 @@ hash_netport4_kadt(struct ip_set *set, const struct sk_buff *skb, const struct hash_netport4 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_netport4_elem e = { - .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK), + .cidr = INIT_CIDR(h->rnets[0], HOST_MASK), }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); @@ -353,7 +353,7 @@ hash_netport6_kadt(struct ip_set *set, const struct sk_buff *skb, const struct hash_netport6 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_netport6_elem e = { - .cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK), + .cidr = INIT_CIDR(h->rnets[0], HOST_MASK), }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); diff --git a/net/netfilter/ipset/ip_set_hash_netportnet.c b/net/netfilter/ipset/ip_set_hash_netportnet.c index bf4f91b78e1d..6291532be7a5 100644 --- a/net/netfilter/ipset/ip_set_hash_netportnet.c +++ b/net/netfilter/ipset/ip_set_hash_netportnet.c @@ -157,8 +157,10 @@ hash_netportnet4_kadt(struct ip_set *set, const struct sk_buff *skb, struct hash_netportnet4_elem e = { }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); - e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK); - e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK); + rcu_read_lock_bh(); + e.cidr[0] = INIT_CIDR(h->rnets[0], HOST_MASK); + e.cidr[1] = INIT_CIDR(h->rnets[1], HOST_MASK); + rcu_read_unlock_bh(); if (adt == IPSET_TEST) e.ccmp = (HOST_MASK << (sizeof(e.cidr[0]) * 8)) | HOST_MASK; @@ -452,8 +454,10 @@ hash_netportnet6_kadt(struct ip_set *set, const struct sk_buff *skb, struct hash_netportnet6_elem e = { }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); - e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK); - e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK); + rcu_read_lock_bh(); + e.cidr[0] = INIT_CIDR(h->rnets[0], HOST_MASK); + e.cidr[1] = INIT_CIDR(h->rnets[1], HOST_MASK); + rcu_read_unlock_bh(); if (adt == IPSET_TEST) e.ccmp = (HOST_MASK << (sizeof(u8) * 8)) | HOST_MASK; diff --git a/net/netfilter/ipset/ip_set_list_set.c b/net/netfilter/ipset/ip_set_list_set.c index 1cef84f15e8c..f070088742d6 100644 --- a/net/netfilter/ipset/ip_set_list_set.c +++ b/net/netfilter/ipset/ip_set_list_set.c @@ -301,9 +301,12 @@ list_set_uadd(struct ip_set *set, void *value, const struct ip_set_ext *ext, e->set = set; INIT_LIST_HEAD(&e->list); list_set_init_extensions(set, ext, e); - if (n) + if (n) { list_set_replace(set, e, n); - else if (next) + return 0; + } + + if (next) list_add_tail_rcu(&e->list, &next->list); else if (prev) list_add_rcu(&e->list, &prev->list); @@ -420,8 +423,7 @@ list_set_flush(struct ip_set *set) list_for_each_entry_safe(e, n, &map->members, list) list_set_del(set, e); - set->elements = 0; - set->ext_size = 0; + DEBUG_NET_WARN_ON_ONCE(set->elements > 0); } static void @@ -455,7 +457,7 @@ list_set_head(struct ip_set *set, struct sk_buff *skb) { const struct list_set *map = set->data; struct nlattr *nested; - size_t memsize = list_set_memsize(map, set->dsize) + set->ext_size; + size_t memsize = list_set_memsize(map, set->dsize) + atomic64_read(&set->ext_size); nested = nla_nest_start(skb, IPSET_ATTR_DATA); if (!nested) diff --git a/net/netfilter/ipvs/ip_vs_app.c b/net/netfilter/ipvs/ip_vs_app.c index d54d7da58334..11cbdbaf561d 100644 --- a/net/netfilter/ipvs/ip_vs_app.c +++ b/net/netfilter/ipvs/ip_vs_app.c @@ -361,14 +361,13 @@ static inline int app_tcp_pkt_out(struct ip_vs_conn *cp, struct sk_buff *skb, struct ip_vs_iphdr *ipvsh) { int diff; - const unsigned int tcp_offset = ip_hdrlen(skb); struct tcphdr *th; __u32 seq; - if (skb_ensure_writable(skb, tcp_offset + sizeof(*th))) + if (skb_ensure_writable(skb, ipvsh->len + sizeof(*th))) return 0; - th = (struct tcphdr *)(skb_network_header(skb) + tcp_offset); + th = (struct tcphdr *)(skb->data + ipvsh->len); /* * Remember seq number in case this pkt gets resized @@ -438,14 +437,13 @@ static inline int app_tcp_pkt_in(struct ip_vs_conn *cp, struct sk_buff *skb, struct ip_vs_iphdr *ipvsh) { int diff; - const unsigned int tcp_offset = ip_hdrlen(skb); struct tcphdr *th; __u32 seq; - if (skb_ensure_writable(skb, tcp_offset + sizeof(*th))) + if (skb_ensure_writable(skb, ipvsh->len + sizeof(*th))) return 0; - th = (struct tcphdr *)(skb_network_header(skb) + tcp_offset); + th = (struct tcphdr *)(skb->data + ipvsh->len); /* * Remember seq number in case this pkt gets resized diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index cb36641f8d1c..6fa3e1dc534c 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -70,25 +70,45 @@ static struct kmem_cache *ip_vs_conn_cachep __read_mostly; * bucket or hash table * - hash table resize works like rehash but always rehashes into new table * - bit lock on bucket serializes all operations that modify the chain + * - on resize, bucket from the old table is locked before bucket from the + * new table * - cp->lock protects conn fields like cp->flags, cp->dest */ -/* Lock conn_tab bucket for conn hash/unhash, not for rehash */ +/** + * conn_tab_lock - Lock conn_tab buckets for conn hash/unhash, not for rehash + * @t: hash table for hn0, new_tbl when new_hash=true + * @t2: hash table for hn1, new_tbl when new_hash2=true + * @cp: connection + * @hash_key: hash key for hn0 + * @hash_key2: hash key for hn1 + * @use2: using hn1 (double hashing) based on the forwarding method + * @new_hash: mode for hn0, hash node (true) or seek node (false) + * @new_hash2: mode for hn1, hash node (true) or seek node (false) + * @head_ret: returned head for hn0 + * @head2_ret: returned head for hn1 + * + * We support 3 modes: + * - seek mode for both nodes, used for unhashing + * - hash mode for both nodes, used for hashing + * - seek hn0 and hash hn1, used when forwarding method is changed + */ static __always_inline void -conn_tab_lock(struct ip_vs_rht *t, struct ip_vs_conn *cp, u32 hash_key, - u32 hash_key2, bool use2, bool new_hash, - struct hlist_bl_head **head_ret, struct hlist_bl_head **head2_ret) +conn_tab_lock(struct ip_vs_rht *t, struct ip_vs_rht *t2, struct ip_vs_conn *cp, + u32 hash_key, u32 hash_key2, bool use2, bool new_hash, + bool new_hash2, struct hlist_bl_head **head_ret, + struct hlist_bl_head **head2_ret) { struct hlist_bl_head *head, *head2; u32 hash_key_new, hash_key_new2; - struct ip_vs_rht *t2 = t; - u32 idx, idx2; + int idx = 0, idx2 = 0; + + /* Advance idx2 when new_hash is not set but hash_key2 + * is for new table + */ + if (new_hash2 && use2 && t != t2) + idx2++; - idx = hash_key & t->mask; - if (use2) - idx2 = hash_key2 & t->mask; - else - idx2 = idx; if (!new_hash) { /* We need to lock the bucket in the right table */ @@ -100,46 +120,45 @@ retry: * both nodes in different tables, use idx/idx2 * for proper lock ordering for heads. */ - idx = hash_key & t->mask; - idx |= IP_VS_RHT_TABLE_ID_MASK; - } - if (use2) { - if (!ip_vs_rht_same_table(t2, hash_key2)) { - /* It is already moved to new table */ - t2 = rcu_dereference(t2->new_tbl); - idx2 = hash_key2 & t2->mask; - idx2 |= IP_VS_RHT_TABLE_ID_MASK; - } - } else { - idx2 = idx; + idx++; } } + if (use2 && !new_hash2 && !ip_vs_rht_same_table(t2, hash_key2)) { + /* It is already moved to new table */ + t2 = rcu_dereference(t2->new_tbl); + idx2++; + } + if (!use2) + idx2 = idx; head = t->buckets + (hash_key & t->mask); head2 = use2 ? t2->buckets + (hash_key2 & t2->mask) : head; - local_bh_disable(); - /* Do not touch seqcount, this is a safe operation */ - - if (idx <= idx2) { + if (idx > idx2 || (head > head2 && idx == idx2)) { + hlist_bl_lock(head2); hlist_bl_lock(head); - if (head != head2) - hlist_bl_lock(head2); } else { - hlist_bl_lock(head2); hlist_bl_lock(head); + if (head != head2) + hlist_bl_lock(head2); } if (!new_hash) { + bool changed; + /* Ensure hash_key is read under lock */ hash_key_new = READ_ONCE(cp->hn0.hash_key); - hash_key_new2 = READ_ONCE(cp->hn1.hash_key); + changed = hash_key != hash_key_new; + if (use2 && !new_hash2) { + hash_key_new2 = READ_ONCE(cp->hn1.hash_key); + changed |= hash_key2 != hash_key_new2; + } else { + hash_key_new2 = hash_key2; + } /* Hash changed ? */ - if (hash_key != hash_key_new || - (hash_key2 != hash_key_new2 && use2)) { + if (changed) { if (head != head2) hlist_bl_unlock(head2); hlist_bl_unlock(head); - local_bh_enable(); hash_key = hash_key_new; hash_key2 = hash_key_new2; goto retry; @@ -155,7 +174,6 @@ static inline void conn_tab_unlock(struct hlist_bl_head *head, if (head != head2) hlist_bl_unlock(head2); hlist_bl_unlock(head); - local_bh_enable(); } static void ip_vs_conn_expire(struct timer_list *t); @@ -268,8 +286,9 @@ static inline int ip_vs_conn_hash(struct ip_vs_conn *cp) use2 = false; } - conn_tab_lock(t, cp, hash_key, hash_key2, use2, true /* new_hash */, - &head, &head2); + local_bh_disable(); + conn_tab_lock(t, t, cp, hash_key, hash_key2, use2, true /* new_hash */, + true /* new_hash2 */, &head, &head2); cp->flags |= IP_VS_CONN_F_HASHED; WRITE_ONCE(cp->hn0.hash_key, hash_key); @@ -280,6 +299,7 @@ static inline int ip_vs_conn_hash(struct ip_vs_conn *cp) hlist_bl_add_head_rcu(&cp->hn1.node, head2); conn_tab_unlock(head, head2); + local_bh_enable(); ret = 1; /* Schedule resizing if load increases */ @@ -306,18 +326,20 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp) return refcount_dec_if_one(&cp->refcnt); rcu_read_lock(); + local_bh_disable(); t = rcu_dereference(ipvs->conn_tab); hash_key = READ_ONCE(cp->hn0.hash_key); hash_key2 = READ_ONCE(cp->hn1.hash_key); use2 = ip_vs_conn_use_hash2(cp); - conn_tab_lock(t, cp, hash_key, hash_key2, use2, false /* new_hash */, - &head, &head2); + conn_tab_lock(t, t, cp, hash_key, hash_key2, use2, false /* new_hash */, + false /* new_hash2 */, &head, &head2); if (cp->flags & IP_VS_CONN_F_HASHED) { /* Decrease refcnt and unlink conn only if we are last user */ - if (refcount_dec_if_one(&cp->refcnt)) { + if (use2 == ip_vs_conn_use_hash2(cp) && + refcount_dec_if_one(&cp->refcnt)) { hlist_bl_del_rcu(&cp->hn0.node); if (use2) hlist_bl_del_rcu(&cp->hn1.node); @@ -328,6 +350,7 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp) conn_tab_unlock(head, head2); + local_bh_enable(); rcu_read_unlock(); return ret; @@ -632,6 +655,7 @@ void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport) int ntbl; int dir; +restart: /* No packets from inside, so we can do it in 2 steps. */ dir = use2 ? 1 : 0; @@ -686,6 +710,23 @@ retry: /* Protect the cp->flags modification */ spin_lock_bh(&cp->lock); + /* Recheck the forwarding method under lock */ + if (use2 != ip_vs_conn_use_hash2(cp)) { + use2 = !use2; + if (use2) { + spin_unlock_bh(&cp->lock); + /* Restart with new use2 value */ + goto restart; + } + if (dir) { + /* Not started yet, so just skip dir 1 */ + spin_unlock_bh(&cp->lock); + dir--; + goto next_dir; + } + /* Just finish dir 0 */ + } + /* Lock seqcount only for the old bucket, even if we are on new table * because it affects the del operation, not the adding. */ @@ -752,6 +793,61 @@ retry: goto next_dir; } +/* Change forwarding method for hashed conn */ +static void ip_vs_conn_change_fwd_mask(struct ip_vs_conn *cp, u32 new_flags) +{ + struct netns_ipvs *ipvs = cp->ipvs; + struct hlist_bl_head *head, *head2; + u32 hash2, hash_key, hash_key2; + struct ip_vs_rht *t, *t2; + + /* See ip_vs_conn_use_hash2() for reference */ + if ((cp->flags & IP_VS_CONN_F_TEMPLATE) || + /* No change in double hashing ? */ + (IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ) == + ((new_flags & IP_VS_CONN_F_FWD_MASK) == IP_VS_CONN_F_MASQ)) { + cp->flags = new_flags; + return; + } + t = rcu_dereference(ipvs->conn_tab); + if (ip_vs_conn_use_hash2(cp)) { + /* Stop double hashing */ + hash_key = READ_ONCE(cp->hn0.hash_key); + hash_key2 = READ_ONCE(cp->hn1.hash_key); + + conn_tab_lock(t, t, cp, hash_key, hash_key2, true /* use2 */, + false /* new_hash */, false /* new_hash2 */, + &head, &head2); + + /* Keep both hash keys in same table */ + hash_key = READ_ONCE(cp->hn0.hash_key); + WRITE_ONCE(cp->hn1.hash_key, hash_key); + hlist_bl_del_rcu(&cp->hn1.node); + cp->flags = new_flags; + + conn_tab_unlock(head, head2); + } else { + /* Start double hashing */ + + hash_key = READ_ONCE(cp->hn0.hash_key); + + t2 = rcu_dereference(t->new_tbl); + hash2 = ip_vs_conn_hashkey_conn(t2, cp, true); + hash_key2 = ip_vs_rht_build_hash_key(t2, hash2); + + /* Change the forwarding method under locked hn0 */ + conn_tab_lock(t, t2, cp, hash_key, hash_key2, true /* use2 */, + false /* new_hash */, true /* new_hash2 */, + &head, &head2); + + WRITE_ONCE(cp->hn1.hash_key, hash_key2); + cp->flags = new_flags; + hlist_bl_add_head_rcu(&cp->hn1.node, head2); + + conn_tab_unlock(head, head2); + } +} + /* Get default load factor to map conn_count/u_thresh to t->size */ static int ip_vs_conn_default_load_factor(struct netns_ipvs *ipvs) { @@ -985,12 +1081,6 @@ static inline void ip_vs_bind_xmit_v6(struct ip_vs_conn *cp) #endif -static inline int ip_vs_dest_totalconns(struct ip_vs_dest *dest) -{ - return atomic_read(&dest->activeconns) - + atomic_read(&dest->inactconns); -} - /* * Bind a connection entry with a virtual service destination * Called just after a new connection entry is created. @@ -1014,6 +1104,9 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest) flags = cp->flags; /* Bind with the destination and its corresponding transmitter */ if (flags & IP_VS_CONN_F_SYNC) { + /* Synced conns are hashed, so they can not get this flag */ + conn_flags &= ~IP_VS_CONN_F_ONE_PACKET; + /* if the connection is not template and is created * by sync, preserve the activity flag. */ @@ -1021,9 +1114,18 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest) conn_flags &= ~IP_VS_CONN_F_INACTIVE; /* connections inherit forwarding method from dest */ flags &= ~(IP_VS_CONN_F_FWD_MASK | IP_VS_CONN_F_NOOUTPUT); + flags |= conn_flags; + /* Changing forwarding method for hashed conn can + * happen only under locks + */ + if (cp->flags & IP_VS_CONN_F_HASHED) + ip_vs_conn_change_fwd_mask(cp, flags); + else + cp->flags = flags; + } else { + flags |= conn_flags; + cp->flags = flags; } - flags |= conn_flags; - cp->flags = flags; cp->dest = dest; IP_VS_DBG_BUF(7, "Bind-dest %s c:%s:%d v:%s:%d " @@ -1039,23 +1141,22 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest) /* Update the connection counters */ if (!(flags & IP_VS_CONN_F_TEMPLATE)) { + int tc; + /* It is a normal connection, so modify the counters * according to the flags, later the protocol can * update them on state change */ if (!(flags & IP_VS_CONN_F_INACTIVE)) atomic_inc(&dest->activeconns); - else - atomic_inc(&dest->inactconns); + tc = atomic_inc_return(&dest->totalconns); + if (tc == READ_ONCE(dest->u_threshold)) + ip_vs_dest_update_overload(dest, 1); } else { /* It is a persistent connection/template, so increase the persistent connection counter */ atomic_inc(&dest->persistconns); } - - if (dest->u_threshold != 0 && - ip_vs_dest_totalconns(dest) >= dest->u_threshold) - dest->flags |= IP_VS_DEST_F_OVERLOAD; } @@ -1136,30 +1237,20 @@ static inline void ip_vs_unbind_dest(struct ip_vs_conn *cp) /* Update the connection counters */ if (!(cp->flags & IP_VS_CONN_F_TEMPLATE)) { - /* It is a normal connection, so decrease the inactconns - or activeconns counter */ - if (cp->flags & IP_VS_CONN_F_INACTIVE) { - atomic_dec(&dest->inactconns); - } else { + int tc; + + /* It is a normal connection, so decrease the counters */ + if (!(cp->flags & IP_VS_CONN_F_INACTIVE)) atomic_dec(&dest->activeconns); - } + tc = atomic_fetch_dec(&dest->totalconns); + if (tc == READ_ONCE(dest->l_threshold_val)) + ip_vs_dest_update_overload(dest, -1); } else { /* It is a persistent connection/template, so decrease the persistent connection counter */ atomic_dec(&dest->persistconns); } - if (dest->l_threshold != 0) { - if (ip_vs_dest_totalconns(dest) < dest->l_threshold) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - } else if (dest->u_threshold != 0) { - if (ip_vs_dest_totalconns(dest) * 4 < dest->u_threshold * 3) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - } else { - if (dest->flags & IP_VS_DEST_F_OVERLOAD) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - } - ip_vs_dest_put(dest); } @@ -1188,7 +1279,7 @@ int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest) * Checking the dest server status. */ if ((dest == NULL) || - !(dest->flags & IP_VS_DEST_F_AVAILABLE) || + !(dest->cflags & IP_VS_DEST_CF_AVAILABLE) || expire_quiescent_template(ipvs, dest) || (cdest && (dest != cdest))) { IP_VS_DBG_BUF(9, "check_template: dest not available for " @@ -1420,8 +1511,8 @@ ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, cp->app = NULL; cp->app_data = NULL; /* reset struct ip_vs_seq */ - cp->in_seq.delta = 0; - cp->out_seq.delta = 0; + memset(&cp->in_seq, 0, sizeof(cp->in_seq)); + memset(&cp->out_seq, 0, sizeof(cp->out_seq)); if (unlikely(flags & IP_VS_CONN_F_NO_CPORT)) { int af_id = ip_vs_af_index(cp->af); @@ -1929,7 +2020,7 @@ repeat: cp = ip_vs_hn0_to_conn(hn); resched_score++; dest = cp->dest; - if (!dest || (dest->flags & IP_VS_DEST_F_AVAILABLE)) + if (!dest || (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) continue; if (atomic_read(&cp->n_control)) diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index d40b404c1bf6..eb806813292a 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -302,7 +302,7 @@ ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb) struct ip_vs_dest *dest = cp->dest; struct netns_ipvs *ipvs = cp->ipvs; - if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) { + if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { struct ip_vs_cpu_stats *s; struct ip_vs_service *svc; @@ -338,7 +338,7 @@ ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb) struct ip_vs_dest *dest = cp->dest; struct netns_ipvs *ipvs = cp->ipvs; - if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) { + if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { struct ip_vs_cpu_stats *s; struct ip_vs_service *svc; @@ -398,10 +398,10 @@ ip_vs_conn_stats(struct ip_vs_conn *cp, struct ip_vs_service *svc) static inline void ip_vs_set_state(struct ip_vs_conn *cp, int direction, const struct sk_buff *skb, - struct ip_vs_proto_data *pd) + struct ip_vs_proto_data *pd, unsigned int iph_len) { if (likely(pd->pp->state_transition)) - pd->pp->state_transition(cp, direction, skb, pd); + pd->pp->state_transition(cp, direction, skb, pd, iph_len); } static inline int @@ -803,7 +803,7 @@ int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb, ip_vs_in_stats(cp, skb); /* set state */ - ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd); + ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd, iph->len); /* transmit the first SYN packet */ ret = cp->packet_xmit(skb, cp, pd->pp, iph); @@ -867,7 +867,7 @@ static int sysctl_nat_icmp_send(struct netns_ipvs *ipvs) { return 0; } #endif -__sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset) +static __sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset) { return csum_fold(skb_checksum(skb, offset, skb->len - offset, 0)); } @@ -923,31 +923,34 @@ static int ip_vs_route_me_harder(struct netns_ipvs *ipvs, int af, * Packet has been made sufficiently writable in caller * - inout: 1=in->out, 0=out->in */ -void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, - struct ip_vs_conn *cp, int inout) +bool ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, + struct ip_vs_conn *cp, int inout, unsigned int toff, + bool has_ports, struct ip_vs_iphdr *ciph) { struct iphdr *iph = ip_hdr(skb); - unsigned int icmp_offset = iph->ihl*4; - struct icmphdr *icmph = (struct icmphdr *)(skb_network_header(skb) + - icmp_offset); - struct iphdr *ciph = (struct iphdr *)(icmph + 1); + struct icmphdr *icmph = (struct icmphdr *)(skb->data + toff); + struct iphdr *cih = (struct iphdr *)(icmph + 1); + /* Before now we may used ihl from skb frag, revalidate it after + * copying it into skb head to prevent out-of-bounds access + */ + if (cih->ihl * 4 != ciph->len - ciph->off) + return false; if (inout) { iph->saddr = cp->vaddr.ip; ip_send_check(iph); - ciph->daddr = cp->vaddr.ip; - ip_send_check(ciph); + cih->daddr = cp->vaddr.ip; + ip_send_check(cih); } else { iph->daddr = cp->daddr.ip; ip_send_check(iph); - ciph->saddr = cp->daddr.ip; - ip_send_check(ciph); + cih->saddr = cp->daddr.ip; + ip_send_check(cih); } /* the TCP/UDP/SCTP port */ - if (IPPROTO_TCP == ciph->protocol || IPPROTO_UDP == ciph->protocol || - IPPROTO_SCTP == ciph->protocol) { - __be16 *ports = (void *)ciph + ciph->ihl*4; + if (has_ports) { + __be16 *ports = (void *)(skb->data + ciph->len); if (inout) ports[1] = cp->vport; @@ -957,48 +960,41 @@ void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, /* And finally the ICMP checksum */ icmph->checksum = 0; - icmph->checksum = ip_vs_checksum_complete(skb, icmp_offset); + icmph->checksum = ip_vs_checksum_complete(skb, toff); skb->ip_summed = CHECKSUM_UNNECESSARY; if (inout) - IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph, - "Forwarding altered outgoing ICMP"); + IP_VS_DBG_PKT(11, AF_INET, pp, skb, ciph->off, + "Forwarding altered outgoing ICMP"); else - IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph, - "Forwarding altered incoming ICMP"); + IP_VS_DBG_PKT(11, AF_INET, pp, skb, ciph->off, + "Forwarding altered incoming ICMP"); + return true; } #ifdef CONFIG_IP_VS_IPV6 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, - struct ip_vs_conn *cp, int inout) + struct ip_vs_conn *cp, int inout, unsigned int toff, + bool has_ports, struct ip_vs_iphdr *ciph) { struct ipv6hdr *iph = ipv6_hdr(skb); - unsigned int icmp_offset = 0; - unsigned int offs = 0; /* header offset*/ - int protocol; struct icmp6hdr *icmph; - struct ipv6hdr *ciph; - unsigned short fragoffs; - - ipv6_find_hdr(skb, &icmp_offset, IPPROTO_ICMPV6, &fragoffs, NULL); - icmph = (struct icmp6hdr *)(skb_network_header(skb) + icmp_offset); - offs = icmp_offset + sizeof(struct icmp6hdr); - ciph = (struct ipv6hdr *)(skb_network_header(skb) + offs); + struct ipv6hdr *cih; - protocol = ipv6_find_hdr(skb, &offs, -1, &fragoffs, NULL); + icmph = (struct icmp6hdr *)(skb->data + toff); + cih = (struct ipv6hdr *)(skb->data + ciph->off); if (inout) { iph->saddr = cp->vaddr.in6; - ciph->daddr = cp->vaddr.in6; + cih->daddr = cp->vaddr.in6; } else { iph->daddr = cp->daddr.in6; - ciph->saddr = cp->daddr.in6; + cih->saddr = cp->daddr.in6; } /* the TCP/UDP/SCTP port */ - if (!fragoffs && (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol || - IPPROTO_SCTP == protocol)) { - __be16 *ports = (void *)(skb_network_header(skb) + offs); + if (has_ports) { + __be16 *ports = (void *)(skb->data + ciph->len); IP_VS_DBG(11, "%s() changed port %d to %d\n", __func__, ntohs(inout ? ports[1] : ports[0]), @@ -1011,19 +1007,17 @@ void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, /* And finally the ICMP checksum */ icmph->icmp6_cksum = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, - skb->len - icmp_offset, + skb->len - toff, IPPROTO_ICMPV6, 0); - skb->csum_start = skb_network_header(skb) - skb->head + icmp_offset; + skb->csum_start = skb_headroom(skb) + toff; skb->csum_offset = offsetof(struct icmp6hdr, icmp6_cksum); skb->ip_summed = CHECKSUM_PARTIAL; if (inout) - IP_VS_DBG_PKT(11, AF_INET6, pp, skb, - (void *)ciph - (void *)iph, + IP_VS_DBG_PKT(11, AF_INET6, pp, skb, ciph->off, "Forwarding altered outgoing ICMPv6"); else - IP_VS_DBG_PKT(11, AF_INET6, pp, skb, - (void *)ciph - (void *)iph, + IP_VS_DBG_PKT(11, AF_INET6, pp, skb, ciph->off, "Forwarding altered incoming ICMPv6"); } #endif @@ -1033,36 +1027,42 @@ void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, */ static int handle_response_icmp(int af, struct sk_buff *skb, union nf_inet_addr *snet, - __u8 protocol, struct ip_vs_conn *cp, + struct ip_vs_conn *cp, struct ip_vs_protocol *pp, - unsigned int offset, unsigned int ihl, - unsigned int hooknum) + struct ip_vs_iphdr *ciph, + unsigned int toff, unsigned int hooknum) { + int iproto = af == AF_INET6 ? IPPROTO_ICMPV6 : IPPROTO_ICMP; unsigned int verdict = NF_DROP; + unsigned int ctoff = ciph->len; + bool has_ports = false; if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) goto after_nat; /* Ensure the checksum is correct */ - if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) { + if (!ip_vs_checksum_common_check(skb, toff, iproto, af)) { /* Failed checksum! */ IP_VS_DBG_BUF(1, "Forward ICMP: failed checksum from %s!\n", IP_VS_DBG_ADDR(af, snet)); goto out; } - if (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol || - IPPROTO_SCTP == protocol) - offset += 2 * sizeof(__u16); - if (skb_ensure_writable(skb, offset)) + if (ciph->protocol == IPPROTO_TCP || ciph->protocol == IPPROTO_UDP || + ciph->protocol == IPPROTO_SCTP) { + ctoff += 2 * sizeof(__u16); + has_ports = true; + } + if (skb_ensure_writable(skb, ctoff)) goto out; #ifdef CONFIG_IP_VS_IPV6 if (af == AF_INET6) - ip_vs_nat_icmp_v6(skb, pp, cp, 1); + ip_vs_nat_icmp_v6(skb, pp, cp, 1, toff, has_ports, ciph); else #endif - ip_vs_nat_icmp(skb, pp, cp, 1); + if (!ip_vs_nat_icmp(skb, pp, cp, 1, toff, has_ports, ciph)) + goto out; if (ip_vs_route_me_harder(cp->ipvs, af, skb, hooknum)) goto out; @@ -1090,15 +1090,15 @@ out: * Currently handles error types - unreachable, quench, ttl exceeded. */ static int ip_vs_out_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, - int *related, unsigned int hooknum) + int *related, unsigned int hooknum, + struct ip_vs_iphdr *ipvsh) { - struct iphdr *iph; struct icmphdr _icmph, *ic; struct iphdr _ciph, *cih; /* The ip header contained within the ICMP */ struct ip_vs_iphdr ciph; struct ip_vs_conn *cp; struct ip_vs_protocol *pp; - unsigned int offset, ihl; + unsigned int offset; union nf_inet_addr snet; *related = 1; @@ -1107,17 +1107,18 @@ static int ip_vs_out_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, if (ip_is_fragment(ip_hdr(skb))) { if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum))) return NF_STOLEN; + if (!ip_vs_fill_iph_skb(AF_INET, skb, false, ipvsh)) + return NF_ACCEPT; } - iph = ip_hdr(skb); - offset = ihl = iph->ihl * 4; + offset = ipvsh->len; ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph); if (ic == NULL) return NF_DROP; IP_VS_DBG(12, "Outgoing ICMP (%d,%d) %pI4->%pI4\n", ic->type, ntohs(icmp_id(ic)), - &iph->saddr, &iph->daddr); + &ipvsh->saddr.ip, &ipvsh->daddr.ip); /* * Work through seeing if this is for us. @@ -1135,33 +1136,34 @@ static int ip_vs_out_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, /* Now find the contained IP header */ offset += sizeof(_icmph); + if (!ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, true, &ciph)) + return NF_ACCEPT; /* The packet looks wrong, ignore */ + cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); - if (cih == NULL) + if (!(cih && cih->version == 4 && + ciph.len - ciph.off >= sizeof(struct iphdr))) return NF_ACCEPT; /* The packet looks wrong, ignore */ - pp = ip_vs_proto_get(cih->protocol); + pp = ip_vs_proto_get(ciph.protocol); if (!pp) return NF_ACCEPT; /* Is the embedded protocol header present? */ - if (unlikely(cih->frag_off & htons(IP_OFFSET) && - pp->dont_defrag)) + if (unlikely(cih->frag_off & htons(IP_OFFSET) && !pp->dont_defrag)) return NF_ACCEPT; IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset, "Checking outgoing ICMP for"); - ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, true, &ciph); - /* The embedded headers contain source and dest in reverse order */ cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto, ipvs, AF_INET, skb, &ciph); if (!cp) return NF_ACCEPT; - snet.ip = iph->saddr; - return handle_response_icmp(AF_INET, skb, &snet, cih->protocol, cp, - pp, ciph.len, ihl, hooknum); + snet.ip = ipvsh->saddr.ip; + return handle_response_icmp(AF_INET, skb, &snet, cp, pp, &ciph, + ipvsh->len, hooknum); } #ifdef CONFIG_IP_VS_IPV6 @@ -1174,7 +1176,6 @@ static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, struct ip_vs_conn *cp; struct ip_vs_protocol *pp; union nf_inet_addr snet; - unsigned int offset; *related = 1; ic = frag_safe_skb_hp(skb, ipvsh->len, sizeof(_icmph), &_icmph); @@ -1210,6 +1211,10 @@ static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, if (!pp) return NF_ACCEPT; + /* Is the embedded protocol header present? */ + if (unlikely(ciph.fragoffs && !pp->dont_defrag)) + return NF_ACCEPT; + /* The embedded headers contain source and dest in reverse order */ cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto, ipvs, AF_INET6, skb, &ciph); @@ -1217,10 +1222,8 @@ static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, return NF_ACCEPT; snet.in6 = ciph.saddr.in6; - offset = ciph.len; - return handle_response_icmp(AF_INET6, skb, &snet, ciph.protocol, cp, - pp, offset, sizeof(struct ipv6hdr), - hooknum); + return handle_response_icmp(AF_INET6, skb, &snet, cp, pp, &ciph, + ipvsh->len, hooknum); } #endif @@ -1484,7 +1487,7 @@ handle_response(int af, struct sk_buff *skb, struct ip_vs_proto_data *pd, after_nat: ip_vs_out_stats(cp, skb); - ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pd); + ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pd, iph->len); skb->ipvs_property = 1; if (!(cp->flags & IP_VS_CONN_F_NFCT)) ip_vs_notrack(skb); @@ -1546,7 +1549,8 @@ ip_vs_out_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *stat #endif if (unlikely(iph.protocol == IPPROTO_ICMP)) { int related; - int verdict = ip_vs_out_icmp(ipvs, skb, &related, hooknum); + int verdict = ip_vs_out_icmp(ipvs, skb, &related, + hooknum, &iph); if (related) return verdict; @@ -1754,9 +1758,8 @@ unk: */ static int ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, - unsigned int hooknum) + unsigned int hooknum, struct ip_vs_iphdr *iph) { - struct iphdr *iph; struct icmphdr _icmph, *ic; struct iphdr _ciph, *cih; /* The ip header contained within the ICMP */ struct ip_vs_iphdr ciph; @@ -1766,7 +1769,9 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, unsigned int offset, offset2, ihl, verdict; bool tunnel, new_cp = false; union nf_inet_addr *raddr; - char *outer_proto = "IPIP"; + char *outer_proto __maybe_unused = "IPIP"; + unsigned int hlen_ipip; + int ulen = 0; *related = 1; @@ -1774,17 +1779,19 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (ip_is_fragment(ip_hdr(skb))) { if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum))) return NF_STOLEN; + if (!ip_vs_fill_iph_skb(AF_INET, skb, false, iph)) + return NF_ACCEPT; } - iph = ip_hdr(skb); - offset = ihl = iph->ihl * 4; + ihl = iph->len; + offset = iph->len; ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph); if (ic == NULL) return NF_DROP; IP_VS_DBG(12, "Incoming ICMP (%d,%d) %pI4->%pI4\n", ic->type, ntohs(icmp_id(ic)), - &iph->saddr, &iph->daddr); + &iph->saddr.ip, &iph->daddr.ip); /* * Work through seeing if this is for us. @@ -1803,7 +1810,10 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Now find the contained IP header */ offset += sizeof(_icmph); cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); - if (cih == NULL) + if (!cih) + return NF_ACCEPT; /* The packet looks wrong, ignore */ + hlen_ipip = cih->ihl * 4; + if (!(cih->version == 4 && hlen_ipip >= sizeof(struct iphdr))) return NF_ACCEPT; /* The packet looks wrong, ignore */ raddr = (union nf_inet_addr *)&cih->daddr; @@ -1821,22 +1831,18 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Only for known tunnel */ if (!dest || dest->tun_type != IP_VS_CONN_F_TUNNEL_TYPE_IPIP) return NF_ACCEPT; - offset += cih->ihl * 4; - cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); - if (cih == NULL) - return NF_ACCEPT; /* The packet looks wrong, ignore */ + offset += hlen_ipip; tunnel = true; } else if ((cih->protocol == IPPROTO_UDP || /* Can be UDP encap */ cih->protocol == IPPROTO_GRE) && /* Can be GRE encap */ /* Error for our tunnel must arrive at LOCAL_IN */ (skb_rtable(skb)->rt_flags & RTCF_LOCAL)) { __u8 iproto; - int ulen; /* Non-first fragment has no UDP/GRE header */ if (unlikely(cih->frag_off & htons(IP_OFFSET))) return NF_ACCEPT; - offset2 = offset + cih->ihl * 4; + offset2 = offset + hlen_ipip; if (cih->protocol == IPPROTO_UDP) { ulen = ipvs_udp_decap(ipvs, skb, offset2, AF_INET, raddr, &iproto); @@ -1850,33 +1856,32 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Skip IP and UDP/GRE tunnel headers */ offset = offset2 + ulen; /* Now we should be at the original IP header */ - cih = skb_header_pointer(skb, offset, sizeof(_ciph), - &_ciph); - if (cih && cih->version == 4 && cih->ihl >= 5 && - iproto == IPPROTO_IPIP) + if (iproto == IPPROTO_IPIP) tunnel = true; else return NF_ACCEPT; } } - pd = ip_vs_proto_data_get(ipvs, cih->protocol); + if (!ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, !tunnel, &ciph)) + return NF_ACCEPT; + pd = ip_vs_proto_data_get(ipvs, ciph.protocol); if (!pd) return NF_ACCEPT; pp = pd->pp; + cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); + if (!(cih && cih->version == 4 && + ciph.len - ciph.off >= sizeof(struct iphdr))) + return NF_ACCEPT; /* The packet looks wrong, ignore */ + /* Is the embedded protocol header present? */ - if (unlikely(cih->frag_off & htons(IP_OFFSET) && - pp->dont_defrag)) + if (unlikely(cih->frag_off & htons(IP_OFFSET) && !pp->dont_defrag)) return NF_ACCEPT; IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset, "Checking incoming ICMP for"); - offset2 = offset; - ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, !tunnel, &ciph); - offset = ciph.len; - /* The embedded headers contain source and dest in reverse order. * For IPIP/UDP/GRE tunnel this is error for request, not for reply. */ @@ -1897,18 +1902,21 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, verdict = NF_DROP; /* Ensure the checksum is correct */ - if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) { + if ((IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ || tunnel) && + !ip_vs_checksum_common_check(skb, ihl, IPPROTO_ICMP, AF_INET)) { /* Failed checksum! */ IP_VS_DBG(1, "Incoming ICMP: failed checksum from %pI4!\n", - &iph->saddr); + &iph->saddr.ip); goto out; } if (tunnel) { + unsigned int hlen_orig = ciph.len - ciph.off; __be32 info = ic->un.gateway; __u8 type = ic->type; __u8 code = ic->code; + offset2 = offset; /* Update the MTU */ if (ic->type == ICMP_DEST_UNREACH && ic->code == ICMP_FRAG_NEEDED) { @@ -1921,6 +1929,9 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, goto ignore_tunnel; offset2 -= ihl + sizeof(_icmph); skb_reset_network_header(skb); + /* Ensure the IP header is present in headroom */ + if (!pskb_may_pull(skb, hlen_ipip)) + goto ignore_tunnel; IP_VS_DBG(12, "ICMP for %s %pI4->%pI4: mtu=%u\n", outer_proto, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, mtu); @@ -1936,8 +1947,8 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (dest_dst) mtu = dst_mtu(dest_dst->dst_cache); } - if (mtu > 68 + sizeof(struct iphdr)) - mtu -= sizeof(struct iphdr); + if (mtu > 68 + hlen_ipip + ulen) + mtu -= hlen_ipip + ulen; info = htonl(mtu); } /* Strip outer IP, ICMP and IPIP/UDP/GRE, go to IP header of @@ -1946,6 +1957,10 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (pskb_pull(skb, offset2) == NULL) goto ignore_tunnel; skb_reset_network_header(skb); + memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); + /* Ensure the IP header is present in headroom */ + if (!pskb_may_pull(skb, hlen_orig)) + goto ignore_tunnel; IP_VS_DBG(12, "Sending ICMP for %pI4->%pI4: t=%u, c=%u, i=%u\n", &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr, type, code, ntohl(info)); @@ -1961,10 +1976,7 @@ ignore_tunnel: /* do the statistics and put it back */ ip_vs_in_stats(cp, skb); - if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol || - IPPROTO_SCTP == cih->protocol) - offset += 2 * sizeof(__u16); - verdict = ip_vs_icmp_xmit(skb, cp, pp, offset, hooknum, &ciph); + verdict = ip_vs_icmp_xmit(skb, cp, pp, iph->len, hooknum, &ciph); out: if (likely(!new_cp)) @@ -2024,8 +2036,8 @@ static int ip_vs_in_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, return NF_ACCEPT; pp = pd->pp; - /* Cannot handle fragmented embedded protocol */ - if (ciph.fragoffs) + /* Is the embedded protocol header present? */ + if (ciph.fragoffs && !pp->dont_defrag) return NF_ACCEPT; IP_VS_DBG_PKT(11, AF_INET6, pp, skb, offset, @@ -2049,23 +2061,22 @@ static int ip_vs_in_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, new_cp = true; } - /* VS/TUN, VS/DR and LOCALNODE just let it go */ - if ((hooknum == NF_INET_LOCAL_OUT) && - (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)) { - verdict = NF_ACCEPT; + verdict = NF_DROP; + + /* Ensure the checksum is correct */ + if (IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ && + !ip_vs_checksum_common_check(skb, iph->len, IPPROTO_ICMPV6, + AF_INET6)) { + /* Failed checksum! */ + IP_VS_DBG(1, "Incoming ICMPv6: failed checksum from %pI6c!\n", + &iph->saddr); goto out; } /* do the statistics and put it back */ ip_vs_in_stats(cp, skb); - /* Need to mangle contained IPv6 header in ICMPv6 packet */ - offset = ciph.len; - if (IPPROTO_TCP == ciph.protocol || IPPROTO_UDP == ciph.protocol || - IPPROTO_SCTP == ciph.protocol) - offset += 2 * sizeof(__u16); /* Also mangle ports */ - - verdict = ip_vs_icmp_xmit_v6(skb, cp, pp, offset, hooknum, &ciph); + verdict = ip_vs_icmp_xmit_v6(skb, cp, pp, iph->len, hooknum, &ciph); out: if (likely(!new_cp)) @@ -2144,7 +2155,7 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state if (unlikely(iph.protocol == IPPROTO_ICMP)) { int related; int verdict = ip_vs_in_icmp(ipvs, skb, &related, - hooknum); + hooknum, &iph); if (related) return verdict; @@ -2192,8 +2203,11 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state } if (resched) { - if (!old_ct) + if (!old_ct) { + spin_lock_bh(&cp->lock); cp->flags &= ~IP_VS_CONN_F_NFCT; + spin_unlock_bh(&cp->lock); + } if (!atomic_read(&cp->n_control)) ip_vs_conn_expire_now(cp); __ip_vs_conn_put(cp); @@ -2204,13 +2218,16 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state } /* Check the server status */ - if (cp && cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) { + if (cp && cp->dest && !(cp->dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { /* the destination server is not available */ if (sysctl_expire_nodest_conn(ipvs)) { bool old_ct = ip_vs_conn_uses_old_conntrack(cp, skb); - if (!old_ct) + if (!old_ct) { + spin_lock_bh(&cp->lock); cp->flags &= ~IP_VS_CONN_F_NFCT; + spin_unlock_bh(&cp->lock); + } ip_vs_conn_expire_now(cp); __ip_vs_conn_put(cp); @@ -2233,7 +2250,7 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state IP_VS_DBG_PKT(11, af, pp, skb, iph.off, "Incoming packet"); ip_vs_in_stats(cp, skb); - ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd); + ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pd, iph.len); if (cp->packet_xmit) ret = cp->packet_xmit(skb, cp, pp, &iph); /* do not touch skb anymore */ @@ -2280,6 +2297,7 @@ ip_vs_forward_icmp(void *priv, struct sk_buff *skb, const struct nf_hook_state *state) { struct netns_ipvs *ipvs = net_ipvs(state->net); + struct ip_vs_iphdr iphdr; int r; /* ipvs enabled in this netns ? */ @@ -2289,10 +2307,9 @@ ip_vs_forward_icmp(void *priv, struct sk_buff *skb, if (state->pf == NFPROTO_IPV4) { if (ip_hdr(skb)->protocol != IPPROTO_ICMP) return NF_ACCEPT; + ip_vs_fill_iph_skb(AF_INET, skb, false, &iphdr); #ifdef CONFIG_IP_VS_IPV6 } else { - struct ip_vs_iphdr iphdr; - ip_vs_fill_iph_skb(AF_INET6, skb, false, &iphdr); if (iphdr.protocol != IPPROTO_ICMPV6) @@ -2302,7 +2319,7 @@ ip_vs_forward_icmp(void *priv, struct sk_buff *skb, #endif } - return ip_vs_in_icmp(ipvs, skb, &r, state->hook); + return ip_vs_in_icmp(ipvs, skb, &r, state->hook, &iphdr); } static const struct nf_hook_ops ip_vs_ops4[] = { diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index bcf40b8c41cf..8f9a8e491ad6 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -1304,6 +1304,40 @@ void ip_vs_stats_free(struct ip_vs_stats *stats) } } +/* Update overload flag based on number of dest conns and lower/upper + * connection thresholds: + * - conns reach u_threshold and exceed it: set the flag + * - conns go below l_threshold (or 75% of u_threshold): clear the flag + */ +static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode) +{ + int conns; + u32 l, u; + + lockdep_assert_held(&dest->dst_lock); + u = READ_ONCE(dest->u_threshold); + if (!u) + goto unset; + l = READ_ONCE(dest->l_threshold_val); + conns = atomic_read(&dest->totalconns); + if (conns >= (mode > 0 ? l : u)) { + dest->flags |= IP_VS_DEST_F_OVERLOAD; + return; + } + if (conns >= (mode < 0 ? u : l)) + return; + +unset: + dest->flags &= ~IP_VS_DEST_F_OVERLOAD; +} + +void ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode) +{ + spin_lock_bh(&dest->dst_lock); + __ip_vs_dest_update_overload(dest, mode); + spin_unlock_bh(&dest->dst_lock); +} + /* * Update a destination in the given service */ @@ -1368,12 +1402,21 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, } /* set the dest status flags */ - dest->flags |= IP_VS_DEST_F_AVAILABLE; + dest->cflags |= IP_VS_DEST_CF_AVAILABLE; - if (udest->u_threshold == 0 || udest->u_threshold > dest->u_threshold) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - dest->u_threshold = udest->u_threshold; - dest->l_threshold = udest->l_threshold; + if (READ_ONCE(dest->u_threshold) != udest->u_threshold || + READ_ONCE(dest->l_threshold) != udest->l_threshold) { + spin_lock_bh(&dest->dst_lock); + WRITE_ONCE(dest->u_threshold, udest->u_threshold); + WRITE_ONCE(dest->l_threshold, udest->l_threshold); + /* Low threshold defaults to 75% of upper threshold */ + WRITE_ONCE(dest->l_threshold_val, + udest->l_threshold ? : + (udest->u_threshold - + (udest->u_threshold >> 2))); + __ip_vs_dest_update_overload(dest, 0); + spin_unlock_bh(&dest->dst_lock); + } dest->af = udest->af; @@ -1445,7 +1488,7 @@ ip_vs_new_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) dest->port = udest->port; atomic_set(&dest->activeconns, 0); - atomic_set(&dest->inactconns, 0); + atomic_set(&dest->totalconns, 0); atomic_set(&dest->persistconns, 0); refcount_set(&dest->refcnt, 1); @@ -1486,6 +1529,9 @@ ip_vs_add_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) return -ERANGE; } + if (udest->u_threshold > INT_MAX) + return -EINVAL; + if (udest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) { if (udest->tun_port == 0) { pr_err("%s(): tunnel port is zero\n", __func__); @@ -1559,6 +1605,9 @@ ip_vs_edit_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) return -ERANGE; } + if (udest->u_threshold > INT_MAX) + return -EINVAL; + if (udest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) { if (udest->tun_port == 0) { pr_err("%s(): tunnel port is zero\n", __func__); @@ -1613,7 +1662,7 @@ static void __ip_vs_unlink_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, int svcupd) { - dest->flags &= ~IP_VS_DEST_F_AVAILABLE; + dest->cflags &= ~IP_VS_DEST_CF_AVAILABLE; spin_lock_bh(&dest->dst_lock); __ip_vs_dst_cache_reset(dest); @@ -3031,7 +3080,7 @@ static int ip_vs_info_seq_show(struct seq_file *seq, void *v) ip_vs_fwd_name(atomic_read(&dest->conn_flags)), atomic_read(&dest->weight), atomic_read(&dest->activeconns), - atomic_read(&dest->inactconns)); + ip_vs_dest_inactconns(dest)); else #endif seq_printf(seq, @@ -3042,7 +3091,7 @@ static int ip_vs_info_seq_show(struct seq_file *seq, void *v) ip_vs_fwd_name(atomic_read(&dest->conn_flags)), atomic_read(&dest->weight), atomic_read(&dest->activeconns), - atomic_read(&dest->inactconns)); + ip_vs_dest_inactconns(dest)); } } @@ -3667,10 +3716,10 @@ __ip_vs_get_dest_entries(struct netns_ipvs *ipvs, const struct ip_vs_get_dests * entry.port = dest->port; entry.conn_flags = atomic_read(&dest->conn_flags); entry.weight = atomic_read(&dest->weight); - entry.u_threshold = dest->u_threshold; - entry.l_threshold = dest->l_threshold; + entry.u_threshold = READ_ONCE(dest->u_threshold); + entry.l_threshold = READ_ONCE(dest->l_threshold); entry.activeconns = atomic_read(&dest->activeconns); - entry.inactconns = atomic_read(&dest->inactconns); + entry.inactconns = ip_vs_dest_inactconns(dest); entry.persistconns = atomic_read(&dest->persistconns); ip_vs_copy_stats(&kstats, &dest->stats); ip_vs_export_stats_user(&entry.stats, &kstats); @@ -4277,12 +4326,14 @@ static int ip_vs_genl_fill_dest(struct sk_buff *skb, struct ip_vs_dest *dest) dest->tun_port) || nla_put_u16(skb, IPVS_DEST_ATTR_TUN_FLAGS, dest->tun_flags) || - nla_put_u32(skb, IPVS_DEST_ATTR_U_THRESH, dest->u_threshold) || - nla_put_u32(skb, IPVS_DEST_ATTR_L_THRESH, dest->l_threshold) || + nla_put_u32(skb, IPVS_DEST_ATTR_U_THRESH, + READ_ONCE(dest->u_threshold)) || + nla_put_u32(skb, IPVS_DEST_ATTR_L_THRESH, + READ_ONCE(dest->l_threshold)) || nla_put_u32(skb, IPVS_DEST_ATTR_ACTIVE_CONNS, atomic_read(&dest->activeconns)) || nla_put_u32(skb, IPVS_DEST_ATTR_INACT_CONNS, - atomic_read(&dest->inactconns)) || + ip_vs_dest_inactconns(dest)) || nla_put_u32(skb, IPVS_DEST_ATTR_PERSIST_CONNS, atomic_read(&dest->persistconns)) || nla_put_u16(skb, IPVS_DEST_ATTR_ADDR_FAMILY, dest->af)) diff --git a/net/netfilter/ipvs/ip_vs_dh.c b/net/netfilter/ipvs/ip_vs_dh.c index e1f62f6b25e2..43abed7a26a6 100644 --- a/net/netfilter/ipvs/ip_vs_dh.c +++ b/net/netfilter/ipvs/ip_vs_dh.c @@ -219,8 +219,8 @@ ip_vs_dh_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, s = (struct ip_vs_dh_state *) svc->sched_data; dest = ip_vs_dh_get(svc->af, s, &iph->daddr); - if (!dest - || !(dest->flags & IP_VS_DEST_F_AVAILABLE) + if (!dest || + !(dest->cflags & IP_VS_DEST_CF_AVAILABLE) || atomic_read(&dest->weight) <= 0 || is_overloaded(dest)) { ip_vs_scheduler_err(svc, "no destination available"); diff --git a/net/netfilter/ipvs/ip_vs_est.c b/net/netfilter/ipvs/ip_vs_est.c index ab09f5182951..05a216a47b45 100644 --- a/net/netfilter/ipvs/ip_vs_est.c +++ b/net/netfilter/ipvs/ip_vs_est.c @@ -191,8 +191,11 @@ static int ip_vs_estimation_kthread(void *data) } /* kthread 0 will handle the calc phase */ - if (ipvs->est_calc_phase) + if (ipvs->est_calc_phase) { ip_vs_est_calc_phase(ipvs); + if (kthread_should_stop() || !READ_ONCE(ipvs->enable)) + return 0; + } } while (1) { @@ -270,6 +273,7 @@ int ip_vs_est_kthread_start(struct netns_ipvs *ipvs, kd->task = NULL; goto out; } + get_task_struct(kd->task); set_user_nice(kd->task, sysctl_est_nice(ipvs)); if (sysctl_est_preferred_cpulist(ipvs)) @@ -286,7 +290,7 @@ void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd) { if (kd->task) { pr_info("stopping estimator thread %d...\n", kd->id); - kthread_stop(kd->task); + kthread_stop_put(kd->task); kd->task = NULL; } } @@ -526,7 +530,7 @@ static void ip_vs_est_kthread_destroy(struct ip_vs_est_kt_data *kd) if (kd) { if (kd->task) { pr_info("stop unused estimator thread %d...\n", kd->id); - kthread_stop(kd->task); + kthread_stop_put(kd->task); } ip_vs_stats_free(kd->calc_stats); kfree(kd); diff --git a/net/netfilter/ipvs/ip_vs_lblc.c b/net/netfilter/ipvs/ip_vs_lblc.c index 15ccb2b2fa1f..693bcc82ccb7 100644 --- a/net/netfilter/ipvs/ip_vs_lblc.c +++ b/net/netfilter/ipvs/ip_vs_lblc.c @@ -502,7 +502,7 @@ ip_vs_lblc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, */ dest = en->dest; - if ((dest->flags & IP_VS_DEST_F_AVAILABLE) && + if ((dest->cflags & IP_VS_DEST_CF_AVAILABLE) && atomic_read(&dest->weight) > 0 && !is_overloaded(dest, svc)) goto out; } diff --git a/net/netfilter/ipvs/ip_vs_lblcr.c b/net/netfilter/ipvs/ip_vs_lblcr.c index c90ea897c3f7..f53f05ceea36 100644 --- a/net/netfilter/ipvs/ip_vs_lblcr.c +++ b/net/netfilter/ipvs/ip_vs_lblcr.c @@ -169,8 +169,8 @@ static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set) if (least->flags & IP_VS_DEST_F_OVERLOAD) continue; - if ((atomic_read(&least->weight) > 0) - && (least->flags & IP_VS_DEST_F_AVAILABLE)) { + if ((atomic_read(&least->weight) > 0) && + (least->cflags & IP_VS_DEST_CF_AVAILABLE)) { loh = ip_vs_dest_conn_overhead(least); goto nextstage; } @@ -186,8 +186,8 @@ static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set) doh = ip_vs_dest_conn_overhead(dest); if (((__s64)loh * atomic_read(&dest->weight) > - (__s64)doh * atomic_read(&least->weight)) - && (dest->flags & IP_VS_DEST_F_AVAILABLE)) { + (__s64)doh * atomic_read(&least->weight)) && + (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { least = dest; loh = doh; } diff --git a/net/netfilter/ipvs/ip_vs_lc.c b/net/netfilter/ipvs/ip_vs_lc.c index 38cc38c5d8bb..18b976a7c4d2 100644 --- a/net/netfilter/ipvs/ip_vs_lc.c +++ b/net/netfilter/ipvs/ip_vs_lc.c @@ -30,7 +30,7 @@ ip_vs_lc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, /* * Simply select the server with the least number of - * (activeconns<<5) + inactconns + * (activeconns*256) + totalconns * Except whose weight is equal to zero. * If the weight is equal to zero, it means that the server is * quiesced, the existing connections to the server still get @@ -56,7 +56,7 @@ ip_vs_lc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, IP_VS_DBG_ADDR(least->af, &least->addr), ntohs(least->port), atomic_read(&least->activeconns), - atomic_read(&least->inactconns)); + ip_vs_dest_inactconns(least)); return least; } diff --git a/net/netfilter/ipvs/ip_vs_proto_sctp.c b/net/netfilter/ipvs/ip_vs_proto_sctp.c index 63c78a1f3918..fb8af6b15a39 100644 --- a/net/netfilter/ipvs/ip_vs_proto_sctp.c +++ b/net/netfilter/ipvs/ip_vs_proto_sctp.c @@ -11,7 +11,7 @@ static int sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int sctphoff); + struct ip_vs_iphdr *iph); static int sctp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -109,7 +109,7 @@ sctp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!sctp_csum_check(cp->af, skb, pp, sctphoff)) + if (!sctp_csum_check(cp->af, skb, pp, iph)) return 0; /* Call application helper if needed */ @@ -121,7 +121,7 @@ sctp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - sctph = (void *) skb_network_header(skb) + sctphoff; + sctph = (void *)skb->data + sctphoff; /* Only update csum if we really have to */ if (sctph->source != cp->vport || payload_csum || @@ -157,7 +157,7 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!sctp_csum_check(cp->af, skb, pp, sctphoff)) + if (!sctp_csum_check(cp->af, skb, pp, iph)) return 0; /* Call application helper if needed */ @@ -169,7 +169,7 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - sctph = (void *) skb_network_header(skb) + sctphoff; + sctph = (void *)skb->data + sctphoff; /* Only update csum if we really have to */ if (sctph->dest != cp->dport || payload_csum || @@ -187,19 +187,22 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int sctphoff) + struct ip_vs_iphdr *iph) { + unsigned int sctphoff = iph->len; struct sctphdr *sh; __le32 cmp, val; + if (!ip_vs_checksum_needed(skb)) + return 1; sh = (struct sctphdr *)(skb->data + sctphoff); cmp = sh->checksum; val = sctp_compute_cksum(skb, sctphoff); if (val != cmp) { /* CRC failure, dump it. */ - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); + IP_VS_DBG_RL_PKT(0, af, pp, skb, iph->off, + "Failed checksum for"); return 0; } return 1; @@ -372,20 +375,15 @@ static const char *sctp_state_name(int state) static inline void set_sctp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, - int direction, const struct sk_buff *skb) + int direction, const struct sk_buff *skb, + unsigned int iph_len) { struct sctp_chunkhdr _sctpch, *sch; unsigned char chunk_type; int event, next_state; - int ihl, cofs; - -#ifdef CONFIG_IP_VS_IPV6 - ihl = cp->af == AF_INET ? ip_hdrlen(skb) : sizeof(struct ipv6hdr); -#else - ihl = ip_hdrlen(skb); -#endif + int cofs; - cofs = ihl + sizeof(struct sctphdr); + cofs = iph_len + sizeof(struct sctphdr); sch = skb_header_pointer(skb, cofs, sizeof(_sctpch), &_sctpch); if (sch == NULL) return; @@ -448,12 +446,10 @@ set_sctp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, if (!(cp->flags & IP_VS_CONN_F_INACTIVE) && (next_state != IP_VS_SCTP_S_ESTABLISHED)) { atomic_dec(&dest->activeconns); - atomic_inc(&dest->inactconns); cp->flags |= IP_VS_CONN_F_INACTIVE; } else if ((cp->flags & IP_VS_CONN_F_INACTIVE) && (next_state == IP_VS_SCTP_S_ESTABLISHED)) { atomic_inc(&dest->activeconns); - atomic_dec(&dest->inactconns); cp->flags &= ~IP_VS_CONN_F_INACTIVE; } } @@ -468,10 +464,11 @@ set_sctp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, static void sctp_state_transition(struct ip_vs_conn *cp, int direction, - const struct sk_buff *skb, struct ip_vs_proto_data *pd) + const struct sk_buff *skb, struct ip_vs_proto_data *pd, + unsigned int iph_len) { spin_lock_bh(&cp->lock); - set_sctp_state(pd, cp, direction, skb); + set_sctp_state(pd, cp, direction, skb, iph_len); spin_unlock_bh(&cp->lock); } diff --git a/net/netfilter/ipvs/ip_vs_proto_tcp.c b/net/netfilter/ipvs/ip_vs_proto_tcp.c index 8cc0a8ce6241..fec0e8b47b71 100644 --- a/net/netfilter/ipvs/ip_vs_proto_tcp.c +++ b/net/netfilter/ipvs/ip_vs_proto_tcp.c @@ -29,7 +29,7 @@ static int tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int tcphoff); + struct ip_vs_iphdr *iph); static int tcp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -166,7 +166,7 @@ tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!tcp_csum_check(cp->af, skb, pp, tcphoff)) + if (!tcp_csum_check(cp->af, skb, pp, iph)) return 0; /* Call application helper if needed */ @@ -179,7 +179,7 @@ tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - tcph = (void *)skb_network_header(skb) + tcphoff; + tcph = (void *)skb->data + tcphoff; tcph->source = cp->vport; /* Adjust TCP checksums */ @@ -244,7 +244,7 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!tcp_csum_check(cp->af, skb, pp, tcphoff)) + if (!tcp_csum_check(cp->af, skb, pp, iph)) return 0; /* @@ -260,7 +260,7 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - tcph = (void *)skb_network_header(skb) + tcphoff; + tcph = (void *)skb->data + tcphoff; tcph->dest = cp->dport; /* @@ -302,41 +302,13 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int tcphoff) + struct ip_vs_iphdr *iph) { - switch (skb->ip_summed) { - case CHECKSUM_NONE: - skb->csum = skb_checksum(skb, tcphoff, skb->len - tcphoff, 0); - fallthrough; - case CHECKSUM_COMPLETE: -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) { - if (csum_ipv6_magic(&ipv6_hdr(skb)->saddr, - &ipv6_hdr(skb)->daddr, - skb->len - tcphoff, - IPPROTO_TCP, - skb->csum)) { - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); - return 0; - } - } else -#endif - if (csum_tcpudp_magic(ip_hdr(skb)->saddr, - ip_hdr(skb)->daddr, - skb->len - tcphoff, - ip_hdr(skb)->protocol, - skb->csum)) { - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); - return 0; - } - break; - default: - /* No need to checksum. */ - break; + if (!ip_vs_checksum_common_check(skb, iph->len, IPPROTO_TCP, af)) { + IP_VS_DBG_RL_PKT(0, af, pp, skb, iph->off, + "Failed checksum for"); + return 0; } - return 1; } @@ -554,12 +526,10 @@ set_tcp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, if (!(cp->flags & IP_VS_CONN_F_INACTIVE) && !tcp_state_active(new_state)) { atomic_dec(&dest->activeconns); - atomic_inc(&dest->inactconns); cp->flags |= IP_VS_CONN_F_INACTIVE; } else if ((cp->flags & IP_VS_CONN_F_INACTIVE) && tcp_state_active(new_state)) { atomic_inc(&dest->activeconns); - atomic_dec(&dest->inactconns); cp->flags &= ~IP_VS_CONN_F_INACTIVE; } } @@ -579,17 +549,12 @@ set_tcp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, static void tcp_state_transition(struct ip_vs_conn *cp, int direction, const struct sk_buff *skb, - struct ip_vs_proto_data *pd) + struct ip_vs_proto_data *pd, + unsigned int iph_len) { struct tcphdr _tcph, *th; -#ifdef CONFIG_IP_VS_IPV6 - int ihl = cp->af == AF_INET ? ip_hdrlen(skb) : sizeof(struct ipv6hdr); -#else - int ihl = ip_hdrlen(skb); -#endif - - th = skb_header_pointer(skb, ihl, sizeof(_tcph), &_tcph); + th = skb_header_pointer(skb, iph_len, sizeof(_tcph), &_tcph); if (th == NULL) return; diff --git a/net/netfilter/ipvs/ip_vs_proto_udp.c b/net/netfilter/ipvs/ip_vs_proto_udp.c index f9de632e38cd..f32785682402 100644 --- a/net/netfilter/ipvs/ip_vs_proto_udp.c +++ b/net/netfilter/ipvs/ip_vs_proto_udp.c @@ -25,7 +25,7 @@ static int udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int udphoff); + struct ip_vs_iphdr *iph); static int udp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -155,7 +155,7 @@ udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!udp_csum_check(cp->af, skb, pp, udphoff)) + if (!udp_csum_check(cp->af, skb, pp, iph)) return 0; /* @@ -170,7 +170,7 @@ udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - udph = (void *)skb_network_header(skb) + udphoff; + udph = (void *)skb->data + udphoff; udph->source = cp->vport; /* @@ -238,7 +238,7 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!udp_csum_check(cp->af, skb, pp, udphoff)) + if (!udp_csum_check(cp->af, skb, pp, iph)) return 0; /* @@ -254,7 +254,7 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - udph = (void *)skb_network_header(skb) + udphoff; + udph = (void *)skb->data + udphoff; udph->dest = cp->dport; /* @@ -298,48 +298,20 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int udphoff) + struct ip_vs_iphdr *iph) { struct udphdr _udph, *uh; - uh = skb_header_pointer(skb, udphoff, sizeof(_udph), &_udph); + uh = skb_header_pointer(skb, iph->len, sizeof(_udph), &_udph); if (uh == NULL) return 0; - if (uh->check != 0) { - switch (skb->ip_summed) { - case CHECKSUM_NONE: - skb->csum = skb_checksum(skb, udphoff, - skb->len - udphoff, 0); - fallthrough; - case CHECKSUM_COMPLETE: -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) { - if (csum_ipv6_magic(&ipv6_hdr(skb)->saddr, - &ipv6_hdr(skb)->daddr, - skb->len - udphoff, - IPPROTO_UDP, - skb->csum)) { - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); - return 0; - } - } else -#endif - if (csum_tcpudp_magic(ip_hdr(skb)->saddr, - ip_hdr(skb)->daddr, - skb->len - udphoff, - ip_hdr(skb)->protocol, - skb->csum)) { - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); - return 0; - } - break; - default: - /* No need to checksum. */ - break; - } + if (!uh->check) + return 1; + if (!ip_vs_checksum_common_check(skb, iph->len, IPPROTO_UDP, af)) { + IP_VS_DBG_RL_PKT(0, af, pp, skb, iph->off, + "Failed checksum for"); + return 0; } return 1; } @@ -444,7 +416,8 @@ static const char * udp_state_name(int state) static void udp_state_transition(struct ip_vs_conn *cp, int direction, const struct sk_buff *skb, - struct ip_vs_proto_data *pd) + struct ip_vs_proto_data *pd, + unsigned int iph_len) { if (unlikely(!pd)) { pr_err("UDP no ns data\n"); diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c index 93038abbf5e0..ea5fdd4f4ce7 100644 --- a/net/netfilter/ipvs/ip_vs_sync.c +++ b/net/netfilter/ipvs/ip_vs_sync.c @@ -878,13 +878,10 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa spin_lock_bh(&cp->lock); if ((cp->flags ^ flags) & IP_VS_CONN_F_INACTIVE && !(flags & IP_VS_CONN_F_TEMPLATE) && dest) { - if (flags & IP_VS_CONN_F_INACTIVE) { + if (flags & IP_VS_CONN_F_INACTIVE) atomic_dec(&dest->activeconns); - atomic_inc(&dest->inactconns); - } else { + else atomic_inc(&dest->activeconns); - atomic_dec(&dest->inactconns); - } } flags &= IP_VS_CONN_F_BACKUP_UPD_MASK; flags |= cp->flags & ~IP_VS_CONN_F_BACKUP_UPD_MASK; diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index ce542ed4b013..04450a48f01a 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -351,7 +351,7 @@ __ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, * stored in dest_trash. */ if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) && - dest->flags & IP_VS_DEST_F_AVAILABLE) + dest->cflags & IP_VS_DEST_CF_AVAILABLE) __ip_vs_dst_set(dest, dest_dst, &rt->dst, 0); else noref = 0; @@ -530,7 +530,7 @@ __ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb, * stored in dest_trash. */ if (!rt_dev_is_down(dst_dev_rcu(&rt->dst)) && - dest->flags & IP_VS_DEST_F_AVAILABLE) + dest->cflags & IP_VS_DEST_CF_AVAILABLE) __ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie); else noref = 0; @@ -736,13 +736,11 @@ int ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh) { - struct iphdr *iph = ip_hdr(skb); - - if (__ip_vs_get_out_rt(cp->ipvs, cp->af, skb, NULL, iph->daddr, + if (__ip_vs_get_out_rt(cp->ipvs, cp->af, skb, NULL, ip_hdr(skb)->daddr, IP_VS_RT_MODE_NON_LOCAL, NULL, ipvsh) < 0) goto tx_error; - ip_send_check(iph); + ip_send_check(ip_hdr(skb)); /* Another hack: avoid icmp_send in ip_fragment */ skb->ignore_df = 1; @@ -1504,20 +1502,22 @@ tx_error: */ int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, - struct ip_vs_protocol *pp, int offset, unsigned int hooknum, - struct ip_vs_iphdr *iph) + struct ip_vs_protocol *pp, unsigned int toff, + unsigned int hooknum, struct ip_vs_iphdr *ciph) { struct rtable *rt; /* Route to the other host */ int rc; int local; int rt_mode, was_input; + bool has_ports = false; + unsigned int wlen; /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be forwarded directly here, because there is no need to translate address/port back */ if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) { if (cp->packet_xmit) - rc = cp->packet_xmit(skb, cp, pp, iph); + rc = cp->packet_xmit(skb, cp, pp, ciph); else rc = NF_ACCEPT; /* do not touch skb anymore */ @@ -1535,7 +1535,7 @@ ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL | IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL; local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip, rt_mode, - NULL, iph); + NULL, ciph); if (local < 0) goto tx_error; rt = skb_rtable(skb); @@ -1566,14 +1566,22 @@ ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, goto tx_error; } + wlen = ciph->len; + if (ciph->protocol == IPPROTO_TCP || ciph->protocol == IPPROTO_UDP || + ciph->protocol == IPPROTO_SCTP) { + wlen += 2 * sizeof(__u16); /* Also mangle ports */ + has_ports = true; + } + /* copy-on-write the packet before mangling it */ - if (skb_ensure_writable(skb, offset)) + if (skb_ensure_writable(skb, wlen)) goto tx_error; if (skb_cow(skb, rt->dst.dev->hard_header_len)) goto tx_error; - ip_vs_nat_icmp(skb, pp, cp, 0); + if (!ip_vs_nat_icmp(skb, pp, cp, 0, toff, has_ports, ciph)) + goto tx_error; /* Another hack: avoid icmp_send in ip_fragment */ skb->ignore_df = 1; @@ -1589,10 +1597,12 @@ ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, #ifdef CONFIG_IP_VS_IPV6 int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, - struct ip_vs_protocol *pp, int offset, unsigned int hooknum, - struct ip_vs_iphdr *ipvsh) + struct ip_vs_protocol *pp, unsigned int toff, + unsigned int hooknum, struct ip_vs_iphdr *ciph) { + bool has_ports = false; struct rt6_info *rt; /* Route to the other host */ + unsigned int wlen; int rc; int local; int rt_mode; @@ -1602,7 +1612,7 @@ ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, translate address/port back */ if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) { if (cp->packet_xmit) - rc = cp->packet_xmit(skb, cp, pp, ipvsh); + rc = cp->packet_xmit(skb, cp, pp, ciph); else rc = NF_ACCEPT; /* do not touch skb anymore */ @@ -1619,7 +1629,7 @@ ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL | IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL; local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest, - &cp->daddr.in6, NULL, ipvsh, 0, rt_mode); + &cp->daddr.in6, NULL, ciph, 0, rt_mode); if (local < 0) goto tx_error; rt = dst_rt6_info(skb_dst(skb)); @@ -1650,14 +1660,21 @@ ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, goto tx_error; } + wlen = ciph->len; + if (ciph->protocol == IPPROTO_TCP || ciph->protocol == IPPROTO_UDP || + ciph->protocol == IPPROTO_SCTP) { + wlen += 2 * sizeof(__u16); /* Also mangle ports */ + has_ports = true; + } + /* copy-on-write the packet before mangling it */ - if (skb_ensure_writable(skb, offset)) + if (skb_ensure_writable(skb, wlen)) goto tx_error; if (skb_cow(skb, rt->dst.dev->hard_header_len)) goto tx_error; - ip_vs_nat_icmp_v6(skb, pp, cp, 0); + ip_vs_nat_icmp_v6(skb, pp, cp, 0, toff, has_ports, ciph); /* Another hack: avoid icmp_send in ip_fragment */ skb->ignore_df = 1; diff --git a/net/netfilter/nf_conncount.c b/net/netfilter/nf_conncount.c index 91582069f6d2..e9ea6d9466e7 100644 --- a/net/netfilter/nf_conncount.c +++ b/net/netfilter/nf_conncount.c @@ -211,8 +211,8 @@ check_connections: /* Not found, but might be about to be confirmed */ if (PTR_ERR(found) == -EAGAIN) { if (nf_ct_tuple_equal(&conn->tuple, &tuple) && - nf_ct_zone_id(&conn->zone, conn->zone.dir) == - nf_ct_zone_id(zone, zone->dir)) + nf_ct_zone_id(&conn->zone, IP_CT_DIR_ORIGINAL) == + nf_ct_zone_id(zone, IP_CT_DIR_ORIGINAL)) goto out_put; /* already exists */ } else { collect++; @@ -223,7 +223,7 @@ check_connections: found_ct = nf_ct_tuplehash_to_ctrack(found); if (nf_ct_tuple_equal(&conn->tuple, &tuple) && - nf_ct_zone_equal(found_ct, zone, zone->dir)) { + nf_ct_zone_equal(found_ct, zone, IP_CT_DIR_ORIGINAL)) { /* * We should not see tuples twice unless someone hooks * this into a table without "-p tcp --syn". diff --git a/net/netfilter/nf_conntrack_bpf.c b/net/netfilter/nf_conntrack_bpf.c index f98d1d4b42c3..c2df7c948281 100644 --- a/net/netfilter/nf_conntrack_bpf.c +++ b/net/netfilter/nf_conntrack_bpf.c @@ -122,42 +122,54 @@ __bpf_nf_ct_alloc_entry(struct net *net, struct bpf_sock_tuple *bpf_tuple, struct nf_conntrack_tuple otuple, rtuple; struct nf_conntrack_zone ct_zone; struct nf_conn *ct; + u8 ct_zone_dir = 0; + u16 ct_zone_id; + s32 netns_id; + u8 l4proto; int err; if (!(opts_len == NF_BPF_CT_OPTS_SZ || opts_len == 12)) return ERR_PTR(-EINVAL); + + netns_id = READ_ONCE(opts->netns_id); + l4proto = READ_ONCE(opts->l4proto); + ct_zone_id = READ_ONCE(opts->ct_zone_id); if (opts_len == NF_BPF_CT_OPTS_SZ) { - if (opts->reserved[0] || opts->reserved[1] || opts->reserved[2]) + ct_zone_dir = READ_ONCE(opts->ct_zone_dir); + if (READ_ONCE(opts->reserved[0]) || + READ_ONCE(opts->reserved[1]) || + READ_ONCE(opts->reserved[2])) return ERR_PTR(-EINVAL); } else { - if (opts->ct_zone_id) + if (ct_zone_id) return ERR_PTR(-EINVAL); } - if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS)) + if (unlikely(netns_id < BPF_F_CURRENT_NETNS)) return ERR_PTR(-EINVAL); - err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto, + err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, l4proto, IP_CT_DIR_ORIGINAL, &otuple); if (err < 0) return ERR_PTR(err); - err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto, + err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, l4proto, IP_CT_DIR_REPLY, &rtuple); if (err < 0) return ERR_PTR(err); - if (opts->netns_id >= 0) { - net = get_net_ns_by_id(net, opts->netns_id); + if (netns_id >= 0) { + net = get_net_ns_by_id(net, netns_id); if (unlikely(!net)) return ERR_PTR(-ENONET); } if (opts_len == NF_BPF_CT_OPTS_SZ) { - if (opts->ct_zone_dir == 0) - opts->ct_zone_dir = NF_CT_DEFAULT_ZONE_DIR; - nf_ct_zone_init(&ct_zone, - opts->ct_zone_id, opts->ct_zone_dir, 0); + if (ct_zone_dir == 0) { + ct_zone_dir = NF_CT_DEFAULT_ZONE_DIR; + opts->ct_zone_dir = ct_zone_dir; + } + nf_ct_zone_init(&ct_zone, ct_zone_id, ct_zone_dir, 0); } else { ct_zone = nf_ct_zone_dflt; } @@ -171,7 +183,7 @@ __bpf_nf_ct_alloc_entry(struct net *net, struct bpf_sock_tuple *bpf_tuple, __nf_ct_set_timeout(ct, timeout * HZ); out: - if (opts->netns_id >= 0) + if (netns_id >= 0) put_net(net); return ct; @@ -186,46 +198,58 @@ static struct nf_conn *__bpf_nf_ct_lookup(struct net *net, struct nf_conntrack_tuple tuple; struct nf_conntrack_zone ct_zone; struct nf_conn *ct; + u8 ct_zone_dir = 0; + u16 ct_zone_id; + s32 netns_id; + u8 l4proto; int err; if (!opts || !bpf_tuple) return ERR_PTR(-EINVAL); if (!(opts_len == NF_BPF_CT_OPTS_SZ || opts_len == 12)) return ERR_PTR(-EINVAL); + + netns_id = READ_ONCE(opts->netns_id); + l4proto = READ_ONCE(opts->l4proto); + ct_zone_id = READ_ONCE(opts->ct_zone_id); if (opts_len == NF_BPF_CT_OPTS_SZ) { - if (opts->reserved[0] || opts->reserved[1] || opts->reserved[2]) + ct_zone_dir = READ_ONCE(opts->ct_zone_dir); + if (READ_ONCE(opts->reserved[0]) || + READ_ONCE(opts->reserved[1]) || + READ_ONCE(opts->reserved[2])) return ERR_PTR(-EINVAL); } else { - if (opts->ct_zone_id) + if (ct_zone_id) return ERR_PTR(-EINVAL); } - if (unlikely(opts->l4proto != IPPROTO_TCP && opts->l4proto != IPPROTO_UDP)) + if (unlikely(l4proto != IPPROTO_TCP && l4proto != IPPROTO_UDP)) return ERR_PTR(-EPROTO); - if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS)) + if (unlikely(netns_id < BPF_F_CURRENT_NETNS)) return ERR_PTR(-EINVAL); - err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto, + err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, l4proto, IP_CT_DIR_ORIGINAL, &tuple); if (err < 0) return ERR_PTR(err); - if (opts->netns_id >= 0) { - net = get_net_ns_by_id(net, opts->netns_id); + if (netns_id >= 0) { + net = get_net_ns_by_id(net, netns_id); if (unlikely(!net)) return ERR_PTR(-ENONET); } if (opts_len == NF_BPF_CT_OPTS_SZ) { - if (opts->ct_zone_dir == 0) - opts->ct_zone_dir = NF_CT_DEFAULT_ZONE_DIR; - nf_ct_zone_init(&ct_zone, - opts->ct_zone_id, opts->ct_zone_dir, 0); + if (ct_zone_dir == 0) { + ct_zone_dir = NF_CT_DEFAULT_ZONE_DIR; + opts->ct_zone_dir = ct_zone_dir; + } + nf_ct_zone_init(&ct_zone, ct_zone_id, ct_zone_dir, 0); } else { ct_zone = nf_ct_zone_dflt; } hash = nf_conntrack_find_get(net, &ct_zone, &tuple); - if (opts->netns_id >= 0) + if (netns_id >= 0) put_net(net); if (!hash) return ERR_PTR(-ENOENT); diff --git a/net/netfilter/nf_conntrack_ecache.c b/net/netfilter/nf_conntrack_ecache.c index 9df159448b89..cc8d8e85169f 100644 --- a/net/netfilter/nf_conntrack_ecache.c +++ b/net/netfilter/nf_conntrack_ecache.c @@ -77,7 +77,7 @@ next: hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode); hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode, &evicted_list); - if (time_after(stop, jiffies)) { + if (time_after(jiffies, stop)) { ret = STATE_RESTART; break; } diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index 7ae68d60586a..8a3b9e33e94f 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -427,7 +427,6 @@ static void nf_ct_expect_insert(struct nf_conntrack_expect *exp, exp->timeout += helper->expect_policy[exp->class].timeout * HZ; hlist_add_head_rcu(&exp->lnode, &master_help->expectations); - master_help->expecting[exp->class]++; hlist_add_head_rcu(&exp->hnode, &nf_ct_expect_hash[h]); cnet = nf_ct_pernet(net); @@ -534,6 +533,7 @@ int nf_ct_expect_related_report(struct nf_conntrack_expect *expect, if (ret < 0) goto out; + master_help->expecting[expect->class]++; nf_ct_expect_insert(expect, master_help); nf_ct_expect_event_report(IPEXP_NEW, expect, portid, report); @@ -546,6 +546,39 @@ out: } EXPORT_SYMBOL_GPL(nf_ct_expect_related_report); +int nf_ct_expect_related_pair(struct nf_conntrack_expect *expect[], + unsigned int flags) +{ + struct nf_conn_help *master_help; + int i, ret; + + spin_lock_bh(&nf_conntrack_expect_lock); + master_help = nfct_help(expect[0]->master); + if (!master_help || master_help != nfct_help(expect[1]->master)) { + ret = -EINVAL; + goto out; + } + + for (i = 0; i < 2; i++) { + ret = __nf_ct_expect_check(expect[i], master_help, flags); + if (ret < 0) { + if (i == 1) + master_help->expecting[expect[0]->class]--; + goto out; + } + master_help->expecting[expect[i]->class]++; + } + + for (i = 0; i < 2; i++) { + nf_ct_expect_insert(expect[i], master_help); + nf_ct_expect_event_report(IPEXP_NEW, expect[i], 0, 0); + } +out: + spin_unlock_bh(&nf_conntrack_expect_lock); + return ret; +} +EXPORT_SYMBOL_GPL(nf_ct_expect_related_pair); + void nf_ct_expect_iterate_destroy(bool (*iter)(struct nf_conntrack_expect *e, void *data), void *data) { diff --git a/net/netfilter/nf_conntrack_proto.c b/net/netfilter/nf_conntrack_proto.c index ad96896516b6..7a40e4e0e33e 100644 --- a/net/netfilter/nf_conntrack_proto.c +++ b/net/netfilter/nf_conntrack_proto.c @@ -79,6 +79,12 @@ void nf_ct_l4proto_log_invalid(const struct sk_buff *skb, struct net *net; va_list args; + /* nfnetlink_log may re-enter conntrack attribute dumping and try to + * take ct->lock again via helpers such as tcp_to_nlattr(), so invalid + * conntrack logs must only be emitted after dropping ct->lock. + */ + lockdep_assert_not_held(&ct->lock); + net = nf_ct_net(ct); if (likely(net->ct.sysctl_log_invalid == 0)) return; diff --git a/net/netfilter/nf_conntrack_proto_sctp.c b/net/netfilter/nf_conntrack_proto_sctp.c index 7e10fa65cbdd..71cc920f6856 100644 --- a/net/netfilter/nf_conntrack_proto_sctp.c +++ b/net/netfilter/nf_conntrack_proto_sctp.c @@ -336,10 +336,12 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, struct sctphdr _sctph; const struct sctp_chunkhdr *sch; struct sctp_chunkhdr _sch; + bool log_invalid = false; u_int32_t offset, count; unsigned int *timeouts; unsigned long map[256 / sizeof(unsigned long)] = { 0 }; bool ignore = false; + u8 invalid_type = 0; if (sctp_error(skb, dataoff, state)) return -NF_ACCEPT; @@ -451,10 +453,8 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, /* Invalid */ if (new_state == SCTP_CONNTRACK_MAX) { - nf_ct_l4proto_log_invalid(skb, ct, state, - "Invalid, old_state %d, dir %d, type %d", - old_state, dir, sch->type); - + log_invalid = true; + invalid_type = sch->type; goto out_unlock; } @@ -529,6 +529,10 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, out_unlock: spin_unlock_bh(&ct->lock); + if (log_invalid) + nf_ct_l4proto_log_invalid(skb, ct, state, + "Invalid, old_state %d, dir %d, type %d", + old_state, dir, invalid_type); out: return -NF_ACCEPT; } diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c index ceeed3d7fe52..30b970e2ade5 100644 --- a/net/netfilter/nf_conntrack_proto_tcp.c +++ b/net/netfilter/nf_conntrack_proto_tcp.c @@ -480,37 +480,81 @@ static void tcp_init_sender(struct ip_ct_tcp_state *sender, } } -__printf(6, 7) -static enum nf_ct_tcp_action nf_tcp_log_invalid(const struct sk_buff *skb, - const struct nf_conn *ct, - const struct nf_hook_state *state, - const struct ip_ct_tcp_state *sender, - enum nf_ct_tcp_action ret, - const char *fmt, ...) +enum nf_tcp_invalid_log_type { + NF_TCP_LOG_NONE, + NF_TCP_LOG_OVERSHOT, + NF_TCP_LOG_SEQ_OVER, + NF_TCP_LOG_ACK_OVER, + NF_TCP_LOG_SEQ_UNDER, + NF_TCP_LOG_ACK_UNDER, +}; + +struct nf_tcp_invalid_log { + enum nf_tcp_invalid_log_type type; + u32 value; +}; + +static enum nf_ct_tcp_action +nf_tcp_store_invalid(const struct nf_conn *ct, + const struct ip_ct_tcp_state *sender, + struct nf_tcp_invalid_log *log, + enum nf_ct_tcp_action ret, + enum nf_tcp_invalid_log_type type, + u32 value) { const struct nf_tcp_net *tn = nf_tcp_pernet(nf_ct_net(ct)); - struct va_format vaf; - va_list args; bool be_liberal; be_liberal = sender->flags & IP_CT_TCP_FLAG_BE_LIBERAL || tn->tcp_be_liberal; if (be_liberal) return NFCT_TCP_ACCEPT; - va_start(args, fmt); - vaf.fmt = fmt; - vaf.va = &args; - nf_ct_l4proto_log_invalid(skb, ct, state, "%pV", &vaf); - va_end(args); - + log->type = type; + log->value = value; return ret; } +static void nf_tcp_log_invalid(const struct sk_buff *skb, + const struct nf_conn *ct, + const struct nf_hook_state *state, + const struct nf_tcp_invalid_log *log) +{ + switch (log->type) { + case NF_TCP_LOG_OVERSHOT: + nf_ct_l4proto_log_invalid(skb, ct, state, + "%u bytes more than expected", + log->value); + break; + case NF_TCP_LOG_SEQ_OVER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "SEQ is over upper bound %u (over the window of the receiver)", + log->value); + break; + case NF_TCP_LOG_ACK_OVER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "ACK is over upper bound %u (ACKed data not seen yet)", + log->value); + break; + case NF_TCP_LOG_SEQ_UNDER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "SEQ is under lower bound %u (already ACKed data retransmitted)", + log->value); + break; + case NF_TCP_LOG_ACK_UNDER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "ignored ACK under lower bound %u (possible overly delayed)", + log->value); + break; + case NF_TCP_LOG_NONE: + break; + } +} + static enum nf_ct_tcp_action tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, unsigned int index, const struct sk_buff *skb, unsigned int dataoff, const struct tcphdr *tcph, - const struct nf_hook_state *hook_state) + struct nf_tcp_invalid_log *log) { struct ip_ct_tcp *state = &ct->proto.tcp; struct ip_ct_tcp_state *sender = &state->seen[dir]; @@ -640,31 +684,29 @@ tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, sender->td_end = end; sender->flags |= IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED; - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, - "%u bytes more than expected", overshot); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_IGNORE, + NF_TCP_LOG_OVERSHOT, overshot); } - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID, - "SEQ is over upper bound %u (over the window of the receiver)", - sender->td_maxend + 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_INVALID, + NF_TCP_LOG_SEQ_OVER, sender->td_maxend + 1); } if (!before(sack, receiver->td_end + 1)) - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID, - "ACK is over upper bound %u (ACKed data not seen yet)", - receiver->td_end + 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_INVALID, + NF_TCP_LOG_ACK_OVER, receiver->td_end + 1); /* Is the ending sequence in the receive window (if available)? */ in_recv_win = !receiver->td_maxwin || after(end, sender->td_end - receiver->td_maxwin - 1); if (!in_recv_win) - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, - "SEQ is under lower bound %u (already ACKed data retransmitted)", - sender->td_end - receiver->td_maxwin - 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_IGNORE, + NF_TCP_LOG_SEQ_UNDER, + sender->td_end - receiver->td_maxwin - 1); if (!after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1)) - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, - "ignored ACK under lower bound %u (possible overly delayed)", - receiver->td_end - MAXACKWINDOW(sender) - 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_IGNORE, + NF_TCP_LOG_ACK_UNDER, + receiver->td_end - MAXACKWINDOW(sender) - 1); /* Take into account window scaling (RFC 1323). */ if (!tcph->syn) @@ -719,11 +761,8 @@ tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, return NFCT_TCP_ACCEPT; } -static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, - enum ip_conntrack_dir dir, - int index, - const struct sk_buff *skb, - const struct nf_hook_state *hook_state) +static bool __cold +nf_tcp_handle_invalid(struct nf_conn *ct, enum ip_conntrack_dir dir, int index) { const unsigned int *timeouts; const struct nf_tcp_net *tn; @@ -732,7 +771,7 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, if (!test_bit(IPS_ASSURED_BIT, &ct->status) || test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status)) - return; + return false; /* We don't want to have connections hanging around in ESTABLISHED * state for long time 'just because' conntrack deemed a FIN/RST @@ -747,7 +786,7 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, case TCP_FIN_SET: break; default: - return; + return false; } if (ct->proto.tcp.last_dir != dir && @@ -755,7 +794,7 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, ct->proto.tcp.last_index == TCP_RST_SET)) { expires = nf_ct_expires(ct); if (expires < 120 * HZ) - return; + return false; tn = nf_tcp_pernet(nf_ct_net(ct)); timeouts = nf_ct_timeout_lookup(ct); @@ -764,16 +803,15 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, timeout = READ_ONCE(timeouts[TCP_CONNTRACK_UNACK]); if (expires > timeout) { - nf_ct_l4proto_log_invalid(skb, ct, hook_state, - "packet (index %d, dir %d) response for index %d lower timeout to %u", - index, dir, ct->proto.tcp.last_index, timeout); - WRITE_ONCE(ct->timeout, timeout + nfct_time_stamp); + return true; } } else { ct->proto.tcp.last_index = index; ct->proto.tcp.last_dir = dir; } + + return false; } /* table of valid flag combinations - PUSH, ECE and CWR are always valid */ @@ -969,7 +1007,9 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct, struct net *net = nf_ct_net(ct); struct nf_tcp_net *tn = nf_tcp_pernet(net); enum tcp_conntrack new_state, old_state; + struct nf_tcp_invalid_log log = {}; unsigned int index, *timeouts; + bool lowered_timeout = false; enum nf_ct_tcp_action res; enum ip_conntrack_dir dir; const struct tcphdr *th; @@ -1252,14 +1292,18 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct, } res = tcp_in_window(ct, dir, index, - skb, dataoff, th, state); + skb, dataoff, th, &log); switch (res) { case NFCT_TCP_IGNORE: spin_unlock_bh(&ct->lock); + nf_tcp_log_invalid(skb, ct, state, &log); return NF_ACCEPT; case NFCT_TCP_INVALID: - nf_tcp_handle_invalid(ct, dir, index, skb, state); + lowered_timeout = nf_tcp_handle_invalid(ct, dir, index); spin_unlock_bh(&ct->lock); + nf_tcp_log_invalid(skb, ct, state, &log); + if (lowered_timeout) + nf_ct_l4proto_log_invalid(skb, ct, state, "lowered timeout to UNACK"); return -NF_ACCEPT; case NFCT_TCP_ACCEPT: break; diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index f3f90a866338..e4a70d1d77b0 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -1663,7 +1663,7 @@ static int sip_help_tcp(struct sk_buff *skb, unsigned int protoff, unsigned int matchoff, matchlen; unsigned int msglen, origlen; const char *dptr, *end; - s16 diff, tdiff = 0; + s32 diff, tdiff = 0; int ret = NF_ACCEPT; unsigned long clen; bool term; diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index 99c5b9d671a0..8b1165f2b5a4 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -127,12 +127,18 @@ static int flow_offload_fill_route(struct flow_offload *flow, switch (route->tuple[dir].xmit_type) { case FLOW_OFFLOAD_XMIT_DIRECT: + if (route->tuple[!dir].in.num_tuns) { + flow_tuple->dst_cache = dst; + flow_tuple->dst_cookie = + flow_offload_dst_cookie(flow_tuple); + } else { + dst_release(dst); + } memcpy(flow_tuple->out.h_dest, route->tuple[dir].out.h_dest, ETH_ALEN); memcpy(flow_tuple->out.h_source, route->tuple[dir].out.h_source, ETH_ALEN); flow_tuple->out.ifidx = route->tuple[dir].out.ifindex; - dst_release(dst); break; case FLOW_OFFLOAD_XMIT_XFRM: case FLOW_OFFLOAD_XMIT_NEIGH: @@ -152,9 +158,7 @@ static int flow_offload_fill_route(struct flow_offload *flow, static void nft_flow_dst_release(struct flow_offload *flow, enum flow_offload_tuple_dir dir) { - if (flow->tuplehash[dir].tuple.xmit_type == FLOW_OFFLOAD_XMIT_NEIGH || - flow->tuplehash[dir].tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM) - dst_release(flow->tuplehash[dir].tuple.dst_cache); + dst_release(flow->tuplehash[dir].tuple.dst_cache); } void flow_offload_route_init(struct flow_offload *flow, @@ -328,27 +332,26 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow) flow->timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow); err = rhashtable_insert_fast(&flow_table->rhashtable, - &flow->tuplehash[0].node, + &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node, nf_flow_offload_rhash_params); if (err < 0) return err; + /* GC only iterates original-direction entries; publish original last. */ err = rhashtable_insert_fast(&flow_table->rhashtable, - &flow->tuplehash[1].node, + &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node, nf_flow_offload_rhash_params); if (err < 0) { rhashtable_remove_fast(&flow_table->rhashtable, - &flow->tuplehash[0].node, + &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node, nf_flow_offload_rhash_params); return err; } nf_ct_refresh(flow->ct, NF_CT_DAY); - if (nf_flowtable_hw_offload(flow_table)) { - __set_bit(NF_FLOW_HW, &flow->flags); + if (nf_flowtable_hw_offload(flow_table)) nf_flow_offload_add(flow_table, flow); - } return 0; } @@ -369,7 +372,8 @@ void flow_offload_refresh(struct nf_flowtable *flow_table, test_bit(NF_FLOW_CLOSING, &flow->flags)) return; - nf_flow_offload_add(flow_table, flow); + if (test_bit(NF_FLOW_HW, &flow->flags)) + nf_flow_offload_refresh(flow_table, flow); } EXPORT_SYMBOL_GPL(flow_offload_refresh); diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c index 29e93ac1e2e4..c9e332fafcb5 100644 --- a/net/netfilter/nf_flow_table_ip.c +++ b/net/netfilter/nf_flow_table_ip.c @@ -299,8 +299,7 @@ static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu) static inline bool nf_flow_dst_check(struct flow_offload_tuple *tuple) { - if (tuple->xmit_type != FLOW_OFFLOAD_XMIT_NEIGH && - tuple->xmit_type != FLOW_OFFLOAD_XMIT_XFRM) + if (!tuple->dst_cache) return true; return dst_check(tuple->dst_cache, tuple->dst_cookie); @@ -311,6 +310,7 @@ static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb, struct dst_entry *dst) { skb_orphan(skb); + skb_dst_drop(skb); skb_dst_set_noref(skb, dst); dst_output(state->net, state->sk, skb); return NF_STOLEN; @@ -590,10 +590,10 @@ static int nf_flow_pppoe_push(struct sk_buff *skb, u16 id, static int nf_flow_tunnel_ipip_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, - __be32 *ip_daddr) + struct dst_entry *dst, __be32 *ip_daddr) { struct iphdr *iph = (struct iphdr *)skb_network_header(skb); - struct rtable *rt = dst_rtable(tuple->dst_cache); + struct rtable *rt = dst_rtable(dst); u8 tos = iph->tos, ttl = iph->ttl; __be16 frag_off = iph->frag_off; u32 headroom = sizeof(*iph); @@ -636,21 +636,22 @@ static int nf_flow_tunnel_ipip_push(struct net *net, struct sk_buff *skb, static int nf_flow_tunnel_v4_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, - __be32 *ip_daddr) + struct dst_entry *dst, __be32 *ip_daddr) { if (tuple->tun_num) - return nf_flow_tunnel_ipip_push(net, skb, tuple, ip_daddr); + return nf_flow_tunnel_ipip_push(net, skb, tuple, dst, ip_daddr); return 0; } static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, + struct dst_entry *dst, struct in6_addr **ip6_daddr) { struct ipv6hdr *ip6h = (struct ipv6hdr *)skb_network_header(skb); - struct rtable *rt = dst_rtable(tuple->dst_cache); __u8 dsfield = ipv6_get_dsfield(ip6h); + struct rtable *rt = dst_rtable(dst); struct flowi6 fl6 = { .daddr = tuple->tun.src_v6, .saddr = tuple->tun.dst_v6, @@ -696,10 +697,11 @@ static int nf_flow_tunnel_ip6ip6_push(struct net *net, struct sk_buff *skb, static int nf_flow_tunnel_v6_push(struct net *net, struct sk_buff *skb, struct flow_offload_tuple *tuple, + struct dst_entry *dst, struct in6_addr **ip6_daddr) { if (tuple->tun_num) - return nf_flow_tunnel_ip6ip6_push(net, skb, tuple, ip6_daddr); + return nf_flow_tunnel_ip6ip6_push(net, skb, tuple, dst, ip6_daddr); return 0; } @@ -842,7 +844,8 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, other_tuple = &flow->tuplehash[!dir].tuple; ip_daddr = other_tuple->src_v4.s_addr; - if (nf_flow_tunnel_v4_push(state->net, skb, other_tuple, &ip_daddr) < 0) + if (nf_flow_tunnel_v4_push(state->net, skb, other_tuple, + tuplehash->tuple.dst_cache, &ip_daddr) < 0) return NF_DROP; switch (tuplehash->tuple.xmit_type) { @@ -859,6 +862,7 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, return NF_DROP; } xmit.dest = neigh->ha; + skb_dst_drop(skb); skb_dst_set_noref(skb, &rt->dst); break; case FLOW_OFFLOAD_XMIT_DIRECT: @@ -1158,6 +1162,7 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, ip6_daddr = &other_tuple->src_v6; if (nf_flow_tunnel_v6_push(state->net, skb, other_tuple, + tuplehash->tuple.dst_cache, &ip6_daddr) < 0) return NF_DROP; @@ -1175,6 +1180,7 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, return NF_DROP; } xmit.dest = neigh->ha; + skb_dst_drop(skb); skb_dst_set_noref(skb, &rt->dst); break; case FLOW_OFFLOAD_XMIT_DIRECT: diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c index 002ec15d988b..801a3dd9ceea 100644 --- a/net/netfilter/nf_flow_table_offload.c +++ b/net/netfilter/nf_flow_table_offload.c @@ -1101,9 +1101,17 @@ nf_flow_offload_work_alloc(struct nf_flowtable *flowtable, return offload; } +static bool nf_flow_offload_unsupported(struct flow_offload *flow) +{ + if (flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.tun_num || + flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.tun_num) + return true; -void nf_flow_offload_add(struct nf_flowtable *flowtable, - struct flow_offload *flow) + return false; +} + +void nf_flow_offload_refresh(struct nf_flowtable *flowtable, + struct flow_offload *flow) { struct flow_offload_work *offload; @@ -1114,6 +1122,16 @@ void nf_flow_offload_add(struct nf_flowtable *flowtable, flow_offload_queue_work(offload); } +void nf_flow_offload_add(struct nf_flowtable *flowtable, + struct flow_offload *flow) +{ + if (nf_flow_offload_unsupported(flow)) + return; + + set_bit(NF_FLOW_HW, &flow->flags); + nf_flow_offload_refresh(flowtable, flow); +} + void nf_flow_offload_del(struct nf_flowtable *flowtable, struct flow_offload *flow) { diff --git a/net/netfilter/nf_log_syslog.c b/net/netfilter/nf_log_syslog.c index e37b09b3203b..5ffde27d450a 100644 --- a/net/netfilter/nf_log_syslog.c +++ b/net/netfilter/nf_log_syslog.c @@ -1014,7 +1014,7 @@ err1: return ret; } -static void __net_exit nf_log_syslog_net_exit(struct net *net) +static void __net_exit nf_log_syslog_net_pre_exit(struct net *net) { nf_log_unset(net, &nf_ip_logger); nf_log_unset(net, &nf_arp_logger); @@ -1025,7 +1025,7 @@ static void __net_exit nf_log_syslog_net_exit(struct net *net) static struct pernet_operations nf_log_syslog_net_ops = { .init = nf_log_syslog_net_init, - .exit = nf_log_syslog_net_exit, + .pre_exit = nf_log_syslog_net_pre_exit, }; static int __init nf_log_syslog_init(void) diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c index 67c04d8143ab..133bd713fe0c 100644 --- a/net/netfilter/nf_nat_sip.c +++ b/net/netfilter/nf_nat_sip.c @@ -289,13 +289,24 @@ next: /* Mangle destination port for Cisco phones, then fix up checksums */ if (dir == IP_CT_DIR_REPLY && ct_sip_info->forced_dport) { + int doff = *dptr - (const char *)skb->data; struct udphdr *uh; + if (doff <= 0) { + DEBUG_NET_WARN_ON_ONCE(1); + return NF_DROP; + } + + /* ct_sip_info->forced_dport only expected with UDP */ + if (nf_ct_protonum(ct) != IPPROTO_UDP) + return NF_DROP; + if (skb_ensure_writable(skb, skb->len)) { nf_ct_helper_log(skb, ct, "cannot mangle packet"); return NF_DROP; } + *dptr = skb->data + doff; uh = (void *)skb->data + protoff; uh->dest = ct_sip_info->forced_dport; @@ -310,7 +321,7 @@ next: } static void nf_nat_sip_seq_adjust(struct sk_buff *skb, unsigned int protoff, - s16 off) + s32 off) { enum ip_conntrack_info ctinfo; struct nf_conn *ct = nf_ct_get(skb, &ctinfo); @@ -581,6 +592,7 @@ static unsigned int nf_nat_sdp_media(struct sk_buff *skb, unsigned int protoff, unsigned int medialen, union nf_inet_addr *rtp_addr) { + struct nf_conntrack_expect *rtp_pair[2] = { rtp_exp, rtcp_exp }; enum ip_conntrack_info ctinfo; struct nf_conn *ct = nf_ct_get(skb, &ctinfo); enum ip_conntrack_dir dir = CTINFO2DIR(ctinfo); @@ -611,24 +623,15 @@ static unsigned int nf_nat_sdp_media(struct sk_buff *skb, unsigned int protoff, int ret; rtp_exp->tuple.dst.u.udp.port = htons(port); - ret = nf_ct_expect_related(rtp_exp, - NF_CT_EXP_F_SKIP_MASTER); - if (ret == -EBUSY) - continue; - else if (ret < 0) { - port = 0; - break; - } rtcp_exp->tuple.dst.u.udp.port = htons(port + 1); - ret = nf_ct_expect_related(rtcp_exp, - NF_CT_EXP_F_SKIP_MASTER); + + ret = nf_ct_expect_related_pair(rtp_pair, + NF_CT_EXP_F_SKIP_MASTER); if (ret == 0) break; - else if (ret == -EBUSY) { - nf_ct_unexpect_related(rtp_exp); + else if (ret == -EBUSY) continue; - } else if (ret < 0) { - nf_ct_unexpect_related(rtp_exp); + else if (ret < 0) { port = 0; break; } diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 4884f7f7aaee..af357f6c5070 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -45,8 +45,6 @@ enum { NFT_VALIDATE_DO, }; -static struct rhltable nft_objname_ht; - static u32 nft_chain_hash(const void *data, u32 len, u32 seed); static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed); static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *); @@ -1635,6 +1633,10 @@ static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info, if (err) goto err_chain_ht; + err = rhltable_init(&table->objname_ht, &nft_objname_ht_params); + if (err < 0) + goto err_obj_ht; + INIT_LIST_HEAD(&table->chains); INIT_LIST_HEAD(&table->sets); INIT_LIST_HEAD(&table->objects); @@ -1653,6 +1655,8 @@ static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info, list_add_tail_rcu(&table->list, &nft_net->tables); return 0; err_trans: + rhltable_destroy(&table->objname_ht); +err_obj_ht: rhltable_destroy(&table->chains_ht); err_chain_ht: kfree(table->udata); @@ -1819,6 +1823,7 @@ static void nf_tables_table_destroy(struct nft_table *table) return; rhltable_destroy(&table->chains_ht); + rhltable_destroy(&table->objname_ht); kfree(table->name); kfree(table->udata); kfree(table); @@ -6563,6 +6568,9 @@ static int nft_get_set_elem(struct nft_ctx *ctx, const struct nft_set *set, if (err < 0) return err; + if (!elem.priv) + return 0; + err = -ENOMEM; skb = nlmsg_new(NLMSG_GOODSIZE, GFP_ATOMIC); if (skb == NULL) @@ -8083,7 +8091,7 @@ void nft_unregister_obj(struct nft_object_type *obj_type) EXPORT_SYMBOL_GPL(nft_unregister_obj); struct nft_object *nft_obj_lookup(const struct net *net, - const struct nft_table *table, + struct nft_table *table, const struct nlattr *nla, u32 objtype, u8 genmask) { @@ -8099,7 +8107,7 @@ struct nft_object *nft_obj_lookup(const struct net *net, !lockdep_commit_lock_is_held(net)); rcu_read_lock(); - list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params); + list = rhltable_lookup(&table->objname_ht, &k, nft_objname_ht_params); if (!list) goto out; @@ -8379,7 +8387,7 @@ static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info, if (err < 0) goto err_trans; - err = rhltable_insert(&nft_objname_ht, &obj->rhlhead, + err = rhltable_insert(&table->objname_ht, &obj->rhlhead, nft_objname_ht_params); if (err < 0) goto err_obj_ht; @@ -8564,8 +8572,8 @@ nf_tables_getobj_single(u32 portid, const struct nfnl_info *info, struct netlink_ext_ack *extack = info->extack; u8 genmask = nft_genmask_cur(info->net); u8 family = info->nfmsg->nfgen_family; - const struct nft_table *table; struct net *net = info->net; + struct nft_table *table; struct nft_object *obj; struct sk_buff *skb2; u32 objtype; @@ -10434,9 +10442,9 @@ static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain) nf_tables_commit_chain_free_rules_old(g0); } -static void nft_obj_del(struct nft_object *obj) +static void nft_obj_del(struct nft_table *table, struct nft_object *obj) { - rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params); + rhltable_remove(&table->objname_ht, &obj->rhlhead, nft_objname_ht_params); list_del_rcu(&obj->list); } @@ -11121,7 +11129,7 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) break; case NFT_MSG_DELOBJ: case NFT_MSG_DESTROYOBJ: - nft_obj_del(nft_trans_obj(trans)); + nft_obj_del(table, nft_trans_obj(trans)); nf_tables_obj_notify(&ctx, nft_trans_obj(trans), trans->msg_type); break; @@ -11413,7 +11421,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action) nft_trans_destroy(trans); } else { nft_use_dec_restore(&table->use); - nft_obj_del(nft_trans_obj(trans)); + nft_obj_del(table, nft_trans_obj(trans)); } break; case NFT_MSG_DELOBJ: @@ -12040,7 +12048,7 @@ static void __nft_release_table(struct net *net, struct nft_table *table) nft_set_destroy(&ctx, set); } list_for_each_entry_safe(obj, ne, &table->objects, list) { - nft_obj_del(obj); + nft_obj_del(table, obj); nft_use_dec(&table->use); nft_obj_destroy(&ctx, obj); } @@ -12222,10 +12230,6 @@ static int __init nf_tables_module_init(void) if (err < 0) goto err_netdev_notifier; - err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params); - if (err < 0) - goto err_rht_objname; - err = nft_offload_init(); if (err < 0) goto err_offload; @@ -12248,8 +12252,6 @@ err_nfnl_subsys: err_netlink_notifier: nft_offload_exit(); err_offload: - rhltable_destroy(&nft_objname_ht); -err_rht_objname: unregister_netdevice_notifier(&nf_tables_flowtable_notifier); err_netdev_notifier: nf_tables_core_module_exit(); @@ -12271,7 +12273,6 @@ static void __exit nf_tables_module_exit(void) unregister_pernet_subsys(&nf_tables_net_ops); cancel_work_sync(&trans_gc_work); rcu_barrier(); - rhltable_destroy(&nft_objname_ht); nf_tables_core_module_exit(); } diff --git a/net/netfilter/nf_tables_offload.c b/net/netfilter/nf_tables_offload.c index 8998a24651ff..0ac3c26dfb3d 100644 --- a/net/netfilter/nf_tables_offload.c +++ b/net/netfilter/nf_tables_offload.c @@ -558,7 +558,7 @@ static void nft_flow_rule_offload_abort(struct net *net, break; } - if (WARN_ON_ONCE(err)) + if (WARN_ON_ONCE(err && err != -ENOMEM)) break; } } diff --git a/net/netfilter/nfnetlink_cthelper.c b/net/netfilter/nfnetlink_cthelper.c index 2cbcca9110db..f062ac210343 100644 --- a/net/netfilter/nfnetlink_cthelper.c +++ b/net/netfilter/nfnetlink_cthelper.c @@ -316,6 +316,8 @@ nfnl_cthelper_update_policy_one(const struct nf_conntrack_expect_policy *policy, new_policy->max_expected = ntohl(nla_get_be32(tb[NFCTH_POLICY_EXPECT_MAX])); + if (!new_policy->max_expected) + new_policy->max_expected = NF_CT_EXPECT_MAX_CNT; if (new_policy->max_expected > NF_CT_EXPECT_MAX_CNT) return -EINVAL; diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c index fa3657599861..6c7fa2ed34f5 100644 --- a/net/netfilter/nfnetlink_log.c +++ b/net/netfilter/nfnetlink_log.c @@ -676,7 +676,7 @@ __build_packet_message(struct nfnl_log_net *log, goto nla_put_failure; if (skb_copy_bits(skb, 0, nla_data(nla), data_len)) - BUG(); + goto nla_put_failure; } nlh->nlmsg_len = inst->skb->tail - old_tail; @@ -698,6 +698,21 @@ static const struct nf_loginfo default_loginfo = { }, }; +static unsigned int nfulnl_get_copy_len(const struct nf_loginfo *li, + const struct sk_buff *skb, + unsigned int copy_len) +{ + unsigned int len = skb->len; + + if ((li->u.ulog.flags & NF_LOG_F_COPY_LEN) && + li->u.ulog.copy_len < copy_len) + copy_len = li->u.ulog.copy_len; + if (!skb_frags_readable(skb)) + len = skb_headlen(skb); + + return min(len, copy_len); +} + /* log handler for internal netfilter logging api */ static void nfulnl_log_packet(struct net *net, @@ -790,14 +805,7 @@ nfulnl_log_packet(struct net *net, break; case NFULNL_COPY_PACKET: - data_len = inst->copy_range; - if ((li->u.ulog.flags & NF_LOG_F_COPY_LEN) && - (li->u.ulog.copy_len < data_len)) - data_len = li->u.ulog.copy_len; - - if (data_len > skb->len) - data_len = skb->len; - + data_len = nfulnl_get_copy_len(li, skb, inst->copy_range); size += nla_total_size(data_len); break; @@ -1162,21 +1170,26 @@ static int __net_init nfnl_log_net_init(struct net *net) return 0; } -static void __net_exit nfnl_log_net_exit(struct net *net) +static void __net_exit nfnl_log_net_pre_exit(struct net *net) { - struct nfnl_log_net *log = nfnl_log_pernet(net); - unsigned int i; - #ifdef CONFIG_PROC_FS remove_proc_entry("nfnetlink_log", net->nf.proc_netfilter); #endif nf_log_unset(net, &nfulnl_logger); +} + +static void __net_exit nfnl_log_net_exit(struct net *net) +{ + struct nfnl_log_net *log = nfnl_log_pernet(net); + unsigned int i; + for (i = 0; i < INSTANCE_BUCKETS; i++) WARN_ON_ONCE(!hlist_empty(&log->instance_table[i])); } static struct pernet_operations nfnl_log_net_ops = { .init = nfnl_log_net_init, + .pre_exit = nfnl_log_net_pre_exit, .exit = nfnl_log_net_exit, .id = &nfnl_log_net_id, .size = sizeof(struct nfnl_log_net), diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index 35d4c6c628ff..b8aaf39cb4d8 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -690,6 +690,17 @@ static int nfqnl_put_master_ifindex(struct sk_buff *nlskb, int attr, } #endif +static unsigned int nfqnl_get_data_len(const struct sk_buff *entskb, + unsigned int copy_range) +{ + unsigned int data_len = entskb->len; + + if (!skb_frags_readable(entskb)) + data_len = skb_headlen(entskb); + + return min(data_len, copy_range); +} + static struct sk_buff * nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, struct nf_queue_entry *entry, @@ -755,10 +766,7 @@ nfqnl_build_packet_message(struct net *net, struct nfqnl_instance *queue, nf_queue_checksum_help(entskb)) return NULL; - data_len = READ_ONCE(queue->copy_range); - if (data_len > entskb->len) - data_len = entskb->len; - + data_len = nfqnl_get_data_len(entskb, READ_ONCE(queue->copy_range)); hlen = skb_zerocopy_headlen(entskb); hlen = min_t(unsigned int, hlen, data_len); size += sizeof(struct nlattr) + hlen; diff --git a/net/netfilter/nft_lookup.c b/net/netfilter/nft_lookup.c index ba512e94b402..19887439847d 100644 --- a/net/netfilter/nft_lookup.c +++ b/net/netfilter/nft_lookup.c @@ -103,13 +103,13 @@ void nft_lookup_eval(const struct nft_expr *expr, bool found; ext = nft_set_do_lookup(net, set, ®s->data[priv->sreg]); + if (!ext) + ext = nft_set_catchall_lookup(net, set); + found = !!ext ^ priv->invert; if (!found) { - ext = nft_set_catchall_lookup(net, set); - if (!ext) { - regs->verdict.code = NFT_BREAK; - return; - } + regs->verdict.code = NFT_BREAK; + return; } if (ext) { diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c index 391539a1ceaa..8a4472fd77d9 100644 --- a/net/netfilter/nft_payload.c +++ b/net/netfilter/nft_payload.c @@ -259,9 +259,7 @@ nla_put_failure: static bool nft_payload_offload_mask(struct nft_offload_reg *reg, u32 priv_len, u32 field_len) { - unsigned int remainder, delta, k; struct nft_data mask = {}; - __be32 remainder_mask; if (priv_len == field_len) { memset(®->mask, 0xff, priv_len); @@ -270,15 +268,7 @@ static bool nft_payload_offload_mask(struct nft_offload_reg *reg, return false; } - memset(&mask, 0xff, field_len); - remainder = priv_len % sizeof(u32); - if (remainder) { - k = priv_len / sizeof(u32); - delta = field_len - priv_len; - remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1)); - mask.data[k] = (__force u32)remainder_mask; - } - + memset(&mask, 0xff, priv_len); memcpy(®->mask, &mask, field_len); return true; diff --git a/net/netfilter/nft_set_rbtree.c b/net/netfilter/nft_set_rbtree.c index 018bbb6df4ce..6222e9bb57bc 100644 --- a/net/netfilter/nft_set_rbtree.c +++ b/net/netfilter/nft_set_rbtree.c @@ -184,10 +184,14 @@ nft_rbtree_get(const struct net *net, const struct nft_set *set, if (!interval || nft_set_elem_expired(interval->from)) return ERR_PTR(-ENOENT); - if (flags & NFT_SET_ELEM_INTERVAL_END) + if (flags & NFT_SET_ELEM_INTERVAL_END) { + if (!interval->to) + return NULL; + rbe = container_of(interval->to, struct nft_rbtree_elem, ext); - else + } else { rbe = container_of(interval->from, struct nft_rbtree_elem, ext); + } return &rbe->priv; } diff --git a/net/netfilter/xt_connmark.c b/net/netfilter/xt_connmark.c index 4277084de2e7..2cf27f7d59b9 100644 --- a/net/netfilter/xt_connmark.c +++ b/net/netfilter/xt_connmark.c @@ -112,6 +112,16 @@ static int connmark_tg_check(const struct xt_tgchk_param *par) return ret; } +static int connmark_tg_check_v2(const struct xt_tgchk_param *par) +{ + const struct xt_connmark_tginfo2 *info = par->targinfo; + + if (info->shift_dir > D_SHIFT_RIGHT || info->shift_bits >= 32) + return -EINVAL; + + return connmark_tg_check(par); +} + static void connmark_tg_destroy(const struct xt_tgdtor_param *par) { nf_ct_netns_put(par->net, par->family); @@ -162,7 +172,7 @@ static struct xt_target connmark_tg_reg[] __read_mostly = { .name = "CONNMARK", .revision = 2, .family = NFPROTO_IPV4, - .checkentry = connmark_tg_check, + .checkentry = connmark_tg_check_v2, .target = connmark_tg_v2, .targetsize = sizeof(struct xt_connmark_tginfo2), .destroy = connmark_tg_destroy, @@ -183,7 +193,7 @@ static struct xt_target connmark_tg_reg[] __read_mostly = { .name = "CONNMARK", .revision = 2, .family = NFPROTO_IPV6, - .checkentry = connmark_tg_check, + .checkentry = connmark_tg_check_v2, .target = connmark_tg_v2, .targetsize = sizeof(struct xt_connmark_tginfo2), .destroy = connmark_tg_destroy, diff --git a/net/netfilter/xt_hashlimit.c b/net/netfilter/xt_hashlimit.c index 2704b4b60d1e..9af0fa895f73 100644 --- a/net/netfilter/xt_hashlimit.c +++ b/net/netfilter/xt_hashlimit.c @@ -117,6 +117,7 @@ struct xt_hashlimit_htable { refcount_t use; u_int8_t family; bool rnd_initialized; + bool ratematch; struct hashlimit_cfg3 cfg; /* config */ @@ -323,6 +324,7 @@ static int htable_create(struct net *net, struct hashlimit_cfg3 *cfg, kvfree(hinfo); return -ENOMEM; } + hinfo->ratematch = !!(cfg->mode & XT_HASHLIMIT_RATE_MATCH); spin_lock_init(&hinfo->lock); switch (revision) { @@ -872,7 +874,10 @@ static int hashlimit_mt_check_common(const struct xt_mtchk_param *par, } /* Check for overflow. */ - if (revision >= 3 && cfg->mode & XT_HASHLIMIT_RATE_MATCH) { + if (cfg->mode & XT_HASHLIMIT_RATE_MATCH) { + if (revision < 3) + return -EINVAL; + if (cfg->avg == 0 || cfg->avg > U32_MAX) { pr_info_ratelimited("invalid rate\n"); return -ERANGE; @@ -905,6 +910,15 @@ static int hashlimit_mt_check_common(const struct xt_mtchk_param *par, mutex_unlock(&hashlimit_mutex); return ret; } + } else { + if ((cfg->mode & XT_HASHLIMIT_RATE_MATCH && + !(*hinfo)->ratematch) || + (!(cfg->mode & XT_HASHLIMIT_RATE_MATCH) && + (*hinfo)->ratematch)) { + mutex_unlock(&hashlimit_mutex); + htable_put(*hinfo); + return -EINVAL; + } } mutex_unlock(&hashlimit_mutex); diff --git a/net/netfilter/xt_nat.c b/net/netfilter/xt_nat.c index b4f7bbc3f3ca..51c7f7ce88d9 100644 --- a/net/netfilter/xt_nat.c +++ b/net/netfilter/xt_nat.c @@ -26,6 +26,15 @@ static int xt_nat_checkentry_v0(const struct xt_tgchk_param *par) static int xt_nat_checkentry(const struct xt_tgchk_param *par) { + switch (par->family) { + case NFPROTO_IPV4: + case NFPROTO_IPV6: + case NFPROTO_INET: + break; + default: + return -EINVAL; + } + return nf_ct_netns_get(par->net, par->family); } diff --git a/net/netfilter/xt_physdev.c b/net/netfilter/xt_physdev.c index dd98f758176c..a388881c68d4 100644 --- a/net/netfilter/xt_physdev.c +++ b/net/netfilter/xt_physdev.c @@ -130,11 +130,6 @@ static int physdev_mt_check(const struct xt_mtchk_param *par) if (X(physoutdev)) return -ENAMETOOLONG; } - - if (X(in_mask)) - return -ENAMETOOLONG; - if (X(out_mask)) - return -ENAMETOOLONG; #undef X if (!brnf_probed) { diff --git a/net/netfilter/xt_rateest.c b/net/netfilter/xt_rateest.c index b1d736c15fcb..7c05b6342578 100644 --- a/net/netfilter/xt_rateest.c +++ b/net/netfilter/xt_rateest.c @@ -16,7 +16,7 @@ xt_rateest_mt(const struct sk_buff *skb, struct xt_action_param *par) { const struct xt_rateest_match_info *info = par->matchinfo; struct gnet_stats_rate_est64 sample = {0}; - u_int32_t bps1, bps2, pps1, pps2; + u64 bps1, bps2, pps1, pps2; bool ret = true; gen_estimator_read(&info->est1->rate_est, &sample); diff --git a/net/netfilter/xt_u32.c b/net/netfilter/xt_u32.c index 117d4615d668..dabbaa742874 100644 --- a/net/netfilter/xt_u32.c +++ b/net/netfilter/xt_u32.c @@ -14,8 +14,8 @@ #include <linux/netfilter/x_tables.h> #include <linux/netfilter/xt_u32.h> -static bool u32_match_it(const struct xt_u32 *data, - const struct sk_buff *skb) +static int u32_match_it(const struct xt_u32 *data, + const struct sk_buff *skb) { const struct xt_u32_test *ct; unsigned int testind; @@ -40,7 +40,8 @@ static bool u32_match_it(const struct xt_u32 *data, return false; if (skb_copy_bits(skb, pos, &n, sizeof(n)) < 0) - BUG(); + return -1; + val = ntohl(n); nnums = ct->nnums; @@ -68,7 +69,7 @@ static bool u32_match_it(const struct xt_u32 *data, if (skb_copy_bits(skb, at + pos, &n, sizeof(n)) < 0) - BUG(); + return -1; val = ntohl(n); break; } @@ -90,9 +91,14 @@ static bool u32_match_it(const struct xt_u32 *data, static bool u32_mt(const struct sk_buff *skb, struct xt_action_param *par) { const struct xt_u32 *data = par->matchinfo; - bool ret; + int ret; ret = u32_match_it(data, skb); + if (ret < 0) { + par->hotdrop = true; + return false; + } + return ret ^ data->invert; } @@ -100,7 +106,7 @@ static int u32_mt_checkentry(const struct xt_mtchk_param *par) { const struct xt_u32 *data = par->matchinfo; const struct xt_u32_test *ct; - unsigned int i; + unsigned int i, j; if (data->ntests > ARRAY_SIZE(data->tests)) return -EINVAL; @@ -111,6 +117,16 @@ static int u32_mt_checkentry(const struct xt_mtchk_param *par) if (ct->nnums > ARRAY_SIZE(ct->location) || ct->nvalues > ARRAY_SIZE(ct->value)) return -EINVAL; + + for (j = 1; j < ct->nnums; ++j) { + switch (ct->location[j].nextop) { + case XT_U32_LEFTSH: + case XT_U32_RIGHTSH: + if (ct->location[j].number >= 32) + return -EINVAL; + break; + } + } } return 0; diff --git a/net/openvswitch/actions.c b/net/openvswitch/actions.c index 140388a18ae0..dc5ff859f114 100644 --- a/net/openvswitch/actions.c +++ b/net/openvswitch/actions.c @@ -837,12 +837,8 @@ static void do_output(struct datapath *dp, struct sk_buff *skb, int out_port, u16 mru = OVS_CB(skb)->mru; u32 cutlen = OVS_CB(skb)->cutlen; - if (unlikely(cutlen > 0)) { - if (skb->len - cutlen > ovs_mac_header_len(key)) - pskb_trim(skb, skb->len - cutlen); - else - pskb_trim(skb, ovs_mac_header_len(key)); - } + if (unlikely(cutlen < skb->len)) + pskb_trim(skb, max(cutlen, ovs_mac_header_len(key))); if (likely(!mru || (skb->len <= mru + vport->dev->hard_header_len))) { @@ -1112,6 +1108,10 @@ static int execute_masked_set_action(struct sk_buff *skb, return err; } +/* When 'last' is true, recirc() should always consume the 'skb'. + * Otherwise, recirc() should keep 'skb' intact regardless what + * actions are executed on recirculation. + */ static int execute_recirc(struct datapath *dp, struct sk_buff *skb, struct sw_flow_key *key, const struct nlattr *a, bool last) @@ -1122,8 +1122,12 @@ static int execute_recirc(struct datapath *dp, struct sk_buff *skb, int err; err = ovs_flow_key_update(skb, key); - if (err) + if (err) { + if (last) + ovs_kfree_skb_reason(skb, + OVS_DROP_ACTION_ERROR); return err; + } } BUG_ON(!is_flow_key_valid(key)); @@ -1234,7 +1238,7 @@ static void execute_psample(struct datapath *dp, struct sk_buff *skb, psample_group.net = ovs_dp_get_net(dp); md.in_ifindex = OVS_CB(skb)->input_vport->dev->ifindex; - md.trunc_size = skb->len - OVS_CB(skb)->cutlen; + md.trunc_size = min(skb->len, OVS_CB(skb)->cutlen); md.rate_as_probability = 1; rate = OVS_CB(skb)->probability ? OVS_CB(skb)->probability : U32_MAX; @@ -1284,22 +1288,21 @@ static int do_execute_actions(struct datapath *dp, struct sk_buff *skb, clone = skb_clone(skb, GFP_ATOMIC); if (clone) do_output(dp, clone, port, key); - OVS_CB(skb)->cutlen = 0; + OVS_CB(skb)->cutlen = U32_MAX; break; } case OVS_ACTION_ATTR_TRUNC: { struct ovs_action_trunc *trunc = nla_data(a); - if (skb->len > trunc->max_len) - OVS_CB(skb)->cutlen = skb->len - trunc->max_len; + OVS_CB(skb)->cutlen = trunc->max_len; break; } case OVS_ACTION_ATTR_USERSPACE: output_userspace(dp, skb, key, a, attr, len, OVS_CB(skb)->cutlen); - OVS_CB(skb)->cutlen = 0; + OVS_CB(skb)->cutlen = U32_MAX; if (nla_is_last(a, rem)) { consume_skb(skb); return 0; @@ -1377,7 +1380,7 @@ static int do_execute_actions(struct datapath *dp, struct sk_buff *skb, if (!is_flow_key_valid(key)) { err = ovs_flow_key_update(skb, key); if (err) - return err; + break; } err = ovs_ct_execute(ovs_dp_get_net(dp), skb, key, @@ -1453,7 +1456,7 @@ static int do_execute_actions(struct datapath *dp, struct sk_buff *skb, case OVS_ACTION_ATTR_PSAMPLE: execute_psample(dp, skb, a); - OVS_CB(skb)->cutlen = 0; + OVS_CB(skb)->cutlen = U32_MAX; if (nla_is_last(a, rem)) { consume_skb(skb); return 0; diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index f0164817d9b7..ae69b2cabab9 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -276,7 +276,7 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key) upcall.portid = ovs_vport_find_upcall_portid(p, skb); upcall.mru = OVS_CB(skb)->mru; - error = ovs_dp_upcall(dp, skb, key, &upcall, 0); + error = ovs_dp_upcall(dp, skb, key, &upcall, U32_MAX); switch (error) { case 0: case -EAGAIN: @@ -457,7 +457,8 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, struct sk_buff *nskb = NULL; struct sk_buff *user_skb = NULL; /* to be queued to userspace */ struct nlattr *nla; - size_t len; + size_t msg_size; + size_t skb_len; unsigned int hlen; int err, dp_ifindex; u64 hash; @@ -478,7 +479,8 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, skb = nskb; } - if (nla_attr_size(skb->len) > USHRT_MAX) { + skb_len = min(skb->len, cutlen); + if (nla_attr_size(skb_len) > USHRT_MAX) { err = -EFBIG; goto out; } @@ -493,13 +495,13 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, * padding logic. Only perform zerocopy if padding is not required. */ if (dp->user_features & OVS_DP_F_UNALIGNED) - hlen = skb_zerocopy_headlen(skb); + hlen = min(skb_zerocopy_headlen(skb), cutlen); else - hlen = skb->len; + hlen = skb_len; - len = upcall_msg_size(upcall_info, hlen - cutlen, - OVS_CB(skb)->acts_origlen); - user_skb = genlmsg_new(len, GFP_ATOMIC); + msg_size = upcall_msg_size(upcall_info, hlen, + OVS_CB(skb)->acts_origlen); + user_skb = genlmsg_new(msg_size, GFP_ATOMIC); if (!user_skb) { err = -ENOMEM; goto out; @@ -560,7 +562,7 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, } /* Add OVS_PACKET_ATTR_LEN when packet is truncated */ - if (cutlen > 0 && + if (skb_len < skb->len && nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN, skb->len)) { err = -ENOBUFS; goto out; @@ -585,9 +587,9 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, err = -ENOBUFS; goto out; } - nla->nla_len = nla_attr_size(skb->len - cutlen); + nla->nla_len = nla_attr_size(skb_len); - err = skb_zerocopy(user_skb, skb, skb->len - cutlen, hlen); + err = skb_zerocopy(user_skb, skb, skb_len, hlen); if (err) goto out; @@ -644,6 +646,7 @@ static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info) packet->ignore_df = 1; } OVS_CB(packet)->mru = mru; + OVS_CB(packet)->cutlen = U32_MAX; if (a[OVS_PACKET_ATTR_HASH]) { hash = nla_get_u64(a[OVS_PACKET_ATTR_HASH]); @@ -1110,9 +1113,8 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) error = -EEXIST; goto err_unlock_ovs; } - /* The flow identifier has to be the same for flow updates. - * Look for any overlapping flow. - */ + + /* Look for any overlapping flow. */ if (unlikely(!ovs_flow_cmp(flow, &match))) { if (ovs_identifier_is_key(&flow->id)) flow = ovs_flow_tbl_lookup_exact(&dp->table, @@ -1124,6 +1126,30 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) goto err_unlock_ovs; } } + + if (unlikely(reply)) { + size_t cur, req; + + cur = ovs_flow_cmd_msg_size(acts, &new_flow->id, + ufid_flags); + req = ovs_flow_cmd_msg_size(acts, &flow->id, + ufid_flags); + if (cur < req) { + struct sk_buff *resized; + + resized = ovs_flow_cmd_alloc_info(acts, + &flow->id, + info, false, + ufid_flags); + if (IS_ERR(resized)) { + error = PTR_ERR(resized); + goto err_unlock_ovs; + } + kfree_skb(reply); + reply = resized; + } + } + /* Update actions. */ old_acts = ovsl_dereference(flow->sf_acts); rcu_assign_pointer(flow->sf_acts, acts); diff --git a/net/openvswitch/datapath.h b/net/openvswitch/datapath.h index db0c3e69d66c..696640e88fa7 100644 --- a/net/openvswitch/datapath.h +++ b/net/openvswitch/datapath.h @@ -118,7 +118,7 @@ struct datapath { * @mru: The maximum received fragement size; 0 if the packet is not * fragmented. * @acts_origlen: The netlink size of the flow actions applied to this skb. - * @cutlen: The number of bytes from the packet end to be removed. + * @cutlen: The number of bytes in the packet to preserve on output. * @probability: The sampling probability that was applied to this skb; 0 means * no sampling has occurred; U32_MAX means 100% probability. * @upcall_pid: Netlink socket PID to use for sending this packet to userspace; diff --git a/net/openvswitch/flow.c b/net/openvswitch/flow.c index 66366982f604..46c1d66aad8c 100644 --- a/net/openvswitch/flow.c +++ b/net/openvswitch/flow.c @@ -889,8 +889,6 @@ static int key_extract_l3l4(struct sk_buff *skb, struct sw_flow_key *key) * Ethernet header * @key: output flow key * - * The caller must ensure that skb->len >= ETH_HLEN. - * * Initializes @skb header fields as follows: * * - skb->mac_header: the L2 header. @@ -910,8 +908,6 @@ static int key_extract_l3l4(struct sk_buff *skb, struct sw_flow_key *key) */ static int key_extract(struct sk_buff *skb, struct sw_flow_key *key) { - struct ethhdr *eth; - /* Flags are always used as part of stats */ key->tp.flags = 0; @@ -926,6 +922,13 @@ static int key_extract(struct sk_buff *skb, struct sw_flow_key *key) skb_reset_network_header(skb); key->eth.type = skb->protocol; } else { + struct ethhdr *eth; + int err; + + err = check_header(skb, ETH_HLEN); + if (unlikely(err)) + return err; + eth = eth_hdr(skb); ether_addr_copy(key->eth.src, eth->h_source); ether_addr_copy(key->eth.dst, eth->h_dest); diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c index 13052408a132..d8079dee700e 100644 --- a/net/openvswitch/flow_netlink.c +++ b/net/openvswitch/flow_netlink.c @@ -2496,13 +2496,56 @@ static inline int add_nested_action_start(struct sw_flow_actions **sfa, return used; } -static inline void add_nested_action_end(struct sw_flow_actions *sfa, - int st_offset) +static inline int add_nested_action_end(struct sw_flow_actions *sfa, + int st_offset) { - struct nlattr *a = (struct nlattr *) ((unsigned char *)sfa->actions + - st_offset); + struct nlattr *a; + u32 attr_len; + + if (WARN_ON_ONCE(st_offset < 0 || + (u32)st_offset > sfa->actions_len)) + return -EINVAL; + + attr_len = sfa->actions_len - (u32)st_offset; + if (WARN_ON_ONCE(attr_len < NLA_HDRLEN)) + return -EINVAL; - a->nla_len = sfa->actions_len - st_offset; + if (attr_len > U16_MAX) + return -EMSGSIZE; + + a = (struct nlattr *)((u8 *)sfa->actions + st_offset); + a->nla_len = attr_len; + return 0; +} + +/* Free the generated action-list tail at @start and truncate it. + * If @nested, @start points to its containing nlattr header. + */ +static void ovs_nla_trim(struct sw_flow_actions *sfa, int start, bool nested) +{ + const struct nlattr *actions; + u32 len; + + if (start < 0) + return; + + if (WARN_ON_ONCE((u32)start > sfa->actions_len)) + return; + + actions = (const struct nlattr *)((u8 *)sfa->actions + start); + len = sfa->actions_len - (u32)start; + + if (nested) { + if (len < NLA_HDRLEN) + goto out; + + actions = (const struct nlattr *)((u8 *)actions + NLA_HDRLEN); + len -= NLA_HDRLEN; + } + + ovs_nla_free_nested_actions(actions, len); +out: + sfa->actions_len = start; } static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr, @@ -2522,6 +2565,7 @@ static int validate_and_copy_sample(struct net *net, const struct nlattr *attr, const struct nlattr *attrs[OVS_SAMPLE_ATTR_MAX + 1]; const struct nlattr *probability, *actions; const struct nlattr *a; + int actions_start; int rem, start, err; struct sample_arg arg; @@ -2565,18 +2609,27 @@ static int validate_and_copy_sample(struct net *net, const struct nlattr *attr, err = ovs_nla_add_action(sfa, OVS_SAMPLE_ATTR_ARG, &arg, sizeof(arg), log); if (err) - return err; + goto err; + actions_start = (*sfa)->actions_len; err = __ovs_nla_copy_actions(net, actions, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (err) + goto err_free; return 0; + +err_free: + ovs_nla_trim(*sfa, actions_start, false); +err: + (*sfa)->actions_len = start; + return err; } static int validate_and_copy_dec_ttl(struct net *net, @@ -2624,18 +2677,31 @@ static int validate_and_copy_dec_ttl(struct net *net, return start; action_start = add_nested_action_start(sfa, OVS_DEC_TTL_ATTR_ACTION, log); - if (action_start < 0) - return action_start; + if (action_start < 0) { + err = action_start; + goto err; + } err = __ovs_nla_copy_actions(net, actions, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; + + err = add_nested_action_end(*sfa, action_start); + if (err) + goto err_free; - add_nested_action_end(*sfa, action_start); - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (err) + goto err_free; return 0; + +err_free: + ovs_nla_trim(*sfa, action_start, true); +err: + (*sfa)->actions_len = start; + return err; } static int validate_and_copy_clone(struct net *net, @@ -2646,6 +2712,7 @@ static int validate_and_copy_clone(struct net *net, u32 mpls_label_count, bool log, bool last, u32 depth) { + int actions_start; int start, err; u32 exec; @@ -2661,17 +2728,26 @@ static int validate_and_copy_clone(struct net *net, err = ovs_nla_add_action(sfa, OVS_CLONE_ATTR_EXEC, &exec, sizeof(exec), log); if (err) - return err; + goto err; + actions_start = (*sfa)->actions_len; err = __ovs_nla_copy_actions(net, attr, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (err) + goto err_free; return 0; + +err_free: + ovs_nla_trim(*sfa, actions_start, false); +err: + (*sfa)->actions_len = start; + return err; } void ovs_match_init(struct sw_flow_match *match, @@ -2763,20 +2839,20 @@ static int validate_and_copy_set_tun(const struct nlattr *attr, tun_dst = metadata_dst_alloc(key.tun_opts_len, METADATA_IP_TUNNEL, GFP_KERNEL); - if (!tun_dst) - return -ENOMEM; + if (!tun_dst) { + err = -ENOMEM; + goto err; + } err = dst_cache_init(&tun_dst->u.tun_info.dst_cache, GFP_KERNEL); - if (err) { - dst_release((struct dst_entry *)tun_dst); - return err; - } + if (err) + goto err_free_tun_dst; a = __add_action(sfa, OVS_KEY_ATTR_TUNNEL_INFO, NULL, sizeof(*ovs_tun), log); if (IS_ERR(a)) { - dst_release((struct dst_entry *)tun_dst); - return PTR_ERR(a); + err = PTR_ERR(a); + goto err_free_tun_dst; } ovs_tun = nla_data(a); @@ -2797,8 +2873,16 @@ static int validate_and_copy_set_tun(const struct nlattr *attr, ip_tunnel_info_opts_set(tun_info, TUN_METADATA_OPTS(&key, key.tun_opts_len), key.tun_opts_len, dst_opt_type); - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (WARN_ON_ONCE(err)) + goto err_free_tun_dst; + + return 0; +err_free_tun_dst: + dst_release((struct dst_entry *)tun_dst); +err: + (*sfa)->actions_len = start; return err; } @@ -2971,7 +3055,7 @@ static int validate_set(const struct nlattr *a, /* Convert non-masked non-tunnel set actions to masked set actions. */ if (!masked && key_type != OVS_KEY_ATTR_TUNNEL) { - int start, len = key_len * 2; + int err, start, len = key_len * 2; struct nlattr *at; *skip_copy = true; @@ -2983,8 +3067,11 @@ static int validate_set(const struct nlattr *a, return start; at = __add_action(sfa, key_type, NULL, len, log); - if (IS_ERR(at)) - return PTR_ERR(at); + if (IS_ERR(at)) { + err = PTR_ERR(at); + (*sfa)->actions_len = start; + return err; + } memcpy(nla_data(at), nla_data(ovs_key), key_len); /* Key. */ memset(nla_data(at) + key_len, 0xff, key_len); /* Mask. */ @@ -2994,7 +3081,11 @@ static int validate_set(const struct nlattr *a, mask->ipv6_label &= htonl(0x000FFFFF); } - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (WARN_ON_ONCE(err)) { + (*sfa)->actions_len = start; + return err; + } } return 0; @@ -3040,7 +3131,8 @@ static int validate_and_copy_check_pkt_len(struct net *net, const struct nlattr *acts_if_greater, *acts_if_lesser_eq; struct nlattr *a[OVS_CHECK_PKT_LEN_ATTR_MAX + 1]; struct check_pkt_len_arg arg; - int nested_acts_start; + int greater_acts_start = -1; + int lesser_acts_start = -1; int start, err; err = nla_parse_deprecated_strict(a, OVS_CHECK_PKT_LEN_ATTR_MAX, @@ -3075,37 +3167,58 @@ static int validate_and_copy_check_pkt_len(struct net *net, err = ovs_nla_add_action(sfa, OVS_CHECK_PKT_LEN_ATTR_ARG, &arg, sizeof(arg), log); if (err) - return err; + goto err_free; - nested_acts_start = add_nested_action_start(sfa, - OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL, log); - if (nested_acts_start < 0) - return nested_acts_start; + lesser_acts_start = + add_nested_action_start(sfa, + OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL, + log); + if (lesser_acts_start < 0) { + err = lesser_acts_start; + goto err_free; + } err = __ovs_nla_copy_actions(net, acts_if_lesser_eq, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; - add_nested_action_end(*sfa, nested_acts_start); + err = add_nested_action_end(*sfa, lesser_acts_start); + if (err) + goto err_free; - nested_acts_start = add_nested_action_start(sfa, - OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER, log); - if (nested_acts_start < 0) - return nested_acts_start; + greater_acts_start = + add_nested_action_start(sfa, + OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER, + log); + if (greater_acts_start < 0) { + err = greater_acts_start; + goto err_free; + } err = __ovs_nla_copy_actions(net, acts_if_greater, key, sfa, eth_type, vlan_tci, mpls_label_count, log, depth + 1); if (err) - return err; + goto err_free; + + err = add_nested_action_end(*sfa, greater_acts_start); + if (err) + goto err_free; - add_nested_action_end(*sfa, nested_acts_start); - add_nested_action_end(*sfa, start); + err = add_nested_action_end(*sfa, start); + if (err) + goto err_free; return 0; + +err_free: + ovs_nla_trim(*sfa, greater_acts_start, true); + ovs_nla_trim(*sfa, lesser_acts_start, true); + ovs_nla_trim(*sfa, start, false); + return err; } static int validate_psample(const struct nlattr *attr) diff --git a/net/openvswitch/meter.c b/net/openvswitch/meter.c index a02c47277337..4aaeeae3af5b 100644 --- a/net/openvswitch/meter.c +++ b/net/openvswitch/meter.c @@ -133,18 +133,10 @@ static void dp_meter_instance_remove(struct dp_meter_instance *ti, static int attach_meter(struct dp_meter_table *tbl, struct dp_meter *meter) { - struct dp_meter_instance *ti = rcu_dereference_ovsl(tbl->ti); - u32 hash = meter_hash(ti, meter->id); + struct dp_meter_instance *ti; + u32 hash; int err; - /* In generally, slots selected should be empty, because - * OvS uses id-pool to fetch a available id. - */ - if (unlikely(rcu_dereference_ovsl(ti->dp_meters[hash]))) - return -EBUSY; - - dp_meter_instance_insert(ti, meter); - /* That function is thread-safe. */ tbl->count++; if (tbl->count >= tbl->max_meters_allowed) { @@ -152,16 +144,29 @@ static int attach_meter(struct dp_meter_table *tbl, struct dp_meter *meter) goto attach_err; } - if (tbl->count >= ti->n_meters && - dp_meter_instance_realloc(tbl, ti->n_meters * 2)) { - err = -ENOMEM; + ti = rcu_dereference_ovsl(tbl->ti); + if (tbl->count >= ti->n_meters) { + err = dp_meter_instance_realloc(tbl, ti->n_meters * 2); + if (err) + goto attach_err; + + ti = rcu_dereference_ovsl(tbl->ti); + } + + hash = meter_hash(ti, meter->id); + + /* In general, selected slots should be empty, because + * OvS uses id-pool to fetch available ids. + */ + if (unlikely(rcu_dereference_ovsl(ti->dp_meters[hash]))) { + err = -EBUSY; goto attach_err; } + dp_meter_instance_insert(ti, meter); return 0; attach_err: - dp_meter_instance_remove(ti, meter); tbl->count--; return err; } diff --git a/net/openvswitch/vport.c b/net/openvswitch/vport.c index 56b2e2d1a749..12741485c939 100644 --- a/net/openvswitch/vport.c +++ b/net/openvswitch/vport.c @@ -502,7 +502,7 @@ int ovs_vport_receive(struct vport *vport, struct sk_buff *skb, OVS_CB(skb)->input_vport = vport; OVS_CB(skb)->mru = 0; - OVS_CB(skb)->cutlen = 0; + OVS_CB(skb)->cutlen = U32_MAX; OVS_CB(skb)->probability = 0; OVS_CB(skb)->upcall_pid = 0; if (unlikely(dev_net(skb->dev) != ovs_dp_get_net(vport->dp))) { diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 8e6f3a734ba0..1168bd6b09cd 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1315,13 +1315,25 @@ static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) return ret; } -static void packet_rcv_try_clear_pressure(struct packet_sock *po) +static void __packet_rcv_try_clear_pressure(struct packet_sock *po) { if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL) packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, false); } +static void packet_rcv_try_clear_pressure(struct packet_sock *po) +{ + struct sock *sk = &po->sk; + + if (!packet_sock_flag(po, PACKET_SOCK_PRESSURE)) + return; + + spin_lock_bh(&sk->sk_receive_queue.lock); + __packet_rcv_try_clear_pressure(po); + spin_unlock_bh(&sk->sk_receive_queue.lock); +} + static void packet_sock_destruct(struct sock *sk) { skb_queue_purge(&sk->sk_error_queue); @@ -1924,19 +1936,20 @@ static void packet_parse_headers(struct sk_buff *skb, struct socket *sock) { int depth; + /* On TX skb->data is the L2 header; anchor it for all socket types. */ + skb_reset_mac_header(skb); + if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) && - sock->type == SOCK_RAW) { - skb_reset_mac_header(skb); + sock->type == SOCK_RAW) skb->protocol = dev_parse_header_protocol(skb); - } + + skb_probe_transport_header(skb); /* Move network header to the right position for VLAN tagged packets */ if (likely(skb->dev->type == ARPHRD_ETHER) && eth_type_vlan(skb->protocol) && vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0) skb_set_network_header(skb, depth); - - skb_probe_transport_header(skb); } /* @@ -1953,8 +1966,9 @@ static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg, struct net_device *dev; struct sockcm_cookie sockc; __be16 proto = 0; - int err; + int hard_header_len; int extra_len = 0; + int err; /* * Get and verify the address. @@ -1997,14 +2011,18 @@ retry: extra_len = 4; /* We're doing our own CRC */ } + /* Keep the allocation-time header length across retry. */ + if (!skb) + hard_header_len = READ_ONCE(dev->hard_header_len); + err = -EMSGSIZE; - if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len) + if (len > dev->mtu + hard_header_len + VLAN_HLEN + extra_len) goto out_unlock; if (!skb) { - size_t reserved = LL_RESERVED_SPACE(dev); + size_t reserved = LL_RESERVED_SPACE_EX(dev, hard_header_len); int tlen = dev->needed_tailroom; - unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0; + unsigned int hhlen = dev->header_ops ? hard_header_len : 0; rcu_read_unlock(); skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL); @@ -2034,7 +2052,7 @@ retry: err = -EINVAL; goto out_unlock; } - if (len > (dev->mtu + dev->hard_header_len + extra_len) && + if (len > (dev->mtu + hard_header_len + extra_len) && !packet_extra_vlan_len_allowed(dev, skb)) { err = -EMSGSIZE; goto out_unlock; @@ -2569,6 +2587,7 @@ static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len, static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, void *frame, struct net_device *dev, void *data, int tp_len, __be16 proto, unsigned char *addr, int hlen, int copylen, + int hard_header_len, const struct sockcm_cookie *sockc) { union tpacket_uhdr ph; @@ -2600,8 +2619,8 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, } else if (copylen) { int hdrlen = min_t(int, copylen, tp_len); - skb_push(skb, dev->hard_header_len); - skb_put(skb, copylen - dev->hard_header_len); + skb_push(skb, hard_header_len); + skb_put(skb, copylen - hard_header_len); err = skb_store_bits(skb, 0, data, hdrlen); if (unlikely(err)) return err; @@ -2641,6 +2660,9 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, len = ((to_write > len_max) ? len_max : to_write); } + if (unlikely(!skb->len)) + return -EINVAL; + packet_parse_headers(skb, sock); return tp_len; @@ -2732,7 +2754,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) void *data; int len_sum = 0; int status = TP_STATUS_AVAILABLE; - int hlen, tlen, copylen = 0; + int hard_header_len, hlen, tlen, copylen = 0; long timeo; mutex_lock(&po->pg_vec_lock); @@ -2779,8 +2801,9 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) goto out_put; } + hard_header_len = READ_ONCE(dev->hard_header_len); if (po->sk.sk_socket->type == SOCK_RAW) - reserve = dev->hard_header_len; + reserve = hard_header_len; size_max = po->tx_ring.frame_size - (po->tp_hdrlen - sizeof(struct sockaddr_ll)); @@ -2817,7 +2840,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) goto tpacket_error; status = TP_STATUS_SEND_REQUEST; - hlen = LL_RESERVED_SPACE(dev); + hlen = LL_RESERVED_SPACE_EX(dev, hard_header_len); tlen = dev->needed_tailroom; if (vnet_hdr_sz) { data += vnet_hdr_sz; @@ -2835,10 +2858,10 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) vnet_hdr.hdr_len); has_vnet_hdr = true; } - copylen = max_t(int, copylen, dev->hard_header_len); + copylen = max_t(int, copylen, hard_header_len); skb = sock_alloc_send_skb(&po->sk, hlen + tlen + sizeof(struct sockaddr_ll) + - (copylen - dev->hard_header_len), + (copylen - hard_header_len), !need_wait, &err); if (unlikely(skb == NULL)) { @@ -2848,7 +2871,8 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) goto out_status; } tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto, - addr, hlen, copylen, &sockc); + addr, hlen, copylen, hard_header_len, + &sockc); if (likely(tp_len >= 0) && tp_len > dev->mtu + reserve && !vnet_hdr_sz && @@ -2956,7 +2980,7 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) int offset = 0; struct packet_sock *po = pkt_sk(sk); int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz); - int hlen, tlen, linear; + int hard_header_len, hlen, tlen, linear; int extra_len = 0; /* @@ -2996,8 +3020,9 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) goto out_unlock; } + hard_header_len = READ_ONCE(dev->hard_header_len); if (sock->type == SOCK_RAW) - reserve = dev->hard_header_len; + reserve = hard_header_len; if (vnet_hdr_sz) { err = packet_snd_vnet_parse(msg, &len, &vnet_hdr, vnet_hdr_sz); if (err) @@ -3018,10 +3043,10 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) goto out_unlock; err = -ENOBUFS; - hlen = LL_RESERVED_SPACE(dev); + hlen = LL_RESERVED_SPACE_EX(dev, hard_header_len); tlen = dev->needed_tailroom; linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len); - linear = max(linear, min_t(int, len, dev->hard_header_len)); + linear = max(linear, min_t(int, len, hard_header_len)); skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear, msg->msg_flags & MSG_DONTWAIT, &err); if (skb == NULL) @@ -3037,7 +3062,7 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) } else if (reserve) { skb_reserve(skb, -reserve); if (len < reserve + sizeof(struct ipv6hdr) && - dev->min_header_len != dev->hard_header_len) + dev->min_header_len != hard_header_len) skb_reset_network_header(skb); } @@ -4304,7 +4329,7 @@ static __poll_t packet_poll(struct file *file, struct socket *sock, TP_STATUS_KERNEL)) mask |= EPOLLIN | EPOLLRDNORM; } - packet_rcv_try_clear_pressure(po); + __packet_rcv_try_clear_pressure(po); spin_unlock_bh(&sk->sk_receive_queue.lock); spin_lock_bh(&sk->sk_write_queue.lock); if (po->tx_ring.pg_vec) { @@ -4544,14 +4569,14 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, rb->frame_max = (req->tp_frame_nr - 1); rb->head = 0; rb->frame_size = req->tp_frame_size; + po->prot_hook.func = (po->rx_ring.pg_vec) ? + tpacket_rcv : packet_rcv; spin_unlock_bh(&rb_queue->lock); swap(rb->pg_vec_order, order); swap(rb->pg_vec_len, req->tp_block_nr); rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE; - po->prot_hook.func = (po->rx_ring.pg_vec) ? - tpacket_rcv : packet_rcv; skb_queue_purge(rb_queue); if (atomic_long_read(&po->mapped)) pr_err("packet_mmap: vma is busy: %ld\n", @@ -4561,7 +4586,11 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, spin_lock(&po->bind_lock); WRITE_ONCE(po->num, num); - if (was_running) + /* + * NETDEV_UNREGISTER may have invalidated the binding while bind_lock + * was dropped above. Do not re-add a fanout hook to a dead device. + */ + if (was_running && READ_ONCE(po->ifindex) != -1) register_prot_hook(sk); spin_unlock(&po->bind_lock); diff --git a/net/phonet/pep.c b/net/phonet/pep.c index 706927139393..31b29e3ca7bc 100644 --- a/net/phonet/pep.c +++ b/net/phonet/pep.c @@ -55,6 +55,8 @@ static unsigned char *pep_get_sb(struct sk_buff *skb, u8 *ptype, u8 *plen, ph = skb_header_pointer(skb, 0, 2, &h); if (ph == NULL || ph->sb_len < 2 || !pskb_may_pull(skb, ph->sb_len)) return NULL; + /* pskb_may_pull() may have reallocated the head; refetch ph. */ + ph = skb_header_pointer(skb, 0, 2, &h); ph->sb_len -= 2; *ptype = ph->sb_type; *plen = ph->sb_len; diff --git a/net/phonet/pn_dev.c b/net/phonet/pn_dev.c index ad44831d6745..1272d49cd038 100644 --- a/net/phonet/pn_dev.c +++ b/net/phonet/pn_dev.c @@ -350,16 +350,34 @@ static struct pernet_operations phonet_net_ops = { /* Initialize Phonet devices list */ int __init phonet_device_init(void) { - int err = register_pernet_subsys(&phonet_net_ops); + int err; + + err = register_pernet_subsys(&phonet_net_ops); if (err) return err; - proc_create_net("pnresource", 0, init_net.proc_net, &pn_res_seq_ops, - sizeof(struct seq_net_private)); - register_netdevice_notifier(&phonet_device_notifier); + if (!proc_create_net("pnresource", 0, init_net.proc_net, + &pn_res_seq_ops, sizeof(struct seq_net_private))) { + err = -ENOMEM; + goto err_pernet; + } + + err = register_netdevice_notifier(&phonet_device_notifier); + if (err) + goto err_proc; + err = phonet_netlink_register(); if (err) - phonet_device_exit(); + goto err_notifier; + + return 0; + +err_notifier: + unregister_netdevice_notifier(&phonet_device_notifier); +err_proc: + remove_proc_entry("pnresource", init_net.proc_net); +err_pernet: + unregister_pernet_subsys(&phonet_net_ops); return err; } @@ -367,8 +385,8 @@ void phonet_device_exit(void) { rtnl_unregister_all(PF_PHONET); unregister_netdevice_notifier(&phonet_device_notifier); - unregister_pernet_subsys(&phonet_net_ops); remove_proc_entry("pnresource", init_net.proc_net); + unregister_pernet_subsys(&phonet_net_ops); } int phonet_route_add(struct net_device *dev, u8 daddr) diff --git a/net/psp/psp_nl.c b/net/psp/psp_nl.c index 9610d8c456ff..43b066353c65 100644 --- a/net/psp/psp_nl.c +++ b/net/psp/psp_nl.c @@ -62,7 +62,14 @@ psp_nl_multicast_per_ns(struct psp_dev *psd, unsigned int group, struct net *main_net; struct sk_buff *ntf; - main_net = dev_net(psd->main_netdev); + /* device may be changing netns in parallel */ + rcu_read_lock(); + main_net = maybe_get_net(dev_net_rcu(psd->main_netdev)); + rcu_read_unlock(); + + if (!main_net) + return; + xa_init(&sent_nets); list_for_each_entry(entry, &psd->assoc_dev_list, dev_list) { @@ -88,10 +95,10 @@ psp_nl_multicast_per_ns(struct psp_dev *psd, unsigned int group, /* Send to main device netns */ ntf = build_ntf(psd, main_net, ctx); - if (!ntf) - return; - genlmsg_multicast_netns(&psp_nl_family, main_net, ntf, 0, group, - GFP_KERNEL); + if (ntf) + genlmsg_multicast_netns(&psp_nl_family, main_net, ntf, 0, group, + GFP_KERNEL); + put_net(main_net); } static struct sk_buff *psp_nl_clone_ntf(struct psp_dev *psd, struct net *net, diff --git a/net/qrtr/af_qrtr.c b/net/qrtr/af_qrtr.c index d02ef9a74c3c..a30fa56e6aa3 100644 --- a/net/qrtr/af_qrtr.c +++ b/net/qrtr/af_qrtr.c @@ -1263,6 +1263,14 @@ static int qrtr_create(struct net *net, struct socket *sock, if (sock->type != SOCK_DGRAM) return -EPROTOTYPE; + /* QRTR keeps its port and node state in module-global variables that + * are not partitioned per network namespace, and the in-kernel name + * service only operates in init_net. Confine the family to init_net so + * a socket in another namespace cannot reach the global control plane. + */ + if (!net_eq(net, &init_net)) + return -EAFNOSUPPORT; + sk = sk_alloc(net, AF_QIPCRTR, GFP_KERNEL, &qrtr_proto, kern); if (!sk) return -ENOMEM; diff --git a/net/qrtr/ns.c b/net/qrtr/ns.c index b3f9bbcf9ab9..c5e7e01db249 100644 --- a/net/qrtr/ns.c +++ b/net/qrtr/ns.c @@ -76,11 +76,11 @@ struct qrtr_node { * requirements. If the requirement changes in the future, these values can be * increased. */ -#define QRTR_NS_MAX_NODES 64 +#define QRTR_NS_MAX_NODES 512 #define QRTR_NS_MAX_SERVERS 256 -#define QRTR_NS_MAX_LOOKUPS 64 +#define QRTR_NS_MAX_LOOKUPS 128 -static u8 node_count; +static u16 node_count; static struct qrtr_node *node_get(unsigned int node_id) { diff --git a/net/rds/ib.c b/net/rds/ib.c index 39f87272e071..8f9cf491984f 100644 --- a/net/rds/ib.c +++ b/net/rds/ib.c @@ -429,6 +429,10 @@ static int rds_ib_laddr_check_cm(struct net *net, const struct in6_addr *addr, sa = (struct sockaddr *)&sin; } else { #if IS_ENABLED(CONFIG_IPV6) + if (!ipv6_mod_enabled()) { + ret = -EADDRNOTAVAIL; + goto out; + } memset(&sin6, 0, sizeof(sin6)); sin6.sin6_family = AF_INET6; sin6.sin6_addr = *addr; diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c index 5667f0173b47..d46146887ba4 100644 --- a/net/rds/ib_cm.c +++ b/net/rds/ib_cm.c @@ -810,6 +810,10 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, dp = event->param.conn.private_data; if (isv6) { #if IS_ENABLED(CONFIG_IPV6) + if (!ipv6_mod_enabled()) { + err = -EOPNOTSUPP; + goto out; + } dp_cmn = &dp->ricp_v6.dp_cmn; saddr6 = &dp->ricp_v6.dp_saddr; daddr6 = &dp->ricp_v6.dp_daddr; diff --git a/net/rds/recv.c b/net/rds/recv.c index 4b3f9e4a8bfd..cf3884d87931 100644 --- a/net/rds/recv.c +++ b/net/rds/recv.c @@ -399,6 +399,21 @@ void rds_recv_incoming(struct rds_connection *conn, struct in6_addr *saddr, goto out; } + /* + * rds_find_bound() uses a global (netns-agnostic) hash table. + * An RDS connection created in netns A can match a socket bound + * in the init netns, delivering inc cross-netns with inc->i_conn + * pointing into netns A. When cleanup_net() then frees that conn, + * any subsequent dereference of inc->i_conn is a use-after-free. + * Drop the inc if the receiving socket lives in a different netns. + */ + if (!net_eq(sock_net(rds_rs_to_sk(rs)), rds_conn_net(conn))) { + rds_stats_inc(s_recv_drop_no_sock); + rds_sock_put(rs); + rs = NULL; + goto out; + } + /* Process extension headers */ rds_recv_incoming_exthdrs(inc, rs); diff --git a/net/rds/tcp.c b/net/rds/tcp.c index a1de114d5e2e..b263634ac750 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -355,21 +355,25 @@ int rds_tcp_laddr_check(struct net *net, const struct in6_addr *addr, /* If the scope_id is specified, check only those addresses * hosted on the specified interface. */ + rcu_read_lock(); if (scope_id != 0) { - rcu_read_lock(); dev = dev_get_by_index_rcu(net, scope_id); /* scope_id is not valid... */ if (!dev) { rcu_read_unlock(); return -EADDRNOTAVAIL; } - rcu_read_unlock(); } #if IS_ENABLED(CONFIG_IPV6) - ret = ipv6_chk_addr(net, addr, dev, 0); - if (ret) - return 0; + if (ipv6_mod_enabled()) { + ret = ipv6_chk_addr(net, addr, dev, 0); + if (ret) { + rcu_read_unlock(); + return 0; + } + } #endif + rcu_read_unlock(); return -EADDRNOTAVAIL; } @@ -655,13 +659,13 @@ static void __net_exit rds_tcp_exit_net(struct net *net) { struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); - rds_tcp_kill_sock(net); - if (rtn->rds_tcp_sysctl) unregister_net_sysctl_table(rtn->rds_tcp_sysctl); if (net != &init_net) kfree(rtn->ctl_table); + + rds_tcp_kill_sock(net); } static struct pernet_operations rds_tcp_net_ops = { diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index ce946b0a03e2..865f05fe37ab 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -1285,9 +1285,11 @@ int rxrpc_io_thread(void *data); void rxrpc_post_response(struct rxrpc_connection *conn, struct sk_buff *skb); static inline void rxrpc_wake_up_io_thread(struct rxrpc_local *local) { - if (!local->io_thread) + struct task_struct *io_thread = READ_ONCE(local->io_thread); + + if (!io_thread) return; - wake_up_process(READ_ONCE(local->io_thread)); + wake_up_process(io_thread); } static inline bool rxrpc_protocol_error(struct sk_buff *skb, enum rxrpc_abort_reason why) diff --git a/net/sched/act_api.c b/net/sched/act_api.c index b68be143a067..600b7804befd 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -41,11 +41,9 @@ int tcf_dev_queue_xmit(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)) } EXPORT_SYMBOL_GPL(tcf_dev_queue_xmit); -static void tcf_action_goto_chain_exec(const struct tc_action *a, +static void tcf_action_goto_chain_exec(const struct tcf_chain *chain, struct tcf_result *res) { - const struct tcf_chain *chain = rcu_dereference_bh(a->goto_chain); - res->goto_tp = rcu_dereference_bh(chain->filter_chain); } @@ -148,10 +146,15 @@ static void offload_action_hw_count_dec(struct tc_action *act, static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act) { - if (is_tcf_pedit(act)) - return tcf_pedit_nkeys(act); - else - return 1; + unsigned int count; + + if (is_tcf_pedit(act)) { + spin_lock_bh(&act->tcfa_lock); + count = tcf_pedit_nkeys_locked(act); + spin_unlock_bh(&act->tcfa_lock); + return count; + } + return 1; } static bool tc_act_skip_hw(u32 flags) @@ -1165,12 +1168,14 @@ repeat: return TC_ACT_OK; } } else if (TC_ACT_EXT_CMP(ret, TC_ACT_GOTO_CHAIN)) { - if (unlikely(!rcu_access_pointer(a->goto_chain))) { + struct tcf_chain *chain = rcu_dereference_bh(a->goto_chain); + + if (unlikely(!chain)) { tcf_set_drop_reason(skb, SKB_DROP_REASON_TC_CHAIN_NOTFOUND); return TC_ACT_SHOT; } - tcf_action_goto_chain_exec(a, res); + tcf_action_goto_chain_exec(chain, res); } if (ret != TC_ACT_PIPE) diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index be535a261fa0..e250969c84ac 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -840,8 +840,15 @@ static int tcf_ct_ipv6_is_fragment(struct sk_buff *skb, bool *frag) return 0; } +/* On error, tells the caller whether it still owns @skb and must free it + * itself. @skb is ours only when the header checks below reject the packet + * before it is handed to the defragmentation engine; once nf_ct_handle_ + * fragments() has been called the skb is either queued (-EINPROGRESS) or has + * already been freed by it. + */ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, - u8 family, u16 zone, bool *defrag) + u8 family, u16 zone, bool *defrag, + bool *skb_is_ours) { enum ip_conntrack_info ctinfo; struct tc_skb_cb cb; @@ -859,8 +866,12 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, err = tcf_ct_ipv4_is_fragment(skb, &frag); else err = tcf_ct_ipv6_is_fragment(skb, &frag); - if (err || !frag) + if (err) { + *skb_is_ours = true; return err; + } + if (!frag) + return 0; cb = *tc_skb_cb(skb); err = nf_ct_handle_fragments(net, skb, zone, family, &proto, &cb.mru); @@ -977,6 +988,7 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, int nh_ofs, err, retval; struct tcf_ct_params *p; bool add_helper = false; + bool skb_is_ours = false; bool skip_add = false; bool defrag = false; struct nf_conn *ct; @@ -1012,9 +1024,18 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, */ nh_ofs = skb_network_offset(skb); skb_pull_rcsum(skb, nh_ofs); - err = tcf_ct_handle_fragments(net, skb, family, p->zone, &defrag); - if (err) + err = tcf_ct_handle_fragments(net, skb, family, p->zone, &defrag, + &skb_is_ours); + if (err) { + /* The skb is still ours only when the header checks rejected + * it; returning TC_ACT_CONSUMED for such a packet would leak + * it, since no caller frees an skb it was told it no longer + * owns. + */ + if (skb_is_ours) + goto drop; goto out_frag; + } err = nf_ct_skb_network_trim(skb, family); if (err) diff --git a/net/sched/act_gact.c b/net/sched/act_gact.c index e949280eb800..565860cccba6 100644 --- a/net/sched/act_gact.c +++ b/net/sched/act_gact.c @@ -89,6 +89,11 @@ static int tcf_gact_init(struct net *net, struct nlattr *nla, p_parm = nla_data(tb[TCA_GACT_PROB]); if (p_parm->ptype >= MAX_RAND) return -EINVAL; + if (!tcf_action_valid(p_parm->paction)) { + NL_SET_ERR_MSG(extack, + "invalid fallback control action"); + return -EINVAL; + } if (TC_ACT_EXT_CMP(p_parm->paction, TC_ACT_GOTO_CHAIN)) { NL_SET_ERR_MSG(extack, "goto chain not allowed on fallback"); diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index 0d652dea4a69..d4d47a9921f4 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -567,9 +567,18 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, { if (bind) { struct flow_action_entry *entry = entry_data; + int nkeys = tcf_pedit_nkeys_locked(act); int k; - for (k = 0; k < tcf_pedit_nkeys(act); k++) { + /* If the required keys exceed the remaining capacity return + * -ENOSPC to abort the offload and fallback to software. + */ + if (nkeys > *index_inc) { + NL_SET_ERR_MSG_MOD(extack, "Not enough space to offload all pedit keys"); + return -ENOSPC; + } + + for (k = 0; k < nkeys; k++) { switch (tcf_pedit_cmd(act, k)) { case TCA_PEDIT_KEY_EX_CMD_SET: entry->id = FLOW_ACTION_MANGLE; @@ -606,7 +615,7 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, return -EOPNOTSUPP; } - for (k = 1; k < tcf_pedit_nkeys(act); k++) { + for (k = 1; k < tcf_pedit_nkeys_locked(act); k++) { if (cmd != tcf_pedit_cmd(act, k)) { NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); return -EOPNOTSUPP; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index b16468a98c55..ce08f6840ef7 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -128,6 +128,12 @@ static int tcf_police_init(struct net *net, struct nlattr *nla, if (tb[TCA_POLICE_RESULT]) { tcfp_result = nla_get_u32(tb[TCA_POLICE_RESULT]); + if (!tcf_action_valid(tcfp_result)) { + NL_SET_ERR_MSG(extack, + "invalid fallback control action"); + err = -EINVAL; + goto failure; + } if (TC_ACT_EXT_CMP(tcfp_result, TC_ACT_GOTO_CHAIN)) { NL_SET_ERR_MSG(extack, "goto chain not allowed on fallback"); diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index 876b30c5709e..b14807761d82 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -342,14 +342,20 @@ static const struct nla_policy tunnel_key_policy[TCA_TUNNEL_KEY_MAX + 1] = { [TCA_TUNNEL_KEY_ENC_TTL] = { .type = NLA_U8 }, }; -static void tunnel_key_release_params(struct tcf_tunnel_key_params *p) +static void tunnel_key_release_params_rcu(struct rcu_head *head) { - if (!p) - return; + struct tcf_tunnel_key_params *p = container_of(head, typeof(*p), rcu); + if (p->tcft_action == TCA_TUNNEL_KEY_ACT_SET) dst_release(&p->tcft_enc_metadata->dst); + kfree(p); +} - kfree_rcu(p, rcu); +static void tunnel_key_release_params(struct tcf_tunnel_key_params *p) +{ + if (!p) + return; + call_rcu(&p->rcu, tunnel_key_release_params_rcu); } static int tunnel_key_init(struct net *net, struct nlattr *nla, diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 3e67600a4a1a..4e6a2812a4f3 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -443,7 +443,22 @@ static void tcf_chain_put(struct tcf_chain *chain); static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held, bool sig_destroy, struct netlink_ext_ack *extack) { - tp->ops->destroy(tp, rtnl_held, extack); + /* A locked classifier's destroy callback (e.g. u32_destroy) uses + * rtnl_dereference() and mutates shared structures (e.g. the + * tc_u_common hash list) that are only safe under rtnl_lock. When an + * unlocked classifier's request (e.g. flower on ingress) loses the + * tcf_chain_tp_insert_unique() race and ends up dropping the last + * reference on a locked classifier's proto, destroy() would run + * without rtnl held. Take it here in that case. + */ + bool not_lockless = !rtnl_held && + !(tp->ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED); + + if (not_lockless) + rtnl_lock(); + tp->ops->destroy(tp, rtnl_held || not_lockless, extack); + if (not_lockless) + rtnl_unlock(); tcf_proto_count_usesw(tp, false); if (sig_destroy) tcf_proto_signal_destroyed(tp->chain, tp); @@ -3886,12 +3901,21 @@ int tc_setup_action(struct flow_action *flow_action, entry = &flow_action->entries[j]; spin_lock_bh(&act->tcfa_lock); + + /* Abort the offload if we have exhausted the allocated capacity */ + if (j >= flow_action->num_entries) { + NL_SET_ERR_MSG_MOD(extack, "Flow action buffer overflow"); + err = -ENOSPC; + goto err_out_locked; + } + err = tcf_act_get_user_cookie(entry, act); if (err) goto err_out_locked; - index = 0; - err = tc_setup_offload_act(act, entry, &index, extack); + index = flow_action->num_entries - j; + err = tc_setup_offload_act(act, entry, &index, + extack); if (err) goto err_out_locked; @@ -3945,10 +3969,13 @@ unsigned int tcf_exts_num_actions(struct tcf_exts *exts) int i; tcf_exts_for_each_action(i, act, exts) { - if (is_tcf_pedit(act)) - num_acts += tcf_pedit_nkeys(act); - else + if (is_tcf_pedit(act)) { + spin_lock_bh(&act->tcfa_lock); + num_acts += tcf_pedit_nkeys_locked(act); + spin_unlock_bh(&act->tcfa_lock); + } else { num_acts++; + } } return num_acts; } @@ -4033,7 +4060,7 @@ struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, stru fl = rcu_dereference_bh(qe->filter_chain); - switch (tcf_classify(skb, NULL, fl, &cl_res, false)) { + switch (tcf_classify_qdisc(skb, fl, &cl_res, false)) { case TC_ACT_SHOT: qdisc_qstats_drop(sch); __qdisc_drop(skb, to_free); @@ -4045,10 +4072,6 @@ struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, stru __qdisc_drop(skb, to_free); *ret = __NET_XMIT_STOLEN; return NULL; - case TC_ACT_REDIRECT: - skb_do_redirect(skb); - *ret = __NET_XMIT_STOLEN; - return NULL; case TC_ACT_CONSUMED: *ret = __NET_XMIT_STOLEN; return NULL; diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index 001d8c4ebfed..6d19155becc8 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -374,7 +374,8 @@ static int cls_bpf_prog_from_ops(struct nlattr **tb, struct cls_bpf_prog *prog) } static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog, - u32 gen_flags, const struct tcf_proto *tp) + u32 gen_flags, const struct tcf_proto *tp, + struct netlink_ext_ack *extack) { struct bpf_prog *fp; char *name = NULL; @@ -388,6 +389,19 @@ static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog, if (IS_ERR(fp)) return PTR_ERR(fp); + if (bpf_prog_is_dev_bound(fp->aux)) { + struct tcf_block *block = tp->chain->block; + struct net_device *dev; + + dev = block->q ? qdisc_dev(block->q) : NULL; + if (!dev || !bpf_offload_dev_match(fp, dev)) { + NL_SET_ERR_MSG(extack, + "Program is bound to a different device"); + bpf_prog_put(fp); + return -EINVAL; + } + } + if (tb[TCA_BPF_NAME]) { name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL); if (!name) { @@ -492,7 +506,7 @@ static int cls_bpf_change(struct net *net, struct sk_buff *in_skb, prog->gen_flags = gen_flags; ret = is_bpf ? cls_bpf_prog_from_ops(tb, prog) : - cls_bpf_prog_from_efd(tb, prog, gen_flags, tp); + cls_bpf_prog_from_efd(tb, prog, gen_flags, tp, extack); if (ret < 0) goto errout_idr; diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index bd6f945bd388..eded7aacd3f7 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -52,6 +52,7 @@ struct route4_filter { struct tcf_result res; struct tcf_exts exts; u32 handle; + bool dying; struct route4_bucket *bkt; struct tcf_proto *tp; struct rcu_work rwork; @@ -66,9 +67,11 @@ static inline int route4_fastmap_hash(u32 id, int iif) static DEFINE_SPINLOCK(fastmap_lock); static void -route4_reset_fastmap(struct route4_head *head) +route4_reset_fastmap(struct route4_head *head, struct route4_filter *f) { spin_lock_bh(&fastmap_lock); + if (f) + f->dying = true; memset(head->fastmap, 0, sizeof(head->fastmap)); spin_unlock_bh(&fastmap_lock); } @@ -81,9 +84,11 @@ route4_set_fastmap(struct route4_head *head, u32 id, int iif, /* fastmap updates must look atomic to aling id, iff, filter */ spin_lock_bh(&fastmap_lock); - head->fastmap[h].id = id; - head->fastmap[h].iif = iif; - head->fastmap[h].filter = f; + if (f == ROUTE4_FAILURE || !f->dying) { + head->fastmap[h].id = id; + head->fastmap[h].iif = iif; + head->fastmap[h].filter = f; + } spin_unlock_bh(&fastmap_lock); } @@ -297,6 +302,13 @@ static void route4_destroy(struct tcf_proto *tp, bool rtnl_held, next = rtnl_dereference(f->next); RCU_INIT_POINTER(b->ht[h2], next); tcf_unbind_filter(tp, &f->res); + /* Mark the filter dying under fastmap_lock so + * any in-flight reader that still holds it + * will skip the republish in route4_set_fastmap(). + */ + spin_lock_bh(&fastmap_lock); + f->dying = true; + spin_unlock_bh(&fastmap_lock); if (tcf_exts_get_net(&f->exts)) route4_queue_work(f); else @@ -307,6 +319,11 @@ static void route4_destroy(struct tcf_proto *tp, bool rtnl_held, kfree_rcu(b, rcu); } } + + /* All filters are unlinked and marked dying, so no in-flight + * reader can republish a stale entry after this reset. + */ + route4_reset_fastmap(head, NULL); kfree_rcu(head, rcu); } @@ -334,11 +351,11 @@ static int route4_delete(struct tcf_proto *tp, void *arg, bool *last, /* unlink it */ RCU_INIT_POINTER(*fp, rtnl_dereference(f->next)); - /* Remove any fastmap lookups that might ref filter - * notice we unlink'd the filter so we can't get it - * back in the fastmap. + /* Clear any fastmap entries that may ref this filter and + * mark it dying so in-flight readers can't republish it + * after the reset. */ - route4_reset_fastmap(head); + route4_reset_fastmap(head, f); /* Delete it */ tcf_unbind_filter(tp, &f->res); @@ -558,7 +575,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, } } - route4_reset_fastmap(head); + route4_reset_fastmap(head, fold); *arg = f; if (fold) { tcf_unbind_filter(tp, &fold->res); diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index 8f30cc82181d..c297d7dbcf91 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -1107,6 +1107,13 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, goto erridr; } + if (s->offshift >= 16) { + NL_SET_ERR_MSG_MOD(extack, + "offshift must be less than 16"); + err = -EINVAL; + goto erridr; + } + n = kzalloc_flex(*n, sel.keys, s->nkeys); if (n == NULL) { err = -ENOBUFS; @@ -1339,6 +1346,9 @@ static void u32_bind_class(void *fh, u32 classid, unsigned long cl, void *q, { struct tc_u_knode *n = fh; + if (TC_U32_KEY(n->handle) == 0) + return; + tc_cls_bind_class(classid, cl, q, &n->res, base); } diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 8a3236456db4..65b35528d125 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -415,12 +415,13 @@ static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab) } static struct qdisc_rate_table *qdisc_rtab_list; +static DEFINE_SPINLOCK(qdisc_rtab_lock); struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab, struct netlink_ext_ack *extack) { - struct qdisc_rate_table *rtab; + struct qdisc_rate_table *rtab, *new_rtab; if (tab == NULL || r->rate == 0 || r->cell_log == 0 || r->cell_log >= 32 || @@ -429,15 +430,20 @@ struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, return NULL; } + new_rtab = kmalloc_obj(*new_rtab); + + spin_lock(&qdisc_rtab_lock); for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) { if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) && !memcmp(&rtab->data, nla_data(tab), TC_RTAB_SIZE)) { rtab->refcnt++; + spin_unlock(&qdisc_rtab_lock); + kfree(new_rtab); return rtab; } } - rtab = kmalloc_obj(*rtab); + rtab = new_rtab; if (rtab) { rtab->rate = *r; rtab->refcnt = 1; @@ -449,6 +455,7 @@ struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, } else { NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table"); } + spin_unlock(&qdisc_rtab_lock); return rtab; } EXPORT_SYMBOL(qdisc_get_rtab); @@ -457,18 +464,25 @@ void qdisc_put_rtab(struct qdisc_rate_table *tab) { struct qdisc_rate_table *rtab, **rtabp; - if (!tab || --tab->refcnt) + if (!tab) return; + spin_lock(&qdisc_rtab_lock); + if (--tab->refcnt) { + spin_unlock(&qdisc_rtab_lock); + return; + } + for (rtabp = &qdisc_rtab_list; (rtab = *rtabp) != NULL; rtabp = &rtab->next) { if (rtab == tab) { *rtabp = rtab->next; - kfree(rtab); - return; + break; } } + spin_unlock(&qdisc_rtab_lock); + kfree(tab); } EXPORT_SYMBOL(qdisc_put_rtab); @@ -1100,6 +1114,9 @@ static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, unsigned int i, num_q, ingress; struct netdev_queue *dev_queue; + if (new) + new->depth = 0; + ingress = 0; num_q = dev->num_tx_queues; if ((q && q->flags & TCQ_F_INGRESS) || @@ -1197,9 +1214,15 @@ skip: NL_SET_ERR_MSG(extack, "STAB not supported on a non root"); return -EINVAL; } + if (new && parent->depth >= 7) { + NL_SET_ERR_MSG(extack, "Qdisc hierarchy is too deep"); + return -E2BIG; + } err = cops->graft(parent, cl, new, &old, extack); if (err) return err; + if (new) + new->depth = parent->depth + 1; notify_and_destroy(net, skb, n, classid, old, new, extack); } return 0; diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index a3c185505afc..f25f60978631 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1287,7 +1287,6 @@ static struct sk_buff *cake_ack_filter(struct cake_sched_data *q, seglen = ipv6_payload_len(skb, ipv6h_check); } else { - WARN_ON(1); /* shouldn't happen */ continue; } @@ -1389,10 +1388,7 @@ static u32 cake_calc_overhead(struct cake_sched_data *qd, u32 len, u32 off) if (qd->min_netlen > len) WRITE_ONCE(qd->min_netlen, len); - len += q->rate_overhead; - - if (len < q->rate_mpu) - len = q->rate_mpu; + len = max((s32)len + q->rate_overhead, (s32)q->rate_mpu); if (q->atm_mode == CAKE_ATM_ATM) { len += 47; @@ -1730,7 +1726,7 @@ static u32 cake_classify(struct Qdisc *sch, struct cake_tin_data **t, goto hash; *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; - result = tcf_classify(skb, NULL, filter, &res, false); + result = tcf_classify_qdisc(skb, filter, &res, false); if (result >= 0) { #ifdef CONFIG_NET_CLS_ACT @@ -2612,9 +2608,11 @@ static void cake_configure_rates(struct Qdisc *sch, u64 rate, bool rate_adjust) break; } - for (c = qd->tin_cnt; c < CAKE_MAX_TINS; c++) { - cake_clear_tin(sch, c); - qd->tins[c].cparams.mtu_time = qd->tins[ft].cparams.mtu_time; + if (!rate_adjust) { + for (c = qd->tin_cnt; c < CAKE_MAX_TINS; c++) { + cake_clear_tin(sch, c); + qd->tins[c].cparams.mtu_time = qd->tins[ft].cparams.mtu_time; + } } qd->rate_ns = qd->tins[ft].tin_rate_ns; diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index 020657f959b5..91b1ef824afa 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -312,7 +312,7 @@ static struct drr_class *drr_classify(struct sk_buff *skb, struct Qdisc *sch, *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; fl = rcu_dereference_bh(q->filter_list); - result = tcf_classify(skb, NULL, fl, &res, false); + result = tcf_classify_qdisc(skb, fl, &res, false); if (result >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c index 27088760eff4..4f678d4ff10e 100644 --- a/net/sched/sch_dualpi2.c +++ b/net/sched/sch_dualpi2.c @@ -364,7 +364,7 @@ static int dualpi2_skb_classify(struct dualpi2_sched_data *q, if (!fl) return NET_XMIT_SUCCESS; - result = tcf_classify(skb, NULL, fl, &res, false); + result = tcf_classify_qdisc(skb, fl, &res, false); if (result >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index cb8cf437ce87..25fcf4079fec 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -391,7 +391,7 @@ static struct ets_class *ets_classify(struct sk_buff *skb, struct Qdisc *sch, *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; if (TC_H_MAJ(skb->priority) != sch->handle) { fl = rcu_dereference_bh(q->filter_list); - err = tcf_classify(skb, NULL, fl, &res, false); + err = tcf_classify_qdisc(skb, fl, &res, false); #ifdef CONFIG_NET_CLS_ACT switch (err) { case TC_ACT_STOLEN: diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index cafd1f943d99..6cce86ba383c 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -91,7 +91,7 @@ static unsigned int fq_codel_classify(struct sk_buff *skb, struct Qdisc *sch, return fq_codel_hash(q, skb) + 1; *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; - result = tcf_classify(skb, NULL, filter, &res, false); + result = tcf_classify_qdisc(skb, filter, &res, false); if (result >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 72f48fa4010b..069e1facd413 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -96,7 +96,7 @@ static unsigned int fq_pie_classify(struct sk_buff *skb, struct Qdisc *sch, return fq_pie_hash(q, skb) + 1; *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; - result = tcf_classify(skb, NULL, filter, &res, false); + result = tcf_classify_qdisc(skb, filter, &res, false); if (result >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c index 7e537295b8b6..e87f5021a199 100644 --- a/net/sched/sch_hfsc.c +++ b/net/sched/sch_hfsc.c @@ -1143,7 +1143,7 @@ hfsc_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; head = &q->root; tcf = rcu_dereference_bh(q->root.filter_list); - while (tcf && (result = tcf_classify(skb, NULL, tcf, &res, false)) >= 0) { + while (tcf && (result = tcf_classify_qdisc(skb, tcf, &res, false)) >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { case TC_ACT_QUEUED: diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index 908b9ba9ba2e..fdac0dc8f35a 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -243,7 +243,7 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, } *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; - while (tcf && (result = tcf_classify(skb, NULL, tcf, &res, false)) >= 0) { + while (tcf && (result = tcf_classify_qdisc(skb, tcf, &res, false)) >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { case TC_ACT_QUEUED: diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c index a467dd122369..66df30939aa5 100644 --- a/net/sched/sch_multiq.c +++ b/net/sched/sch_multiq.c @@ -36,7 +36,7 @@ multiq_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) int err; *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; - err = tcf_classify(skb, NULL, fl, &res, false); + err = tcf_classify_qdisc(skb, fl, &res, false); #ifdef CONFIG_NET_CLS_ACT switch (err) { case TC_ACT_STOLEN: diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index e4dd56a89072..79437c587e7e 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -39,7 +39,7 @@ prio_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; if (TC_H_MAJ(skb->priority) != sch->handle) { fl = rcu_dereference_bh(q->filter_list); - err = tcf_classify(skb, NULL, fl, &res, false); + err = tcf_classify_qdisc(skb, fl, &res, false); #ifdef CONFIG_NET_CLS_ACT switch (err) { case TC_ACT_STOLEN: diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c index cb56787e1d25..6f3b7273cb16 100644 --- a/net/sched/sch_qfq.c +++ b/net/sched/sch_qfq.c @@ -709,7 +709,7 @@ static struct qfq_class *qfq_classify(struct sk_buff *skb, struct Qdisc *sch, *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; fl = rcu_dereference_bh(q->filter_list); - result = tcf_classify(skb, NULL, fl, &res, false); + result = tcf_classify_qdisc(skb, fl, &res, false); if (result >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c index b1d465094276..ed39869199c0 100644 --- a/net/sched/sch_sfb.c +++ b/net/sched/sch_sfb.c @@ -260,7 +260,7 @@ static bool sfb_classify(struct sk_buff *skb, struct tcf_proto *fl, struct tcf_result res; int result; - result = tcf_classify(skb, NULL, fl, &res, false); + result = tcf_classify_qdisc(skb, fl, &res, false); if (result >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index 758b88f21865..77675f9a4c46 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -171,7 +171,7 @@ static unsigned int sfq_classify(struct sk_buff *skb, struct Qdisc *sch, return sfq_hash(q, skb) + 1; *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; - result = tcf_classify(skb, NULL, fl, &res, false); + result = tcf_classify_qdisc(skb, fl, &res, false); if (result >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 24ba31f8c828..5c42a29a981c 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -311,14 +311,14 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) int subq = skb_get_queue_mapping(skb); struct sk_buff *skb_res = NULL; - rcu_read_lock_bh(); - - start = rcu_dereference_bh(master->slaves); - restart: nores = 0; busy = 0; + rcu_read_lock(); + + start = rcu_dereference(master->slaves); + q = start; if (!q) goto drop; @@ -345,17 +345,17 @@ restart: netdev_start_xmit(skb, slave, slave_txq, false) == NETDEV_TX_OK) { __netif_tx_unlock(slave_txq); - spin_lock_bh(&master->slaves_lock); + spin_lock(&master->slaves_lock); if (rcu_dereference_protected(master->slaves, lockdep_is_held(&master->slaves_lock)) == q) rcu_assign_pointer(master->slaves, rcu_dereference_protected(NEXT_SLAVE(q), lockdep_is_held(&master->slaves_lock))); - spin_unlock_bh(&master->slaves_lock); + spin_unlock(&master->slaves_lock); netif_wake_queue(dev); master->tx_packets++; master->tx_bytes += length; - rcu_read_unlock_bh(); + rcu_read_unlock(); return NETDEV_TX_OK; } __netif_tx_unlock(slave_txq); @@ -364,37 +364,38 @@ restart: busy = 1; break; case 1: - spin_lock_bh(&master->slaves_lock); + spin_lock(&master->slaves_lock); if (rcu_dereference_protected(master->slaves, lockdep_is_held(&master->slaves_lock)) == q) rcu_assign_pointer(master->slaves, rcu_dereference_protected(NEXT_SLAVE(q), lockdep_is_held(&master->slaves_lock))); - spin_unlock_bh(&master->slaves_lock); - rcu_read_unlock_bh(); + spin_unlock(&master->slaves_lock); + rcu_read_unlock(); return NETDEV_TX_OK; default: nores = 1; break; } __skb_pull(skb, skb_network_offset(skb)); - } while ((q = rcu_dereference_bh(NEXT_SLAVE(q))) != start); + } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); if (nores && skb_res == NULL) { skb_res = skb; + rcu_read_unlock(); goto restart; } if (busy) { netif_stop_queue(dev); - rcu_read_unlock_bh(); + rcu_read_unlock(); return NETDEV_TX_BUSY; } master->tx_errors++; drop: master->tx_dropped++; - rcu_read_unlock_bh(); + rcu_read_unlock(); dev_kfree_skb(skb); return NETDEV_TX_OK; } diff --git a/net/sctp/associola.c b/net/sctp/associola.c index 62d3cc155809..5be0bed2685e 100644 --- a/net/sctp/associola.c +++ b/net/sctp/associola.c @@ -543,6 +543,9 @@ void sctp_assoc_rm_peer(struct sctp_association *asoc, asoc->addip_last_asconf->transport == peer) asoc->addip_last_asconf->transport = NULL; + if (asoc->new_transport == peer) + asoc->new_transport = NULL; + /* If we have something on the transmitted list, we have to * save it off. The best place is the active path. */ @@ -573,6 +576,10 @@ void sctp_assoc_rm_peer(struct sctp_association *asoc, if (ch->transport == peer) ch->transport = NULL; + list_for_each_entry(ch, &asoc->outqueue.control_chunk_list, list) + if (ch->transport == peer) + ch->transport = NULL; + asoc->peer.transport_count--; sctp_ulpevent_notify_peer_addr_change(peer, SCTP_ADDR_REMOVED, 0); @@ -614,6 +621,9 @@ struct sctp_transport *sctp_assoc_add_peer(struct sctp_association *asoc, return peer; } + if (asoc->peer.transport_count == U16_MAX) + return NULL; + peer = sctp_transport_new(asoc->base.net, addr, gfp); if (!peer) return NULL; @@ -1706,6 +1716,8 @@ void sctp_asconf_queue_teardown(struct sctp_association *asoc) sctp_assoc_free_asconf_queue(asoc); /* Free any cached ASCONF chunk. */ - if (asoc->addip_last_asconf) + if (asoc->addip_last_asconf) { sctp_chunk_free(asoc->addip_last_asconf); + asoc->addip_last_asconf = NULL; + } } diff --git a/net/sctp/auth.c b/net/sctp/auth.c index be9782760f50..cc4229ee116d 100644 --- a/net/sctp/auth.c +++ b/net/sctp/auth.c @@ -377,6 +377,81 @@ nomem: return -ENOMEM; } +static bool sctp_auth_chunk_id_forbidden(__u8 chunk_id) +{ + switch (chunk_id) { + case SCTP_CID_INIT: + case SCTP_CID_INIT_ACK: + case SCTP_CID_SHUTDOWN_COMPLETE: + case SCTP_CID_AUTH: + return true; + default: + return false; + } +} + +/* Verify AUTH parameters copied from a state cookie before they are restored + * into an association. When cookie authentication is disabled these fields + * are peer-controlled, so they must satisfy the same constraints as locally + * generated AUTH parameters. + */ +bool sctp_auth_verify_cookie_params(const struct sctp_endpoint *ep, + const struct sctp_cookie *cookie) +{ + const struct sctp_paramhdr *random; + const struct sctp_hmac_algo_param *hmacs; + const struct sctp_chunks_param *chunks; + u16 hmacs_len, chunks_len; + u16 n_hmacs, n_chunks, i; + bool has_sha1 = false; + + if (sctp_sk(ep->base.sk)->cookie_auth_enable || !ep->auth_enable) + return true; + + random = (const struct sctp_paramhdr *)cookie->auth_random; + if (random->type != SCTP_PARAM_RANDOM || + ntohs(random->length) != sizeof(*random) + SCTP_AUTH_RANDOM_LENGTH) + return false; + + hmacs = (const struct sctp_hmac_algo_param *)cookie->auth_hmacs; + hmacs_len = ntohs(hmacs->param_hdr.length); + if (hmacs->param_hdr.type != SCTP_PARAM_HMAC_ALGO || + hmacs_len < sizeof(struct sctp_paramhdr) + + sizeof(hmacs->hmac_ids[0]) || + hmacs_len > sizeof(cookie->auth_hmacs) || + (hmacs_len - sizeof(struct sctp_paramhdr)) % + sizeof(hmacs->hmac_ids[0])) + return false; + + n_hmacs = (hmacs_len - sizeof(struct sctp_paramhdr)) / + sizeof(hmacs->hmac_ids[0]); + for (i = 0; i < n_hmacs; i++) { + u16 hmac_id = ntohs(hmacs->hmac_ids[i]); + + if (!sctp_hmac_supported(hmac_id)) + return false; + if (hmac_id == SCTP_AUTH_HMAC_ID_SHA1) + has_sha1 = true; + } + if (!has_sha1) + return false; + + chunks = (const struct sctp_chunks_param *)cookie->auth_chunks; + chunks_len = ntohs(chunks->param_hdr.length); + if (chunks->param_hdr.type != SCTP_PARAM_CHUNKS || + chunks_len < sizeof(struct sctp_paramhdr) || + chunks_len > sizeof(cookie->auth_chunks)) + return false; + + n_chunks = chunks_len - sizeof(struct sctp_paramhdr); + for (i = 0; i < n_chunks; i++) { + if (sctp_auth_chunk_id_forbidden(chunks->chunks[i])) + return false; + } + + return true; +} + /* Public interface to create the association shared key. * See code above for the algorithm. @@ -672,7 +747,7 @@ int sctp_auth_ep_add_chunkid(struct sctp_endpoint *ep, __u8 chunk_id) /* Check if we can add this chunk to the array */ param_len = ntohs(p->param_hdr.length); nchunks = param_len - sizeof(struct sctp_paramhdr); - if (nchunks == SCTP_NUM_CHUNK_TYPES) + if (nchunks == SCTP_AUTH_MAX_CHUNKS) return -EINVAL; p->chunks[nchunks] = chunk_id; diff --git a/net/sctp/outqueue.c b/net/sctp/outqueue.c index f6b8c13dafa4..e92fb9da4647 100644 --- a/net/sctp/outqueue.c +++ b/net/sctp/outqueue.c @@ -650,6 +650,7 @@ static int __sctp_outq_flush_rtx(struct sctp_outq *q, struct sctp_packet *pkt, if (chunk->tsn_gap_acked) { list_move_tail(&chunk->transmitted_list, &transport->transmitted); + chunk->transport = transport; continue; } diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index cf335494bffe..27c26e12f95d 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c @@ -1383,10 +1383,6 @@ static int __net_init sctp_defaults_init(struct net *net) net->sctp.l3mdev_accept = 1; #endif - status = sctp_sysctl_net_register(net); - if (status) - goto err_sysctl_register; - /* Allocate and initialise sctp mibs. */ status = init_sctp_mibs(net); if (status) @@ -1420,8 +1416,6 @@ err_init_proc: cleanup_sctp_mibs(net); #endif err_init_mibs: - sctp_sysctl_net_unregister(net); -err_sysctl_register: return status; } @@ -1436,7 +1430,6 @@ static void __net_exit sctp_defaults_exit(struct net *net) net->sctp.proc_net_sctp = NULL; #endif cleanup_sctp_mibs(net); - sctp_sysctl_net_unregister(net); } static struct pernet_operations sctp_defaults_ops = { @@ -1450,16 +1443,28 @@ static int __net_init sctp_ctrlsock_init(struct net *net) /* Initialize the control inode/socket for handling OOTB packets. */ status = sctp_ctl_sock_init(net); - if (status) + if (status) { pr_err("Failed to initialize the SCTP control sock\n"); + return status; + } + + status = sctp_sysctl_net_register(net); + if (status) { + inet_ctl_sock_destroy(net->sctp.ctl_sock); + net->sctp.ctl_sock = NULL; + } return status; } static void __net_exit sctp_ctrlsock_exit(struct net *net) { + sctp_sysctl_net_unregister(net); + sctp_udp_sock_stop(net); + /* Free the control endpoint. */ inet_ctl_sock_destroy(net->sctp.ctl_sock); + net->sctp.ctl_sock = NULL; } static struct pernet_operations sctp_ctrlsock_ops = { diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c index 8adac9e0cd66..236e25abc7a4 100644 --- a/net/sctp/sm_make_chunk.c +++ b/net/sctp/sm_make_chunk.c @@ -1802,9 +1802,9 @@ struct sctp_association *sctp_unpack_cookie( goto fail; } - /* Check to see if the cookie is stale. If there is already - * an association, there is no need to check cookie's expiration - * for init collision case of lost COOKIE ACK. + /* Check to see if the cookie is stale. RFC 9260 Section 5.2.4 + * exempts an expired cookie only when both Verification Tags match + * the current association. * If skb has been timestamped, then use the stamp, otherwise * use current time. This introduces a small possibility that * a cookie may be considered expired, but this would only slow @@ -1815,7 +1815,10 @@ struct sctp_association *sctp_unpack_cookie( else kt = ktime_get_real(); - if (!asoc && ktime_before(bear_cookie->expiration, kt)) { + if ((!asoc || + asoc->c.my_vtag != bear_cookie->my_vtag || + asoc->c.peer_vtag != bear_cookie->peer_vtag) && + ktime_before(bear_cookie->expiration, kt)) { suseconds_t usecs = ktime_to_us(ktime_sub(kt, bear_cookie->expiration)); __be32 n = htonl(usecs); @@ -1849,6 +1852,9 @@ struct sctp_association *sctp_unpack_cookie( /* Set up our peer's port number. */ retval->peer.port = ntohs(chunk->sctp_hdr->source); + if (!sctp_auth_verify_cookie_params(ep, bear_cookie)) + goto malformed; + /* Populate the association from the cookie. */ memcpy(&retval->c, bear_cookie, sizeof(*bear_cookie)); @@ -2168,7 +2174,13 @@ static enum sctp_ierror sctp_verify_param(struct net *net, case SCTP_PARAM_HEARTBEAT_INFO: case SCTP_PARAM_UNRECOGNIZED_PARAMETERS: case SCTP_PARAM_ECN_CAPABLE: + break; case SCTP_PARAM_ADAPTATION_LAYER_IND: + if (ntohs(param.p->length) != sizeof(*param.aind)) { + sctp_process_inv_paramlength(asoc, param.p, + chunk, err_chunk); + retval = SCTP_IERROR_ABORT; + } break; case SCTP_PARAM_SUPPORTED_EXT: @@ -3153,6 +3165,12 @@ static __be16 sctp_process_asconf_param(struct sctp_association *asoc, if (!peer) return SCTP_ERROR_DNS_FAILED; + /* Don't free asconf->transport; a later wildcard DEL-IP + * parameter reuses it. + */ + if (peer == asconf->transport) + return SCTP_ERROR_REQ_REFUSED; + sctp_assoc_rm_peer(asoc, peer); break; case SCTP_PARAM_SET_PRIMARY: @@ -3321,12 +3339,11 @@ struct sctp_chunk *sctp_process_asconf(struct sctp_association *asoc, goto done; } done: - asoc->peer.addip_serial++; - /* If we are sending a new ASCONF_ACK hold a reference to it in assoc * after freeing the reference to old asconf ack if any. */ if (asconf_ack) { + asoc->peer.addip_serial++; sctp_chunk_hold(asconf_ack); list_add_tail(&asconf_ack->transmitted_list, &asoc->asconf_ack_list); diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index d23d935e128e..3a8e16b29660 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -74,7 +74,8 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale( const struct sctp_association *asoc, const union sctp_subtype type, void *arg, - struct sctp_cmd_seq *commands); + struct sctp_cmd_seq *commands, + struct sctp_errhdr *err); static enum sctp_disposition sctp_sf_shut_8_4_5( struct net *net, const struct sctp_endpoint *ep, @@ -642,7 +643,7 @@ static bool sctp_auth_chunk_verify(struct net *net, struct sctp_chunk *chunk, struct sctp_chunk auth; if (!chunk->auth_chunk) - return true; + return !sctp_auth_recv_cid(chunk->chunk_hdr->type, asoc); /* SCTP-AUTH: auth_chunk pointer is only set when the cookie-echo * is supposed to be authenticated and we have to do delayed @@ -2529,9 +2530,15 @@ enum sctp_disposition sctp_sf_cookie_echoed_err( * errors. */ sctp_walk_errors(err, chunk->chunk_hdr) { - if (SCTP_ERROR_STALE_COOKIE == err->cause) - return sctp_sf_do_5_2_6_stale(net, ep, asoc, type, - arg, commands); + if (err->cause != SCTP_ERROR_STALE_COOKIE) + continue; + /* The staleness is only meaningful if the cause is long + * enough to hold it; a shorter one is malformed. + */ + if (ntohs(err->length) < sizeof(*err) + sizeof(__be32)) + break; + return sctp_sf_do_5_2_6_stale(net, ep, asoc, type, + arg, commands, err); } /* It is possible to have malformed error causes, and that @@ -2573,13 +2580,13 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale( const struct sctp_association *asoc, const union sctp_subtype type, void *arg, - struct sctp_cmd_seq *commands) + struct sctp_cmd_seq *commands, + struct sctp_errhdr *err) { int attempts = asoc->init_err_counter + 1; - struct sctp_chunk *chunk = arg, *reply; struct sctp_cookie_preserve_param bht; struct sctp_bind_addr *bp; - struct sctp_errhdr *err; + struct sctp_chunk *reply; u32 stale; if (attempts > asoc->max_init_attempts) { @@ -2590,8 +2597,6 @@ static enum sctp_disposition sctp_sf_do_5_2_6_stale( return SCTP_DISPOSITION_DELETE_TCB; } - err = (struct sctp_errhdr *)(chunk->skb->data); - /* When calculating the time extension, an implementation * SHOULD use the RTT information measured based on the * previous COOKIE ECHO / ERROR exchange, and should add no @@ -6140,8 +6145,12 @@ enum sctp_disposition sctp_sf_t4_timer_expire( struct sctp_cmd_seq *commands) { struct sctp_chunk *chunk = asoc->addip_last_asconf; - struct sctp_transport *transport = chunk->transport; + struct sctp_transport *transport; + + if (!chunk) + return SCTP_DISPOSITION_CONSUME; + transport = chunk->transport; SCTP_INC_STATS(net, SCTP_MIB_T4_RTO_EXPIREDS); /* ADDIP 4.1 B1) Increment the error counters and perform path failure diff --git a/net/sctp/stream.c b/net/sctp/stream.c index 5c2fdedea088..34ffe6c945a4 100644 --- a/net/sctp/stream.c +++ b/net/sctp/stream.c @@ -308,7 +308,8 @@ int sctp_send_reset_streams(struct sctp_association *asoc, goto out; param_len += str_nums * sizeof(__u16) + - sizeof(struct sctp_strreset_inreq); + (out ? sizeof(struct sctp_strreset_inreq) + : sizeof(struct sctp_strreset_outreq)); } if (param_len > SCTP_MAX_CHUNK_LEN - @@ -639,6 +640,9 @@ struct sctp_chunk *sctp_process_strreset_inreq( nums = (ntohs(param.p->length) - sizeof(*inreq)) / sizeof(__u16); str_p = inreq->list_of_streams; + if (nums * sizeof(__u16) + sizeof(struct sctp_strreset_outreq) > + SCTP_MAX_CHUNK_LEN - sizeof(struct sctp_reconf_chunk)) + goto out; for (i = 0; i < nums; i++) { if (ntohs(str_p[i]) >= stream->outcnt) { result = SCTP_STRRESET_ERR_WRONG_SSN; diff --git a/net/sctp/sysctl.c b/net/sctp/sysctl.c index 15e7db9a3ab2..fca840484ebf 100644 --- a/net/sctp/sysctl.c +++ b/net/sctp/sysctl.c @@ -615,11 +615,16 @@ int sctp_sysctl_net_register(struct net *net) void sctp_sysctl_net_unregister(struct net *net) { + struct ctl_table_header *header = net->sctp.sysctl_header; const struct ctl_table *table; - table = net->sctp.sysctl_header->ctl_table_arg; - unregister_net_sysctl_table(net->sctp.sysctl_header); + if (!header) + return; + + table = header->ctl_table_arg; + unregister_net_sysctl_table(header); kfree(table); + net->sctp.sysctl_header = NULL; } static struct ctl_table_header *sctp_sysctl_header; diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index b5db69073e20..00403175b740 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -1931,11 +1931,12 @@ static void smc_listen_out(struct smc_sock *new_smc) atomic_dec(&lsmc->queued_smc_hs); release_sock(newsmcsk); /* lock in smc_listen_work() */ + lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING); if (lsmc->sk.sk_state == SMC_LISTEN) { - lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING); smc_accept_enqueue(&lsmc->sk, newsmcsk); release_sock(&lsmc->sk); } else { /* no longer listening */ + release_sock(&lsmc->sk); smc_close_non_accepted(newsmcsk); } diff --git a/net/smc/smc_cdc.c b/net/smc/smc_cdc.c index 619b3bab3824..32d6d03df321 100644 --- a/net/smc/smc_cdc.c +++ b/net/smc/smc_cdc.c @@ -470,9 +470,9 @@ static void smc_cdc_rx_handler(struct ib_wc *wc, void *buf) { struct smc_link *link = (struct smc_link *)wc->qp->qp_context; struct smc_cdc_msg *cdc = buf; + struct smc_sock *smc = NULL; struct smc_connection *conn; struct smc_link_group *lgr; - struct smc_sock *smc; if (wc->byte_len < offsetof(struct smc_cdc_msg, reserved)) return; /* short message */ @@ -483,21 +483,26 @@ static void smc_cdc_rx_handler(struct ib_wc *wc, void *buf) lgr = smc_get_lgr(link); read_lock_bh(&lgr->conns_lock); conn = smc_lgr_find_conn(ntohl(cdc->token), lgr); - read_unlock_bh(&lgr->conns_lock); - if (!conn || conn->out_of_sync) + if (!conn || conn->out_of_sync) { + read_unlock_bh(&lgr->conns_lock); return; + } smc = container_of(conn, struct smc_sock, conn); + sock_hold(&smc->sk); + read_unlock_bh(&lgr->conns_lock); if (cdc->prod_flags.failover_validation) { smc_cdc_msg_validate(smc, cdc, link); - return; + goto out; } if (smc_cdc_before(ntohs(cdc->seqno), conn->local_rx_ctrl.seqno)) /* received seqno is old */ - return; + goto out; smc_cdc_msg_recv(smc, cdc); +out: + sock_put(&smc->sk); } static struct smc_wr_rx_handler smc_cdc_rx_handlers[] = { diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index cf6b620fef05..b4208cb186c5 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -1572,10 +1572,10 @@ static void __smc_lgr_terminate(struct smc_link_group *lgr, bool soft) read_lock_bh(&lgr->conns_lock); node = rb_first(&lgr->conns_all); while (node) { - read_unlock_bh(&lgr->conns_lock); conn = rb_entry(node, struct smc_connection, alert_node); smc = container_of(conn, struct smc_sock, conn); sock_hold(&smc->sk); /* sock_put below */ + read_unlock_bh(&lgr->conns_lock); lock_sock(&smc->sk); smc_conn_kill(conn, soft); release_sock(&smc->sk); diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c index 954b2ff1815c..aa6d83af55ed 100644 --- a/net/smc/smc_llc.c +++ b/net/smc/smc_llc.c @@ -1927,7 +1927,8 @@ static void smc_llc_event_handler(struct smc_llc_qentry *qentry) return; case SMC_LLC_CONFIRM_LINK: case SMC_LLC_ADD_LINK_CONT: - if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { + if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE && + !lgr->llc_flow_lcl.qentry) { /* a flow is waiting for this message */ smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); wake_up(&lgr->llc_msg_waiter); diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c index c1d9b923938d..5c9e4d8b57de 100644 --- a/net/smc/smc_rx.c +++ b/net/smc/smc_rx.c @@ -150,7 +150,12 @@ static const struct pipe_buf_operations smc_pipe_ops = { static void smc_rx_spd_release(struct splice_pipe_desc *spd, unsigned int i) { + struct smc_spd_priv *priv = (struct smc_spd_priv *)spd->partial[i].private; + struct sock *sk = &priv->smc->sk; + + kfree(priv); put_page(spd->pages[i]); + sock_put(sk); } static int smc_rx_splice(struct pipe_inode_info *pipe, char *src, size_t len, @@ -209,6 +214,10 @@ static int smc_rx_splice(struct pipe_inode_info *pipe, char *src, size_t len, offset = 0; } } + for (i = 0; i < nr_pages; i++) { + get_page(pages[i]); + sock_hold(&smc->sk); + } spd.nr_pages_max = nr_pages; spd.nr_pages = nr_pages; spd.pages = pages; @@ -217,16 +226,8 @@ static int smc_rx_splice(struct pipe_inode_info *pipe, char *src, size_t len, spd.spd_release = smc_rx_spd_release; bytes = splice_to_pipe(pipe, &spd); - if (bytes > 0) { - sock_hold(&smc->sk); - if (!lgr->is_smcd && smc->conn.rmb_desc->is_vm) { - for (i = 0; i < PAGE_ALIGN(bytes + offset) / PAGE_SIZE; i++) - get_page(pages[i]); - } else { - get_page(smc->conn.rmb_desc->pages); - } + if (bytes > 0) atomic_add(bytes, &smc->conn.splice_pending); - } kfree(priv); kfree(partial); kfree(pages); diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index bc8ca470718b..efa26899bc7d 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -1026,8 +1026,23 @@ rpc_free_auth(struct rpc_clnt *clnt) return NULL; } -/* - * Release reference to the RPC client +/** + * rpc_hold_client - acquire a reference on an rpc_clnt + * @clnt: rpc_clnt to pin + * + * Pairs with rpc_release_client(). + */ +void rpc_hold_client(struct rpc_clnt *clnt) +{ + refcount_inc(&clnt->cl_count); +} + +/** + * rpc_release_client - release a reference on an rpc_clnt + * @clnt: rpc_clnt to release + * + * Pairs with rpc_hold_client(). The rpc_clnt's resources are + * freed once its reference count drops to zero. */ void rpc_release_client(struct rpc_clnt *clnt) diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 2e1fe6013361..359407aae03e 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -2734,8 +2734,11 @@ static void xs_tcp_tls_setup_socket(struct work_struct *work) lower_xprt = rcu_dereference(lower_clnt->cl_xprt); rcu_read_unlock(); - if (wait_on_bit_lock(&lower_xprt->state, XPRT_LOCKED, TASK_KILLABLE)) + if (wait_on_bit_lock(&lower_xprt->state, XPRT_LOCKED, TASK_KILLABLE)) { + /* XPRT_LOCKED was never acquired. */ + rpc_shutdown_client(lower_clnt); goto out_unlock; + } status = xs_tls_handshake_sync(lower_xprt, &upper_xprt->xprtsec); if (status) { @@ -2758,6 +2761,7 @@ static void xs_tcp_tls_setup_socket(struct work_struct *work) out_unlock: current_restore_flags(pflags, PF_MEMALLOC); upper_transport->clnt = NULL; + rpc_release_client(upper_clnt); xprt_unlock_connect(upper_xprt, upper_transport); return; @@ -2805,7 +2809,15 @@ static void xs_connect(struct rpc_xprt *xprt, struct rpc_task *task) } else dprintk("RPC: xs_connect scheduled xprt %p\n", xprt); - transport->clnt = task->tk_client; + /* + * Only the TLS connect_worker reads transport->clnt; pinning + * the upper rpc_clnt unconditionally would form a cycle with + * cl_xprt and prevent xprt destruction. + */ + if (xprt->xprtsec.policy != RPC_XPRTSEC_NONE) { + rpc_hold_client(task->tk_client); + transport->clnt = task->tk_client; + } queue_delayed_work(xprtiod_workqueue, &transport->connect_worker, delay); diff --git a/net/tipc/netlink.c b/net/tipc/netlink.c index 8336a9664703..1307dd1a9613 100644 --- a/net/tipc/netlink.c +++ b/net/tipc/netlink.c @@ -113,12 +113,16 @@ const struct nla_policy tipc_nl_node_policy[TIPC_NLA_NODE_MAX + 1] = { }; /* Properties valid for media, bearer and link */ +static const struct netlink_range_validation tipc_nl_mtu_range = { + .max = U16_MAX, +}; + const struct nla_policy tipc_nl_prop_policy[TIPC_NLA_PROP_MAX + 1] = { [TIPC_NLA_PROP_UNSPEC] = { .type = NLA_UNSPEC }, [TIPC_NLA_PROP_PRIO] = { .type = NLA_U32 }, [TIPC_NLA_PROP_TOL] = { .type = NLA_U32 }, [TIPC_NLA_PROP_WIN] = { .type = NLA_U32 }, - [TIPC_NLA_PROP_MTU] = { .type = NLA_U32 }, + [TIPC_NLA_PROP_MTU] = NLA_POLICY_FULL_RANGE(NLA_U32, &tipc_nl_mtu_range), [TIPC_NLA_PROP_BROADCAST] = { .type = NLA_U32 }, [TIPC_NLA_PROP_BROADCAST_RATIO] = { .type = NLA_U32 } }; diff --git a/net/tipc/netlink_compat.c b/net/tipc/netlink_compat.c index 2a786c56c8c5..d9a4f94ea2d4 100644 --- a/net/tipc/netlink_compat.c +++ b/net/tipc/netlink_compat.c @@ -221,6 +221,10 @@ static int __tipc_nl_compat_dumpit(struct tipc_nl_compat_cmd_dump *cmd, int rem; len = (*cmd->dumpit)(buf, &cb); + if (len < 0) { + err = len; + goto err_out; + } nlmsg_for_each_msg(nlmsg, nlmsg_hdr(buf), len, rem) { err = nlmsg_parse_deprecated(nlmsg, GENL_HDRLEN, diff --git a/net/tipc/node.c b/net/tipc/node.c index 8e4ef2630ae4..683a136e53ef 100644 --- a/net/tipc/node.c +++ b/net/tipc/node.c @@ -1061,18 +1061,23 @@ static void __tipc_node_link_down(struct tipc_node *n, int *bearer_id, static void tipc_node_link_down(struct tipc_node *n, int bearer_id, bool delete) { - struct tipc_link_entry *le = &n->links[bearer_id]; struct tipc_media_addr *maddr = NULL; - struct tipc_link *l = le->link; int old_bearer_id = bearer_id; + struct tipc_link_entry *le; struct sk_buff_head xmitq; - - if (!l) - return; + struct tipc_link *l; __skb_queue_head_init(&xmitq); + /* Synchronize the link lookup with bearer teardown. */ tipc_node_write_lock(n); + le = &n->links[bearer_id]; + l = le->link; + if (!l) { + tipc_node_write_unlock_fast(n); + return; + } + if (!tipc_link_is_establishing(l)) { __tipc_node_link_down(n, &bearer_id, &xmitq, &maddr); } else { diff --git a/net/tipc/socket.c b/net/tipc/socket.c index e564341e0216..d5d70eb230b5 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c @@ -502,6 +502,7 @@ static int tipc_sk_create(struct net *net, struct socket *sock, tipc_set_sk_state(sk, TIPC_OPEN); if (tipc_sk_insert(tsk)) { sk_free(sk); + sock->sk = NULL; pr_warn("Socket create failed; port number exhausted\n"); return -EINVAL; } @@ -795,7 +796,7 @@ static __poll_t tipc_poll(struct file *file, struct socket *sock, __poll_t revents = 0; sock_poll_wait(file, sock, wait); - trace_tipc_sk_poll(sk, NULL, TIPC_DUMP_ALL, " "); + trace_tipc_sk_poll(sk, NULL, TIPC_DUMP_NONE, " "); if (sk->sk_shutdown & RCV_SHUTDOWN) revents |= EPOLLRDHUP | EPOLLIN | EPOLLRDNORM; @@ -1935,7 +1936,7 @@ static int tipc_recvmsg(struct socket *sock, struct msghdr *m, if (likely(!err)) { int offset = skb_cb->bytes_read; - copy = min_t(int, dlen - offset, buflen); + copy = min_t(size_t, dlen - offset, buflen); rc = skb_copy_datagram_msg(skb, hlen + offset, m, copy); if (unlikely(rc)) goto exit; @@ -2067,7 +2068,7 @@ static int tipc_recvstream(struct socket *sock, struct msghdr *m, /* Copy data if msg ok, otherwise return error/partial data */ if (likely(!err)) { offset = skb_cb->bytes_read; - copy = min_t(int, dlen - offset, buflen - copied); + copy = min_t(size_t, dlen - offset, buflen - copied); rc = skb_copy_datagram_msg(skb, hlen + offset, m, copy); if (unlikely(rc)) break; diff --git a/net/tipc/udp_media.c b/net/tipc/udp_media.c index 62ae7f5b5840..230645cc01c9 100644 --- a/net/tipc/udp_media.c +++ b/net/tipc/udp_media.c @@ -94,6 +94,8 @@ struct udp_replicast { * @ifindex: local address scope * @work: used to schedule deferred work on a bearer * @rcast: associated udp_replicast container + * @rcast_lock: serialize updates to @rcast.list against concurrent updaters + * @disabled: bearer is being torn down; reject further @rcast.list additions */ struct udp_bearer { struct tipc_bearer __rcu *bearer; @@ -101,6 +103,8 @@ struct udp_bearer { u32 ifindex; struct work_struct work; struct udp_replicast rcast; + spinlock_t rcast_lock; + bool disabled; }; static int tipc_udp_is_mcast_addr(struct udp_media_addr *addr) @@ -278,26 +282,6 @@ out: return err; } -static bool tipc_udp_is_known_peer(struct tipc_bearer *b, - struct udp_media_addr *addr) -{ - struct udp_replicast *rcast, *tmp; - struct udp_bearer *ub; - - ub = rcu_dereference_rtnl(b->media_ptr); - if (!ub) { - pr_err_ratelimited("UDP bearer instance not found\n"); - return false; - } - - list_for_each_entry_safe(rcast, tmp, &ub->rcast.list, list) { - if (!memcmp(&rcast->addr, addr, sizeof(struct udp_media_addr))) - return true; - } - - return false; -} - static int tipc_udp_rcast_add(struct tipc_bearer *b, struct udp_media_addr *addr) { @@ -308,16 +292,34 @@ static int tipc_udp_rcast_add(struct tipc_bearer *b, if (!ub) return -ENODEV; + spin_lock_bh(&ub->rcast_lock); + if (ub->disabled) { + spin_unlock_bh(&ub->rcast_lock); + return 0; + } + list_for_each_entry(rcast, &ub->rcast.list, list) { + if (!memcmp(&rcast->addr, addr, sizeof(*addr))) { + spin_unlock_bh(&ub->rcast_lock); + return 0; + } + } + rcast = kmalloc_obj(*rcast, GFP_ATOMIC); - if (!rcast) + if (!rcast) { + spin_unlock_bh(&ub->rcast_lock); return -ENOMEM; + } if (dst_cache_init(&rcast->dst_cache, GFP_ATOMIC)) { + spin_unlock_bh(&ub->rcast_lock); kfree(rcast); return -ENOMEM; } memcpy(&rcast->addr, addr, sizeof(struct udp_media_addr)); + list_add_rcu(&rcast->list, &ub->rcast.list); + b->bcast_addr.broadcast = TIPC_REPLICAST_SUPPORT; + spin_unlock_bh(&ub->rcast_lock); if (ntohs(addr->proto) == ETH_P_IP) pr_info("New replicast peer: %pI4\n", &rcast->addr.ipv4); @@ -325,8 +327,6 @@ static int tipc_udp_rcast_add(struct tipc_bearer *b, else if (ntohs(addr->proto) == ETH_P_IPV6) pr_info("New replicast peer: %pI6\n", &rcast->addr.ipv6); #endif - b->bcast_addr.broadcast = TIPC_REPLICAST_SUPPORT; - list_add_rcu(&rcast->list, &ub->rcast.list); return 0; } @@ -361,9 +361,6 @@ static int tipc_udp_rcast_disc(struct tipc_bearer *b, struct sk_buff *skb) return 0; } - if (likely(tipc_udp_is_known_peer(b, &src))) - return 0; - return tipc_udp_rcast_add(b, &src); } @@ -644,9 +641,6 @@ int tipc_udp_nl_bearer_add(struct tipc_bearer *b, struct nlattr *attr) return -EINVAL; } - if (tipc_udp_is_known_peer(b, &addr)) - return 0; - return tipc_udp_rcast_add(b, &addr); } @@ -679,6 +673,7 @@ static int tipc_udp_enable(struct net *net, struct tipc_bearer *b, return -ENOMEM; INIT_LIST_HEAD(&ub->rcast.list); + spin_lock_init(&ub->rcast_lock); if (!attrs[TIPC_NLA_BEARER_UDP_OPTS]) goto err; @@ -819,10 +814,13 @@ static void cleanup_bearer(struct work_struct *work) struct udp_replicast *rcast, *tmp; struct tipc_net *tn; + spin_lock_bh(&ub->rcast_lock); list_for_each_entry_safe(rcast, tmp, &ub->rcast.list, list) { list_del_rcu(&rcast->list); call_rcu_hurry(&rcast->rcu, rcast_free_rcu); } + ub->disabled = true; + spin_unlock_bh(&ub->rcast_lock); tn = tipc_net(sock_net(ub->sk)); diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 741aef09bfd3..37bb06a8e8f5 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -595,13 +595,15 @@ void tls_device_splice_eof(struct socket *sock) struct tls_context *tls_ctx = tls_get_ctx(sk); struct iov_iter iter = {}; - if (!tls_is_partially_sent_record(tls_ctx)) + if (!tls_is_partially_sent_record(tls_ctx) && + !tls_is_pending_open_record(tls_ctx)) return; mutex_lock(&tls_ctx->tx_lock); lock_sock(sk); - if (tls_is_partially_sent_record(tls_ctx)) { + if (tls_is_partially_sent_record(tls_ctx) || + tls_is_pending_open_record(tls_ctx)) { iov_iter_bvec(&iter, ITER_SOURCE, NULL, 0, 0); tls_push_data(sk, &iter, 0, 0, TLS_RECORD_TYPE_DATA); } diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 9324e4ed20a3..d1ad31986cf2 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -458,7 +458,7 @@ int tls_tx_records(struct sock *sk, int flags) } tx_err: - if (rc < 0 && rc != -EAGAIN) + if (rc < 0 && rc != -EAGAIN && rc != -EINTR && rc != -ERESTARTSYS) tls_err_abort(sk, rc); return rc; @@ -832,6 +832,14 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, if (!sk_stream_memory_free(sk)) goto wait_for_sndbuf; + /* open record may be full if we couldn't push it in the last sendmsg call */ + if (sk_msg_full(msg_pl)) { + full_record = true; + sk_msg_trim(sk, msg_en, + msg_pl->sg.size + prot->overhead_size); + goto copied; + } + alloc_encrypted: ret = tls_alloc_encrypted_msg(sk, required_size); if (ret) { @@ -921,6 +929,12 @@ fallback_to_reg_send: msg_pl, try_to_copy); if (ret < 0) goto trim_sgl; + + if (sk_msg_full(msg_pl)) { + full_record = true; + sk_msg_trim(sk, msg_en, + msg_pl->sg.size + prot->overhead_size); + } } /* Open records defined only if successfully copied, otherwise @@ -1442,6 +1456,8 @@ tls_decrypt_sw(struct sock *sk, struct tls_context *tls_ctx, /* If opportunistic TLS 1.3 ZC failed retry without ZC */ if (unlikely(darg->zc && prot->version == TLS_1_3_VERSION && darg->tail != TLS_RECORD_TYPE_DATA)) { + iov_iter_revert(&msg->msg_iter, strp_msg(darg->skb)->full_len - + prot->overhead_size); darg->zc = false; if (!darg->tail) TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXNOPADVIOL); @@ -1998,6 +2014,11 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, if (err < 0) return err; + /* If crypto failed the connection is broken */ + err = ctx->async_wait.err; + if (err) + goto splice_read_end; + if (!skb_queue_empty(&ctx->rx_list)) { skb = __skb_dequeue(&ctx->rx_list); } else { @@ -2115,6 +2136,17 @@ int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc, goto read_sock_requeue; } + /* An empty data record (legal in TLS 1.3) gives a zero + * read_actor return, indistinguishable from the consumer + * stalling; the used <= 0 path would requeue it at the + * head of rx_list and block all later records. Consume it + * here instead. + */ + if (rxm->full_len == 0) { + consume_skb(skb); + continue; + } + used = read_actor(desc, skb, rxm->offset, rxm->full_len); if (used <= 0) { if (!copied) diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index f7a9d55eee8a..10ed9421e43a 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -823,6 +823,7 @@ static int unix_listen(struct socket *sock, int backlog) if (err) goto out; unix_state_lock(sk); + err = -EINVAL; if (sk->sk_state != TCP_CLOSE && sk->sk_state != TCP_LISTEN) goto out_unlock; if (backlog > sk->sk_max_ack_backlog) diff --git a/net/unix/garbage.c b/net/unix/garbage.c index 0783555e2526..9fcaaf55cba5 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -186,6 +186,7 @@ static void unix_del_edge(struct scm_fp_list *fpl, struct unix_edge *edge) if (!vertex->out_degree) { edge->predecessor->vertex = NULL; list_move_tail(&vertex->entry, &fpl->vertices); + list_del(&vertex->scc_entry); } } diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c index 57f2d6ec3ffc..96c9fe8d357c 100644 --- a/net/vmw_vsock/virtio_transport.c +++ b/net/vmw_vsock/virtio_transport.c @@ -346,12 +346,13 @@ static void virtio_transport_tx_work(struct work_struct *work) struct virtqueue *vq; bool added = false; - vq = vsock->vqs[VSOCK_VQ_TX]; mutex_lock(&vsock->tx_lock); if (!vsock->tx_run) goto out; + vq = vsock->vqs[VSOCK_VQ_TX]; + do { struct sk_buff *skb; unsigned int len; @@ -451,13 +452,13 @@ static void virtio_transport_event_work(struct work_struct *work) container_of(work, struct virtio_vsock, event_work); struct virtqueue *vq; - vq = vsock->vqs[VSOCK_VQ_EVENT]; - mutex_lock(&vsock->event_lock); if (!vsock->event_run) goto out; + vq = vsock->vqs[VSOCK_VQ_EVENT]; + do { struct virtio_vsock_event *event; unsigned int len; @@ -634,12 +635,12 @@ static void virtio_transport_rx_work(struct work_struct *work) container_of(work, struct virtio_vsock, rx_work); struct virtqueue *vq; - vq = vsock->vqs[VSOCK_VQ_RX]; - mutex_lock(&vsock->rx_lock); if (!vsock->rx_run) - goto out; + goto out_nofill; + + vq = vsock->vqs[VSOCK_VQ_RX]; do { virtqueue_disable_cb(vq); @@ -691,6 +692,7 @@ static void virtio_transport_rx_work(struct work_struct *work) out: if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2) virtio_vsock_rx_fill(vsock); +out_nofill: mutex_unlock(&vsock->rx_lock); } diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index 09475007165b..8becad81279c 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -26,6 +26,13 @@ /* Threshold for detecting small packets to copy */ #define GOOD_COPY_LEN 128 +/* Max payload that can be collapsed into a single linear skb, using the same + * allocation threshold as virtio_vsock_alloc_skb() to avoid adding pressure + * on the page allocator. + */ +#define MAX_COLLAPSE_LEN \ + SKB_MAX_ORDER(VIRTIO_VSOCK_SKB_HEADROOM, PAGE_ALLOC_COSTLY_ORDER) + static void virtio_transport_cancel_close_work(struct vsock_sock *vsk, bool cancel_timeout); static s64 virtio_transport_has_space(struct virtio_vsock_sock *vvs); @@ -420,6 +427,145 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, return ret; } +static bool virtio_transport_can_collapse(struct sk_buff *skb) +{ + /* skbs that are partially consumed, mark a SEQPACKET message boundary, + * or are already large enough should not be collapsed: they either + * need special accounting, carry protocol state, or already have a + * good data-to-overhead ratio. + */ + if (VIRTIO_VSOCK_SKB_CB(skb)->offset) + return false; + if (le32_to_cpu(virtio_vsock_hdr(skb)->flags) & VIRTIO_VSOCK_SEQ_EOM) + return false; + if (skb->len >= MAX_COLLAPSE_LEN) + return false; + return true; +} + +/* Iterate through the packets in the queue starting from the current skb to + * count the number of bytes we can collapse. + */ +static unsigned int +virtio_transport_collapse_size(struct sk_buff *skb, struct sk_buff_head *queue) +{ + unsigned int target = skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset; + + while ((skb = skb_peek_next(skb, queue)) && + virtio_transport_can_collapse(skb)) { + unsigned int len = skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset; + + if (len > MAX_COLLAPSE_LEN - target) + return target; + + target += len; + } + + return target; +} + +/* Called under lock_sock to compact the receive queue by merging small skbs. + * @min_to_free: minimum number of skbs to eliminate from the queue. May free + * more to fill each collapsed skb to capacity. + */ +static void +virtio_transport_collapse_rx_queue(struct virtio_vsock_sock *vvs, + u32 min_to_free) +{ + struct sk_buff *skb, *next_skb, *new_skb = NULL; + struct sk_buff_head new_queue; + u32 saved = 0; + + __skb_queue_head_init(&new_queue); + + skb_queue_walk_safe(&vvs->rx_queue, skb, next_skb) { + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); + u32 src_off = VIRTIO_VSOCK_SKB_CB(skb)->offset; + u32 src_len = skb->len - src_off; + bool keep; + + keep = !virtio_transport_can_collapse(skb); + if (keep) { + /* Finalize pending collapsed skb to preserve packet + * ordering. + */ + if (new_skb) { + __skb_queue_tail(&new_queue, new_skb); + new_skb = NULL; + saved--; + } + goto next; + } + + /* Finalize if this packet won't fit in the remaining tailroom, + * so we can allocate a right-sized new_skb. + */ + if (new_skb && src_len > skb_tailroom(new_skb)) { + __skb_queue_tail(&new_queue, new_skb); + new_skb = NULL; + saved--; + } + + if (!new_skb) { + unsigned int alloc_size; + + /* Check after finalizing to opportunistically fill + * each collapsed skb to capacity, merging more skbs + * than strictly required. + */ + if (saved >= min_to_free) + break; + + alloc_size = virtio_transport_collapse_size(skb, &vvs->rx_queue); + + /* Only this skb's data is eligible, nothing to merge + * with. Keep as-is. + */ + if (alloc_size <= src_len) { + keep = true; + goto next; + } + + new_skb = virtio_vsock_alloc_linear_skb(alloc_size + + VIRTIO_VSOCK_SKB_HEADROOM, GFP_KERNEL); + if (!new_skb) + break; + + memcpy(virtio_vsock_hdr(new_skb), hdr, + sizeof(struct virtio_vsock_hdr)); + virtio_vsock_hdr(new_skb)->len = 0; + } + + /* Cannot fail since src_off/src_len are within bounds, but if + * it does, discard new_skb to avoid queuing corrupted data. + */ + if (WARN_ON_ONCE(skb_copy_bits(skb, src_off, + skb_put(new_skb, src_len), + src_len))) { + kfree_skb(new_skb); + new_skb = NULL; + break; + } + + le32_add_cpu(&virtio_vsock_hdr(new_skb)->len, src_len); + virtio_vsock_hdr(new_skb)->flags |= hdr->flags; + +next: + __skb_unlink(skb, &vvs->rx_queue); + if (keep) { + __skb_queue_tail(&new_queue, skb); + } else { + consume_skb(skb); + saved++; + } + } + + if (new_skb) + __skb_queue_tail(&new_queue, new_skb); + + skb_queue_splice(&new_queue, &vvs->rx_queue); +} + static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs, u32 len) { @@ -1354,12 +1500,29 @@ virtio_transport_recv_enqueue(struct vsock_sock *vsk, { struct virtio_vsock_sock *vvs = vsk->trans; bool can_enqueue, free_pkt = false; + u32 len, queue_max, queue_len; struct virtio_vsock_hdr *hdr; - u32 len; hdr = virtio_vsock_hdr(skb); len = le32_to_cpu(hdr->len); + /* virtio_transport_inc_rx_pkt() rejects packets when the per-skb + * overhead (skb_queue_len * SKB_TRUESIZE(0)) exceeds buf_alloc. + * Proactively collapse the queue before that happens. + * No rx_lock needed: lock_sock is held by caller, preventing + * concurrent enqueue or dequeue. + */ + queue_max = vvs->buf_alloc / SKB_TRUESIZE(0); + queue_len = skb_queue_len(&vvs->rx_queue); + if (queue_len >= queue_max) { + /* Walking a large queue may take a significant amount of time + * and cache misses, causing traffic burstiness. Limit the + * collapse to freeing room for this packet and the next one. + * It may free more to fill each collapsed skb to capacity. + */ + virtio_transport_collapse_rx_queue(vvs, queue_len + 2 - queue_max); + } + spin_lock_bh(&vvs->rx_lock); can_enqueue = virtio_transport_inc_rx_pkt(vvs, len); diff --git a/net/wireless/core.c b/net/wireless/core.c index 3dcf63b04c41..610238d723ff 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -1335,6 +1335,7 @@ void wiphy_unregister(struct wiphy *wiphy) /* this has nothing to do now but make sure it's gone */ cancel_work_sync(&rdev->wiphy_work); + cancel_work_sync(&rdev->sched_scan_res_wk); cancel_work_sync(&rdev->rfkill_block); cancel_work_sync(&rdev->conn_work); flush_work(&rdev->event_work); @@ -1424,6 +1425,7 @@ static void _cfg80211_unregister_wdev(struct wireless_dev *wdev, list_del_rcu(&wdev->list); synchronize_net(); rdev->devlist_generation++; + wiphy_work_cancel(wdev->wiphy, &wdev->disconnect_wk); cfg80211_mlme_purge_registrations(wdev); @@ -1613,7 +1615,7 @@ void cfg80211_init_wdev(struct wireless_dev *wdev) INIT_LIST_HEAD(&wdev->mgmt_registrations); INIT_LIST_HEAD(&wdev->pmsr_list); spin_lock_init(&wdev->pmsr_lock); - INIT_WORK(&wdev->pmsr_free_wk, cfg80211_pmsr_free_wk); + wiphy_work_init(&wdev->pmsr_free_wk, cfg80211_pmsr_free_wk); #ifdef CONFIG_CFG80211_WEXT wdev->wext.default_key = -1; @@ -1637,7 +1639,7 @@ void cfg80211_init_wdev(struct wireless_dev *wdev) wdev->iftype == NL80211_IFTYPE_ADHOC) && !wdev->use_4addr) wdev->netdev->priv_flags |= IFF_DONT_BRIDGE; - INIT_WORK(&wdev->disconnect_wk, cfg80211_autodisconnect_wk); + wiphy_work_init(&wdev->disconnect_wk, cfg80211_autodisconnect_wk); } void cfg80211_register_wdev(struct cfg80211_registered_device *rdev, @@ -1743,11 +1745,11 @@ static int cfg80211_netdev_notifier_call(struct notifier_block *nb, break; case NETDEV_GOING_DOWN: cfg80211_leave(rdev, wdev, -1); - scoped_guard(wiphy, &rdev->wiphy) + scoped_guard(wiphy, &rdev->wiphy) { cfg80211_remove_links(wdev); - /* since we just did cfg80211_leave() nothing to do there */ - cancel_work_sync(&wdev->disconnect_wk); - cancel_work_sync(&wdev->pmsr_free_wk); + /* since we just did cfg80211_leave() nothing to do there */ + wiphy_work_cancel(wdev->wiphy, &wdev->disconnect_wk); + } break; case NETDEV_DOWN: wiphy_lock(&rdev->wiphy); diff --git a/net/wireless/core.h b/net/wireless/core.h index df47ed6208a5..ac6ce9f967ec 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -428,7 +428,7 @@ void __cfg80211_port_authorized(struct wireless_dev *wdev, const u8 *peer_addr, const u8 *td_bitmap, u8 td_bitmap_len); int cfg80211_mgd_wext_connect(struct cfg80211_registered_device *rdev, struct wireless_dev *wdev); -void cfg80211_autodisconnect_wk(struct work_struct *work); +void cfg80211_autodisconnect_wk(struct wiphy *wiphy, struct wiphy_work *work); /* SME implementation */ void cfg80211_conn_work(struct work_struct *work); @@ -586,7 +586,7 @@ cfg80211_get_6ghz_power_type(const u8 *elems, size_t elems_len, void cfg80211_release_pmsr(struct wireless_dev *wdev, u32 portid); void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev); -void cfg80211_pmsr_free_wk(struct work_struct *work); +void cfg80211_pmsr_free_wk(struct wiphy *wiphy, struct wiphy_work *work); void cfg80211_remove_link(struct wireless_dev *wdev, unsigned int link_id); void cfg80211_remove_links(struct wireless_dev *wdev); diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c index 2a2c173058ba..7824b7ac2770 100644 --- a/net/wireless/mlme.c +++ b/net/wireless/mlme.c @@ -32,15 +32,11 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, .timeout_reason = NL80211_TIMEOUT_UNSPECIFIED, .req_ie = data->req_ies, .req_ie_len = data->req_ies_len, - .resp_ie = mgmt->u.assoc_resp.variable, - .resp_ie_len = data->len - - offsetof(struct ieee80211_mgmt, - u.assoc_resp.variable), - .status = le16_to_cpu(mgmt->u.assoc_resp.status_code), .ap_mld_addr = data->ap_mld_addr, .assoc_encrypted = data->assoc_encrypted, }; unsigned int link_id; + bool is_s1g = false; for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) { cr.links[link_id].status = data->links[link_id].status; @@ -61,16 +57,32 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, if (cr.links[link_id].bss->channel->band == NL80211_BAND_S1GHZ) { WARN_ON(link_id); - cr.resp_ie = (u8 *)&mgmt->u.s1g_assoc_resp.variable; - cr.resp_ie_len = data->len - - offsetof(struct ieee80211_mgmt, - u.s1g_assoc_resp.variable); + is_s1g = true; } if (cr.ap_mld_addr) cr.valid_links |= BIT(link_id); } + if (is_s1g) { + if (data->len < offsetof(struct ieee80211_mgmt, + u.s1g_assoc_resp.variable)) + goto free_bss; + cr.resp_ie = (u8 *)&mgmt->u.s1g_assoc_resp.variable; + cr.resp_ie_len = data->len - + offsetof(struct ieee80211_mgmt, + u.s1g_assoc_resp.variable); + } else { + if (data->len < offsetof(struct ieee80211_mgmt, + u.assoc_resp.variable)) + goto free_bss; + cr.resp_ie = mgmt->u.assoc_resp.variable; + cr.resp_ie_len = data->len - + offsetof(struct ieee80211_mgmt, + u.assoc_resp.variable); + } + cr.status = le16_to_cpu(mgmt->u.assoc_resp.status_code); + trace_cfg80211_send_rx_assoc(dev, data); /* @@ -79,22 +91,24 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, * and got a reject -- we only try again with an assoc * frame instead of reassoc. */ - if (cfg80211_sme_rx_assoc_resp(wdev, cr.status)) { - for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) { - struct cfg80211_bss *bss = data->links[link_id].bss; - - if (!bss) - continue; - - cfg80211_unhold_bss(bss_from_pub(bss)); - cfg80211_put_bss(wiphy, bss); - } - return; - } + if (cfg80211_sme_rx_assoc_resp(wdev, cr.status)) + goto free_bss; nl80211_send_rx_assoc(rdev, dev, data); /* update current_bss etc., consumes the bss reference */ __cfg80211_connect_result(dev, &cr, cr.status == WLAN_STATUS_SUCCESS); + return; + +free_bss: + for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) { + struct cfg80211_bss *bss = data->links[link_id].bss; + + if (!bss) + continue; + + cfg80211_unhold_bss(bss_from_pub(bss)); + cfg80211_put_bss(wiphy, bss); + } } EXPORT_SYMBOL(cfg80211_rx_assoc_resp); @@ -151,19 +165,35 @@ void cfg80211_rx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len) { struct wireless_dev *wdev = dev->ieee80211_ptr; struct ieee80211_mgmt *mgmt = (void *)buf; + __le16 fc; lockdep_assert_wiphy(wdev->wiphy); - trace_cfg80211_rx_mlme_mgmt(dev, buf, len); + if (len < sizeof(fc)) + return; + + fc = mgmt->frame_control; - if (WARN_ON(len < 2)) + if (ieee80211_is_auth(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.auth.status_code)) + return; + } else if (ieee80211_is_deauth(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code)) + return; + } else if (ieee80211_is_disassoc(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.disassoc.reason_code)) + return; + } else { return; + } + + trace_cfg80211_rx_mlme_mgmt(dev, buf, len); - if (ieee80211_is_auth(mgmt->frame_control)) + if (ieee80211_is_auth(fc)) cfg80211_process_auth(wdev, buf, len); - else if (ieee80211_is_deauth(mgmt->frame_control)) + else if (ieee80211_is_deauth(fc)) cfg80211_process_deauth(wdev, buf, len, false); - else if (ieee80211_is_disassoc(mgmt->frame_control)) + else cfg80211_process_disassoc(wdev, buf, len, false); } EXPORT_SYMBOL(cfg80211_rx_mlme_mgmt); @@ -216,15 +246,28 @@ void cfg80211_tx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len, { struct wireless_dev *wdev = dev->ieee80211_ptr; struct ieee80211_mgmt *mgmt = (void *)buf; + __le16 fc; lockdep_assert_wiphy(wdev->wiphy); - trace_cfg80211_tx_mlme_mgmt(dev, buf, len, reconnect); + if (len < sizeof(fc)) + return; - if (WARN_ON(len < 2)) + fc = mgmt->frame_control; + + if (ieee80211_is_deauth(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code)) + return; + } else if (ieee80211_is_disassoc(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.disassoc.reason_code)) + return; + } else { return; + } + + trace_cfg80211_tx_mlme_mgmt(dev, buf, len, reconnect); - if (ieee80211_is_deauth(mgmt->frame_control)) + if (ieee80211_is_deauth(fc)) cfg80211_process_deauth(wdev, buf, len, reconnect); else cfg80211_process_disassoc(wdev, buf, len, reconnect); diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 53b4b3f76697..5adcb6bd0fc5 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -461,7 +461,9 @@ nl80211_ftm_responder_policy[NL80211_FTM_RESP_ATTR_MAX + 1] = { static const struct nla_policy nl80211_pmsr_ftm_req_attr_policy[NL80211_PMSR_FTM_REQ_ATTR_MAX + 1] = { [NL80211_PMSR_FTM_REQ_ATTR_ASAP] = { .type = NLA_FLAG }, - [NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE] = { .type = NLA_U32 }, + [NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE] = + NLA_POLICY_RANGE(NLA_U32, NL80211_PREAMBLE_LEGACY, + NL80211_PREAMBLE_HE), [NL80211_PMSR_FTM_REQ_ATTR_NUM_BURSTS_EXP] = NLA_POLICY_MAX(NLA_U8, 15), [NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD] = { .type = NLA_U16 }, @@ -630,7 +632,7 @@ nl80211_mbssid_config_policy[NL80211_MBSSID_CONFIG_ATTR_MAX + 1] = { [NL80211_MBSSID_CONFIG_ATTR_TX_IFINDEX] = { .type = NLA_U32 }, [NL80211_MBSSID_CONFIG_ATTR_EMA] = { .type = NLA_FLAG }, [NL80211_MBSSID_CONFIG_ATTR_TX_LINK_ID] = - NLA_POLICY_MAX(NLA_U8, IEEE80211_MLD_MAX_NUM_LINKS), + NLA_POLICY_RANGE(NLA_U8, 0, IEEE80211_MLD_MAX_NUM_LINKS - 1), }; static const struct nla_policy @@ -6510,7 +6512,8 @@ static int nl80211_parse_mbssid_config(struct wiphy *wiphy, } static struct cfg80211_mbssid_elems * -nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs) +nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs, + struct netlink_ext_ack *extack) { struct nlattr *nl_elems; struct cfg80211_mbssid_elems *elems; @@ -6521,6 +6524,12 @@ nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs) return ERR_PTR(-EINVAL); nla_for_each_nested(nl_elems, attrs, rem_elems) { + int ret; + + ret = validate_ie_attr(nl_elems, extack); + if (ret) + return ERR_PTR(ret); + if (num_elems >= 255) return ERR_PTR(-EINVAL); num_elems++; @@ -6787,7 +6796,8 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev, if (attrs[NL80211_ATTR_MBSSID_ELEMS]) { struct cfg80211_mbssid_elems *mbssid = nl80211_parse_mbssid_elems(&rdev->wiphy, - attrs[NL80211_ATTR_MBSSID_ELEMS]); + attrs[NL80211_ATTR_MBSSID_ELEMS], + extack); if (IS_ERR(mbssid)) return PTR_ERR(mbssid); @@ -6803,8 +6813,10 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev, if (IS_ERR(rnr)) return PTR_ERR(rnr); - if (rnr && rnr->cnt < bcn->mbssid_ies->cnt) + if (rnr && rnr->cnt < bcn->mbssid_ies->cnt) { + kfree(rnr); return -EINVAL; + } bcn->rnr_ies = rnr; } @@ -22942,7 +22954,8 @@ static int nl80211_netlink_notify(struct notifier_block * nb, wdev->nl_owner_dead = true; schedule_work(&rdev->destroy_work); } else if (wdev->conn_owner_nlportid == notify->portid) { - schedule_work(&wdev->disconnect_wk); + wiphy_work_queue(wdev->wiphy, + &wdev->disconnect_wk); } cfg80211_release_pmsr(wdev, notify->portid); diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c index c8447448f3a5..97449bcb9a22 100644 --- a/net/wireless/pmsr.c +++ b/net/wireless/pmsr.c @@ -125,6 +125,7 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev, NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_FTM_REQ_ATTR_REQUEST_LCI], "FTM: LCI request not supported"); + return -EOPNOTSUPP; } out->ftm.request_civicloc = @@ -133,6 +134,7 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev, NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_FTM_REQ_ATTR_REQUEST_CIVICLOC], "FTM: civic location request not supported"); + return -EOPNOTSUPP; } out->ftm.trigger_based = @@ -261,8 +263,9 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev, "FTM: nominal time is required for PD NTB ranging"); return -EINVAL; } - out->ftm.nominal_time = - nla_get_u32(tb[NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME]); + if (tb[NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME]) + out->ftm.nominal_time = + nla_get_u32(tb[NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME]); if (tb[NL80211_PMSR_FTM_REQ_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS]) out->ftm.min_time_between_measurements = @@ -310,6 +313,7 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, { struct nlattr *tb[NL80211_PMSR_PEER_ATTR_MAX + 1]; struct nlattr *req[NL80211_PMSR_REQ_ATTR_MAX + 1]; + bool have_measurement_type = false; struct nlattr *treq; int err, rem; @@ -376,6 +380,14 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, } nla_for_each_nested(treq, req[NL80211_PMSR_REQ_ATTR_DATA], rem) { + if (have_measurement_type) { + NL_SET_ERR_MSG_ATTR(info->extack, treq, + "multiple measurement types in request data"); + return -EINVAL; + } + + have_measurement_type = true; + switch (nla_type(treq)) { case NL80211_PMSR_TYPE_FTM: err = pmsr_parse_ftm(rdev, treq, out, info); @@ -385,10 +397,16 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, "unsupported measurement type"); err = -EINVAL; } + if (err) + return err; } - if (err) - return err; + if (!have_measurement_type) { + NL_SET_ERR_MSG_ATTR(info->extack, + req[NL80211_PMSR_REQ_ATTR_DATA], + "missing measurement type in request data"); + return -EINVAL; + } return 0; } @@ -402,6 +420,7 @@ int nl80211_pmsr_start(struct sk_buff *skb, struct genl_info *info) const struct cfg80211_pmsr_capabilities *capa; struct cfg80211_pmsr_request *req; struct nlattr *peers, *peer; + u64 cookie; capa = rdev->wiphy.pmsr_capa; @@ -427,6 +446,11 @@ int nl80211_pmsr_start(struct sk_buff *skb, struct genl_info *info) } } + if (!count) { + NL_SET_ERR_MSG_ATTR(info->extack, peers, "No peers specified"); + return -EINVAL; + } + req = kzalloc_flex(*req, peers, count); if (!req) return -ENOMEM; @@ -498,14 +522,27 @@ int nl80211_pmsr_start(struct sk_buff *skb, struct genl_info *info) } req->cookie = cfg80211_assign_cookie(rdev); req->nl_portid = info->snd_portid; + cookie = req->cookie; + + /* + * Add to the list before the driver call; under races or broken + * drivers, completion may free the request before rdev_start_pmsr() + * returns. Use the saved cookie below. + */ + spin_lock_bh(&wdev->pmsr_lock); + list_add_tail(&req->list, &wdev->pmsr_list); + spin_unlock_bh(&wdev->pmsr_lock); err = rdev_start_pmsr(rdev, wdev, req); - if (err) + if (err) { + /* An error return leaves the request owned by this path. */ + spin_lock_bh(&wdev->pmsr_lock); + list_del(&req->list); + spin_unlock_bh(&wdev->pmsr_lock); goto out_err; + } - list_add_tail(&req->list, &wdev->pmsr_list); - - nl_set_extack_cookie_u64(info->extack, req->cookie); + nl_set_extack_cookie_u64(info->extack, cookie); return 0; out_err: kfree(req); @@ -807,13 +844,11 @@ static void cfg80211_pmsr_process_abort(struct wireless_dev *wdev) } } -void cfg80211_pmsr_free_wk(struct work_struct *work) +void cfg80211_pmsr_free_wk(struct wiphy *wiphy, struct wiphy_work *work) { struct wireless_dev *wdev = container_of(work, struct wireless_dev, pmsr_free_wk); - guard(wiphy)(wdev->wiphy); - cfg80211_pmsr_process_abort(wdev); } @@ -829,7 +864,7 @@ void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev) } spin_unlock_bh(&wdev->pmsr_lock); - cancel_work_sync(&wdev->pmsr_free_wk); + wiphy_work_cancel(wdev->wiphy, &wdev->pmsr_free_wk); if (found) cfg80211_pmsr_process_abort(wdev); @@ -844,7 +879,7 @@ void cfg80211_release_pmsr(struct wireless_dev *wdev, u32 portid) list_for_each_entry(req, &wdev->pmsr_list, list) { if (req->nl_portid == portid) { req->nl_portid = 0; - schedule_work(&wdev->pmsr_free_wk); + wiphy_work_queue(wdev->wiphy, &wdev->pmsr_free_wk); } } spin_unlock_bh(&wdev->pmsr_lock); diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 05b7dc6b766c..071083cc3367 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -205,7 +205,7 @@ bool cfg80211_is_element_inherited(const struct element *elem, return true; if (elem->id == WLAN_EID_EXTENSION) { - if (!ext_id_len) + if (!ext_id_len || !elem->datalen) return true; loop_len = ext_id_len; list = &non_inherit_elem->data[3 + id_len]; @@ -326,8 +326,11 @@ cfg80211_gen_new_ie(const u8 *ie, size_t ielen, /* For ML probe response, match the MLE in the frame body with * MLD id being 'bssid_index' */ - if (parent->id == WLAN_EID_EXTENSION && parent->datalen > 1 && + if (parent->id == WLAN_EID_EXTENSION && parent->data[0] == WLAN_EID_EXT_EHT_MULTI_LINK && + ieee80211_mle_type_ok(parent->data + 1, + IEEE80211_ML_CONTROL_TYPE_BASIC, + parent->datalen - 1) && bssid_index == ieee80211_mle_get_mld_id(parent->data + 1)) { if (!cfg80211_copy_elem_with_frags(parent, ie, ielen, @@ -3311,14 +3314,15 @@ cfg80211_inform_bss_frame_data(struct wiphy *wiphy, bssid = ext->u.s1g_beacon.sa; capability = le16_to_cpu(compat->compat_info); beacon_interval = le16_to_cpu(compat->beacon_int); + tsf = le32_to_cpu(ext->u.s1g_beacon.timestamp); + tsf |= (u64)le32_to_cpu(compat->tsf_completion) << 32; } else { bssid = mgmt->bssid; beacon_interval = le16_to_cpu(mgmt->u.probe_resp.beacon_int); capability = le16_to_cpu(mgmt->u.probe_resp.capab_info); + tsf = le64_to_cpu(mgmt->u.probe_resp.timestamp); } - tsf = le64_to_cpu(mgmt->u.probe_resp.timestamp); - if (ieee80211_is_probe_resp(mgmt->frame_control)) ftype = CFG80211_BSS_FTYPE_PRESP; else if (ext) @@ -3612,8 +3616,10 @@ int cfg80211_wext_siwscan(struct net_device *dev, /* translate "Scan for SSID" request */ if (wreq) { if (wrqu->data.flags & IW_SCAN_THIS_ESSID) { - if (wreq->essid_len > IEEE80211_MAX_SSID_LEN) - return -EINVAL; + if (wreq->essid_len > IEEE80211_MAX_SSID_LEN) { + err = -EINVAL; + goto out; + } memcpy(creq->req.ssids[0].ssid, wreq->essid, wreq->essid_len); creq->req.ssids[0].ssid_len = wreq->essid_len; diff --git a/net/wireless/sme.c b/net/wireless/sme.c index b451df3096dd..2a719b5c487e 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -1578,13 +1578,11 @@ int cfg80211_disconnect(struct cfg80211_registered_device *rdev, * Used to clean up after the connection / connection attempt owner socket * disconnects */ -void cfg80211_autodisconnect_wk(struct work_struct *work) +void cfg80211_autodisconnect_wk(struct wiphy *wiphy, struct wiphy_work *work) { struct wireless_dev *wdev = container_of(work, struct wireless_dev, disconnect_wk); - struct cfg80211_registered_device *rdev = wiphy_to_rdev(wdev->wiphy); - - guard(wiphy)(wdev->wiphy); + struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); if (wdev->conn_owner_nlportid) { switch (wdev->iftype) { diff --git a/net/wireless/wext-sme.c b/net/wireless/wext-sme.c index 573b6b15a446..b5914f3658db 100644 --- a/net/wireless/wext-sme.c +++ b/net/wireless/wext-sme.c @@ -319,6 +319,15 @@ int cfg80211_wext_siwgenie(struct net_device *dev, return 0; if (ie_len) { + const struct element *elem; + + for_each_element(elem, extra, ie_len) { + /* nothing */ + } + + if (!for_each_element_completed(elem, extra, ie_len)) + return -EINVAL; + ie = kmemdup(extra, ie_len, GFP_KERNEL); if (!ie) return -ENOMEM; diff --git a/net/x25/af_x25.c b/net/x25/af_x25.c index c31d2af5dd22..033e7d059f58 100644 --- a/net/x25/af_x25.c +++ b/net/x25/af_x25.c @@ -363,6 +363,7 @@ static void x25_destroy_timer(struct timer_list *t) struct sock *sk = timer_container_of(sk, t, sk_timer); x25_destroy_socket_from_timer(sk); + sock_put(sk); } /* @@ -398,9 +399,8 @@ static void __x25_destroy_socket(struct sock *sk) if (sk_has_allocations(sk)) { /* Defer: outstanding buffers */ - sk->sk_timer.expires = jiffies + 10 * HZ; sk->sk_timer.function = x25_destroy_timer; - add_timer(&sk->sk_timer); + sk_reset_timer(sk, &sk->sk_timer, jiffies + 10 * HZ); } else { /* drop last reference so sock_put will free */ __sock_put(sk); @@ -1768,15 +1768,19 @@ void x25_kill_by_neigh(struct x25_neigh *nb) { struct sock *s; +again: write_lock_bh(&x25_list_lock); sk_for_each(s, &x25_list) { if (x25_sk(s)->neighbour == nb) { + sock_hold(s); write_unlock_bh(&x25_list_lock); lock_sock(s); - x25_disconnect(s, ENETUNREACH, 0, 0); + if (x25_sk(s)->neighbour == nb) + x25_disconnect(s, ENETUNREACH, 0, 0); release_sock(s); - write_lock_bh(&x25_list_lock); + sock_put(s); + goto again; } } write_unlock_bh(&x25_list_lock); diff --git a/net/x25/x25_timer.c b/net/x25/x25_timer.c index 2ec63a1f4c6d..7896cd43f1cc 100644 --- a/net/x25/x25_timer.c +++ b/net/x25/x25_timer.c @@ -36,45 +36,45 @@ void x25_init_timers(struct sock *sk) void x25_start_heartbeat(struct sock *sk) { - mod_timer(&sk->sk_timer, jiffies + 5 * HZ); + sk_reset_timer(sk, &sk->sk_timer, jiffies + 5 * HZ); } void x25_stop_heartbeat(struct sock *sk) { - timer_delete(&sk->sk_timer); + sk_stop_timer(sk, &sk->sk_timer); } void x25_start_t2timer(struct sock *sk) { struct x25_sock *x25 = x25_sk(sk); - mod_timer(&x25->timer, jiffies + x25->t2); + sk_reset_timer(sk, &x25->timer, jiffies + x25->t2); } void x25_start_t21timer(struct sock *sk) { struct x25_sock *x25 = x25_sk(sk); - mod_timer(&x25->timer, jiffies + x25->t21); + sk_reset_timer(sk, &x25->timer, jiffies + x25->t21); } void x25_start_t22timer(struct sock *sk) { struct x25_sock *x25 = x25_sk(sk); - mod_timer(&x25->timer, jiffies + x25->t22); + sk_reset_timer(sk, &x25->timer, jiffies + x25->t22); } void x25_start_t23timer(struct sock *sk) { struct x25_sock *x25 = x25_sk(sk); - mod_timer(&x25->timer, jiffies + x25->t23); + sk_reset_timer(sk, &x25->timer, jiffies + x25->t23); } void x25_stop_timer(struct sock *sk) { - timer_delete(&x25_sk(sk)->timer); + sk_stop_timer(sk, &x25_sk(sk)->timer); } unsigned long x25_display_timer(struct sock *sk) @@ -108,7 +108,7 @@ static void x25_heartbeat_expiry(struct timer_list *t) sock_flag(sk, SOCK_DEAD))) { bh_unlock_sock(sk); x25_destroy_socket_from_timer(sk); - return; + goto out; } break; @@ -120,8 +120,14 @@ static void x25_heartbeat_expiry(struct timer_list *t) break; } restart_heartbeat: - x25_start_heartbeat(sk); + /* Do not rearm once __x25_destroy_socket() has unlinked the socket: + * it is past its cancel point and owns the teardown from there on. + */ + if (sk_hashed(sk)) + x25_start_heartbeat(sk); bh_unlock_sock(sk); +out: + sock_put(sk); } /* @@ -166,4 +172,5 @@ static void x25_timer_expiry(struct timer_list *t) } else x25_do_timer_expiry(sk); bh_unlock_sock(sk); + sock_put(sk); } diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index 58da2f4f4397..cd8643360eb3 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -210,6 +210,8 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) if (mr->flags & XDP_UMEM_TX_METADATA_LEN) { if (mr->tx_metadata_len >= 256 || mr->tx_metadata_len % 8) return -EINVAL; + if (mr->tx_metadata_len < 16) + return -EINVAL; umem->tx_metadata_len = mr->tx_metadata_len; } diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index b970f30ea9b9..7855ee09c4b6 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -499,6 +499,23 @@ void __xsk_map_flush(struct list_head *flush_list) void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries) { + u32 reclaim_descs = READ_ONCE(pool->reclaim_descs); + + if (unlikely(reclaim_descs)) { + u32 pending_descs = READ_ONCE(pool->tx_zc_pending_descs); + + if (nb_entries < pending_descs) { + WRITE_ONCE(pool->tx_zc_pending_descs, + pending_descs - nb_entries); + xskq_prod_submit_n(pool->cq, nb_entries); + return; + } + + WRITE_ONCE(pool->tx_zc_pending_descs, 0); + nb_entries += reclaim_descs; + WRITE_ONCE(pool->reclaim_descs, 0); + } + xskq_prod_submit_n(pool->cq, nb_entries); } EXPORT_SYMBOL(xsk_tx_completed); @@ -574,24 +591,157 @@ static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entr return nb_pkts; } +static void xsk_tx_commit_batch(struct xsk_buff_pool *pool, + struct xsk_tx_batch *batch) +{ + u32 nb_descs = xsk_tx_batch_cq_descs(batch); + u32 cq_cached_prod; + + if (!nb_descs) + return; + + cq_cached_prod = pool->cq->cached_prod; + xskq_prod_write_addr_batch(pool->cq, pool->tx_descs, nb_descs); + + if (unlikely(batch->reclaim_descs)) { + u32 cq_pending_descs; + + /* CQ is positional. Descriptors already written but not + * submitted must complete before any reclaim-only descriptors + * appended below. + */ + cq_pending_descs = cq_cached_prod - xskq_get_prod(pool->cq); + + WRITE_ONCE(pool->tx_zc_pending_descs, + batch->tx_descs + cq_pending_descs); + WRITE_ONCE(pool->reclaim_descs, batch->reclaim_descs); + if (unlikely(!pool->tx_zc_pending_descs)) + xsk_tx_completed(pool, 0); + } +} + +static struct xsk_tx_batch +__xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_sock *xs, + struct xdp_desc *descs, u32 max_descs) +{ + struct xsk_tx_batch batch = {}; + u32 entries; + + entries = xskq_cons_nb_entries(xs->tx, max_descs); + if (!entries) + return batch; + + batch = xskq_cons_read_desc_batch(xs, pool, descs, max_descs); + if (!xsk_tx_batch_cq_descs(&batch)) { + xs->tx->queue_empty_descs++; + } else { + __xskq_cons_release(xs->tx); + xs->sk.sk_write_space(&xs->sk); + } + return batch; +} + +static struct xsk_tx_batch +xsk_tx_peek_release_shared_desc_batch(struct xsk_buff_pool *pool, u32 max_descs) +{ + u32 cq_descs_before, cq_descs_after; + struct xsk_tx_batch sum_batch = {}; + bool budget_exhausted; + u32 per_socket_budget; + struct xdp_sock *xs; + + /* The fairness quota must allow one maximum-sized valid packet. */ + per_socket_budget = max_t(u32, MAX_PER_SOCKET_BUDGET, + pool->xdp_zc_max_segs); + +again: + budget_exhausted = false; + cq_descs_before = xsk_tx_batch_cq_descs(&sum_batch); + list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) { + u32 budget, budget_left, offset, remaining, used; + struct xsk_tx_batch curr_batch; + + /* Once reclaim-only descriptors have been appended to the CQ + * address area, do not append driver-visible Tx descriptors + * from another socket after them. xsk_tx_completed() relies on + * all driver-visible descriptors preceding all reclaim-only + * descriptors in CQ order. + */ + if (sum_batch.reclaim_descs) + break; + + /* be gentle when playing with pool->tx_descs */ + offset = xsk_tx_batch_cq_descs(&sum_batch); + if (offset >= max_descs) + break; + + if (xs->tx_budget_spent >= per_socket_budget) { + if (xskq_cons_nb_entries(xs->tx, 1)) + budget_exhausted = true; + continue; + } + + budget_left = per_socket_budget - xs->tx_budget_spent; + remaining = max_descs - offset; + budget = min(remaining, budget_left); + + curr_batch = __xsk_tx_peek_release_desc_batch(pool, xs, + pool->tx_descs + offset, + budget); + used = xsk_tx_batch_cq_descs(&curr_batch); + if (!used) { + if (curr_batch.budget_limited && budget_left < remaining) + budget_exhausted = true; + continue; + } + + xs->tx_budget_spent += used; + sum_batch.tx_descs += curr_batch.tx_descs; + sum_batch.reclaim_descs = curr_batch.reclaim_descs; + } + + cq_descs_after = xsk_tx_batch_cq_descs(&sum_batch); + + if (sum_batch.reclaim_descs || cq_descs_after >= max_descs) + return sum_batch; + + /* Continue filling the batch while this pass made progress */ + if (cq_descs_before != cq_descs_after) + goto again; + + if (!budget_exhausted) + return sum_batch; + + list_for_each_entry_rcu(xs, &pool->xsk_tx_list, tx_list) + xs->tx_budget_spent = 0; + goto again; +} + u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 nb_pkts) { + struct xsk_tx_batch batch = {}; struct xdp_sock *xs; + bool umem_shared; rcu_read_lock(); - if (!list_is_singular(&pool->xsk_tx_list)) { - /* Fallback to the non-batched version */ - rcu_read_unlock(); - return xsk_tx_peek_release_fallback(pool, nb_pkts); - } + if (unlikely(READ_ONCE(pool->reclaim_descs))) + goto out; - xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list); - if (!xs) { - nb_pkts = 0; + xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, + tx_list); + if (!xs) goto out; - } - nb_pkts = xskq_cons_nb_entries(xs->tx, nb_pkts); + nb_pkts = min(nb_pkts, pool->tx_descs_nentries); + if (!nb_pkts) + goto out; + + umem_shared = !list_is_singular(&pool->xsk_tx_list); + + if (umem_shared && !(pool->umem->flags & XDP_UMEM_SG_FLAG)) { + rcu_read_unlock(); + return xsk_tx_peek_release_fallback(pool, nb_pkts); + } /* This is the backpressure mechanism for the Tx path. Try to * reserve space in the completion queue for all packets, but @@ -603,19 +753,16 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 nb_pkts) if (!nb_pkts) goto out; - nb_pkts = xskq_cons_read_desc_batch(xs->tx, pool, nb_pkts); - if (!nb_pkts) { - xs->tx->queue_empty_descs++; - goto out; - } - - __xskq_cons_release(xs->tx); - xskq_prod_write_addr_batch(pool->cq, pool->tx_descs, nb_pkts); - xs->sk.sk_write_space(&xs->sk); + batch = umem_shared ? + xsk_tx_peek_release_shared_desc_batch(pool, nb_pkts) : + __xsk_tx_peek_release_desc_batch(pool, xs, + pool->tx_descs, + nb_pkts); + xsk_tx_commit_batch(pool, &batch); out: rcu_read_unlock(); - return nb_pkts; + return batch.tx_descs; } EXPORT_SYMBOL(xsk_tx_peek_release_desc_batch); @@ -737,6 +884,19 @@ static void xsk_cq_submit_addr_locked(struct xsk_buff_pool *pool, spin_unlock_irqrestore(&pool->cq_prod_lock, flags); } +static void xsk_cq_submit_addr_single_locked(struct xsk_buff_pool *pool, + struct xdp_desc *desc) +{ + unsigned long flags; + u32 idx; + + spin_lock_irqsave(&pool->cq_prod_lock, flags); + idx = xskq_get_prod(pool->cq); + xskq_prod_write_addr(pool->cq, idx, desc->addr); + xskq_prod_submit_n(pool->cq, 1); + spin_unlock_irqrestore(&pool->cq_prod_lock, flags); +} + static void xsk_cq_cancel_locked(struct xsk_buff_pool *pool, u32 n) { spin_lock(&pool->cq->cq_cached_prod_lock); @@ -794,8 +954,11 @@ static void xsk_consume_skb(struct sk_buff *skb) static void xsk_drop_skb(struct sk_buff *skb) { - xdp_sk(skb->sk)->tx->invalid_descs += xsk_get_num_desc(skb); - xsk_consume_skb(skb); + struct xdp_sock *xs = xdp_sk(skb->sk); + + xs->tx->invalid_descs += xsk_get_num_desc(skb); + consume_skb(skb); + xs->skb = NULL; } static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, @@ -804,15 +967,16 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, { struct xsk_tx_metadata *meta = NULL; u16 csum_start, csum_offset; + u64 flags; if (unlikely(pool->tx_metadata_len == 0)) return -EINVAL; meta = buffer - pool->tx_metadata_len; - if (unlikely(!xsk_buff_valid_tx_metadata(meta))) + if (unlikely(!xsk_buff_valid_tx_metadata(pool, meta, &flags))) return -EINVAL; - if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) { + if (flags & XDP_TXMD_FLAGS_CHECKSUM) { csum_start = READ_ONCE(meta->request.csum_start); csum_offset = READ_ONCE(meta->request.csum_offset); @@ -833,8 +997,10 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, } } - if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) - skb->skb_mstamp_ns = meta->request.launch_time; + if (flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + skb->skb_mstamp_ns = READ_ONCE(meta->request.launch_time); + if (!(flags & XDP_TXMD_FLAGS_TIMESTAMP)) + meta = NULL; xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); return 0; @@ -877,7 +1043,7 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, return ERR_PTR(-ENOMEM); /* in case of -EOVERFLOW that could happen below, - * xsk_consume_skb() will release this node as whole skb + * xsk_drop_skb() will release this node as whole skb * would be dropped, which implies freeing all list elements */ xsk_addr->addrs[xsk_addr->num_descs] = desc->addr; @@ -969,6 +1135,8 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, goto free_err; } + xsk_addr->addrs[xsk_addr->num_descs] = desc->addr; + if (unlikely(nr_frags == (MAX_SKB_FRAGS - 1) && xp_mb_desc(desc))) { err = -EOVERFLOW; goto free_err; @@ -986,8 +1154,6 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, skb_add_rx_frag(skb, nr_frags, page, 0, len, PAGE_SIZE); refcount_add(PAGE_SIZE, &xs->sk.sk_wmem_alloc); - - xsk_addr->addrs[xsk_addr->num_descs] = desc->addr; } } @@ -1025,13 +1191,14 @@ free_err: static int __xsk_generic_xmit(struct sock *sk) { struct xdp_sock *xs = xdp_sk(sk); - bool sent_frame = false; struct xdp_desc desc; struct sk_buff *skb; + u32 cached_cons; u32 max_batch; int err = 0; mutex_lock(&xs->mutex); + cached_cons = xs->tx->cached_cons; /* Since we dropped the RCU read lock, the socket state might have changed. */ if (unlikely(!xsk_is_bound(xs))) { @@ -1060,11 +1227,21 @@ static int __xsk_generic_xmit(struct sock *sk) goto out; } + if (unlikely(xs->drain_cont)) { + xsk_cq_submit_addr_single_locked(xs->pool, &desc); + xs->tx->invalid_descs++; + xskq_cons_release(xs->tx); + xs->drain_cont = xp_mb_desc(&desc); + continue; + } + skb = xsk_build_skb(xs, &desc); if (IS_ERR(skb)) { err = PTR_ERR(skb); if (err != -EOVERFLOW) goto out; + if (xp_mb_desc(&desc)) + xs->drain_cont = true; err = 0; continue; } @@ -1093,18 +1270,33 @@ static int __xsk_generic_xmit(struct sock *sk) goto out; } - sent_frame = true; xs->skb = NULL; } if (xskq_has_descs(xs->tx)) { + bool drain = xs->skb || xs->drain_cont || xp_mb_desc(&desc); + + err = xsk_cq_reserve_locked(xs->pool); + if (err) { + xs->tx->invalid_descs--; + if (xs->skb) + xsk_drop_skb(xs->skb); + xs->drain_cont = drain; + err = -EAGAIN; + goto out; + } + if (xs->skb) xsk_drop_skb(xs->skb); + + xsk_cq_submit_addr_single_locked(xs->pool, &desc); + xskq_cons_release(xs->tx); + xs->drain_cont = xp_mb_desc(&desc); } out: - if (sent_frame) + if (xs->tx->cached_cons != cached_cons) __xsk_tx_release(xs); mutex_unlock(&xs->mutex); @@ -1483,7 +1675,8 @@ static int xsk_bind(struct socket *sock, struct sockaddr_unsized *addr, int addr * and/or device. */ xs->pool = xp_create_and_assign_umem(xs, - umem_xs->umem); + umem_xs->umem, + dev->xdp_zc_max_segs); if (!xs->pool) { err = -ENOMEM; sockfd_put(sock); @@ -1515,7 +1708,8 @@ static int xsk_bind(struct socket *sock, struct sockaddr_unsized *addr, int addr * utilizes */ if (xs->tx && !xs->pool->tx_descs) { - err = xp_alloc_tx_descs(xs->pool, xs); + err = xp_alloc_tx_descs(xs->pool, xs, + dev->xdp_zc_max_segs); if (err) { xp_put_pool(xs->pool); xs->pool = NULL; @@ -1533,7 +1727,9 @@ static int xsk_bind(struct socket *sock, struct sockaddr_unsized *addr, int addr goto out_unlock; } else { /* This xsk has its own umem. */ - xs->pool = xp_create_and_assign_umem(xs, xs->umem); + xs->pool = xp_create_and_assign_umem(xs, xs->umem, + dev->xdp_zc_max_segs); + if (!xs->pool) { err = -ENOMEM; goto out_unlock; diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index 1f28a9641571..78c14f106395 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -42,17 +42,22 @@ void xp_destroy(struct xsk_buff_pool *pool) kvfree(pool); } -int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs) +int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs, + u32 max_segs) { - pool->tx_descs = kvzalloc_objs(*pool->tx_descs, xs->tx->nentries); + u32 nentries = max(xs->tx->nentries, max_segs); + + pool->tx_descs = kvzalloc_objs(*pool->tx_descs, nentries); if (!pool->tx_descs) return -ENOMEM; + pool->tx_descs_nentries = nentries; return 0; } struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, - struct xdp_umem *umem) + struct xdp_umem *umem, + u32 max_segs) { bool unaligned = umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG; struct xsk_buff_pool *pool; @@ -69,7 +74,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, goto out; if (xs->tx) - if (xp_alloc_tx_descs(pool, xs)) + if (xp_alloc_tx_descs(pool, xs, max_segs)) goto out; pool->chunk_mask = ~((u64)umem->chunk_size - 1); @@ -760,11 +765,11 @@ EXPORT_SYMBOL(xp_raw_get_dma); * @addr: desc address (from userspace) * * Helper for getting desc's DMA address and metadata pointer, if present. - * Saves one call on hotpath, double calculation of the actual address, - * and inline checks for metadata presence and sanity. + * Saves one call on hotpath and double calculation of the actual address. + * Metadata is validated later by xsk_tx_metadata_request(). * * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata - * pointer, if it is present and valid (initialized to %NULL otherwise). + * pointer, if it is present (initialized to %NULL otherwise). */ struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) { diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h index 3e3fbb73d23e..1bc42c8902f4 100644 --- a/net/xdp/xsk_queue.h +++ b/net/xdp/xsk_queue.h @@ -58,6 +58,17 @@ struct parsed_desc { u32 valid; }; +struct xsk_tx_batch { + u32 tx_descs; + u32 reclaim_descs; + bool budget_limited; +}; + +static inline u32 xsk_tx_batch_cq_descs(const struct xsk_tx_batch *batch) +{ + return batch->tx_descs + batch->reclaim_descs; +} + /* The structure of the shared state of the rings are a simple * circular buffer, as outlined in * Documentation/core-api/circular-buffers.rst. For the Rx and @@ -263,17 +274,18 @@ static inline void parse_desc(struct xsk_queue *q, struct xsk_buff_pool *pool, parsed->mb = xp_mb_desc(desc); } -static inline -u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool, - u32 max) +static inline struct xsk_tx_batch +xskq_cons_read_desc_batch(struct xdp_sock *xs, struct xsk_buff_pool *pool, + struct xdp_desc *descs, u32 max) { - u32 cached_cons = q->cached_cons, nb_entries = 0; - struct xdp_desc *descs = pool->tx_descs; - u32 total_descs = 0, nr_frags = 0; + bool drain = READ_ONCE(xs->drain_cont); + u32 cached_cons, nb_entries = 0; + struct xsk_tx_batch batch = {}; + struct xsk_queue *q = xs->tx; + u32 nr_frags = 0; + + cached_cons = q->cached_cons; - /* track first entry, if stumble upon *any* invalid descriptor, rewind - * current packet that consists of frags and stop the processing - */ while (cached_cons != q->cached_prod && nb_entries < max) { struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring; u32 idx = cached_cons & q->ring_mask; @@ -283,25 +295,42 @@ u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool, cached_cons++; parse_desc(q, pool, &descs[nb_entries], &parsed); if (unlikely(!parsed.valid)) - break; + drain = true; + + nr_frags++; + nb_entries++; if (likely(!parsed.mb)) { - total_descs += (nr_frags + 1); - nr_frags = 0; - } else { - nr_frags++; - if (nr_frags == pool->xdp_zc_max_segs) { + if (unlikely(drain)) { + batch.reclaim_descs = nr_frags; + WRITE_ONCE(xs->drain_cont, false); nr_frags = 0; break; } + + batch.tx_descs += nr_frags; + nr_frags = 0; + continue; + } + + if (nr_frags == pool->xdp_zc_max_segs) + drain = true; + } + + if (nr_frags) { + if (drain) { + batch.reclaim_descs = nr_frags; + WRITE_ONCE(xs->drain_cont, true); + } else { + if (nb_entries == max) + batch.budget_limited = true; + cached_cons -= nr_frags; } - nb_entries++; } - cached_cons -= nr_frags; /* Release valid plus any invalid entries */ xskq_cons_release_n(q, cached_cons - q->cached_cons); - return total_descs; + return batch; } /* Functions for consumers */ diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 630f3dd31cc5..f153bf695b9d 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -182,7 +182,7 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur err = x->type_offload->xmit(x, skb, esp_features); if (err) { if (err == -EINPROGRESS) - return NULL; + return ERR_PTR(-EINPROGRESS); XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); kfree_skb(skb); @@ -224,7 +224,15 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur pskb = skb2; } - return skb; + /* skb_gso_segment() set skb->prev to the last segment, but async + * crypto may have stolen it above without updating ->prev. Repoint + * it at the last retained segment so validate_xmit_skb_list() does + * not chain onto a segment now owned by the crypto engine. + */ + if (skb) + skb->prev = pskb; + + return skb ? skb : ERR_PTR(-EINPROGRESS); } EXPORT_SYMBOL_GPL(validate_xmit_xfrm); @@ -313,6 +321,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, } xso->dev = dev; + xso->ifindex = dev->ifindex; netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC); if (xuo->flags & XFRM_OFFLOAD_INBOUND) diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c index ad810d1f97c0..597aedeac26e 100644 --- a/net/xfrm/xfrm_iptfs.c +++ b/net/xfrm/xfrm_iptfs.c @@ -480,6 +480,7 @@ static int iptfs_skb_add_frags(struct sk_buff *skb, } __skb_frag_ref(tofrag); shinfo->nr_frags++; + shinfo->flags |= SKBFL_SHARED_FRAG; /* see if we are done */ fraglen = tofrag->len; diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index 458931062a04..eb1b6f67739e 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -55,8 +55,10 @@ static int nat_keepalive_send_ipv4(struct sk_buff *skb, ka->encap_sport, sock_net_uid(net, NULL)); rt = ip_route_output_key(net, &fl4); - if (IS_ERR(rt)) + if (IS_ERR(rt)) { + kfree_skb(skb); return PTR_ERR(rt); + } skb_dst_set(skb, &rt->dst); @@ -101,6 +103,7 @@ static int nat_keepalive_send_ipv6(struct sk_buff *skb, dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL); if (IS_ERR(dst)) { local_unlock_nested_bh(&nat_keepalive_sk_ipv6.bh_lock); + kfree_skb(skb); return PTR_ERR(dst); } @@ -118,7 +121,6 @@ static void nat_keepalive_send(struct nat_keepalive *ka) sizeof(struct ipv6hdr)) + sizeof(struct udphdr); const u8 nat_ka_payload = 0xFF; - int err = -EAFNOSUPPORT; struct sk_buff *skb; struct udphdr *uh; @@ -140,16 +142,17 @@ static void nat_keepalive_send(struct nat_keepalive *ka) switch (ka->family) { case AF_INET: - err = nat_keepalive_send_ipv4(skb, ka); + nat_keepalive_send_ipv4(skb, ka); break; #if IS_ENABLED(CONFIG_IPV6) case AF_INET6: - err = nat_keepalive_send_ipv6(skb, ka, uh); + nat_keepalive_send_ipv6(skb, ka, uh); break; #endif - } - if (err) + default: kfree_skb(skb); + break; + } } struct nat_keepalive_work_ctx { diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 7ef861a0e823..932a313b9460 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -1329,8 +1329,8 @@ static void xfrm_hash_rebuild(struct work_struct *work) } } - if (policy->selector.prefixlen_d < dbits || - policy->selector.prefixlen_s < sbits) + if (policy->selector.prefixlen_d >= dbits && + policy->selector.prefixlen_s >= sbits) continue; bin = xfrm_policy_inexact_alloc_bin(policy, dir); diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index c58cd024e3c6..36a4f6793ede 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -1547,6 +1547,7 @@ found: xso->type = XFRM_DEV_OFFLOAD_PACKET; xso->dir = xdo->dir; xso->dev = dev; + xso->ifindex = dev->ifindex; xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ; netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC); error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x, @@ -2071,8 +2072,11 @@ static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig, x->mode_cbs = orig->mode_cbs; if (x->mode_cbs && x->mode_cbs->clone_state) { - if (x->mode_cbs->clone_state(x, orig)) + if (x->mode_cbs->clone_state(x, orig)) { + if (!x->mode_data) + x->mode_cbs = NULL; goto error; + } } x->props.reqid = m->new_reqid; @@ -3010,7 +3014,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (sockptr_is_null(optval) && !optlen) { xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL); xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL); - __sk_dst_reset(sk); + sk_dst_reset(sk); return 0; } @@ -3050,7 +3054,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (err >= 0) { xfrm_sk_policy_insert(sk, err, pol); xfrm_pol_put(pol); - __sk_dst_reset(sk); + sk_dst_reset(sk); err = 0; } @@ -3291,6 +3295,8 @@ int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) if (x->mode_cbs->init_state) err = x->mode_cbs->init_state(x); module_put(x->mode_cbs->owner); + if (err && !x->mode_data) + x->mode_cbs = NULL; } error: return err; diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 6384795ee6b2..d6db63304ba6 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -1201,17 +1201,26 @@ static int copy_sec_ctx(struct xfrm_sec_ctx *s, struct sk_buff *skb) return 0; } -static void xso_to_xuo(const struct xfrm_dev_offload *xso, - struct xfrm_user_offload *xuo) +static void xso_to_xuo_ifindex(const struct xfrm_dev_offload *xso, int ifindex, + struct xfrm_user_offload *xuo) { - xuo->ifindex = xso->dev->ifindex; + xuo->ifindex = ifindex; if (xso->dir == XFRM_DEV_OFFLOAD_IN) xuo->flags = XFRM_OFFLOAD_INBOUND; if (xso->type == XFRM_DEV_OFFLOAD_PACKET) xuo->flags |= XFRM_OFFLOAD_PACKET; } -static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) +#ifdef CONFIG_XFRM_MIGRATE +static void xso_to_xuo(const struct xfrm_dev_offload *xso, + struct xfrm_user_offload *xuo) +{ + xso_to_xuo_ifindex(xso, xso->dev->ifindex, xuo); +} +#endif + +static int copy_user_offload_ifindex(const struct xfrm_dev_offload *xso, + int ifindex, struct sk_buff *skb) { struct xfrm_user_offload *xuo; struct nlattr *attr; @@ -1222,11 +1231,22 @@ static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) xuo = nla_data(attr); memset(xuo, 0, sizeof(*xuo)); - xso_to_xuo(xso, xuo); + xso_to_xuo_ifindex(xso, ifindex, xuo); return 0; } +static int copy_user_offload(struct xfrm_dev_offload *xso, struct sk_buff *skb) +{ + return copy_user_offload_ifindex(xso, xso->dev->ifindex, skb); +} + +static int copy_user_state_offload(const struct xfrm_dev_offload *xso, + struct sk_buff *skb) +{ + return copy_user_offload_ifindex(xso, READ_ONCE(xso->ifindex), skb); +} + static bool xfrm_redact(void) { return IS_ENABLED(CONFIG_SECURITY) && @@ -1433,8 +1453,8 @@ static int copy_to_user_state_extra(struct xfrm_state *x, &x->replay); if (ret) goto out; - if(x->xso.dev) - ret = copy_user_offload(&x->xso, skb); + if (READ_ONCE(x->xso.dev)) + ret = copy_user_state_offload(&x->xso, skb); if (ret) goto out; if (x->if_id) { @@ -2104,13 +2124,12 @@ static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family, switch (ut[i].mode) { case XFRM_MODE_TUNNEL: case XFRM_MODE_BEET: + case XFRM_MODE_IPTFS: if (ut[i].optional && dir == XFRM_POLICY_OUT) { NL_SET_ERR_MSG(extack, "Mode in optional template not allowed in outbound policy"); return -EINVAL; } break; - case XFRM_MODE_IPTFS: - break; default: if (ut[i].family != prev_family) { NL_SET_ERR_MSG(extack, "Mode in template doesn't support a family change"); @@ -4046,8 +4065,8 @@ static inline unsigned int xfrm_sa_len(struct xfrm_state *x) l += nla_total_size(sizeof(*x->coaddr)); if (x->props.extra_flags) l += nla_total_size(sizeof(x->props.extra_flags)); - if (x->xso.dev) - l += nla_total_size(sizeof(struct xfrm_user_offload)); + if (READ_ONCE(x->xso.dev)) + l += nla_total_size(sizeof(struct xfrm_user_offload)); if (x->props.smark.v | x->props.smark.m) { l += nla_total_size(sizeof(x->props.smark.v)); l += nla_total_size(sizeof(x->props.smark.m)); |