diff options
Diffstat (limited to 'net/smc')
-rw-r--r-- | net/smc/smc_ib.c | 1 | ||||
-rw-r--r-- | net/smc/smc_ib.h | 2 | ||||
-rw-r--r-- | net/smc/smc_pnet.c | 439 | ||||
-rw-r--r-- | net/smc/smc_pnet.h | 1 |
4 files changed, 249 insertions, 194 deletions
diff --git a/net/smc/smc_ib.c b/net/smc/smc_ib.c index 0b244be24fe0..53f429c04843 100644 --- a/net/smc/smc_ib.c +++ b/net/smc/smc_ib.c @@ -564,7 +564,6 @@ static void smc_ib_remove_dev(struct ib_device *ibdev, void *client_data) spin_lock(&smc_ib_devices.lock); list_del_init(&smcibdev->list); /* remove from smc_ib_devices */ spin_unlock(&smc_ib_devices.lock); - smc_pnet_remove_by_ibdev(smcibdev); smc_ib_cleanup_per_ibdev(smcibdev); ib_unregister_event_handler(&smcibdev->event_handler); kfree(smcibdev); diff --git a/net/smc/smc_ib.h b/net/smc/smc_ib.h index bac7fd65a4c0..da60ab9e8d70 100644 --- a/net/smc/smc_ib.h +++ b/net/smc/smc_ib.h @@ -42,6 +42,8 @@ struct smc_ib_device { /* ib-device infos for smc */ /* mac address per port*/ u8 pnetid[SMC_MAX_PORTS][SMC_MAX_PNETID_LEN]; /* pnetid per port */ + bool pnetid_by_user[SMC_MAX_PORTS]; + /* pnetid defined by user? */ u8 initialized : 1; /* ib dev CQ, evthdl done */ struct work_struct port_event_work; unsigned long port_event_mask; diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c index 632c3109dee5..327bf2afe820 100644 --- a/net/smc/smc_pnet.c +++ b/net/smc/smc_pnet.c @@ -24,6 +24,10 @@ #include "smc_ib.h" #include "smc_ism.h" +#define SMC_ASCII_BLANK 32 + +static struct net_device *pnet_find_base_ndev(struct net_device *ndev); + static struct nla_policy smc_pnet_policy[SMC_PNETID_MAX + 1] = { [SMC_PNETID_NAME] = { .type = NLA_NUL_STRING, @@ -56,13 +60,14 @@ static struct smc_pnettable { }; /** - * struct smc_pnetentry - pnet identifier name entry + * struct smc_user_pnetentry - pnet identifier name entry for/from user * @list: List node. * @pnet_name: Pnet identifier name * @ndev: pointer to network device. * @smcibdev: Pointer to IB device. + * @ib_port: Port of IB device. */ -struct smc_pnetentry { +struct smc_user_pnetentry { struct list_head list; char pnet_name[SMC_MAX_PNETID_LEN + 1]; struct net_device *ndev; @@ -70,33 +75,26 @@ struct smc_pnetentry { u8 ib_port; }; -/* Check if two RDMA device entries are identical. Use device name and port - * number for comparison. - */ -static bool smc_pnet_same_ibname(struct smc_pnetentry *pnetelem, char *ibname, - u8 ibport) -{ - return pnetelem->ib_port == ibport && - !strncmp(pnetelem->smcibdev->ibdev->name, ibname, - sizeof(pnetelem->smcibdev->ibdev->name)); -} +/* pnet entry stored in pnet table */ +struct smc_pnetentry { + struct list_head list; + char pnet_name[SMC_MAX_PNETID_LEN + 1]; + struct net_device *ndev; +}; -/* Find a pnetid in the pnet table. - */ -static struct smc_pnetentry *smc_pnet_find_pnetid(char *pnet_name) +/* Check if two given pnetids match */ +static bool smc_pnet_match(u8 *pnetid1, u8 *pnetid2) { - struct smc_pnetentry *pnetelem, *found_pnetelem = NULL; + int i; - read_lock(&smc_pnettable.lock); - list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) { - if (!strncmp(pnetelem->pnet_name, pnet_name, - sizeof(pnetelem->pnet_name))) { - found_pnetelem = pnetelem; + for (i = 0; i < SMC_MAX_PNETID_LEN; i++) { + if ((pnetid1[i] == 0 || pnetid1[i] == SMC_ASCII_BLANK) && + (pnetid2[i] == 0 || pnetid2[i] == SMC_ASCII_BLANK)) break; - } + if (pnetid1[i] != pnetid2[i]) + return false; } - read_unlock(&smc_pnettable.lock); - return found_pnetelem; + return true; } /* Remove a pnetid from the pnet table. @@ -104,21 +102,39 @@ static struct smc_pnetentry *smc_pnet_find_pnetid(char *pnet_name) static int smc_pnet_remove_by_pnetid(char *pnet_name) { struct smc_pnetentry *pnetelem, *tmp_pe; + struct smc_ib_device *ibdev; int rc = -ENOENT; + int ibport; + /* remove netdevices */ write_lock(&smc_pnettable.lock); list_for_each_entry_safe(pnetelem, tmp_pe, &smc_pnettable.pnetlist, list) { - if (!strncmp(pnetelem->pnet_name, pnet_name, - sizeof(pnetelem->pnet_name))) { + if (!pnet_name || + smc_pnet_match(pnetelem->pnet_name, pnet_name)) { list_del(&pnetelem->list); dev_put(pnetelem->ndev); kfree(pnetelem); rc = 0; - break; } } write_unlock(&smc_pnettable.lock); + /* remove ib devices */ + spin_lock(&smc_ib_devices.lock); + list_for_each_entry(ibdev, &smc_ib_devices.list, list) { + for (ibport = 0; ibport < SMC_MAX_PORTS; ibport++) { + if (ibdev->pnetid_by_user[ibport] && + (!pnet_name || + smc_pnet_match(pnet_name, + ibdev->pnetid[ibport]))) { + memset(ibdev->pnetid[ibport], 0, + SMC_MAX_PNETID_LEN); + ibdev->pnetid_by_user[ibport] = false; + rc = 0; + } + } + } + spin_unlock(&smc_ib_devices.lock); return rc; } @@ -144,53 +160,66 @@ static int smc_pnet_remove_by_ndev(struct net_device *ndev) return rc; } -/* Remove a pnet entry mentioning a given ib device from the pnet table. +/* Append a pnetid to the end of the pnet table if not already on this list. */ -int smc_pnet_remove_by_ibdev(struct smc_ib_device *ibdev) +static int smc_pnet_enter(struct smc_user_pnetentry *new_pnetelem) { - struct smc_pnetentry *pnetelem, *tmp_pe; - int rc = -ENOENT; - - write_lock(&smc_pnettable.lock); - list_for_each_entry_safe(pnetelem, tmp_pe, &smc_pnettable.pnetlist, - list) { - if (pnetelem->smcibdev == ibdev) { - list_del(&pnetelem->list); - dev_put(pnetelem->ndev); - kfree(pnetelem); - rc = 0; - break; + u8 pnet_null[SMC_MAX_PNETID_LEN] = {0}; + u8 ndev_pnetid[SMC_MAX_PNETID_LEN]; + struct smc_pnetentry *tmp_pnetelem; + struct smc_pnetentry *pnetelem; + struct net_device *ndev; + bool new_netdev = true; + bool new_ibdev = false; + + if (new_pnetelem->smcibdev) { + struct smc_ib_device *ib_dev = new_pnetelem->smcibdev; + int ib_port = new_pnetelem->ib_port; + + spin_lock(&smc_ib_devices.lock); + if (smc_pnet_match(ib_dev->pnetid[ib_port - 1], pnet_null)) { + memcpy(ib_dev->pnetid[ib_port - 1], + new_pnetelem->pnet_name, SMC_MAX_PNETID_LEN); + ib_dev->pnetid_by_user[ib_port - 1] = true; + new_ibdev = true; } + spin_unlock(&smc_ib_devices.lock); } - write_unlock(&smc_pnettable.lock); - return rc; -} -/* Append a pnetid to the end of the pnet table if not already on this list. - */ -static int smc_pnet_enter(struct smc_pnetentry *new_pnetelem) -{ - struct smc_pnetentry *pnetelem; - int rc = -EEXIST; + if (!new_pnetelem->ndev) + return new_ibdev ? 0 : -EEXIST; + + /* check if (base) netdev already has a pnetid. If there is one, we do + * not want to add a pnet table entry + */ + ndev = pnet_find_base_ndev(new_pnetelem->ndev); + if (!smc_pnetid_by_dev_port(ndev->dev.parent, ndev->dev_port, + ndev_pnetid)) + return new_ibdev ? 0 : -EEXIST; + + /* add a new netdev entry to the pnet table if there isn't one */ + tmp_pnetelem = kzalloc(sizeof(*pnetelem), GFP_KERNEL); + if (!tmp_pnetelem) + return -ENOMEM; + memcpy(tmp_pnetelem->pnet_name, new_pnetelem->pnet_name, + SMC_MAX_PNETID_LEN); + tmp_pnetelem->ndev = new_pnetelem->ndev; write_lock(&smc_pnettable.lock); list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) { - if (!strncmp(pnetelem->pnet_name, new_pnetelem->pnet_name, - sizeof(new_pnetelem->pnet_name)) || - !strncmp(pnetelem->ndev->name, new_pnetelem->ndev->name, - sizeof(new_pnetelem->ndev->name)) || - smc_pnet_same_ibname(pnetelem, - new_pnetelem->smcibdev->ibdev->name, - new_pnetelem->ib_port)) { - dev_put(pnetelem->ndev); - goto found; - } + if (pnetelem->ndev == new_pnetelem->ndev) + new_netdev = false; } - list_add_tail(&new_pnetelem->list, &smc_pnettable.pnetlist); - rc = 0; -found: - write_unlock(&smc_pnettable.lock); - return rc; + if (new_netdev) { + dev_hold(tmp_pnetelem->ndev); + list_add_tail(&tmp_pnetelem->list, &smc_pnettable.pnetlist); + write_unlock(&smc_pnettable.lock); + } else { + write_unlock(&smc_pnettable.lock); + kfree(tmp_pnetelem); + } + + return (new_netdev || new_ibdev) ? 0 : -EEXIST; } /* The limit for pnetid is 16 characters. @@ -241,7 +270,8 @@ out: /* Parse the supplied netlink attributes and fill a pnetentry structure. * For ethernet and infiniband device names verify that the devices exist. */ -static int smc_pnet_fill_entry(struct net *net, struct smc_pnetentry *pnetelem, +static int smc_pnet_fill_entry(struct net *net, + struct smc_user_pnetentry *pnetelem, struct nlattr *tb[]) { char *string, *ibname; @@ -258,30 +288,29 @@ static int smc_pnet_fill_entry(struct net *net, struct smc_pnetentry *pnetelem, goto error; rc = -EINVAL; - if (!tb[SMC_PNETID_ETHNAME]) - goto error; - rc = -ENOENT; - string = (char *)nla_data(tb[SMC_PNETID_ETHNAME]); - pnetelem->ndev = dev_get_by_name(net, string); - if (!pnetelem->ndev) - goto error; - - rc = -EINVAL; - if (!tb[SMC_PNETID_IBNAME]) - goto error; - rc = -ENOENT; - ibname = (char *)nla_data(tb[SMC_PNETID_IBNAME]); - ibname = strim(ibname); - pnetelem->smcibdev = smc_pnet_find_ib(ibname); - if (!pnetelem->smcibdev) - goto error; + if (tb[SMC_PNETID_ETHNAME]) { + string = (char *)nla_data(tb[SMC_PNETID_ETHNAME]); + pnetelem->ndev = dev_get_by_name(net, string); + if (!pnetelem->ndev) + goto error; + } rc = -EINVAL; - if (!tb[SMC_PNETID_IBPORT]) - goto error; - pnetelem->ib_port = nla_get_u8(tb[SMC_PNETID_IBPORT]); - if (pnetelem->ib_port < 1 || pnetelem->ib_port > SMC_MAX_PORTS) - goto error; + if (tb[SMC_PNETID_IBNAME]) { + ibname = (char *)nla_data(tb[SMC_PNETID_IBNAME]); + ibname = strim(ibname); + pnetelem->smcibdev = smc_pnet_find_ib(ibname); + if (!pnetelem->smcibdev) + goto error; + if (pnetelem->smcibdev) { + if (!tb[SMC_PNETID_IBPORT]) + goto error; + pnetelem->ib_port = nla_get_u8(tb[SMC_PNETID_IBPORT]); + if (pnetelem->ib_port < 1 || + pnetelem->ib_port > SMC_MAX_PORTS) + goto error; + } + } return 0; @@ -292,71 +321,44 @@ error: } /* Convert an smc_pnetentry to a netlink attribute sequence */ -static int smc_pnet_set_nla(struct sk_buff *msg, struct smc_pnetentry *pnetelem) +static int smc_pnet_set_nla(struct sk_buff *msg, + struct smc_user_pnetentry *pnetelem) { - if (nla_put_string(msg, SMC_PNETID_NAME, pnetelem->pnet_name) || - nla_put_string(msg, SMC_PNETID_ETHNAME, pnetelem->ndev->name) || - nla_put_string(msg, SMC_PNETID_IBNAME, - pnetelem->smcibdev->ibdev->name) || - nla_put_u8(msg, SMC_PNETID_IBPORT, pnetelem->ib_port)) + if (nla_put_string(msg, SMC_PNETID_NAME, pnetelem->pnet_name)) return -1; - return 0; -} - -/* Retrieve one PNETID entry */ -static int smc_pnet_get(struct sk_buff *skb, struct genl_info *info) -{ - struct smc_pnetentry *pnetelem; - struct sk_buff *msg; - void *hdr; - int rc; - - if (!info->attrs[SMC_PNETID_NAME]) - return -EINVAL; - pnetelem = smc_pnet_find_pnetid( - (char *)nla_data(info->attrs[SMC_PNETID_NAME])); - if (!pnetelem) - return -ENOENT; - msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); - if (!msg) - return -ENOMEM; - - hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq, - &smc_pnet_nl_family, 0, SMC_PNETID_GET); - if (!hdr) { - rc = -EMSGSIZE; - goto err_out; + if (pnetelem->ndev) { + if (nla_put_string(msg, SMC_PNETID_ETHNAME, + pnetelem->ndev->name)) + return -1; + } else { + if (nla_put_string(msg, SMC_PNETID_ETHNAME, "n/a")) + return -1; } - - if (smc_pnet_set_nla(msg, pnetelem)) { - rc = -ENOBUFS; - goto err_out; + if (pnetelem->smcibdev) { + if (nla_put_string(msg, SMC_PNETID_IBNAME, + pnetelem->smcibdev->ibdev->name) || + nla_put_u8(msg, SMC_PNETID_IBPORT, pnetelem->ib_port)) + return -1; + } else { + if (nla_put_string(msg, SMC_PNETID_IBNAME, "n/a") || + nla_put_u8(msg, SMC_PNETID_IBPORT, 0xff)) + return -1; } - genlmsg_end(msg, hdr); - return genlmsg_reply(msg, info); - -err_out: - nlmsg_free(msg); - return rc; + return 0; } static int smc_pnet_add(struct sk_buff *skb, struct genl_info *info) { struct net *net = genl_info_net(info); - struct smc_pnetentry *pnetelem; + struct smc_user_pnetentry pnetelem; int rc; - pnetelem = kzalloc(sizeof(*pnetelem), GFP_KERNEL); - if (!pnetelem) - return -ENOMEM; - rc = smc_pnet_fill_entry(net, pnetelem, info->attrs); + rc = smc_pnet_fill_entry(net, &pnetelem, info->attrs); if (!rc) - rc = smc_pnet_enter(pnetelem); - if (rc) { - kfree(pnetelem); - return rc; - } + rc = smc_pnet_enter(&pnetelem); + if (pnetelem.ndev) + dev_put(pnetelem.ndev); return rc; } @@ -376,7 +378,7 @@ static int smc_pnet_dump_start(struct netlink_callback *cb) static int smc_pnet_dumpinfo(struct sk_buff *skb, u32 portid, u32 seq, u32 flags, - struct smc_pnetentry *pnetelem) + struct smc_user_pnetentry *pnetelem) { void *hdr; @@ -392,42 +394,107 @@ static int smc_pnet_dumpinfo(struct sk_buff *skb, return 0; } -static int smc_pnet_dump(struct sk_buff *skb, struct netlink_callback *cb) +static int _smc_pnet_dump(struct sk_buff *skb, u32 portid, u32 seq, u8 *pnetid, + int start_idx) { + struct smc_user_pnetentry tmp_entry; struct smc_pnetentry *pnetelem; + struct smc_ib_device *ibdev; int idx = 0; + int ibport; + /* dump netdevices */ read_lock(&smc_pnettable.lock); list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) { - if (idx++ < cb->args[0]) + if (pnetid && !smc_pnet_match(pnetelem->pnet_name, pnetid)) + continue; + if (idx++ < start_idx) continue; - if (smc_pnet_dumpinfo(skb, NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, NLM_F_MULTI, - pnetelem)) { + memset(&tmp_entry, 0, sizeof(tmp_entry)); + memcpy(&tmp_entry.pnet_name, pnetelem->pnet_name, + SMC_MAX_PNETID_LEN); + tmp_entry.ndev = pnetelem->ndev; + if (smc_pnet_dumpinfo(skb, portid, seq, NLM_F_MULTI, + &tmp_entry)) { --idx; break; } } - cb->args[0] = idx; read_unlock(&smc_pnettable.lock); + + /* dump ib devices */ + spin_lock(&smc_ib_devices.lock); + list_for_each_entry(ibdev, &smc_ib_devices.list, list) { + for (ibport = 0; ibport < SMC_MAX_PORTS; ibport++) { + if (ibdev->pnetid_by_user[ibport]) { + if (pnetid && + !smc_pnet_match(ibdev->pnetid[ibport], + pnetid)) + continue; + if (idx++ < start_idx) + continue; + memset(&tmp_entry, 0, sizeof(tmp_entry)); + memcpy(&tmp_entry.pnet_name, + ibdev->pnetid[ibport], + SMC_MAX_PNETID_LEN); + tmp_entry.smcibdev = ibdev; + tmp_entry.ib_port = ibport + 1; + if (smc_pnet_dumpinfo(skb, portid, seq, + NLM_F_MULTI, + &tmp_entry)) { + --idx; + break; + } + } + } + } + spin_unlock(&smc_ib_devices.lock); + + return idx; +} + +static int smc_pnet_dump(struct sk_buff *skb, struct netlink_callback *cb) +{ + int idx; + + idx = _smc_pnet_dump(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, NULL, cb->args[0]); + + cb->args[0] = idx; return skb->len; } +/* Retrieve one PNETID entry */ +static int smc_pnet_get(struct sk_buff *skb, struct genl_info *info) +{ + struct sk_buff *msg; + void *hdr; + + if (!info->attrs[SMC_PNETID_NAME]) + return -EINVAL; + + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!msg) + return -ENOMEM; + + _smc_pnet_dump(msg, info->snd_portid, info->snd_seq, + nla_data(info->attrs[SMC_PNETID_NAME]), 0); + + /* finish multi part message and send it */ + hdr = nlmsg_put(msg, info->snd_portid, info->snd_seq, NLMSG_DONE, 0, + NLM_F_MULTI); + if (!hdr) { + nlmsg_free(msg); + return -EMSGSIZE; + } + return genlmsg_reply(msg, info); +} + /* Remove and delete all pnetids from pnet table. */ static int smc_pnet_flush(struct sk_buff *skb, struct genl_info *info) { - struct smc_pnetentry *pnetelem, *tmp_pe; - - write_lock(&smc_pnettable.lock); - list_for_each_entry_safe(pnetelem, tmp_pe, &smc_pnettable.pnetlist, - list) { - list_del(&pnetelem->list); - dev_put(pnetelem->ndev); - kfree(pnetelem); - } - write_unlock(&smc_pnettable.lock); - return 0; + return smc_pnet_remove_by_pnetid(NULL); } /* SMC_PNETID generic netlink operation definition */ @@ -534,6 +601,25 @@ static struct net_device *pnet_find_base_ndev(struct net_device *ndev) return ndev; } +static int smc_pnet_find_ndev_pnetid_by_table(struct net_device *netdev, + u8 *pnetid) +{ + struct smc_pnetentry *pnetelem; + int rc = -ENOENT; + + read_lock(&smc_pnettable.lock); + list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) { + if (netdev == pnetelem->ndev) { + /* get pnetid of netdev device */ + memcpy(pnetid, pnetelem->pnet_name, SMC_MAX_PNETID_LEN); + rc = 0; + break; + } + } + read_unlock(&smc_pnettable.lock); + return rc; +} + /* Determine the corresponding IB device port based on the hardware PNETID. * Searching stops at the first matching active IB device port with vlan_id * configured. @@ -549,7 +635,8 @@ static void smc_pnet_find_roce_by_pnetid(struct net_device *ndev, ndev = pnet_find_base_ndev(ndev); if (smc_pnetid_by_dev_port(ndev->dev.parent, ndev->dev_port, - ndev_pnetid)) + ndev_pnetid) && + smc_pnet_find_ndev_pnetid_by_table(ndev, ndev_pnetid)) return; /* pnetid could not be determined */ spin_lock(&smc_ib_devices.lock); @@ -557,8 +644,7 @@ static void smc_pnet_find_roce_by_pnetid(struct net_device *ndev, for (i = 1; i <= SMC_MAX_PORTS; i++) { if (!rdma_is_port_valid(ibdev->ibdev, i)) continue; - if (!memcmp(ibdev->pnetid[i - 1], ndev_pnetid, - SMC_MAX_PNETID_LEN) && + if (smc_pnet_match(ibdev->pnetid[i - 1], ndev_pnetid) && smc_ib_port_active(ibdev, i) && !smc_ib_determine_gid(ibdev, i, vlan_id, gid, NULL)) { @@ -593,31 +679,6 @@ static void smc_pnet_find_ism_by_pnetid(struct net_device *ndev, spin_unlock(&smcd_dev_list.lock); } -/* Lookup of coupled ib_device via SMC pnet table */ -static void smc_pnet_find_roce_by_table(struct net_device *netdev, - struct smc_ib_device **smcibdev, - u8 *ibport, unsigned short vlan_id, - u8 gid[]) -{ - struct smc_pnetentry *pnetelem; - - read_lock(&smc_pnettable.lock); - list_for_each_entry(pnetelem, &smc_pnettable.pnetlist, list) { - if (netdev == pnetelem->ndev) { - if (smc_ib_port_active(pnetelem->smcibdev, - pnetelem->ib_port) && - !smc_ib_determine_gid(pnetelem->smcibdev, - pnetelem->ib_port, vlan_id, - gid, NULL)) { - *smcibdev = pnetelem->smcibdev; - *ibport = pnetelem->ib_port; - } - break; - } - } - read_unlock(&smc_pnettable.lock); -} - /* PNET table analysis for a given sock: * determine ib_device and port belonging to used internal TCP socket * ethernet interface. @@ -636,13 +697,7 @@ void smc_pnet_find_roce_resource(struct sock *sk, if (!dst->dev) goto out_rel; - /* if possible, lookup via hardware-defined pnetid */ smc_pnet_find_roce_by_pnetid(dst->dev, smcibdev, ibport, vlan_id, gid); - if (*smcibdev) - goto out_rel; - - /* lookup via SMC PNET table */ - smc_pnet_find_roce_by_table(dst->dev, smcibdev, ibport, vlan_id, gid); out_rel: dst_release(dst); diff --git a/net/smc/smc_pnet.h b/net/smc/smc_pnet.h index 8ff777636e32..37044e4ee50f 100644 --- a/net/smc/smc_pnet.h +++ b/net/smc/smc_pnet.h @@ -31,7 +31,6 @@ static inline int smc_pnetid_by_dev_port(struct device *dev, int smc_pnet_init(void) __init; void smc_pnet_exit(void); -int smc_pnet_remove_by_ibdev(struct smc_ib_device *ibdev); void smc_pnet_find_roce_resource(struct sock *sk, struct smc_ib_device **smcibdev, u8 *ibport, unsigned short vlan_id, u8 gid[]); |