Date: Wed, 20 Jan 2016 15:02:43 +0000 (UTC) From: =?UTF-8?Q?Roger_Pau_Monn=c3=a9?= <royger@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r294442 - head/sys/dev/xen/netfront Message-ID: <201601201502.u0KF2h8p092321@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: royger Date: Wed Jan 20 15:02:43 2016 New Revision: 294442 URL: https://svnweb.freebsd.org/changeset/base/294442 Log: xen-netfront: add multiqueue support Add support for multiple TX and RX queue pairs. The default number of queues is set to 4, but can be easily changed from the sysctl node hw.xn.num_queues. Also heavily refactor netfront driver: break out a bunch of helper functions and different structures. Use threads to handle TX and RX. Remove some dead code and fix quite a few bugs as I go along. Submitted by: Wei Liu <wei.liu2@citrix.com> Reviewed by: royger Sponsored by: Citrix Systems R&D Relnotes: Yes Differential Revision: https://reviews.freebsd.org/D4193 Modified: head/sys/dev/xen/netfront/netfront.c Modified: head/sys/dev/xen/netfront/netfront.c ============================================================================== --- head/sys/dev/xen/netfront/netfront.c Wed Jan 20 14:49:16 2016 (r294441) +++ head/sys/dev/xen/netfront/netfront.c Wed Jan 20 15:02:43 2016 (r294442) @@ -1,5 +1,6 @@ /*- * Copyright (c) 2004-2006 Kip Macy + * Copyright (c) 2015 Wei Liu <wei.liu2@citrix.com> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -39,15 +40,14 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/socket.h> #include <sys/sysctl.h> +#include <sys/taskqueue.h> #include <net/if.h> #include <net/if_var.h> #include <net/if_arp.h> #include <net/ethernet.h> #include <net/if_media.h> - #include <net/bpf.h> - #include <net/if_types.h> #include <netinet/in.h> @@ -86,6 +86,12 @@ __FBSDID("$FreeBSD$"); static int xn_enable_lro = 1; TUNABLE_INT("hw.xn.enable_lro", &xn_enable_lro); +/* + * Number of pairs of queues. + */ +static unsigned long xn_num_queues = 4; +TUNABLE_ULONG("hw.xn.num_queues", &xn_num_queues); + /** * \brief The maximum allowed data fragments in a single transmit * request. @@ -100,149 +106,167 @@ TUNABLE_INT("hw.xn.enable_lro", &xn_enab #define net_ratelimit() 0 +struct netfront_rxq; +struct netfront_txq; struct netfront_info; struct netfront_rx_info; -static void xn_txeof(struct netfront_info *); -static void xn_rxeof(struct netfront_info *); -static void network_alloc_rx_buffers(struct netfront_info *); - -static void xn_tick_locked(struct netfront_info *); -static void xn_tick(void *); - -static void xn_intr(void *); +static void xn_txeof(struct netfront_txq *); +static void xn_rxeof(struct netfront_rxq *); +static void xn_alloc_rx_buffers(struct netfront_rxq *); + +static void xn_release_rx_bufs(struct netfront_rxq *); +static void xn_release_tx_bufs(struct netfront_txq *); + +static void xn_rxq_intr(void *); +static void xn_txq_intr(void *); +static int xn_intr(void *); static inline int xn_count_frags(struct mbuf *m); -static int xn_assemble_tx_request(struct netfront_info *sc, - struct mbuf *m_head); -static void xn_start_locked(struct ifnet *); -static void xn_start(struct ifnet *); -static int xn_ioctl(struct ifnet *, u_long, caddr_t); +static int xn_assemble_tx_request(struct netfront_txq *, struct mbuf *); +static int xn_ioctl(struct ifnet *, u_long, caddr_t); static void xn_ifinit_locked(struct netfront_info *); static void xn_ifinit(void *); static void xn_stop(struct netfront_info *); static void xn_query_features(struct netfront_info *np); -static int xn_configure_features(struct netfront_info *np); -#ifdef notyet -static void xn_watchdog(struct ifnet *); -#endif - -#ifdef notyet -static void netfront_closing(device_t dev); -#endif +static int xn_configure_features(struct netfront_info *np); static void netif_free(struct netfront_info *info); static int netfront_detach(device_t dev); +static int xn_txq_mq_start_locked(struct netfront_txq *, struct mbuf *); +static int xn_txq_mq_start(struct ifnet *, struct mbuf *); + static int talk_to_backend(device_t dev, struct netfront_info *info); static int create_netdev(device_t dev); static void netif_disconnect_backend(struct netfront_info *info); -static int setup_device(device_t dev, struct netfront_info *info); -static void free_ring(int *ref, void *ring_ptr_ref); - -static int xn_ifmedia_upd(struct ifnet *ifp); +static int setup_device(device_t dev, struct netfront_info *info, + unsigned long); +static int xn_ifmedia_upd(struct ifnet *ifp); static void xn_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr); -/* Xenolinux helper functions */ -int network_connect(struct netfront_info *); +int xn_connect(struct netfront_info *); -static void xn_free_rx_ring(struct netfront_info *); - -static void xn_free_tx_ring(struct netfront_info *); - -static int xennet_get_responses(struct netfront_info *np, - struct netfront_rx_info *rinfo, RING_IDX rp, RING_IDX *cons, - struct mbuf **list); +static int xn_get_responses(struct netfront_rxq *, + struct netfront_rx_info *, RING_IDX, RING_IDX *, + struct mbuf **); #define virt_to_mfn(x) (vtophys(x) >> PAGE_SHIFT) #define INVALID_P2M_ENTRY (~0UL) -/* - * Mbuf pointers. We need these to keep track of the virtual addresses - * of our mbuf chains since we can only convert from virtual to physical, - * not the other way around. The size must track the free index arrays. - */ -struct xn_chain_data { - struct mbuf *xn_tx_chain[NET_TX_RING_SIZE+1]; - int xn_tx_chain_cnt; - struct mbuf *xn_rx_chain[NET_RX_RING_SIZE+1]; +struct xn_rx_stats +{ + u_long rx_packets; /* total packets received */ + u_long rx_bytes; /* total bytes received */ + u_long rx_errors; /* bad packets received */ }; -struct netfront_stats +struct xn_tx_stats { - u_long rx_packets; /* total packets received */ - u_long tx_packets; /* total packets transmitted */ - u_long rx_bytes; /* total bytes received */ - u_long tx_bytes; /* total bytes transmitted */ - u_long rx_errors; /* bad packets received */ - u_long tx_errors; /* packet transmit problems */ + u_long tx_packets; /* total packets transmitted */ + u_long tx_bytes; /* total bytes transmitted */ + u_long tx_errors; /* packet transmit problems */ +}; + +#define XN_QUEUE_NAME_LEN 8 /* xn{t,r}x_%u, allow for two digits */ +struct netfront_rxq { + struct netfront_info *info; + u_int id; + char name[XN_QUEUE_NAME_LEN]; + struct mtx lock; + + int ring_ref; + netif_rx_front_ring_t ring; + xen_intr_handle_t xen_intr_handle; + + grant_ref_t gref_head; + grant_ref_t grant_ref[NET_TX_RING_SIZE + 1]; + + struct mbuf *mbufs[NET_RX_RING_SIZE + 1]; + struct mbufq batch; /* batch queue */ + int target; + + xen_pfn_t pfn_array[NET_RX_RING_SIZE]; + + struct lro_ctrl lro; + + struct taskqueue *tq; + struct task intrtask; + + struct xn_rx_stats stats; +}; + +struct netfront_txq { + struct netfront_info *info; + u_int id; + char name[XN_QUEUE_NAME_LEN]; + struct mtx lock; + + int ring_ref; + netif_tx_front_ring_t ring; + xen_intr_handle_t xen_intr_handle; + + grant_ref_t gref_head; + grant_ref_t grant_ref[NET_TX_RING_SIZE + 1]; + + struct mbuf *mbufs[NET_TX_RING_SIZE + 1]; + int mbufs_cnt; + struct buf_ring *br; + + struct taskqueue *tq; + struct task intrtask; + struct task defrtask; + + bool full; + + struct xn_tx_stats stats; }; struct netfront_info { - struct ifnet *xn_ifp; - struct lro_ctrl xn_lro; + struct ifnet *xn_ifp; - struct netfront_stats stats; - u_int tx_full; + struct mtx sc_lock; - netif_tx_front_ring_t tx; - netif_rx_front_ring_t rx; + u_int num_queues; + struct netfront_rxq *rxq; + struct netfront_txq *txq; - struct mtx tx_lock; - struct mtx rx_lock; - struct mtx sc_lock; - - xen_intr_handle_t xen_intr_handle; - u_int carrier; - u_int maxfrags; + u_int carrier; + u_int maxfrags; /* Receive-ring batched refills. */ #define RX_MIN_TARGET 32 #define RX_MAX_TARGET NET_RX_RING_SIZE - int rx_min_target; - int rx_max_target; - int rx_target; - - grant_ref_t gref_tx_head; - grant_ref_t grant_tx_ref[NET_TX_RING_SIZE + 1]; - grant_ref_t gref_rx_head; - grant_ref_t grant_rx_ref[NET_TX_RING_SIZE + 1]; + int rx_min_target; + int rx_max_target; device_t xbdev; - int tx_ring_ref; - int rx_ring_ref; uint8_t mac[ETHER_ADDR_LEN]; - struct xn_chain_data xn_cdata; /* mbufs */ - struct mbufq xn_rx_batch; /* batch queue */ int xn_if_flags; - struct callout xn_stat_ch; - xen_pfn_t rx_pfn_array[NET_RX_RING_SIZE]; struct ifmedia sc_media; bool xn_resume; }; -#define rx_mbufs xn_cdata.xn_rx_chain -#define tx_mbufs xn_cdata.xn_tx_chain +struct netfront_rx_info { + struct netif_rx_response rx; + struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1]; +}; -#define XN_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_lock) -#define XN_RX_UNLOCK(_sc) mtx_unlock(&(_sc)->rx_lock) +#define XN_RX_LOCK(_q) mtx_lock(&(_q)->lock) +#define XN_RX_UNLOCK(_q) mtx_unlock(&(_q)->lock) -#define XN_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_lock) -#define XN_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_lock) +#define XN_TX_LOCK(_q) mtx_lock(&(_q)->lock) +#define XN_TX_TRYLOCK(_q) mtx_trylock(&(_q)->lock) +#define XN_TX_UNLOCK(_q) mtx_unlock(&(_q)->lock) #define XN_LOCK(_sc) mtx_lock(&(_sc)->sc_lock); #define XN_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_lock); #define XN_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_lock, MA_OWNED); -#define XN_RX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->rx_lock, MA_OWNED); -#define XN_TX_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->tx_lock, MA_OWNED); - -struct netfront_rx_info { - struct netif_rx_response rx; - struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1]; -}; +#define XN_RX_LOCK_ASSERT(_q) mtx_assert(&(_q)->lock, MA_OWNED); +#define XN_TX_LOCK_ASSERT(_q) mtx_assert(&(_q)->lock, MA_OWNED); #define netfront_carrier_on(netif) ((netif)->carrier = 1) #define netfront_carrier_off(netif) ((netif)->carrier = 0) @@ -253,6 +277,7 @@ struct netfront_rx_info { static inline void add_id_to_freelist(struct mbuf **list, uintptr_t id) { + KASSERT(id != 0, ("%s: the head item (0) must always be free.", __func__)); list[id] = list[0]; @@ -272,30 +297,33 @@ get_id_from_freelist(struct mbuf **list) } static inline int -xennet_rxidx(RING_IDX idx) +xn_rxidx(RING_IDX idx) { + return idx & (NET_RX_RING_SIZE - 1); } static inline struct mbuf * -xennet_get_rx_mbuf(struct netfront_info *np, RING_IDX ri) +xn_get_rx_mbuf(struct netfront_rxq *rxq, RING_IDX ri) { - int i = xennet_rxidx(ri); + int i; struct mbuf *m; - m = np->rx_mbufs[i]; - np->rx_mbufs[i] = NULL; + i = xn_rxidx(ri); + m = rxq->mbufs[i]; + rxq->mbufs[i] = NULL; return (m); } static inline grant_ref_t -xennet_get_rx_ref(struct netfront_info *np, RING_IDX ri) +xn_get_rx_ref(struct netfront_rxq *rxq, RING_IDX ri) { - int i = xennet_rxidx(ri); - grant_ref_t ref = np->grant_rx_ref[i]; + int i = xn_rxidx(ri); + grant_ref_t ref = rxq->grant_ref[i]; + KASSERT(ref != GRANT_REF_INVALID, ("Invalid grant reference!\n")); - np->grant_rx_ref[i] = GRANT_REF_INVALID; - return ref; + rxq->grant_ref[i] = GRANT_REF_INVALID; + return (ref); } #define IPRINTK(fmt, args...) \ @@ -392,7 +420,7 @@ netfront_attach(device_t dev) int err; err = create_netdev(dev); - if (err) { + if (err != 0) { xenbus_dev_fatal(dev, err, "creating netdev"); return (err); } @@ -402,19 +430,29 @@ netfront_attach(device_t dev) OID_AUTO, "enable_lro", CTLFLAG_RW, &xn_enable_lro, 0, "Large Receive Offload"); + SYSCTL_ADD_ULONG(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "num_queues", CTLFLAG_RD, + &xn_num_queues, "Number of pairs of queues"); + return (0); } static int netfront_suspend(device_t dev) { - struct netfront_info *info = device_get_softc(dev); + struct netfront_info *np = device_get_softc(dev); + u_int i; - XN_RX_LOCK(info); - XN_TX_LOCK(info); - netfront_carrier_off(info); - XN_TX_UNLOCK(info); - XN_RX_UNLOCK(info); + for (i = 0; i < np->num_queues; i++) { + XN_RX_LOCK(&np->rxq[i]); + XN_TX_LOCK(&np->txq[i]); + } + netfront_carrier_off(np); + for (i = 0; i < np->num_queues; i++) { + XN_RX_UNLOCK(&np->rxq[i]); + XN_TX_UNLOCK(&np->txq[i]); + } return (0); } @@ -434,6 +472,61 @@ netfront_resume(device_t dev) return (0); } +static int +write_queue_xenstore_keys(device_t dev, + struct netfront_rxq *rxq, + struct netfront_txq *txq, + struct xs_transaction *xst, bool hierarchy) +{ + int err; + const char *message; + const char *node = xenbus_get_node(dev); + char *path; + size_t path_size; + + KASSERT(rxq->id == txq->id, ("Mismatch between RX and TX queue ids")); + /* Split event channel support is not yet there. */ + KASSERT(rxq->xen_intr_handle == txq->xen_intr_handle, + ("Split event channels are not supported")); + + if (hierarchy) { + path_size = strlen(node) + 10; + path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO); + snprintf(path, path_size, "%s/queue-%u", node, rxq->id); + } else { + path_size = strlen(node) + 1; + path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO); + snprintf(path, path_size, "%s", node); + } + + err = xs_printf(*xst, path, "tx-ring-ref","%u", txq->ring_ref); + if (err != 0) { + message = "writing tx ring-ref"; + goto error; + } + err = xs_printf(*xst, path, "rx-ring-ref","%u", rxq->ring_ref); + if (err != 0) { + message = "writing rx ring-ref"; + goto error; + } + err = xs_printf(*xst, path, "event-channel", "%u", + xen_intr_port(rxq->xen_intr_handle)); + if (err != 0) { + message = "writing event-channel"; + goto error; + } + + free(path, M_DEVBUF); + + return (0); + +error: + free(path, M_DEVBUF); + xenbus_dev_fatal(dev, err, "%s", message); + + return (err); +} + /* Common code used when first setting up, and when resuming. */ static int talk_to_backend(device_t dev, struct netfront_info *info) @@ -442,134 +535,427 @@ talk_to_backend(device_t dev, struct net struct xs_transaction xst; const char *node = xenbus_get_node(dev); int err; + unsigned long num_queues, max_queues = 0; + unsigned int i; err = xen_net_read_mac(dev, info->mac); - if (err) { + if (err != 0) { xenbus_dev_fatal(dev, err, "parsing %s/mac", node); goto out; } - /* Create shared ring, alloc event channel. */ - err = setup_device(dev, info); - if (err) + err = xs_scanf(XST_NIL, xenbus_get_otherend_path(info->xbdev), + "multi-queue-max-queues", NULL, "%lu", &max_queues); + if (err != 0) + max_queues = 1; + num_queues = xn_num_queues; + if (num_queues > max_queues) + num_queues = max_queues; + + err = setup_device(dev, info, num_queues); + if (err != 0) goto out; again: err = xs_transaction_start(&xst); - if (err) { + if (err != 0) { xenbus_dev_fatal(dev, err, "starting transaction"); - goto destroy_ring; + goto free; } - err = xs_printf(xst, node, "tx-ring-ref","%u", - info->tx_ring_ref); - if (err) { - message = "writing tx ring-ref"; - goto abort_transaction; - } - err = xs_printf(xst, node, "rx-ring-ref","%u", - info->rx_ring_ref); - if (err) { - message = "writing rx ring-ref"; - goto abort_transaction; - } - err = xs_printf(xst, node, - "event-channel", "%u", - xen_intr_port(info->xen_intr_handle)); - if (err) { - message = "writing event-channel"; - goto abort_transaction; + + if (info->num_queues == 1) { + err = write_queue_xenstore_keys(dev, &info->rxq[0], + &info->txq[0], &xst, false); + if (err != 0) + goto abort_transaction_no_def_error; + } else { + err = xs_printf(xst, node, "multi-queue-num-queues", + "%u", info->num_queues); + if (err != 0) { + message = "writing multi-queue-num-queues"; + goto abort_transaction; + } + + for (i = 0; i < info->num_queues; i++) { + err = write_queue_xenstore_keys(dev, &info->rxq[i], + &info->txq[i], &xst, true); + if (err != 0) + goto abort_transaction_no_def_error; + } } + err = xs_printf(xst, node, "request-rx-copy", "%u", 1); - if (err) { + if (err != 0) { message = "writing request-rx-copy"; goto abort_transaction; } err = xs_printf(xst, node, "feature-rx-notify", "%d", 1); - if (err) { + if (err != 0) { message = "writing feature-rx-notify"; goto abort_transaction; } err = xs_printf(xst, node, "feature-sg", "%d", 1); - if (err) { + if (err != 0) { message = "writing feature-sg"; goto abort_transaction; } err = xs_printf(xst, node, "feature-gso-tcpv4", "%d", 1); - if (err) { + if (err != 0) { message = "writing feature-gso-tcpv4"; goto abort_transaction; } err = xs_transaction_end(xst, 0); - if (err) { + if (err != 0) { if (err == EAGAIN) goto again; xenbus_dev_fatal(dev, err, "completing transaction"); - goto destroy_ring; + goto free; } return 0; abort_transaction: - xs_transaction_end(xst, 1); xenbus_dev_fatal(dev, err, "%s", message); - destroy_ring: + abort_transaction_no_def_error: + xs_transaction_end(xst, 1); + free: netif_free(info); out: - return err; + return (err); +} + +static void +xn_rxq_tq_intr(void *xrxq, int pending) +{ + struct netfront_rxq *rxq = xrxq; + + XN_RX_LOCK(rxq); + xn_rxeof(rxq); + XN_RX_UNLOCK(rxq); +} + +static void +xn_txq_start(struct netfront_txq *txq) +{ + struct netfront_info *np = txq->info; + struct ifnet *ifp = np->xn_ifp; + + XN_TX_LOCK_ASSERT(txq); + if (!drbr_empty(ifp, txq->br)) + xn_txq_mq_start_locked(txq, NULL); +} + +static void +xn_txq_tq_intr(void *xtxq, int pending) +{ + struct netfront_txq *txq = xtxq; + + XN_TX_LOCK(txq); + if (RING_HAS_UNCONSUMED_RESPONSES(&txq->ring)) + xn_txeof(txq); + xn_txq_start(txq); + XN_TX_UNLOCK(txq); +} + +static void +xn_txq_tq_deferred(void *xtxq, int pending) +{ + struct netfront_txq *txq = xtxq; + + XN_TX_LOCK(txq); + xn_txq_start(txq); + XN_TX_UNLOCK(txq); +} + +static void +disconnect_rxq(struct netfront_rxq *rxq) +{ + + xn_release_rx_bufs(rxq); + gnttab_free_grant_references(rxq->gref_head); + gnttab_end_foreign_access_ref(rxq->ring_ref); + /* + * No split event channel support at the moment, handle will + * be unbound in tx. So no need to call xen_intr_unbind here, + * but we do want to reset the handler to 0. + */ + rxq->xen_intr_handle = 0; +} + +static void +destroy_rxq(struct netfront_rxq *rxq) +{ + + free(rxq->ring.sring, M_DEVBUF); + taskqueue_drain_all(rxq->tq); + taskqueue_free(rxq->tq); +} + +static void +destroy_rxqs(struct netfront_info *np) +{ + int i; + + for (i = 0; i < np->num_queues; i++) + destroy_rxq(&np->rxq[i]); + + free(np->rxq, M_DEVBUF); + np->rxq = NULL; } static int -setup_device(device_t dev, struct netfront_info *info) +setup_rxqs(device_t dev, struct netfront_info *info, + unsigned long num_queues) { - netif_tx_sring_t *txs; + int q, i; + int error; netif_rx_sring_t *rxs; + struct netfront_rxq *rxq; + + info->rxq = malloc(sizeof(struct netfront_rxq) * num_queues, + M_DEVBUF, M_WAITOK|M_ZERO); + + for (q = 0; q < num_queues; q++) { + rxq = &info->rxq[q]; + + rxq->id = q; + rxq->info = info; + rxq->target = RX_MIN_TARGET; + rxq->ring_ref = GRANT_REF_INVALID; + rxq->ring.sring = NULL; + snprintf(rxq->name, XN_QUEUE_NAME_LEN, "xnrx_%u", q); + mtx_init(&rxq->lock, rxq->name, "netfront receive lock", + MTX_DEF); + + for (i = 0; i <= NET_RX_RING_SIZE; i++) { + rxq->mbufs[i] = NULL; + rxq->grant_ref[i] = GRANT_REF_INVALID; + } + + mbufq_init(&rxq->batch, INT_MAX); + + /* Start resources allocation */ + + if (gnttab_alloc_grant_references(RX_MAX_TARGET, + &rxq->gref_head) != 0) { + device_printf(dev, "allocating rx gref"); + error = ENOMEM; + goto fail; + } + + rxs = (netif_rx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, + M_WAITOK|M_ZERO); + SHARED_RING_INIT(rxs); + FRONT_RING_INIT(&rxq->ring, rxs, PAGE_SIZE); + + error = xenbus_grant_ring(dev, virt_to_mfn(rxs), + &rxq->ring_ref); + if (error != 0) { + device_printf(dev, "granting rx ring page"); + goto fail_grant_ring; + } + + TASK_INIT(&rxq->intrtask, 0, xn_rxq_tq_intr, rxq); + rxq->tq = taskqueue_create_fast(rxq->name, M_WAITOK, + taskqueue_thread_enqueue, &rxq->tq); + + error = taskqueue_start_threads(&rxq->tq, 1, PI_NET, + "%s rxq %d", device_get_nameunit(dev), rxq->id); + if (error != 0) { + device_printf(dev, "failed to start rx taskq %d\n", + rxq->id); + goto fail_start_thread; + } + } + + return (0); + +fail_start_thread: + gnttab_end_foreign_access_ref(rxq->ring_ref); + taskqueue_drain_all(rxq->tq); + taskqueue_free(rxq->tq); +fail_grant_ring: + gnttab_free_grant_references(rxq->gref_head); + free(rxq->ring.sring, M_DEVBUF); +fail: + for (; q >= 0; q--) { + disconnect_rxq(&info->rxq[q]); + destroy_rxq(&info->rxq[q]); + } + + free(info->rxq, M_DEVBUF); + return (error); +} + +static void +disconnect_txq(struct netfront_txq *txq) +{ + + xn_release_tx_bufs(txq); + gnttab_free_grant_references(txq->gref_head); + gnttab_end_foreign_access_ref(txq->ring_ref); + xen_intr_unbind(&txq->xen_intr_handle); +} + +static void +destroy_txq(struct netfront_txq *txq) +{ + + free(txq->ring.sring, M_DEVBUF); + buf_ring_free(txq->br, M_DEVBUF); + taskqueue_drain_all(txq->tq); + taskqueue_free(txq->tq); +} + +static void +destroy_txqs(struct netfront_info *np) +{ + int i; + + for (i = 0; i < np->num_queues; i++) + destroy_txq(&np->txq[i]); + + free(np->txq, M_DEVBUF); + np->txq = NULL; +} + +static int +setup_txqs(device_t dev, struct netfront_info *info, + unsigned long num_queues) +{ + int q, i; int error; + netif_tx_sring_t *txs; + struct netfront_txq *txq; - info->tx_ring_ref = GRANT_REF_INVALID; - info->rx_ring_ref = GRANT_REF_INVALID; - info->rx.sring = NULL; - info->tx.sring = NULL; - - txs = (netif_tx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO); - if (!txs) { - error = ENOMEM; - xenbus_dev_fatal(dev, error, "allocating tx ring page"); - goto fail; - } - SHARED_RING_INIT(txs); - FRONT_RING_INIT(&info->tx, txs, PAGE_SIZE); - error = xenbus_grant_ring(dev, virt_to_mfn(txs), &info->tx_ring_ref); - if (error) - goto fail; - - rxs = (netif_rx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO); - if (!rxs) { - error = ENOMEM; - xenbus_dev_fatal(dev, error, "allocating rx ring page"); - goto fail; - } - SHARED_RING_INIT(rxs); - FRONT_RING_INIT(&info->rx, rxs, PAGE_SIZE); - - error = xenbus_grant_ring(dev, virt_to_mfn(rxs), &info->rx_ring_ref); - if (error) - goto fail; - - error = xen_intr_alloc_and_bind_local_port(dev, - xenbus_get_otherend_id(dev), /*filter*/NULL, xn_intr, info, - INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY, &info->xen_intr_handle); - - if (error) { - xenbus_dev_fatal(dev, error, - "xen_intr_alloc_and_bind_local_port failed"); - goto fail; + info->txq = malloc(sizeof(struct netfront_txq) * num_queues, + M_DEVBUF, M_WAITOK|M_ZERO); + + for (q = 0; q < num_queues; q++) { + txq = &info->txq[q]; + + txq->id = q; + txq->info = info; + + txq->ring_ref = GRANT_REF_INVALID; + txq->ring.sring = NULL; + + snprintf(txq->name, XN_QUEUE_NAME_LEN, "xntx_%u", q); + + mtx_init(&txq->lock, txq->name, "netfront transmit lock", + MTX_DEF); + + for (i = 0; i <= NET_TX_RING_SIZE; i++) { + txq->mbufs[i] = (void *) ((u_long) i+1); + txq->grant_ref[i] = GRANT_REF_INVALID; + } + txq->mbufs[NET_TX_RING_SIZE] = (void *)0; + + /* Start resources allocation. */ + + if (gnttab_alloc_grant_references(NET_TX_RING_SIZE, + &txq->gref_head) != 0) { + device_printf(dev, "failed to allocate tx grant refs\n"); + error = ENOMEM; + goto fail; + } + + txs = (netif_tx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, + M_WAITOK|M_ZERO); + SHARED_RING_INIT(txs); + FRONT_RING_INIT(&txq->ring, txs, PAGE_SIZE); + + error = xenbus_grant_ring(dev, virt_to_mfn(txs), + &txq->ring_ref); + if (error != 0) { + device_printf(dev, "failed to grant tx ring\n"); + goto fail_grant_ring; + } + + txq->br = buf_ring_alloc(NET_TX_RING_SIZE, M_DEVBUF, + M_WAITOK, &txq->lock); + TASK_INIT(&txq->defrtask, 0, xn_txq_tq_deferred, txq); + TASK_INIT(&txq->intrtask, 0, xn_txq_tq_intr, txq); + + txq->tq = taskqueue_create_fast(txq->name, M_WAITOK, + taskqueue_thread_enqueue, &txq->tq); + + error = taskqueue_start_threads(&txq->tq, 1, PI_NET, + "%s txq %d", device_get_nameunit(dev), txq->id); + if (error != 0) { + device_printf(dev, "failed to start tx taskq %d\n", + txq->id); + goto fail_start_thread; + } + + error = xen_intr_alloc_and_bind_local_port(dev, + xenbus_get_otherend_id(dev), xn_intr, /* handler */ NULL, + &info->txq[q], + INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY, + &txq->xen_intr_handle); + + if (error != 0) { + device_printf(dev, "xen_intr_alloc_and_bind_local_port failed\n"); + goto fail_bind_port; + } } return (0); - fail: - netif_free(info); +fail_bind_port: + taskqueue_drain_all(txq->tq); +fail_start_thread: + gnttab_free_grant_references(txq->gref_head); + free(txq->ring.sring, M_DEVBUF); + gnttab_end_foreign_access_ref(txq->ring_ref); + buf_ring_free(txq->br, M_DEVBUF); + taskqueue_free(txq->tq); +fail_grant_ring: + gnttab_free_grant_references(txq->gref_head); + free(txq->ring.sring, M_DEVBUF); +fail: + for (; q >= 0; q--) { + disconnect_txq(&info->txq[q]); + destroy_txq(&info->txq[q]); + } + + free(info->txq, M_DEVBUF); + return (error); +} + +static int +setup_device(device_t dev, struct netfront_info *info, + unsigned long num_queues) +{ + int error; + int q; + + if (info->txq) + destroy_txqs(info); + + if (info->rxq) + destroy_rxqs(info); + + info->num_queues = 0; + + error = setup_rxqs(dev, info, num_queues); + if (error != 0) + goto out; + error = setup_txqs(dev, info, num_queues); + if (error != 0) + goto out; + + info->num_queues = num_queues; + + /* No split event channel at the moment. */ + for (q = 0; q < num_queues; q++) + info->rxq[q].xen_intr_handle = info->txq[q].xen_intr_handle; + + return (0); + +out: + KASSERT(error != 0, ("Error path taken without providing an error code")); return (error); } @@ -614,7 +1000,7 @@ netfront_backend_changed(device_t dev, X case XenbusStateInitWait: if (xenbus_get_state(dev) != XenbusStateInitialising) break; - if (network_connect(sc) != 0) + if (xn_connect(sc) != 0) break; xenbus_set_state(dev, XenbusStateConnected); break; @@ -629,42 +1015,6 @@ netfront_backend_changed(device_t dev, X } } -static void -xn_free_rx_ring(struct netfront_info *sc) -{ -#if 0 - int i; - - for (i = 0; i < NET_RX_RING_SIZE; i++) { - if (sc->xn_cdata.rx_mbufs[i] != NULL) { - m_freem(sc->rx_mbufs[i]); - sc->rx_mbufs[i] = NULL; - } - } - - sc->rx.rsp_cons = 0; - sc->xn_rx_if->req_prod = 0; - sc->xn_rx_if->event = sc->rx.rsp_cons ; -#endif -} - -static void -xn_free_tx_ring(struct netfront_info *sc) -{ -#if 0 - int i; - - for (i = 0; i < NET_TX_RING_SIZE; i++) { - if (sc->tx_mbufs[i] != NULL) { - m_freem(sc->tx_mbufs[i]); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201601201502.u0KF2h8p092321>