Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 20 Jan 2016 15:02:43 +0000 (UTC)
From:      =?UTF-8?Q?Roger_Pau_Monn=c3=a9?= <royger@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r294442 - head/sys/dev/xen/netfront
Message-ID:  <201601201502.u0KF2h8p092321@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: royger
Date: Wed Jan 20 15:02:43 2016
New Revision: 294442
URL: https://svnweb.freebsd.org/changeset/base/294442

Log:
  xen-netfront: add multiqueue support
  
  Add support for multiple TX and RX queue pairs. The default number of queues
  is set to 4, but can be easily changed from the sysctl node hw.xn.num_queues.
  
  Also heavily refactor netfront driver: break out a bunch of helper
  functions and different structures. Use threads to handle TX and RX.
  Remove some dead code and fix quite a few bugs as I go along.
  
  Submitted by:		Wei Liu <wei.liu2@citrix.com>
  Reviewed by:		royger
  Sponsored by:		Citrix Systems R&D
  Relnotes:		Yes
  Differential Revision:	https://reviews.freebsd.org/D4193

Modified:
  head/sys/dev/xen/netfront/netfront.c

Modified: head/sys/dev/xen/netfront/netfront.c
==============================================================================
--- head/sys/dev/xen/netfront/netfront.c	Wed Jan 20 14:49:16 2016	(r294441)
+++ head/sys/dev/xen/netfront/netfront.c	Wed Jan 20 15:02:43 2016	(r294442)
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2004-2006 Kip Macy
+ * Copyright (c) 2015 Wei Liu <wei.liu2@citrix.com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,15 +40,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/socket.h>
 #include <sys/sysctl.h>
+#include <sys/taskqueue.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_arp.h>
 #include <net/ethernet.h>
 #include <net/if_media.h>
-
 #include <net/bpf.h>
-
 #include <net/if_types.h>
 
 #include <netinet/in.h>
@@ -86,6 +86,12 @@ __FBSDID("$FreeBSD$");
 static int xn_enable_lro = 1;
 TUNABLE_INT("hw.xn.enable_lro", &xn_enable_lro);
 
+/*
+ * Number of pairs of queues.
+ */
+static unsigned long xn_num_queues = 4;
+TUNABLE_ULONG("hw.xn.num_queues", &xn_num_queues);
+
 /**
  * \brief The maximum allowed data fragments in a single transmit
  *        request.
@@ -100,149 +106,167 @@ TUNABLE_INT("hw.xn.enable_lro", &xn_enab
 
 #define net_ratelimit() 0
 
+struct netfront_rxq;
+struct netfront_txq;
 struct netfront_info;
 struct netfront_rx_info;
 
-static void xn_txeof(struct netfront_info *);
-static void xn_rxeof(struct netfront_info *);
-static void network_alloc_rx_buffers(struct netfront_info *);
-
-static void xn_tick_locked(struct netfront_info *);
-static void xn_tick(void *);
-
-static void xn_intr(void *);
+static void xn_txeof(struct netfront_txq *);
+static void xn_rxeof(struct netfront_rxq *);
+static void xn_alloc_rx_buffers(struct netfront_rxq *);
+
+static void xn_release_rx_bufs(struct netfront_rxq *);
+static void xn_release_tx_bufs(struct netfront_txq *);
+
+static void xn_rxq_intr(void *);
+static void xn_txq_intr(void *);
+static int  xn_intr(void *);
 static inline int xn_count_frags(struct mbuf *m);
-static int  xn_assemble_tx_request(struct netfront_info *sc,
-				   struct mbuf *m_head);
-static void xn_start_locked(struct ifnet *);
-static void xn_start(struct ifnet *);
-static int  xn_ioctl(struct ifnet *, u_long, caddr_t);
+static int xn_assemble_tx_request(struct netfront_txq *, struct mbuf *);
+static int xn_ioctl(struct ifnet *, u_long, caddr_t);
 static void xn_ifinit_locked(struct netfront_info *);
 static void xn_ifinit(void *);
 static void xn_stop(struct netfront_info *);
 static void xn_query_features(struct netfront_info *np);
-static int  xn_configure_features(struct netfront_info *np);
-#ifdef notyet
-static void xn_watchdog(struct ifnet *);
-#endif
-
-#ifdef notyet
-static void netfront_closing(device_t dev);
-#endif
+static int xn_configure_features(struct netfront_info *np);
 static void netif_free(struct netfront_info *info);
 static int netfront_detach(device_t dev);
 
+static int xn_txq_mq_start_locked(struct netfront_txq *, struct mbuf *);
+static int xn_txq_mq_start(struct ifnet *, struct mbuf *);
+
 static int talk_to_backend(device_t dev, struct netfront_info *info);
 static int create_netdev(device_t dev);
 static void netif_disconnect_backend(struct netfront_info *info);
-static int setup_device(device_t dev, struct netfront_info *info);
-static void free_ring(int *ref, void *ring_ptr_ref);
-
-static int  xn_ifmedia_upd(struct ifnet *ifp);
+static int setup_device(device_t dev, struct netfront_info *info,
+    unsigned long);
+static int xn_ifmedia_upd(struct ifnet *ifp);
 static void xn_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr);
 
-/* Xenolinux helper functions */
-int network_connect(struct netfront_info *);
+int xn_connect(struct netfront_info *);
 
-static void xn_free_rx_ring(struct netfront_info *);
-
-static void xn_free_tx_ring(struct netfront_info *);
-
-static int xennet_get_responses(struct netfront_info *np,
-	struct netfront_rx_info *rinfo, RING_IDX rp, RING_IDX *cons,
-	struct mbuf **list);
+static int xn_get_responses(struct netfront_rxq *,
+    struct netfront_rx_info *, RING_IDX, RING_IDX *,
+    struct mbuf **);
 
 #define virt_to_mfn(x) (vtophys(x) >> PAGE_SHIFT)
 
 #define INVALID_P2M_ENTRY (~0UL)
 
-/*
- * Mbuf pointers. We need these to keep track of the virtual addresses
- * of our mbuf chains since we can only convert from virtual to physical,
- * not the other way around.  The size must track the free index arrays.
- */
-struct xn_chain_data {
-	struct mbuf    *xn_tx_chain[NET_TX_RING_SIZE+1];
-	int		xn_tx_chain_cnt;
-	struct mbuf    *xn_rx_chain[NET_RX_RING_SIZE+1];
+struct xn_rx_stats
+{
+	u_long	rx_packets;	/* total packets received	*/
+	u_long	rx_bytes;	/* total bytes received 	*/
+	u_long	rx_errors;	/* bad packets received		*/
 };
 
-struct netfront_stats
+struct xn_tx_stats
 {
-	u_long	rx_packets;		/* total packets received	*/
-	u_long	tx_packets;		/* total packets transmitted	*/
-	u_long	rx_bytes;		/* total bytes received 	*/
-	u_long	tx_bytes;		/* total bytes transmitted	*/
-	u_long	rx_errors;		/* bad packets received		*/
-	u_long	tx_errors;		/* packet transmit problems	*/
+	u_long	tx_packets;	/* total packets transmitted	*/
+	u_long	tx_bytes;	/* total bytes transmitted	*/
+	u_long	tx_errors;	/* packet transmit problems	*/
+};
+
+#define XN_QUEUE_NAME_LEN  8	/* xn{t,r}x_%u, allow for two digits */
+struct netfront_rxq {
+	struct netfront_info 	*info;
+	u_int			id;
+	char			name[XN_QUEUE_NAME_LEN];
+	struct mtx		lock;
+
+	int			ring_ref;
+	netif_rx_front_ring_t 	ring;
+	xen_intr_handle_t	xen_intr_handle;
+
+	grant_ref_t 		gref_head;
+	grant_ref_t 		grant_ref[NET_TX_RING_SIZE + 1];
+
+	struct mbuf		*mbufs[NET_RX_RING_SIZE + 1];
+	struct mbufq		batch;		/* batch queue */
+	int                     target;
+
+	xen_pfn_t		pfn_array[NET_RX_RING_SIZE];
+
+	struct lro_ctrl		lro;
+
+	struct taskqueue 	*tq;
+	struct task		intrtask;
+
+	struct xn_rx_stats	stats;
+};
+
+struct netfront_txq {
+	struct netfront_info 	*info;
+	u_int 			id;
+	char			name[XN_QUEUE_NAME_LEN];
+	struct mtx		lock;
+
+	int			ring_ref;
+	netif_tx_front_ring_t	ring;
+	xen_intr_handle_t 	xen_intr_handle;
+
+	grant_ref_t		gref_head;
+	grant_ref_t		grant_ref[NET_TX_RING_SIZE + 1];
+
+	struct mbuf		*mbufs[NET_TX_RING_SIZE + 1];
+	int			mbufs_cnt;
+	struct buf_ring		*br;
+
+	struct taskqueue 	*tq;
+	struct task       	intrtask;
+	struct task       	defrtask;
+
+	bool			full;
+
+	struct xn_tx_stats	stats;
 };
 
 struct netfront_info {
-	struct ifnet *xn_ifp;
-	struct lro_ctrl xn_lro;
+	struct ifnet 		*xn_ifp;
 
-	struct netfront_stats stats;
-	u_int tx_full;
+	struct mtx   		sc_lock;
 
-	netif_tx_front_ring_t tx;
-	netif_rx_front_ring_t rx;
+	u_int  num_queues;
+	struct netfront_rxq 	*rxq;
+	struct netfront_txq 	*txq;
 
-	struct mtx   tx_lock;
-	struct mtx   rx_lock;
-	struct mtx   sc_lock;
-
-	xen_intr_handle_t xen_intr_handle;
-	u_int carrier;
-	u_int maxfrags;
+	u_int			carrier;
+	u_int			maxfrags;
 
 	/* Receive-ring batched refills. */
 #define RX_MIN_TARGET 32
 #define RX_MAX_TARGET NET_RX_RING_SIZE
-	int rx_min_target;
-	int rx_max_target;
-	int rx_target;
-
-	grant_ref_t gref_tx_head;
-	grant_ref_t grant_tx_ref[NET_TX_RING_SIZE + 1];
-	grant_ref_t gref_rx_head;
-	grant_ref_t grant_rx_ref[NET_TX_RING_SIZE + 1];
+	int			rx_min_target;
+	int			rx_max_target;
 
 	device_t		xbdev;
-	int			tx_ring_ref;
-	int			rx_ring_ref;
 	uint8_t			mac[ETHER_ADDR_LEN];
-	struct xn_chain_data	xn_cdata;	/* mbufs */
-	struct mbufq		xn_rx_batch;	/* batch queue */
 
 	int			xn_if_flags;
-	struct callout	        xn_stat_ch;
 
-	xen_pfn_t		rx_pfn_array[NET_RX_RING_SIZE];
 	struct ifmedia		sc_media;
 
 	bool			xn_resume;
 };
 
-#define rx_mbufs xn_cdata.xn_rx_chain
-#define tx_mbufs xn_cdata.xn_tx_chain
+struct netfront_rx_info {
+	struct netif_rx_response rx;
+	struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
+};
 
-#define XN_RX_LOCK(_sc)           mtx_lock(&(_sc)->rx_lock)
-#define XN_RX_UNLOCK(_sc)         mtx_unlock(&(_sc)->rx_lock)
+#define XN_RX_LOCK(_q)         mtx_lock(&(_q)->lock)
+#define XN_RX_UNLOCK(_q)       mtx_unlock(&(_q)->lock)
 
-#define XN_TX_LOCK(_sc)           mtx_lock(&(_sc)->tx_lock)
-#define XN_TX_UNLOCK(_sc)         mtx_unlock(&(_sc)->tx_lock)
+#define XN_TX_LOCK(_q)         mtx_lock(&(_q)->lock)
+#define XN_TX_TRYLOCK(_q)      mtx_trylock(&(_q)->lock)
+#define XN_TX_UNLOCK(_q)       mtx_unlock(&(_q)->lock)
 
 #define XN_LOCK(_sc)           mtx_lock(&(_sc)->sc_lock);
 #define XN_UNLOCK(_sc)         mtx_unlock(&(_sc)->sc_lock);
 
 #define XN_LOCK_ASSERT(_sc)    mtx_assert(&(_sc)->sc_lock, MA_OWNED);
-#define XN_RX_LOCK_ASSERT(_sc)    mtx_assert(&(_sc)->rx_lock, MA_OWNED);
-#define XN_TX_LOCK_ASSERT(_sc)    mtx_assert(&(_sc)->tx_lock, MA_OWNED);
-
-struct netfront_rx_info {
-	struct netif_rx_response rx;
-	struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
-};
+#define XN_RX_LOCK_ASSERT(_q)  mtx_assert(&(_q)->lock, MA_OWNED);
+#define XN_TX_LOCK_ASSERT(_q)  mtx_assert(&(_q)->lock, MA_OWNED);
 
 #define netfront_carrier_on(netif)	((netif)->carrier = 1)
 #define netfront_carrier_off(netif)	((netif)->carrier = 0)
@@ -253,6 +277,7 @@ struct netfront_rx_info {
 static inline void
 add_id_to_freelist(struct mbuf **list, uintptr_t id)
 {
+
 	KASSERT(id != 0,
 		("%s: the head item (0) must always be free.", __func__));
 	list[id] = list[0];
@@ -272,30 +297,33 @@ get_id_from_freelist(struct mbuf **list)
 }
 
 static inline int
-xennet_rxidx(RING_IDX idx)
+xn_rxidx(RING_IDX idx)
 {
+
 	return idx & (NET_RX_RING_SIZE - 1);
 }
 
 static inline struct mbuf *
-xennet_get_rx_mbuf(struct netfront_info *np, RING_IDX ri)
+xn_get_rx_mbuf(struct netfront_rxq *rxq, RING_IDX ri)
 {
-	int i = xennet_rxidx(ri);
+	int i;
 	struct mbuf *m;
 
-	m = np->rx_mbufs[i];
-	np->rx_mbufs[i] = NULL;
+	i = xn_rxidx(ri);
+	m = rxq->mbufs[i];
+	rxq->mbufs[i] = NULL;
 	return (m);
 }
 
 static inline grant_ref_t
-xennet_get_rx_ref(struct netfront_info *np, RING_IDX ri)
+xn_get_rx_ref(struct netfront_rxq *rxq, RING_IDX ri)
 {
-	int i = xennet_rxidx(ri);
-	grant_ref_t ref = np->grant_rx_ref[i];
+	int i = xn_rxidx(ri);
+	grant_ref_t ref = rxq->grant_ref[i];
+
 	KASSERT(ref != GRANT_REF_INVALID, ("Invalid grant reference!\n"));
-	np->grant_rx_ref[i] = GRANT_REF_INVALID;
-	return ref;
+	rxq->grant_ref[i] = GRANT_REF_INVALID;
+	return (ref);
 }
 
 #define IPRINTK(fmt, args...) \
@@ -392,7 +420,7 @@ netfront_attach(device_t dev)
 	int err;
 
 	err = create_netdev(dev);
-	if (err) {
+	if (err != 0) {
 		xenbus_dev_fatal(dev, err, "creating netdev");
 		return (err);
 	}
@@ -402,19 +430,29 @@ netfront_attach(device_t dev)
 	    OID_AUTO, "enable_lro", CTLFLAG_RW,
 	    &xn_enable_lro, 0, "Large Receive Offload");
 
+	SYSCTL_ADD_ULONG(device_get_sysctl_ctx(dev),
+	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
+	    OID_AUTO, "num_queues", CTLFLAG_RD,
+	    &xn_num_queues, "Number of pairs of queues");
+
 	return (0);
 }
 
 static int
 netfront_suspend(device_t dev)
 {
-	struct netfront_info *info = device_get_softc(dev);
+	struct netfront_info *np = device_get_softc(dev);
+	u_int i;
 
-	XN_RX_LOCK(info);
-	XN_TX_LOCK(info);
-	netfront_carrier_off(info);
-	XN_TX_UNLOCK(info);
-	XN_RX_UNLOCK(info);
+	for (i = 0; i < np->num_queues; i++) {
+		XN_RX_LOCK(&np->rxq[i]);
+		XN_TX_LOCK(&np->txq[i]);
+	}
+	netfront_carrier_off(np);
+	for (i = 0; i < np->num_queues; i++) {
+		XN_RX_UNLOCK(&np->rxq[i]);
+		XN_TX_UNLOCK(&np->txq[i]);
+	}
 	return (0);
 }
 
@@ -434,6 +472,61 @@ netfront_resume(device_t dev)
 	return (0);
 }
 
+static int
+write_queue_xenstore_keys(device_t dev,
+    struct netfront_rxq *rxq,
+    struct netfront_txq *txq,
+    struct xs_transaction *xst, bool hierarchy)
+{
+	int err;
+	const char *message;
+	const char *node = xenbus_get_node(dev);
+	char *path;
+	size_t path_size;
+
+	KASSERT(rxq->id == txq->id, ("Mismatch between RX and TX queue ids"));
+	/* Split event channel support is not yet there. */
+	KASSERT(rxq->xen_intr_handle == txq->xen_intr_handle,
+	    ("Split event channels are not supported"));
+
+	if (hierarchy) {
+		path_size = strlen(node) + 10;
+		path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO);
+		snprintf(path, path_size, "%s/queue-%u", node, rxq->id);
+	} else {
+		path_size = strlen(node) + 1;
+		path = malloc(path_size, M_DEVBUF, M_WAITOK|M_ZERO);
+		snprintf(path, path_size, "%s", node);
+	}
+
+	err = xs_printf(*xst, path, "tx-ring-ref","%u", txq->ring_ref);
+	if (err != 0) {
+		message = "writing tx ring-ref";
+		goto error;
+	}
+	err = xs_printf(*xst, path, "rx-ring-ref","%u", rxq->ring_ref);
+	if (err != 0) {
+		message = "writing rx ring-ref";
+		goto error;
+	}
+	err = xs_printf(*xst, path, "event-channel", "%u",
+	    xen_intr_port(rxq->xen_intr_handle));
+	if (err != 0) {
+		message = "writing event-channel";
+		goto error;
+	}
+
+	free(path, M_DEVBUF);
+
+	return (0);
+
+error:
+	free(path, M_DEVBUF);
+	xenbus_dev_fatal(dev, err, "%s", message);
+
+	return (err);
+}
+
 /* Common code used when first setting up, and when resuming. */
 static int
 talk_to_backend(device_t dev, struct netfront_info *info)
@@ -442,134 +535,427 @@ talk_to_backend(device_t dev, struct net
 	struct xs_transaction xst;
 	const char *node = xenbus_get_node(dev);
 	int err;
+	unsigned long num_queues, max_queues = 0;
+	unsigned int i;
 
 	err = xen_net_read_mac(dev, info->mac);
-	if (err) {
+	if (err != 0) {
 		xenbus_dev_fatal(dev, err, "parsing %s/mac", node);
 		goto out;
 	}
 
-	/* Create shared ring, alloc event channel. */
-	err = setup_device(dev, info);
-	if (err)
+	err = xs_scanf(XST_NIL, xenbus_get_otherend_path(info->xbdev),
+	    "multi-queue-max-queues", NULL, "%lu", &max_queues);
+	if (err != 0)
+		max_queues = 1;
+	num_queues = xn_num_queues;
+	if (num_queues > max_queues)
+		num_queues = max_queues;
+
+	err = setup_device(dev, info, num_queues);
+	if (err != 0)
 		goto out;
 
  again:
 	err = xs_transaction_start(&xst);
-	if (err) {
+	if (err != 0) {
 		xenbus_dev_fatal(dev, err, "starting transaction");
-		goto destroy_ring;
+		goto free;
 	}
-	err = xs_printf(xst, node, "tx-ring-ref","%u",
-			info->tx_ring_ref);
-	if (err) {
-		message = "writing tx ring-ref";
-		goto abort_transaction;
-	}
-	err = xs_printf(xst, node, "rx-ring-ref","%u",
-			info->rx_ring_ref);
-	if (err) {
-		message = "writing rx ring-ref";
-		goto abort_transaction;
-	}
-	err = xs_printf(xst, node,
-			"event-channel", "%u",
-			xen_intr_port(info->xen_intr_handle));
-	if (err) {
-		message = "writing event-channel";
-		goto abort_transaction;
+
+	if (info->num_queues == 1) {
+		err = write_queue_xenstore_keys(dev, &info->rxq[0],
+		    &info->txq[0], &xst, false);
+		if (err != 0)
+			goto abort_transaction_no_def_error;
+	} else {
+		err = xs_printf(xst, node, "multi-queue-num-queues",
+		    "%u", info->num_queues);
+		if (err != 0) {
+			message = "writing multi-queue-num-queues";
+			goto abort_transaction;
+		}
+
+		for (i = 0; i < info->num_queues; i++) {
+			err = write_queue_xenstore_keys(dev, &info->rxq[i],
+			    &info->txq[i], &xst, true);
+			if (err != 0)
+				goto abort_transaction_no_def_error;
+		}
 	}
+
 	err = xs_printf(xst, node, "request-rx-copy", "%u", 1);
-	if (err) {
+	if (err != 0) {
 		message = "writing request-rx-copy";
 		goto abort_transaction;
 	}
 	err = xs_printf(xst, node, "feature-rx-notify", "%d", 1);
-	if (err) {
+	if (err != 0) {
 		message = "writing feature-rx-notify";
 		goto abort_transaction;
 	}
 	err = xs_printf(xst, node, "feature-sg", "%d", 1);
-	if (err) {
+	if (err != 0) {
 		message = "writing feature-sg";
 		goto abort_transaction;
 	}
 	err = xs_printf(xst, node, "feature-gso-tcpv4", "%d", 1);
-	if (err) {
+	if (err != 0) {
 		message = "writing feature-gso-tcpv4";
 		goto abort_transaction;
 	}
 
 	err = xs_transaction_end(xst, 0);
-	if (err) {
+	if (err != 0) {
 		if (err == EAGAIN)
 			goto again;
 		xenbus_dev_fatal(dev, err, "completing transaction");
-		goto destroy_ring;
+		goto free;
 	}
 
 	return 0;
 
  abort_transaction:
-	xs_transaction_end(xst, 1);
 	xenbus_dev_fatal(dev, err, "%s", message);
- destroy_ring:
+ abort_transaction_no_def_error:
+	xs_transaction_end(xst, 1);
+ free:
 	netif_free(info);
  out:
-	return err;
+	return (err);
+}
+
+static void
+xn_rxq_tq_intr(void *xrxq, int pending)
+{
+	struct netfront_rxq *rxq = xrxq;
+
+	XN_RX_LOCK(rxq);
+	xn_rxeof(rxq);
+	XN_RX_UNLOCK(rxq);
+}
+
+static void
+xn_txq_start(struct netfront_txq *txq)
+{
+	struct netfront_info *np = txq->info;
+	struct ifnet *ifp = np->xn_ifp;
+
+	XN_TX_LOCK_ASSERT(txq);
+	if (!drbr_empty(ifp, txq->br))
+		xn_txq_mq_start_locked(txq, NULL);
+}
+
+static void
+xn_txq_tq_intr(void *xtxq, int pending)
+{
+	struct netfront_txq *txq = xtxq;
+
+	XN_TX_LOCK(txq);
+	if (RING_HAS_UNCONSUMED_RESPONSES(&txq->ring))
+		xn_txeof(txq);
+	xn_txq_start(txq);
+	XN_TX_UNLOCK(txq);
+}
+
+static void
+xn_txq_tq_deferred(void *xtxq, int pending)
+{
+	struct netfront_txq *txq = xtxq;
+
+	XN_TX_LOCK(txq);
+	xn_txq_start(txq);
+	XN_TX_UNLOCK(txq);
+}
+
+static void
+disconnect_rxq(struct netfront_rxq *rxq)
+{
+
+	xn_release_rx_bufs(rxq);
+	gnttab_free_grant_references(rxq->gref_head);
+	gnttab_end_foreign_access_ref(rxq->ring_ref);
+	/*
+	 * No split event channel support at the moment, handle will
+	 * be unbound in tx. So no need to call xen_intr_unbind here,
+	 * but we do want to reset the handler to 0.
+	 */
+	rxq->xen_intr_handle = 0;
+}
+
+static void
+destroy_rxq(struct netfront_rxq *rxq)
+{
+
+	free(rxq->ring.sring, M_DEVBUF);
+	taskqueue_drain_all(rxq->tq);
+	taskqueue_free(rxq->tq);
+}
+
+static void
+destroy_rxqs(struct netfront_info *np)
+{
+	int i;
+
+	for (i = 0; i < np->num_queues; i++)
+		destroy_rxq(&np->rxq[i]);
+
+	free(np->rxq, M_DEVBUF);
+	np->rxq = NULL;
 }
 
 static int
-setup_device(device_t dev, struct netfront_info *info)
+setup_rxqs(device_t dev, struct netfront_info *info,
+	   unsigned long num_queues)
 {
-	netif_tx_sring_t *txs;
+	int q, i;
+	int error;
 	netif_rx_sring_t *rxs;
+	struct netfront_rxq *rxq;
+
+	info->rxq = malloc(sizeof(struct netfront_rxq) * num_queues,
+	    M_DEVBUF, M_WAITOK|M_ZERO);
+
+	for (q = 0; q < num_queues; q++) {
+		rxq = &info->rxq[q];
+
+		rxq->id = q;
+		rxq->info = info;
+		rxq->target = RX_MIN_TARGET;
+		rxq->ring_ref = GRANT_REF_INVALID;
+		rxq->ring.sring = NULL;
+		snprintf(rxq->name, XN_QUEUE_NAME_LEN, "xnrx_%u", q);
+		mtx_init(&rxq->lock, rxq->name, "netfront receive lock",
+		    MTX_DEF);
+
+		for (i = 0; i <= NET_RX_RING_SIZE; i++) {
+			rxq->mbufs[i] = NULL;
+			rxq->grant_ref[i] = GRANT_REF_INVALID;
+		}
+
+		mbufq_init(&rxq->batch, INT_MAX);
+
+		/* Start resources allocation */
+
+		if (gnttab_alloc_grant_references(RX_MAX_TARGET,
+		    &rxq->gref_head) != 0) {
+			device_printf(dev, "allocating rx gref");
+			error = ENOMEM;
+			goto fail;
+		}
+
+		rxs = (netif_rx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF,
+		    M_WAITOK|M_ZERO);
+		SHARED_RING_INIT(rxs);
+		FRONT_RING_INIT(&rxq->ring, rxs, PAGE_SIZE);
+
+		error = xenbus_grant_ring(dev, virt_to_mfn(rxs),
+		    &rxq->ring_ref);
+		if (error != 0) {
+			device_printf(dev, "granting rx ring page");
+			goto fail_grant_ring;
+		}
+
+		TASK_INIT(&rxq->intrtask, 0, xn_rxq_tq_intr, rxq);
+		rxq->tq = taskqueue_create_fast(rxq->name, M_WAITOK,
+		    taskqueue_thread_enqueue, &rxq->tq);
+
+		error = taskqueue_start_threads(&rxq->tq, 1, PI_NET,
+		    "%s rxq %d", device_get_nameunit(dev), rxq->id);
+		if (error != 0) {
+			device_printf(dev, "failed to start rx taskq %d\n",
+			    rxq->id);
+			goto fail_start_thread;
+		}
+	}
+
+	return (0);
+
+fail_start_thread:
+	gnttab_end_foreign_access_ref(rxq->ring_ref);
+	taskqueue_drain_all(rxq->tq);
+	taskqueue_free(rxq->tq);
+fail_grant_ring:
+	gnttab_free_grant_references(rxq->gref_head);
+	free(rxq->ring.sring, M_DEVBUF);
+fail:
+	for (; q >= 0; q--) {
+		disconnect_rxq(&info->rxq[q]);
+		destroy_rxq(&info->rxq[q]);
+	}
+
+	free(info->rxq, M_DEVBUF);
+	return (error);
+}
+
+static void
+disconnect_txq(struct netfront_txq *txq)
+{
+
+	xn_release_tx_bufs(txq);
+	gnttab_free_grant_references(txq->gref_head);
+	gnttab_end_foreign_access_ref(txq->ring_ref);
+	xen_intr_unbind(&txq->xen_intr_handle);
+}
+
+static void
+destroy_txq(struct netfront_txq *txq)
+{
+
+	free(txq->ring.sring, M_DEVBUF);
+	buf_ring_free(txq->br, M_DEVBUF);
+	taskqueue_drain_all(txq->tq);
+	taskqueue_free(txq->tq);
+}
+
+static void
+destroy_txqs(struct netfront_info *np)
+{
+	int i;
+
+	for (i = 0; i < np->num_queues; i++)
+		destroy_txq(&np->txq[i]);
+
+	free(np->txq, M_DEVBUF);
+	np->txq = NULL;
+}
+
+static int
+setup_txqs(device_t dev, struct netfront_info *info,
+	   unsigned long num_queues)
+{
+	int q, i;
 	int error;
+	netif_tx_sring_t *txs;
+	struct netfront_txq *txq;
 
-	info->tx_ring_ref = GRANT_REF_INVALID;
-	info->rx_ring_ref = GRANT_REF_INVALID;
-	info->rx.sring = NULL;
-	info->tx.sring = NULL;
-
-	txs = (netif_tx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO);
-	if (!txs) {
-		error = ENOMEM;
-		xenbus_dev_fatal(dev, error, "allocating tx ring page");
-		goto fail;
-	}
-	SHARED_RING_INIT(txs);
-	FRONT_RING_INIT(&info->tx, txs, PAGE_SIZE);
-	error = xenbus_grant_ring(dev, virt_to_mfn(txs), &info->tx_ring_ref);
-	if (error)
-		goto fail;
-
-	rxs = (netif_rx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO);
-	if (!rxs) {
-		error = ENOMEM;
-		xenbus_dev_fatal(dev, error, "allocating rx ring page");
-		goto fail;
-	}
-	SHARED_RING_INIT(rxs);
-	FRONT_RING_INIT(&info->rx, rxs, PAGE_SIZE);
-
-	error = xenbus_grant_ring(dev, virt_to_mfn(rxs), &info->rx_ring_ref);
-	if (error)
-		goto fail;
-
-	error = xen_intr_alloc_and_bind_local_port(dev,
-	    xenbus_get_otherend_id(dev), /*filter*/NULL, xn_intr, info,
-	    INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY, &info->xen_intr_handle);
-
-	if (error) {
-		xenbus_dev_fatal(dev, error,
-				 "xen_intr_alloc_and_bind_local_port failed");
-		goto fail;
+	info->txq = malloc(sizeof(struct netfront_txq) * num_queues,
+	    M_DEVBUF, M_WAITOK|M_ZERO);
+
+	for (q = 0; q < num_queues; q++) {
+		txq = &info->txq[q];
+
+		txq->id = q;
+		txq->info = info;
+
+		txq->ring_ref = GRANT_REF_INVALID;
+		txq->ring.sring = NULL;
+
+		snprintf(txq->name, XN_QUEUE_NAME_LEN, "xntx_%u", q);
+
+		mtx_init(&txq->lock, txq->name, "netfront transmit lock",
+		    MTX_DEF);
+
+		for (i = 0; i <= NET_TX_RING_SIZE; i++) {
+			txq->mbufs[i] = (void *) ((u_long) i+1);
+			txq->grant_ref[i] = GRANT_REF_INVALID;
+		}
+		txq->mbufs[NET_TX_RING_SIZE] = (void *)0;
+
+		/* Start resources allocation. */
+
+		if (gnttab_alloc_grant_references(NET_TX_RING_SIZE,
+		    &txq->gref_head) != 0) {
+			device_printf(dev, "failed to allocate tx grant refs\n");
+			error = ENOMEM;
+			goto fail;
+		}
+
+		txs = (netif_tx_sring_t *)malloc(PAGE_SIZE, M_DEVBUF,
+		    M_WAITOK|M_ZERO);
+		SHARED_RING_INIT(txs);
+		FRONT_RING_INIT(&txq->ring, txs, PAGE_SIZE);
+
+		error = xenbus_grant_ring(dev, virt_to_mfn(txs),
+		    &txq->ring_ref);
+		if (error != 0) {
+			device_printf(dev, "failed to grant tx ring\n");
+			goto fail_grant_ring;
+		}
+
+		txq->br = buf_ring_alloc(NET_TX_RING_SIZE, M_DEVBUF,
+		    M_WAITOK, &txq->lock);
+		TASK_INIT(&txq->defrtask, 0, xn_txq_tq_deferred, txq);
+		TASK_INIT(&txq->intrtask, 0, xn_txq_tq_intr, txq);
+
+		txq->tq = taskqueue_create_fast(txq->name, M_WAITOK,
+		    taskqueue_thread_enqueue, &txq->tq);
+
+		error = taskqueue_start_threads(&txq->tq, 1, PI_NET,
+		    "%s txq %d", device_get_nameunit(dev), txq->id);
+		if (error != 0) {
+			device_printf(dev, "failed to start tx taskq %d\n",
+			    txq->id);
+			goto fail_start_thread;
+		}
+
+		error = xen_intr_alloc_and_bind_local_port(dev,
+			    xenbus_get_otherend_id(dev), xn_intr, /* handler */ NULL,
+			    &info->txq[q],
+			    INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY,
+			    &txq->xen_intr_handle);
+
+		if (error != 0) {
+			device_printf(dev, "xen_intr_alloc_and_bind_local_port failed\n");
+			goto fail_bind_port;
+		}
 	}
 
 	return (0);
 
- fail:
-	netif_free(info);
+fail_bind_port:
+	taskqueue_drain_all(txq->tq);
+fail_start_thread:
+	gnttab_free_grant_references(txq->gref_head);
+	free(txq->ring.sring, M_DEVBUF);
+	gnttab_end_foreign_access_ref(txq->ring_ref);
+	buf_ring_free(txq->br, M_DEVBUF);
+	taskqueue_free(txq->tq);
+fail_grant_ring:
+	gnttab_free_grant_references(txq->gref_head);
+	free(txq->ring.sring, M_DEVBUF);
+fail:
+	for (; q >= 0; q--) {
+		disconnect_txq(&info->txq[q]);
+		destroy_txq(&info->txq[q]);
+	}
+
+	free(info->txq, M_DEVBUF);
+	return (error);
+}
+
+static int
+setup_device(device_t dev, struct netfront_info *info,
+    unsigned long num_queues)
+{
+	int error;
+	int q;
+
+	if (info->txq)
+		destroy_txqs(info);
+
+	if (info->rxq)
+		destroy_rxqs(info);
+
+	info->num_queues = 0;
+
+	error = setup_rxqs(dev, info, num_queues);
+	if (error != 0)
+		goto out;
+	error = setup_txqs(dev, info, num_queues);
+	if (error != 0)
+		goto out;
+
+	info->num_queues = num_queues;
+
+	/* No split event channel at the moment. */
+	for (q = 0; q < num_queues; q++)
+		info->rxq[q].xen_intr_handle = info->txq[q].xen_intr_handle;
+
+	return (0);
+
+out:
+	KASSERT(error != 0, ("Error path taken without providing an error code"));
 	return (error);
 }
 
@@ -614,7 +1000,7 @@ netfront_backend_changed(device_t dev, X
 	case XenbusStateInitWait:
 		if (xenbus_get_state(dev) != XenbusStateInitialising)
 			break;
-		if (network_connect(sc) != 0)
+		if (xn_connect(sc) != 0)
 			break;
 		xenbus_set_state(dev, XenbusStateConnected);
 		break;
@@ -629,42 +1015,6 @@ netfront_backend_changed(device_t dev, X
 	}
 }
 
-static void
-xn_free_rx_ring(struct netfront_info *sc)
-{
-#if 0
-	int i;
-
-	for (i = 0; i < NET_RX_RING_SIZE; i++) {
-		if (sc->xn_cdata.rx_mbufs[i] != NULL) {
-			m_freem(sc->rx_mbufs[i]);
-			sc->rx_mbufs[i] = NULL;
-		}
-	}
-
-	sc->rx.rsp_cons = 0;
-	sc->xn_rx_if->req_prod = 0;
-	sc->xn_rx_if->event = sc->rx.rsp_cons ;
-#endif
-}
-
-static void
-xn_free_tx_ring(struct netfront_info *sc)
-{
-#if 0
-	int i;
-
-	for (i = 0; i < NET_TX_RING_SIZE; i++) {
-		if (sc->tx_mbufs[i] != NULL) {
-			m_freem(sc->tx_mbufs[i]);

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201601201502.u0KF2h8p092321>