Skip site navigation (1)Skip section navigation (2)
Date:      Mon, 30 May 2011 21:34:44 +0000 (UTC)
From:      Navdeep Parhar <np@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r222510 - head/sys/dev/cxgbe
Message-ID:  <201105302134.p4ULYigj094113@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: np
Date: Mon May 30 21:34:44 2011
New Revision: 222510
URL: http://svn.freebsd.org/changeset/base/222510

Log:
  - Specialized ingress queues that take interrupts for other ingress
    queues.  Try to have a set of these per port when possible, fall back
    to sharing a common pool between all ports otherwise.
  
  - One control queue per port (used to be one per hardware channel).
  
  - t4_eth_rx now handles Ethernet rx only.
  
  - sysctls to display pidx/cidx for some queues.
  
  MFC after:	1 week

Modified:
  head/sys/dev/cxgbe/adapter.h
  head/sys/dev/cxgbe/t4_main.c
  head/sys/dev/cxgbe/t4_sge.c

Modified: head/sys/dev/cxgbe/adapter.h
==============================================================================
--- head/sys/dev/cxgbe/adapter.h	Mon May 30 21:07:26 2011	(r222509)
+++ head/sys/dev/cxgbe/adapter.h	Mon May 30 21:34:44 2011	(r222510)
@@ -110,6 +110,9 @@ enum {
 	FW_IQ_QSIZE = 256,
 	FW_IQ_ESIZE = 64,	/* At least 64 mandated by the firmware spec */
 
+	INTR_IQ_QSIZE = 64,
+	INTR_IQ_ESIZE = 64,	/* Handles some CPLs too, do not reduce */
+
 	CTRL_EQ_QSIZE = 128,
 	CTRL_EQ_ESIZE = 64,
 
@@ -141,7 +144,7 @@ enum {
 	/* adapter flags */
 	FULL_INIT_DONE	= (1 << 0),
 	FW_OK		= (1 << 1),
-	INTR_FWD	= (1 << 2),
+	INTR_SHARED	= (1 << 2),	/* one set of intrq's for all ports */
 
 	CXGBE_BUSY	= (1 << 9),
 
@@ -384,12 +387,10 @@ struct sge_ctrlq {
 
 	/* stats for common events first */
 
-	uint64_t total_wrs;	/* # of work requests sent down this queue */
 
 	/* stats for not-that-common events */
 
 	uint32_t no_desc;	/* out of hardware descriptors */
-	uint32_t too_long;	/* WR longer than hardware max */
 } __aligned(CACHE_LINE_SIZE);
 
 struct sge {
@@ -403,7 +404,7 @@ struct sge {
 
 	struct sge_iq fwq;	/* Firmware event queue */
 	struct sge_ctrlq *ctrlq;/* Control queues */
-	struct sge_iq *fiq;	/* Forwarded interrupt queues (INTR_FWD) */
+	struct sge_iq *intrq;	/* Interrupt queues */
 	struct sge_txq *txq;	/* NIC tx queues */
 	struct sge_rxq *rxq;	/* NIC rx queues */
 
@@ -457,7 +458,9 @@ struct adapter {
 	struct t4_virt_res vres;
 
 	struct sysctl_ctx_list ctx; /* from first_port_up to last_port_down */
+	struct sysctl_oid *oid_fwq;
 	struct sysctl_oid *oid_ctrlq;
+	struct sysctl_oid *oid_intrq;
 
 	struct mtx sc_lock;
 	char lockname[16];
@@ -503,7 +506,10 @@ struct adapter {
 	rxq = &pi->adapter->sge.rxq[pi->first_rxq]; \
 	for (iter = 0; iter < pi->nrxq; ++iter, ++rxq)
 
-#define NFIQ(sc) ((sc)->intr_count > 1 ? (sc)->intr_count - 1 : 1)
+/* One for errors, one for firmware events */
+#define T4_EXTRA_INTR 2
+#define NINTRQ(sc) ((sc)->intr_count > T4_EXTRA_INTR ? \
+    (sc)->intr_count - T4_EXTRA_INTR : 1)
 
 static inline uint32_t
 t4_read_reg(struct adapter *sc, uint32_t reg)
@@ -600,12 +606,9 @@ int t4_teardown_adapter_queues(struct ad
 int t4_setup_eth_queues(struct port_info *);
 int t4_teardown_eth_queues(struct port_info *);
 void t4_intr_all(void *);
-void t4_intr_fwd(void *);
+void t4_intr(void *);
 void t4_intr_err(void *);
 void t4_intr_evt(void *);
-void t4_intr_data(void *);
-void t4_evt_rx(void *);
-void t4_eth_rx(void *);
 int t4_mgmt_tx(struct adapter *, struct mbuf *);
 int t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *);
 void t4_update_fl_bufsize(struct ifnet *);

Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c	Mon May 30 21:07:26 2011	(r222509)
+++ head/sys/dev/cxgbe/t4_main.c	Mon May 30 21:34:44 2011	(r222510)
@@ -214,12 +214,12 @@ SYSCTL_UINT(_hw_cxgbe, OID_AUTO, interru
     "interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively)");
 
 /*
- * Force the driver to use interrupt forwarding.
+ * Force the driver to use the same set of interrupts for all ports.
  */
-static int intr_fwd = 0;
-TUNABLE_INT("hw.cxgbe.interrupt_forwarding", &intr_fwd);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, interrupt_forwarding, CTLFLAG_RDTUN,
-    &intr_fwd, 0, "always use forwarded interrupts");
+static int intr_shared = 0;
+TUNABLE_INT("hw.cxgbe.interrupts_shared", &intr_shared);
+SYSCTL_UINT(_hw_cxgbe, OID_AUTO, interrupts_shared, CTLFLAG_RDTUN,
+    &intr_shared, 0, "interrupts shared between all ports");
 
 static unsigned int filter_mode = HW_TPL_FR_MT_PR_IV_P_FC;
 TUNABLE_INT("hw.cxgbe.filter_mode", &filter_mode);
@@ -229,7 +229,7 @@ SYSCTL_UINT(_hw_cxgbe, OID_AUTO, filter_
 struct intrs_and_queues {
 	int intr_type;		/* INTx, MSI, or MSI-X */
 	int nirq;		/* Number of vectors */
-	int intr_fwd;		/* Interrupts forwarded */
+	int intr_shared;	/* Interrupts shared between all ports */
 	int ntxq10g;		/* # of NIC txq's for each 10G port */
 	int nrxq10g;		/* # of NIC rxq's for each 10G port */
 	int ntxq1g;		/* # of NIC txq's for each 1G port */
@@ -516,8 +516,8 @@ t4_attach(device_t dev)
 			device_printf(dev, "unable to initialize port %d: %d\n",
 			    i, rc);
 			free(pi, M_CXGBE);
-			sc->port[i] = NULL;	/* indicates init failed */
-			continue;
+			sc->port[i] = NULL;
+			goto done;
 		}
 
 		snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
@@ -584,15 +584,15 @@ t4_attach(device_t dev)
 	s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
 	s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
 	s->neq = s->ntxq + s->nrxq;	/* the free list in an rxq is an eq */
-	s->neq += NCHAN;		/* control queues, 1 per hw channel */
+	s->neq += sc->params.nports;	/* control queues, 1 per port */
 	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
-	if (iaq.intr_fwd) {
-		sc->flags |= INTR_FWD;
-		s->niq += NFIQ(sc);		/* forwarded interrupt queues */
-		s->fiq = malloc(NFIQ(sc) * sizeof(struct sge_iq), M_CXGBE,
-		    M_ZERO | M_WAITOK);
-	}
-	s->ctrlq = malloc(NCHAN * sizeof(struct sge_ctrlq), M_CXGBE,
+	if (iaq.intr_shared)
+		sc->flags |= INTR_SHARED;
+	s->niq += NINTRQ(sc);		/* interrupt queues */
+
+	s->intrq = malloc(NINTRQ(sc) * sizeof(struct sge_iq), M_CXGBE,
+	    M_ZERO | M_WAITOK);
+	s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_ctrlq), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 	s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
 	    M_ZERO | M_WAITOK);
@@ -702,7 +702,7 @@ t4_detach(device_t dev)
 	free(sc->sge.rxq, M_CXGBE);
 	free(sc->sge.txq, M_CXGBE);
 	free(sc->sge.ctrlq, M_CXGBE);
-	free(sc->sge.fiq, M_CXGBE);
+	free(sc->sge.intrq, M_CXGBE);
 	free(sc->sge.iqmap, M_CXGBE);
 	free(sc->sge.eqmap, M_CXGBE);
 	free(sc->tids.ftid_tab, M_CXGBE);
@@ -1238,33 +1238,32 @@ cfg_itype_and_nqueues(struct adapter *sc
 		nrxq10g = min(nc, max_nrxq_10g);
 		nrxq1g = min(nc, max_nrxq_1g);
 
-		/* Extra 2 is for a) error interrupt b) firmware event */
-		iaq->nirq = n10g * nrxq10g + n1g * nrxq1g + 2;
-		if (iaq->nirq <= navail && intr_fwd == 0) {
+		iaq->nirq = n10g * nrxq10g + n1g * nrxq1g + T4_EXTRA_INTR;
+		if (iaq->nirq <= navail && intr_shared == 0) {
 
 			if (itype == INTR_MSI && !powerof2(iaq->nirq))
-				goto fwd;
+				goto share;
 
 			/* One for err, one for fwq, and one for each rxq */
 
-			iaq->intr_fwd = 0;
+			iaq->intr_shared = 0;
 			iaq->nrxq10g = nrxq10g;
 			iaq->nrxq1g = nrxq1g;
 
 		} else {
-fwd:
-			iaq->intr_fwd = 1;
+share:
+			iaq->intr_shared = 1;
 
-			if (navail > nc) {
+			if (navail >= nc + T4_EXTRA_INTR) {
 				if (itype == INTR_MSIX)
-					navail = nc + 1;
+					navail = nc + T4_EXTRA_INTR;
 
 				/* navail is and must remain a pow2 for MSI */
 				if (itype == INTR_MSI) {
 					KASSERT(powerof2(navail),
 					    ("%d not power of 2", navail));
 
-					while (navail / 2 > nc)
+					while (navail / 2 >= nc + T4_EXTRA_INTR)
 						navail /= 2;
 				}
 			}
@@ -1297,7 +1296,7 @@ fwd:
 			 * the kernel is willing to allocate (it's in navail).
 			 */
 			pci_release_msi(sc->dev);
-			goto fwd;
+			goto share;
 		}
 
 		device_printf(sc->dev,
@@ -1930,16 +1929,18 @@ cxgbe_uninit_synchronized(struct port_in
 	return (0);
 }
 
-#define T4_ALLOC_IRQ(sc, irqid, rid, handler, arg, name) do { \
-	rc = t4_alloc_irq(sc, &sc->irq[irqid], rid, handler, arg, name); \
+#define T4_ALLOC_IRQ(sc, irq, rid, handler, arg, name) do { \
+	rc = t4_alloc_irq(sc, irq, rid, handler, arg, name); \
 	if (rc != 0) \
 		goto done; \
 } while (0)
 static int
 first_port_up(struct adapter *sc)
 {
-	int rc, i;
-	char name[8];
+	int rc, i, rid, p, q;
+	char s[8];
+	struct irq *irq;
+	struct sge_iq *intrq;
 
 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 
@@ -1953,39 +1954,52 @@ first_port_up(struct adapter *sc)
 	/*
 	 * Setup interrupts.
 	 */
+	irq = &sc->irq[0];
+	rid = sc->intr_type == INTR_INTX ? 0 : 1;
 	if (sc->intr_count == 1) {
-		KASSERT(sc->flags & INTR_FWD,
-		    ("%s: single interrupt but not forwarded?", __func__));
-		T4_ALLOC_IRQ(sc, 0, 0, t4_intr_all, sc, "all");
+		KASSERT(sc->flags & INTR_SHARED,
+		    ("%s: single interrupt but not shared?", __func__));
+
+		T4_ALLOC_IRQ(sc, irq, rid, t4_intr_all, sc, "all");
 	} else {
 		/* Multiple interrupts.  The first one is always error intr */
-		T4_ALLOC_IRQ(sc, 0, 1, t4_intr_err, sc, "err");
+		T4_ALLOC_IRQ(sc, irq, rid, t4_intr_err, sc, "err");
+		irq++;
+		rid++;
+
+		/* Firmware event queue normally has an interrupt of its own */
+		if (sc->intr_count > T4_EXTRA_INTR) {
+			T4_ALLOC_IRQ(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
+			    "evt");
+			irq++;
+			rid++;
+		}
+
+		intrq = &sc->sge.intrq[0];
+		if (sc->flags & INTR_SHARED) {
 
-		if (sc->flags & INTR_FWD) {
-			/* The rest are shared by the fwq and all data intr */
-			for (i = 1; i < sc->intr_count; i++) {
-				snprintf(name, sizeof(name), "mux%d", i - 1);
-				T4_ALLOC_IRQ(sc, i, i + 1, t4_intr_fwd,
-				    &sc->sge.fiq[i - 1], name);
+			/* All ports share these interrupt queues */
+
+			for (i = 0; i < NINTRQ(sc); i++) {
+				snprintf(s, sizeof(s), "*.%d", i);
+				T4_ALLOC_IRQ(sc, irq, rid, t4_intr, intrq, s);
+				irq++;
+				rid++;
+				intrq++;
 			}
 		} else {
-			struct port_info *pi;
-			int p, q;
 
-			T4_ALLOC_IRQ(sc, 1, 2, t4_intr_evt, &sc->sge.fwq,
-			    "evt");
+			/* Each port has its own set of interrupt queues */
 
-			p = q = 0;
-			pi = sc->port[p];
-			for (i = 2; i < sc->intr_count; i++) {
-				snprintf(name, sizeof(name), "p%dq%d", p, q);
-				if (++q >= pi->nrxq) {
-					p++;
-					q = 0;
-					pi = sc->port[p];
+			for (p = 0; p < sc->params.nports; p++) {
+				for (q = 0; q < sc->port[p]->nrxq; q++) {
+					snprintf(s, sizeof(s), "%d.%d", p, q);
+					T4_ALLOC_IRQ(sc, irq, rid, t4_intr,
+					    intrq, s);
+					irq++;
+					rid++;
+					intrq++;
 				}
-				T4_ALLOC_IRQ(sc, i, i + 1, t4_intr_data,
-				    &sc->sge.rxq[i - 2], name);
 			}
 		}
 	}
@@ -3121,7 +3135,7 @@ set_filter_wr(struct adapter *sc, int fi
 		V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));
 	fwr->smac_sel = 0;
 	fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
-	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
+	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.intrq[0].abs_id));
 	fwr->maci_to_matchtypem =
 	    htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
 		V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
@@ -3181,7 +3195,7 @@ del_filter_wr(struct adapter *sc, int fi
 	m->m_len = m->m_pkthdr.len = sizeof(*fwr);
 	bzero(fwr, sizeof (*fwr));
 
-	t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
+	t4_mk_filtdelwr(ftid, fwr, sc->sge.intrq[0].abs_id);
 
 	f->pending = 1;
 	rc = t4_mgmt_tx(sc, m);

Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c	Mon May 30 21:07:26 2011	(r222509)
+++ head/sys/dev/cxgbe/t4_sge.c	Mon May 30 21:34:44 2011	(r222510)
@@ -91,6 +91,8 @@ struct sgl {
 	bus_dma_segment_t seg[TX_SGL_SEGS];
 };
 
+static void t4_evt_rx(void *);
+static void t4_eth_rx(void *);
 static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int,
     int, iq_intr_handler_t *, char *);
 static inline void init_fl(struct sge_fl *, int, char *);
@@ -102,8 +104,10 @@ static int free_ring(struct adapter *, b
 static int alloc_iq_fl(struct port_info *, struct sge_iq *, struct sge_fl *,
     int, int);
 static int free_iq_fl(struct port_info *, struct sge_iq *, struct sge_fl *);
-static int alloc_iq(struct sge_iq *, int);
-static int free_iq(struct sge_iq *);
+static int alloc_intrq(struct adapter *, int, int, int);
+static int free_intrq(struct sge_iq *);
+static int alloc_fwq(struct adapter *, int);
+static int free_fwq(struct sge_iq *);
 static int alloc_rxq(struct port_info *, struct sge_rxq *, int, int);
 static int free_rxq(struct port_info *, struct sge_rxq *);
 static int alloc_ctrlq(struct adapter *, struct sge_ctrlq *, int);
@@ -139,9 +143,10 @@ static void write_eqflush_wr(struct sge_
 static __be64 get_flit(bus_dma_segment_t *, int, int);
 static int handle_sge_egr_update(struct adapter *,
     const struct cpl_sge_egr_update *);
+static void handle_cpl(struct adapter *, struct sge_iq *);
 
 static int ctrl_tx(struct adapter *, struct sge_ctrlq *, struct mbuf *);
-static int sysctl_abs_id(SYSCTL_HANDLER_ARGS);
+static int sysctl_uint16(SYSCTL_HANDLER_ARGS);
 
 extern void filter_rpl(struct adapter *, const struct cpl_set_tcb_rpl *);
 
@@ -243,8 +248,7 @@ t4_destroy_dma_tag(struct adapter *sc)
 
 /*
  * Allocate and initialize the firmware event queue, control queues, and the
- * forwarded interrupt queues (if any).  The adapter owns all these queues as
- * they are not associated with any particular port.
+ * interrupt queues.  The adapter owns all of these queues.
  *
  * Returns errno on failure.  Resources allocated up to that point may still be
  * allocated.  Caller is responsible for cleanup in case this function fails.
@@ -252,8 +256,8 @@ t4_destroy_dma_tag(struct adapter *sc)
 int
 t4_setup_adapter_queues(struct adapter *sc)
 {
-	int i, rc;
-	struct sge_iq *iq, *fwq;
+	int i, j, rc, intr_idx, qsize;
+	struct sge_iq *iq;
 	struct sge_ctrlq *ctrlq;
 	iq_intr_handler_t *handler;
 	char name[16];
@@ -264,47 +268,76 @@ t4_setup_adapter_queues(struct adapter *
 		struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
 		struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
 
+		sc->oid_fwq = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO,
+		    "fwq", CTLFLAG_RD, NULL, "firmware event queue");
 		sc->oid_ctrlq = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO,
 		    "ctrlq", CTLFLAG_RD, NULL, "ctrl queues");
+		sc->oid_intrq = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO,
+		    "intrq", CTLFLAG_RD, NULL, "interrupt queues");
 	}
 
-	fwq = &sc->sge.fwq;
-	if (sc->flags & INTR_FWD) {
-		iq = &sc->sge.fiq[0];
-
-		/*
-		 * Forwarded interrupt queues - allocate 1 if there's only 1
-		 * vector available, one less than the number of vectors
-		 * otherwise (the first vector is reserved for the error
-		 * interrupt in that case).
-		 */
-		i = sc->intr_count > 1 ? 1 : 0;
-		for (; i < sc->intr_count; i++, iq++) {
-
-			snprintf(name, sizeof(name), "%s fiq%d",
+	/*
+	 * Interrupt queues
+	 */
+	intr_idx = sc->intr_count - NINTRQ(sc);
+	if (sc->flags & INTR_SHARED) {
+		qsize = max((sc->sge.nrxq + 1) * 2, INTR_IQ_QSIZE);
+		for (i = 0; i < NINTRQ(sc); i++, intr_idx++) {
+			snprintf(name, sizeof(name), "%s intrq%d",
 			    device_get_nameunit(sc->dev), i);
-			init_iq(iq, sc, 0, 0, (sc->sge.nrxq + 1) * 2, 16, NULL,
-			    name);
 
-			rc = alloc_iq(iq, i);
+			iq = &sc->sge.intrq[i];
+			init_iq(iq, sc, 0, 0, qsize, INTR_IQ_ESIZE, NULL, name);
+			rc = alloc_intrq(sc, i % sc->params.nports, i,
+			    intr_idx);
+
 			if (rc != 0) {
 				device_printf(sc->dev,
-				    "failed to create fwd intr queue %d: %d\n",
-				    i, rc);
+				    "failed to create %s: %d\n", name, rc);
 				return (rc);
 			}
 		}
-
-		handler = t4_evt_rx;
-		i = 0;	/* forward fwq's interrupt to the first fiq */
 	} else {
-		handler = NULL;
-		i = 1;	/* fwq should use vector 1 (0 is used by error) */
+		int qidx = 0;
+		struct port_info *pi;
+
+		for (i = 0; i < sc->params.nports; i++) {
+			pi = sc->port[i];
+			qsize = max((pi->nrxq + 1) * 2, INTR_IQ_QSIZE);
+			for (j = 0; j < pi->nrxq; j++, qidx++, intr_idx++) {
+				snprintf(name, sizeof(name), "%s intrq%d",
+				    device_get_nameunit(pi->dev), j);
+
+				iq = &sc->sge.intrq[qidx];
+				init_iq(iq, sc, 0, 0, qsize, INTR_IQ_ESIZE,
+				    NULL, name);
+				rc = alloc_intrq(sc, i, qidx, intr_idx);
+
+				if (rc != 0) {
+					device_printf(sc->dev,
+					    "failed to create %s: %d\n",
+					    name, rc);
+					return (rc);
+				}
+			}
+		}
 	}
 
+	/*
+	 * Firmware event queue
+	 */
 	snprintf(name, sizeof(name), "%s fwq", device_get_nameunit(sc->dev));
-	init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE, handler, name);
-	rc = alloc_iq(fwq, i);
+	if (sc->intr_count > T4_EXTRA_INTR) {
+		handler = NULL;
+		intr_idx = 1;
+	} else {
+		handler = t4_evt_rx;
+		intr_idx = 0;
+	}
+
+	iq = &sc->sge.fwq;
+	init_iq(iq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE, handler, name);
+	rc = alloc_fwq(sc, intr_idx);
 	if (rc != 0) {
 		device_printf(sc->dev,
 		    "failed to create firmware event queue: %d\n", rc);
@@ -313,10 +346,10 @@ t4_setup_adapter_queues(struct adapter *
 	}
 
 	/*
-	 * Control queues - one per hardware channel.
+	 * Control queues - one per port.
 	 */
 	ctrlq = &sc->sge.ctrlq[0];
-	for (i = 0; i < NCHAN; i++, ctrlq++) {
+	for (i = 0; i < sc->params.nports; i++, ctrlq++) {
 		snprintf(name, sizeof(name), "%s ctrlq%d",
 		    device_get_nameunit(sc->dev), i);
 		init_eq(&ctrlq->eq, CTRL_EQ_QSIZE, name);
@@ -344,21 +377,22 @@ t4_teardown_adapter_queues(struct adapte
 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
 
 	/* Do this before freeing the queues */
-	if (sc->oid_ctrlq) {
+	if (sc->oid_fwq || sc->oid_ctrlq || sc->oid_intrq) {
 		sysctl_ctx_free(&sc->ctx);
+		sc->oid_fwq = NULL;
 		sc->oid_ctrlq = NULL;
+		sc->oid_intrq = NULL;
 	}
 
-	for (i = 0; i < NCHAN; i++)
+	for (i = 0; i < sc->params.nports; i++)
 		free_ctrlq(sc, &sc->sge.ctrlq[i]);
 
 	iq = &sc->sge.fwq;
-	free_iq(iq);
-	if (sc->flags & INTR_FWD) {
-		for (i = 0; i < NFIQ(sc); i++) {
-			iq = &sc->sge.fiq[i];
-			free_iq(iq);
-		}
+	free_fwq(iq);
+
+	for (i = 0; i < NINTRQ(sc); i++) {
+		iq = &sc->sge.intrq[i];
+		free_intrq(iq);
 	}
 
 	return (0);
@@ -388,23 +422,19 @@ t4_setup_eth_queues(struct port_info *pi
 		snprintf(name, sizeof(name), "%s rxq%d-iq",
 		    device_get_nameunit(pi->dev), i);
 		init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
-		    pi->qsize_rxq, RX_IQ_ESIZE,
-		    sc->flags & INTR_FWD ? t4_eth_rx : NULL, name);
+		    pi->qsize_rxq, RX_IQ_ESIZE, t4_eth_rx, name);
 
 		snprintf(name, sizeof(name), "%s rxq%d-fl",
 		    device_get_nameunit(pi->dev), i);
 		init_fl(&rxq->fl, pi->qsize_rxq / 8, name);
 
-		if (sc->flags & INTR_FWD)
-			intr_idx = (pi->first_rxq + i) % NFIQ(sc);
-		else
-			intr_idx = pi->first_rxq + i + 2;
+		intr_idx = pi->first_rxq + i;
+		if (sc->flags & INTR_SHARED)
+			intr_idx %= NINTRQ(sc);
 
 		rc = alloc_rxq(pi, rxq, intr_idx, i);
 		if (rc != 0)
 			goto done;
-
-		intr_idx++;
 	}
 
 	for_each_txq(pi, i, txq) {
@@ -452,25 +482,26 @@ t4_teardown_eth_queues(struct port_info 
 	return (0);
 }
 
-/* Deals with errors and forwarded interrupts */
+/* Deals with errors and the first (and only) interrupt queue */
 void
 t4_intr_all(void *arg)
 {
 	struct adapter *sc = arg;
 
 	t4_intr_err(arg);
-	t4_intr_fwd(&sc->sge.fiq[0]);
+	t4_intr(&sc->sge.intrq[0]);
 }
 
-/* Deals with forwarded interrupts on the given ingress queue */
+/* Deals with interrupts, and a few CPLs, on the given interrupt queue */
 void
-t4_intr_fwd(void *arg)
+t4_intr(void *arg)
 {
 	struct sge_iq *iq = arg, *q;
 	struct adapter *sc = iq->adapter;
 	struct rsp_ctrl *ctrl;
+	const struct rss_header *rss;
 	int ndesc_pending = 0, ndesc_total = 0;
-	int qid;
+	int qid, rsp_type;
 
 	if (!atomic_cmpset_32(&iq->state, IQS_IDLE, IQS_BUSY))
 		return;
@@ -479,17 +510,23 @@ t4_intr_fwd(void *arg)
 
 		rmb();
 
-		/* Only interrupt muxing expected on this queue */
-		KASSERT(G_RSPD_TYPE(ctrl->u.type_gen) == X_RSPD_TYPE_INTR,
-		    ("unexpected event on forwarded interrupt queue: %x",
-		    G_RSPD_TYPE(ctrl->u.type_gen)));
+		rss = (const void *)iq->cdesc;
+		rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
+
+		if (__predict_false(rsp_type == X_RSPD_TYPE_CPL)) {
+			handle_cpl(sc, iq);
+			goto nextdesc;
+		}
 
 		qid = ntohl(ctrl->pldbuflen_qid) - sc->sge.iq_start;
 		q = sc->sge.iqmap[qid];
 
-		q->handler(q);
+		if (atomic_cmpset_32(&q->state, IQS_IDLE, IQS_BUSY)) {
+			q->handler(q);
+			atomic_cmpset_32(&q->state, IQS_BUSY, IQS_IDLE);
+		}
 
-		ndesc_total++;
+nextdesc:	ndesc_total++;
 		if (++ndesc_pending >= iq->qsize / 4) {
 			t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
 			    V_CIDXINC(ndesc_pending) |
@@ -514,9 +551,7 @@ t4_intr_err(void *arg)
 {
 	struct adapter *sc = arg;
 
-	if (sc->intr_type == INTR_INTX)
-		t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
-
+	t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
 	t4_slow_intr_handler(sc);
 }
 
@@ -526,70 +561,32 @@ t4_intr_evt(void *arg)
 {
 	struct sge_iq *iq = arg;
 
-	if (!atomic_cmpset_32(&iq->state, IQS_IDLE, IQS_BUSY))
-		return;
-
-	t4_evt_rx(arg);
-
-	atomic_cmpset_32(&iq->state, IQS_BUSY, IQS_IDLE);
-}
-
-void
-t4_intr_data(void *arg)
-{
-	struct sge_iq *iq = arg;
-
-	if (!atomic_cmpset_32(&iq->state, IQS_IDLE, IQS_BUSY))
-		return;
-
-	t4_eth_rx(arg);
-
-	atomic_cmpset_32(&iq->state, IQS_BUSY, IQS_IDLE);
+	if (atomic_cmpset_32(&iq->state, IQS_IDLE, IQS_BUSY)) {
+		t4_evt_rx(arg);
+		atomic_cmpset_32(&iq->state, IQS_BUSY, IQS_IDLE);
+	}
 }
 
-void
+static void
 t4_evt_rx(void *arg)
 {
 	struct sge_iq *iq = arg;
 	struct adapter *sc = iq->adapter;
 	struct rsp_ctrl *ctrl;
-	const struct rss_header *rss;
 	int ndesc_pending = 0, ndesc_total = 0;
 
 	KASSERT(iq == &sc->sge.fwq, ("%s: unexpected ingress queue", __func__));
 
 	while (is_new_response(iq, &ctrl)) {
+		int rsp_type;
 
 		rmb();
 
-		rss = (const void *)iq->cdesc;
+		rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
+		if (__predict_false(rsp_type != X_RSPD_TYPE_CPL))
+			panic("%s: unexpected rsp_type %d", __func__, rsp_type);
 
-		/* Should only get CPL on this queue */
-		KASSERT(G_RSPD_TYPE(ctrl->u.type_gen) == X_RSPD_TYPE_CPL,
-		    ("%s: unexpected type %d", __func__,
-		    G_RSPD_TYPE(ctrl->u.type_gen)));
-
-		switch (rss->opcode) {
-		case CPL_FW4_MSG:
-		case CPL_FW6_MSG: {
-			const struct cpl_fw6_msg *cpl;
-
-			cpl = (const void *)(rss + 1);
-			if (cpl->type == FW6_TYPE_CMD_RPL)
-				t4_handle_fw_rpl(sc, cpl->data);
-
-			break;
-			}
-		case CPL_SGE_EGR_UPDATE:
-			handle_sge_egr_update(sc, (const void *)(rss + 1));
-			break;
-		case CPL_SET_TCB_RPL:
-			filter_rpl(sc, (const void *) (rss + 1));
-			break;
-		default:
-			device_printf(sc->dev,
-			    "can't handle CPL opcode %d.", rss->opcode);
-		}
+		handle_cpl(sc, iq);
 
 		ndesc_total++;
 		if (++ndesc_pending >= iq->qsize / 4) {
@@ -600,6 +597,7 @@ t4_evt_rx(void *arg)
 				V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
 			ndesc_pending = 0;
 		}
+
 		iq_next(iq);
 	}
 
@@ -613,7 +611,7 @@ t4_evt_rx(void *arg)
 #define RX_COPY_THRESHOLD MINCLSIZE
 #endif
 
-void
+static void
 t4_eth_rx(void *arg)
 {
 	struct sge_rxq *rxq = arg;
@@ -644,17 +642,9 @@ t4_eth_rx(void *arg)
 		rss = (const void *)iq->cdesc;
 		i = G_RSPD_TYPE(ctrl->u.type_gen);
 
-		if (__predict_false(i == X_RSPD_TYPE_CPL)) {
-
-			/* Can't be anything except an egress update */
-			KASSERT(rss->opcode == CPL_SGE_EGR_UPDATE,
-			    ("%s: unexpected CPL %x", __func__, rss->opcode));
-
-			handle_sge_egr_update(sc, (const void *)(rss + 1));
-			goto nextdesc;
-		}
 		KASSERT(i == X_RSPD_TYPE_FLBUF && rss->opcode == CPL_RX_PKT,
-		    ("%s: unexpected CPL %x rsp %d", __func__, rss->opcode, i));
+		    ("%s: unexpected type %d CPL opcode 0x%x",
+		    __func__, i, rss->opcode));
 
 		sd_next = sd + 1;
 		if (__predict_false(fl->cidx + 1 == fl->cap))
@@ -786,16 +776,15 @@ t4_eth_rx(void *arg)
 			refill_fl(sc, fl, 64, 32);
 		FL_UNLOCK(fl);
 
-nextdesc:	ndescs++;
-		iq_next(iq);
-
-		if (ndescs > 32) {
+		if (++ndescs > 32) {
 			t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
 			    V_CIDXINC(ndescs) |
 			    V_INGRESSQID((u32)iq->cntxt_id) |
 			    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
 			ndescs = 0;
 		}
+
+		iq_next(iq);
 	}
 
 #ifdef INET
@@ -1008,7 +997,7 @@ t4_update_fl_bufsize(struct ifnet *ifp)
 
 /*
  * A non-NULL handler indicates this iq will not receive direct interrupts, the
- * handler will be invoked by a forwarded interrupt queue.
+ * handler will be invoked by an interrupt queue.
  */
 static inline void
 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
@@ -1100,7 +1089,7 @@ free_ring(struct adapter *sc, bus_dma_ta
  *
  * If the ingress queue will take interrupts directly (iq->handler == NULL) then
  * the intr_idx specifies the vector, starting from 0.  Otherwise it specifies
- * the index of the queue to which its interrupts will be forwarded.
+ * the index of the interrupt queue to which its interrupts will be forwarded.
  */
 static int
 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
@@ -1112,10 +1101,6 @@ alloc_iq_fl(struct port_info *pi, struct
 	struct adapter *sc = iq->adapter;
 	__be32 v = 0;
 
-	/* The adapter queues are nominally allocated in port[0]'s name */
-	if (pi == NULL)
-		pi = sc->port[0];
-
 	len = iq->qsize * iq->esize;
 	rc = alloc_ring(sc, len, &iq->desc_tag, &iq->desc_map, &iq->ba,
 	    (void **)&iq->desc);
@@ -1135,10 +1120,10 @@ alloc_iq_fl(struct port_info *pi, struct
 		v |= F_FW_IQ_CMD_IQASYNCH;
 
 	if (iq->handler) {
-		KASSERT(intr_idx < NFIQ(sc),
+		KASSERT(intr_idx < NINTRQ(sc),
 		    ("%s: invalid indirect intr_idx %d", __func__, intr_idx));
 		v |= F_FW_IQ_CMD_IQANDST;
-		v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.fiq[intr_idx].abs_id);
+		v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.intrq[intr_idx].abs_id);
 	} else {
 		KASSERT(intr_idx < sc->intr_count,
 		    ("%s: invalid direct intr_idx %d", __func__, intr_idx));
@@ -1333,13 +1318,61 @@ free_iq_fl(struct port_info *pi, struct 
 }
 
 static int
-alloc_iq(struct sge_iq *iq, int intr_idx)
+alloc_intrq(struct adapter *sc, int port_idx, int intrq_idx, int intr_idx)
 {
-	return alloc_iq_fl(NULL, iq, NULL, intr_idx, -1);
+	int rc;
+	struct sysctl_oid *oid;
+	struct sysctl_oid_list *children;
+	char name[16];
+	struct sge_iq *intrq = &sc->sge.intrq[intrq_idx];
+
+	rc = alloc_iq_fl(sc->port[port_idx], intrq, NULL, intr_idx, -1);
+	if (rc != 0)
+		return (rc);
+
+	children = SYSCTL_CHILDREN(sc->oid_intrq);
+
+	snprintf(name, sizeof(name), "%d", intrq_idx);
+	oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, name, CTLFLAG_RD,
+	    NULL, "interrupt queue");
+	children = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cidx",
+	    CTLTYPE_INT | CTLFLAG_RD, &intrq->cidx, 0, sysctl_uint16, "I",
+	    "consumer index");
+
+	return (rc);
 }
 
 static int
-free_iq(struct sge_iq *iq)
+free_intrq(struct sge_iq *iq)
+{
+	return free_iq_fl(NULL, iq, NULL);
+
+}
+
+static int
+alloc_fwq(struct adapter *sc, int intr_idx)
+{
+	int rc;
+	struct sysctl_oid_list *children;
+	struct sge_iq *fwq = &sc->sge.fwq;
+
+	rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
+	if (rc != 0)
+		return (rc);
+
+	children = SYSCTL_CHILDREN(sc->oid_fwq);
+
+	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cidx",
+	    CTLTYPE_INT | CTLFLAG_RD, &fwq->cidx, 0, sysctl_uint16, "I",
+	    "consumer index");
+
+	return (rc);
+}
+
+static int
+free_fwq(struct sge_iq *iq)
 {
 	return free_iq_fl(NULL, iq, NULL);
 }
@@ -1375,7 +1408,7 @@ alloc_rxq(struct port_info *pi, struct s
 	children = SYSCTL_CHILDREN(oid);
 
 	SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "abs_id",
-	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.abs_id, 0, sysctl_abs_id, "I",
+	    CTLTYPE_INT | CTLFLAG_RD, &rxq->iq.abs_id, 0, sysctl_uint16, "I",
 	    "absolute id of the queue");
 #ifdef INET
 	SYSCTL_ADD_INT(&pi->ctx, children, OID_AUTO, "lro_queued", CTLFLAG_RD,
@@ -1433,7 +1466,10 @@ alloc_ctrlq(struct adapter *sc, struct s
 	eq->cap = eq->qsize - SPG_LEN / CTRL_EQ_ESIZE;
 	eq->spg = (void *)&eq->desc[eq->cap];
 	eq->avail = eq->cap - 1;	/* one less to avoid cidx = pidx */
-	eq->iqid = sc->sge.fwq.cntxt_id;
+	if (sc->flags & INTR_SHARED)
+		eq->iqid = sc->sge.intrq[idx % NINTRQ(sc)].cntxt_id;
+	else
+		eq->iqid = sc->sge.intrq[sc->port[idx]->first_rxq].cntxt_id;
 
 	bzero(&c, sizeof(c));
 
@@ -1446,8 +1482,8 @@ alloc_ctrlq(struct adapter *sc, struct s
 	c.physeqid_pkd = htobe32(0);
 	c.fetchszm_to_iqid =
 	    htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
-		V_FW_EQ_CTRL_CMD_PCIECHN(idx) | F_FW_EQ_CTRL_CMD_FETCHRO |
-		V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
+		V_FW_EQ_CTRL_CMD_PCIECHN(sc->port[idx]->tx_chan) |
+		F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
 	c.dcaen_to_eqsize =
 	    htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
 		V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
@@ -1479,13 +1515,12 @@ alloc_ctrlq(struct adapter *sc, struct s
 	    NULL, "ctrl queue");
 	children = SYSCTL_CHILDREN(oid);
 
-	SYSCTL_ADD_UQUAD(&sc->ctx, children, OID_AUTO, "total_wrs", CTLFLAG_RD,
-	    &ctrlq->total_wrs, "total # of work requests");
+	SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "pidx",
+	    CTLTYPE_INT | CTLFLAG_RD, &ctrlq->eq.pidx, 0, sysctl_uint16, "I",
+	    "producer index");
 	SYSCTL_ADD_UINT(&sc->ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
 	    &ctrlq->no_desc, 0,
 	    "# of times ctrlq ran out of hardware descriptors");
-	SYSCTL_ADD_UINT(&sc->ctx, children, OID_AUTO, "too_long", CTLFLAG_RD,
-	    &ctrlq->too_long, 0, "# of oversized work requests");
 
 	return (rc);
 }
@@ -1526,6 +1561,7 @@ alloc_txq(struct port_info *pi, struct s
 	char name[16];
 	struct sysctl_oid *oid;
 	struct sysctl_oid_list *children;
+	struct sge_iq *intrq;
 
 	txq->ifp = pi->ifp;
 	TASK_INIT(&txq->resume_tx, 0, cxgbe_txq_start, txq);
@@ -1544,7 +1580,12 @@ alloc_txq(struct port_info *pi, struct s
 	txq->sdesc = malloc(eq->cap * sizeof(struct tx_sdesc), M_CXGBE,
 	    M_ZERO | M_WAITOK);
 	txq->br = buf_ring_alloc(eq->qsize, M_CXGBE, M_WAITOK, &eq->eq_lock);
-	eq->iqid = sc->sge.rxq[pi->first_rxq].iq.cntxt_id;
+
+	intrq = &sc->sge.intrq[0];
+	if (sc->flags & INTR_SHARED)
+		eq->iqid = intrq[(pi->first_txq + idx) % NINTRQ(sc)].cntxt_id;
+	else
+		eq->iqid = intrq[pi->first_rxq + (idx % pi->nrxq)].cntxt_id;
 
 	rc = bus_dma_tag_create(sc->dmat, 1, 0, BUS_SPACE_MAXADDR,
 	    BUS_SPACE_MAXADDR, NULL, NULL, 64 * 1024, TX_SGL_SEGS,
@@ -2695,6 +2736,32 @@ handle_sge_egr_update(struct adapter *sc
 	return (0);
 }
 
+static void
+handle_cpl(struct adapter *sc, struct sge_iq *iq)
+{
+	const struct rss_header *rss = (const void *)iq->cdesc;
+	const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
+
+	switch (rss->opcode) {
+	case CPL_FW4_MSG:
+	case CPL_FW6_MSG:
+		if (cpl->type == FW6_TYPE_CMD_RPL)
+			t4_handle_fw_rpl(sc, cpl->data);
+		break;
+
+	case CPL_SGE_EGR_UPDATE:
+		handle_sge_egr_update(sc, (const void *)cpl);
+		break;
+
+	case CPL_SET_TCB_RPL:
+		filter_rpl(sc, (const void *)cpl);
+		break;
+
+	default:
+		panic("%s: unexpected CPL opcode 0x%x", __func__, rss->opcode);
+	}
+}
+
 /*
  * m0 is freed on successful transmission.
  */
@@ -2710,7 +2777,8 @@ ctrl_tx(struct adapter *sc, struct sge_c
 	M_ASSERTPKTHDR(m0);
 
 	if (m0->m_pkthdr.len > SGE_MAX_WR_LEN) {
-		ctrlq->too_long++;
+		log(LOG_ERR, "%s: %s work request too long (%d)",
+		    device_get_nameunit(sc->dev), __func__, m0->m_pkthdr.len);
 		return (EMSGSIZE);
 	}
 	ndesc = howmany(m0->m_pkthdr.len, CTRL_EQ_ESIZE);
@@ -2738,7 +2806,6 @@ ctrl_tx(struct adapter *sc, struct sge_c
 		eq->pidx -= eq->cap;
 
 	eq->pending += ndesc;
-	ctrlq->total_wrs++;
 	ring_eq_db(sc, eq);
 failed:
 	EQ_UNLOCK(eq);
@@ -2749,7 +2816,7 @@ failed:
 }
 
 static int
-sysctl_abs_id(SYSCTL_HANDLER_ARGS)
+sysctl_uint16(SYSCTL_HANDLER_ARGS)

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201105302134.p4ULYigj094113>