Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 11 Aug 2018 04:55:47 +0000 (UTC)
From:      Navdeep Parhar <np@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r337609 - head/sys/dev/cxgbe
Message-ID:  <201808110455.w7B4tloY058893@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: np
Date: Sat Aug 11 04:55:47 2018
New Revision: 337609
URL: https://svnweb.freebsd.org/changeset/base/337609

Log:
  cxgbe(4): Create two variants of service_iq, one for queues with
  freelists and one for those without.
  
  MFH:		3 weeks
  Sponsored by:	Chelsio Communications

Modified:
  head/sys/dev/cxgbe/adapter.h
  head/sys/dev/cxgbe/t4_main.c
  head/sys/dev/cxgbe/t4_netmap.c
  head/sys/dev/cxgbe/t4_sge.c

Modified: head/sys/dev/cxgbe/adapter.h
==============================================================================
--- head/sys/dev/cxgbe/adapter.h	Sat Aug 11 04:26:29 2018	(r337608)
+++ head/sys/dev/cxgbe/adapter.h	Sat Aug 11 04:55:47 2018	(r337609)
@@ -670,6 +670,7 @@ struct sge_wrq {
 
 #define INVALID_NM_RXQ_CNTXT_ID ((uint16_t)(-1))
 struct sge_nm_rxq {
+	volatile int nm_state;	/* NM_OFF, NM_ON, or NM_BUSY */
 	struct vi_info *vi;
 
 	struct iq_desc *iq_desc;
@@ -797,7 +798,6 @@ struct adapter {
 	struct irq {
 		struct resource *res;
 		int rid;
-		volatile int nm_state;	/* NM_OFF, NM_ON, or NM_BUSY */
 		void *tag;
 		struct sge_rxq *rxq;
 		struct sge_nm_rxq *nm_rxq;
@@ -1186,9 +1186,10 @@ void release_tid(struct adapter *, int, struct sge_wrq
 
 #ifdef DEV_NETMAP
 /* t4_netmap.c */
+struct sge_nm_rxq;
 void cxgbe_nm_attach(struct vi_info *);
 void cxgbe_nm_detach(struct vi_info *);
-void t4_nm_intr(void *);
+void service_nm_rxq(struct sge_nm_rxq *);
 #endif
 
 /* t4_sge.c */
@@ -1207,7 +1208,10 @@ int t4_setup_vi_queues(struct vi_info *);
 int t4_teardown_vi_queues(struct vi_info *);
 void t4_intr_all(void *);
 void t4_intr(void *);
+#ifdef DEV_NETMAP
+void t4_nm_intr(void *);
 void t4_vi_intr(void *);
+#endif
 void t4_intr_err(void *);
 void t4_intr_evt(void *);
 void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *);

Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c	Sat Aug 11 04:26:29 2018	(r337608)
+++ head/sys/dev/cxgbe/t4_main.c	Sat Aug 11 04:55:47 2018	(r337609)
@@ -684,8 +684,8 @@ struct {
 
 #ifdef TCP_OFFLOAD
 /*
- * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
- * exactly the same for both rxq and ofld_rxq.
+ * service_iq_fl() has an iq and needs the fl.  Offset of fl from the iq should
+ * be exactly the same for both rxq and ofld_rxq.
  */
 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
@@ -4819,9 +4819,26 @@ t4_setup_intr_handlers(struct adapter *sc)
 #ifdef DEV_NETMAP
 					if (q < vi->nnmrxq)
 						irq->nm_rxq = nm_rxq++;
+
+					if (irq->nm_rxq != NULL &&
+					    irq->rxq == NULL) {
+						/* Netmap rx only */
+						rc = t4_alloc_irq(sc, irq, rid,
+						    t4_nm_intr, irq->nm_rxq, s);
+					}
+					if (irq->nm_rxq != NULL &&
+					    irq->rxq != NULL) {
+						/* NIC and Netmap rx */
+						rc = t4_alloc_irq(sc, irq, rid,
+						    t4_vi_intr, irq, s);
+					}
 #endif
-					rc = t4_alloc_irq(sc, irq, rid,
-					    t4_vi_intr, irq, s);
+					if (irq->rxq != NULL &&
+					    irq->nm_rxq == NULL) {
+						/* NIC rx only */
+						rc = t4_alloc_irq(sc, irq, rid,
+						    t4_intr, irq->rxq, s);
+					}
 					if (rc != 0)
 						return (rc);
 #ifdef RSS

Modified: head/sys/dev/cxgbe/t4_netmap.c
==============================================================================
--- head/sys/dev/cxgbe/t4_netmap.c	Sat Aug 11 04:26:29 2018	(r337608)
+++ head/sys/dev/cxgbe/t4_netmap.c	Sat Aug 11 04:55:47 2018	(r337609)
@@ -358,8 +358,6 @@ cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi
 	nm_set_native_flags(na);
 
 	for_each_nm_rxq(vi, i, nm_rxq) {
-		struct irq *irq = &sc->irq[vi->first_intr + i];
-
 		kring = na->rx_rings[nm_rxq->nid];
 		if (!nm_kring_pending_on(kring) ||
 		    nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID)
@@ -387,7 +385,7 @@ cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi
 		t4_write_reg(sc, sc->sge_kdoorbell_reg,
 		    nm_rxq->fl_db_val | V_PIDX(j));
 
-		atomic_cmpset_int(&irq->nm_state, NM_OFF, NM_ON);
+		atomic_cmpset_int(&nm_rxq->nm_state, NM_OFF, NM_ON);
 	}
 
 	for_each_nm_txq(vi, i, nm_txq) {
@@ -459,14 +457,12 @@ cxgbe_netmap_off(struct adapter *sc, struct vi_info *v
 		free_nm_txq_hwq(vi, nm_txq);
 	}
 	for_each_nm_rxq(vi, i, nm_rxq) {
-		struct irq *irq = &sc->irq[vi->first_intr + i];
-
 		kring = na->rx_rings[nm_rxq->nid];
 		if (!nm_kring_pending_off(kring) ||
 		    nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID)
 			continue;
 
-		while (!atomic_cmpset_int(&irq->nm_state, NM_ON, NM_OFF))
+		while (!atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_OFF))
 			pause("nmst", 1);
 
 		free_nm_rxq_hwq(vi, nm_rxq);
@@ -955,9 +951,8 @@ handle_nm_sge_egr_update(struct adapter *sc, struct if
 }
 
 void
-t4_nm_intr(void *arg)
+service_nm_rxq(struct sge_nm_rxq *nm_rxq)
 {
-	struct sge_nm_rxq *nm_rxq = arg;
 	struct vi_info *vi = nm_rxq->vi;
 	struct adapter *sc = vi->pi->adapter;
 	struct ifnet *ifp = vi->ifp;

Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c	Sat Aug 11 04:26:29 2018	(r337608)
+++ head/sys/dev/cxgbe/t4_sge.c	Sat Aug 11 04:55:47 2018	(r337609)
@@ -204,6 +204,7 @@ struct sgl {
 };
 
 static int service_iq(struct sge_iq *, int);
+static int service_iq_fl(struct sge_iq *, int);
 static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t);
 static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
 static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int);
@@ -1333,8 +1334,12 @@ t4_teardown_vi_queues(struct vi_info *vi)
 }
 
 /*
- * Deals with errors and the firmware event queue.  All data rx queues forward
- * their interrupt to the firmware event queue.
+ * Interrupt handler when the driver is using only 1 interrupt.  This is a very
+ * unusual scenario.
+ *
+ * a) Deals with errors, if any.
+ * b) Services firmware event queue, which is taking interrupts for all other
+ *    queues.
  */
 void
 t4_intr_all(void *arg)
@@ -1342,14 +1347,16 @@ t4_intr_all(void *arg)
 	struct adapter *sc = arg;
 	struct sge_iq *fwq = &sc->sge.fwq;
 
+	MPASS(sc->intr_count == 1);
+
 	t4_intr_err(arg);
-	if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) {
-		service_iq(fwq, 0);
-		atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE);
-	}
+	t4_intr_evt(fwq);
 }
 
-/* Deals with error interrupts */
+/*
+ * Interrupt handler for errors (installed directly when multiple interrupts are
+ * being used, or called by t4_intr_all).
+ */
 void
 t4_intr_err(void *arg)
 {
@@ -1359,6 +1366,10 @@ t4_intr_err(void *arg)
 	t4_slow_intr_handler(sc);
 }
 
+/*
+ * Interrupt handler for iq-only queues.  The firmware event queue is the only
+ * such queue right now.
+ */
 void
 t4_intr_evt(void *arg)
 {
@@ -1370,90 +1381,74 @@ t4_intr_evt(void *arg)
 	}
 }
 
+/*
+ * Interrupt handler for iq+fl queues.
+ */
 void
 t4_intr(void *arg)
 {
 	struct sge_iq *iq = arg;
 
 	if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
-		service_iq(iq, 0);
+		service_iq_fl(iq, 0);
 		atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
 	}
 }
 
+#ifdef DEV_NETMAP
+/*
+ * Interrupt handler for netmap rx queues.
+ */
 void
-t4_vi_intr(void *arg)
+t4_nm_intr(void *arg)
 {
-	struct irq *irq = arg;
+	struct sge_nm_rxq *nm_rxq = arg;
 
-#ifdef DEV_NETMAP
-	if (atomic_cmpset_int(&irq->nm_state, NM_ON, NM_BUSY)) {
-		t4_nm_intr(irq->nm_rxq);
-		atomic_cmpset_int(&irq->nm_state, NM_BUSY, NM_ON);
+	if (atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_BUSY)) {
+		service_nm_rxq(nm_rxq);
+		atomic_cmpset_int(&nm_rxq->nm_state, NM_BUSY, NM_ON);
 	}
-#endif
-	if (irq->rxq != NULL)
-		t4_intr(irq->rxq);
 }
 
-static inline int
-sort_before_lro(struct lro_ctrl *lro)
+/*
+ * Interrupt handler for vectors shared between NIC and netmap rx queues.
+ */
+void
+t4_vi_intr(void *arg)
 {
+	struct irq *irq = arg;
 
-	return (lro->lro_mbuf_max != 0);
+	MPASS(irq->nm_rxq != NULL);
+	t4_nm_intr(irq->nm_rxq);
+
+	MPASS(irq->rxq != NULL);
+	t4_intr(irq->rxq);
 }
+#endif
 
 /*
- * Deals with anything and everything on the given ingress queue.
+ * Deals with interrupts on an iq-only (no freelist) queue.
  */
 static int
 service_iq(struct sge_iq *iq, int budget)
 {
 	struct sge_iq *q;
-	struct sge_rxq *rxq = iq_to_rxq(iq);	/* Use iff iq is part of rxq */
-	struct sge_fl *fl;			/* Use iff IQ_HAS_FL */
 	struct adapter *sc = iq->adapter;
 	struct iq_desc *d = &iq->desc[iq->cidx];
 	int ndescs = 0, limit;
-	int rsp_type, refill;
+	int rsp_type;
 	uint32_t lq;
-	uint16_t fl_hw_cidx;
-	struct mbuf *m0;
 	STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
-#if defined(INET) || defined(INET6)
-	const struct timeval lro_timeout = {0, sc->lro_timeout};
-	struct lro_ctrl *lro = &rxq->lro;
-#endif
 
 	KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
+	KASSERT((iq->flags & IQ_HAS_FL) == 0,
+	    ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq,
+	    iq->flags));
+	MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
+	MPASS((iq->flags & IQ_LRO_ENABLED) == 0);
 
 	limit = budget ? budget : iq->qsize / 16;
 
-	if (iq->flags & IQ_HAS_FL) {
-		fl = &rxq->fl;
-		fl_hw_cidx = fl->hw_cidx;	/* stable snapshot */
-	} else {
-		fl = NULL;
-		fl_hw_cidx = 0;			/* to silence gcc warning */
-	}
-
-#if defined(INET) || defined(INET6)
-	if (iq->flags & IQ_ADJ_CREDIT) {
-		MPASS(sort_before_lro(lro));
-		iq->flags &= ~IQ_ADJ_CREDIT;
-		if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) {
-			tcp_lro_flush_all(lro);
-			t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) |
-			    V_INGRESSQID((u32)iq->cntxt_id) |
-			    V_SEINTARM(iq->intr_params));
-			return (0);
-		}
-		ndescs = 1;
-	}
-#else
-	MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
-#endif
-
 	/*
 	 * We always come back and check the descriptor ring for new indirect
 	 * interrupts and other responses after running a single handler.
@@ -1463,74 +1458,40 @@ service_iq(struct sge_iq *iq, int budget)
 
 			rmb();
 
-			refill = 0;
-			m0 = NULL;
 			rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
 			lq = be32toh(d->rsp.pldbuflen_qid);
 
 			switch (rsp_type) {
 			case X_RSPD_TYPE_FLBUF:
+				panic("%s: data for an iq (%p) with no freelist",
+				    __func__, iq);
 
-				KASSERT(iq->flags & IQ_HAS_FL,
-				    ("%s: data for an iq (%p) with no freelist",
-				    __func__, iq));
+				/* NOTREACHED */
 
-				m0 = get_fl_payload(sc, fl, lq);
-				if (__predict_false(m0 == NULL))
-					goto process_iql;
-				refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2;
-#ifdef T4_PKT_TIMESTAMP
-				/*
-				 * 60 bit timestamp for the payload is
-				 * *(uint64_t *)m0->m_pktdat.  Note that it is
-				 * in the leading free-space in the mbuf.  The
-				 * kernel can clobber it during a pullup,
-				 * m_copymdata, etc.  You need to make sure that
-				 * the mbuf reaches you unmolested if you care
-				 * about the timestamp.
-				 */
-				*(uint64_t *)m0->m_pktdat =
-				    be64toh(ctrl->u.last_flit) &
-				    0xfffffffffffffff;
-#endif
-
-				/* fall through */
-
 			case X_RSPD_TYPE_CPL:
 				KASSERT(d->rss.opcode < NUM_CPL_CMDS,
 				    ("%s: bad opcode %02x.", __func__,
 				    d->rss.opcode));
-				t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
+				t4_cpl_handler[d->rss.opcode](iq, &d->rss, NULL);
 				break;
 
 			case X_RSPD_TYPE_INTR:
-
 				/*
-				 * Interrupts should be forwarded only to queues
-				 * that are not forwarding their interrupts.
-				 * This means service_iq can recurse but only 1
-				 * level deep.
-				 */
-				KASSERT(budget == 0,
-				    ("%s: budget %u, rsp_type %u", __func__,
-				    budget, rsp_type));
-
-				/*
 				 * There are 1K interrupt-capable queues (qids 0
 				 * through 1023).  A response type indicating a
 				 * forwarded interrupt with a qid >= 1K is an
 				 * iWARP async notification.
 				 */
-				if (lq >= 1024) {
-                                        t4_an_handler(iq, &d->rsp);
-                                        break;
-                                }
+				if (__predict_true(lq >= 1024)) {
+					t4_an_handler(iq, &d->rsp);
+					break;
+				}
 
 				q = sc->sge.iqmap[lq - sc->sge.iq_start -
 				    sc->sge.iq_base];
 				if (atomic_cmpset_int(&q->state, IQS_IDLE,
 				    IQS_BUSY)) {
-					if (service_iq(q, q->qsize / 16) == 0) {
+					if (service_iq_fl(q, q->qsize / 16) == 0) {
 						atomic_cmpset_int(&q->state,
 						    IQS_BUSY, IQS_IDLE);
 					} else {
@@ -1563,33 +1524,12 @@ service_iq(struct sge_iq *iq, int budget)
 				    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
 				ndescs = 0;
 
-#if defined(INET) || defined(INET6)
-				if (iq->flags & IQ_LRO_ENABLED &&
-				    !sort_before_lro(lro) &&
-				    sc->lro_timeout != 0) {
-					tcp_lro_flush_inactive(lro,
-					    &lro_timeout);
-				}
-#endif
-
 				if (budget) {
-					if (iq->flags & IQ_HAS_FL) {
-						FL_LOCK(fl);
-						refill_fl(sc, fl, 32);
-						FL_UNLOCK(fl);
-					}
 					return (EINPROGRESS);
 				}
 			}
-			if (refill) {
-				FL_LOCK(fl);
-				refill_fl(sc, fl, 32);
-				FL_UNLOCK(fl);
-				fl_hw_cidx = fl->hw_cidx;
-			}
 		}
 
-process_iql:
 		if (STAILQ_EMPTY(&iql))
 			break;
 
@@ -1599,13 +1539,168 @@ process_iql:
 		 */
 		q = STAILQ_FIRST(&iql);
 		STAILQ_REMOVE_HEAD(&iql, link);
-		if (service_iq(q, q->qsize / 8) == 0)
+		if (service_iq_fl(q, q->qsize / 8) == 0)
 			atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
 		else
 			STAILQ_INSERT_TAIL(&iql, q, link);
 	}
 
+	t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
+	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
+
+	return (0);
+}
+
+static inline int
+sort_before_lro(struct lro_ctrl *lro)
+{
+
+	return (lro->lro_mbuf_max != 0);
+}
+
+/*
+ * Deals with interrupts on an iq+fl queue.
+ */
+static int
+service_iq_fl(struct sge_iq *iq, int budget)
+{
+	struct sge_rxq *rxq = iq_to_rxq(iq);
+	struct sge_fl *fl;
+	struct adapter *sc = iq->adapter;
+	struct iq_desc *d = &iq->desc[iq->cidx];
+	int ndescs = 0, limit;
+	int rsp_type, refill, starved;
+	uint32_t lq;
+	uint16_t fl_hw_cidx;
+	struct mbuf *m0;
 #if defined(INET) || defined(INET6)
+	const struct timeval lro_timeout = {0, sc->lro_timeout};
+	struct lro_ctrl *lro = &rxq->lro;
+#endif
+
+	KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
+	MPASS(iq->flags & IQ_HAS_FL);
+
+	limit = budget ? budget : iq->qsize / 16;
+	fl = &rxq->fl;
+	fl_hw_cidx = fl->hw_cidx;	/* stable snapshot */
+
+#if defined(INET) || defined(INET6)
+	if (iq->flags & IQ_ADJ_CREDIT) {
+		MPASS(sort_before_lro(lro));
+		iq->flags &= ~IQ_ADJ_CREDIT;
+		if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) {
+			tcp_lro_flush_all(lro);
+			t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) |
+			    V_INGRESSQID((u32)iq->cntxt_id) |
+			    V_SEINTARM(iq->intr_params));
+			return (0);
+		}
+		ndescs = 1;
+	}
+#else
+	MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
+#endif
+
+	while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
+
+		rmb();
+
+		refill = 0;
+		m0 = NULL;
+		rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
+		lq = be32toh(d->rsp.pldbuflen_qid);
+
+		switch (rsp_type) {
+		case X_RSPD_TYPE_FLBUF:
+
+			m0 = get_fl_payload(sc, fl, lq);
+			if (__predict_false(m0 == NULL))
+				goto out;
+			refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2;
+#ifdef T4_PKT_TIMESTAMP
+			/*
+			 * 60 bit timestamp for the payload is
+			 * *(uint64_t *)m0->m_pktdat.  Note that it is
+			 * in the leading free-space in the mbuf.  The
+			 * kernel can clobber it during a pullup,
+			 * m_copymdata, etc.  You need to make sure that
+			 * the mbuf reaches you unmolested if you care
+			 * about the timestamp.
+			 */
+			*(uint64_t *)m0->m_pktdat =
+			    be64toh(ctrl->u.last_flit) & 0xfffffffffffffff;
+#endif
+
+			/* fall through */
+
+		case X_RSPD_TYPE_CPL:
+			KASSERT(d->rss.opcode < NUM_CPL_CMDS,
+			    ("%s: bad opcode %02x.", __func__, d->rss.opcode));
+			t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
+			break;
+
+		case X_RSPD_TYPE_INTR:
+
+			/*
+			 * There are 1K interrupt-capable queues (qids 0
+			 * through 1023).  A response type indicating a
+			 * forwarded interrupt with a qid >= 1K is an
+			 * iWARP async notification.  That is the only
+			 * acceptable indirect interrupt on this queue.
+			 */
+			if (__predict_false(lq < 1024)) {
+				panic("%s: indirect interrupt on iq_fl %p "
+				    "with qid %u", __func__, iq, lq);
+			}
+
+			t4_an_handler(iq, &d->rsp);
+			break;
+
+		default:
+			KASSERT(0, ("%s: illegal response type %d on iq %p",
+			    __func__, rsp_type, iq));
+			log(LOG_ERR, "%s: illegal response type %d on iq %p",
+			    device_get_nameunit(sc->dev), rsp_type, iq);
+			break;
+		}
+
+		d++;
+		if (__predict_false(++iq->cidx == iq->sidx)) {
+			iq->cidx = 0;
+			iq->gen ^= F_RSPD_GEN;
+			d = &iq->desc[0];
+		}
+		if (__predict_false(++ndescs == limit)) {
+			t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
+			    V_INGRESSQID(iq->cntxt_id) |
+			    V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
+			ndescs = 0;
+
+#if defined(INET) || defined(INET6)
+			if (iq->flags & IQ_LRO_ENABLED &&
+			    !sort_before_lro(lro) &&
+			    sc->lro_timeout != 0) {
+				tcp_lro_flush_inactive(lro, &lro_timeout);
+			}
+#endif
+			if (budget) {
+				FL_LOCK(fl);
+				refill_fl(sc, fl, 32);
+				FL_UNLOCK(fl);
+
+				return (EINPROGRESS);
+			}
+		}
+		if (refill) {
+			FL_LOCK(fl);
+			refill_fl(sc, fl, 32);
+			FL_UNLOCK(fl);
+			fl_hw_cidx = fl->hw_cidx;
+		}
+	}
+out:
+#if defined(INET) || defined(INET6)
 	if (iq->flags & IQ_LRO_ENABLED) {
 		if (ndescs > 0 && lro->lro_mbuf_count > 8) {
 			MPASS(sort_before_lro(lro));
@@ -1621,15 +1716,11 @@ process_iql:
 	t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
 	    V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
 
-	if (iq->flags & IQ_HAS_FL) {
-		int starved;
-
-		FL_LOCK(fl);
-		starved = refill_fl(sc, fl, 64);
-		FL_UNLOCK(fl);
-		if (__predict_false(starved != 0))
-			add_fl_to_sfl(sc, fl);
-	}
+	FL_LOCK(fl);
+	starved = refill_fl(sc, fl, 64);
+	FL_UNLOCK(fl);
+	if (__predict_false(starved != 0))
+		add_fl_to_sfl(sc, fl);
 
 	return (0);
 }



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201808110455.w7B4tloY058893>