Date: Sun, 23 Sep 2007 06:35:50 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 126712 for review Message-ID: <200709230635.l8N6Zoqk024565@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=126712 Change 126712 by kmacy@kmacy_home:ethng on 2007/09/23 06:35:48 switch over to handling buffers in terms of buffer aggregates rather than mbuf chain - reduces our mbuf working set as well as reducing the number of cache misses when doing tx cleaning - some residual fixes remain to be made for compatibility Affected files ... .. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_adapter.h#18 edit .. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_main.c#20 edit .. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_sge.c#22 edit .. //depot/projects/ethng/src/sys/dev/cxgb/sys/mvec.h#4 edit .. //depot/projects/ethng/src/sys/dev/cxgb/sys/uipc_mvec.c#3 edit Differences ... ==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_adapter.h#18 (text+ko) ==== @@ -287,6 +287,9 @@ uint32_t txq_enqueued; unsigned long txq_frees; struct mtx lock; + struct sg_ent txq_sgl[TX_MAX_SEGS / 2 + 1]; + bus_dma_segment_t txq_segs[TX_MAX_SEGS]; + struct mbuf *txq_m_vec[TX_WR_COUNT_MAX]; #define TXQ_NAME_LEN 32 char lockbuf[TXQ_NAME_LEN]; }; @@ -535,12 +538,12 @@ void t3b_intr(void *data); void t3_intr_msi(void *data); void t3_intr_msix(void *data); -int t3_encap(struct sge_qset *, struct mbuf **, int, int *free); +int t3_encap(struct sge_qset *, struct mbuf **, int); int t3_sge_init_adapter(adapter_t *); int t3_sge_init_port(struct port_info *); void t3_sge_deinit_sw(adapter_t *); -int t3_free_tx_desc(struct sge_txq *q, int n, struct mbuf **m_vec, int m_vec_size, int *desc_reclaimed); +void t3_free_tx_desc(struct sge_txq *q, int n); void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro); ==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_main.c#20 (text+ko) ==== @@ -51,6 +51,7 @@ #include <sys/sysctl.h> #include <sys/queue.h> #include <sys/taskqueue.h> +#include <sys/proc.h> #include <net/bpf.h> #include <net/ethernet.h> @@ -213,11 +214,7 @@ * The driver uses an auto-queue algorithm by default. * To disable it and force a single queue-set per port, use singleq = 1. */ -#ifdef IFNET_MULTIQUEUE static int singleq = 0; -#else -static int singleq = 1; -#endif TUNABLE_INT("hw.cxgb.singleq", &singleq); SYSCTL_UINT(_hw_cxgb, OID_AUTO, singleq, CTLFLAG_RDTUN, &singleq, 0, "use a single queue-set per port"); @@ -639,14 +636,14 @@ { int i; + #ifdef IFNET_MULTIQUEUE cxgb_pcpu_shutdown_threads(sc); #endif - ADAPTER_LOCK(sc); - /* - * drops the lock - */ +/* + * drops the lock + */ cxgb_down_locked(sc); #ifdef MSI_SUPPORTED @@ -671,7 +668,7 @@ * Wait for last callout */ - tsleep(&sc, 0, "cxgb unload", 3*hz); + DELAY(hz*100); for (i = 0; i < (sc)->params.nports; ++i) { if (sc->portdev[i] != NULL) @@ -705,8 +702,6 @@ MTX_DESTROY(&sc->sge.reg_lock); MTX_DESTROY(&sc->elmer_lock); ADAPTER_LOCK_DEINIT(sc); - - return; } /** @@ -1027,6 +1022,9 @@ } ether_ifdetach(p->ifp); + printf("waiting for callout to stop ..."); + DELAY(10000); + printf("done\n"); /* * the lock may be acquired in ifdetach */ @@ -1586,11 +1584,6 @@ for (i = 0; i < sc->params.nports; i++) taskqueue_drain(sc->tq, &sc->port[i].timer_reclaim_task); } -#ifdef notyet - - if (sc->port[i].tq != NULL) -#endif - } static int @@ -1708,8 +1701,7 @@ device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id); t3_port_intr_enable(sc, p->port_id); - callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, - cxgb_tick, sc); + callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; @@ -1736,7 +1728,6 @@ ADAPTER_LOCK_ASSERT_NOTOWNED(p->adapter); ifp = p->ifp; - t3_port_intr_disable(p->adapter, p->port_id); ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); p->phy.ops->power_down(&p->phy, 1); @@ -1819,8 +1810,7 @@ if (ifp->if_drv_flags & IFF_DRV_RUNNING) { adapter_t *sc = p->adapter; - callout_reset(&sc->cxgb_tick_ch, - sc->params.stats_update_period * hz, + callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); } PORT_UNLOCK(p); @@ -1875,42 +1865,24 @@ cxgb_tx_common(struct ifnet *ifp, struct sge_qset *qs, uint32_t txmax) { struct sge_txq *txq; - int err, in_use_init, count, i, free; - struct mbuf *m_vec[TX_WR_COUNT_MAX]; + int err, in_use_init, count, i; + struct mbuf **m_vec; txq = &qs->txq[TXQ_ETH]; + m_vec = txq->txq_m_vec; in_use_init = txq->in_use; err = 0; while ((txq->in_use - in_use_init < txmax) && (txq->size > txq->in_use + TX_MAX_DESC)) { - free = 0; count = cxgb_dequeue_packet(ifp, txq, m_vec); + if (count == 0) break; -#ifdef notyet - /* - * Convert chain to M_IOVEC - */ - KASSERT((m->m_flags & M_IOVEC) == 0, ("IOVEC set too early")); - m0 = m; - if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES && - m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) { - if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) { - m = m0; - m_collapse(m, TX_MAX_SEGS, &m0); - } else - break; - } - m = m0; -#endif - if ((err = t3_encap(qs, m_vec, count, &free)) != 0) + ETHER_BPF_MTAP(ifp, m_vec[0]); + + if ((err = t3_encap(qs, m_vec, count)) != 0) break; txq->txq_enqueued += count; - for (i = 0; i < count; i++) - BPF_MTAP(ifp, m_vec[i]); - if (free) - m_freem(m_vec[0]); - } #ifndef IFNET_MULTIQUEUE if (__predict_false(err)) { @@ -2154,12 +2126,26 @@ cxgb_tick(void *arg) { adapter_t *sc = (adapter_t *)arg; + int i, running = 0; + + for_each_port(sc, i) { + + struct port_info *p = &sc->port[i]; + struct ifnet *ifp = p->ifp; + PORT_LOCK(p); + + if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) + running = 1; + PORT_UNLOCK(p); + } + if (running == 0) + return; + taskqueue_enqueue(sc->tq, &sc->tick_task); if (sc->open_device_map != 0) - callout_reset(&sc->cxgb_tick_ch, sc->params.stats_update_period * hz, - cxgb_tick, sc); + callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc); } static void ==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_sge.c#22 (text+ko) ==== @@ -57,18 +57,17 @@ #include <dev/pci/pcireg.h> #include <dev/pci/pcivar.h> +#include <vm/vm.h> +#include <vm/pmap.h> + #ifdef CONFIG_DEFINED #include <cxgb_include.h> +#include <sys/mvec.h> #else #include <dev/cxgb/cxgb_include.h> +#include <dev/cxgb/sys/mvec.h> #endif -#include <vm/vm.h> -#include <vm/pmap.h> - - - - uint32_t collapse_free = 0; uint32_t mb_free_vec_free = 0; int txq_fills = 0; @@ -76,14 +75,15 @@ static int bogus_imm = 0; #ifndef DISABLE_MBUF_IOVEC static int recycle_enable = 1; +#endif extern int cxgb_txq_mbuf_ring_size; -#endif + #define USE_GTS 0 #define SGE_RX_SM_BUF_SIZE 1536 #define SGE_RX_DROP_THRES 16 -#define SGE_RX_COPY_THRES MHLEN +#define SGE_RX_COPY_THRES 128 /* * Period of the Tx buffer reclaim timer. This timer does not need to run @@ -134,9 +134,7 @@ #define RSPQ_SOP_EOP G_RSPD_SOP_EOP(F_RSPD_SOP|F_RSPD_EOP) struct tx_sw_desc { /* SW state per Tx descriptor */ - struct mbuf *m[TX_WR_COUNT_MAX]; - - int count; + struct mbuf_iovec mi; bus_dmamap_t map; int flags; }; @@ -211,16 +209,15 @@ static __inline int reclaim_completed_tx(struct sge_txq *q, int nbufs, struct mbuf **mvec) { - int reclaimed, reclaim = desc_reclaimable(q); - int n = 0; + int reclaim = desc_reclaimable(q); mtx_assert(&q->lock, MA_OWNED); if (reclaim > 0) { - n = t3_free_tx_desc(q, reclaim, mvec, nbufs, &reclaimed); - q->cleaned += reclaimed; - q->in_use -= reclaimed; + t3_free_tx_desc(q, reclaim); + q->cleaned += reclaim; + q->in_use -= reclaim; } - return (n); + return (reclaim); } /** @@ -315,7 +312,7 @@ if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP) return (0); - m = m_gethdr(M_NOWAIT, MT_DATA); + m = m_gethdr(M_DONTWAIT, MT_DATA); len = G_RSPD_LEN(ntohl(resp->len_cq)); if (m) { @@ -423,11 +420,15 @@ q->polling = adap->params.rev > 0; - if (adap->params.nports > 2) + if (adap->params.nports > 2) { + q->coalesce_nsecs = 50000; + } else { +#ifdef INVARIANTS q->coalesce_nsecs = 50000; - else +#else q->coalesce_nsecs = 5000; - +#endif + } q->rspq_size = RSPQ_Q_SIZE; q->fl_size = FL_Q_SIZE; q->jumbo_size = JUMBO_Q_SIZE; @@ -832,6 +833,7 @@ callout_init(&sc->sge_timer_ch, CALLOUT_MPSAFE); callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc); + mi_init(); return (0); } @@ -853,6 +855,8 @@ for (i = 0; i < sc->params.nports; i++) if (sc->port[i].tq != NULL) taskqueue_drain(sc->port[i].tq, &sc->port[i].timer_reclaim_task); + + mi_deinit(); } /** @@ -876,7 +880,7 @@ static __inline void sge_txq_reclaim_(struct sge_txq *txq) { - int reclaimable, i, n; + int reclaimable, n; struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; struct port_info *pi; @@ -890,10 +894,14 @@ } if (n == 0) return; - +#if 0 + { + int i; for (i = 0; i < n; i++) { m_freem_vec(m_vec[i]); } + } +#endif if (pi && pi->ifp->if_drv_flags & IFF_DRV_OACTIVE && txq->size - txq->in_use >= TX_START_MAX_DESC) { txq_fills++; @@ -1018,7 +1026,7 @@ * packet. Ethernet packets require addition of WR and CPL headers. */ static __inline unsigned int -calc_tx_descs(const struct mbuf *m, int nsegs) +calc_tx_descs(const struct mbuf *m, int nsegs, int tsoinfo) { unsigned int flits; @@ -1027,7 +1035,7 @@ flits = sgl_len(nsegs) + 2; #ifdef TSO_SUPPORTED - if (m->m_pkthdr.csum_flags & (CSUM_TSO)) + if (tsoinfo) flits++; #endif return flits_to_desc(flits); @@ -1038,25 +1046,24 @@ struct tx_sw_desc *txsd, bus_dma_segment_t *segs, int *nsegs) { struct mbuf *m0; - int err, pktlen; + int err, pktlen, pass = 0; +retry: + err = 0; m0 = *m; pktlen = m0->m_pkthdr.len; +#if defined(__i386__) || defined(__amd64__) + if (busdma_map_sg_collapse(m, segs, nsegs) == 0) { + goto done; + } else +#endif + err = bus_dmamap_load_mvec_sg(txq->entry_tag, txsd->map, m0, segs, nsegs, 0); - err = bus_dmamap_load_mvec_sg(txq->entry_tag, txsd->map, m0, segs, nsegs, 0); -#ifdef DEBUG - if (err) { - int n = 0; - struct mbuf *mtmp = m0; - while(mtmp) { - n++; - mtmp = mtmp->m_next; - } - printf("map_mbufs: bus_dmamap_load_mbuf_sg failed with %d - pkthdr.len==%d nmbufs=%d\n", - err, m0->m_pkthdr.len, n); + if (err == 0) { + goto done; } -#endif - if (err == EFBIG) { + if (err == EFBIG && pass == 0) { + pass = 1; /* Too many segments, try to defrag */ m0 = m_defrag(m0, M_DONTWAIT); if (m0 == NULL) { @@ -1065,22 +1072,20 @@ return (ENOBUFS); } *m = m0; - err = bus_dmamap_load_mbuf_sg(txq->entry_tag, txsd->map, m0, segs, nsegs, 0); - } - - if (err == ENOMEM) { + goto retry; + } else if (err == ENOMEM) { return (err); - } - - if (err) { + } if (err) { if (cxgb_debug) printf("map failure err=%d pktlen=%d\n", err, pktlen); - m_freem_vec(m0); + m_freem(m0); *m = NULL; return (err); } - +done: +#if !defined(__i386__) && !defined(__amd64__) bus_dmamap_sync(txq->entry_tag, txsd->map, BUS_DMASYNC_PREWRITE); +#endif txsd->flags |= TX_SW_DESC_MAPPED; return (0); @@ -1101,12 +1106,19 @@ { int i, idx; - for (idx = 0, i = 0; i < nsegs; i++, idx ^= 1) { + for (idx = 0, i = 0; i < nsegs; i++) { + /* + * firmware doesn't like empty segments + */ + if (segs[i].ds_len == 0) + continue; if (i && idx == 0) ++sgp; - + sgp->len[idx] = htobe32(segs[i].ds_len); sgp->addr[idx] = htobe64(segs[i].ds_addr); + + idx ^= 1; } if (idx) @@ -1225,8 +1237,7 @@ * is freed all clusters will be freed * with it */ - txsd->m[0] = NULL; - txsd->count = 0; + txsd->mi.mi_base = NULL; wrp = (struct work_request_hdr *)txd; wrp->wr_hi = htonl(V_WR_DATATYPE(1) | V_WR_SGLSFLT(1)) | wr_hi; @@ -1243,7 +1254,13 @@ } } - +static void +dump_mi(struct mbuf_iovec *mi) +{ + DPRINTF("mi_flags=0x%08x mi_data=%p mi_len=%d mi_type=%d\n", + mi->mi_flags, mi->mi_base + mi->mi_offset, mi->mi_len, mi->mi_type); +} + /* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */ #define TCPPKTHDRSIZE (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + 20 + 20) @@ -1253,12 +1270,19 @@ if ((m)->m_flags & M_VLANTAG) \ cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \ } while (0) + +#define GET_VTAG_MI(cntrl, mi) \ +do { \ + if ((mi)->mi_flags & M_VLANTAG) \ + cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((mi)->mi_ether_vtag); \ +} while (0) #else #define GET_VTAG(cntrl, m) +#define GET_VTAG_MI(cntrl, m) #endif int -t3_encap(struct sge_qset *qs, struct mbuf **m, int count, int *free) +t3_encap(struct sge_qset *qs, struct mbuf **m, int count) { adapter_t *sc; struct mbuf *m0; @@ -1270,16 +1294,14 @@ struct work_request_hdr *wrp; struct tx_sw_desc *txsd; - struct sg_ent *sgp, sgl[TX_MAX_SEGS / 2 + 1]; - bus_dma_segment_t segs[TX_MAX_SEGS]; + struct sg_ent *sgp, *sgl; + bus_dma_segment_t *segs; uint32_t wr_hi, wr_lo, sgl_flits; struct tx_desc *txd; - - -#if defined(IFNET_MULTIQUEUE) && defined(STRICT_AFFINITY) - KASSERT(qs->qs_cpuid == curcpu, ("cpu qset mismatch cpuid=%d curcpu=%d", qs->qs_cpuid, curcpu)); -#endif + struct mbuf_vec *mv; + struct mbuf_iovec *mi; + DPRINTF("t3_encap cpu=%d ", curcpu); pi = qs->port; @@ -1287,47 +1309,62 @@ txq = &qs->txq[TXQ_ETH]; txsd = &txq->sdesc[txq->pidx]; txd = &txq->desc[txq->pidx]; - + sgl = txq->txq_sgl; + segs = txq->txq_segs; + m0 = *m; DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset); DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan); - /* - * XXX handle checksum, TSO, and VLAN here - * - */ + cntrl = V_TXPKT_INTF(pi->txpkt_intf); - - /* - * XXX need to add VLAN support for 6.x - */ +/* + * XXX need to add VLAN support for 6.x + */ #ifdef VLAN_SUPPORTED - if (m[0]->m_pkthdr.csum_flags & (CSUM_TSO)) - tso_info = V_LSO_MSS(m[0]->m_pkthdr.tso_segsz); + if (m0->m_pkthdr.csum_flags & (CSUM_TSO)) + tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); #endif - txsd->count = count; if (count > 1) { + if ((err = busdma_map_sg_vec(m, &m0, segs, count))) + return (err); + nsegs = count; + } else if ((err = busdma_map_sg_collapse(&m0, segs, &nsegs))) { + printf("failed ... err=%d\n", err); + return (err); + } + if (m0->m_type == MT_DATA) { + DPRINTF("mbuf type=%d tags:%d head=%p", m0->m_type, !SLIST_EMPTY(&m0->m_pkthdr.tags), + SLIST_FIRST(&m0->m_pkthdr.tags)); + mi_collapse_mbuf(&txsd->mi, m0); + } else { + mv = mtomv(m0); + txsd->mi.mi_flags = m0->m_flags; + txsd->mi.mi_base = (caddr_t)m0; + txsd->mi.mi_type = m0->m_type; + txsd->mi.mi_len = m0->m_pkthdr.len; + } + mi = &txsd->mi; + dump_mi(&txsd->mi); + + if (count > 1) { struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd; - int i; - + int i, fidx; + wrp = (struct work_request_hdr *)txd; - + flits = count*2 + 1; txq_prod(txq, 1, &txqs); - for (i = 0; i < count; i++) { + for (fidx = 1, i = 0; i < count; i++, mi++, fidx += 2) { struct cpl_tx_pkt_batch_entry *cbe = &cpl_batch->pkt_entry[i]; - - cntrl = V_TXPKT_INTF(pi->port_id); - GET_VTAG(cntrl, m[i]); - cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT) | (1 << 24); + + cntrl = V_TXPKT_INTF(pi->txpkt_intf); + GET_VTAG_MI(cntrl, mi); + cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); cbe->cntrl = htonl(cntrl); - cbe->len = htonl(m[i]->m_pkthdr.len | 0x80000000); - m_set_priority(m[i], txqs.pidx); - txsd->m[i] = m[i]; - /* - * XXX - NOT PORTABLE outside of x86 - */ - cbe->addr = htobe64(pmap_kextract(mtod(m[i], vm_offset_t))); + cbe->len = htonl(mi->mi_len | 0x80000000); + txd->flit[fidx] |= htobe64(1 << 24); + cbe->addr = htobe64(segs[i].ds_addr); } wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | @@ -1345,24 +1382,27 @@ struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd; struct ip *ip; struct tcphdr *tcp; - char *pkthdr, tmp[TCPPKTHDRSIZE]; /* is this too large for the stack? */ + char *pkthdr; txd->flit[2] = 0; - m0 = m[0]; - GET_VTAG(cntrl, m0); + GET_VTAG_MI(cntrl, mi); cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO); hdr->cntrl = htonl(cntrl); mlen = m0->m_pkthdr.len; hdr->len = htonl(mlen | 0x80000000); - if (__predict_false(m0->m_len < TCPPKTHDRSIZE)) { - pkthdr = &tmp[0]; - m_copydata(m0, 0, TCPPKTHDRSIZE, pkthdr); - } else { - pkthdr = mtod(m0, char *); - } + DPRINTF("tso buf len=%d\n", mlen); + if (__predict_false(mi->mi_len < TCPPKTHDRSIZE)) { + /* + * XXX + * + */ + pkthdr = NULL; + panic("discontig packet - fixxorz"); + } else + pkthdr = m0->m_data; - if (__predict_false(m0->m_flags & M_VLANTAG)) { + if (__predict_false(mi->mi_flags & M_VLANTAG)) { eth_type = CPL_ETH_II_VLAN; ip = (struct ip *)(pkthdr + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); @@ -1380,9 +1420,8 @@ flits = 3; } else { struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd; - - m0 = m[0]; - GET_VTAG(cntrl, m0); + + GET_VTAG_MI(cntrl, mi); cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); cpl->cntrl = htonl(cntrl); mlen = m0->m_pkthdr.len; @@ -1390,13 +1429,23 @@ if (mlen <= WR_LEN - sizeof(*cpl)) { txq_prod(txq, 1, &txqs); - txq->sdesc[txqs.pidx].count = 0; - if (m0->m_len == m0->m_pkthdr.len) - memcpy(&txd->flit[2], mtod(m0, uint8_t *), mlen); - else + DPRINTF("mlen==%d max=%ld\n", mlen, (WR_LEN - sizeof(*cpl))); + if (mi->mi_type != MT_IOVEC && + mi->mi_type != MT_CLIOVEC) + memcpy(&txd->flit[2], mi_data(mi), mlen); + else { + /* + * XXX mbuf_iovec + */ +#if 0 m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); - *free = 1; +#endif + printf("bailing on m_copydata\n"); + } + m_freem_iovec(&txsd->mi); + txsd->mi.mi_base = NULL; + flits = (mlen + 7) / 8 + 2; cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | @@ -1407,17 +1456,24 @@ wr_gen2(txd, txqs.gen); check_ring_tx_db(sc, txq); + DPRINTF("pio buf\n"); return (0); } + DPRINTF("regular buf\n"); flits = 2; } wrp = (struct work_request_hdr *)txd; - - if ((err = busdma_map_mbufs(m, txq, txsd, segs, &nsegs)) != 0) { + +#ifdef nomore + /* + * XXX need to move into one of the helper routines above + * + */ + if ((err = busdma_map_mbufs(m, txq, txsd, segs, &nsegs)) != 0) return (err); - } m0 = *m; - ndesc = calc_tx_descs(m0, nsegs); +#endif + ndesc = calc_tx_descs(mi, nsegs, tso_info); sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl; make_sgl(sgp, segs, nsegs); @@ -1426,16 +1482,16 @@ DPRINTF("make_sgl success nsegs==%d ndesc==%d\n", nsegs, ndesc); txq_prod(txq, ndesc, &txqs); - txsd = &txq->sdesc[txqs.pidx]; wr_hi = htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | txqs.compl); wr_lo = htonl(V_WR_TID(txq->token)); - txsd->count = count; - txsd->m[0] = m0; - m_set_priority(m0, txqs.pidx); - write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo); check_ring_tx_db(pi->adapter, txq); + if ((m0->m_type == MT_DATA) && (m0->m_flags & M_EXT)) { + m0->m_flags = 0; + m_free(m0); + } + return (0); } @@ -1798,13 +1854,11 @@ * * Returns number of buffers of reclaimed */ -int -t3_free_tx_desc(struct sge_txq *q, int reclaimable, struct mbuf **m_vec, - int m_vec_size, int *desc_reclaimed) +void +t3_free_tx_desc(struct sge_txq *q, int reclaimable) { struct tx_sw_desc *txsd; unsigned int cidx; - int i, iter, reclaimed, nbufs; #ifdef T3_TRACE T3_TRACE2(sc->tb[q->cntxt_id & 7], @@ -1812,30 +1866,29 @@ #endif cidx = q->cidx; txsd = &q->sdesc[cidx]; - prefetch(txsd); - reclaimed = nbufs = 0; - for (iter = reclaimed = 0; reclaimed < reclaimable; iter++) { - if ((iter & 0x1) == 0) { - prefetch(txsd + 1); - prefetch(txsd + 2); - prefetch(txsd + 3); - prefetch(txsd + 4); - } + DPRINTF("reclaiming %d WR\n", reclaimable); + while (reclaimable--) { DPRINTF("cidx=%d d=%p\n", cidx, txsd); - if (txsd->count > 0) { - if (nbufs + txsd->count > m_vec_size) - break; + if (txsd->mi.mi_base != NULL) { if (txsd->flags & TX_SW_DESC_MAPPED) { bus_dmamap_unload(q->entry_tag, txsd->map); txsd->flags &= ~TX_SW_DESC_MAPPED; } - for (i = 0; i < txsd->count; i++, nbufs++) { - prefetch(txsd->m[i]); - m_vec[nbufs] = txsd->m[i]; - } - txsd->count = 0; +#ifdef notyet + critical_enter(); + /* + * transfer mbuf_vec contents to cpu local ring + * XXX + * + */ + critical_exit(); +#else + m_freem_iovec(&txsd->mi); +#endif -#ifdef DIAGNOSTIC + txsd->mi.mi_base = NULL; + +#if defined(DIAGNOSTIC) && 0 if (m_get_priority(txsd->m[0]) != cidx) printf("pri=%d cidx=%d\n", (int)m_get_priority(txsd->m[0]), cidx); #endif @@ -1848,12 +1901,9 @@ cidx = 0; txsd = q->sdesc; } - reclaimed++; } - *desc_reclaimed = reclaimed; q->cidx = cidx; - return (nbufs); } /** @@ -1905,8 +1955,7 @@ struct txq_state txqs; if (immediate(m)) { - q->sdesc[pidx].m[0] = NULL; - q->sdesc[pidx].count = 0; + q->sdesc[pidx].mi.mi_base = NULL; write_imm(d, m, m->m_len, gen); return; } @@ -1970,7 +2019,7 @@ unsigned int pidx, gen; struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; bus_dma_segment_t segs[TX_MAX_SEGS]; - int i, cleaned; + int cleaned; struct tx_sw_desc *stx = &q->sdesc[q->pidx]; mtx_lock(&q->lock); @@ -2009,10 +2058,14 @@ write_ofld_wr(adap, m, q, pidx, gen, ndesc, segs, nsegs); check_ring_tx_db(adap, q); - +#if 0 + { + int i; for (i = 0; i < cleaned; i++) { m_freem_vec(m_vec[i]); } + } +#endif return (0); } @@ -2025,15 +2078,14 @@ static void restart_offloadq(void *data, int npending) { - struct mbuf *m; struct sge_qset *qs = data; struct sge_txq *q = &qs->txq[TXQ_OFLD]; adapter_t *adap = qs->port->adapter; struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; bus_dma_segment_t segs[TX_MAX_SEGS]; - int nsegs, i, cleaned; struct tx_sw_desc *stx = &q->sdesc[q->pidx]; + int nsegs, cleaned; mtx_lock(&q->lock); again: cleaned = reclaim_completed_tx(q, TX_CLEAN_MAX_DESC, m_vec); @@ -2076,10 +2128,12 @@ #endif t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); +#if 0 for (i = 0; i < cleaned; i++) { m_freem_vec(m_vec[i]); } +#endif } /** @@ -2408,7 +2462,9 @@ m->m_pkthdr.rcvif = ifp; m->m_pkthdr.header = mtod(m, uint8_t *) + sizeof(*cpl) + ethpad; +#ifndef DISABLE_MBUF_IOVEC m_explode(m); +#endif /* * adjust after conversion to mbuf chain */ @@ -2677,8 +2733,7 @@ } else if (flags & F_RSPD_IMM_DATA_VALID) { #ifdef DISABLE_MBUF_IOVEC - if (cxgb_debug) - printf("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", r->rss_hdr.opcode, rspq->cidx); + printf("IMM DATA VALID opcode=0x%x rspq->cidx=%d\n", r->rss_hdr.opcode, rspq->cidx); if(get_imm_packet(adap, r, &rspq->rspq_mh) == 0) { rspq->next_holdoff = NOMEM_INTR_DELAY; @@ -2692,7 +2747,7 @@ if (rspq->rspq_mbuf == NULL) rspq->rspq_mbuf = m_gethdr(M_DONTWAIT, MT_DATA); - else + else m = m_gethdr(M_DONTWAIT, MT_DATA); /* @@ -2713,29 +2768,27 @@ #ifdef DISABLE_MBUF_IOVEC struct mbuf *m; - m = m_gethdr(M_NOWAIT, MT_DATA); + m = m_gethdr(M_DONTWAIT, MT_DATA); if (m == NULL) { log(LOG_WARNING, "failed to get mbuf for packet\n"); break; + } else { + m->m_next = m->m_nextpkt = NULL; } eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mh, r, m); #else - if (rspq->rspq_mbuf == NULL) + if (rspq->rspq_mbuf == NULL) rspq->rspq_mbuf = m_gethdr(M_DONTWAIT, MT_DATA); if (rspq->rspq_mbuf == NULL) { + rspq->next_holdoff = NOMEM_INTR_DELAY; log(LOG_WARNING, "failed to get mbuf for packet\n"); break; + } else { + rspq->rspq_mbuf->m_pkthdr.rss_hash = rss_hash; + rspq->rspq_mbuf->m_next = rspq->rspq_mbuf->m_nextpkt = NULL; } - if (rspq->rspq_mbuf == NULL) { - if ((rspq->rspq_mbuf = m_gethdr(M_DONTWAIT, MT_DATA)) == NULL) { - DPRINTF("0x%x:%d\n", rss_hash & ((1<<7)-1), curcpu); - rspq->next_holdoff = NOMEM_INTR_DELAY; - break; - } else - rspq->rspq_mbuf->m_pkthdr.rss_hash = rss_hash; - } eop = get_packet(adap, drop_thresh, qs, rspq->rspq_mbuf, r); #endif ethpad = 2; @@ -2743,7 +2796,6 @@ DPRINTF("pure response\n"); rspq->pure_rsps++; } - if (flags & RSPD_CTRL_MASK) { sleeping |= flags & RSPD_GTS_MASK; handle_rsp_cntrl_info(qs, flags); @@ -2757,7 +2809,6 @@ rspq->gen ^= 1; r = rspq->desc; } >>> TRUNCATED FOR MAIL (1000 lines) <<<
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200709230635.l8N6Zoqk024565>