Date: Mon, 17 Sep 2007 20:40:31 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 126532 for review Message-ID: <200709172040.l8HKeVcQ078853@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=126532 Change 126532 by kmacy@kmacy:storage:toestack on 2007/09/17 20:39:57 fix compile errors from IFC Affected files ... .. //depot/projects/toestack/sys/dev/cxgb/cxgb_main.c#6 edit .. //depot/projects/toestack/sys/dev/cxgb/cxgb_offload.c#4 edit .. //depot/projects/toestack/sys/dev/cxgb/cxgb_sge.c#3 edit .. //depot/projects/toestack/sys/dev/cxgb/ulp/toecore/toedev.c#2 edit Differences ... ==== //depot/projects/toestack/sys/dev/cxgb/cxgb_main.c#6 (text+ko) ==== @@ -51,6 +51,7 @@ #include <sys/sysctl.h> #include <sys/queue.h> #include <sys/taskqueue.h> +#include <sys/proc.h> #include <net/bpf.h> #include <net/ethernet.h> ==== //depot/projects/toestack/sys/dev/cxgb/cxgb_offload.c#4 (text+ko) ==== @@ -109,7 +109,7 @@ TAILQ_INSERT_TAIL(&client_list, client, client_entry); if (client->add) { - TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) { + TAILQ_FOREACH(tdev, &ofld_dev_list, entry) { if (offload_activated(tdev)) client->add(tdev); } @@ -133,7 +133,7 @@ TAILQ_REMOVE(&client_list, client, client_entry); if (client->remove) { - TAILQ_FOREACH(tdev, &ofld_dev_list, ofld_entry) { + TAILQ_FOREACH(tdev, &ofld_dev_list, entry) { if (offload_activated(tdev)) client->remove(tdev); } @@ -481,7 +481,7 @@ cxgb_set_dummy_ops(struct toedev *dev) { dev->recv = rx_offload_blackhole; - dev->neigh_update = dummy_neigh_update; + dev->arp_update = dummy_neigh_update; } /* @@ -1410,7 +1410,7 @@ TOE_DATA(dev) = t; dev->recv = process_rx; - dev->neigh_update = t3_l2t_update; + dev->arp_update = t3_l2t_update; #if 0 offload_proc_dev_setup(dev); #endif @@ -1460,7 +1460,7 @@ mtx_lock(&cxgb_db_lock); snprintf(tdev->name, sizeof(tdev->name), "ofld_dev%d", unit++); - TAILQ_INSERT_TAIL(&ofld_dev_list, tdev, ofld_entry); + TAILQ_INSERT_TAIL(&ofld_dev_list, tdev, entry); mtx_unlock(&cxgb_db_lock); } @@ -1468,7 +1468,7 @@ unregister_tdev(struct toedev *tdev) { mtx_lock(&cxgb_db_lock); - TAILQ_REMOVE(&ofld_dev_list, tdev, ofld_entry); + TAILQ_REMOVE(&ofld_dev_list, tdev, entry); mtx_unlock(&cxgb_db_lock); } @@ -1498,7 +1498,7 @@ offload_proc_dev_exit(tdev); #endif tdev->recv = NULL; - tdev->neigh_update = NULL; + tdev->arp_update = NULL; unregister_tdev(tdev); } @@ -1620,7 +1620,7 @@ len = sprintf(buf, "Device Interfaces\n"); mtx_lock(&cxgb_db_lock); - TAILQ_FOREACH(dev, &ofld_dev_list, ofld_entry) { + TAILQ_FOREACH(dev, &ofld_dev_list, entry) { len += sprintf(buf + len, "%-16s", dev->name); read_lock(&dev_base_lock); for (ndev = dev_base; ndev; ndev = ndev->next) { ==== //depot/projects/toestack/sys/dev/cxgb/cxgb_sge.c#3 (text+ko) ==== @@ -83,7 +83,7 @@ #define SGE_RX_SM_BUF_SIZE 1536 #define SGE_RX_DROP_THRES 16 -#define SGE_RX_COPY_THRES 128 +#define SGE_RX_COPY_THRES MHLEN /* * Period of the Tx buffer reclaim timer. This timer does not need to run @@ -135,17 +135,25 @@ struct tx_sw_desc { /* SW state per Tx descriptor */ struct mbuf *m[TX_WR_COUNT_MAX]; + int count; bus_dmamap_t map; int flags; }; struct rx_sw_desc { /* SW state per Rx descriptor */ - void *cl; - bus_dmamap_t map; - int flags; + union { + void *cl; + uint32_t *ref; + } u; + uint8_t *data; + bus_dmamap_t map; + int flags; }; +#define rxsd_cl u.cl +#define rxsd_ref u.ref + struct txq_state { unsigned int compl; unsigned int gen; @@ -356,8 +364,12 @@ memcpy(mtod(m, uint8_t *), resp->imm_data, len); break; case RSPQ_EOP: - memcpy(cl, resp->imm_data, len); - m_iovappend(m, cl, MSIZE, len, 0); + memcpy(cl, resp->imm_data, len); + /* + * XXX + */ + panic("bad append"); + m_iovappend(m, cl, MSIZE, len, 0, NULL); break; default: bogus_imm++; @@ -507,6 +519,7 @@ qs->rspq.polling = 0 /* p->polling */; } +#if !defined(__i386__) && !defined(__amd64__) static void refill_fl_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) { @@ -517,7 +530,7 @@ cb_arg->nseg = nseg; } - +#endif /** * refill_fl - refill an SGE free-buffer list * @sc: the controller softc @@ -534,6 +547,7 @@ struct rx_desc *d = &q->desc[q->pidx]; struct refill_fl_cb_arg cb_arg; void *cl; + uint32_t *ref; int err; cb_arg.error = 0; @@ -541,10 +555,12 @@ /* * We only allocate a cluster, mbuf allocation happens after rx */ - if ((cl = m_cljget(NULL, M_DONTWAIT, q->buf_size)) == NULL) { + if ((cl = m_cljget(NULL, M_DONTWAIT, q->zone)) == NULL) { log(LOG_WARNING, "Failed to allocate cluster\n"); goto done; } + ref = cl; + if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) { if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) { log(LOG_WARNING, "bus_dmamap_create failed %d\n", err); @@ -553,7 +569,8 @@ } sd->flags |= RX_SW_DESC_MAP_CREATED; } - err = bus_dmamap_load(q->entry_tag, sd->map, cl, q->buf_size, +#if !defined(__i386__) && !defined(__amd64__) + err = bus_dmamap_load(q->entry_tag, sd->map, (uint32_t *)cl + 1, q->buf_size, refill_fl_cb, &cb_arg, 0); if (err != 0 || cb_arg.error) { @@ -563,9 +580,12 @@ */ return; } - +#else + cb_arg.seg.ds_addr = pmap_kextract((vm_offset_t)((uint32_t *)cl + 1)); +#endif sd->flags |= RX_SW_DESC_INUSE; - sd->cl = cl; + sd->rxsd_cl = cl; + sd->data = (uint8_t *)(sd->rxsd_ref + 1); d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff); d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff); d->len_gen = htobe32(V_FLD_GEN1(q->gen)); @@ -607,9 +627,9 @@ if (d->flags & RX_SW_DESC_INUSE) { bus_dmamap_unload(q->entry_tag, d->map); bus_dmamap_destroy(q->entry_tag, d->map); - uma_zfree(q->zone, d->cl); + uma_zfree(q->zone, d->rxsd_cl); } - d->cl = NULL; + d->rxsd_cl = NULL; if (++cidx == q->size) cidx = 0; } @@ -621,11 +641,17 @@ refill_fl(adap, fl, min(16U, fl->size - fl->credits)); } +static __inline void +__refill_fl_lt(adapter_t *adap, struct sge_fl *fl, int max) +{ + if ((fl->size - fl->credits) < max) + refill_fl(adap, fl, min(max, fl->size - fl->credits)); +} + void refill_fl_service(adapter_t *adap, struct sge_fl *fl) { - return; /* XXX no-op for now */ - __refill_fl(adap, fl); + __refill_fl_lt(adap, fl, 512); } #ifndef DISABLE_MBUF_IOVEC @@ -759,7 +785,7 @@ { adapter_t *sc = arg; #ifndef IFNET_MULTIQUEUE - struct port_info *p; + struct port_info *pi; struct sge_qset *qs; struct sge_txq *txq; int i, j; @@ -775,7 +801,7 @@ (qs->fl[1].credits < qs->fl[1].size)); if (reclaim_eth || reclaim_ofl || refill_rx) { p = &sc->port[i]; - taskqueue_enqueue(p->tq, &p->timer_reclaim_task); + taskqueue_enqueue(pi->tq, &pi->timer_reclaim_task); break; } } @@ -810,9 +836,9 @@ } int -t3_sge_init_port(struct port_info *p) +t3_sge_init_port(struct port_info *pi) { - TASK_INIT(&p->timer_reclaim_task, 0, sge_timer_reclaim, p); + TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi); return (0); } @@ -852,9 +878,9 @@ { int reclaimable, i, n; struct mbuf *m_vec[TX_CLEAN_MAX_DESC]; - struct port_info *p; + struct port_info *pi; - p = txq->port; + pi = txq->port; reclaim_more: n = 0; reclaimable = desc_reclaimable(txq); @@ -868,11 +894,11 @@ for (i = 0; i < n; i++) { m_freem_vec(m_vec[i]); } - if (p && p->ifp->if_drv_flags & IFF_DRV_OACTIVE && + if (pi && pi->ifp->if_drv_flags & IFF_DRV_OACTIVE && txq->size - txq->in_use >= TX_START_MAX_DESC) { txq_fills++; - p->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; - taskqueue_enqueue(p->tq, &p->start_task); + pi->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + taskqueue_enqueue(pi->tq, &pi->start_task); } if (n) @@ -890,9 +916,9 @@ static void sge_timer_reclaim(void *arg, int ncount) { - struct port_info *p = arg; - int i, nqsets = p->nqsets; - adapter_t *sc = p->adapter; + struct port_info *pi = arg; + int i, nqsets = pi->nqsets; + adapter_t *sc = pi->adapter; struct sge_qset *qs; struct sge_txq *txq; struct mtx *lock; @@ -1231,7 +1257,6 @@ #define GET_VTAG(cntrl, m) #endif - int t3_encap(struct sge_qset *qs, struct mbuf **m, int count, int *free) { @@ -1239,7 +1264,7 @@ struct mbuf *m0; struct sge_txq *txq; struct txq_state txqs; - struct port_info *p; + struct port_info *pi; unsigned int ndesc, flits, cntrl, mlen; int err, nsegs, tso_info = 0; @@ -1250,25 +1275,26 @@ uint32_t wr_hi, wr_lo, sgl_flits; struct tx_desc *txd; - DPRINTF("t3_encap port_id=%d qsidx=%d ", p->port_id, p->first_qset); + #if defined(IFNET_MULTIQUEUE) && defined(STRICT_AFFINITY) KASSERT(qs->qs_cpuid == curcpu, ("cpu qset mismatch cpuid=%d curcpu=%d", qs->qs_cpuid, curcpu)); #endif DPRINTF("t3_encap cpu=%d ", curcpu); - p = qs->port; - sc = p->adapter; + pi = qs->port; + sc = pi->adapter; txq = &qs->txq[TXQ_ETH]; txsd = &txq->sdesc[txq->pidx]; txd = &txq->desc[txq->pidx]; - DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", mlen, p->txpkt_intf, p->tx_chan); + DPRINTF("t3_encap port_id=%d qsidx=%d ", pi->port_id, pi->first_qset); + DPRINTF("mlen=%d txpkt_intf=%d tx_chan=%d\n", m[0]->m_pkthdr.len, pi->txpkt_intf, pi->tx_chan); /* * XXX handle checksum, TSO, and VLAN here * */ - cntrl = V_TXPKT_INTF(p->txpkt_intf); + cntrl = V_TXPKT_INTF(pi->txpkt_intf); /* * XXX need to add VLAN support for 6.x @@ -1291,7 +1317,7 @@ for (i = 0; i < count; i++) { struct cpl_tx_pkt_batch_entry *cbe = &cpl_batch->pkt_entry[i]; - cntrl = V_TXPKT_INTF(p->port_id); + cntrl = V_TXPKT_INTF(pi->txpkt_intf); GET_VTAG(cntrl, m[i]); cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT) | (1 << 24); cbe->cntrl = htonl(cntrl); @@ -1370,7 +1396,6 @@ memcpy(&txd->flit[2], mtod(m0, uint8_t *), mlen); else m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); - *free = 1; flits = (mlen + 7) / 8 + 2; cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(mlen & 7) | @@ -1409,7 +1434,7 @@ m_set_priority(m0, txqs.pidx); write_wr_hdr_sgl(ndesc, txd, &txqs, txq, sgl, flits, sgl_flits, wr_hi, wr_lo); - check_ring_tx_db(p->adapter, txq); + check_ring_tx_db(pi->adapter, txq); return (0); } @@ -1779,7 +1804,7 @@ { struct tx_sw_desc *txsd; unsigned int cidx; - int i, reclaimed, nbufs; + int i, iter, reclaimed, nbufs; #ifdef T3_TRACE T3_TRACE2(sc->tb[q->cntxt_id & 7], @@ -1787,10 +1812,16 @@ #endif cidx = q->cidx; txsd = &q->sdesc[cidx]; + prefetch(txsd); reclaimed = nbufs = 0; - - for (reclaimed = 0; reclaimed < reclaimable;) { - DPRINTF("cidx=%d d=%p\n", cidx, d); + for (iter = reclaimed = 0; reclaimed < reclaimable; iter++) { + if ((iter & 0x1) == 0) { + prefetch(txsd + 1); + prefetch(txsd + 2); + prefetch(txsd + 3); + prefetch(txsd + 4); + } + DPRINTF("cidx=%d d=%p\n", cidx, txsd); if (txsd->count > 0) { if (nbufs + txsd->count > m_vec_size) break; @@ -1798,13 +1829,17 @@ bus_dmamap_unload(q->entry_tag, txsd->map); txsd->flags &= ~TX_SW_DESC_MAPPED; } - if (m_get_priority(txsd->m[0]) == cidx) { - for (i = 0; i < txsd->count; i++, nbufs++) - m_vec[nbufs] = txsd->m[i]; - txsd->count = 0; - } else { + for (i = 0; i < txsd->count; i++, nbufs++) { + prefetch(txsd->m[i]); + m_vec[nbufs] = txsd->m[i]; + } + txsd->count = 0; + +#ifdef DIAGNOSTIC + if (m_get_priority(txsd->m[0]) != cidx) printf("pri=%d cidx=%d\n", (int)m_get_priority(txsd->m[0]), cidx); - } +#endif + } else q->txq_skipped++; @@ -2254,13 +2289,18 @@ q->txq[TXQ_ETH].stop_thres = nports * flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); - q->fl[0].buf_size = MCLBYTES; + q->fl[0].buf_size = MCLBYTES - sizeof(uint32_t); q->fl[0].zone = zone_clust; q->fl[0].type = EXT_CLUSTER; - q->fl[1].buf_size = MJUMPAGESIZE; - q->fl[1].zone = zone_jumbop; - q->fl[1].type = EXT_JUMBOP; - + if (jumbo_phys_contig) { + q->fl[1].buf_size = MJUM9BYTES - sizeof(uint32_t); + q->fl[1].zone = zone_jumbo9; + q->fl[1].type = EXT_JUMBO9; + } else { + q->fl[1].buf_size = MJUMPAGESIZE - sizeof(uint32_t); + q->fl[1].zone = zone_jumbop; + q->fl[1].type = EXT_JUMBOP; + } q->lro.enabled = lro_default; mtx_lock(&sc->sge.reg_lock); @@ -2372,7 +2412,9 @@ /* * adjust after conversion to mbuf chain */ - m_adj(m, sizeof(*cpl) + ethpad); + m->m_pkthdr.len -= (sizeof(*cpl) + ethpad); + m->m_len -= (sizeof(*cpl) + ethpad); + m->m_data += (sizeof(*cpl) + ethpad); (*ifp->if_input)(ifp, m); } @@ -2406,17 +2448,24 @@ uint32_t len = G_RSPD_LEN(len_cq); uint32_t flags = ntohl(r->flags); uint8_t sopeop = G_RSPD_SOP_EOP(flags); + uint32_t *ref; int ret = 0; - prefetch(sd->cl); + prefetch(sd->rxsd_cl); fl->credits--; bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(fl->entry_tag, sd->map); - m_cljset(m, sd->cl, fl->type); + ref = sd->rxsd_ref; + m_cljset(m, sd->rxsd_cl, fl->type, sd->rxsd_ref); + *ref = 1; m->m_len = len; - + /* + * bump past the refcnt address + */ + m->m_data = sd->data; + switch(sopeop) { case RSPQ_SOP_EOP: DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); @@ -2474,9 +2523,11 @@ uint32_t flags = ntohl(r->flags); uint8_t sopeop = G_RSPD_SOP_EOP(flags); void *cl; + uint32_t *ref = NULL; int ret = 0; - - prefetch(sd->cl); + + prefetch((sd + 1)->rxsd_cl); + prefetch((sd + 2)->rxsd_cl); DPRINTF("rx cpu=%d\n", curcpu); fl->credits--; @@ -2484,17 +2535,22 @@ if (recycle_enable && len <= SGE_RX_COPY_THRES && sopeop == RSPQ_SOP_EOP) { cl = mtod(m, void *); - memcpy(cl, sd->cl, len); + memcpy(cl, sd->data, len); recycle_rx_buf(adap, fl, fl->cidx); } else { - cl = sd->cl; bus_dmamap_unload(fl->entry_tag, sd->map); + cl = sd->rxsd_cl; + ref = sd->rxsd_ref; } + switch(sopeop) { case RSPQ_SOP_EOP: DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); - if (cl == sd->cl) - m_cljset(m, cl, fl->type); + if (cl == sd->rxsd_cl) { + m_cljset(m, cl, fl->type, ref); + *ref = 1; + m->m_data = sd->data; + } m->m_len = m->m_pkthdr.len = len; ret = 1; goto done; @@ -2513,8 +2569,8 @@ ret = 1; break; } - m_iovappend(m, cl, fl->buf_size, len, 0); - + m_iovappend(m, cl, fl->buf_size, len, sizeof(uint32_t), ref); + *ref = 1; done: if (++fl->cidx == fl->size) fl->cidx = 0; @@ -2710,7 +2766,7 @@ if (eop) { prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *)); - prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *) + L1_CACHE_BYTES); + prefetch(mtod(rspq->rspq_mh.mh_head, uint8_t *) + L1_CACHE_BYTES); if (eth) { t3_rx_eth_lro(adap, rspq, rspq->rspq_mh.mh_head, ethpad, @@ -2727,10 +2783,10 @@ ngathered = rx_offload(&adap->tdev, rspq, rspq->rspq_mh.mh_head, offload_mbufs, ngathered); } - __refill_fl(adap, &qs->fl[0]); - __refill_fl(adap, &qs->fl[1]); } + __refill_fl_lt(adap, &qs->fl[0], 32); + __refill_fl_lt(adap, &qs->fl[1], 32); --budget_left; } @@ -2744,6 +2800,8 @@ if (__predict_false(qs->txq_stopped != 0)) restart_tx(qs); + __refill_fl_lt(adap, &qs->fl[0], 512); + __refill_fl_lt(adap, &qs->fl[1], 512); budget -= budget_left; return (budget); } ==== //depot/projects/toestack/sys/dev/cxgb/ulp/toecore/toedev.c#2 (text+ko) ==== @@ -38,6 +38,7 @@ #include <sys/module.h> #include <sys/queue.h> #include <sys/mbuf.h> +#include <sys/proc.h> #include <sys/socket.h> #include <sys/sockio.h>
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200709172040.l8HKeVcQ078853>