From owner-p4-projects@FreeBSD.ORG Sun Apr 15 06:35:43 2007 Return-Path: X-Original-To: p4-projects@freebsd.org Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 94A4716A403; Sun, 15 Apr 2007 06:35:43 +0000 (UTC) X-Original-To: perforce@freebsd.org Delivered-To: perforce@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id 3848916A400 for ; Sun, 15 Apr 2007 06:35:43 +0000 (UTC) (envelope-from mjacob@freebsd.org) Received: from repoman.freebsd.org (repoman.freebsd.org [69.147.83.41]) by mx1.freebsd.org (Postfix) with ESMTP id 1C68613C448 for ; Sun, 15 Apr 2007 06:35:43 +0000 (UTC) (envelope-from mjacob@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.13.8/8.13.8) with ESMTP id l3F6ZgNq006244 for ; Sun, 15 Apr 2007 06:35:42 GMT (envelope-from mjacob@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.13.8/8.13.8/Submit) id l3F6ZgJP006241 for perforce@freebsd.org; Sun, 15 Apr 2007 06:35:42 GMT (envelope-from mjacob@freebsd.org) Date: Sun, 15 Apr 2007 06:35:42 GMT Message-Id: <200704150635.l3F6ZgJP006241@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to mjacob@freebsd.org using -f From: Matt Jacob To: Perforce Change Reviews Cc: Subject: PERFORCE change 118142 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 15 Apr 2007 06:35:43 -0000 http://perforce.freebsd.org/chv.cgi?CH=118142 Change 118142 by mjacob@mjexp on 2007/04/15 06:34:53 More IFC. Affected files ... .. //depot/projects/mjexp/sys/dev/cxgb/cxgb_main.c#3 integrate .. //depot/projects/mjexp/sys/dev/cxgb/cxgb_sge.c#5 integrate .. //depot/projects/mjexp/sys/dev/cxgb/sys/uipc_mvec.c#3 integrate .. //depot/projects/mjexp/sys/sys/mbuf.h#8 integrate Differences ... ==== //depot/projects/mjexp/sys/dev/cxgb/cxgb_main.c#3 (text+ko) ==== @@ -32,7 +32,7 @@ ***************************************************************************/ #include -__FBSDID("$FreeBSD: src/sys/dev/cxgb/cxgb_main.c,v 1.13 2007/04/14 20:40:22 kmacy Exp $"); +__FBSDID("$FreeBSD: src/sys/dev/cxgb/cxgb_main.c,v 1.14 2007/04/15 05:45:09 kmacy Exp $"); #include #include @@ -167,6 +167,7 @@ #define SGE_MSIX_COUNT (SGE_QSETS + 1) +extern int collapse_mbufs; /* * The driver uses the best interrupt scheme available on a platform in the * order MSI-X, MSI, legacy pin interrupts. This parameter determines which @@ -1339,8 +1340,8 @@ } m_sanity(m0, 0); m0 = m; -#endif - if (m->m_pkthdr.len > MCLBYTES && +#endif + if (collapse_mbufs && m->m_pkthdr.len > MCLBYTES && m_collapse(m, TX_MAX_SEGS, &m0) == EFBIG) { if ((m0 = m_defrag(m, M_NOWAIT)) != NULL) { m = m0; ==== //depot/projects/mjexp/sys/dev/cxgb/cxgb_sge.c#5 (text+ko) ==== @@ -32,7 +32,7 @@ ***************************************************************************/ #include -__FBSDID("$FreeBSD: src/sys/dev/cxgb/cxgb_sge.c,v 1.11 2007/04/14 20:40:22 kmacy Exp $"); +__FBSDID("$FreeBSD: src/sys/dev/cxgb/cxgb_sge.c,v 1.12 2007/04/15 05:45:10 kmacy Exp $"); #include #include @@ -71,7 +71,7 @@ uint32_t collapse_free = 0; uint32_t mb_free_vec_free = 0; - +int collapse_mbufs = 1; #define USE_GTS 0 @@ -837,13 +837,6 @@ m0 = *m; pktlen = m0->m_pkthdr.len; - if ((stx->flags & TX_SW_DESC_MAP_CREATED) == 0) { - if ((err = bus_dmamap_create(txq->entry_tag, 0, &stx->map))) { - log(LOG_WARNING, "bus_dmamap_create failed %d\n", err); - return (err); - } - stx->flags |= TX_SW_DESC_MAP_CREATED; - } err = bus_dmamap_load_mvec_sg(txq->entry_tag, stx->map, m0, segs, nsegs, 0); #ifdef DEBUG if (err) { @@ -2264,7 +2257,10 @@ "mb_free_vec_free", CTLFLAG_RD, &mb_free_vec_free, 0, "frees during mb_free_vec"); - + SYSCTL_ADD_INT(ctx, children, OID_AUTO, + "collapse_mbufs", + CTLFLAG_RW, &collapse_mbufs, + 0, "collapse mbuf chains into iovecs"); } /** ==== //depot/projects/mjexp/sys/dev/cxgb/sys/uipc_mvec.c#3 (text+ko) ==== @@ -29,7 +29,7 @@ ***************************************************************************/ #include -__FBSDID("$FreeBSD: src/sys/dev/cxgb/sys/uipc_mvec.c,v 1.5 2007/04/14 20:38:38 kmacy Exp $"); +__FBSDID("$FreeBSD: src/sys/dev/cxgb/sys/uipc_mvec.c,v 1.7 2007/04/15 05:46:34 kmacy Exp $"); #include #include @@ -42,11 +42,7 @@ #include #include - #include "opt_zero.h" -#ifdef ZERO_COPY_SOCKETS -#error "ZERO_COPY_SOCKETS not supported with mvec" -#endif #ifdef DEBUG #define DPRINTF printf @@ -137,19 +133,7 @@ m->m_next = head; head = m; M_SANITY(m, 0); -#ifdef INVARIANTS - len = head->m_len; - m = m->m_next; - while (m) { - KASSERT((m->m_flags & M_PKTHDR) == 0, - ("pkthdr set on intermediate mbuf - post")); - len += m->m_len; - m = m->m_next; - - } - if (len != head->m_pkthdr.len) - panic("len=%d pktlen=%d", len, head->m_pkthdr.len); -#endif + return (0); } @@ -161,12 +145,14 @@ for (i = 0; i < max; i++) { if (m == NULL) break; -#ifndef PACKET_ZONE_DISABLED +#ifndef MBUF_PACKET_ZONE_DISABLE if ((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_PACKET)) return (EINVAL); #endif - if (m->m_len == 0) - DPRINTF("m=%p is len=0\n", m); +#ifdef ZERO_COPY_SOCKETS + if ((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_SFBUF)) + return (EINVAL); +#endif M_SANITY(m, 0); vec[i] = m; m = m->m_next; @@ -398,13 +384,28 @@ } #if (!defined(__sparc64__) && !defined(__sun4v__)) -struct mvec_sg_cb_arg { - bus_dma_segment_t *segs; - int error; - int index; - int nseg; +#include + +#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 +#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 + +struct bounce_zone { + STAILQ_ENTRY(bounce_zone) links; + STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; + int total_bpages; + int free_bpages; + int reserved_bpages; + int active_bpages; + int total_bounced; + int total_deferred; + bus_size_t alignment; + bus_size_t boundary; + bus_addr_t lowaddr; + char zoneid[8]; + char lowaddrid[20]; + struct sysctl_ctx_list sysctl_tree; + struct sysctl_oid *sysctl_tree_top; }; - struct bus_dma_tag { bus_dma_tag_t parent; bus_size_t alignment; @@ -425,23 +426,137 @@ struct bounce_zone *bounce_zone; }; -static void -mvec_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +struct bus_dmamap { + struct bp_list bpages; + int pagesneeded; + int pagesreserved; + bus_dma_tag_t dmat; + void *buf; /* unmapped buffer pointer */ + bus_size_t buflen; /* unmapped buffer length */ + bus_dmamap_callback_t *callback; + void *callback_arg; + STAILQ_ENTRY(bus_dmamap) links; +}; + +static struct bus_dmamap nobounce_dmamap; + +static __inline int +run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) +{ + int retval; + + retval = 0; + + do { + if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) + || ((paddr & (dmat->alignment - 1)) != 0)) + && (dmat->filter == NULL + || (*dmat->filter)(dmat->filterarg, paddr) != 0)) + retval = 1; + + dmat = dmat->parent; + } while (retval == 0 && dmat != NULL); + return (retval); +} + +static __inline int +_bus_dmamap_load_buffer(bus_dma_tag_t dmat, + bus_dmamap_t map, + void *buf, bus_size_t buflen, + pmap_t pmap, + int flags, + bus_addr_t *lastaddrp, + bus_dma_segment_t *segs, + int *segp, + int first) { - struct mvec_sg_cb_arg *cb_arg = arg; - - cb_arg->error = error; - cb_arg->segs[cb_arg->index] = segs[0]; - cb_arg->nseg = nseg; - KASSERT(nseg == 1, ("nseg=%d", nseg)); + bus_size_t sgsize; + bus_addr_t curaddr, lastaddr, baddr, bmask; + vm_offset_t vaddr; + int needbounce = 0; + int seg; + + if (map == NULL) + map = &nobounce_dmamap; + + /* Reserve Necessary Bounce Pages */ + if (map->pagesneeded != 0) + panic("don't support bounce pages"); + + vaddr = (vm_offset_t)buf; + lastaddr = *lastaddrp; + bmask = ~(dmat->boundary - 1); + + for (seg = *segp; buflen > 0 ; ) { + /* + * Get the physical address for this segment. + */ + if (pmap) + curaddr = pmap_extract(pmap, vaddr); + else + curaddr = pmap_kextract(vaddr); + + + /* + * Compute the segment size, and adjust counts. + */ + sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); + if (buflen < sgsize) + sgsize = buflen; + + /* + * Make sure we don't cross any boundaries. + */ + if (dmat->boundary > 0) { + baddr = (curaddr + dmat->boundary) & bmask; + if (sgsize > (baddr - curaddr)) + sgsize = (baddr - curaddr); + } + + if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) + panic("no bounce page support"); + + /* + * Insert chunk into a segment, coalescing with + * previous segment if possible. + */ + if (first) { + segs[seg].ds_addr = curaddr; + segs[seg].ds_len = sgsize; + first = 0; + } else { + if (needbounce == 0 && curaddr == lastaddr && + (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && + (dmat->boundary == 0 || + (segs[seg].ds_addr & bmask) == (curaddr & bmask))) + segs[seg].ds_len += sgsize; + else { + if (++seg >= dmat->nsegments) + break; + segs[seg].ds_addr = curaddr; + segs[seg].ds_len = sgsize; + } + } + + lastaddr = curaddr + sgsize; + vaddr += sgsize; + buflen -= sgsize; + } + + *segp = seg; + *lastaddrp = lastaddr; + + /* + * Did we fit? + */ + return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ } int bus_dmamap_load_mvec_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags) { - int error; - struct mvec_sg_cb_arg cb_arg; + int error, i; M_ASSERTPKTHDR(m0); @@ -452,51 +567,43 @@ *nsegs = 0; error = 0; if (m0->m_pkthdr.len <= dmat->maxsize) { + int first = 1; + bus_addr_t lastaddr = 0; struct mbuf *m; - cb_arg.segs = segs; + for (m = m0; m != NULL && error == 0; m = m->m_next) { struct mbuf_vec *mv; - int count, first, i; + int count, firstcl; if (!(m->m_len > 0)) continue; mv = mtomv(m); count = mv->mv_count; - first = mv->mv_first; + firstcl = mv->mv_first; KASSERT(count <= MAX_MBUF_IOV, ("count=%d too large", count)); - for (i = first; i < count; i++) { + for (i = firstcl; i < count && error == 0; i++) { void *data = mv->mv_vec[i].mi_base + mv->mv_vec[i].mi_offset; - int size = mv->mv_vec[i].mi_len; + int len = mv->mv_vec[i].mi_len; - if (size == 0) + if (len == 0) continue; - DPRINTF("mapping data=%p size=%d\n", data, size); - cb_arg.index = *nsegs; - error = bus_dmamap_load(dmat, map, - data, size, mvec_cb, &cb_arg, flags); - (*nsegs)++; - - if (*nsegs >= dmat->nsegments) { - DPRINTF("*nsegs=%d dmat->nsegments=%d index=%d\n", - *nsegs, dmat->nsegments, cb_arg.index); - error = EFBIG; - goto err_out; - } - if (error || cb_arg.error) - goto err_out; + DPRINTF("mapping data=%p len=%d\n", data, len); + error = _bus_dmamap_load_buffer(dmat, NULL, + data, len, NULL, flags, &lastaddr, + segs, nsegs, first); + DPRINTF("%d: addr=0x%lx len=%ld\n", i, segs[i].ds_addr, + segs[i].ds_len); + first = 0; } } } else { error = EINVAL; } + + (*nsegs)++; + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, dmat->flags, error, *nsegs); return (error); - -err_out: - if (cb_arg.error) - return (cb_arg.error); - - return (error); } #endif /* !__sparc64__ && !__sun4v__ */ ==== //depot/projects/mjexp/sys/sys/mbuf.h#8 (text+ko) ==== @@ -28,7 +28,7 @@ * SUCH DAMAGE. * * @(#)mbuf.h 8.5 (Berkeley) 2/19/95 - * $FreeBSD: src/sys/sys/mbuf.h,v 1.210 2007/04/14 20:31:05 kmacy Exp $ + * $FreeBSD: src/sys/sys/mbuf.h,v 1.211 2007/04/15 06:30:28 kmacy Exp $ */ #ifndef _SYS_MBUF_H_ @@ -341,9 +341,7 @@ extern uma_zone_t zone_jumbo16; extern uma_zone_t zone_ext_refcnt; -#ifndef MBUF_PACKET_ZONE_DISABLE static __inline struct mbuf *m_getcl(int how, short type, int flags); -#endif static __inline struct mbuf *m_get(int how, short type); static __inline struct mbuf *m_gethdr(int how, short type); static __inline struct mbuf *m_getjcl(int how, short type, int flags, @@ -453,7 +451,6 @@ return ((struct mbuf *)(uma_zalloc_arg(zone_mbuf, &args, how))); } -#ifndef MBUF_PACKET_ZONE_DISABLE static __inline struct mbuf * m_getcl(int how, short type, int flags) { @@ -463,9 +460,6 @@ args.type = type; return ((struct mbuf *)(uma_zalloc_arg(zone_pack, &args, how))); } -#else -#define m_getcl(how, type, flags) m_getjcl(how, type, flags, MCLBYTES) -#endif /* * m_getjcl() returns an mbuf with a cluster of the specified size attached.