Date: Tue, 18 Jun 2013 00:36:54 +0000 (UTC) From: Scott Long <scottl@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-9@freebsd.org Subject: svn commit: r251874 - in stable/9/sys: arm/arm cam cam/ctl cam/scsi conf dev/aac dev/advansys dev/aha dev/ahb dev/ahci dev/aic dev/aic7xxx dev/amr dev/arcmsr dev/ata dev/buslogic dev/ciss dev/dpt d... Message-ID: <201306180036.r5I0askx009647@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: scottl Date: Tue Jun 18 00:36:53 2013 New Revision: 251874 URL: http://svnweb.freebsd.org/changeset/base/251874 Log: Big MFC of the physbio changes necessary for unmapped I/O. These changes have been in production at Netflix for several months with significant success. MFC r246713: Reform the busdma API so that new types may be added without modifying every architecture's busdma_machdep.c. It is done by unifying the bus_dmamap_load_buffer() routines so that they may be called from MI code. The MD busdma is then given a chance to do any final processing in the complete() callback. MFC r249538: Some compilers issue a warning when wider integer is casted to narrow pointer. Supposedly shut down the warning by casting through uintptr_t. MFC r251479: Simplify the checking of flags for cam_periph_mapmem(). This gets rid of a lot of code redundancy and grossness at very minor expense. MFC r251837: MFC r251842: Add infrastructure for doing compatibility shims, as has been sorely needed for the last 10 years. Far too much of the internal API is exposed, and every small adjustment causes applications to stop working. To kick this off, bump the API version to 0x17 as should have been done with r246713, but add shims to compensate. Thanks to the shims, there should be no visible change in application behavior. Submitted by: kib, jeffr Approved by: kib Obtained from: Netflix Added: stable/9/sys/cam/cam_compat.c - copied unchanged from r251837, head/sys/cam/cam_compat.c stable/9/sys/cam/cam_compat.h - copied unchanged from r251837, head/sys/cam/cam_compat.h stable/9/sys/kern/subr_bus_dma.c - copied, changed from r246713, head/sys/kern/subr_bus_dma.c stable/9/sys/sys/memdesc.h - copied unchanged from r246713, head/sys/sys/memdesc.h Modified: stable/9/sys/arm/arm/busdma_machdep.c stable/9/sys/cam/cam_ccb.h stable/9/sys/cam/cam_periph.c stable/9/sys/cam/cam_xpt.c stable/9/sys/cam/ctl/ctl_frontend_cam_sim.c stable/9/sys/cam/ctl/scsi_ctl.c stable/9/sys/cam/scsi/scsi_pass.c stable/9/sys/cam/scsi/scsi_sg.c stable/9/sys/cam/scsi/scsi_target.c stable/9/sys/conf/files stable/9/sys/dev/aac/aac_cam.c stable/9/sys/dev/advansys/advansys.c stable/9/sys/dev/advansys/adwcam.c stable/9/sys/dev/aha/aha.c stable/9/sys/dev/ahb/ahb.c stable/9/sys/dev/ahci/ahci.c stable/9/sys/dev/aic/aic.c stable/9/sys/dev/aic7xxx/aic79xx_osm.c stable/9/sys/dev/aic7xxx/aic7xxx_osm.c stable/9/sys/dev/amr/amr_cam.c stable/9/sys/dev/arcmsr/arcmsr.c stable/9/sys/dev/ata/ata-dma.c stable/9/sys/dev/ata/atapi-cam.c stable/9/sys/dev/buslogic/bt.c stable/9/sys/dev/ciss/ciss.c stable/9/sys/dev/ciss/cissvar.h stable/9/sys/dev/dpt/dpt_scsi.c stable/9/sys/dev/firewire/sbp.c stable/9/sys/dev/hpt27xx/osm_bsd.c stable/9/sys/dev/hptiop/hptiop.c stable/9/sys/dev/hptmv/entry.c stable/9/sys/dev/hptrr/hptrr_osm_bsd.c stable/9/sys/dev/iir/iir.c stable/9/sys/dev/isci/isci_io_request.c stable/9/sys/dev/isp/isp_pci.c stable/9/sys/dev/isp/isp_sbus.c stable/9/sys/dev/mfi/mfi.c stable/9/sys/dev/mfi/mfi_cam.c stable/9/sys/dev/mly/mly.c stable/9/sys/dev/mly/mlyvar.h stable/9/sys/dev/mps/mps.c stable/9/sys/dev/mps/mps_sas.c stable/9/sys/dev/mps/mpsvar.h stable/9/sys/dev/mpt/mpt_cam.c stable/9/sys/dev/mvs/mvs.c stable/9/sys/dev/siis/siis.c stable/9/sys/dev/sym/sym_hipd.c stable/9/sys/dev/trm/trm.c stable/9/sys/dev/twa/tw_osl.h stable/9/sys/dev/twa/tw_osl_cam.c stable/9/sys/dev/twa/tw_osl_freebsd.c stable/9/sys/dev/tws/tws.h stable/9/sys/dev/tws/tws_cam.c stable/9/sys/dev/virtio/scsi/virtio_scsi.c stable/9/sys/dev/wds/wd7000.c stable/9/sys/ia64/ia64/busdma_machdep.c stable/9/sys/kern/subr_uio.c stable/9/sys/mips/mips/busdma_machdep.c stable/9/sys/modules/cam/Makefile stable/9/sys/powerpc/powerpc/busdma_machdep.c stable/9/sys/powerpc/ps3/ps3cdrom.c stable/9/sys/sparc64/include/bus_dma.h stable/9/sys/sparc64/sparc64/bus_machdep.c stable/9/sys/sparc64/sparc64/iommu.c stable/9/sys/sys/bus_dma.h stable/9/sys/sys/uio.h stable/9/sys/x86/x86/busdma_machdep.c Directory Properties: stable/9/sys/ (props changed) stable/9/sys/conf/ (props changed) stable/9/sys/dev/ (props changed) stable/9/sys/dev/isp/ (props changed) stable/9/sys/modules/ (props changed) stable/9/sys/sys/ (props changed) Modified: stable/9/sys/arm/arm/busdma_machdep.c ============================================================================== --- stable/9/sys/arm/arm/busdma_machdep.c Mon Jun 17 23:21:18 2013 (r251873) +++ stable/9/sys/arm/arm/busdma_machdep.c Tue Jun 18 00:36:53 2013 (r251874) @@ -61,12 +61,12 @@ __FBSDID("$FreeBSD$"); #include <sys/interrupt.h> #include <sys/lock.h> #include <sys/proc.h> +#include <sys/memdesc.h> #include <sys/mutex.h> -#include <sys/mbuf.h> -#include <sys/uio.h> #include <sys/ktr.h> #include <sys/kernel.h> #include <sys/sysctl.h> +#include <sys/uio.h> #include <vm/uma.h> #include <vm/vm.h> @@ -125,10 +125,17 @@ struct bounce_page { vm_offset_t vaddr_nocache; /* kva of bounce buffer uncached */ bus_addr_t busaddr; /* Physical address */ vm_offset_t datavaddr; /* kva of client data */ + bus_addr_t dataaddr; /* client physical address */ bus_size_t datacount; /* client data count */ STAILQ_ENTRY(bounce_page) links; }; +struct sync_list { + vm_offset_t vaddr; /* kva of bounce buffer */ + bus_addr_t busaddr; /* Physical address */ + bus_size_t datacount; /* client data count */ +}; + int busdma_swi_pending; struct bounce_zone { @@ -158,24 +165,21 @@ static SYSCTL_NODE(_hw, OID_AUTO, busdma SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, "Total bounce pages"); -#define DMAMAP_LINEAR 0x1 -#define DMAMAP_MBUF 0x2 -#define DMAMAP_UIO 0x4 -#define DMAMAP_CACHE_ALIGNED 0x10 -#define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO) #define DMAMAP_COHERENT 0x8 +#define DMAMAP_CACHE_ALIGNED 0x10 + struct bus_dmamap { struct bp_list bpages; int pagesneeded; int pagesreserved; bus_dma_tag_t dmat; + struct memdesc mem; int flags; - void *buffer; - int len; STAILQ_ENTRY(bus_dmamap) links; bus_dmamap_callback_t *callback; void *callback_arg; - + int sync_count; + struct sync_list *slist; }; static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; @@ -191,7 +195,8 @@ static int alloc_bounce_pages(bus_dma_ta static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit); static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, - vm_offset_t vaddr, bus_size_t size); + vm_offset_t vaddr, bus_addr_t addr, + bus_size_t size); static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); /* Default tag, as most drivers provide no parent tag. */ @@ -564,13 +569,20 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat) int bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) { + struct sync_list *slist; bus_dmamap_t map; int error = 0; + slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); + if (slist == NULL) + return (ENOMEM); + map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT); *mapp = map; - if (map == NULL) + if (map == NULL) { + free(slist, M_DEVBUF); return (ENOMEM); + } /* * If the tag's segments haven't been allocated yet we need to do it @@ -580,6 +592,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in dmat->segments = malloc(dmat->nsegments * sizeof(*dmat->segments), M_DEVBUF, M_NOWAIT); if (dmat->segments == NULL) { + free(slist, M_DEVBUF); uma_zfree(dmamap_zone, map); *mapp = NULL; return (ENOMEM); @@ -599,6 +612,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in if (dmat->bounce_zone == NULL) { if ((error = alloc_bounce_zone(dmat)) != 0) { + free(slist, M_DEVBUF); uma_zfree(dmamap_zone, map); *mapp = NULL; return (error); @@ -633,6 +647,8 @@ bus_dmamap_create(bus_dma_tag_t dmat, in } bz->map_count++; } + map->sync_count = 0; + map->slist = slist; CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", __func__, dmat, dmat->flags, error); @@ -647,11 +663,12 @@ int bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) { - if (STAILQ_FIRST(&map->bpages) != NULL) { + if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY); return (EBUSY); } + free(map->slist, M_DEVBUF); uma_zfree(dmamap_zone, map); if (dmat->bounce_zone) dmat->bounce_zone->map_count--; @@ -668,6 +685,7 @@ int bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, bus_dmamap_t *mapp) { + struct sync_list *slist; void * vaddr; struct busdma_bufzone *bufzone; busdma_bufalloc_t ba; @@ -679,7 +697,6 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi mflags = M_NOWAIT; else mflags = M_WAITOK; - /* * If the tag's segments haven't been allocated yet we need to do it * now, because we can't sleep for resources at map load time. @@ -688,10 +705,14 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi dmat->segments = malloc(dmat->nsegments * sizeof(*dmat->segments), M_DEVBUF, mflags); + slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT); + if (slist == NULL) + return (ENOMEM); map = uma_zalloc_arg(dmamap_zone, dmat, mflags); - if (map == NULL) + if (map == NULL) { + free(slist, M_DEVBUF); return (ENOMEM); - + } if (flags & BUS_DMA_COHERENT) { memattr = VM_MEMATTR_UNCACHEABLE; ba = coherent_allocator; @@ -738,12 +759,14 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, memattr); } - if (vaddr == NULL) { + free(slist, M_DEVBUF); uma_zfree(dmamap_zone, map); map = NULL; + } else { + map->slist = slist; + map->sync_count = 0; } - *vaddrp = vaddr; *mapp = map; @@ -762,10 +785,11 @@ bus_dmamem_free(bus_dma_tag_t dmat, void if (map->flags & DMAMAP_COHERENT) ba = coherent_allocator; - else + else ba = standard_allocator; - uma_zfree(dmamap_zone, map); + uma_zfree(dmamap_zone, map); + free(map->slist, M_DEVBUF); /* Be careful not to access map from here on. */ bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); @@ -777,7 +801,37 @@ bus_dmamem_free(bus_dma_tag_t dmat, void kmem_free(kernel_map, (vm_offset_t)vaddr, dmat->maxsize); } -static int +static void +_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags) +{ + bus_addr_t curaddr; + bus_size_t sgsize; + + if ((map->pagesneeded == 0)) { + CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", + dmat->lowaddr, dmat->boundary, dmat->alignment); + CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", + map, map->pagesneeded); + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + curaddr = buf; + while (buflen != 0) { + sgsize = MIN(buflen, dmat->maxsegsz); + if (run_filter(dmat, curaddr) != 0) { + sgsize = MIN(sgsize, PAGE_SIZE); + map->pagesneeded++; + } + curaddr += sgsize; + buflen -= sgsize; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } +} + +static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, void *buf, bus_size_t buflen, int flags) { @@ -798,75 +852,183 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm vendaddr = (vm_offset_t)buf + buflen; while (vaddr < vendaddr) { - if (__predict_true(pmap == pmap_kernel())) + if (__predict_true(pmap == kernel_pmap)) paddr = pmap_kextract(vaddr); else paddr = pmap_extract(pmap, vaddr); - if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && - run_filter(dmat, paddr) != 0) + if (run_filter(dmat, paddr) != 0) map->pagesneeded++; vaddr += PAGE_SIZE; } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } +} + +static int +_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) +{ /* Reserve Necessary Bounce Pages */ - if (map->pagesneeded != 0) { - mtx_lock(&bounce_lock); - if (flags & BUS_DMA_NOWAIT) { - if (reserve_bounce_pages(dmat, map, 0) != 0) { - mtx_unlock(&bounce_lock); - return (ENOMEM); - } - } else { - if (reserve_bounce_pages(dmat, map, 1) != 0) { - /* Queue us for resources */ - STAILQ_INSERT_TAIL(&bounce_map_waitinglist, - map, links); - mtx_unlock(&bounce_lock); - return (EINPROGRESS); - } + mtx_lock(&bounce_lock); + if (flags & BUS_DMA_NOWAIT) { + if (reserve_bounce_pages(dmat, map, 0) != 0) { + mtx_unlock(&bounce_lock); + return (ENOMEM); + } + } else { + if (reserve_bounce_pages(dmat, map, 1) != 0) { + /* Queue us for resources */ + STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); + mtx_unlock(&bounce_lock); + return (EINPROGRESS); } - mtx_unlock(&bounce_lock); } + mtx_unlock(&bounce_lock); + + return (0); +} +/* + * Add a single contiguous physical range to the segment list. + */ +static int +_bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, + bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) +{ + bus_addr_t baddr, bmask; + int seg; + + /* + * Make sure we don't cross any boundaries. + */ + bmask = ~(dmat->boundary - 1); + if (dmat->boundary > 0) { + baddr = (curaddr + dmat->boundary) & bmask; + if (sgsize > (baddr - curaddr)) + sgsize = (baddr - curaddr); + } + if (dmat->ranges) { + struct arm32_dma_range *dr; + + dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, + curaddr); + if (dr == NULL) + return (EINVAL); + /* + * In a valid DMA range. Translate the physical + * memory address to an address in the DMA window. + */ + curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; + + } + + seg = *segp; + /* + * Insert chunk into a segment, coalescing with + * the previous segment if possible. + */ + if (seg >= 0 && + curaddr == segs[seg].ds_addr + segs[seg].ds_len && + (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && + (dmat->boundary == 0 || + (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { + segs[seg].ds_len += sgsize; + } else { + if (++seg >= dmat->nsegments) + return (EFBIG); + segs[seg].ds_addr = curaddr; + segs[seg].ds_len = sgsize; + } + *segp = seg; return (0); } /* - * Utility function to load a linear buffer. lastaddrp holds state - * between invocations (for multiple-buffer loads). segp contains + * Utility function to load a physical buffer. segp contains + * the starting segment on entrace, and the ending segment on exit. + */ +int +_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) +{ + bus_size_t sgsize; + bus_addr_t curaddr; + int error; + + if (segs == NULL) + segs = dmat->segments; + + if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + + while (buflen > 0) { + curaddr = buf; + sgsize = MIN(buflen, dmat->maxsegsz); + if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && + map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + sgsize = MIN(sgsize, PAGE_SIZE); + curaddr = add_bounce_page(dmat, map, 0, curaddr, + sgsize); + } + sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, + segp); + if (sgsize == 0) + break; + buf += sgsize; + buflen -= sgsize; + } + + /* + * Did we fit? + */ + if (buflen != 0) { + _bus_dmamap_unload(dmat, map); + return (EFBIG); /* XXX better return value here? */ + } + return (0); +} +/* + * Utility function to load a linear buffer. segp contains * the starting segment on entrance, and the ending segment on exit. - * first indicates if this is the first invocation of this function. */ -static __inline int -bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs, - bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, - int flags, vm_offset_t *lastaddrp, int *segp) +int +_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, + bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, + int *segp) { bus_size_t sgsize; - bus_addr_t curaddr, lastaddr, baddr, bmask; + bus_addr_t curaddr; + struct sync_list *sl; vm_offset_t vaddr = (vm_offset_t)buf; - int seg; int error = 0; - lastaddr = *lastaddrp; - bmask = ~(dmat->boundary - 1); + if (segs == NULL) + segs = dmat->segments; + if ((flags & BUS_DMA_LOAD_MBUF) != 0) + map->flags |= DMAMAP_CACHE_ALIGNED; if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { - error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, - flags); - if (error) - return (error); + _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } } CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); - for (seg = *segp; buflen > 0 ; ) { + while (buflen > 0) { /* * Get the physical address for this segment. */ - if (__predict_true(pmap == pmap_kernel())) { + if (__predict_true(pmap == kernel_pmap)) { curaddr = pmap_kextract(vaddr); } else { curaddr = pmap_extract(pmap, vaddr); @@ -882,260 +1044,63 @@ bus_dmamap_load_buffer(bus_dma_tag_t dma if (buflen < sgsize) sgsize = buflen; - /* - * Make sure we don't cross any boundaries. - */ - if (dmat->boundary > 0) { - baddr = (curaddr + dmat->boundary) & bmask; - if (sgsize > (baddr - curaddr)) - sgsize = (baddr - curaddr); - } if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && - map->pagesneeded != 0 && run_filter(dmat, curaddr)) - curaddr = add_bounce_page(dmat, map, vaddr, sgsize); - - if (dmat->ranges) { - struct arm32_dma_range *dr; - - dr = _bus_dma_inrange(dmat->ranges, dmat->_nranges, - curaddr); - if (dr == NULL) - return (EINVAL); - /* - * In a valid DMA range. Translate the physical - * memory address to an address in the DMA window. - */ - curaddr = (curaddr - dr->dr_sysbase) + dr->dr_busbase; - - } - - /* - * Insert chunk into a segment, coalescing with - * the previous segment if possible. - */ - if (seg >= 0 && curaddr == lastaddr && - (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && - (dmat->boundary == 0 || - (segs[seg].ds_addr & bmask) == - (curaddr & bmask))) { - segs[seg].ds_len += sgsize; - goto segdone; + map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + curaddr = add_bounce_page(dmat, map, vaddr, curaddr, + sgsize); } else { - if (++seg >= dmat->nsegments) - break; - segs[seg].ds_addr = curaddr; - segs[seg].ds_len = sgsize; - } - if (error) + sl = &map->slist[map->sync_count - 1]; + if (map->sync_count == 0 || + vaddr != sl->vaddr + sl->datacount) { + if (++map->sync_count > dmat->nsegments) + goto cleanup; + sl++; + sl->vaddr = vaddr; + sl->datacount = sgsize; + sl->busaddr = curaddr; + } else + sl->datacount += sgsize; + } + sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, + segp); + if (sgsize == 0) break; -segdone: - lastaddr = curaddr + sgsize; vaddr += sgsize; buflen -= sgsize; } - *segp = seg; - *lastaddrp = lastaddr; - +cleanup: /* * Did we fit? */ - if (buflen != 0) - error = EFBIG; /* XXX better return value here? */ - return (error); + if (buflen != 0) { + _bus_dmamap_unload(dmat, map); + return (EFBIG); /* XXX better return value here? */ + } + return (0); } -/* - * Map the buffer buf into bus space using the dmamap map. - */ -int -bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, - bus_size_t buflen, bus_dmamap_callback_t *callback, - void *callback_arg, int flags) +void +__bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, + struct memdesc *mem, bus_dmamap_callback_t *callback, + void *callback_arg) { - vm_offset_t lastaddr = 0; - int error, nsegs = -1; KASSERT(dmat != NULL, ("dmatag is NULL")); KASSERT(map != NULL, ("dmamap is NULL")); + map->mem = *mem; map->callback = callback; map->callback_arg = callback_arg; - map->flags &= ~DMAMAP_TYPE_MASK; - map->flags |= DMAMAP_LINEAR; - map->buffer = buf; - map->len = buflen; - error = bus_dmamap_load_buffer(dmat, - dmat->segments, map, buf, buflen, kernel_pmap, - flags, &lastaddr, &nsegs); - if (error == EINPROGRESS) - return (error); - if (error) - (*callback)(callback_arg, NULL, 0, error); - else - (*callback)(callback_arg, dmat->segments, nsegs + 1, error); - - CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, nsegs + 1, error); - - return (error); } -/* - * Like bus_dmamap_load(), but for mbufs. - * - * Note that the manpage states that BUS_DMA_NOWAIT is implied for mbufs. - * - * We know that the way the system allocates and uses mbufs implies that we can - * treat them as DMAMAP_CACHE_ALIGNED in terms of handling partial cache line - * flushes. Even though the flush may reference the data area within the mbuf - * that isn't aligned to a cache line, we know the overall mbuf itself is - * properly aligned, and we know that the CPU will not touch the header fields - * before the data area while the DMA is in progress. - */ -int -bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, - bus_dmamap_callback2_t *callback, void *callback_arg, - int flags) -{ - int nsegs = -1, error = 0; - - M_ASSERTPKTHDR(m0); - - flags |= BUS_DMA_NOWAIT; - map->flags &= ~DMAMAP_TYPE_MASK; - map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; - map->buffer = m0; - map->len = 0; - if (m0->m_pkthdr.len <= dmat->maxsize) { - vm_offset_t lastaddr = 0; - struct mbuf *m; - - for (m = m0; m != NULL && error == 0; m = m->m_next) { - if (m->m_len > 0) { - error = bus_dmamap_load_buffer(dmat, - dmat->segments, map, m->m_data, m->m_len, - pmap_kernel(), flags, &lastaddr, &nsegs); - map->len += m->m_len; - } - } - } else { - error = EINVAL; - } - - if (error) { - /* - * force "no valid mappings" on error in callback. - */ - (*callback)(callback_arg, NULL, 0, 0, error); - } else { - (*callback)(callback_arg, dmat->segments, nsegs + 1, - m0->m_pkthdr.len, error); - } - CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); - - return (error); -} - -int -bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, - struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, - int flags) +bus_dma_segment_t * +_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, + bus_dma_segment_t *segs, int nsegs, int error) { - int error = 0; - M_ASSERTPKTHDR(m0); - - flags |= BUS_DMA_NOWAIT; - *nsegs = -1; - map->flags &= ~DMAMAP_TYPE_MASK; - map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; - map->buffer = m0; - map->len = 0; - if (m0->m_pkthdr.len <= dmat->maxsize) { - vm_offset_t lastaddr = 0; - struct mbuf *m; - - for (m = m0; m != NULL && error == 0; m = m->m_next) { - if (m->m_len > 0) { - error = bus_dmamap_load_buffer(dmat, segs, map, - m->m_data, m->m_len, - pmap_kernel(), flags, &lastaddr, - nsegs); - map->len += m->m_len; - } - } - } else { - error = EINVAL; - } - - /* XXX FIXME: Having to increment nsegs is really annoying */ - ++*nsegs; - CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, *nsegs); - return (error); -} - -/* - * Like bus_dmamap_load(), but for uios. - */ -int -bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, - bus_dmamap_callback2_t *callback, void *callback_arg, - int flags) -{ - vm_offset_t lastaddr = 0; - int nsegs, i, error; - bus_size_t resid; - struct iovec *iov; - struct pmap *pmap; - - resid = uio->uio_resid; - iov = uio->uio_iov; - map->flags &= ~DMAMAP_TYPE_MASK; - map->flags |= DMAMAP_UIO; - map->buffer = uio; - map->len = 0; - - if (uio->uio_segflg == UIO_USERSPACE) { - KASSERT(uio->uio_td != NULL, - ("bus_dmamap_load_uio: USERSPACE but no proc")); - pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); - } else - pmap = kernel_pmap; - - error = 0; - nsegs = -1; - for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { - /* - * Now at the first iovec to load. Load each iovec - * until we have exhausted the residual count. - */ - bus_size_t minlen = - resid < iov[i].iov_len ? resid : iov[i].iov_len; - caddr_t addr = (caddr_t) iov[i].iov_base; - - if (minlen > 0) { - error = bus_dmamap_load_buffer(dmat, dmat->segments, - map, addr, minlen, pmap, flags, &lastaddr, &nsegs); - - map->len += minlen; - resid -= minlen; - } - } - - if (error) { - /* - * force "no valid mappings" on error in callback. - */ - (*callback)(callback_arg, NULL, 0, 0, error); - } else { - (*callback)(callback_arg, dmat->segments, nsegs+1, - uio->uio_resid, error); - } - CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); - return (error); + if (segs == NULL) + segs = dmat->segments; + return (segs); } /* @@ -1146,24 +1111,25 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, b { struct bounce_page *bpage; - map->flags &= ~DMAMAP_TYPE_MASK; while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { STAILQ_REMOVE_HEAD(&map->bpages, links); free_bounce_page(dmat, bpage); } + map->sync_count = 0; return; } static void -bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op, int bufaligned) +bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, + int bufaligned) { char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; register_t s; int partial; if ((op & BUS_DMASYNC_PREWRITE) && !(op & BUS_DMASYNC_PREREAD)) { - cpu_dcache_wb_range((vm_offset_t)buf, len); - cpu_l2cache_wb_range((vm_offset_t)buf, len); + cpu_dcache_wb_range(buf, len); + cpu_l2cache_wb_range(buf, len); } /* @@ -1186,38 +1152,37 @@ bus_dmamap_sync_buf(void *buf, int len, if (op & BUS_DMASYNC_PREREAD) { if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { - cpu_dcache_inv_range((vm_offset_t)buf, len); - cpu_l2cache_inv_range((vm_offset_t)buf, len); + cpu_dcache_inv_range(buf, len); + cpu_l2cache_inv_range(buf, len); } else { - cpu_dcache_wbinv_range((vm_offset_t)buf, len); - cpu_l2cache_wbinv_range((vm_offset_t)buf, len); + cpu_dcache_wbinv_range(buf, len); + cpu_l2cache_wbinv_range(buf, len); } } if (op & BUS_DMASYNC_POSTREAD) { if (partial && !bufaligned) { s = intr_disable(); - if ((vm_offset_t)buf & arm_dcache_align_mask) - memcpy(_tmp_cl, (void *)((vm_offset_t)buf & + if (buf & arm_dcache_align_mask) + memcpy(_tmp_cl, (void *)(buf & ~arm_dcache_align_mask), - (vm_offset_t)buf & arm_dcache_align_mask); - if (((vm_offset_t)buf + len) & arm_dcache_align_mask) + buf & arm_dcache_align_mask); + if ((buf + len) & arm_dcache_align_mask) memcpy(_tmp_clend, - (void *)((vm_offset_t)buf + len), - arm_dcache_align - (((vm_offset_t)(buf) + - len) & arm_dcache_align_mask)); + (void *)(buf + len), + arm_dcache_align - + ((buf + len) & arm_dcache_align_mask)); } - cpu_dcache_inv_range((vm_offset_t)buf, len); - cpu_l2cache_inv_range((vm_offset_t)buf, len); + cpu_dcache_inv_range(buf, len); + cpu_l2cache_inv_range(buf, len); if (partial && !bufaligned) { - if ((vm_offset_t)buf & arm_dcache_align_mask) - memcpy((void *)((vm_offset_t)buf & + if (buf & arm_dcache_align_mask) + memcpy((void *)(buf & ~arm_dcache_align_mask), _tmp_cl, - (vm_offset_t)buf & arm_dcache_align_mask); - if (((vm_offset_t)buf + len) & arm_dcache_align_mask) - memcpy((void *)((vm_offset_t)buf + len), + buf & arm_dcache_align_mask); + if ((buf + len) & arm_dcache_align_mask) + memcpy((void *)(buf + len), _tmp_clend, arm_dcache_align - - (((vm_offset_t)(buf) + len) & - arm_dcache_align_mask)); + ((buf + len) & arm_dcache_align_mask)); intr_restore(s); } } @@ -1230,10 +1195,18 @@ _bus_dmamap_sync_bp(bus_dma_tag_t dmat, STAILQ_FOREACH(bpage, &map->bpages, links) { if (op & BUS_DMASYNC_PREWRITE) { - bcopy((void *)bpage->datavaddr, - (void *)(bpage->vaddr_nocache != 0 ? - bpage->vaddr_nocache : bpage->vaddr), - bpage->datacount); + if (bpage->datavaddr != 0) + bcopy((void *)bpage->datavaddr, + (void *)(bpage->vaddr_nocache != 0 ? + bpage->vaddr_nocache : + bpage->vaddr), + bpage->datacount); + else + physcopyout(bpage->dataaddr, + (void *)(bpage->vaddr_nocache != 0 ? + bpage->vaddr_nocache : + bpage->vaddr), + bpage->datacount); if (bpage->vaddr_nocache == 0) { cpu_dcache_wb_range(bpage->vaddr, bpage->datacount); @@ -1249,36 +1222,23 @@ _bus_dmamap_sync_bp(bus_dma_tag_t dmat, cpu_l2cache_inv_range(bpage->vaddr, bpage->datacount); } - bcopy((void *)(bpage->vaddr_nocache != 0 ? - bpage->vaddr_nocache : bpage->vaddr), - (void *)bpage->datavaddr, bpage->datacount); + if (bpage->datavaddr != 0) + bcopy((void *)(bpage->vaddr_nocache != 0 ? + bpage->vaddr_nocache : bpage->vaddr), + (void *)bpage->datavaddr, bpage->datacount); + else + physcopyin((void *)(bpage->vaddr_nocache != 0 ? + bpage->vaddr_nocache : bpage->vaddr), + bpage->dataaddr, bpage->datacount); dmat->bounce_zone->total_bounced++; } } } -static __inline int -_bus_dma_buf_is_in_bp(bus_dmamap_t map, void *buf, int len) -{ - struct bounce_page *bpage; - - STAILQ_FOREACH(bpage, &map->bpages, links) { - if ((vm_offset_t)buf >= bpage->datavaddr && - (vm_offset_t)buf + len <= bpage->datavaddr + - bpage->datacount) - return (1); - } - return (0); - -} - void _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { - struct mbuf *m; - struct uio *uio; - int resid; - struct iovec *iov; + struct sync_list *sl, *end; int bufaligned; if (op == BUS_DMASYNC_POSTWRITE) @@ -1289,40 +1249,11 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus _bus_dmamap_sync_bp(dmat, map, op); CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); - switch(map->flags & DMAMAP_TYPE_MASK) { - case DMAMAP_LINEAR: - if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len))) - bus_dmamap_sync_buf(map->buffer, map->len, op, + if (map->sync_count) { + end = &map->slist[map->sync_count]; + for (sl = &map->slist[0]; sl != end; sl++) + bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op, bufaligned); - break; - case DMAMAP_MBUF: - m = map->buffer; - while (m) { - if (m->m_len > 0 && - !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len))) - bus_dmamap_sync_buf(m->m_data, m->m_len, op, - bufaligned); - m = m->m_next; - } - break; - case DMAMAP_UIO: - uio = map->buffer; - iov = uio->uio_iov; - resid = uio->uio_resid; - for (int i = 0; i < uio->uio_iovcnt && resid != 0; i++) { - bus_size_t minlen = resid < iov[i].iov_len ? resid : - iov[i].iov_len; - if (minlen > 0) { - if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base, - minlen)) - bus_dmamap_sync_buf(iov[i].iov_base, - minlen, op, bufaligned); - resid -= minlen; - } - } - break; - default: - break; } drain: @@ -1489,7 +1420,7 @@ reserve_bounce_pages(bus_dma_tag_t dmat, static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, - bus_size_t size) + bus_addr_t addr, bus_size_t size) { struct bounce_zone *bz; struct bounce_page *bpage; @@ -1522,6 +1453,7 @@ add_bounce_page(bus_dma_tag_t dmat, bus_ bpage->busaddr |= vaddr & PAGE_MASK; } bpage->datavaddr = vaddr; + bpage->dataaddr = addr; bpage->datacount = size; STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); return (bpage->busaddr); @@ -1575,8 +1507,8 @@ busdma_swi(void) mtx_unlock(&bounce_lock); dmat = map->dmat; (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); - bus_dmamap_load(map->dmat, map, map->buffer, map->len, - map->callback, map->callback_arg, /*flags*/0); + bus_dmamap_load_mem(map->dmat, map, &map->mem, + map->callback, map->callback_arg, BUS_DMA_WAITOK); (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); mtx_lock(&bounce_lock); } Modified: stable/9/sys/cam/cam_ccb.h ============================================================================== --- stable/9/sys/cam/cam_ccb.h Mon Jun 17 23:21:18 2013 (r251873) +++ stable/9/sys/cam/cam_ccb.h Tue Jun 18 00:36:53 2013 (r251874) @@ -64,13 +64,19 @@ typedef enum { * Perform transport negotiation * with this command. */ - CAM_SCATTER_VALID = 0x00000010,/* Scatter/gather list is valid */ + CAM_DATA_ISPHYS = 0x00000010,/* Data type with physical addrs */ CAM_DIS_AUTOSENSE = 0x00000020,/* Disable autosense feature */ CAM_DIR_BOTH = 0x00000000,/* Data direction (00:IN/OUT) */ CAM_DIR_IN = 0x00000040,/* Data direction (01:DATA IN) */ CAM_DIR_OUT = 0x00000080,/* Data direction (10:DATA OUT) */ CAM_DIR_NONE = 0x000000C0,/* Data direction (11:no data) */ CAM_DIR_MASK = 0x000000C0,/* Data direction Mask */ + CAM_DATA_VADDR = 0x00000000,/* Data type (000:Virtual) */ *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201306180036.r5I0askx009647>