Date: Sat, 29 Dec 2012 22:18:02 +0000 (UTC) From: Jeff Roberson <jeff@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r244838 - in projects/physbio/sys: arm/arm ia64/ia64 mips/mips powerpc/powerpc x86/x86 Message-ID: <201212292218.qBTMI2XE060710@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jeff Date: Sat Dec 29 22:18:02 2012 New Revision: 244838 URL: http://svnweb.freebsd.org/changeset/base/244838 Log: - Add support for bouncing physical addresses to all architectures that support bounce buffers. Sponsored by: EMC / Isilon Storage Division Modified: projects/physbio/sys/arm/arm/busdma_machdep-v6.c projects/physbio/sys/arm/arm/busdma_machdep.c projects/physbio/sys/ia64/ia64/busdma_machdep.c projects/physbio/sys/mips/mips/busdma_machdep.c projects/physbio/sys/powerpc/powerpc/busdma_machdep.c projects/physbio/sys/x86/x86/busdma_machdep.c Modified: projects/physbio/sys/arm/arm/busdma_machdep-v6.c ============================================================================== --- projects/physbio/sys/arm/arm/busdma_machdep-v6.c Sat Dec 29 21:22:38 2012 (r244837) +++ projects/physbio/sys/arm/arm/busdma_machdep-v6.c Sat Dec 29 22:18:02 2012 (r244838) @@ -166,8 +166,12 @@ static bus_addr_t add_bounce_page(bus_dm bus_size_t size); static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); -static int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, +static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, int flags); +static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, + vm_paddr_t buf, bus_size_t buflen, int flags); +static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, + int flags); static __inline int _bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) @@ -649,7 +653,37 @@ bus_dmamem_free(bus_dma_tag_t dmat, void CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); } -static int +static void +_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags) +{ + bus_addr_t curaddr; + bus_size_t sgsize; + + if (map->pagesneeded == 0) { + CTR5(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d" + " map= %p, pagesneeded= %d", + dmat->lowaddr, dmat->boundary, dmat->alignment, + map, map->pagesneeded); + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + curaddr = buf; + while (buflen != 0) { + sgsize = MIN(buflen, dmat->maxsegsz); + if (run_filter(dmat, curaddr) != 0) { + sgsize = MIN(sgsize, PAGE_SIZE); + map->pagesneeded++; + } + curaddr += sgsize; + buflen -= sgsize; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); + } +} + +static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, int flags) { @@ -674,8 +708,7 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm paddr = pmap_kextract(vaddr); else paddr = pmap_extract(map->pmap, vaddr); - if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && - run_filter(dmat, paddr) != 0) { + if (run_filter(dmat, paddr) != 0) { map->pagesneeded++; } vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); @@ -683,27 +716,29 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm } CTR1(KTR_BUSDMA, "pagesneeded= %d", map->pagesneeded); } +} + +static int +_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) +{ /* Reserve Necessary Bounce Pages */ - if (map->pagesneeded != 0) { - mtx_lock(&bounce_lock); - if (flags & BUS_DMA_NOWAIT) { - if (reserve_bounce_pages(dmat, map, 0) != 0) { - map->pagesneeded = 0; - mtx_unlock(&bounce_lock); - return (ENOMEM); - } - } else { - if (reserve_bounce_pages(dmat, map, 1) != 0) { - /* Queue us for resources */ - STAILQ_INSERT_TAIL(&bounce_map_waitinglist, - map, links); - mtx_unlock(&bounce_lock); - return (EINPROGRESS); - } + mtx_lock(&bounce_lock); + if (flags & BUS_DMA_NOWAIT) { + if (reserve_bounce_pages(dmat, map, 0) != 0) { + map->pagesneeded = 0; + mtx_unlock(&bounce_lock); + return (ENOMEM); + } + } else { + if (reserve_bounce_pages(dmat, map, 1) != 0) { + /* Queue us for resources */ + STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); + mtx_unlock(&bounce_lock); + return (EINPROGRESS); } - mtx_unlock(&bounce_lock); } + mtx_unlock(&bounce_lock); return (0); } @@ -784,18 +819,34 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat { bus_addr_t curaddr; bus_size_t sgsize; + int error; if (segs == NULL) segs = dmat->segments; - curaddr = buf; + if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + while (buflen > 0) { + curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); + if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && + map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + sgsize = MIN(sgsize, PAGE_SIZE); + curaddr = add_bounce_page(dmat, map, 0, curaddr, + sgsize); + } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; - curaddr += sgsize; + buf += sgsize; buflen -= sgsize; } @@ -832,9 +883,12 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm segs = dmat->segments; if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { - error = _bus_dmamap_count_pages(dmat, map, buf, buflen, flags); - if (error) - return (error); + _bus_dmamap_count_pages(dmat, map, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } } sl = NULL; Modified: projects/physbio/sys/arm/arm/busdma_machdep.c ============================================================================== --- projects/physbio/sys/arm/arm/busdma_machdep.c Sat Dec 29 21:22:38 2012 (r244837) +++ projects/physbio/sys/arm/arm/busdma_machdep.c Sat Dec 29 22:18:02 2012 (r244838) @@ -698,7 +698,37 @@ bus_dmamem_free(bus_dma_tag_t dmat, void CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); } -static int +static void +_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags) +{ + bus_addr_t curaddr; + bus_size_t sgsize; + + if ((map->pagesneeded == 0)) { + CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", + dmat->lowaddr, dmat->boundary, dmat->alignment); + CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", + map, map->pagesneeded); + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + curaddr = buf; + while (buflen != 0) { + sgsize = MIN(buflen, dmat->maxsegsz); + if (run_filter(dmat, curaddr) != 0) { + sgsize = MIN(sgsize, PAGE_SIZE); + map->pagesneeded++; + } + curaddr += sgsize; + buflen -= sgsize; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } +} + +static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, void *buf, bus_size_t buflen, int flags) { @@ -723,33 +753,34 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm paddr = pmap_kextract(vaddr); else paddr = pmap_extract(pmap, vaddr); - if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && - run_filter(dmat, paddr) != 0) + if (run_filter(dmat, paddr) != 0) map->pagesneeded++; vaddr += PAGE_SIZE; } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } +} + +static int +_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) +{ /* Reserve Necessary Bounce Pages */ - if (map->pagesneeded != 0) { - mtx_lock(&bounce_lock); - if (flags & BUS_DMA_NOWAIT) { - if (reserve_bounce_pages(dmat, map, 0) != 0) { - mtx_unlock(&bounce_lock); - return (ENOMEM); - } - } else { - if (reserve_bounce_pages(dmat, map, 1) != 0) { - /* Queue us for resources */ - STAILQ_INSERT_TAIL(&bounce_map_waitinglist, - map, links); - mtx_unlock(&bounce_lock); - return (EINPROGRESS); - } + mtx_lock(&bounce_lock); + if (flags & BUS_DMA_NOWAIT) { + if (reserve_bounce_pages(dmat, map, 0) != 0) { + mtx_unlock(&bounce_lock); + return (ENOMEM); + } + } else { + if (reserve_bounce_pages(dmat, map, 1) != 0) { + /* Queue us for resources */ + STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); + mtx_unlock(&bounce_lock); + return (EINPROGRESS); } - mtx_unlock(&bounce_lock); } + mtx_unlock(&bounce_lock); return (0); } @@ -819,18 +850,34 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat { bus_size_t sgsize; bus_addr_t curaddr; + int error; if (segs == NULL) segs = dmat->segments; - curaddr = buf; + if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + while (buflen > 0) { + curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); + if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && + map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + sgsize = MIN(sgsize, PAGE_SIZE); + curaddr = add_bounce_page(dmat, map, 0, curaddr, + sgsize); + } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; - curaddr += sgsize; + buf += sgsize; buflen -= sgsize; } @@ -865,10 +912,12 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm segs = dmat->segments; if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { - error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, - flags); - if (error) - return (error); + _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } } CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); Modified: projects/physbio/sys/ia64/ia64/busdma_machdep.c ============================================================================== --- projects/physbio/sys/ia64/ia64/busdma_machdep.c Sat Dec 29 21:22:38 2012 (r244837) +++ projects/physbio/sys/ia64/ia64/busdma_machdep.c Sat Dec 29 22:18:02 2012 (r244838) @@ -479,6 +479,88 @@ bus_dmamem_free(bus_dma_tag_t dmat, void } } +static void +_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags) +{ + bus_addr_t curaddr; + bus_size_t sgsize; + + if ((dmat->lowaddr < paddr_max || dmat->boundary > 0 || + dmat->alignment > 1) && map != &nobounce_dmamap && + map->pagesneeded == 0) { + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + curaddr = buf; + while (buflen != 0) { + sgsize = MIN(buflen, dmat->maxsegsz); + if (run_filter(dmat, curaddr, 0) != 0) { + sgsize = MIN(sgsize, PAGE_SIZE); + map->pagesneeded++; + } + curaddr += sgsize; + buflen -= sgsize; + } + } +} + +static void +_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, + void *buf, bus_size_t buflen, int flags) +{ + vm_offset_t vaddr; + vm_offset_t vendaddr; + bus_addr_t paddr; + + if ((dmat->lowaddr < paddr_max || dmat->boundary > 0 || + dmat->alignment > 1) && map != &nobounce_dmamap && + map->pagesneeded == 0) { + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + vaddr = trunc_page((vm_offset_t)buf); + vendaddr = (vm_offset_t)buf + buflen; + + while (vaddr < vendaddr) { + if (pmap == kernel_pmap) + paddr = pmap_kextract(vaddr); + else + paddr = pmap_extract(pmap, vaddr); + if (run_filter(dmat, paddr, 0) != 0) + map->pagesneeded++; + vaddr += PAGE_SIZE; + } + } +} + +static void +_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) +{ + + /* Reserve Necessary Bounce Pages */ + mtx_lock(&bounce_lock); + if (flags & BUS_DMA_NOWAIT) { + if (reserve_bounce_pages(dmat, map, 0) != 0) { + mtx_unlock(&bounce_lock); + return (ENOMEM); + } + } else { + if (reserve_bounce_pages(dmat, map, 1) != 0) { + /* Queue us for resources */ + STAILQ_INSERT_TAIL(&bounce_map_waitinglist, + map, links); + mtx_unlock(&bounce_lock); + return (EINPROGRESS); + } + } + mtx_unlock(&bounce_lock); + + return (0) +} + /* * Add a single contiguous physical range to the segment list. */ @@ -536,6 +618,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat { bus_addr_t curaddr; bus_size_t sgsize; + int error; if (map == NULL) map = &nobounce_dmamap; @@ -543,14 +626,29 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat if (segs == NULL) segs = dmat->segments; - curaddr = buf; + if (map != &nobounce_dmamap) { + _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + while (buflen > 0) { + curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); + if (map->pagesneeded != 0 && + run_filter(dmat, curaddr, sgsize)) { + sgsize = MIN(sgsize, PAGE_SIZE); + curaddr = add_bounce_page(dmat, map, 0, curaddr, + sgsize); + } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; - curaddr += sgsize; + buf += sgsize; buflen -= sgsize; } @@ -579,51 +677,17 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm if (segs == NULL) segs = dmat->segments; - if ((dmat->lowaddr < paddr_max || dmat->boundary > 0 || - dmat->alignment > 1) && map != &nobounce_dmamap && - map->pagesneeded == 0) { - vm_offset_t vendaddr; - - /* - * Count the number of bounce pages - * needed in order to complete this transfer - */ - vaddr = trunc_page((vm_offset_t)buf); - vendaddr = (vm_offset_t)buf + buflen; - - while (vaddr < vendaddr) { - if (pmap == kernel_pmap) - paddr = pmap_kextract(vaddr); - else - paddr = pmap_extract(pmap, vaddr); - if (run_filter(dmat, paddr, 0) != 0) - map->pagesneeded++; - vaddr += PAGE_SIZE; + if (map != &nobounce_dmamap) { + _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); } } vaddr = (vm_offset_t)buf; - /* Reserve Necessary Bounce Pages */ - if (map->pagesneeded != 0) { - mtx_lock(&bounce_lock); - if (flags & BUS_DMA_NOWAIT) { - if (reserve_bounce_pages(dmat, map, 0) != 0) { - mtx_unlock(&bounce_lock); - return (ENOMEM); - } - } else { - if (reserve_bounce_pages(dmat, map, 1) != 0) { - /* Queue us for resources */ - STAILQ_INSERT_TAIL(&bounce_map_waitinglist, - map, links); - mtx_unlock(&bounce_lock); - return (EINPROGRESS); - } - } - mtx_unlock(&bounce_lock); - } - while (buflen > 0) { /* * Get the physical address for this segment. Modified: projects/physbio/sys/mips/mips/busdma_machdep.c ============================================================================== --- projects/physbio/sys/mips/mips/busdma_machdep.c Sat Dec 29 21:22:38 2012 (r244837) +++ projects/physbio/sys/mips/mips/busdma_machdep.c Sat Dec 29 22:18:02 2012 (r244838) @@ -693,7 +693,37 @@ bus_dmamem_free(bus_dma_tag_t dmat, void CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); } -static int +static void +_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags) +{ + bus_addr_t curaddr; + bus_size_t sgsize; + + if ((map->pagesneeded == 0)) { + CTR3(KTR_BUSDMA, "lowaddr= %d, boundary= %d, alignment= %d", + dmat->lowaddr, dmat->boundary, dmat->alignment); + CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", + map, map->pagesneeded); + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + curaddr = buf; + while (buflen != 0) { + sgsize = MIN(buflen, dmat->maxsegsz); + if (run_filter(dmat, curaddr) != 0) { + sgsize = MIN(sgsize, PAGE_SIZE); + map->pagesneeded++; + } + curaddr += sgsize; + buflen -= sgsize; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } +} + +static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, void *buf, bus_size_t buflen, int flags) { @@ -728,26 +758,29 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } +} + +static int +_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map,int flags) +{ /* Reserve Necessary Bounce Pages */ - if (map->pagesneeded != 0) { - mtx_lock(&bounce_lock); - if (flags & BUS_DMA_NOWAIT) { - if (reserve_bounce_pages(dmat, map, 0) != 0) { - mtx_unlock(&bounce_lock); - return (ENOMEM); - } - } else { - if (reserve_bounce_pages(dmat, map, 1) != 0) { - /* Queue us for resources */ - STAILQ_INSERT_TAIL(&bounce_map_waitinglist, - map, links); - mtx_unlock(&bounce_lock); - return (EINPROGRESS); - } + mtx_lock(&bounce_lock); + if (flags & BUS_DMA_NOWAIT) { + if (reserve_bounce_pages(dmat, map, 0) != 0) { + mtx_unlock(&bounce_lock); + return (ENOMEM); + } + } else { + if (reserve_bounce_pages(dmat, map, 1) != 0) { + /* Queue us for resources */ + STAILQ_INSERT_TAIL(&bounce_map_waitinglist, + map, links); + mtx_unlock(&bounce_lock); + return (EINPROGRESS); } - mtx_unlock(&bounce_lock); } + mtx_unlock(&bounce_lock); return (0); } @@ -803,18 +836,34 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat { bus_addr_t curaddr; bus_size_t sgsize; + int error; if (segs == NULL) segs = dmat->segments; - curaddr = buf; + if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + while (buflen > 0) { + curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); + if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && + map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + sgsize = MIN(sgsize, PAGE_SIZE); + curaddr = add_bounce_page(dmat, map, 0, curaddr, + sgsize); + } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; - curaddr += sgsize; + buf += sgsize; buflen -= sgsize; } @@ -849,10 +898,12 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm segs = dmat->segments; if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { - error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, - flags); - if (error) - return (error); + _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } } CTR3(KTR_BUSDMA, "lowaddr= %d boundary= %d, " "alignment= %d", dmat->lowaddr, dmat->boundary, dmat->alignment); Modified: projects/physbio/sys/powerpc/powerpc/busdma_machdep.c ============================================================================== --- projects/physbio/sys/powerpc/powerpc/busdma_machdep.c Sat Dec 29 21:22:38 2012 (r244837) +++ projects/physbio/sys/powerpc/powerpc/busdma_machdep.c Sat Dec 29 22:18:02 2012 (r244838) @@ -564,6 +564,99 @@ bus_dmamem_free(bus_dma_tag_t dmat, void CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); } +static void +_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags) +{ + bus_addr_t curaddr; + bus_size_t sgsize; + + if (map->pagesneeded == 0) { + CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " + "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), + dmat->boundary, dmat->alignment); + CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded); + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + curaddr = buf; + while (buflen != 0) { + sgsize = MIN(buflen, dmat->maxsegsz); + if (run_filter(dmat, curaddr) != 0) { + sgsize = MIN(sgsize, PAGE_SIZE); + map->pagesneeded++; + } + curaddr += sgsize; + buflen -= sgsize; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } +} + +static void +_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, + void *buf, bus_size_t buflen, int flags) +{ + vm_offset_t vaddr; + vm_offset_t vendaddr; + bus_addr_t paddr; + + if (map->pagesneeded == 0) { + CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " + "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), + dmat->boundary, dmat->alignment); + CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded); + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + vaddr = (vm_offset_t)buf; + vendaddr = (vm_offset_t)buf + buflen; + + while (vaddr < vendaddr) { + bus_size_t sg_len; + + sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); + if (pmap == kernel_pmap) + paddr = pmap_kextract(vaddr); + else + paddr = pmap_extract(pmap, vaddr); + if (run_filter(dmat, paddr) != 0) { + sg_len = roundup2(sg_len, dmat->alignment); + map->pagesneeded++; + } + vaddr += sg_len; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } +} + +static int +_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) +{ + + /* Reserve Necessary Bounce Pages */ + mtx_lock(&bounce_lock); + if (flags & BUS_DMA_NOWAIT) { + if (reserve_bounce_pages(dmat, map, 0) != 0) { + mtx_unlock(&bounce_lock); + return (ENOMEM); + } + } else { + if (reserve_bounce_pages(dmat, map, 1) != 0) { + /* Queue us for resources */ + STAILQ_INSERT_TAIL(&bounce_map_waitinglist, + map, links); + mtx_unlock(&bounce_lock); + return (EINPROGRESS); + } + } + mtx_unlock(&bounce_lock); + + return (0); +} + /* * Add a single contiguous physical range to the segment list. */ @@ -624,18 +717,33 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat { bus_addr_t curaddr; bus_size_t sgsize; + int error; if (segs == NULL) segs = map->segments; - curaddr = buf; + if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + while (buflen > 0) { + curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); + if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + sgsize = MIN(sgsize, PAGE_SIZE); + curaddr = add_bounce_page(dmat, map, 0, curaddr, + sgsize); + } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; - curaddr += sgsize; + buf += sgsize; buflen -= sgsize; } @@ -666,55 +774,13 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm if (segs == NULL) segs = map->segments; - if (map->pagesneeded == 0 && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) { - vm_offset_t vendaddr; - - CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " - "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), - dmat->boundary, dmat->alignment); - CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded); - /* - * Count the number of bounce pages - * needed in order to complete this transfer - */ - vaddr = (vm_offset_t)buf; - vendaddr = (vm_offset_t)buf + buflen; - - while (vaddr < vendaddr) { - bus_size_t sg_len; - - sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); - if (pmap == kernel_pmap) - paddr = pmap_kextract(vaddr); - else - paddr = pmap_extract(pmap, vaddr); - if (run_filter(dmat, paddr) != 0) { - sg_len = roundup2(sg_len, dmat->alignment); - map->pagesneeded++; - } - vaddr += sg_len; + if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); } - CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); - } - - /* Reserve Necessary Bounce Pages */ - if (map->pagesneeded != 0) { - mtx_lock(&bounce_lock); - if (flags & BUS_DMA_NOWAIT) { - if (reserve_bounce_pages(dmat, map, 0) != 0) { - mtx_unlock(&bounce_lock); - return (ENOMEM); - } - } else { - if (reserve_bounce_pages(dmat, map, 1) != 0) { - /* Queue us for resources */ - STAILQ_INSERT_TAIL(&bounce_map_waitinglist, - map, links); - mtx_unlock(&bounce_lock); - return (EINPROGRESS); - } - } - mtx_unlock(&bounce_lock); } vaddr = (vm_offset_t)buf; Modified: projects/physbio/sys/x86/x86/busdma_machdep.c ============================================================================== --- projects/physbio/sys/x86/x86/busdma_machdep.c Sat Dec 29 21:22:38 2012 (r244837) +++ projects/physbio/sys/x86/x86/busdma_machdep.c Sat Dec 29 22:18:02 2012 (r244838) @@ -144,8 +144,14 @@ static bus_addr_t add_bounce_page(bus_dm bus_size_t size); static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); -int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, - void *buf, bus_size_t buflen, int flags); +static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, + pmap_t pmap, void *buf, bus_size_t buflen, + int flags); +static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, + vm_paddr_t buf, bus_size_t buflen, + int flags); +static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, + int flags); #ifdef XEN #undef pmap_kextract @@ -579,7 +585,33 @@ bus_dmamem_free(bus_dma_tag_t dmat, void CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); } -int +static void +_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags) +{ + bus_addr_t curaddr; + bus_size_t sgsize; + + if ((map != &nobounce_dmamap && map->pagesneeded == 0)) { + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + curaddr = buf; + while (buflen != 0) { + sgsize = MIN(buflen, dmat->maxsegsz); + if (run_filter(dmat, curaddr)) { + sgsize = MIN(sgsize, PAGE_SIZE); + map->pagesneeded++; + } + curaddr += sgsize; + buflen -= sgsize; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } +} + +static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, void *buf, bus_size_t buflen, int flags) { @@ -608,8 +640,7 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm paddr = pmap_kextract(vaddr); else paddr = pmap_extract(pmap, vaddr); - if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && - run_filter(dmat, paddr) != 0) { + if (run_filter(dmat, paddr) != 0) { sg_len = roundup2(sg_len, dmat->alignment); map->pagesneeded++; } @@ -617,26 +648,28 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } +} + +static int +_bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) +{ /* Reserve Necessary Bounce Pages */ - if (map->pagesneeded != 0) { - mtx_lock(&bounce_lock); - if (flags & BUS_DMA_NOWAIT) { - if (reserve_bounce_pages(dmat, map, 0) != 0) { - mtx_unlock(&bounce_lock); - return (ENOMEM); - } - } else { - if (reserve_bounce_pages(dmat, map, 1) != 0) { - /* Queue us for resources */ - STAILQ_INSERT_TAIL(&bounce_map_waitinglist, - map, links); - mtx_unlock(&bounce_lock); - return (EINPROGRESS); - } + mtx_lock(&bounce_lock); + if (flags & BUS_DMA_NOWAIT) { + if (reserve_bounce_pages(dmat, map, 0) != 0) { + mtx_unlock(&bounce_lock); + return (ENOMEM); + } + } else { + if (reserve_bounce_pages(dmat, map, 1) != 0) { + /* Queue us for resources */ + STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); + mtx_unlock(&bounce_lock); + return (EINPROGRESS); } - mtx_unlock(&bounce_lock); } + mtx_unlock(&bounce_lock); return (0); } @@ -701,6 +734,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat { bus_size_t sgsize; bus_addr_t curaddr; + int error; if (map == NULL || map == &contig_dmamap) map = &nobounce_dmamap; @@ -708,14 +742,29 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat if (segs == NULL) segs = dmat->segments; - curaddr = buf; + if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + while (buflen > 0) { + curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); + if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && + map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + sgsize = MIN(sgsize, PAGE_SIZE); + curaddr = add_bounce_page(dmat, map, 0, curaddr, + sgsize); + } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, segp); if (sgsize == 0) break; - curaddr += sgsize; + buf += sgsize; buflen -= sgsize; } @@ -750,9 +799,12 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm segs = dmat->segments; if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { - error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); - if (error) - return (error); + _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201212292218.qBTMI2XE060710>