Date: Tue, 6 Apr 2010 14:35:30 +0000 (UTC) From: Nathan Whitehorn <nwhitehorn@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r206272 - in projects/ppc64/sys/powerpc: aim booke powerpc Message-ID: <201004061435.o36EZU9T092513@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: nwhitehorn Date: Tue Apr 6 14:35:30 2010 New Revision: 206272 URL: http://svn.freebsd.org/changeset/base/206272 Log: Add support for bounce pages to make DMA usable on memory above 4 GB (2 GB on Apple machines) by replacing the existing PowerPC busdma_machdep.c with the one from amd64. Eventually we will need support for IOMMUs, in particular to run under most PowerPC hypervisors, but this will do for now. This commit is the final piece of the powerpc64 port. Obtained from: amd64 Modified: projects/ppc64/sys/powerpc/aim/vm_machdep.c projects/ppc64/sys/powerpc/booke/vm_machdep.c projects/ppc64/sys/powerpc/powerpc/busdma_machdep.c Modified: projects/ppc64/sys/powerpc/aim/vm_machdep.c ============================================================================== --- projects/ppc64/sys/powerpc/aim/vm_machdep.c Tue Apr 6 14:07:48 2010 (r206271) +++ projects/ppc64/sys/powerpc/aim/vm_machdep.c Tue Apr 6 14:35:30 2010 (r206272) @@ -371,10 +371,9 @@ sf_buf_free(struct sf_buf *sf) void swi_vm(void *dummy) { -#if 0 /* XXX: Don't have busdma stuff yet */ + if (busdma_swi_pending != 0) busdma_swi(); -#endif } /* Modified: projects/ppc64/sys/powerpc/booke/vm_machdep.c ============================================================================== --- projects/ppc64/sys/powerpc/booke/vm_machdep.c Tue Apr 6 14:07:48 2010 (r206271) +++ projects/ppc64/sys/powerpc/booke/vm_machdep.c Tue Apr 6 14:35:30 2010 (r206272) @@ -357,10 +357,9 @@ sf_buf_free(struct sf_buf *sf) void swi_vm(void *dummy) { -#if 0 /* XXX: Don't have busdma stuff yet */ + if (busdma_swi_pending != 0) busdma_swi(); -#endif } /* Modified: projects/ppc64/sys/powerpc/powerpc/busdma_machdep.c ============================================================================== --- projects/ppc64/sys/powerpc/powerpc/busdma_machdep.c Tue Apr 6 14:07:48 2010 (r206271) +++ projects/ppc64/sys/powerpc/powerpc/busdma_machdep.c Tue Apr 6 14:35:30 2010 (r206272) @@ -1,5 +1,4 @@ /*- - * Copyright (c) 2002 Peter Grehan * Copyright (c) 1997, 1998 Justin T. Gibbs. * All rights reserved. * @@ -23,27 +22,28 @@ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. - * - * From i386/busdma_machdep.c,v 1.26 2002/04/19 22:58:09 alfred */ -#include <sys/cdefs.h> -__FBSDID("$FreeBSD$"); - /* - * Bus dma support routines + * From amd64/busdma_machdep.c, r204214 */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + #include <sys/param.h> #include <sys/systm.h> #include <sys/malloc.h> #include <sys/bus.h> #include <sys/interrupt.h> +#include <sys/kernel.h> +#include <sys/ktr.h> #include <sys/lock.h> #include <sys/proc.h> #include <sys/mutex.h> #include <sys/mbuf.h> #include <sys/uio.h> +#include <sys/sysctl.h> #include <vm/vm.h> #include <vm/vm_page.h> @@ -51,34 +51,122 @@ __FBSDID("$FreeBSD$"); #include <machine/atomic.h> #include <machine/bus.h> -#include <machine/cpufunc.h> +#include <machine/md_var.h> + +#define MAX_BPAGES 8192 + +struct bounce_zone; struct bus_dma_tag { - bus_dma_tag_t parent; - bus_size_t alignment; - bus_size_t boundary; - bus_addr_t lowaddr; - bus_addr_t highaddr; + bus_dma_tag_t parent; + bus_size_t alignment; + bus_size_t boundary; + bus_addr_t lowaddr; + bus_addr_t highaddr; bus_dma_filter_t *filter; - void *filterarg; - bus_size_t maxsize; - u_int nsegments; - bus_size_t maxsegsz; - int flags; - int ref_count; - int map_count; + void *filterarg; + bus_size_t maxsize; + u_int nsegments; + bus_size_t maxsegsz; + int flags; + int ref_count; + int map_count; bus_dma_lock_t *lockfunc; void *lockfuncarg; + bus_dma_segment_t *segments; + struct bounce_zone *bounce_zone; +}; + +struct bounce_page { + vm_offset_t vaddr; /* kva of bounce buffer */ + bus_addr_t busaddr; /* Physical address */ + vm_offset_t datavaddr; /* kva of client data */ + bus_size_t datacount; /* client data count */ + STAILQ_ENTRY(bounce_page) links; +}; + +int busdma_swi_pending; + +struct bounce_zone { + STAILQ_ENTRY(bounce_zone) links; + STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; + int total_bpages; + int free_bpages; + int reserved_bpages; + int active_bpages; + int total_bounced; + int total_deferred; + int map_count; + bus_size_t alignment; + bus_addr_t lowaddr; + char zoneid[8]; + char lowaddrid[20]; + struct sysctl_ctx_list sysctl_tree; + struct sysctl_oid *sysctl_tree_top; }; +static struct mtx bounce_lock; +static int total_bpages; +static int busdma_zonecount; +static STAILQ_HEAD(, bounce_zone) bounce_zone_list; + +SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); +SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, + "Total bounce pages"); + struct bus_dmamap { - bus_dma_tag_t dmat; - void *buf; /* unmapped buffer pointer */ - bus_size_t buflen; /* unmapped buffer length */ - bus_dmamap_callback_t *callback; - void *callback_arg; + struct bp_list bpages; + int pagesneeded; + int pagesreserved; + bus_dma_tag_t dmat; + void *buf; /* unmapped buffer pointer */ + bus_size_t buflen; /* unmapped buffer length */ + bus_dmamap_callback_t *callback; + void *callback_arg; + STAILQ_ENTRY(bus_dmamap) links; }; +static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; +static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; +static struct bus_dmamap nobounce_dmamap; + +static void init_bounce_pages(void *dummy); +static int alloc_bounce_zone(bus_dma_tag_t dmat); +static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); +static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, + int commit); +static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, + vm_offset_t vaddr, bus_size_t size); +static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); +static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); + +/* + * Return true if a match is made. + * + * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'. + * + * If paddr is within the bounds of the dma tag then call the filter callback + * to check for a match, if there is no filter callback then assume a match. + */ +static __inline int +run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) +{ + int retval; + + retval = 0; + + do { + if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) + || ((paddr & (dmat->alignment - 1)) != 0)) + && (dmat->filter == NULL + || (*dmat->filter)(dmat->filterarg, paddr) != 0)) + retval = 1; + + dmat = dmat->parent; + } while (retval == 0 && dmat != NULL); + return (retval); +} + /* * Convenience function for manipulating driver locks from busdma (during * busdma_swi, for example). Drivers that don't provide their own locks @@ -112,42 +200,53 @@ busdma_lock_mutex(void *arg, bus_dma_loc static void dflt_lock(void *arg, bus_dma_lock_op_t op) { -#ifdef INVARIANTS panic("driver error: busdma dflt_lock called"); -#else - printf("DRIVER_ERROR: busdma dflt_lock called\n"); -#endif } +#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 +#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 /* * Allocate a device specific dma_tag. */ int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, - bus_size_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, - bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, - int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, - void *lockfuncarg, bus_dma_tag_t *dmat) + bus_size_t boundary, bus_addr_t lowaddr, + bus_addr_t highaddr, bus_dma_filter_t *filter, + void *filterarg, bus_size_t maxsize, int nsegments, + bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, + void *lockfuncarg, bus_dma_tag_t *dmat) { bus_dma_tag_t newtag; int error = 0; + /* Basic sanity checking */ + if (boundary != 0 && boundary < maxsegsz) + maxsegsz = boundary; + + if (maxsegsz == 0) { + return (EINVAL); + } + /* Return a NULL tag on failure */ *dmat = NULL; - newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT); - if (newtag == NULL) + newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, + M_ZERO | M_NOWAIT); + if (newtag == NULL) { + CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", + __func__, newtag, 0, error); return (ENOMEM); + } newtag->parent = parent; newtag->alignment = alignment; newtag->boundary = boundary; - newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); - newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); + newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); + newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); newtag->filter = filter; newtag->filterarg = filterarg; - newtag->maxsize = maxsize; - newtag->nsegments = nsegments; + newtag->maxsize = maxsize; + newtag->nsegments = nsegments; newtag->maxsegsz = maxsegsz; newtag->flags = flags; newtag->ref_count = 1; /* Count ourself */ @@ -159,61 +258,107 @@ bus_dma_tag_create(bus_dma_tag_t parent, newtag->lockfunc = dflt_lock; newtag->lockfuncarg = NULL; } + newtag->segments = NULL; - /* - * Take into account any restrictions imposed by our parent tag - */ - if (parent != NULL) { - newtag->lowaddr = min(parent->lowaddr, newtag->lowaddr); - newtag->highaddr = max(parent->highaddr, newtag->highaddr); + /* Take into account any restrictions imposed by our parent tag */ + if (parent != NULL) { + newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); + newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); if (newtag->boundary == 0) newtag->boundary = parent->boundary; else if (parent->boundary != 0) - newtag->boundary = MIN(parent->boundary, + newtag->boundary = MIN(parent->boundary, newtag->boundary); - if (newtag->filter == NULL) { - /* - * Short circuit looking at our parent directly - * since we have encapsulated all of its information - */ - newtag->filter = parent->filter; - newtag->filterarg = parent->filterarg; - newtag->parent = parent->parent; + if (newtag->filter == NULL) { + /* + * Short circuit looking at our parent directly + * since we have encapsulated all of its information + */ + newtag->filter = parent->filter; + newtag->filterarg = parent->filterarg; + newtag->parent = parent->parent; } if (newtag->parent != NULL) atomic_add_int(&parent->ref_count, 1); } - *dmat = newtag; + if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) + || newtag->alignment > 1) + newtag->flags |= BUS_DMA_COULD_BOUNCE; + + if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && + (flags & BUS_DMA_ALLOCNOW) != 0) { + struct bounce_zone *bz; + + /* Must bounce */ + + if ((error = alloc_bounce_zone(newtag)) != 0) { + free(newtag, M_DEVBUF); + return (error); + } + bz = newtag->bounce_zone; + + if (ptoa(bz->total_bpages) < maxsize) { + int pages; + + pages = atop(maxsize) - bz->total_bpages; + + /* Add pages to our bounce pool */ + if (alloc_bounce_pages(newtag, pages) < pages) + error = ENOMEM; + } + /* Performed initial allocation */ + newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; + } + + if (error != 0) { + free(newtag, M_DEVBUF); + } else { + *dmat = newtag; + } + CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", + __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); return (error); } int bus_dma_tag_destroy(bus_dma_tag_t dmat) { + bus_dma_tag_t dmat_copy; + int error; + + error = 0; + dmat_copy = dmat; + if (dmat != NULL) { - - if (dmat->map_count != 0) - return (EBUSY); - - while (dmat != NULL) { - bus_dma_tag_t parent; - - parent = dmat->parent; - atomic_subtract_int(&dmat->ref_count, 1); - if (dmat->ref_count == 0) { - free(dmat, M_DEVBUF); - /* - * Last reference count, so - * release our reference - * count on our parent. - */ - dmat = parent; - } else - dmat = NULL; - } - } - return (0); + + if (dmat->map_count != 0) { + error = EBUSY; + goto out; + } + + while (dmat != NULL) { + bus_dma_tag_t parent; + + parent = dmat->parent; + atomic_subtract_int(&dmat->ref_count, 1); + if (dmat->ref_count == 0) { + if (dmat->segments != NULL) + free(dmat->segments, M_DEVBUF); + free(dmat, M_DEVBUF); + /* + * Last reference count, so + * release our reference + * count on our parent. + */ + dmat = parent; + } else + dmat = NULL; + } + } +out: + CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); + return (error); } /* @@ -223,10 +368,83 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat) int bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) { - *mapp = NULL; - dmat->map_count++; + int error; - return (0); + error = 0; + + if (dmat->segments == NULL) { + dmat->segments = (bus_dma_segment_t *)malloc( + sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, + M_NOWAIT); + if (dmat->segments == NULL) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", + __func__, dmat, ENOMEM); + return (ENOMEM); + } + } + + /* + * Bouncing might be required if the driver asks for an active + * exclusion region, a data alignment that is stricter than 1, and/or + * an active address boundary. + */ + if (dmat->flags & BUS_DMA_COULD_BOUNCE) { + + /* Must bounce */ + struct bounce_zone *bz; + int maxpages; + + if (dmat->bounce_zone == NULL) { + if ((error = alloc_bounce_zone(dmat)) != 0) + return (error); + } + bz = dmat->bounce_zone; + + *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, + M_NOWAIT | M_ZERO); + if (*mapp == NULL) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", + __func__, dmat, ENOMEM); + return (ENOMEM); + } + + /* Initialize the new map */ + STAILQ_INIT(&((*mapp)->bpages)); + + /* + * Attempt to add pages to our pool on a per-instance + * basis up to a sane limit. + */ + if (dmat->alignment > 1) + maxpages = MAX_BPAGES; + else + maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr)); + if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 + || (bz->map_count > 0 && bz->total_bpages < maxpages)) { + int pages; + + pages = MAX(atop(dmat->maxsize), 1); + pages = MIN(maxpages - bz->total_bpages, pages); + pages = MAX(pages, 1); + if (alloc_bounce_pages(dmat, pages) < pages) + error = ENOMEM; + + if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { + if (error == 0) + dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; + } else { + error = 0; + } + } + bz->map_count++; + } else { + *mapp = NULL; + } + if (error == 0) + dmat->map_count++; + CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", + __func__, dmat, dmat->flags, error); + return (error); } /* @@ -236,13 +454,22 @@ bus_dmamap_create(bus_dma_tag_t dmat, in int bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) { - if (map != NULL) { - panic("dmamap_destroy: NULL?\n"); - } - dmat->map_count--; - return (0); + if (map != NULL && map != &nobounce_dmamap) { + if (STAILQ_FIRST(&map->bpages) != NULL) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", + __func__, dmat, EBUSY); + return (EBUSY); + } + if (dmat->bounce_zone) + dmat->bounce_zone->map_count--; + free(map, M_DEVBUF); + } + dmat->map_count--; + CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); + return (0); } + /* * Allocate a piece of memory that can be efficiently mapped into * bus device space based on the constraints lited in the dma tag. @@ -250,7 +477,7 @@ bus_dmamap_destroy(bus_dma_tag_t dmat, b */ int bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, - bus_dmamap_t *mapp) + bus_dmamap_t *mapp) { int mflags; @@ -258,56 +485,87 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi mflags = M_NOWAIT; else mflags = M_WAITOK; + + /* If we succeed, no mapping/bouncing will be required */ + *mapp = NULL; + + if (dmat->segments == NULL) { + dmat->segments = (bus_dma_segment_t *)malloc( + sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, + mflags); + if (dmat->segments == NULL) { + CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", + __func__, dmat, dmat->flags, ENOMEM); + return (ENOMEM); + } + } if (flags & BUS_DMA_ZERO) mflags |= M_ZERO; - *mapp = NULL; - /* * XXX: * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact * alignment guarantees of malloc need to be nailed down, and the * code below should be rewritten to take that into account. * - * In the meantime, we'll return an error if malloc gets it wrong. + * In the meantime, we'll warn the user if malloc gets it wrong. */ - if (dmat->maxsize <= PAGE_SIZE && - dmat->alignment < dmat->maxsize) { + if ((dmat->maxsize <= PAGE_SIZE) && + (dmat->alignment < dmat->maxsize) && + dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) { *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); - } else { - /* - * XXX Use Contigmalloc until it is merged into this facility - * and handles multi-seg allocations. Nobody is doing - * multi-seg allocations yet though. - */ - *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, - 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, - dmat->boundary); - } - - if (*vaddr == NULL) - return (ENOMEM); - - if ((uintptr_t)*vaddr % dmat->alignment) - printf("XXX: %s: alignment not respected!\n", __func__); - - return (0); + } else { + /* + * XXX Use Contigmalloc until it is merged into this facility + * and handles multi-seg allocations. Nobody is doing + * multi-seg allocations yet though. + * XXX Certain AGP hardware does. + */ + *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, + 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, + dmat->boundary); + } + if (*vaddr == NULL) { + CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", + __func__, dmat, dmat->flags, ENOMEM); + return (ENOMEM); + } else if ((uintptr_t)*vaddr & (dmat->alignment - 1)) { + printf("bus_dmamem_alloc failed to align memory properly.\n"); + } +#ifdef NOTYET + if (flags & BUS_DMA_NOCACHE) + pmap_change_attr((vm_offset_t)*vaddr, dmat->maxsize, + PAT_UNCACHEABLE); +#endif + CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", + __func__, dmat, dmat->flags, 0); + return (0); } /* - * Free a piece of memory and it's allocated dmamap, that was allocated + * Free a piece of memory and it's allociated dmamap, that was allocated * via bus_dmamem_alloc. Make the same choice for free/contigfree. */ void bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) { - if (map != NULL) - panic("bus_dmamem_free: Invalid map freed\n"); - if (dmat->maxsize <= PAGE_SIZE && - dmat->alignment < dmat->maxsize) + /* + * dmamem does not need to be bounced, so the map should be + * NULL + */ + if (map != NULL) + panic("bus_dmamem_free: Invalid map freed\n"); +#ifdef NOTYET + pmap_change_attr((vm_offset_t)vaddr, dmat->maxsize, PAT_WRITE_BACK); +#endif + if ((dmat->maxsize <= PAGE_SIZE) && + (dmat->alignment < dmat->maxsize) && + dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) free(vaddr, M_DEVBUF); - else + else { contigfree(vaddr, dmat->maxsize, M_DEVBUF); + } + CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); } /* @@ -316,26 +574,89 @@ bus_dmamem_free(bus_dma_tag_t dmat, void * the starting segment on entrance, and the ending segment on exit. * first indicates if this is the first invocation of this function. */ -static int -bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t segs[], - void *buf, bus_size_t buflen, struct thread *td, int flags, - vm_offset_t *lastaddrp, int *segp, int first) +static __inline int +_bus_dmamap_load_buffer(bus_dma_tag_t dmat, + bus_dmamap_t map, + void *buf, bus_size_t buflen, + pmap_t pmap, + int flags, + bus_addr_t *lastaddrp, + bus_dma_segment_t *segs, + int *segp, + int first) { bus_size_t sgsize; bus_addr_t curaddr, lastaddr, baddr, bmask; - vm_offset_t vaddr = (vm_offset_t)buf; + vm_offset_t vaddr; + bus_addr_t paddr; int seg; - pmap_t pmap; - if (td != NULL) - pmap = vmspace_pmap(td->td_proc->p_vmspace); - else - pmap = NULL; + if (map == NULL) + map = &nobounce_dmamap; + + if ((map != &nobounce_dmamap && map->pagesneeded == 0) + && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) { + vm_offset_t vendaddr; + + CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " + "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), + dmat->boundary, dmat->alignment); + CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d", + map, &nobounce_dmamap, map->pagesneeded); + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + vaddr = (vm_offset_t)buf; + vendaddr = (vm_offset_t)buf + buflen; + + while (vaddr < vendaddr) { + bus_size_t sg_len; + sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); + if (pmap) + paddr = pmap_extract(pmap, vaddr); + else + paddr = pmap_kextract(vaddr); + if (run_filter(dmat, paddr) != 0) { + sg_len = roundup2(sg_len, dmat->alignment); + map->pagesneeded++; + } + vaddr += sg_len; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } + + /* Reserve Necessary Bounce Pages */ + if (map->pagesneeded != 0) { + mtx_lock(&bounce_lock); + if (flags & BUS_DMA_NOWAIT) { + if (reserve_bounce_pages(dmat, map, 0) != 0) { + mtx_unlock(&bounce_lock); + return (ENOMEM); + } + } else { + if (reserve_bounce_pages(dmat, map, 1) != 0) { + /* Queue us for resources */ + map->dmat = dmat; + map->buf = buf; + map->buflen = buflen; + STAILQ_INSERT_TAIL(&bounce_map_waitinglist, + map, links); + mtx_unlock(&bounce_lock); + return (EINPROGRESS); + } + } + mtx_unlock(&bounce_lock); + } + + vaddr = (vm_offset_t)buf; lastaddr = *lastaddrp; bmask = ~(dmat->boundary - 1); for (seg = *segp; buflen > 0 ; ) { + bus_size_t max_sgsize; + /* * Get the physical address for this segment. */ @@ -347,11 +668,15 @@ bus_dmamap_load_buffer(bus_dma_tag_t dma /* * Compute the segment size, and adjust counts. */ - sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); - if (sgsize > dmat->maxsegsz) - sgsize = dmat->maxsegsz; - if (buflen < sgsize) - sgsize = buflen; + max_sgsize = MIN(buflen, dmat->maxsegsz); + sgsize = PAGE_SIZE - ((vm_offset_t)curaddr & PAGE_MASK); + if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + sgsize = roundup2(sgsize, dmat->alignment); + sgsize = MIN(sgsize, max_sgsize); + curaddr = add_bounce_page(dmat, map, vaddr, sgsize); + } else { + sgsize = MIN(sgsize, max_sgsize); + } /* * Make sure we don't cross any boundaries. @@ -364,7 +689,7 @@ bus_dmamap_load_buffer(bus_dma_tag_t dma /* * Insert chunk into a segment, coalescing with - * the previous segment if possible. + * previous segment if possible. */ if (first) { segs[seg].ds_addr = curaddr; @@ -403,59 +728,71 @@ bus_dmamap_load_buffer(bus_dma_tag_t dma */ int bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, - bus_size_t buflen, bus_dmamap_callback_t *callback, - void *callback_arg, int flags) + bus_size_t buflen, bus_dmamap_callback_t *callback, + void *callback_arg, int flags) { -#ifdef __CC_SUPPORTS_DYNAMIC_ARRAY_INIT - bus_dma_segment_t dm_segments[dmat->nsegments]; -#else - bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; -#endif - vm_offset_t lastaddr; - int error, nsegs; + bus_addr_t lastaddr = 0; + int error, nsegs = 0; - if (map != NULL) - panic("bus_dmamap_load: Invalid map\n"); + if (map != NULL) { + flags |= BUS_DMA_WAITOK; + map->callback = callback; + map->callback_arg = callback_arg; + } - lastaddr = (vm_offset_t)0; - nsegs = 0; - error = bus_dmamap_load_buffer(dmat, dm_segments, buf, buflen, - NULL, flags, &lastaddr, &nsegs, 1); + error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, + &lastaddr, dmat->segments, &nsegs, 1); - if (error == 0) - (*callback)(callback_arg, dm_segments, nsegs + 1, 0); + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, dmat->flags, error, nsegs + 1); + + if (error == EINPROGRESS) { + return (error); + } + + if (error) + (*callback)(callback_arg, dmat->segments, 0, error); else - (*callback)(callback_arg, NULL, 0, error); + (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); + + /* + * Return ENOMEM to the caller so that it can pass it up the stack. + * This error only happens when NOWAIT is set, so deferal is disabled. + */ + if (error == ENOMEM) + return (error); return (0); } + /* - * Like bus_dmamap_load(), but for mbufs. + * Like _bus_dmamap_load(), but for mbufs. */ int -bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, - bus_dmamap_callback2_t *callback, void *callback_arg, int flags) +bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, + struct mbuf *m0, + bus_dmamap_callback2_t *callback, void *callback_arg, + int flags) { -#ifdef __CC_SUPPORTS_DYNAMIC_ARRAY_INIT - bus_dma_segment_t dm_segments[dmat->nsegments]; -#else - bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; -#endif - int nsegs = 0, error = 0; + int nsegs, error; M_ASSERTPKTHDR(m0); + flags |= BUS_DMA_NOWAIT; + nsegs = 0; + error = 0; if (m0->m_pkthdr.len <= dmat->maxsize) { int first = 1; - vm_offset_t lastaddr = 0; + bus_addr_t lastaddr = 0; struct mbuf *m; for (m = m0; m != NULL && error == 0; m = m->m_next) { if (m->m_len > 0) { - error = bus_dmamap_load_buffer(dmat, - dm_segments, m->m_data, m->m_len, NULL, - flags, &lastaddr, &nsegs, first); + error = _bus_dmamap_load_buffer(dmat, map, + m->m_data, m->m_len, + NULL, flags, &lastaddr, + dmat->segments, &nsegs, first); first = 0; } } @@ -464,90 +801,96 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat, } if (error) { - /* - * force "no valid mappings" on error in callback. - */ - (*callback)(callback_arg, dm_segments, 0, 0, error); + /* force "no valid mappings" in callback */ + (*callback)(callback_arg, dmat->segments, 0, 0, error); } else { - (*callback)(callback_arg, dm_segments, nsegs+1, - m0->m_pkthdr.len, error); + (*callback)(callback_arg, dmat->segments, + nsegs+1, m0->m_pkthdr.len, error); } + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, dmat->flags, error, nsegs + 1); return (error); } int -bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, - bus_dma_segment_t *segs, int *nsegs, int flags) +bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, + struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, + int flags) { - int error = 0; + int error; M_ASSERTPKTHDR(m0); + flags |= BUS_DMA_NOWAIT; *nsegs = 0; - + error = 0; if (m0->m_pkthdr.len <= dmat->maxsize) { int first = 1; - vm_offset_t lastaddr = 0; + bus_addr_t lastaddr = 0; struct mbuf *m; for (m = m0; m != NULL && error == 0; m = m->m_next) { if (m->m_len > 0) { - error = bus_dmamap_load_buffer(dmat, - segs, m->m_data, m->m_len, NULL, - flags, &lastaddr, nsegs, first); + error = _bus_dmamap_load_buffer(dmat, map, + m->m_data, m->m_len, + NULL, flags, &lastaddr, + segs, nsegs, first); first = 0; } } - ++*nsegs; } else { error = EINVAL; } + /* XXX FIXME: Having to increment nsegs is really annoying */ + ++*nsegs; + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, dmat->flags, error, *nsegs); return (error); } /* - * Like bus_dmamap_load(), but for uios. + * Like _bus_dmamap_load(), but for uios. */ int -bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio, - bus_dmamap_callback2_t *callback, void *callback_arg, int flags) +bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, + struct uio *uio, + bus_dmamap_callback2_t *callback, void *callback_arg, + int flags) { - vm_offset_t lastaddr; -#ifdef __CC_SUPPORTS_DYNAMIC_ARRAY_INIT - bus_dma_segment_t dm_segments[dmat->nsegments]; -#else - bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS]; -#endif - int nsegs, i, error, first; + bus_addr_t lastaddr = 0; + int nsegs, error, first, i; bus_size_t resid; struct iovec *iov; - struct thread *td = NULL; + pmap_t pmap; + flags |= BUS_DMA_NOWAIT; resid = uio->uio_resid; iov = uio->uio_iov; if (uio->uio_segflg == UIO_USERSPACE) { - td = uio->uio_td; - KASSERT(td != NULL, *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201004061435.o36EZU9T092513>