Date: Thu, 20 Dec 2012 00:38:08 +0000 (UTC) From: Olivier Houchard <cognet@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r244471 - head/sys/arm/arm Message-ID: <201212200038.qBK0c8op023177@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cognet Date: Thu Dec 20 00:38:08 2012 New Revision: 244471 URL: http://svnweb.freebsd.org/changeset/base/244471 Log: Busdma enhancements, especially for managing small uncacheable buffers. - Use the new architecture-agnostic buffer pool manager that uses uma(9) to manage a set of power-of-2 sized buffers for bus_dmamem_alloc(). - Create pools of buffers backed by both regular and uncacheable memory, and use them to handle regular versus BUS_DMA_COHERENT allocations. - Use uma(9) to manage a pool of bus_dmamap structs instead of local code to manage a static list of 500 items (it took 3300 maps to get to multi-user mode, so the static pool wasn't much of an optimization). - Small BUS_DMA_COHERENT allocations no longer waste an entire page per allocation, or set pages to uncached when they contain data other than DMA buffers. There's no longer a need for drivers to work around the inefficiency by allocing large buffers then sub-dividing them. - Because we know the alignment and padding of buffers allocated by bus_dmamem_alloc() (whether coherent or regular memory, and whether obtained from the pool allocator or directly from the kernel) we can avoid doing partial cacheline flushes on them. - Add a fast-out to _bus_dma_could_bounce() (and some comments about what the routine really does because the old misplaced comment was wrong). - Everywhere the dma tag alignment is used, the interpretation is that an alignment of 1 means no special alignment. If the tag is created with an alignment argument of zero, store it in the tag as one, and remove all the code scattered around that changed 0->1 at point of use. - Remove stack-allocated arrays of segments, use a local array of two segments within the tag struct, or dynamically allocate an array at first use if nsegments > 2. On an arm system I tested, only 5 of 97 tags used more than two segments. On my x86 desktop it was only 7 of 111 tags. Submitted by: Ian Lepore <freebsd@damnhippie.dyndns.org> Modified: head/sys/arm/arm/busdma_machdep.c Modified: head/sys/arm/arm/busdma_machdep.c ============================================================================== --- head/sys/arm/arm/busdma_machdep.c Thu Dec 20 00:37:11 2012 (r244470) +++ head/sys/arm/arm/busdma_machdep.c Thu Dec 20 00:38:08 2012 (r244471) @@ -1,4 +1,5 @@ /*- + * Copyright (c) 2012 Ian Lepore * Copyright (c) 2004 Olivier Houchard * Copyright (c) 2002 Peter Grehan * Copyright (c) 1997, 1998 Justin T. Gibbs. @@ -32,7 +33,23 @@ __FBSDID("$FreeBSD$"); /* - * ARM bus dma support routines + * ARM bus dma support routines. + * + * XXX Things to investigate / fix some day... + * - What is the earliest that this API can be called? Could there be any + * fallout from changing the SYSINIT() order from SI_SUB_VM to SI_SUB_KMEM? + * - The manpage mentions the BUS_DMA_NOWAIT flag only in the context of the + * bus_dmamap_load() function. This code has historically (and still does) + * honor it in bus_dmamem_alloc(). If we got rid of that we could lose some + * error checking because some resource management calls would become WAITOK + * and thus "cannot fail." + * - The decisions made by _bus_dma_can_bounce() should be made once, at tag + * creation time, and the result stored in the tag. + * - It should be possible to take some shortcuts when mapping a buffer we know + * came from the uma(9) allocators based on what we know about such buffers + * (aligned, contiguous, etc). + * - The allocation of bounce pages could probably be cleaned up, then we could + * retire arm_remap_nocache(). */ #define _ARM32_BUS_DMA_PRIVATE @@ -40,6 +57,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/malloc.h> #include <sys/bus.h> +#include <sys/busdma_bufalloc.h> #include <sys/interrupt.h> #include <sys/lock.h> #include <sys/proc.h> @@ -50,7 +68,10 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/sysctl.h> +#include <vm/uma.h> #include <vm/vm.h> +#include <vm/vm_extern.h> +#include <vm/vm_kern.h> #include <vm/vm_page.h> #include <vm/vm_map.h> @@ -81,7 +102,6 @@ struct bus_dma_tag { int map_count; bus_dma_lock_t *lockfunc; void *lockfuncarg; - bus_dma_segment_t *segments; /* * DMA range for this tag. If the page doesn't fall within * one of these ranges, an error is returned. The caller @@ -91,6 +111,13 @@ struct bus_dma_tag { struct arm32_dma_range *ranges; int _nranges; struct bounce_zone *bounce_zone; + /* + * Most tags need one or two segments, and can use the local tagsegs + * array. For tags with a larger limit, we'll allocate a bigger array + * on first use. + */ + bus_dma_segment_t *segments; + bus_dma_segment_t tagsegs[2]; }; struct bounce_page { @@ -134,7 +161,7 @@ SYSCTL_INT(_hw_busdma, OID_AUTO, total_b #define DMAMAP_LINEAR 0x1 #define DMAMAP_MBUF 0x2 #define DMAMAP_UIO 0x4 -#define DMAMAP_ALLOCATED 0x10 +#define DMAMAP_CACHE_ALIGNED 0x10 #define DMAMAP_TYPE_MASK (DMAMAP_LINEAR|DMAMAP_MBUF|DMAMAP_UIO) #define DMAMAP_COHERENT 0x8 struct bus_dmamap { @@ -144,9 +171,6 @@ struct bus_dmamap { bus_dma_tag_t dmat; int flags; void *buffer; - void *origbuffer; - void *allocbuffer; - TAILQ_ENTRY(bus_dmamap) freelist; int len; STAILQ_ENTRY(bus_dmamap) links; bus_dmamap_callback_t *callback; @@ -157,12 +181,6 @@ struct bus_dmamap { static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; -static TAILQ_HEAD(,bus_dmamap) dmamap_freelist = - TAILQ_HEAD_INITIALIZER(dmamap_freelist); - -#define BUSDMA_STATIC_MAPS 500 -static struct bus_dmamap map_pool[BUSDMA_STATIC_MAPS]; - static struct mtx busdma_mtx; MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF); @@ -179,6 +197,87 @@ static void free_bounce_page(bus_dma_tag /* Default tag, as most drivers provide no parent tag. */ bus_dma_tag_t arm_root_dma_tag; +//---------------------------------------------------------------------------- +// Begin block of code useful to transplant to other implementations. + +static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */ + +static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ +static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ + +/* + * This is the ctor function passed to uma_zcreate() for the pool of dma maps. + * It'll need platform-specific changes if this code is copied. + */ +static int +dmamap_ctor(void *mem, int size, void *arg, int flags) +{ + bus_dmamap_t map; + bus_dma_tag_t dmat; + + map = (bus_dmamap_t)mem; + dmat = (bus_dma_tag_t)arg; + + dmat->map_count++; + + map->dmat = dmat; + map->flags = 0; + STAILQ_INIT(&map->bpages); + + return (0); +} + +/* + * This is the dtor function passed to uma_zcreate() for the pool of dma maps. + * It may need platform-specific changes if this code is copied . + */ +static void +dmamap_dtor(void *mem, int size, void *arg) +{ + bus_dmamap_t map; + + map = (bus_dmamap_t)mem; + + map->dmat->map_count--; +} + +static void +busdma_init(void *dummy) +{ + + /* Create a cache of maps for bus_dmamap_create(). */ + dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap), + dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0); + + /* Create a cache of buffers in standard (cacheable) memory. */ + standard_allocator = busdma_bufalloc_create("buffer", + arm_dcache_align, /* minimum_alignment */ + NULL, /* uma_alloc func */ + NULL, /* uma_free func */ + 0); /* uma_zcreate_flags */ + + /* + * Create a cache of buffers in uncacheable memory, to implement the + * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. + */ + coherent_allocator = busdma_bufalloc_create("coherent", + arm_dcache_align, /* minimum_alignment */ + busdma_bufalloc_alloc_uncacheable, + busdma_bufalloc_free_uncacheable, + 0); /* uma_zcreate_flags */ +} + +/* + * This init historically used SI_SUB_VM, but now the init code requires + * malloc(9) using M_DEVBUF memory, which is set up later than SI_SUB_VM, by + * SI_SUB_KMEM and SI_ORDER_SECOND, so we'll go right after that by using + * SI_SUB_KMEM and SI_ORDER_THIRD. + */ +SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_THIRD, busdma_init, NULL); + +// End block of code useful to transplant to other implementations. +//---------------------------------------------------------------------------- + /* * Return true if a match is made. * @@ -206,30 +305,26 @@ run_filter(bus_dma_tag_t dmat, bus_addr_ return (retval); } -static void -arm_dmamap_freelist_init(void *dummy) -{ - int i; - - for (i = 0; i < BUSDMA_STATIC_MAPS; i++) - TAILQ_INSERT_HEAD(&dmamap_freelist, &map_pool[i], freelist); -} - -SYSINIT(busdma, SI_SUB_VM, SI_ORDER_ANY, arm_dmamap_freelist_init, NULL); - /* - * Check to see if the specified page is in an allowed DMA range. + * This routine checks the exclusion zone constraints from a tag against the + * physical RAM available on the machine. If a tag specifies an exclusion zone + * but there's no RAM in that zone, then we avoid allocating resources to bounce + * a request, and we can use any memory allocator (as opposed to needing + * kmem_alloc_contig() just because it can allocate pages in an address range). + * + * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the + * same value on 32-bit architectures) as their lowaddr constraint, and we can't + * possibly have RAM at an address higher than the highest address we can + * express, so we take a fast out. */ - -static __inline int -bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dma_segment_t *segs, - bus_dmamap_t map, void *buf, bus_size_t buflen, struct pmap *pmap, - int flags, vm_offset_t *lastaddrp, int *segp); - static __inline int _bus_dma_can_bounce(vm_offset_t lowaddr, vm_offset_t highaddr) { int i; + + if (lowaddr >= BUS_SPACE_MAXADDR) + return (0); + for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { if ((lowaddr >= phys_avail[i] && lowaddr <= phys_avail[i + 1]) || (lowaddr < phys_avail[i] && @@ -294,38 +389,6 @@ dflt_lock(void *arg, bus_dma_lock_op_t o #endif } -static __inline bus_dmamap_t -_busdma_alloc_dmamap(void) -{ - bus_dmamap_t map; - - mtx_lock(&busdma_mtx); - map = TAILQ_FIRST(&dmamap_freelist); - if (map) - TAILQ_REMOVE(&dmamap_freelist, map, freelist); - mtx_unlock(&busdma_mtx); - if (!map) { - map = malloc(sizeof(*map), M_DEVBUF, M_NOWAIT | M_ZERO); - if (map) - map->flags = DMAMAP_ALLOCATED; - } else - map->flags = 0; - STAILQ_INIT(&map->bpages); - return (map); -} - -static __inline void -_busdma_free_dmamap(bus_dmamap_t map) -{ - if (map->flags & DMAMAP_ALLOCATED) - free(map, M_DEVBUF); - else { - mtx_lock(&busdma_mtx); - TAILQ_INSERT_HEAD(&dmamap_freelist, map, freelist); - mtx_unlock(&busdma_mtx); - } -} - /* * Allocate a device specific dma_tag. */ @@ -354,7 +417,7 @@ bus_dma_tag_create(bus_dma_tag_t parent, } newtag->parent = parent; - newtag->alignment = alignment; + newtag->alignment = alignment ? alignment : 1; newtag->boundary = boundary; newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1); newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1); @@ -375,9 +438,19 @@ bus_dma_tag_create(bus_dma_tag_t parent, newtag->lockfunc = dflt_lock; newtag->lockfuncarg = NULL; } - newtag->segments = NULL; - - /* + /* + * If all the segments we need fit into the local tagsegs array, set the + * pointer now. Otherwise NULL the pointer and an array of segments + * will be allocated later, on first use. We don't pre-allocate now + * because some tags exist just to pass contraints to children in the + * device hierarchy, and they tend to use BUS_SPACE_UNRESTRICTED and we + * sure don't want to try to allocate an array for that. + */ + if (newtag->nsegments <= nitems(newtag->tagsegs)) + newtag->segments = newtag->tagsegs; + else + newtag->segments = NULL; + /* * Take into account any restrictions imposed by our parent tag */ if (parent != NULL) { @@ -450,6 +523,7 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat) #endif if (dmat != NULL) { + if (dmat->map_count != 0) return (EBUSY); @@ -459,7 +533,8 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat) parent = dmat->parent; atomic_subtract_int(&dmat->ref_count, 1); if (dmat->ref_count == 0) { - if (dmat->segments != NULL) + if (dmat->segments != NULL && + dmat->segments != dmat->tagsegs) free(dmat->segments, M_DEVBUF); free(dmat, M_DEVBUF); /* @@ -485,29 +560,19 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat) int bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) { - bus_dmamap_t newmap; + bus_dmamap_t map; int error = 0; - if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc( - sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, - M_NOWAIT); - if (dmat->segments == NULL) { - CTR3(KTR_BUSDMA, "%s: tag %p error %d", - __func__, dmat, ENOMEM); - return (ENOMEM); - } - } + map = uma_zalloc_arg(dmamap_zone, dmat, M_WAITOK); + *mapp = map; - newmap = _busdma_alloc_dmamap(); - if (newmap == NULL) { - CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); - return (ENOMEM); - } - *mapp = newmap; - newmap->dmat = dmat; - newmap->allocbuffer = NULL; - dmat->map_count++; + /* + * If the tag's segments haven't been allocated yet we need to do it + * now, because we can't sleep for resources at map load time. + */ + if (dmat->segments == NULL) + dmat->segments = malloc(dmat->nsegments * + sizeof(*dmat->segments), M_DEVBUF, M_WAITOK); /* * Bouncing might be required if the driver asks for an active @@ -522,7 +587,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in if (dmat->bounce_zone == NULL) { if ((error = alloc_bounce_zone(dmat)) != 0) { - _busdma_free_dmamap(newmap); + uma_zfree(dmamap_zone, map); *mapp = NULL; return (error); } @@ -575,118 +640,129 @@ bus_dmamap_destroy(bus_dma_tag_t dmat, b __func__, dmat, EBUSY); return (EBUSY); } - _busdma_free_dmamap(map); + uma_zfree(dmamap_zone, map); if (dmat->bounce_zone) dmat->bounce_zone->map_count--; - dmat->map_count--; CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); return (0); } /* - * Allocate a piece of memory that can be efficiently mapped into - * bus device space based on the constraints lited in the dma tag. - * A dmamap to for use with dmamap_load is also allocated. + * Allocate a piece of memory that can be efficiently mapped into bus device + * space based on the constraints listed in the dma tag. Returns a pointer to + * the allocated memory, and a pointer to an associated bus_dmamap. */ int -bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, +bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddrp, int flags, bus_dmamap_t *mapp) { - bus_dmamap_t newmap = NULL; - + void * vaddr; + struct busdma_bufzone *bufzone; + busdma_bufalloc_t ba; + bus_dmamap_t map; int mflags; + vm_memattr_t memattr; if (flags & BUS_DMA_NOWAIT) mflags = M_NOWAIT; else mflags = M_WAITOK; - if (dmat->segments == NULL) { - dmat->segments = (bus_dma_segment_t *)malloc( - sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF, - mflags); - if (dmat->segments == NULL) { - CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", - __func__, dmat, dmat->flags, ENOMEM); - return (ENOMEM); - } + + /* + * If the tag's segments haven't been allocated yet we need to do it + * now, because we can't sleep for resources at map load time. + */ + if (dmat->segments == NULL) + dmat->segments = malloc(dmat->nsegments * + sizeof(*dmat->segments), M_DEVBUF, mflags); + + map = uma_zalloc_arg(dmamap_zone, dmat, mflags); + if (map == NULL) + return (ENOMEM); + + if (flags & BUS_DMA_COHERENT) { + memattr = VM_MEMATTR_UNCACHEABLE; + ba = coherent_allocator; + map->flags |= DMAMAP_COHERENT; + } else { + memattr = VM_MEMATTR_DEFAULT; + ba = standard_allocator; } + /* All buffers we allocate are cache-aligned. */ + map->flags |= DMAMAP_CACHE_ALIGNED; + if (flags & BUS_DMA_ZERO) mflags |= M_ZERO; - newmap = _busdma_alloc_dmamap(); - if (newmap == NULL) { - CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", - __func__, dmat, dmat->flags, ENOMEM); - return (ENOMEM); + /* + * Try to find a bufzone in the allocator that holds a cache of buffers + * of the right size for this request. If the buffer is too big to be + * held in the allocator cache, this returns NULL. + */ + bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); + + /* + * Allocate the buffer from the uma(9) allocator if... + * - It's small enough to be in the allocator (bufzone not NULL). + * - The alignment constraint isn't larger than the allocation size + * (the allocator aligns buffers to their size boundaries). + * - There's no need to handle lowaddr/highaddr exclusion zones. + * else allocate non-contiguous pages if... + * - The page count that could get allocated doesn't exceed nsegments. + * - The alignment constraint isn't larger than a page boundary. + * - There are no boundary-crossing constraints. + * else allocate a block of contiguous pages because one or more of the + * constraints is something that only the contig allocator can fulfill. + */ + if (bufzone != NULL && dmat->alignment <= bufzone->size && + !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { + vaddr = uma_zalloc(bufzone->umazone, mflags); + } else if (dmat->nsegments >= btoc(dmat->maxsize) && + dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) { + vaddr = (void *)kmem_alloc_attr(kernel_map, dmat->maxsize, + mflags, 0, dmat->lowaddr, memattr); + } else { + vaddr = (void *)kmem_alloc_contig(kernel_map, dmat->maxsize, + mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary, + memattr); } - dmat->map_count++; - *mapp = newmap; - newmap->dmat = dmat; - - if (dmat->maxsize <= PAGE_SIZE && - (dmat->alignment < dmat->maxsize) && - !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) { - *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags); - } else { - /* - * XXX Use Contigmalloc until it is merged into this facility - * and handles multi-seg allocations. Nobody is doing - * multi-seg allocations yet though. - */ - *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, - 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul, - dmat->boundary); - } - if (*vaddr == NULL) { - if (newmap != NULL) { - _busdma_free_dmamap(newmap); - dmat->map_count--; - } - *mapp = NULL; - return (ENOMEM); + + if (vaddr == NULL) { + uma_zfree(dmamap_zone, map); + map = NULL; } - if (flags & BUS_DMA_COHERENT) { - void *tmpaddr = arm_remap_nocache( - (void *)((vm_offset_t)*vaddr &~ PAGE_MASK), - dmat->maxsize + ((vm_offset_t)*vaddr & PAGE_MASK)); - - if (tmpaddr) { - tmpaddr = (void *)((vm_offset_t)(tmpaddr) + - ((vm_offset_t)*vaddr & PAGE_MASK)); - newmap->origbuffer = *vaddr; - newmap->allocbuffer = tmpaddr; - *vaddr = tmpaddr; - } else - newmap->origbuffer = newmap->allocbuffer = NULL; - } else - newmap->origbuffer = newmap->allocbuffer = NULL; - return (0); + + *vaddrp = vaddr; + *mapp = map; + + return (vaddr == NULL ? ENOMEM : 0); } /* - * Free a piece of memory and it's allocated dmamap, that was allocated - * via bus_dmamem_alloc. Make the same choice for free/contigfree. + * Free a piece of memory that was allocated via bus_dmamem_alloc, along with + * its associated map. */ void bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) { - if (map->allocbuffer) { - KASSERT(map->allocbuffer == vaddr, - ("Trying to freeing the wrong DMA buffer")); - vaddr = map->origbuffer; - arm_unmap_nocache(map->allocbuffer, - dmat->maxsize + ((vm_offset_t)vaddr & PAGE_MASK)); - } - if (dmat->maxsize <= PAGE_SIZE && - dmat->alignment < dmat->maxsize && + struct busdma_bufzone *bufzone; + busdma_bufalloc_t ba; + + if (map->flags & DMAMAP_COHERENT) + ba = coherent_allocator; + else + ba = standard_allocator; + uma_zfree(dmamap_zone, map); + + /* Be careful not to access map from here on. */ + + bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize); + + if (bufzone != NULL && dmat->alignment <= bufzone->size && !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) - free(vaddr, M_DEVBUF); - else { - contigfree(vaddr, dmat->maxsize, M_DEVBUF); - } - dmat->map_count--; - _busdma_free_dmamap(map); - CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); + uma_zfree(bufzone->umazone, vaddr); + else + kmem_free(kernel_map, (vm_offset_t)vaddr, dmat->maxsize); } static int @@ -795,10 +871,6 @@ bus_dmamap_load_buffer(bus_dma_tag_t dma else curaddr = (*pde & L1_S_FRAME) | (vaddr & L1_S_OFFSET); - if (*pde & L1_S_CACHE_MASK) { - map->flags &= - ~DMAMAP_COHERENT; - } } else { pte = *ptep; KASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV, @@ -807,18 +879,9 @@ bus_dmamap_load_buffer(bus_dma_tag_t dma == L2_TYPE_L)) { curaddr = (pte & L2_L_FRAME) | (vaddr & L2_L_OFFSET); - if (pte & L2_L_CACHE_MASK) { - map->flags &= - ~DMAMAP_COHERENT; - - } } else { curaddr = (pte & L2_S_FRAME) | (vaddr & L2_S_OFFSET); - if (pte & L2_S_CACHE_MASK) { - map->flags &= - ~DMAMAP_COHERENT; - } } } } else { @@ -914,7 +977,7 @@ bus_dmamap_load(bus_dma_tag_t dmat, bus_ map->callback = callback; map->callback_arg = callback_arg; map->flags &= ~DMAMAP_TYPE_MASK; - map->flags |= DMAMAP_LINEAR|DMAMAP_COHERENT; + map->flags |= DMAMAP_LINEAR; map->buffer = buf; map->len = buflen; error = bus_dmamap_load_buffer(dmat, @@ -935,6 +998,15 @@ bus_dmamap_load(bus_dma_tag_t dmat, bus_ /* * Like bus_dmamap_load(), but for mbufs. + * + * Note that the manpage states that BUS_DMA_NOWAIT is implied for mbufs. + * + * We know that the way the system allocates and uses mbufs implies that we can + * treat them as DMAMAP_CACHE_ALIGNED in terms of handling partial cache line + * flushes. Even though the flush may reference the data area within the mbuf + * that isn't aligned to a cache line, we know the overall mbuf itself is + * properly aligned, and we know that the CPU will not touch the header fields + * before the data area while the DMA is in progress. */ int bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, @@ -945,8 +1017,9 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat, M_ASSERTPKTHDR(m0); + flags |= BUS_DMA_NOWAIT; map->flags &= ~DMAMAP_TYPE_MASK; - map->flags |= DMAMAP_MBUF | DMAMAP_COHERENT; + map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; map->buffer = m0; map->len = 0; if (m0->m_pkthdr.len <= dmat->maxsize) { @@ -969,7 +1042,7 @@ bus_dmamap_load_mbuf(bus_dma_tag_t dmat, /* * force "no valid mappings" on error in callback. */ - (*callback)(callback_arg, dmat->segments, 0, 0, error); + (*callback)(callback_arg, NULL, 0, 0, error); } else { (*callback)(callback_arg, dmat->segments, nsegs + 1, m0->m_pkthdr.len, error); @@ -991,7 +1064,7 @@ bus_dmamap_load_mbuf_sg(bus_dma_tag_t dm flags |= BUS_DMA_NOWAIT; *nsegs = -1; map->flags &= ~DMAMAP_TYPE_MASK; - map->flags |= DMAMAP_MBUF | DMAMAP_COHERENT; + map->flags |= DMAMAP_MBUF | DMAMAP_CACHE_ALIGNED; map->buffer = m0; map->len = 0; if (m0->m_pkthdr.len <= dmat->maxsize) { @@ -1035,7 +1108,7 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat, resid = uio->uio_resid; iov = uio->uio_iov; map->flags &= ~DMAMAP_TYPE_MASK; - map->flags |= DMAMAP_UIO|DMAMAP_COHERENT; + map->flags |= DMAMAP_UIO; map->buffer = uio; map->len = 0; @@ -1058,7 +1131,7 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat, caddr_t addr = (caddr_t) iov[i].iov_base; if (minlen > 0) { - error = bus_dmamap_load_buffer(dmat, dmat->segments, + error = bus_dmamap_load_buffer(dmat, dmat->segments, map, addr, minlen, pmap, flags, &lastaddr, &nsegs); map->len += minlen; @@ -1070,7 +1143,7 @@ bus_dmamap_load_uio(bus_dma_tag_t dmat, /* * force "no valid mappings" on error in callback. */ - (*callback)(callback_arg, dmat->segments, 0, 0, error); + (*callback)(callback_arg, NULL, 0, 0, error); } else { (*callback)(callback_arg, dmat->segments, nsegs+1, uio->uio_resid, error); @@ -1098,7 +1171,7 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, b } static void -bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op) +bus_dmamap_sync_buf(void *buf, int len, bus_dmasync_op_t op, int bufaligned) { char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; register_t s; @@ -1108,7 +1181,25 @@ bus_dmamap_sync_buf(void *buf, int len, cpu_dcache_wb_range((vm_offset_t)buf, len); cpu_l2cache_wb_range((vm_offset_t)buf, len); } + + /* + * If the caller promises the buffer is properly aligned to a cache line + * (even if the call parms make it look like it isn't) we can avoid + * attempting to preserve the non-DMA part of the cache line in the + * POSTREAD case, but we MUST still do a writeback in the PREREAD case. + * + * This covers the case of mbufs, where we know how they're aligned and + * know the CPU doesn't touch the header in front of the DMA data area + * during the IO, but it may have touched it right before invoking the + * sync, so a PREREAD writeback is required. + * + * It also handles buffers we created in bus_dmamem_alloc(), which are + * always aligned and padded to cache line size even if the IO length + * isn't a multiple of cache line size. In this case the PREREAD + * writeback probably isn't required, but it's harmless. + */ partial = (((vm_offset_t)buf) | len) & arm_dcache_align_mask; + if (op & BUS_DMASYNC_PREREAD) { if (!(op & BUS_DMASYNC_PREWRITE) && !partial) { cpu_dcache_inv_range((vm_offset_t)buf, len); @@ -1119,7 +1210,7 @@ bus_dmamap_sync_buf(void *buf, int len, } } if (op & BUS_DMASYNC_POSTREAD) { - if (partial) { + if (partial && !bufaligned) { s = intr_disable(); if ((vm_offset_t)buf & arm_dcache_align_mask) memcpy(_tmp_cl, (void *)((vm_offset_t)buf & @@ -1133,7 +1224,7 @@ bus_dmamap_sync_buf(void *buf, int len, } cpu_dcache_inv_range((vm_offset_t)buf, len); cpu_l2cache_inv_range((vm_offset_t)buf, len); - if (partial) { + if (partial && !bufaligned) { if ((vm_offset_t)buf & arm_dcache_align_mask) memcpy((void *)((vm_offset_t)buf & ~arm_dcache_align_mask), _tmp_cl, @@ -1204,25 +1295,29 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus struct uio *uio; int resid; struct iovec *iov; - + int bufaligned; + if (op == BUS_DMASYNC_POSTWRITE) return; + if (map->flags & DMAMAP_COHERENT) + goto drain; if (STAILQ_FIRST(&map->bpages)) _bus_dmamap_sync_bp(dmat, map, op); - if (map->flags & DMAMAP_COHERENT) - return; CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags); + bufaligned = (map->flags & DMAMAP_CACHE_ALIGNED); switch(map->flags & DMAMAP_TYPE_MASK) { case DMAMAP_LINEAR: if (!(_bus_dma_buf_is_in_bp(map, map->buffer, map->len))) - bus_dmamap_sync_buf(map->buffer, map->len, op); + bus_dmamap_sync_buf(map->buffer, map->len, op, + bufaligned); break; case DMAMAP_MBUF: m = map->buffer; while (m) { if (m->m_len > 0 && !(_bus_dma_buf_is_in_bp(map, m->m_data, m->m_len))) - bus_dmamap_sync_buf(m->m_data, m->m_len, op); + bus_dmamap_sync_buf(m->m_data, m->m_len, op, + bufaligned); m = m->m_next; } break; @@ -1237,7 +1332,7 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus if (!_bus_dma_buf_is_in_bp(map, iov[i].iov_base, minlen)) bus_dmamap_sync_buf(iov[i].iov_base, - minlen, op); + minlen, op, bufaligned); resid -= minlen; } } @@ -1245,6 +1340,9 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus default: break; } + +drain: + cpu_drain_writebuf(); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201212200038.qBK0c8op023177>