From owner-svn-src-head@FreeBSD.ORG Wed Aug 7 15:44:58 2013 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTP id 914942A7; Wed, 7 Aug 2013 15:44:58 +0000 (UTC) (envelope-from cognet@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id 7E3A0242B; Wed, 7 Aug 2013 15:44:58 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.7/8.14.7) with ESMTP id r77FiwQe072983; Wed, 7 Aug 2013 15:44:58 GMT (envelope-from cognet@svn.freebsd.org) Received: (from cognet@localhost) by svn.freebsd.org (8.14.7/8.14.5/Submit) id r77FiwMK072982; Wed, 7 Aug 2013 15:44:58 GMT (envelope-from cognet@svn.freebsd.org) Message-Id: <201308071544.r77FiwMK072982@svn.freebsd.org> From: Olivier Houchard Date: Wed, 7 Aug 2013 15:44:58 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r254061 - head/sys/arm/arm X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 07 Aug 2013 15:44:58 -0000 Author: cognet Date: Wed Aug 7 15:44:58 2013 New Revision: 254061 URL: http://svnweb.freebsd.org/changeset/base/254061 Log: Don't bother trying to work around buffers which are not aligned on a cache line boundary. It has never been 100% correct, and it can't work on SMP, because nothing prevents another core from accessing data from an unrelated buffer in the same cache line while we invalidated it. Just use bounce pages instead. Reviewed by: ian Approved by: mux (mentor) (implicit) Modified: head/sys/arm/arm/busdma_machdep-v6.c Modified: head/sys/arm/arm/busdma_machdep-v6.c ============================================================================== --- head/sys/arm/arm/busdma_machdep-v6.c Wed Aug 7 12:03:34 2013 (r254060) +++ head/sys/arm/arm/busdma_machdep-v6.c Wed Aug 7 15:44:58 2013 (r254061) @@ -67,8 +67,6 @@ __FBSDID("$FreeBSD$"); #define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 -#define FIX_DMAP_BUS_DMASYNC_POSTREAD - struct bounce_zone; struct bus_dma_tag { @@ -179,7 +177,7 @@ static bus_addr_t add_bounce_page(bus_dm vm_offset_t vaddr, bus_addr_t addr, bus_size_t size); static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); -int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); +int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr, bus_size_t size, int coherent); static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, bus_size_t buflen, int flags); static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, @@ -269,7 +267,7 @@ _bus_dma_inrange(struct arm32_dma_range * to check for a match, if there is no filter callback then assume a match. */ int -run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) +run_filter(bus_dma_tag_t dmat, bus_addr_t paddr, bus_size_t size, int coherent) { int retval; @@ -277,7 +275,9 @@ run_filter(bus_dma_tag_t dmat, bus_addr_ do { if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) - || ((paddr & (dmat->alignment - 1)) != 0)) + || ((paddr & (dmat->alignment - 1)) != 0) || + (!coherent && (size & arm_dcache_align_mask)) || + (!coherent && (paddr & arm_dcache_align_mask))) && (dmat->filter == NULL || (*dmat->filter)(dmat->filterarg, paddr) != 0)) retval = 1; @@ -426,8 +426,7 @@ bus_dma_tag_create(bus_dma_tag_t parent, || newtag->alignment > 1) newtag->flags |= BUS_DMA_COULD_BOUNCE; - if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && - (flags & BUS_DMA_ALLOCNOW) != 0) { + if ((flags & BUS_DMA_ALLOCNOW) != 0) { struct bounce_zone *bz; /* Must bounce */ @@ -503,6 +502,41 @@ out: return (error); } +static int allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp) +{ + struct bounce_zone *bz; + int maxpages; + int error; + + if (dmat->bounce_zone == NULL) + if ((error = alloc_bounce_zone(dmat)) != 0) + return (error); + bz = dmat->bounce_zone; + /* Initialize the new map */ + STAILQ_INIT(&(mapp->bpages)); + + /* + * Attempt to add pages to our pool on a per-instance + * basis up to a sane limit. + */ + maxpages = MAX_BPAGES; + if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 + || (bz->map_count > 0 && bz->total_bpages < maxpages)) { + int pages; + + pages = MAX(atop(dmat->maxsize), 1); + pages = MIN(maxpages - bz->total_bpages, pages); + pages = MAX(pages, 1); + if (alloc_bounce_pages(dmat, pages) < pages) + return (ENOMEM); + + if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) + dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; + } + bz->map_count++; + return (0); +} + /* * Allocate a handle for mapping from kva/uva/physical * address space into bus device space. @@ -511,9 +545,7 @@ int bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) { int mapsize; - int error; - - error = 0; + int error = 0; mapsize = sizeof(**mapp) + (sizeof(struct sync_list) * dmat->nsegments); *mapp = (bus_dmamap_t)malloc(mapsize, M_DEVBUF, M_NOWAIT | M_ZERO); @@ -540,52 +572,12 @@ bus_dmamap_create(bus_dma_tag_t dmat, in * exclusion region, a data alignment that is stricter than 1, and/or * an active address boundary. */ - if (dmat->flags & BUS_DMA_COULD_BOUNCE) { - - /* Must bounce */ - struct bounce_zone *bz; - int maxpages; - - if (dmat->bounce_zone == NULL) { - if ((error = alloc_bounce_zone(dmat)) != 0) { - free(*mapp, M_DEVBUF); - *mapp = NULL; - return (error); - } - } - bz = dmat->bounce_zone; - - /* Initialize the new map */ - STAILQ_INIT(&((*mapp)->bpages)); - - /* - * Attempt to add pages to our pool on a per-instance - * basis up to a sane limit. - */ - maxpages = MAX_BPAGES; - if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 - || (bz->map_count > 0 && bz->total_bpages < maxpages)) { - int pages; - - pages = MAX(atop(dmat->maxsize), 1); - pages = MIN(maxpages - bz->total_bpages, pages); - pages = MAX(pages, 1); - if (alloc_bounce_pages(dmat, pages) < pages) - error = ENOMEM; - - if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { - if (error == 0) - dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; - } else { - error = 0; - } - } - bz->map_count++; + error = allocate_bz_and_pages(dmat, *mapp); + if (error != 0) { + free(*mapp, M_DEVBUF); + *mapp = NULL; + return (error); } - if (error == 0) - dmat->map_count++; - CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", - __func__, dmat, dmat->flags, error); return (error); } @@ -624,6 +616,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi vm_memattr_t memattr; int mflags; int mapsize; + int error; if (flags & BUS_DMA_NOWAIT) mflags = M_NOWAIT; @@ -641,6 +634,13 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi } (*mapp)->sync_count = 0; + /* We may need bounce pages, even for allocated memory */ + error = allocate_bz_and_pages(dmat, *mapp); + if (error != 0) { + free(*mapp, M_DEVBUF); + *mapp = NULL; + return (error); + } if (dmat->segments == NULL) { dmat->segments = (bus_dma_segment_t *)malloc( @@ -666,10 +666,6 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi ba = standard_allocator; (*mapp)->flags = 0; } -#ifdef notyet - /* All buffers we allocate are cache-aligned. */ - map->flags |= DMAMAP_CACHE_ALIGNED; -#endif /* * Try to find a bufzone in the allocator that holds a cache of buffers @@ -770,7 +766,8 @@ _bus_dmamap_count_phys(bus_dma_tag_t dma curaddr = buf; while (buflen != 0) { sgsize = MIN(buflen, dmat->maxsegsz); - if (run_filter(dmat, curaddr) != 0) { + if (run_filter(dmat, curaddr, sgsize, + map->flags & DMAMAP_COHERENT) != 0) { sgsize = MIN(sgsize, PAGE_SIZE); map->pagesneeded++; } @@ -806,7 +803,10 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm paddr = pmap_kextract(vaddr); else paddr = pmap_extract(map->pmap, vaddr); - if (run_filter(dmat, paddr) != 0) { + if (run_filter(dmat, paddr, + min(vendaddr - vaddr, + (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK))), + map->flags & DMAMAP_COHERENT) != 0) { map->pagesneeded++; } vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); @@ -922,7 +922,8 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat if (segs == NULL) segs = dmat->segments; - if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + if (((map->flags & DMAMAP_COHERENT) == 0) || + (dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); if (map->pagesneeded != 0) { error = _bus_dmamap_reserve_pages(dmat, map, flags); @@ -934,8 +935,10 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat while (buflen > 0) { curaddr = buf; sgsize = MIN(buflen, dmat->maxsegsz); - if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && - map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + if ((((map->flags & DMAMAP_COHERENT) == 0) || + ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) && + map->pagesneeded != 0 && run_filter(dmat, curaddr, + sgsize, map->flags & DMAMAP_COHERENT)) { sgsize = MIN(sgsize, PAGE_SIZE); curaddr = add_bounce_page(dmat, map, 0, curaddr, sgsize); @@ -982,7 +985,8 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm map->pmap = pmap; - if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + if (!(map->flags & DMAMAP_COHERENT) || + (dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { _bus_dmamap_count_pages(dmat, map, buf, buflen, flags); if (map->pagesneeded != 0) { error = _bus_dmamap_reserve_pages(dmat, map, flags); @@ -1012,8 +1016,10 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm if (buflen < sgsize) sgsize = buflen; - if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && - map->pagesneeded != 0 && run_filter(dmat, curaddr)) { + if ((((map->flags & DMAMAP_COHERENT) == 0) || + ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) && + map->pagesneeded != 0 && run_filter(dmat, curaddr, + sgsize, map->flags & DMAMAP_COHERENT)) { curaddr = add_bounce_page(dmat, map, vaddr, curaddr, sgsize); } else { @@ -1156,12 +1162,6 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus { struct bounce_page *bpage; struct sync_list *sl, *end; - bus_size_t len, unalign; - vm_offset_t buf, ebuf; -#ifdef FIX_DMAP_BUS_DMASYNC_POSTREAD - vm_offset_t bbuf; - char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align]; -#endif /* * If the buffer was from user space, it is possible that this is not * the same vm map, especially on a POST operation. It's not clear that @@ -1252,45 +1252,9 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus case BUS_DMASYNC_PREREAD: while (sl != end) { - /* write back the unaligned portions */ - vm_paddr_t physaddr = sl->busaddr, ephysaddr; - buf = sl->vaddr; - len = sl->datacount; - ebuf = buf + len; /* end of buffer */ - ephysaddr = physaddr + len; - unalign = buf & arm_dcache_align_mask; - if (unalign) { - /* wbinv leading fragment */ - buf &= ~arm_dcache_align_mask; - physaddr &= ~arm_dcache_align_mask; - cpu_dcache_wbinv_range(buf, - arm_dcache_align); - l2cache_wbinv_range(buf, physaddr, - arm_dcache_align); - buf += arm_dcache_align; - physaddr += arm_dcache_align; - /* number byte in buffer wbinv */ - unalign = arm_dcache_align - unalign; - if (len > unalign) - len -= unalign; - else - len = 0; - } - unalign = ebuf & arm_dcache_align_mask; - if (ebuf > buf && unalign) { - /* wbinv trailing fragment */ - len -= unalign; - ebuf -= unalign; - ephysaddr -= unalign; - cpu_dcache_wbinv_range(ebuf, - arm_dcache_align); - l2cache_wbinv_range(ebuf, ephysaddr, - arm_dcache_align); - } - if (ebuf > buf) { - cpu_dcache_inv_range(buf, len); - l2cache_inv_range(buf, physaddr, len); - } + cpu_dcache_inv_range(sl->vaddr, sl->datacount); + l2cache_inv_range(sl->vaddr, sl->busaddr, + sl->datacount); sl++; } break; @@ -1304,60 +1268,6 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus } break; -#ifdef FIX_DMAP_BUS_DMASYNC_POSTREAD - case BUS_DMASYNC_POSTREAD: - while (sl != end) { - /* write back the unaligned portions */ - vm_paddr_t physaddr; - register_t s = 0; - - buf = sl->vaddr; - len = sl->datacount; - physaddr = sl->busaddr; - bbuf = buf & ~arm_dcache_align_mask; - ebuf = buf + len; - physaddr = physaddr & ~arm_dcache_align_mask; - - - if ((buf & arm_dcache_align_mask) || - (ebuf & arm_dcache_align_mask)) { - s = intr_disable(); - unalign = buf & arm_dcache_align_mask; - if (unalign) { - memcpy(_tmp_cl, (void *)bbuf, unalign); - len += unalign; /* inv entire cache line */ - } - - unalign = ebuf & arm_dcache_align_mask; - if (unalign) { - unalign = arm_dcache_align - unalign; - memcpy(_tmp_clend, (void *)ebuf, unalign); - len += unalign; /* inv entire cache line */ - } - } - - /* inv are cache length aligned */ - cpu_dcache_inv_range(bbuf, len); - l2cache_inv_range(bbuf, physaddr, len); - - if ((buf & arm_dcache_align_mask) || - (ebuf & arm_dcache_align_mask)) { - unalign = (vm_offset_t)buf & arm_dcache_align_mask; - if (unalign) - memcpy((void *)bbuf, _tmp_cl, unalign); - - unalign = ebuf & arm_dcache_align_mask; - if (unalign) - memcpy((void *)ebuf, _tmp_clend, - arm_dcache_align - unalign); - - intr_restore(s); - } - sl++; - } - break; -#endif /* FIX_DMAP_BUS_DMASYNC_POSTREAD */ - default: break; }