From owner-svn-src-all@freebsd.org Mon Nov 9 12:20:00 2015 Return-Path: Delivered-To: svn-src-all@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id E6EDBA2970D; Mon, 9 Nov 2015 12:19:59 +0000 (UTC) (envelope-from royger@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id BB734131A; Mon, 9 Nov 2015 12:19:59 +0000 (UTC) (envelope-from royger@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id tA9CJwGs067037; Mon, 9 Nov 2015 12:19:58 GMT (envelope-from royger@FreeBSD.org) Received: (from royger@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id tA9CJwe7067036; Mon, 9 Nov 2015 12:19:58 GMT (envelope-from royger@FreeBSD.org) Message-Id: <201511091219.tA9CJwe7067036@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: royger set sender to royger@FreeBSD.org using -f From: =?UTF-8?Q?Roger_Pau_Monn=c3=a9?= Date: Mon, 9 Nov 2015 12:19:58 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r290610 - head/sys/x86/x86 X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 09 Nov 2015 12:20:00 -0000 Author: royger Date: Mon Nov 9 12:19:58 2015 New Revision: 290610 URL: https://svnweb.freebsd.org/changeset/base/290610 Log: x86/dma_bounce: rework _bus_dmamap_load_ma implementation The implementation of bus_dmamap_load_ma_triv currently calls _bus_dmamap_load_phys on each page that is part of the passed in buffer. Since each page is treated as an individual buffer, the resulting behaviour is different from the behaviour of _bus_dmamap_load_buffer. This breaks certain drivers, like Xen blkfront. If an unmapped buffer of size 4096 that starts at offset 13 into the first page is passed to the current _bus_dmamap_load_ma implementation (so the ma array contains two pages), the result is that two segments are created, one with a size of 4083 and the other with size 13 (because two independant calls to _bus_dmamap_load_phys are performed, one for each physical page). If the same is done with a mapped buffer and calling _bus_dmamap_load_buffer the result is that only one segment is created, with a size of 4096. This patch relegates the usage of bus_dmamap_load_ma_triv in x86 bounce buffer code to drivers requesting BUS_DMA_KEEP_PG_OFFSET and implements _bus_dmamap_load_ma so that it's behaviour is the same as the mapped version (_bus_dmamap_load_buffer). This patch only modifies the x86 bounce buffer code, other arches are left untouched. Sponsored by: Citrix Systems R&D Reviewed by: kib, jah (previous version) MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D888 Modified: head/sys/x86/x86/busdma_bounce.c Modified: head/sys/x86/x86/busdma_bounce.c ============================================================================== --- head/sys/x86/x86/busdma_bounce.c Mon Nov 9 11:29:54 2015 (r290609) +++ head/sys/x86/x86/busdma_bounce.c Mon Nov 9 12:19:58 2015 (r290610) @@ -79,8 +79,8 @@ struct bounce_page { vm_offset_t vaddr; /* kva of bounce buffer */ bus_addr_t busaddr; /* Physical address */ vm_offset_t datavaddr; /* kva of client data */ - vm_page_t datapage; /* physical page of client data */ vm_offset_t dataoffs; /* page offset of client data */ + vm_page_t datapage[2]; /* physical page(s) of client data */ bus_size_t datacount; /* client data count */ STAILQ_ENTRY(bounce_page) links; }; @@ -135,8 +135,8 @@ static int alloc_bounce_pages(bus_dma_ta static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit); static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, - vm_offset_t vaddr, bus_addr_t addr, - bus_size_t size); + vm_offset_t vaddr, bus_addr_t addr1, + bus_addr_t addr2, bus_size_t size); static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, @@ -527,6 +527,51 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm } } +static void +_bus_dmamap_count_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, + int ma_offs, bus_size_t buflen, int flags) +{ + bus_size_t sg_len, max_sgsize; + int page_index; + vm_paddr_t paddr; + + if ((map != &nobounce_dmamap && map->pagesneeded == 0)) { + CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " + "alignment= %d", dmat->common.lowaddr, + ptoa((vm_paddr_t)Maxmem), + dmat->common.boundary, dmat->common.alignment); + CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d", + map, &nobounce_dmamap, map->pagesneeded); + + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + page_index = 0; + while (buflen > 0) { + paddr = VM_PAGE_TO_PHYS(ma[page_index]) + ma_offs; + sg_len = PAGE_SIZE - ma_offs; + max_sgsize = MIN(buflen, dmat->common.maxsegsz); + sg_len = MIN(sg_len, max_sgsize); + if (bus_dma_run_filter(&dmat->common, paddr) != 0) { + sg_len = roundup2(sg_len, + dmat->common.alignment); + sg_len = MIN(sg_len, max_sgsize); + KASSERT((sg_len & (dmat->common.alignment - 1)) + == 0, ("Segment size is not aligned")); + map->pagesneeded++; + } + if (((ma_offs + sg_len) & ~PAGE_MASK) != 0) + page_index++; + ma_offs = (ma_offs + sg_len) & PAGE_MASK; + KASSERT(buflen >= sg_len, + ("Segment length overruns original buffer")); + buflen -= sg_len; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } +} + static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) { @@ -632,7 +677,7 @@ bounce_bus_dmamap_load_phys(bus_dma_tag_ map->pagesneeded != 0 && bus_dma_run_filter(&dmat->common, curaddr)) { sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); - curaddr = add_bounce_page(dmat, map, 0, curaddr, + curaddr = add_bounce_page(dmat, map, 0, curaddr, 0, sgsize); } sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, @@ -701,7 +746,7 @@ bounce_bus_dmamap_load_buffer(bus_dma_ta bus_dma_run_filter(&dmat->common, curaddr)) { sgsize = roundup2(sgsize, dmat->common.alignment); sgsize = MIN(sgsize, max_sgsize); - curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, + curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 0, sgsize); } else { sgsize = MIN(sgsize, max_sgsize); @@ -720,6 +765,88 @@ bounce_bus_dmamap_load_buffer(bus_dma_ta return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ } +static int +bounce_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, + struct vm_page **ma, bus_size_t buflen, int ma_offs, int flags, + bus_dma_segment_t *segs, int *segp) +{ + vm_paddr_t paddr, next_paddr; + int error, page_index; + bus_size_t sgsize, max_sgsize; + + if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) { + /* + * If we have to keep the offset of each page this function + * is not suitable, switch back to bus_dmamap_load_ma_triv + * which is going to do the right thing in this case. + */ + error = bus_dmamap_load_ma_triv(dmat, map, ma, buflen, ma_offs, + flags, segs, segp); + return (error); + } + + if (map == NULL) + map = &nobounce_dmamap; + + if (segs == NULL) + segs = dmat->segments; + + if ((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_ma(dmat, map, ma, ma_offs, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + + page_index = 0; + while (buflen > 0) { + /* + * Compute the segment size, and adjust counts. + */ + paddr = VM_PAGE_TO_PHYS(ma[page_index]) + ma_offs; + max_sgsize = MIN(buflen, dmat->common.maxsegsz); + sgsize = PAGE_SIZE - ma_offs; + if (((dmat->bounce_flags & BUS_DMA_COULD_BOUNCE) != 0) && + map->pagesneeded != 0 && + bus_dma_run_filter(&dmat->common, paddr)) { + sgsize = roundup2(sgsize, dmat->common.alignment); + sgsize = MIN(sgsize, max_sgsize); + KASSERT((sgsize & (dmat->common.alignment - 1)) == 0, + ("Segment size is not aligned")); + /* + * Check if two pages of the user provided buffer + * are used. + */ + if ((ma_offs + sgsize) > PAGE_SIZE) + next_paddr = + VM_PAGE_TO_PHYS(ma[page_index + 1]); + else + next_paddr = 0; + paddr = add_bounce_page(dmat, map, 0, paddr, + next_paddr, sgsize); + } else { + sgsize = MIN(sgsize, max_sgsize); + } + sgsize = _bus_dmamap_addseg(dmat, map, paddr, sgsize, segs, + segp); + if (sgsize == 0) + break; + KASSERT(buflen >= sgsize, + ("Segment length overruns original buffer")); + buflen -= sgsize; + if (((ma_offs + sgsize) & ~PAGE_MASK) != 0) + page_index++; + ma_offs = (ma_offs + sgsize) & PAGE_MASK; + } + + /* + * Did we fit? + */ + return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ +} + static void bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) @@ -763,6 +890,7 @@ bounce_bus_dmamap_sync(bus_dma_tag_t dma { struct bounce_page *bpage; vm_offset_t datavaddr, tempvaddr; + bus_size_t datacount1, datacount2; if ((bpage = STAILQ_FIRST(&map->bpages)) == NULL) return; @@ -778,17 +906,38 @@ bounce_bus_dmamap_sync(bus_dma_tag_t dma while (bpage != NULL) { tempvaddr = 0; datavaddr = bpage->datavaddr; + datacount1 = bpage->datacount; if (datavaddr == 0) { tempvaddr = - pmap_quick_enter_page(bpage->datapage); + pmap_quick_enter_page(bpage->datapage[0]); datavaddr = tempvaddr | bpage->dataoffs; + datacount1 = min(PAGE_SIZE - bpage->dataoffs, + datacount1); } bcopy((void *)datavaddr, - (void *)bpage->vaddr, bpage->datacount); + (void *)bpage->vaddr, datacount1); if (tempvaddr != 0) pmap_quick_remove_page(tempvaddr); + + if (bpage->datapage[1] == 0) { + KASSERT(datacount1 == bpage->datacount, + ("Mismatch between data size and provided memory space")); + goto next_w; + } + + /* + * We are dealing with an unmapped buffer that expands + * over two pages. + */ + datavaddr = pmap_quick_enter_page(bpage->datapage[1]); + datacount2 = bpage->datacount - datacount1; + bcopy((void *)datavaddr, + (void *)(bpage->vaddr + datacount1), datacount2); + pmap_quick_remove_page(datavaddr); + +next_w: bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; @@ -798,17 +947,38 @@ bounce_bus_dmamap_sync(bus_dma_tag_t dma while (bpage != NULL) { tempvaddr = 0; datavaddr = bpage->datavaddr; + datacount1 = bpage->datacount; if (datavaddr == 0) { tempvaddr = - pmap_quick_enter_page(bpage->datapage); + pmap_quick_enter_page(bpage->datapage[0]); datavaddr = tempvaddr | bpage->dataoffs; + datacount1 = min(PAGE_SIZE - bpage->dataoffs, + datacount1); } - bcopy((void *)bpage->vaddr, - (void *)datavaddr, bpage->datacount); + bcopy((void *)bpage->vaddr, (void *)datavaddr, + datacount1); if (tempvaddr != 0) pmap_quick_remove_page(tempvaddr); + + if (bpage->datapage[1] == 0) { + KASSERT(datacount1 == bpage->datacount, + ("Mismatch between data size and provided memory space")); + goto next_r; + } + + /* + * We are dealing with an unmapped buffer that expands + * over two pages. + */ + datavaddr = pmap_quick_enter_page(bpage->datapage[1]); + datacount2 = bpage->datacount - datacount1; + bcopy((void *)(bpage->vaddr + datacount1), + (void *)datavaddr, datacount2); + pmap_quick_remove_page(datavaddr); + +next_r: bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; @@ -972,7 +1142,7 @@ reserve_bounce_pages(bus_dma_tag_t dmat, static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, - bus_addr_t addr, bus_size_t size) + bus_addr_t addr1, bus_addr_t addr2, bus_size_t size) { struct bounce_zone *bz; struct bounce_page *bpage; @@ -1002,12 +1172,16 @@ add_bounce_page(bus_dma_tag_t dmat, bus_ if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) { /* Page offset needs to be preserved. */ - bpage->vaddr |= addr & PAGE_MASK; - bpage->busaddr |= addr & PAGE_MASK; + bpage->vaddr |= addr1 & PAGE_MASK; + bpage->busaddr |= addr1 & PAGE_MASK; + KASSERT(addr2 == 0, + ("Trying to bounce multiple pages with BUS_DMA_KEEP_PG_OFFSET")); } bpage->datavaddr = vaddr; - bpage->datapage = PHYS_TO_VM_PAGE(addr); - bpage->dataoffs = addr & PAGE_MASK; + bpage->datapage[0] = PHYS_TO_VM_PAGE(addr1); + KASSERT((addr2 & PAGE_MASK) == 0, ("Second page is not aligned")); + bpage->datapage[1] = PHYS_TO_VM_PAGE(addr2); + bpage->dataoffs = addr1 & PAGE_MASK; bpage->datacount = size; STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); return (bpage->busaddr); @@ -1079,7 +1253,7 @@ struct bus_dma_impl bus_dma_bounce_impl .mem_free = bounce_bus_dmamem_free, .load_phys = bounce_bus_dmamap_load_phys, .load_buffer = bounce_bus_dmamap_load_buffer, - .load_ma = bus_dmamap_load_ma_triv, + .load_ma = bounce_bus_dmamap_load_ma, .map_waitok = bounce_bus_dmamap_waitok, .map_complete = bounce_bus_dmamap_complete, .map_unload = bounce_bus_dmamap_unload,