Date: Fri, 01 May 2026 21:15:16 +0000 From: Adrian Chadd <adrian@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 3bcb7c2a3320 - main - powerpc: initial straight port of busdma_machdep.c -> busdma_bounce.c Message-ID: <69f517e4.1836b.6f5ef3b1@gitrepo.freebsd.org>
index | next in thread | raw e-mail
The branch main has been updated by adrian: URL: https://cgit.FreeBSD.org/src/commit/?id=3bcb7c2a33206d52cecb349b77b2a631728bc7d1 commit 3bcb7c2a33206d52cecb349b77b2a631728bc7d1 Author: Adrian Chadd <adrian@FreeBSD.org> AuthorDate: 2026-02-14 17:03:27 +0000 Commit: Adrian Chadd <adrian@FreeBSD.org> CommitDate: 2026-05-01 21:14:23 +0000 powerpc: initial straight port of busdma_machdep.c -> busdma_bounce.c This is a straight port of the code and doesn't yet handle different implementations (which will be in a subsequent commit.) Locally tested: * G5 SMP (2x PPC970mp) * power8 / power9 pseries QEMU VM * power8 powernv Differential Revision: https://reviews.freebsd.org/D55313 --- sys/conf/files.powerpc | 1 + sys/powerpc/include/bus_dma.h | 145 ++++++- sys/powerpc/include/bus_dma_impl.h | 82 ++++ sys/powerpc/powerpc/busdma_bounce.c | 787 +++++++++++++++++++++++++++++++++++ sys/powerpc/powerpc/busdma_machdep.c | 711 ++----------------------------- sys/powerpc/pseries/phyp_vscsi.c | 1 - 6 files changed, 1039 insertions(+), 688 deletions(-) diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc index 698433422101..164a5d01d4b9 100644 --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -361,6 +361,7 @@ powerpc/powernv/xive.c optional powernv powerpc/powerpc/altivec.c standard powerpc/powerpc/autoconf.c standard powerpc/powerpc/bus_machdep.c standard +powerpc/powerpc/busdma_bounce.c standard powerpc/powerpc/busdma_machdep.c standard powerpc/powerpc/clock.c standard powerpc/powerpc/copyinout.c optional aim diff --git a/sys/powerpc/include/bus_dma.h b/sys/powerpc/include/bus_dma.h index 09bc7b15e94e..c12161011a99 100644 --- a/sys/powerpc/include/bus_dma.h +++ b/sys/powerpc/include/bus_dma.h @@ -2,6 +2,7 @@ * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2005 Scott Long + * Copyright (c) 2026 Raptor Engineering, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -29,9 +30,149 @@ #ifndef _POWERPC_BUS_DMA_H_ #define _POWERPC_BUS_DMA_H_ +#define WANT_INLINE_DMAMAP #include <sys/bus_dma.h> -#include <sys/bus_dma_internal.h> -int bus_dma_tag_set_iommu(bus_dma_tag_t, device_t iommu, void *cookie); +#include <machine/bus_dma_impl.h> + +/* + * Allocate a handle for mapping from kva/uva/physical + * address space into bus device space. + */ +static inline int +bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->map_create(dmat, flags, mapp)); +} + +/* + * Destroy a handle for mapping from kva/uva/physical + * address space into bus device space. + */ +static inline int +bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->map_destroy(dmat, map)); +} + +/* + * Allocate a piece of memory that can be efficiently mapped into + * bus device space based on the constraints listed in the dma tag. + * A dmamap to for use with dmamap_load is also allocated. + */ +static inline int +bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, + bus_dmamap_t *mapp) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->mem_alloc(dmat, vaddr, flags, mapp)); +} + +/* + * Free a piece of memory and it's allociated dmamap, that was allocated + * via bus_dmamem_alloc. + */ +static inline void +bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + tc->impl->mem_free(dmat, vaddr, map); +} + +/* + * Release the mapping held by map. + */ +static inline void +bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + tc->impl->map_unload(dmat, map); +} + +static inline void +bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + tc->impl->map_sync(dmat, map, op); +} + +static inline int +_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->load_phys(dmat, map, buf, buflen, flags, segs, + segp)); +} + +static inline int +_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma, + bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs, + int *segp) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->load_ma(dmat, map, ma, tlen, ma_offs, flags, + segs, segp)); +} + +static inline int +_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, + bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs, + int *segp) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs, + segp)); +} + +static inline void +_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, + struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + tc->impl->map_waitok(dmat, map, mem, callback, callback_arg); +} + +static inline bus_dma_segment_t * +_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, + bus_dma_segment_t *segs, int nsegs, int error) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->map_complete(dmat, map, segs, nsegs, error)); +} + +static inline int +bus_dma_tag_set_iommu(bus_dma_tag_t dmat, device_t iommu, void *cookie) +{ + struct bus_dma_tag_common *tc; + + tc = (struct bus_dma_tag_common *)dmat; + return (tc->impl->set_iommu(dmat, iommu, cookie)); + +} #endif /* _POWERPC_BUS_DMA_H_ */ diff --git a/sys/powerpc/include/bus_dma_impl.h b/sys/powerpc/include/bus_dma_impl.h new file mode 100644 index 000000000000..e24bd6dd8806 --- /dev/null +++ b/sys/powerpc/include/bus_dma_impl.h @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2013 The FreeBSD Foundation + * + * This software was developed by Konstantin Belousov <kib@FreeBSD.org> + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _MACHINE_BUS_DMA_IMPL_H_ +#define _MACHINE_BUS_DMA_IMPL_H_ + +/* Note: must be first entry in bus_dma_tag */ +struct bus_dma_tag_common { + struct bus_dma_impl *impl; + bus_size_t alignment; + bus_addr_t boundary; + bus_addr_t lowaddr; + bus_addr_t highaddr; + bus_size_t maxsize; + u_int nsegments; + bus_size_t maxsegsz; + int flags; + bus_dma_lock_t *lockfunc; + void *lockfuncarg; +}; + +struct bus_dma_impl { + int (*tag_create)(bus_dma_tag_t parent, + bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr, + bus_addr_t highaddr, bus_size_t maxsize, int nsegments, + bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, + void *lockfuncarg, bus_dma_tag_t *dmat); + int (*tag_destroy)(bus_dma_tag_t dmat); + int (*tag_set_domain)(bus_dma_tag_t, int domain); + int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp); + int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map); + int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags, + bus_dmamap_t *mapp); + void (*mem_free)(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map); + int (*load_ma)(bus_dma_tag_t dmat, bus_dmamap_t map, + struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, + bus_dma_segment_t *segs, int *segp); + int (*load_phys)(bus_dma_tag_t dmat, bus_dmamap_t map, + vm_paddr_t buf, bus_size_t buflen, int flags, + bus_dma_segment_t *segs, int *segp); + int (*load_buffer)(bus_dma_tag_t dmat, bus_dmamap_t map, + void *buf, bus_size_t buflen, struct pmap *pmap, int flags, + bus_dma_segment_t *segs, int *segp); + void (*map_waitok)(bus_dma_tag_t dmat, bus_dmamap_t map, + struct memdesc *mem, bus_dmamap_callback_t *callback, + void *callback_arg); + bus_dma_segment_t *(*map_complete)(bus_dma_tag_t dmat, bus_dmamap_t map, + bus_dma_segment_t *segs, int nsegs, int error); + void (*map_unload)(bus_dma_tag_t dmat, bus_dmamap_t map); + void (*map_sync)(bus_dma_tag_t dmat, bus_dmamap_t map, + bus_dmasync_op_t op); + int (*set_iommu)(bus_dma_tag_t dmat, device_t iommu, void *cookie); +}; + +extern struct bus_dma_impl bus_dma_bounce_impl; + +#endif diff --git a/sys/powerpc/powerpc/busdma_bounce.c b/sys/powerpc/powerpc/busdma_bounce.c new file mode 100644 index 000000000000..cdc25a5d8fb7 --- /dev/null +++ b/sys/powerpc/powerpc/busdma_bounce.c @@ -0,0 +1,787 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 1997, 1998 Justin T. Gibbs. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification, immediately at the beginning of the file. + * 2. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * From amd64/busdma_machdep.c, r204214 + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/bus.h> +#include <sys/interrupt.h> +#include <sys/kernel.h> +#include <sys/ktr.h> +#include <sys/lock.h> +#include <sys/proc.h> +#include <sys/memdesc.h> +#include <sys/mutex.h> +#include <sys/sysctl.h> +#include <sys/uio.h> + +#include <vm/vm.h> +#include <vm/vm_extern.h> +#include <vm/vm_kern.h> +#include <vm/vm_page.h> +#include <vm/vm_map.h> + +#include <machine/atomic.h> +#include <machine/bus.h> +#include <machine/cpufunc.h> +#include <machine/md_var.h> + +#include "iommu_if.h" + +#define MAX_BPAGES MIN(8192, physmem/40) + +struct bus_dmamap { + STAILQ_HEAD(, bounce_page) bpages; + int pagesneeded; + int pagesreserved; + bus_dma_tag_t dmat; + struct memdesc mem; + bus_dma_segment_t *segments; + int nsegs; + bus_dmamap_callback_t *callback; + void *callback_arg; + __sbintime_t queued_time; + STAILQ_ENTRY(bus_dmamap) links; + int contigalloc; +}; + +static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); + +#define dmat_alignment(dmat) ((dmat)->common.alignment) +/* XXX TODO: bounce flags? */ +#define dmat_bounce_flags(dmat) (0) +#define dmat_boundary(dmat) ((dmat)->common.boundary) +#define dmat_flags(dmat) ((dmat)->common.flags) +#define dmat_highaddr(dmat) ((dmat)->common.highaddr) +#define dmat_lowaddr(dmat) ((dmat)->common.lowaddr) +#define dmat_lockfunc(dmat) ((dmat)->common.lockfunc) +#define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg) +#define dmat_maxsegsz(dmat) ((dmat)->common.maxsegsz) +#define dmat_nsegments(dmat) ((dmat)->common.nsegments) + +/* + * Note: at this point the code requires a 'struct bus_dma_tag' to exist + * or the included code will not function correctly. + */ + +struct bus_dma_tag { + struct bus_dma_tag_common common; + int map_count; + int bounce_flags; + struct bounce_zone *bounce_zone; + device_t iommu; + void *iommu_cookie; +}; + +static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, + "Busdma parameters"); + +#include "../../kern/subr_busdma_bounce.c" + +/* + * Returns true if the address falls within the tag's exclusion window, or + * fails to meet its alignment requirements. + */ +static __inline bool +must_bounce(bus_dma_tag_t dmat, bus_addr_t paddr) +{ + + if (dmat->iommu == NULL && paddr > dmat->common.lowaddr && + paddr <= dmat->common.highaddr) + return (true); + if (!vm_addr_align_ok(paddr, dmat->common.alignment)) + return (true); + + return (false); +} + +#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 +#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 + +static int +bounce_bus_dma_zone_setup(bus_dma_tag_t newtag) +{ + struct bounce_zone *bz; + const u_long maxsize = newtag->common.maxsize; + int error = 0; + + if ((error = alloc_bounce_zone(newtag)) != 0) { + return (error); + } + bz = newtag->bounce_zone; + + if (ptoa(bz->total_bpages) < maxsize) { + int pages; + + pages = atop(maxsize) - bz->total_bpages; + + /* Add pages to our bounce pool */ + if (alloc_bounce_pages(newtag, pages) < pages) + error = ENOMEM; + } + /* Performed initial allocation */ + newtag->common.flags |= BUS_DMA_MIN_ALLOC_COMP; + + return (error); +} + +/* + * Allocate a device specific dma_tag. + * + * TODO: this does ALL of the work, rather than it being split into + * common and bounce specific. That'll need fixing. + */ +static int +bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, + bus_addr_t boundary, bus_addr_t lowaddr, + bus_addr_t highaddr, bus_size_t maxsize, int nsegments, + bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, + void *lockfuncarg, bus_dma_tag_t *dmat) +{ + bus_dma_tag_t newtag; + int error = 0; + + /* Basic sanity checking */ + if (boundary != 0 && boundary < maxsegsz) + maxsegsz = boundary; + + if (maxsegsz == 0) { + return (EINVAL); + } + + /* Return a NULL tag on failure */ + *dmat = NULL; + + newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, + M_ZERO | M_NOWAIT); + if (newtag == NULL) { + CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", + __func__, newtag, 0, error); + return (ENOMEM); + } + + newtag->common.alignment = alignment; + newtag->common.boundary = boundary; + newtag->common.lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); + newtag->common.highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); + newtag->common.maxsize = maxsize; + newtag->common.nsegments = nsegments; + newtag->common.maxsegsz = maxsegsz; + newtag->common.flags = flags; + newtag->map_count = 0; + newtag->common.impl = &bus_dma_bounce_impl; + if (lockfunc != NULL) { + newtag->common.lockfunc = lockfunc; + newtag->common.lockfuncarg = lockfuncarg; + } else { + newtag->common.lockfunc = _busdma_dflt_lock; + newtag->common.lockfuncarg = NULL; + } + + /* Take into account any restrictions imposed by our parent tag */ + if (parent != NULL) { + newtag->common.lowaddr = MIN(parent->common.lowaddr, newtag->common.lowaddr); + newtag->common.highaddr = MAX(parent->common.highaddr, newtag->common.highaddr); + if (newtag->common.boundary == 0) + newtag->common.boundary = parent->common.boundary; + else if (parent->common.boundary != 0) + newtag->common.boundary = MIN(parent->common.boundary, + newtag->common.boundary); + + newtag->iommu = parent->iommu; + newtag->iommu_cookie = parent->iommu_cookie; + } + + if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) && newtag->iommu == NULL) + newtag->common.flags |= BUS_DMA_COULD_BOUNCE; + + if (newtag->common.alignment > 1) + newtag->common.flags |= BUS_DMA_COULD_BOUNCE; + + if (((newtag->common.flags & BUS_DMA_COULD_BOUNCE) != 0) && + (flags & BUS_DMA_ALLOCNOW) != 0) { + /* Must bounce */ + error = bounce_bus_dma_zone_setup(newtag); + } + + if (error != 0) { + free(newtag, M_DEVBUF); + } else { + *dmat = newtag; + } + CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", + __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), error); + return (error); +} + +static int +bounce_bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) +{ + + /* TODO */ + return (0); +} + +static int +bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) +{ + int error = 0; + + if (dmat != NULL) { + if (dmat->map_count != 0) { + error = EBUSY; + goto out; + } + + free(dmat, M_DEVBUF); + } +out: + CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat, error); + return (error); +} + +/* + * Allocate a handle for mapping from kva/uva/physical + * address space into bus device space. + */ +static int +bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) +{ + int error; + + error = 0; + + *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF, + M_NOWAIT | M_ZERO); + if (*mapp == NULL) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", + __func__, dmat, ENOMEM); + return (ENOMEM); + } + + /* Initialize the new map */ + STAILQ_INIT(&((*mapp)->bpages)); + + /* + * Bouncing might be required if the driver asks for an active + * exclusion region, a data alignment that is stricter than 1, and/or + * an active address boundary. + */ + if (dmat->common.flags & BUS_DMA_COULD_BOUNCE) { + /* Must bounce */ + struct bounce_zone *bz; + int maxpages; + + if (dmat->bounce_zone == NULL) { + if ((error = alloc_bounce_zone(dmat)) != 0) + return (error); + } + bz = dmat->bounce_zone; + + /* + * Attempt to add pages to our pool on a per-instance + * basis up to a sane limit. + */ + if (dmat->common.alignment > 1) + maxpages = MAX_BPAGES; + else + maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->common.lowaddr)); + if ((dmat->common.flags & BUS_DMA_MIN_ALLOC_COMP) == 0 + || (bz->map_count > 0 && bz->total_bpages < maxpages)) { + int pages; + + pages = MAX(atop(dmat->common.maxsize), 1); + pages = MIN(maxpages - bz->total_bpages, pages); + pages = MAX(pages, 1); + if (alloc_bounce_pages(dmat, pages) < pages) + error = ENOMEM; + + if ((dmat->common.flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { + if (error == 0) + dmat->common.flags |= BUS_DMA_MIN_ALLOC_COMP; + } else { + error = 0; + } + } + bz->map_count++; + } + + (*mapp)->nsegs = 0; + (*mapp)->segments = (bus_dma_segment_t *)malloc( + sizeof(bus_dma_segment_t) * dmat->common.nsegments, M_DEVBUF, + M_NOWAIT); + if ((*mapp)->segments == NULL) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", + __func__, dmat, ENOMEM); + return (ENOMEM); + } + + if (error == 0) + dmat->map_count++; + CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", + __func__, dmat, dmat->common.flags, error); + return (error); +} + +/* + * Destroy a handle for mapping from kva/uva/physical + * address space into bus device space. + */ +static int +bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) +{ + if (dmat->common.flags & BUS_DMA_COULD_BOUNCE) { + if (STAILQ_FIRST(&map->bpages) != NULL) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", + __func__, dmat, EBUSY); + return (EBUSY); + } + if (dmat->bounce_zone) + dmat->bounce_zone->map_count--; + } + free(map->segments, M_DEVBUF); + free(map, M_DEVBUF); + dmat->map_count--; + CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); + return (0); +} + +/* + * Allocate a piece of memory that can be efficiently mapped into + * bus device space based on the constraints lited in the dma tag. + * A dmamap to for use with dmamap_load is also allocated. + */ +static int +bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, + bus_dmamap_t *mapp) +{ + vm_memattr_t attr; + int mflags; + + if (flags & BUS_DMA_NOWAIT) + mflags = M_NOWAIT; + else + mflags = M_WAITOK; + + bus_dmamap_create(dmat, flags, mapp); + + if (flags & BUS_DMA_ZERO) + mflags |= M_ZERO; + if (flags & BUS_DMA_NOCACHE) + attr = VM_MEMATTR_UNCACHEABLE; + else + attr = VM_MEMATTR_DEFAULT; + + /* + * XXX: + * (dmat->common.alignment <= dmat->common.maxsize) is just a quick hack; the exact + * alignment guarantees of malloc need to be nailed down, and the + * code below should be rewritten to take that into account. + * + * In the meantime, we'll warn the user if malloc gets it wrong. + */ + if ((dmat->common.maxsize <= PAGE_SIZE) && + (dmat->common.alignment <= dmat->common.maxsize) && + dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && + attr == VM_MEMATTR_DEFAULT) { + *vaddr = malloc(dmat->common.maxsize, M_DEVBUF, mflags); + } else { + /* + * XXX Use Contigmalloc until it is merged into this facility + * and handles multi-seg allocations. Nobody is doing + * multi-seg allocations yet though. + * XXX Certain AGP hardware does. + */ + *vaddr = kmem_alloc_contig(dmat->common.maxsize, mflags, 0ul, + dmat->common.lowaddr, dmat->common.alignment ? dmat->common.alignment : 1ul, + dmat->common.boundary, attr); + (*mapp)->contigalloc = 1; + } + if (*vaddr == NULL) { + CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", + __func__, dmat, dmat->common.flags, ENOMEM); + return (ENOMEM); + } else if (!vm_addr_align_ok(vtophys(*vaddr), dmat->common.alignment)) { + printf("bus_dmamem_alloc failed to align memory properly.\n"); + } + CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", + __func__, dmat, dmat->common.flags, 0); + return (0); +} + +/* + * Free a piece of memory and it's allociated dmamap, that was allocated + * via bus_dmamem_alloc. + */ +static void +bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) +{ + + if (!map->contigalloc) + free(vaddr, M_DEVBUF); + else + kmem_free(vaddr, dmat->common.maxsize); + bus_dmamap_destroy(dmat, map); + CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->common.flags); +} + +static void +_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, + bus_size_t buflen, int flags) +{ + bus_addr_t curaddr; + bus_size_t sgsize; + + if (map->pagesneeded == 0) { + CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " + "alignment= %d", dmat->common.lowaddr, ptoa((vm_paddr_t)Maxmem), + dmat->common.boundary, dmat->common.alignment); + CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded); + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + curaddr = buf; + while (buflen != 0) { + sgsize = buflen; + if (must_bounce(dmat, curaddr)) { + sgsize = MIN(sgsize, + PAGE_SIZE - (curaddr & PAGE_MASK)); + map->pagesneeded++; + } + curaddr += sgsize; + buflen -= sgsize; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } +} + +static void +_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, + void *buf, bus_size_t buflen, int flags) +{ + vm_offset_t vaddr; + vm_offset_t vendaddr; + bus_addr_t paddr; + + if (map->pagesneeded == 0) { + CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " + "alignment= %d", dmat->common.lowaddr, ptoa((vm_paddr_t)Maxmem), + dmat->common.boundary, dmat->common.alignment); + CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded); + /* + * Count the number of bounce pages + * needed in order to complete this transfer + */ + vaddr = (vm_offset_t)buf; + vendaddr = (vm_offset_t)buf + buflen; + + while (vaddr < vendaddr) { + bus_size_t sg_len; + + sg_len = MIN(vendaddr - vaddr, + PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); + if (pmap == kernel_pmap) + paddr = pmap_kextract(vaddr); + else + paddr = pmap_extract(pmap, vaddr); + if (must_bounce(dmat, paddr)) { + sg_len = roundup2(sg_len, dmat->common.alignment); + map->pagesneeded++; + } + vaddr += sg_len; + } + CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); + } +} + +/* + * Utility function to load a physical buffer. segp contains + * the starting segment on entrace, and the ending segment on exit. + */ +static int +bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, + bus_dmamap_t map, + vm_paddr_t buf, bus_size_t buflen, + int flags, + bus_dma_segment_t *segs, + int *segp) +{ + bus_addr_t curaddr; + bus_size_t sgsize; + int error; + + if (segs == NULL) + segs = map->segments; + + if ((dmat->common.flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + + while (buflen > 0) { + curaddr = buf; + sgsize = buflen; + if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) { + sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); + curaddr = add_bounce_page(dmat, map, 0, curaddr, + sgsize); + } + if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs, + segp)) + break; + buf += sgsize; + buflen -= sgsize; + } + + /* + * Did we fit? + */ + return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ +} + +static int +bounce_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, + struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags, + bus_dma_segment_t *segs, int *segp) +{ + + return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags, + segs, segp)); +} + +/* + * Utility function to load a linear buffer. segp contains + * the starting segment on entrance, and the ending segment on exit. + */ +static int +bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, + bus_dmamap_t map, + void *buf, bus_size_t buflen, + pmap_t pmap, + int flags, + bus_dma_segment_t *segs, + int *segp) +{ + bus_size_t sgsize; + bus_addr_t curaddr; + char *kvaddr, *vaddr; + int error; + + if (segs == NULL) + segs = map->segments; + + if ((dmat->common.flags & BUS_DMA_COULD_BOUNCE) != 0) { + _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); + if (map->pagesneeded != 0) { + error = _bus_dmamap_reserve_pages(dmat, map, flags); + if (error) + return (error); + } + } + + vaddr = buf; + + while (buflen > 0) { + /* + * Get the physical address for this segment. + */ + if (pmap == kernel_pmap) { + curaddr = pmap_kextract((vm_offset_t)vaddr); + kvaddr = vaddr; + } else { + curaddr = pmap_extract(pmap, (vm_offset_t)vaddr); + kvaddr = NULL; + } + + /* + * Compute the segment size, and adjust counts. + */ + sgsize = MIN(buflen, PAGE_SIZE - (curaddr & PAGE_MASK)); + if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) { + sgsize = roundup2(sgsize, dmat->common.alignment); + sgsize = MIN(sgsize, buflen); + curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, + sgsize); + } + + if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs, + segp)) + break; + vaddr += sgsize; + buflen -= MIN(sgsize, buflen); /* avoid underflow */ + } + + /* + * Did we fit? + */ + return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ +} + +static void +bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, + struct memdesc *mem, bus_dmamap_callback_t *callback, + void *callback_arg) +{ + + if (dmat->common.flags & BUS_DMA_COULD_BOUNCE) { + map->dmat = dmat; + map->mem = *mem; + map->callback = callback; + map->callback_arg = callback_arg; + } +} + +static bus_dma_segment_t * +bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, + bus_dma_segment_t *segs, int nsegs, int error) +{ + + map->nsegs = nsegs; + if (segs != NULL) + memcpy(map->segments, segs, map->nsegs*sizeof(segs[0])); + if (dmat->iommu != NULL) + IOMMU_MAP(dmat->iommu, map->segments, &map->nsegs, + dmat->common.lowaddr, dmat->common.highaddr, dmat->common.alignment, + dmat->common.boundary, dmat->iommu_cookie); + + if (segs != NULL) + memcpy(segs, map->segments, map->nsegs*sizeof(segs[0])); + else + segs = map->segments; + + return (segs); *** 870 LINES SKIPPED ***home | help
Want to link to this message? Use this
URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?69f517e4.1836b.6f5ef3b1>