Date: Wed, 8 May 2019 15:22:27 +0000 (UTC) From: Ruslan Bukin <br@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r347337 - head/sys/dev/xdma Message-ID: <201905081522.x48FMRWm002422@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: br Date: Wed May 8 15:22:27 2019 New Revision: 347337 URL: https://svnweb.freebsd.org/changeset/base/347337 Log: o Implement a bounce buffer based on device reserved memory. Grab device reserved physical memory regions from FDT using standard "memory-region" property and use vmem(9) to allocate buffers from it. The same vmem could be used by DMA engine drivers to allocate memory for DMA descriptors. This is required for platforms that provide uncached memory region reserved exclusively for DMA operations. o Change sleepable sx(9) lock type to non-sleepable mutex(9) since network drivers usually hold mutex during DMA operations. So we don't take sleepable lock after non-sleepable. Tested on U.S. Government Furnished Equipment (GFE) 64-bit RISC-V cores. Sponsored by: DARPA, AFRL Modified: head/sys/dev/xdma/xdma.c head/sys/dev/xdma/xdma.h head/sys/dev/xdma/xdma_bank.c head/sys/dev/xdma/xdma_bio.c head/sys/dev/xdma/xdma_mbuf.c head/sys/dev/xdma/xdma_queue.c head/sys/dev/xdma/xdma_sg.c Modified: head/sys/dev/xdma/xdma.c ============================================================================== --- head/sys/dev/xdma/xdma.c Wed May 8 15:20:16 2019 (r347336) +++ head/sys/dev/xdma/xdma.c Wed May 8 15:22:27 2019 (r347337) @@ -1,7 +1,8 @@ /*- - * Copyright (c) 2016-2018 Ruslan Bukin <br@bsdpad.com> - * All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause * + * Copyright (c) 2016-2019 Ruslan Bukin <br@bsdpad.com> + * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 * ("CTSRD"), as part of the DARPA CRASH research programme. @@ -43,7 +44,6 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/sysctl.h> #include <sys/systm.h> -#include <sys/sx.h> #include <machine/bus.h> @@ -61,12 +61,14 @@ __FBSDID("$FreeBSD$"); * Multiple xDMA controllers may work with single DMA device, * so we have global lock for physical channel management. */ -static struct sx xdma_sx; +static struct mtx xdma_mtx; -#define XDMA_LOCK() sx_xlock(&xdma_sx) -#define XDMA_UNLOCK() sx_xunlock(&xdma_sx) -#define XDMA_ASSERT_LOCKED() sx_xassert(&xdma_sx, MA_OWNED) +#define XDMA_LOCK() mtx_lock(&xdma_mtx) +#define XDMA_UNLOCK() mtx_unlock(&xdma_mtx) +#define XDMA_ASSERT_LOCKED() mtx_assert(&xdma_mtx, MA_OWNED) +#define FDT_REG_CELLS 4 + /* * Allocate virtual xDMA channel. */ @@ -95,11 +97,11 @@ xdma_channel_alloc(xdma_controller_t *xdma, uint32_t c TAILQ_INIT(&xchan->ie_handlers); - sx_init(&xchan->sx_lock, "xDMA chan"); - sx_init(&xchan->sx_qin_lock, "xDMA qin"); - sx_init(&xchan->sx_qout_lock, "xDMA qout"); - sx_init(&xchan->sx_bank_lock, "xDMA bank"); - sx_init(&xchan->sx_proc_lock, "xDMA proc"); + mtx_init(&xchan->mtx_lock, "xDMA chan", NULL, MTX_DEF); + mtx_init(&xchan->mtx_qin_lock, "xDMA qin", NULL, MTX_DEF); + mtx_init(&xchan->mtx_qout_lock, "xDMA qout", NULL, MTX_DEF); + mtx_init(&xchan->mtx_bank_lock, "xDMA bank", NULL, MTX_DEF); + mtx_init(&xchan->mtx_proc_lock, "xDMA proc", NULL, MTX_DEF); TAILQ_INIT(&xchan->bank); TAILQ_INIT(&xchan->queue_in); @@ -138,11 +140,11 @@ xdma_channel_free(xdma_channel_t *xchan) xdma_teardown_all_intr(xchan); - sx_destroy(&xchan->sx_lock); - sx_destroy(&xchan->sx_qin_lock); - sx_destroy(&xchan->sx_qout_lock); - sx_destroy(&xchan->sx_bank_lock); - sx_destroy(&xchan->sx_proc_lock); + mtx_destroy(&xchan->mtx_lock); + mtx_destroy(&xchan->mtx_qin_lock); + mtx_destroy(&xchan->mtx_qout_lock); + mtx_destroy(&xchan->mtx_bank_lock); + mtx_destroy(&xchan->mtx_proc_lock); TAILQ_REMOVE(&xdma->channels, xchan, xchan_next); @@ -303,6 +305,95 @@ xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cel return (ret); } +static int +xdma_handle_mem_node(vmem_t *vmem, phandle_t memory) +{ + pcell_t reg[FDT_REG_CELLS * FDT_MEM_REGIONS]; + pcell_t *regp; + int addr_cells, size_cells; + int i, reg_len, ret, tuple_size, tuples; + vmem_addr_t mem_start; + vmem_size_t mem_size; + + if ((ret = fdt_addrsize_cells(OF_parent(memory), &addr_cells, + &size_cells)) != 0) + return (ret); + + if (addr_cells > 2) + return (ERANGE); + + tuple_size = sizeof(pcell_t) * (addr_cells + size_cells); + reg_len = OF_getproplen(memory, "reg"); + if (reg_len <= 0 || reg_len > sizeof(reg)) + return (ERANGE); + + if (OF_getprop(memory, "reg", reg, reg_len) <= 0) + return (ENXIO); + + tuples = reg_len / tuple_size; + regp = (pcell_t *)® + for (i = 0; i < tuples; i++) { + ret = fdt_data_to_res(regp, addr_cells, size_cells, + &mem_start, &mem_size); + if (ret != 0) + return (ret); + + vmem_add(vmem, mem_start, mem_size, 0); + regp += addr_cells + size_cells; + } + + return (0); +} + +vmem_t * +xdma_get_memory(device_t dev) +{ + phandle_t mem_node, node; + pcell_t mem_handle; + vmem_t *vmem; + + node = ofw_bus_get_node(dev); + if (node <= 0) { + device_printf(dev, + "%s called on not ofw based device.\n", __func__); + return (NULL); + } + + if (!OF_hasprop(node, "memory-region")) + return (NULL); + + if (OF_getencprop(node, "memory-region", (void *)&mem_handle, + sizeof(mem_handle)) <= 0) + return (NULL); + + vmem = vmem_create("xDMA vmem", 0, 0, PAGE_SIZE, + PAGE_SIZE, M_BESTFIT | M_WAITOK); + if (vmem == NULL) + return (NULL); + + mem_node = OF_node_from_xref(mem_handle); + if (xdma_handle_mem_node(vmem, mem_node) != 0) { + vmem_destroy(vmem); + return (NULL); + } + + return (vmem); +} + +void +xdma_put_memory(vmem_t *vmem) +{ + + vmem_destroy(vmem); +} + +void +xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem) +{ + + xchan->vmem = vmem; +} + /* * Allocate xdma controller. */ @@ -400,7 +491,7 @@ static void xdma_init(void) { - sx_init(&xdma_sx, "xDMA"); + mtx_init(&xdma_mtx, "xDMA", NULL, MTX_DEF); } SYSINIT(xdma, SI_SUB_DRIVERS, SI_ORDER_FIRST, xdma_init, NULL); Modified: head/sys/dev/xdma/xdma.h ============================================================================== --- head/sys/dev/xdma/xdma.h Wed May 8 15:20:16 2019 (r347336) +++ head/sys/dev/xdma/xdma.h Wed May 8 15:22:27 2019 (r347337) @@ -1,7 +1,8 @@ /*- - * Copyright (c) 2016-2018 Ruslan Bukin <br@bsdpad.com> - * All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause * + * Copyright (c) 2016-2019 Ruslan Bukin <br@bsdpad.com> + * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 * ("CTSRD"), as part of the DARPA CRASH research programme. @@ -34,6 +35,7 @@ #define _DEV_XDMA_XDMA_H_ #include <sys/proc.h> +#include <sys/vmem.h> enum xdma_direction { XDMA_MEM_TO_MEM, @@ -73,6 +75,7 @@ struct xdma_controller { device_t dev; /* DMA consumer device_t. */ device_t dma_dev; /* A real DMA device_t. */ void *data; /* OFW MD part. */ + vmem_t *vmem; /* Bounce memory. */ /* List of virtual channels allocated. */ TAILQ_HEAD(xdma_channel_list, xdma_channel) channels; @@ -84,6 +87,9 @@ struct xchan_buf { bus_dmamap_t map; uint32_t nsegs; uint32_t nsegs_left; + vm_offset_t vaddr; + vm_offset_t paddr; + vm_size_t size; }; struct xdma_request { @@ -117,6 +123,7 @@ struct xdma_sglist { struct xdma_channel { xdma_controller_t *xdma; + vmem_t *vmem; uint32_t flags; #define XCHAN_BUFS_ALLOCATED (1 << 0) @@ -139,11 +146,11 @@ struct xdma_channel { TAILQ_HEAD(, xdma_intr_handler) ie_handlers; TAILQ_ENTRY(xdma_channel) xchan_next; - struct sx sx_lock; - struct sx sx_qin_lock; - struct sx sx_qout_lock; - struct sx sx_bank_lock; - struct sx sx_proc_lock; + struct mtx mtx_lock; + struct mtx mtx_qin_lock; + struct mtx mtx_qout_lock; + struct mtx mtx_bank_lock; + struct mtx mtx_proc_lock; /* Request queue. */ bus_dma_tag_t dma_tag_bufs; @@ -176,30 +183,30 @@ struct xdma_intr_handler { static MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework"); -#define XCHAN_LOCK(xchan) sx_xlock(&(xchan)->sx_lock) -#define XCHAN_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_lock) +#define XCHAN_LOCK(xchan) mtx_lock(&(xchan)->mtx_lock) +#define XCHAN_UNLOCK(xchan) mtx_unlock(&(xchan)->mtx_lock) #define XCHAN_ASSERT_LOCKED(xchan) \ - sx_assert(&(xchan)->sx_lock, SX_XLOCKED) + mtx_assert(&(xchan)->mtx_lock, MA_OWNED) -#define QUEUE_IN_LOCK(xchan) sx_xlock(&(xchan)->sx_qin_lock) -#define QUEUE_IN_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_qin_lock) +#define QUEUE_IN_LOCK(xchan) mtx_lock(&(xchan)->mtx_qin_lock) +#define QUEUE_IN_UNLOCK(xchan) mtx_unlock(&(xchan)->mtx_qin_lock) #define QUEUE_IN_ASSERT_LOCKED(xchan) \ - sx_assert(&(xchan)->sx_qin_lock, SX_XLOCKED) + mtx_assert(&(xchan)->mtx_qin_lock, MA_OWNED) -#define QUEUE_OUT_LOCK(xchan) sx_xlock(&(xchan)->sx_qout_lock) -#define QUEUE_OUT_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_qout_lock) +#define QUEUE_OUT_LOCK(xchan) mtx_lock(&(xchan)->mtx_qout_lock) +#define QUEUE_OUT_UNLOCK(xchan) mtx_unlock(&(xchan)->mtx_qout_lock) #define QUEUE_OUT_ASSERT_LOCKED(xchan) \ - sx_assert(&(xchan)->sx_qout_lock, SX_XLOCKED) + mtx_assert(&(xchan)->mtx_qout_lock, MA_OWNED) -#define QUEUE_BANK_LOCK(xchan) sx_xlock(&(xchan)->sx_bank_lock) -#define QUEUE_BANK_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_bank_lock) +#define QUEUE_BANK_LOCK(xchan) mtx_lock(&(xchan)->mtx_bank_lock) +#define QUEUE_BANK_UNLOCK(xchan) mtx_unlock(&(xchan)->mtx_bank_lock) #define QUEUE_BANK_ASSERT_LOCKED(xchan) \ - sx_assert(&(xchan)->sx_bank_lock, SX_XLOCKED) + mtx_assert(&(xchan)->mtx_bank_lock, MA_OWNED) -#define QUEUE_PROC_LOCK(xchan) sx_xlock(&(xchan)->sx_proc_lock) -#define QUEUE_PROC_UNLOCK(xchan) sx_xunlock(&(xchan)->sx_proc_lock) +#define QUEUE_PROC_LOCK(xchan) mtx_lock(&(xchan)->mtx_proc_lock) +#define QUEUE_PROC_UNLOCK(xchan) mtx_unlock(&(xchan)->mtx_proc_lock) #define QUEUE_PROC_ASSERT_LOCKED(xchan) \ - sx_assert(&(xchan)->sx_proc_lock, SX_XLOCKED) + mtx_assert(&(xchan)->mtx_proc_lock, MA_OWNED) #define XDMA_SGLIST_MAXLEN 2048 #define XDMA_MAX_SEG 128 @@ -207,11 +214,14 @@ static MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework") /* xDMA controller ops */ xdma_controller_t *xdma_ofw_get(device_t dev, const char *prop); int xdma_put(xdma_controller_t *xdma); +vmem_t * xdma_get_memory(device_t dev); +void xdma_put_memory(vmem_t *vmem); /* xDMA channel ops */ xdma_channel_t * xdma_channel_alloc(xdma_controller_t *, uint32_t caps); int xdma_channel_free(xdma_channel_t *); int xdma_request(xdma_channel_t *xchan, struct xdma_request *r); +void xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem); /* SG interface */ int xdma_prep_sg(xdma_channel_t *, uint32_t, Modified: head/sys/dev/xdma/xdma_bank.c ============================================================================== --- head/sys/dev/xdma/xdma_bank.c Wed May 8 15:20:16 2019 (r347336) +++ head/sys/dev/xdma/xdma_bank.c Wed May 8 15:22:27 2019 (r347337) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2018 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2018-2019 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * This software was developed by SRI International and the University of @@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$"); #include <sys/bus.h> #include <sys/kernel.h> #include <sys/malloc.h> -#include <sys/sx.h> #include <machine/bus.h> Modified: head/sys/dev/xdma/xdma_bio.c ============================================================================== --- head/sys/dev/xdma/xdma_bio.c Wed May 8 15:20:16 2019 (r347336) +++ head/sys/dev/xdma/xdma_bio.c Wed May 8 15:22:27 2019 (r347337) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2017-2018 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2017-2019 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * This software was developed by SRI International and the University of @@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$"); #include <sys/bus.h> #include <sys/kernel.h> #include <sys/malloc.h> -#include <sys/sx.h> #include <machine/bus.h> Modified: head/sys/dev/xdma/xdma_mbuf.c ============================================================================== --- head/sys/dev/xdma/xdma_mbuf.c Wed May 8 15:20:16 2019 (r347336) +++ head/sys/dev/xdma/xdma_mbuf.c Wed May 8 15:22:27 2019 (r347337) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2017-2018 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2017-2019 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * This software was developed by SRI International and the University of @@ -36,7 +36,6 @@ __FBSDID("$FreeBSD$"); #include <sys/conf.h> #include <sys/bus.h> #include <sys/kernel.h> -#include <sys/sx.h> #include <sys/mbuf.h> #include <machine/bus.h> Modified: head/sys/dev/xdma/xdma_queue.c ============================================================================== --- head/sys/dev/xdma/xdma_queue.c Wed May 8 15:20:16 2019 (r347336) +++ head/sys/dev/xdma/xdma_queue.c Wed May 8 15:22:27 2019 (r347337) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2018 Ruslan Bukin <br@bsdpad.com> + * Copyright (c) 2018-2019 Ruslan Bukin <br@bsdpad.com> * All rights reserved. * * This software was developed by SRI International and the University of @@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$"); #include <sys/bus.h> #include <sys/kernel.h> #include <sys/malloc.h> -#include <sys/sx.h> #include <machine/bus.h> Modified: head/sys/dev/xdma/xdma_sg.c ============================================================================== --- head/sys/dev/xdma/xdma_sg.c Wed May 8 15:20:16 2019 (r347336) +++ head/sys/dev/xdma/xdma_sg.c Wed May 8 15:22:27 2019 (r347337) @@ -1,7 +1,8 @@ /*- - * Copyright (c) 2018 Ruslan Bukin <br@bsdpad.com> - * All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause * + * Copyright (c) 2018-2019 Ruslan Bukin <br@bsdpad.com> + * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 * ("CTSRD"), as part of the DARPA CRASH research programme. @@ -38,10 +39,14 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/mbuf.h> -#include <sys/sx.h> +#include <sys/rwlock.h> #include <machine/bus.h> +#include <vm/vm.h> +#include <vm/vm_extern.h> +#include <vm/vm_page.h> + #ifdef FDT #include <dev/fdt/fdt_common.h> #include <dev/ofw/ofw_bus.h> @@ -58,25 +63,71 @@ struct seg_load_request { uint32_t error; }; +static void +xchan_bufs_free_reserved(xdma_channel_t *xchan) +{ + struct xdma_request *xr; + vm_size_t size; + int i; + + for (i = 0; i < xchan->xr_num; i++) { + xr = &xchan->xr_mem[i]; + size = xr->buf.size; + if (xr->buf.vaddr) { + pmap_kremove_device(xr->buf.vaddr, size); + kva_free(xr->buf.vaddr, size); + xr->buf.vaddr = 0; + } + if (xr->buf.paddr) { + vmem_free(xchan->vmem, xr->buf.paddr, size); + xr->buf.paddr = 0; + } + xr->buf.size = 0; + } +} + static int -_xchan_bufs_alloc(xdma_channel_t *xchan) +xchan_bufs_alloc_reserved(xdma_channel_t *xchan) { xdma_controller_t *xdma; struct xdma_request *xr; + vmem_addr_t addr; + vm_size_t size; int i; xdma = xchan->xdma; + if (xchan->vmem == NULL) + return (ENOBUFS); + for (i = 0; i < xchan->xr_num; i++) { xr = &xchan->xr_mem[i]; - /* TODO: bounce buffer */ + size = round_page(xchan->maxsegsize); + if (vmem_alloc(xchan->vmem, size, + M_BESTFIT | M_NOWAIT, &addr)) { + device_printf(xdma->dev, + "%s: Can't allocate memory\n", __func__); + xchan_bufs_free_reserved(xchan); + return (ENOMEM); + } + + xr->buf.size = size; + xr->buf.paddr = addr; + xr->buf.vaddr = kva_alloc(size); + if (xr->buf.vaddr == 0) { + device_printf(xdma->dev, + "%s: Can't allocate KVA\n", __func__); + xchan_bufs_free_reserved(xchan); + return (ENOMEM); + } + pmap_kenter_device(xr->buf.vaddr, size, addr); } return (0); } static int -_xchan_bufs_alloc_busdma(xdma_channel_t *xchan) +xchan_bufs_alloc_busdma(xdma_channel_t *xchan) { xdma_controller_t *xdma; struct xdma_request *xr; @@ -138,9 +189,10 @@ xchan_bufs_alloc(xdma_channel_t *xchan) } if (xchan->caps & XCHAN_CAP_BUSDMA) - ret = _xchan_bufs_alloc_busdma(xchan); - else - ret = _xchan_bufs_alloc(xchan); + ret = xchan_bufs_alloc_busdma(xchan); + else { + ret = xchan_bufs_alloc_reserved(xchan); + } if (ret != 0) { device_printf(xdma->dev, "%s: Can't allocate bufs.\n", __func__); @@ -169,12 +221,8 @@ xchan_bufs_free(xdma_channel_t *xchan) bus_dmamap_destroy(xchan->dma_tag_bufs, b->map); } bus_dma_tag_destroy(xchan->dma_tag_bufs); - } else { - for (i = 0; i < xchan->xr_num; i++) { - xr = &xchan->xr_mem[i]; - /* TODO: bounce buffer */ - } - } + } else + xchan_bufs_free_reserved(xchan); xchan->flags &= ~XCHAN_BUFS_ALLOCATED; @@ -296,6 +344,11 @@ xchan_seg_done(xdma_channel_t *xchan, bus_dmamap_sync(xchan->dma_tag_bufs, b->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(xchan->dma_tag_bufs, b->map); + } else { + if (xr->req_type == XR_TYPE_MBUF && + xr->direction == XDMA_DEV_TO_MEM) + m_copyback(xr->m, 0, st->transferred, + (void *)xr->buf.vaddr); } xr->status.error = st->error; xr->status.transferred = st->transferred; @@ -437,7 +490,13 @@ _xdma_load_data(xdma_channel_t *xchan, struct xdma_req switch (xr->req_type) { case XR_TYPE_MBUF: - seg[0].ds_addr = mtod(m, bus_addr_t); + if ((xchan->caps & XCHAN_CAP_NOBUFS) == 0) { + if (xr->direction == XDMA_MEM_TO_DEV) + m_copydata(m, 0, m->m_pkthdr.len, + (void *)xr->buf.vaddr); + seg[0].ds_addr = (bus_addr_t)xr->buf.paddr; + } else + seg[0].ds_addr = mtod(m, bus_addr_t); seg[0].ds_len = m->m_pkthdr.len; break; case XR_TYPE_BIO: @@ -494,6 +553,7 @@ xdma_process(xdma_channel_t *xchan, xdma = xchan->xdma; n = 0; + c = 0; ret = XDMA_CHANNEL_CAPACITY(xdma->dma_dev, xchan, &capacity); if (ret != 0) {
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201905081522.x48FMRWm002422>