From owner-svn-src-head@freebsd.org Tue Nov 1 19:18:55 2016 Return-Path: Delivered-To: svn-src-head@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id E73E4C28A48; Tue, 1 Nov 2016 19:18:55 +0000 (UTC) (envelope-from cem@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 979F01F3E; Tue, 1 Nov 2016 19:18:55 +0000 (UTC) (envelope-from cem@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id uA1JIstt092272; Tue, 1 Nov 2016 19:18:54 GMT (envelope-from cem@FreeBSD.org) Received: (from cem@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id uA1JIs3O092270; Tue, 1 Nov 2016 19:18:54 GMT (envelope-from cem@FreeBSD.org) Message-Id: <201611011918.uA1JIs3O092270@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: cem set sender to cem@FreeBSD.org using -f From: "Conrad E. Meyer" Date: Tue, 1 Nov 2016 19:18:54 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r308179 - head/sys/dev/ioat X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 01 Nov 2016 19:18:56 -0000 Author: cem Date: Tue Nov 1 19:18:54 2016 New Revision: 308179 URL: https://svnweb.freebsd.org/changeset/base/308179 Log: ioat(4): Allocate contiguous descriptors This allows us to make strong assertions about descriptor address validity. Additionally, future generations of the ioat(4) hardware will require contiguous descriptors. Reviewed by: markj Sponsored by: Dell EMC Isilon Modified: head/sys/dev/ioat/ioat.c head/sys/dev/ioat/ioat_internal.h Modified: head/sys/dev/ioat/ioat.c ============================================================================== --- head/sys/dev/ioat/ioat.c Tue Nov 1 19:18:52 2016 (r308178) +++ head/sys/dev/ioat/ioat.c Tue Nov 1 19:18:54 2016 (r308179) @@ -81,13 +81,11 @@ static void ioat_process_events(struct i static inline uint32_t ioat_get_active(struct ioat_softc *ioat); static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat); static void ioat_free_ring(struct ioat_softc *, uint32_t size, - struct ioat_descriptor **); -static void ioat_free_ring_entry(struct ioat_softc *ioat, - struct ioat_descriptor *desc); -static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *, - int mflags); + struct ioat_descriptor *); static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags); -static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat, +static union ioat_hw_descriptor *ioat_get_descriptor(struct ioat_softc *, + uint32_t index); +static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *, uint32_t index); static void ioat_halted_debug(struct ioat_softc *, uint32_t); static void ioat_poll_timer_callback(void *arg); @@ -349,7 +347,12 @@ ioat_detach(device_t device) bus_dma_tag_destroy(ioat->comp_update_tag); } - bus_dma_tag_destroy(ioat->hw_desc_tag); + if (ioat->hw_desc_ring != NULL) { + bus_dmamap_unload(ioat->hw_desc_tag, ioat->hw_desc_map); + bus_dmamem_free(ioat->hw_desc_tag, ioat->hw_desc_ring, + ioat->hw_desc_map); + bus_dma_tag_destroy(ioat->hw_desc_tag); + } return (0); } @@ -383,8 +386,8 @@ ioat_start_channel(struct ioat_softc *io /* Submit 'NULL' operation manually to avoid quiescing flag */ desc = ioat_get_ring_entry(ioat, ioat->head); + hw_desc = &ioat_get_descriptor(ioat, ioat->head)->dma; dmadesc = &desc->bus_dmadesc; - hw_desc = desc->u.dma; dmadesc->callback_fn = NULL; dmadesc->callback_arg = NULL; @@ -421,9 +424,10 @@ static int ioat3_attach(device_t device) { struct ioat_softc *ioat; - struct ioat_descriptor **ring; - struct ioat_descriptor *next; + struct ioat_descriptor *ring; struct ioat_dma_hw_descriptor *dma_hw_desc; + void *hw_desc; + size_t ringsz; int i, num_descriptors; int error; uint8_t xfercap; @@ -478,36 +482,41 @@ ioat3_attach(device_t device) return (error); ioat->ring_size_order = g_ioat_ring_order; - num_descriptors = 1 << ioat->ring_size_order; + ringsz = sizeof(struct ioat_dma_hw_descriptor) * num_descriptors; - bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0, - BUS_SPACE_MAXADDR_40BIT, BUS_SPACE_MAXADDR, NULL, NULL, - sizeof(struct ioat_dma_hw_descriptor), 1, - sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL, - &ioat->hw_desc_tag); + error = bus_dma_tag_create(bus_get_dma_tag(ioat->device), + 2 * 1024 * 1024, 0x0, BUS_SPACE_MAXADDR_40BIT, BUS_SPACE_MAXADDR, + NULL, NULL, ringsz, 1, ringsz, 0, NULL, NULL, &ioat->hw_desc_tag); + if (error != 0) + return (error); + + error = bus_dmamem_alloc(ioat->hw_desc_tag, &hw_desc, + BUS_DMA_ZERO | BUS_DMA_WAITOK, &ioat->hw_desc_map); + if (error != 0) + return (error); + + error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc, + ringsz, ioat_dmamap_cb, &ioat->hw_desc_bus_addr, BUS_DMA_WAITOK); + if (error) + return (error); + + ioat->hw_desc_ring = hw_desc; ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT, M_ZERO | M_WAITOK); ring = ioat->ring; for (i = 0; i < num_descriptors; i++) { - ring[i] = ioat_alloc_ring_entry(ioat, M_WAITOK); - if (ring[i] == NULL) - return (ENOMEM); - - ring[i]->id = i; + memset(&ring[i].bus_dmadesc, 0, sizeof(ring[i].bus_dmadesc)); + ring[i].id = i; } - for (i = 0; i < num_descriptors - 1; i++) { - next = ring[i + 1]; - dma_hw_desc = ring[i]->u.dma; - - dma_hw_desc->next = next->hw_desc_bus_addr; + for (i = 0; i < num_descriptors; i++) { + dma_hw_desc = &ioat->hw_desc_ring[i].dma; + dma_hw_desc->next = RING_PHYS_ADDR(ioat, i + 1); } - ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr; - ioat->head = ioat->hw_head = 0; ioat->tail = 0; ioat->last_seen = 0; @@ -673,6 +682,12 @@ ioat_process_events(struct ioat_softc *i comp_update = *ioat->comp_update; status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; + if (status < ioat->hw_desc_bus_addr || + status >= ioat->hw_desc_bus_addr + (1 << ioat->ring_size_order) * + sizeof(struct ioat_generic_hw_descriptor)) + panic("Bogus completion address %jx (channel %u)", + (uintmax_t)status, ioat->chan_idx); + if (status == ioat->last_seen) { /* * If we landed in process_events and nothing has been @@ -683,8 +698,7 @@ ioat_process_events(struct ioat_softc *i CTR4(KTR_IOAT, "%s channel=%u hw_status=0x%lx last_seen=0x%lx", __func__, ioat->chan_idx, comp_update, ioat->last_seen); - desc = ioat_get_ring_entry(ioat, ioat->tail - 1); - while (desc->hw_desc_bus_addr != status) { + while (RING_PHYS_ADDR(ioat, ioat->tail - 1) != status) { desc = ioat_get_ring_entry(ioat, ioat->tail); dmadesc = &desc->bus_dmadesc; CTR5(KTR_IOAT, "channel=%u completing desc idx %u (%p) ok cb %p(%p)", @@ -701,7 +715,7 @@ ioat_process_events(struct ioat_softc *i ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat)); if (completed != 0) { - ioat->last_seen = desc->hw_desc_bus_addr; + ioat->last_seen = RING_PHYS_ADDR(ioat, ioat->tail - 1); ioat->stats.descriptors_processed += completed; } @@ -986,7 +1000,7 @@ ioat_op_generic(struct ioat_softc *ioat, return (NULL); desc = ioat_get_ring_entry(ioat, ioat->head); - hw_desc = desc->u.generic; + hw_desc = &ioat_get_descriptor(ioat, ioat->head)->generic; hw_desc->u.control_raw = 0; hw_desc->u.control_generic.op = op; @@ -1022,7 +1036,7 @@ ioat_null(bus_dmaengine_t dmaengine, bus if (desc == NULL) return (NULL); - hw_desc = desc->u.dma; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma; hw_desc->u.control.null = 1; ioat_submit_single(ioat); return (&desc->bus_dmadesc); @@ -1050,7 +1064,7 @@ ioat_copy(bus_dmaengine_t dmaengine, bus if (desc == NULL) return (NULL); - hw_desc = desc->u.dma; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma; if (g_ioat_debug_level >= 3) dump_descriptor(hw_desc); @@ -1088,7 +1102,7 @@ ioat_copy_8k_aligned(bus_dmaengine_t dma if (desc == NULL) return (NULL); - hw_desc = desc->u.dma; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma; if (src2 != src1 + PAGE_SIZE) { hw_desc->u.control.src_page_break = 1; hw_desc->next_src_addr = src2; @@ -1165,7 +1179,7 @@ ioat_copy_crc(bus_dmaengine_t dmaengine, if (desc == NULL) return (NULL); - hw_desc = desc->u.crc32; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32; if ((flags & DMA_CRC_INLINE) == 0) hw_desc->crc_address = crcptr; @@ -1244,7 +1258,7 @@ ioat_crc(bus_dmaengine_t dmaengine, bus_ if (desc == NULL) return (NULL); - hw_desc = desc->u.crc32; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32; if ((flags & DMA_CRC_INLINE) == 0) hw_desc->crc_address = crcptr; @@ -1292,7 +1306,7 @@ ioat_blockfill(bus_dmaengine_t dmaengine if (desc == NULL) return (NULL); - hw_desc = desc->u.fill; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->fill; if (g_ioat_debug_level >= 3) dump_descriptor(hw_desc); @@ -1317,60 +1331,6 @@ ioat_get_ring_space(struct ioat_softc *i return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1); } -static struct ioat_descriptor * -ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags) -{ - struct ioat_generic_hw_descriptor *hw_desc; - struct ioat_descriptor *desc; - int error, busdmaflag; - - error = ENOMEM; - hw_desc = NULL; - - if ((mflags & M_WAITOK) != 0) - busdmaflag = BUS_DMA_WAITOK; - else - busdmaflag = BUS_DMA_NOWAIT; - - desc = malloc(sizeof(*desc), M_IOAT, mflags); - if (desc == NULL) - goto out; - - bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, - BUS_DMA_ZERO | busdmaflag, &ioat->hw_desc_map); - if (hw_desc == NULL) - goto out; - - memset(&desc->bus_dmadesc, 0, sizeof(desc->bus_dmadesc)); - desc->u.generic = hw_desc; - - error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc, - sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, - busdmaflag); - if (error) - goto out; - -out: - if (error) { - ioat_free_ring_entry(ioat, desc); - return (NULL); - } - return (desc); -} - -static void -ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc) -{ - - if (desc == NULL) - return; - - if (desc->u.generic) - bus_dmamem_free(ioat->hw_desc_tag, desc->u.generic, - ioat->hw_desc_map); - free(desc, M_IOAT); -} - /* * Reserves space in this IOAT descriptor ring by ensuring enough slots remain * for 'num_descs'. @@ -1451,14 +1411,9 @@ out: static void ioat_free_ring(struct ioat_softc *ioat, uint32_t size, - struct ioat_descriptor **ring) + struct ioat_descriptor *ring) { - uint32_t i; - for (i = 0; i < size; i++) { - if (ring[i] != NULL) - ioat_free_ring_entry(ioat, ring[i]); - } free(ring, M_IOAT); } @@ -1466,13 +1421,20 @@ static struct ioat_descriptor * ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index) { - return (ioat->ring[index % (1 << ioat->ring_size_order)]); + return (&ioat->ring[index % (1 << ioat->ring_size_order)]); +} + +static union ioat_hw_descriptor * +ioat_get_descriptor(struct ioat_softc *ioat, uint32_t index) +{ + + return (&ioat->hw_desc_ring[index % (1 << ioat->ring_size_order)]); } static void ioat_halted_debug(struct ioat_softc *ioat, uint32_t chanerr) { - struct ioat_descriptor *desc; + union ioat_hw_descriptor *desc; ioat_log_message(0, "Channel halted (%b)\n", (int)chanerr, IOAT_CHANERR_STR); @@ -1481,11 +1443,11 @@ ioat_halted_debug(struct ioat_softc *ioa mtx_assert(&ioat->cleanup_lock, MA_OWNED); - desc = ioat_get_ring_entry(ioat, ioat->tail + 0); - dump_descriptor(desc->u.raw); + desc = ioat_get_descriptor(ioat, ioat->tail + 0); + dump_descriptor(desc); - desc = ioat_get_ring_entry(ioat, ioat->tail + 1); - dump_descriptor(desc->u.raw); + desc = ioat_get_descriptor(ioat, ioat->tail + 1); + dump_descriptor(desc); } static void @@ -1643,7 +1605,7 @@ ioat_reset_hw(struct ioat_softc *ioat) ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); - ioat_write_chainaddr(ioat, ioat->ring[0]->hw_desc_bus_addr); + ioat_write_chainaddr(ioat, RING_PHYS_ADDR(ioat, 0)); error = 0; CTR2(KTR_IOAT, "%s channel=%u configured channel", __func__, ioat->chan_idx); @@ -2018,34 +1980,37 @@ DB_SHOW_COMMAND(ioat, db_show_ioat) db_printf(" ring_size_order: %u\n", sc->ring_size_order); db_printf(" last_seen: 0x%lx\n", sc->last_seen); db_printf(" ring: %p\n", sc->ring); + db_printf(" descriptors: %p\n", sc->hw_desc_ring); + db_printf(" descriptors (phys): 0x%jx\n", + (uintmax_t)sc->hw_desc_bus_addr); db_printf(" ring[%u] (tail):\n", sc->tail % (1 << sc->ring_size_order)); db_printf(" id: %u\n", ioat_get_ring_entry(sc, sc->tail)->id); db_printf(" addr: 0x%lx\n", - ioat_get_ring_entry(sc, sc->tail)->hw_desc_bus_addr); + RING_PHYS_ADDR(sc, sc->tail)); db_printf(" next: 0x%lx\n", - ioat_get_ring_entry(sc, sc->tail)->u.generic->next); + ioat_get_descriptor(sc, sc->tail)->generic.next); db_printf(" ring[%u] (head - 1):\n", (sc->head - 1) % (1 << sc->ring_size_order)); db_printf(" id: %u\n", ioat_get_ring_entry(sc, sc->head - 1)->id); db_printf(" addr: 0x%lx\n", - ioat_get_ring_entry(sc, sc->head - 1)->hw_desc_bus_addr); + RING_PHYS_ADDR(sc, sc->head - 1)); db_printf(" next: 0x%lx\n", - ioat_get_ring_entry(sc, sc->head - 1)->u.generic->next); + ioat_get_descriptor(sc, sc->head - 1)->generic.next); db_printf(" ring[%u] (head):\n", (sc->head) % (1 << sc->ring_size_order)); db_printf(" id: %u\n", ioat_get_ring_entry(sc, sc->head)->id); db_printf(" addr: 0x%lx\n", - ioat_get_ring_entry(sc, sc->head)->hw_desc_bus_addr); + RING_PHYS_ADDR(sc, sc->head)); db_printf(" next: 0x%lx\n", - ioat_get_ring_entry(sc, sc->head)->u.generic->next); + ioat_get_descriptor(sc, sc->head)->generic.next); for (idx = 0; idx < (1 << sc->ring_size_order); idx++) if ((*sc->comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK) - == ioat_get_ring_entry(sc, idx)->hw_desc_bus_addr) + == RING_PHYS_ADDR(sc, idx)) db_printf(" ring[%u] == hardware tail\n", idx); db_printf(" cleanup_lock: "); Modified: head/sys/dev/ioat/ioat_internal.h ============================================================================== --- head/sys/dev/ioat/ioat_internal.h Tue Nov 1 19:18:52 2016 (r308178) +++ head/sys/dev/ioat/ioat_internal.h Tue Nov 1 19:18:54 2016 (r308179) @@ -413,19 +413,7 @@ struct bus_dmadesc { struct ioat_descriptor { struct bus_dmadesc bus_dmadesc; - union { - struct ioat_generic_hw_descriptor *generic; - struct ioat_dma_hw_descriptor *dma; - struct ioat_fill_hw_descriptor *fill; - struct ioat_crc32_hw_descriptor *crc32; - struct ioat_xor_hw_descriptor *xor; - struct ioat_xor_ext_hw_descriptor *xor_ext; - struct ioat_pq_hw_descriptor *pq; - struct ioat_pq_ext_hw_descriptor *pq_ext; - struct ioat_raw_hw_descriptor *raw; - } u; uint32_t id; - bus_addr_t hw_desc_bus_addr; }; /* Unused by this driver at this time. */ @@ -500,7 +488,22 @@ struct ioat_softc { uint32_t ring_size_order; bus_addr_t last_seen; - struct ioat_descriptor **ring; + struct ioat_descriptor *ring; + + union ioat_hw_descriptor { + struct ioat_generic_hw_descriptor generic; + struct ioat_dma_hw_descriptor dma; + struct ioat_fill_hw_descriptor fill; + struct ioat_crc32_hw_descriptor crc32; + struct ioat_xor_hw_descriptor xor; + struct ioat_xor_ext_hw_descriptor xor_ext; + struct ioat_pq_hw_descriptor pq; + struct ioat_pq_ext_hw_descriptor pq_ext; + struct ioat_raw_hw_descriptor raw; + } *hw_desc_ring; + bus_addr_t hw_desc_bus_addr; +#define RING_PHYS_ADDR(sc, i) (sc)->hw_desc_bus_addr + \ + (((i) % (1 << (sc)->ring_size_order)) * sizeof(struct ioat_dma_hw_descriptor)) struct mtx cleanup_lock; volatile uint32_t refcnt;