Date: Thu, 21 Feb 2019 00:44:26 +0000 (UTC) From: Alexander Motin <mav@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org Subject: svn commit: r344401 - stable/11/sys/dev/ioat Message-ID: <201902210044.x1L0iQY3022372@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mav Date: Thu Feb 21 00:44:26 2019 New Revision: 344401 URL: https://svnweb.freebsd.org/changeset/base/344401 Log: MFC r302669,302677-302686,303761,304602,304603,305027-305028,305259, 305710,305711,308067-308070,308178,308179,308230,308553,309338,309526, 343125 (by cem): Synchronize ioat(4) with head. Most of these changes are 3 years old, just never got merged. Modified: stable/11/sys/dev/ioat/ioat.c stable/11/sys/dev/ioat/ioat.h stable/11/sys/dev/ioat/ioat_hw.h stable/11/sys/dev/ioat/ioat_internal.h Directory Properties: stable/11/ (props changed) Modified: stable/11/sys/dev/ioat/ioat.c ============================================================================== --- stable/11/sys/dev/ioat/ioat.c Thu Feb 21 00:17:24 2019 (r344400) +++ stable/11/sys/dev/ioat/ioat.c Thu Feb 21 00:44:26 2019 (r344401) @@ -33,6 +33,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/bus.h> #include <sys/conf.h> +#include <sys/fail.h> #include <sys/ioccom.h> #include <sys/kernel.h> #include <sys/lock.h> @@ -62,7 +63,6 @@ __FBSDID("$FreeBSD$"); #define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFULL #endif #define IOAT_REFLK (&ioat->submit_lock) -#define IOAT_SHRINK_PERIOD (10 * hz) static int ioat_probe(device_t device); static int ioat_attach(device_t device); @@ -81,23 +81,14 @@ static void ioat_process_events(struct ioat_softc *ioa static inline uint32_t ioat_get_active(struct ioat_softc *ioat); static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat); static void ioat_free_ring(struct ioat_softc *, uint32_t size, - struct ioat_descriptor **); -static void ioat_free_ring_entry(struct ioat_softc *ioat, - struct ioat_descriptor *desc); -static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *, - int mflags); + struct ioat_descriptor *); static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags); -static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat, +static union ioat_hw_descriptor *ioat_get_descriptor(struct ioat_softc *, uint32_t index); -static struct ioat_descriptor **ioat_prealloc_ring(struct ioat_softc *, - uint32_t size, boolean_t need_dscr, int mflags); -static int ring_grow(struct ioat_softc *, uint32_t oldorder, - struct ioat_descriptor **); -static int ring_shrink(struct ioat_softc *, uint32_t oldorder, - struct ioat_descriptor **); +static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *, + uint32_t index); static void ioat_halted_debug(struct ioat_softc *, uint32_t); static void ioat_poll_timer_callback(void *arg); -static void ioat_shrink_timer_callback(void *arg); static void dump_descriptor(void *hw_desc); static void ioat_submit_single(struct ioat_softc *ioat); static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, @@ -134,6 +125,10 @@ int g_ioat_debug_level = 0; SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level, 0, "Set log level (0-3) for ioat(4). Higher is more verbose."); +unsigned g_ioat_ring_order = 13; +SYSCTL_UINT(_hw_ioat, OID_AUTO, ring_order, CTLFLAG_RDTUN, &g_ioat_ring_order, + 0, "Set IOAT ring order. (1 << this) == ring size."); + /* * OS <-> Driver interface structures */ @@ -335,7 +330,6 @@ ioat_detach(device_t device) ioat_teardown_intr(ioat); callout_drain(&ioat->poll_timer); - callout_drain(&ioat->shrink_timer); pci_disable_busmaster(device); @@ -353,7 +347,12 @@ ioat_detach(device_t device) bus_dma_tag_destroy(ioat->comp_update_tag); } - bus_dma_tag_destroy(ioat->hw_desc_tag); + if (ioat->hw_desc_ring != NULL) { + bus_dmamap_unload(ioat->hw_desc_tag, ioat->hw_desc_map); + bus_dmamem_free(ioat->hw_desc_tag, ioat->hw_desc_ring, + ioat->hw_desc_map); + bus_dma_tag_destroy(ioat->hw_desc_tag); + } return (0); } @@ -387,8 +386,8 @@ ioat_start_channel(struct ioat_softc *ioat) /* Submit 'NULL' operation manually to avoid quiescing flag */ desc = ioat_get_ring_entry(ioat, ioat->head); + hw_desc = &ioat_get_descriptor(ioat, ioat->head)->dma; dmadesc = &desc->bus_dmadesc; - hw_desc = desc->u.dma; dmadesc->callback_fn = NULL; dmadesc->callback_arg = NULL; @@ -425,9 +424,10 @@ static int ioat3_attach(device_t device) { struct ioat_softc *ioat; - struct ioat_descriptor **ring; - struct ioat_descriptor *next; + struct ioat_descriptor *ring; struct ioat_dma_hw_descriptor *dma_hw_desc; + void *hw_desc; + size_t ringsz; int i, num_descriptors; int error; uint8_t xfercap; @@ -452,7 +452,6 @@ ioat3_attach(device_t device) mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); mtx_init(&ioat->cleanup_lock, "ioat_cleanup", NULL, MTX_DEF); callout_init(&ioat->poll_timer, 1); - callout_init(&ioat->shrink_timer, 1); TASK_INIT(&ioat->reset_task, 0, ioat_reset_hw_task, ioat); /* Establish lock order for Witness */ @@ -461,7 +460,7 @@ ioat3_attach(device_t device) mtx_unlock(&ioat->cleanup_lock); mtx_unlock(&ioat->submit_lock); - ioat->is_resize_pending = FALSE; + ioat->is_submitter_processing = FALSE; ioat->is_completion_pending = FALSE; ioat->is_reset_pending = FALSE; ioat->is_channel_running = FALSE; @@ -482,37 +481,43 @@ ioat3_attach(device_t device) if (error != 0) return (error); - ioat->ring_size_order = IOAT_MIN_ORDER; - + ioat->ring_size_order = g_ioat_ring_order; num_descriptors = 1 << ioat->ring_size_order; + ringsz = sizeof(struct ioat_dma_hw_descriptor) * num_descriptors; - bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0, - BUS_SPACE_MAXADDR_40BIT, BUS_SPACE_MAXADDR, NULL, NULL, - sizeof(struct ioat_dma_hw_descriptor), 1, - sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL, + error = bus_dma_tag_create(bus_get_dma_tag(ioat->device), + 2 * 1024 * 1024, 0x0, (bus_addr_t)BUS_SPACE_MAXADDR_40BIT, + BUS_SPACE_MAXADDR, NULL, NULL, ringsz, 1, ringsz, 0, NULL, NULL, &ioat->hw_desc_tag); + if (error != 0) + return (error); + error = bus_dmamem_alloc(ioat->hw_desc_tag, &hw_desc, + BUS_DMA_ZERO | BUS_DMA_WAITOK, &ioat->hw_desc_map); + if (error != 0) + return (error); + + error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc, + ringsz, ioat_dmamap_cb, &ioat->hw_desc_bus_addr, BUS_DMA_WAITOK); + if (error) + return (error); + + ioat->hw_desc_ring = hw_desc; + ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT, M_ZERO | M_WAITOK); ring = ioat->ring; for (i = 0; i < num_descriptors; i++) { - ring[i] = ioat_alloc_ring_entry(ioat, M_WAITOK); - if (ring[i] == NULL) - return (ENOMEM); - - ring[i]->id = i; + memset(&ring[i].bus_dmadesc, 0, sizeof(ring[i].bus_dmadesc)); + ring[i].id = i; } - for (i = 0; i < num_descriptors - 1; i++) { - next = ring[i + 1]; - dma_hw_desc = ring[i]->u.dma; - - dma_hw_desc->next = next->hw_desc_bus_addr; + for (i = 0; i < num_descriptors; i++) { + dma_hw_desc = &ioat->hw_desc_ring[i].dma; + dma_hw_desc->next = RING_PHYS_ADDR(ioat, i + 1); } - ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr; - ioat->head = ioat->hw_head = 0; ioat->tail = 0; ioat->last_seen = 0; @@ -662,8 +667,6 @@ ioat_process_events(struct ioat_softc *ioat) boolean_t pending; int error; - CTR0(KTR_IOAT, __func__); - mtx_lock(&ioat->cleanup_lock); /* @@ -680,31 +683,42 @@ ioat_process_events(struct ioat_softc *ioat) comp_update = *ioat->comp_update; status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; + if (status < ioat->hw_desc_bus_addr || + status >= ioat->hw_desc_bus_addr + (1 << ioat->ring_size_order) * + sizeof(struct ioat_generic_hw_descriptor)) + panic("Bogus completion address %jx (channel %u)", + (uintmax_t)status, ioat->chan_idx); + if (status == ioat->last_seen) { /* * If we landed in process_events and nothing has been * completed, check for a timeout due to channel halt. */ - comp_update = ioat_get_chansts(ioat); goto out; } + CTR4(KTR_IOAT, "%s channel=%u hw_status=0x%lx last_seen=0x%lx", + __func__, ioat->chan_idx, comp_update, ioat->last_seen); - while (1) { + while (RING_PHYS_ADDR(ioat, ioat->tail - 1) != status) { desc = ioat_get_ring_entry(ioat, ioat->tail); dmadesc = &desc->bus_dmadesc; - CTR1(KTR_IOAT, "completing desc %d", ioat->tail); + CTR5(KTR_IOAT, "channel=%u completing desc idx %u (%p) ok cb %p(%p)", + ioat->chan_idx, ioat->tail, dmadesc, dmadesc->callback_fn, + dmadesc->callback_arg); if (dmadesc->callback_fn != NULL) dmadesc->callback_fn(dmadesc->callback_arg, 0); completed++; ioat->tail++; - if (desc->hw_desc_bus_addr == status) - break; } + CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__, + ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat)); - ioat->last_seen = desc->hw_desc_bus_addr; - ioat->stats.descriptors_processed += completed; + if (completed != 0) { + ioat->last_seen = RING_PHYS_ADDR(ioat, ioat->tail - 1); + ioat->stats.descriptors_processed += completed; + } out: ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); @@ -719,8 +733,6 @@ out: pending = (ioat_get_active(ioat) != 0); if (!pending && ioat->is_completion_pending) { ioat->is_completion_pending = FALSE; - callout_reset(&ioat->shrink_timer, IOAT_SHRINK_PERIOD, - ioat_shrink_timer_callback, ioat); callout_stop(&ioat->poll_timer); } mtx_unlock(&ioat->submit_lock); @@ -736,6 +748,12 @@ out: wakeup(&ioat->tail); } + /* + * The device doesn't seem to reliably push suspend/halt statuses to + * the channel completion memory address, so poll the device register + * here. + */ + comp_update = ioat_get_chansts(ioat) & IOAT_CHANSTS_STATUS; if (!is_ioat_halted(comp_update) && !is_ioat_suspended(comp_update)) return; @@ -745,19 +763,30 @@ out: * Fatal programming error on this DMA channel. Flush any outstanding * work with error status and restart the engine. */ - ioat_log_message(0, "Channel halted due to fatal programming error\n"); mtx_lock(&ioat->submit_lock); mtx_lock(&ioat->cleanup_lock); ioat->quiescing = TRUE; + /* + * This is safe to do here because we have both locks and the submit + * queue is quiesced. We know that we will drain all outstanding + * events, so ioat_reset_hw can't deadlock. It is necessary to + * protect other ioat_process_event threads from racing ioat_reset_hw, + * reading an indeterminate hw state, and attempting to continue + * issuing completions. + */ + ioat->resetting_cleanup = TRUE; chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); - ioat_halted_debug(ioat, chanerr); + if (1 <= g_ioat_debug_level) + ioat_halted_debug(ioat, chanerr); ioat->stats.last_halt_chanerr = chanerr; while (ioat_get_active(ioat) > 0) { desc = ioat_get_ring_entry(ioat, ioat->tail); dmadesc = &desc->bus_dmadesc; - CTR1(KTR_IOAT, "completing err desc %d", ioat->tail); + CTR5(KTR_IOAT, "channel=%u completing desc idx %u (%p) err cb %p(%p)", + ioat->chan_idx, ioat->tail, dmadesc, dmadesc->callback_fn, + dmadesc->callback_arg); if (dmadesc->callback_fn != NULL) dmadesc->callback_fn(dmadesc->callback_arg, @@ -768,7 +797,14 @@ out: ioat->stats.descriptors_processed++; ioat->stats.descriptors_error++; } + CTR5(KTR_IOAT, "%s channel=%u head=%u tail=%u active=%u", __func__, + ioat->chan_idx, ioat->head, ioat->tail, ioat_get_active(ioat)); + if (ioat->is_completion_pending) { + ioat->is_completion_pending = FALSE; + callout_stop(&ioat->poll_timer); + } + /* Clear error status */ ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr); @@ -869,6 +905,15 @@ ioat_get_max_io_size(bus_dmaengine_t dmaengine) return (ioat->max_xfer_size); } +uint32_t +ioat_get_capabilities(bus_dmaengine_t dmaengine) +{ + struct ioat_softc *ioat; + + ioat = to_ioat_softc(dmaengine); + return (ioat->capabilities); +} + int ioat_set_interrupt_coalesce(bus_dmaengine_t dmaengine, uint16_t delay) { @@ -902,7 +947,8 @@ ioat_acquire(bus_dmaengine_t dmaengine) ioat = to_ioat_softc(dmaengine); mtx_lock(&ioat->submit_lock); - CTR0(KTR_IOAT, __func__); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); + ioat->acq_head = ioat->head; } int @@ -926,8 +972,22 @@ ioat_release(bus_dmaengine_t dmaengine) struct ioat_softc *ioat; ioat = to_ioat_softc(dmaengine); - CTR0(KTR_IOAT, __func__); - ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->hw_head); + CTR4(KTR_IOAT, "%s channel=%u dispatch1 hw_head=%u head=%u", __func__, + ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head); + KFAIL_POINT_CODE(DEBUG_FP, ioat_release, /* do nothing */); + CTR4(KTR_IOAT, "%s channel=%u dispatch2 hw_head=%u head=%u", __func__, + ioat->chan_idx, ioat->hw_head & UINT16_MAX, ioat->head); + + if (ioat->acq_head != ioat->head) { + ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, + (uint16_t)ioat->hw_head); + + if (!ioat->is_completion_pending) { + ioat->is_completion_pending = TRUE; + callout_reset(&ioat->poll_timer, 1, + ioat_poll_timer_callback, ioat); + } + } mtx_unlock(&ioat->submit_lock); } @@ -960,7 +1020,7 @@ ioat_op_generic(struct ioat_softc *ioat, uint8_t op, return (NULL); desc = ioat_get_ring_entry(ioat, ioat->head); - hw_desc = desc->u.generic; + hw_desc = &ioat_get_descriptor(ioat, ioat->head)->generic; hw_desc->u.control_raw = 0; hw_desc->u.control_generic.op = op; @@ -988,15 +1048,15 @@ ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_cal struct ioat_descriptor *desc; struct ioat_softc *ioat; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); desc = ioat_op_generic(ioat, IOAT_OP_COPY, 8, 0, 0, callback_fn, callback_arg, flags); if (desc == NULL) return (NULL); - hw_desc = desc->u.dma; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma; hw_desc->u.control.null = 1; ioat_submit_single(ioat); return (&desc->bus_dmadesc); @@ -1011,7 +1071,6 @@ ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst, struct ioat_descriptor *desc; struct ioat_softc *ioat; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); if (((src | dst) & (0xffffull << 48)) != 0) { @@ -1025,11 +1084,13 @@ ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst, if (desc == NULL) return (NULL); - hw_desc = desc->u.dma; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma; if (g_ioat_debug_level >= 3) dump_descriptor(hw_desc); ioat_submit_single(ioat); + CTR6(KTR_IOAT, "%s channel=%u desc=%p dest=%lx src=%lx len=%lx", + __func__, ioat->chan_idx, &desc->bus_dmadesc, dst, src, len); return (&desc->bus_dmadesc); } @@ -1042,8 +1103,8 @@ ioat_copy_8k_aligned(bus_dmaengine_t dmaengine, bus_ad struct ioat_descriptor *desc; struct ioat_softc *ioat; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); if (((src1 | src2 | dst1 | dst2) & (0xffffull << 48)) != 0) { ioat_log_message(0, "%s: High 16 bits of src/dst invalid\n", @@ -1061,7 +1122,7 @@ ioat_copy_8k_aligned(bus_dmaengine_t dmaengine, bus_ad if (desc == NULL) return (NULL); - hw_desc = desc->u.dma; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->dma; if (src2 != src1 + PAGE_SIZE) { hw_desc->u.control.src_page_break = 1; hw_desc->next_src_addr = src2; @@ -1089,8 +1150,8 @@ ioat_copy_crc(bus_dmaengine_t dmaengine, bus_addr_t ds uint32_t teststore; uint8_t op; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); if ((ioat->capabilities & IOAT_DMACAP_MOVECRC) == 0) { ioat_log_message(0, "%s: Device lacks MOVECRC capability\n", @@ -1138,7 +1199,7 @@ ioat_copy_crc(bus_dmaengine_t dmaengine, bus_addr_t ds if (desc == NULL) return (NULL); - hw_desc = desc->u.crc32; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32; if ((flags & DMA_CRC_INLINE) == 0) hw_desc->crc_address = crcptr; @@ -1168,8 +1229,8 @@ ioat_crc(bus_dmaengine_t dmaengine, bus_addr_t src, bu uint32_t teststore; uint8_t op; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); if ((ioat->capabilities & IOAT_DMACAP_CRC) == 0) { ioat_log_message(0, "%s: Device lacks CRC capability\n", @@ -1217,7 +1278,7 @@ ioat_crc(bus_dmaengine_t dmaengine, bus_addr_t src, bu if (desc == NULL) return (NULL); - hw_desc = desc->u.crc32; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->crc32; if ((flags & DMA_CRC_INLINE) == 0) hw_desc->crc_address = crcptr; @@ -1245,8 +1306,8 @@ ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t d struct ioat_descriptor *desc; struct ioat_softc *ioat; - CTR0(KTR_IOAT, __func__); ioat = to_ioat_softc(dmaengine); + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); if ((ioat->capabilities & IOAT_DMACAP_BFILL) == 0) { ioat_log_message(0, "%s: Device lacks BFILL capability\n", @@ -1265,7 +1326,7 @@ ioat_blockfill(bus_dmaengine_t dmaengine, bus_addr_t d if (desc == NULL) return (NULL); - hw_desc = desc->u.fill; + hw_desc = &ioat_get_descriptor(ioat, desc->id)->fill; if (g_ioat_debug_level >= 3) dump_descriptor(hw_desc); @@ -1290,60 +1351,6 @@ ioat_get_ring_space(struct ioat_softc *ioat) return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1); } -static struct ioat_descriptor * -ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags) -{ - struct ioat_generic_hw_descriptor *hw_desc; - struct ioat_descriptor *desc; - int error, busdmaflag; - - error = ENOMEM; - hw_desc = NULL; - - if ((mflags & M_WAITOK) != 0) - busdmaflag = BUS_DMA_WAITOK; - else - busdmaflag = BUS_DMA_NOWAIT; - - desc = malloc(sizeof(*desc), M_IOAT, mflags); - if (desc == NULL) - goto out; - - bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, - BUS_DMA_ZERO | busdmaflag, &ioat->hw_desc_map); - if (hw_desc == NULL) - goto out; - - memset(&desc->bus_dmadesc, 0, sizeof(desc->bus_dmadesc)); - desc->u.generic = hw_desc; - - error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc, - sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, - busdmaflag); - if (error) - goto out; - -out: - if (error) { - ioat_free_ring_entry(ioat, desc); - return (NULL); - } - return (desc); -} - -static void -ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc) -{ - - if (desc == NULL) - return; - - if (desc->u.generic) - bus_dmamem_free(ioat->hw_desc_tag, desc->u.generic, - ioat->hw_desc_map); - free(desc, M_IOAT); -} - /* * Reserves space in this IOAT descriptor ring by ensuring enough slots remain * for 'num_descs'. @@ -1363,114 +1370,70 @@ ioat_free_ring_entry(struct ioat_softc *ioat, struct i static int ioat_reserve_space(struct ioat_softc *ioat, uint32_t num_descs, int mflags) { - struct ioat_descriptor **new_ring; - uint32_t order; + boolean_t dug; int error; mtx_assert(&ioat->submit_lock, MA_OWNED); error = 0; + dug = FALSE; - if (num_descs < 1 || num_descs > (1 << IOAT_MAX_ORDER)) { + if (num_descs < 1 || num_descs >= (1 << ioat->ring_size_order)) { error = EINVAL; goto out; } - if (ioat->quiescing) { - error = ENXIO; - goto out; - } for (;;) { + if (ioat->quiescing) { + error = ENXIO; + goto out; + } + if (ioat_get_ring_space(ioat) >= num_descs) goto out; - order = ioat->ring_size_order; - if (ioat->is_resize_pending || order == IOAT_MAX_ORDER) { - if ((mflags & M_WAITOK) != 0) { - msleep(&ioat->tail, &ioat->submit_lock, 0, - "ioat_rsz", 0); - continue; - } + CTR3(KTR_IOAT, "%s channel=%u starved (%u)", __func__, + ioat->chan_idx, num_descs); - error = EAGAIN; - break; - } - - ioat->is_resize_pending = TRUE; - for (;;) { + if (!dug && !ioat->is_submitter_processing) { + ioat->is_submitter_processing = TRUE; mtx_unlock(&ioat->submit_lock); - new_ring = ioat_prealloc_ring(ioat, 1 << (order + 1), - TRUE, mflags); + CTR2(KTR_IOAT, "%s channel=%u attempting to process events", + __func__, ioat->chan_idx); + ioat_process_events(ioat); mtx_lock(&ioat->submit_lock); - KASSERT(ioat->ring_size_order == order, - ("is_resize_pending should protect order")); - - if (new_ring == NULL) { - KASSERT((mflags & M_WAITOK) == 0, - ("allocation failed")); - error = EAGAIN; - break; - } - - error = ring_grow(ioat, order, new_ring); - if (error == 0) - break; + dug = TRUE; + KASSERT(ioat->is_submitter_processing == TRUE, + ("is_submitter_processing")); + ioat->is_submitter_processing = FALSE; + wakeup(&ioat->tail); + continue; } - ioat->is_resize_pending = FALSE; - wakeup(&ioat->tail); - if (error) + + if ((mflags & M_WAITOK) == 0) { + error = EAGAIN; break; + } + CTR2(KTR_IOAT, "%s channel=%u blocking on completions", + __func__, ioat->chan_idx); + msleep(&ioat->tail, &ioat->submit_lock, 0, + "ioat_full", 0); + continue; } out: mtx_assert(&ioat->submit_lock, MA_OWNED); + KASSERT(!ioat->quiescing || error == ENXIO, + ("reserved during quiesce")); return (error); } -static struct ioat_descriptor ** -ioat_prealloc_ring(struct ioat_softc *ioat, uint32_t size, boolean_t need_dscr, - int mflags) -{ - struct ioat_descriptor **ring; - uint32_t i; - int error; - - KASSERT(size > 0 && powerof2(size), ("bogus size")); - - ring = malloc(size * sizeof(*ring), M_IOAT, M_ZERO | mflags); - if (ring == NULL) - return (NULL); - - if (need_dscr) { - error = ENOMEM; - for (i = size / 2; i < size; i++) { - ring[i] = ioat_alloc_ring_entry(ioat, mflags); - if (ring[i] == NULL) - goto out; - ring[i]->id = i; - } - } - error = 0; - -out: - if (error != 0 && ring != NULL) { - ioat_free_ring(ioat, size, ring); - ring = NULL; - } - return (ring); -} - static void ioat_free_ring(struct ioat_softc *ioat, uint32_t size, - struct ioat_descriptor **ring) + struct ioat_descriptor *ring) { - uint32_t i; - for (i = 0; i < size; i++) { - if (ring[i] != NULL) - ioat_free_ring_entry(ioat, ring[i]); - } free(ring, M_IOAT); } @@ -1478,164 +1441,20 @@ static struct ioat_descriptor * ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index) { - return (ioat->ring[index % (1 << ioat->ring_size_order)]); + return (&ioat->ring[index % (1 << ioat->ring_size_order)]); } -static int -ring_grow(struct ioat_softc *ioat, uint32_t oldorder, - struct ioat_descriptor **newring) +static union ioat_hw_descriptor * +ioat_get_descriptor(struct ioat_softc *ioat, uint32_t index) { - struct ioat_descriptor *tmp, *next; - struct ioat_dma_hw_descriptor *hw; - uint32_t oldsize, newsize, head, tail, i, end; - int error; - CTR0(KTR_IOAT, __func__); - - mtx_assert(&ioat->submit_lock, MA_OWNED); - - if (oldorder != ioat->ring_size_order || oldorder >= IOAT_MAX_ORDER) { - error = EINVAL; - goto out; - } - - oldsize = (1 << oldorder); - newsize = (1 << (oldorder + 1)); - - mtx_lock(&ioat->cleanup_lock); - - head = ioat->head & (oldsize - 1); - tail = ioat->tail & (oldsize - 1); - - /* Copy old descriptors to new ring */ - for (i = 0; i < oldsize; i++) - newring[i] = ioat->ring[i]; - - /* - * If head has wrapped but tail hasn't, we must swap some descriptors - * around so that tail can increment directly to head. - */ - if (head < tail) { - for (i = 0; i <= head; i++) { - tmp = newring[oldsize + i]; - - newring[oldsize + i] = newring[i]; - newring[oldsize + i]->id = oldsize + i; - - newring[i] = tmp; - newring[i]->id = i; - } - head += oldsize; - } - - KASSERT(head >= tail, ("invariants")); - - /* Head didn't wrap; we only need to link in oldsize..newsize */ - if (head < oldsize) { - i = oldsize - 1; - end = newsize; - } else { - /* Head did wrap; link newhead..newsize and 0..oldhead */ - i = head; - end = newsize + (head - oldsize) + 1; - } - - /* - * Fix up hardware ring, being careful not to trample the active - * section (tail -> head). - */ - for (; i < end; i++) { - KASSERT((i & (newsize - 1)) < tail || - (i & (newsize - 1)) >= head, ("trampling snake")); - - next = newring[(i + 1) & (newsize - 1)]; - hw = newring[i & (newsize - 1)]->u.dma; - hw->next = next->hw_desc_bus_addr; - } - - free(ioat->ring, M_IOAT); - ioat->ring = newring; - ioat->ring_size_order = oldorder + 1; - ioat->tail = tail; - ioat->head = head; - error = 0; - - mtx_unlock(&ioat->cleanup_lock); -out: - if (error) - ioat_free_ring(ioat, (1 << (oldorder + 1)), newring); - return (error); + return (&ioat->hw_desc_ring[index % (1 << ioat->ring_size_order)]); } -static int -ring_shrink(struct ioat_softc *ioat, uint32_t oldorder, - struct ioat_descriptor **newring) -{ - struct ioat_dma_hw_descriptor *hw; - struct ioat_descriptor *ent, *next; - uint32_t oldsize, newsize, current_idx, new_idx, i; - int error; - - CTR0(KTR_IOAT, __func__); - - mtx_assert(&ioat->submit_lock, MA_OWNED); - - if (oldorder != ioat->ring_size_order || oldorder <= IOAT_MIN_ORDER) { - error = EINVAL; - goto out_unlocked; - } - - oldsize = (1 << oldorder); - newsize = (1 << (oldorder - 1)); - - mtx_lock(&ioat->cleanup_lock); - - /* Can't shrink below current active set! */ - if (ioat_get_active(ioat) >= newsize) { - error = ENOMEM; - goto out; - } - - /* - * Copy current descriptors to the new ring, dropping the removed - * descriptors. - */ - for (i = 0; i < newsize; i++) { - current_idx = (ioat->tail + i) & (oldsize - 1); - new_idx = (ioat->tail + i) & (newsize - 1); - - newring[new_idx] = ioat->ring[current_idx]; - newring[new_idx]->id = new_idx; - } - - /* Free deleted descriptors */ - for (i = newsize; i < oldsize; i++) { - ent = ioat_get_ring_entry(ioat, ioat->tail + i); - ioat_free_ring_entry(ioat, ent); - } - - /* Fix up hardware ring. */ - hw = newring[(ioat->tail + newsize - 1) & (newsize - 1)]->u.dma; - next = newring[(ioat->tail + newsize) & (newsize - 1)]; - hw->next = next->hw_desc_bus_addr; - - free(ioat->ring, M_IOAT); - ioat->ring = newring; - ioat->ring_size_order = oldorder - 1; - error = 0; - -out: - mtx_unlock(&ioat->cleanup_lock); -out_unlocked: - if (error) - ioat_free_ring(ioat, (1 << (oldorder - 1)), newring); - return (error); -} - static void ioat_halted_debug(struct ioat_softc *ioat, uint32_t chanerr) { - struct ioat_descriptor *desc; + union ioat_hw_descriptor *desc; ioat_log_message(0, "Channel halted (%b)\n", (int)chanerr, IOAT_CHANERR_STR); @@ -1644,11 +1463,11 @@ ioat_halted_debug(struct ioat_softc *ioat, uint32_t ch mtx_assert(&ioat->cleanup_lock, MA_OWNED); - desc = ioat_get_ring_entry(ioat, ioat->tail + 0); - dump_descriptor(desc->u.raw); + desc = ioat_get_descriptor(ioat, ioat->tail + 0); + dump_descriptor(desc); - desc = ioat_get_ring_entry(ioat, ioat->tail + 1); - dump_descriptor(desc->u.raw); + desc = ioat_get_descriptor(ioat, ioat->tail + 1); + dump_descriptor(desc); } static void @@ -1662,52 +1481,6 @@ ioat_poll_timer_callback(void *arg) ioat_process_events(ioat); } -static void -ioat_shrink_timer_callback(void *arg) -{ - struct ioat_descriptor **newring; - struct ioat_softc *ioat; - uint32_t order; - - ioat = arg; - ioat_log_message(1, "%s\n", __func__); - - /* Slowly scale the ring down if idle. */ - mtx_lock(&ioat->submit_lock); - - /* Don't run while the hardware is being reset. */ - if (ioat->resetting) { - mtx_unlock(&ioat->submit_lock); - return; - } - - order = ioat->ring_size_order; - if (ioat->is_resize_pending || order == IOAT_MIN_ORDER) { - mtx_unlock(&ioat->submit_lock); - goto out; - } - ioat->is_resize_pending = TRUE; - mtx_unlock(&ioat->submit_lock); - - newring = ioat_prealloc_ring(ioat, 1 << (order - 1), FALSE, - M_NOWAIT); - - mtx_lock(&ioat->submit_lock); - KASSERT(ioat->ring_size_order == order, - ("resize_pending protects order")); - - if (newring != NULL) - ring_shrink(ioat, order, newring); - - ioat->is_resize_pending = FALSE; - mtx_unlock(&ioat->submit_lock); - -out: - if (ioat->ring_size_order > IOAT_MIN_ORDER) - callout_reset(&ioat->poll_timer, IOAT_SHRINK_PERIOD, - ioat_shrink_timer_callback, ioat); -} - /* * Support Functions */ @@ -1715,17 +1488,15 @@ static void ioat_submit_single(struct ioat_softc *ioat) { + mtx_assert(&ioat->submit_lock, MA_OWNED); + ioat_get(ioat, IOAT_ACTIVE_DESCR_REF); atomic_add_rel_int(&ioat->head, 1); atomic_add_rel_int(&ioat->hw_head, 1); + CTR5(KTR_IOAT, "%s channel=%u head=%u hw_head=%u tail=%u", __func__, + ioat->chan_idx, ioat->head, ioat->hw_head & UINT16_MAX, + ioat->tail); - if (!ioat->is_completion_pending) { - ioat->is_completion_pending = TRUE; - callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback, - ioat); - callout_stop(&ioat->shrink_timer); - } - ioat->stats.descriptors_submitted++; } @@ -1737,6 +1508,8 @@ ioat_reset_hw(struct ioat_softc *ioat) unsigned timeout; int error; + CTR2(KTR_IOAT, "%s channel=%u", __func__, ioat->chan_idx); + mtx_lock(IOAT_REFLK); while (ioat->resetting && !ioat->destroying) msleep(&ioat->resetting, IOAT_REFLK, 0, "IRH_drain", 0); @@ -1758,6 +1531,9 @@ ioat_reset_hw(struct ioat_softc *ioat) ioat->resetting_cleanup = TRUE; mtx_unlock(&ioat->cleanup_lock); + CTR2(KTR_IOAT, "%s channel=%u quiesced and drained", __func__, + ioat->chan_idx); + status = ioat_get_chansts(ioat); if (is_ioat_active(status) || is_ioat_idle(status)) ioat_suspend(ioat); @@ -1778,6 +1554,9 @@ ioat_reset_hw(struct ioat_softc *ioat) chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr); + CTR2(KTR_IOAT, "%s channel=%u hardware suspended", __func__, + ioat->chan_idx); + /* * IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors * that can cause stability issues for IOAT v3. @@ -1797,6 +1576,8 @@ ioat_reset_hw(struct ioat_softc *ioat) } ioat_reset(ioat); + CTR2(KTR_IOAT, "%s channel=%u hardware reset", __func__, + ioat->chan_idx); /* Wait at most 20 ms */ for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++) @@ -1840,26 +1621,30 @@ ioat_reset_hw(struct ioat_softc *ioat) ioat->tail = ioat->head = ioat->hw_head = 0; ioat->last_seen = 0; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201902210044.x1L0iQY3022372>