From owner-svn-src-head@freebsd.org Wed Jan 16 05:44:15 2019 Return-Path: Delivered-To: svn-src-head@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 2F02814A3DA4; Wed, 16 Jan 2019 05:44:15 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id C43CB8D9BE; Wed, 16 Jan 2019 05:44:14 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 95FB725E69; Wed, 16 Jan 2019 05:44:14 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id x0G5iE3X095382; Wed, 16 Jan 2019 05:44:14 GMT (envelope-from kib@FreeBSD.org) Received: (from kib@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id x0G5iErW095381; Wed, 16 Jan 2019 05:44:14 GMT (envelope-from kib@FreeBSD.org) Message-Id: <201901160544.x0G5iErW095381@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: kib set sender to kib@FreeBSD.org using -f From: Konstantin Belousov Date: Wed, 16 Jan 2019 05:44:14 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r343085 - head/sys/net X-SVN-Group: head X-SVN-Commit-Author: kib X-SVN-Commit-Paths: head/sys/net X-SVN-Commit-Revision: 343085 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Rspamd-Queue-Id: C43CB8D9BE X-Spamd-Bar: -- Authentication-Results: mx1.freebsd.org X-Spamd-Result: default: False [-2.98 / 15.00]; local_wl_from(0.00)[FreeBSD.org]; NEURAL_HAM_MEDIUM(-1.00)[-0.999,0]; NEURAL_HAM_SHORT(-0.98)[-0.980,0]; ASN(0.00)[asn:11403, ipnet:2610:1c1:1::/48, country:US]; NEURAL_HAM_LONG(-1.00)[-0.999,0] X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 16 Jan 2019 05:44:15 -0000 Author: kib Date: Wed Jan 16 05:44:14 2019 New Revision: 343085 URL: https://svnweb.freebsd.org/changeset/base/343085 Log: Improve iflib busdma(9) KPI use. - Specify BUS_DMA_NOWAIT for bus_dmamap_load() on rx refill, since callbacks are not supposed to be used. - Match tso/non-tso tags to corresponding tx map operations. Create separate tso maps for tx descriptors. In particular, do not use non-tso tag to load, unload, or destroy a map created with tso tag. - Add missed bus_dmamap_sync() calls. Submitted by: marius. Reported and tested by: pho Reviewed by: marius Sponsored by: The FreeBSD Foundation MFC after: 1 week Modified: head/sys/net/iflib.c Modified: head/sys/net/iflib.c ============================================================================== --- head/sys/net/iflib.c Wed Jan 16 05:17:27 2019 (r343084) +++ head/sys/net/iflib.c Wed Jan 16 05:44:14 2019 (r343085) @@ -282,6 +282,7 @@ typedef struct iflib_sw_rx_desc_array { typedef struct iflib_sw_tx_desc_array { bus_dmamap_t *ifsd_map; /* bus_dma maps for packet */ + bus_dmamap_t *ifsd_tso_map; /* bus_dma maps for TSO packet */ struct mbuf **ifsd_m; /* pkthdr mbufs */ } if_txsd_vec_t; @@ -1491,6 +1492,8 @@ iflib_fast_intr_rxtx(void *arg) ctx = rxq->ifr_ctx; + bus_dmamap_sync(rxq->ifr_ifdi->idi_tag, rxq->ifr_ifdi->idi_map, + BUS_DMASYNC_POSTREAD); if (!ctx->isc_txd_credits_update(ctx->ifc_softc, txqid, false)) { IFDI_TX_QUEUE_INTR_ENABLE(ctx, txqid); continue; @@ -1583,6 +1586,7 @@ iflib_txsd_alloc(iflib_txq_t txq) device_t dev = ctx->ifc_dev; bus_size_t tsomaxsize; int err, nsegments, ntsosegments; + bool tso; nsegments = scctx->isc_tx_nsegments; ntsosegments = scctx->isc_tx_tso_segments_max; @@ -1617,8 +1621,8 @@ iflib_txsd_alloc(iflib_txq_t txq) (uintmax_t)sctx->isc_tx_maxsize, nsegments, (uintmax_t)sctx->isc_tx_maxsegsize); goto fail; } - if ((if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) && - (err = bus_dma_tag_create(bus_get_dma_tag(dev), + tso = (if_getcapabilities(ctx->ifc_ifp) & IFCAP_TSO) != 0; + if (tso && (err = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ @@ -1631,7 +1635,6 @@ iflib_txsd_alloc(iflib_txq_t txq) NULL, /* lockfuncarg */ &txq->ift_tso_desc_tag))) { device_printf(dev,"Unable to allocate TX TSO DMA tag: %d\n", err); - goto fail; } if (!(txq->ift_sds.ifsd_m = @@ -1643,19 +1646,38 @@ iflib_txsd_alloc(iflib_txq_t txq) } /* Create the descriptor buffer dma maps */ - if (!(txq->ift_sds.ifsd_map = - (bus_dmamap_t *) malloc(sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], M_IFLIB, M_NOWAIT | M_ZERO))) { + if ((txq->ift_sds.ifsd_map = (bus_dmamap_t *)malloc( + sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], + M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { device_printf(dev, "Unable to allocate tx_buffer map memory\n"); err = ENOMEM; goto fail; } + if (tso && (txq->ift_sds.ifsd_tso_map = (bus_dmamap_t *)malloc( + sizeof(bus_dmamap_t) * scctx->isc_ntxd[txq->ift_br_offset], + M_IFLIB, M_NOWAIT | M_ZERO)) == NULL) { + device_printf(dev, "Unable to allocate TSO tx_buffer " + "map memory\n"); + err = ENOMEM; + goto fail; + } + for (int i = 0; i < scctx->isc_ntxd[txq->ift_br_offset]; i++) { - err = bus_dmamap_create(txq->ift_desc_tag, 0, &txq->ift_sds.ifsd_map[i]); + err = bus_dmamap_create(txq->ift_desc_tag, 0, + &txq->ift_sds.ifsd_map[i]); if (err != 0) { device_printf(dev, "Unable to create TX DMA map\n"); goto fail; } + if (!tso) + continue; + err = bus_dmamap_create(txq->ift_tso_desc_tag, 0, + &txq->ift_sds.ifsd_tso_map[i]); + if (err != 0) { + device_printf(dev, "Unable to create TSO TX DMA map\n"); + goto fail; + } } return (0); fail: @@ -1673,10 +1695,22 @@ iflib_txsd_destroy(if_ctx_t ctx, iflib_txq_t txq, int if (txq->ift_sds.ifsd_map != NULL) map = txq->ift_sds.ifsd_map[i]; if (map != NULL) { + bus_dmamap_sync(txq->ift_desc_tag, map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txq->ift_desc_tag, map); bus_dmamap_destroy(txq->ift_desc_tag, map); txq->ift_sds.ifsd_map[i] = NULL; } + + map = NULL; + if (txq->ift_sds.ifsd_tso_map != NULL) + map = txq->ift_sds.ifsd_tso_map[i]; + if (map != NULL) { + bus_dmamap_sync(txq->ift_tso_desc_tag, map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(txq->ift_tso_desc_tag, map); + bus_dmamap_destroy(txq->ift_tso_desc_tag, map); + txq->ift_sds.ifsd_tso_map[i] = NULL; + } } static void @@ -1690,6 +1724,10 @@ iflib_txq_destroy(iflib_txq_t txq) free(txq->ift_sds.ifsd_map, M_IFLIB); txq->ift_sds.ifsd_map = NULL; } + if (txq->ift_sds.ifsd_tso_map != NULL) { + free(txq->ift_sds.ifsd_tso_map, M_IFLIB); + txq->ift_sds.ifsd_tso_map = NULL; + } if (txq->ift_sds.ifsd_m != NULL) { free(txq->ift_sds.ifsd_m, M_IFLIB); txq->ift_sds.ifsd_m = NULL; @@ -1715,11 +1753,15 @@ iflib_txsd_free(if_ctx_t ctx, iflib_txq_t txq, int i) if (txq->ift_sds.ifsd_map != NULL) { bus_dmamap_sync(txq->ift_desc_tag, - txq->ift_sds.ifsd_map[i], - BUS_DMASYNC_POSTWRITE); - bus_dmamap_unload(txq->ift_desc_tag, - txq->ift_sds.ifsd_map[i]); + txq->ift_sds.ifsd_map[i], BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(txq->ift_desc_tag, txq->ift_sds.ifsd_map[i]); } + if (txq->ift_sds.ifsd_tso_map != NULL) { + bus_dmamap_sync(txq->ift_tso_desc_tag, + txq->ift_sds.ifsd_tso_map[i], BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(txq->ift_tso_desc_tag, + txq->ift_sds.ifsd_tso_map[i]); + } m_free(*mp); DBG_COUNTER_INC(tx_frees); *mp = NULL; @@ -1926,10 +1968,8 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int coun cb_arg.error = 0; MPASS(sd_map != NULL); err = bus_dmamap_load(fl->ifl_desc_tag, sd_map[frag_idx], - cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, 0); - bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx], - BUS_DMASYNC_PREREAD); - + cl, fl->ifl_buf_size, _rxq_refill_cb, &cb_arg, + BUS_DMA_NOWAIT); if (err != 0 || cb_arg.error) { /* * !zone_pack ? @@ -1939,6 +1979,8 @@ _iflib_fl_refill(if_ctx_t ctx, iflib_fl_t fl, int coun break; } + bus_dmamap_sync(fl->ifl_desc_tag, sd_map[frag_idx], + BUS_DMASYNC_PREREAD); sd_ba[frag_idx] = bus_addr = cb_arg.seg.ds_addr; sd_cl[frag_idx] = cl; #if MEMORY_LOGGING @@ -2030,6 +2072,7 @@ static void iflib_fl_bufs_free(iflib_fl_t fl) { iflib_dma_info_t idi = fl->ifl_ifdi; + bus_dmamap_t sd_map; uint32_t i; for (i = 0; i < fl->ifl_size; i++) { @@ -2037,7 +2080,9 @@ iflib_fl_bufs_free(iflib_fl_t fl) caddr_t *sd_cl = &fl->ifl_sds.ifsd_cl[i]; if (*sd_cl != NULL) { - bus_dmamap_t sd_map = fl->ifl_sds.ifsd_map[i]; + sd_map = fl->ifl_sds.ifsd_map[i]; + bus_dmamap_sync(fl->ifl_desc_tag, sd_map, + BUS_DMASYNC_POSTREAD); bus_dmamap_unload(fl->ifl_desc_tag, sd_map); if (*sd_cl != NULL) uma_zfree(fl->ifl_zone, *sd_cl); @@ -2140,19 +2185,32 @@ static void iflib_rx_sds_free(iflib_rxq_t rxq) { iflib_fl_t fl; - int i; + int i, j; if (rxq->ifr_fl != NULL) { for (i = 0; i < rxq->ifr_nfl; i++) { fl = &rxq->ifr_fl[i]; if (fl->ifl_desc_tag != NULL) { + if (fl->ifl_sds.ifsd_map != NULL) { + for (j = 0; j < fl->ifl_size; i++) { + if (fl->ifl_sds.ifsd_map[i] == + NULL) + continue; + bus_dmamap_sync( + fl->ifl_desc_tag, + fl->ifl_sds.ifsd_map[i], + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload( + fl->ifl_desc_tag, + fl->ifl_sds.ifsd_map[i]); + } + } bus_dma_tag_destroy(fl->ifl_desc_tag); fl->ifl_desc_tag = NULL; } free(fl->ifl_sds.ifsd_m, M_IFLIB); free(fl->ifl_sds.ifsd_cl, M_IFLIB); free(fl->ifl_sds.ifsd_ba, M_IFLIB); - /* XXX destroy maps first */ free(fl->ifl_sds.ifsd_map, M_IFLIB); fl->ifl_sds.ifsd_m = NULL; fl->ifl_sds.ifsd_cl = NULL; @@ -2430,11 +2488,10 @@ rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int map = fl->ifl_sds.ifsd_map[cidx]; di = fl->ifl_ifdi; next = (cidx + CACHE_LINE_SIZE) & (fl->ifl_size-1); - bus_dmamap_sync(di->idi_tag, di->idi_map, - BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); /* not valid assert if bxe really does SGE from non-contiguous elements */ MPASS(fl->ifl_cidx == cidx); + bus_dmamap_sync(fl->ifl_desc_tag, map, BUS_DMASYNC_POSTREAD); if (unload) bus_dmamap_unload(fl->ifl_desc_tag, map); fl->ifl_cidx = (fl->ifl_cidx + 1) & (fl->ifl_size-1); @@ -2442,7 +2499,7 @@ rxd_frag_to_sd(iflib_rxq_t rxq, if_rxd_frag_t irf, int fl->ifl_gen = 0; bus_dmamap_sync(fl->ifl_ifdi->idi_tag, fl->ifl_ifdi->idi_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); - bit_clear(fl->ifl_rx_bitmap, cidx); + bit_clear(fl->ifl_rx_bitmap, cidx); } static struct mbuf * @@ -2519,6 +2576,9 @@ iflib_rxd_pkt_get(iflib_rxq_t rxq, if_rxd_info_t ri) m->m_data += 2; #endif memcpy(m->m_data, *sd.ifsd_cl, ri->iri_len); + bus_dmamap_sync(rxq->ifr_fl->ifl_desc_tag, + rxq->ifr_fl->ifl_sds.ifsd_map[ri->iri_frags[0].irf_idx], + BUS_DMASYNC_PREREAD); m->m_len = ri->iri_frags[0].irf_len; } else { m = assemble_segments(rxq, ri, &sd); @@ -2587,6 +2647,7 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) if_ctx_t ctx = rxq->ifr_ctx; if_shared_ctx_t sctx = ctx->ifc_sctx; if_softc_ctx_t scctx = &ctx->ifc_softc_ctx; + iflib_dma_info_t di; int avail, i; qidx_t *cidxp; struct if_rxd_info ri; @@ -2631,6 +2692,9 @@ iflib_rxeof(iflib_rxq_t rxq, qidx_t budget) ri.iri_cidx = *cidxp; ri.iri_ifp = ifp; ri.iri_frags = rxq->ifr_frags; + di = rxq->ifr_fl[rxq->ifr_frags[0].irf_flid].ifl_ifdi; + bus_dmamap_sync(di->idi_tag, di->idi_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); err = ctx->isc_rxd_pkt_get(ctx->ifc_softc, &ri); if (err) @@ -3006,16 +3070,17 @@ iflib_remove_mbuf(iflib_txq_t txq) { int ntxd, pidx; struct mbuf *m, **ifsd_m; - bus_dmamap_t *ifsd_map; ifsd_m = txq->ift_sds.ifsd_m; ntxd = txq->ift_size; pidx = txq->ift_pidx & (ntxd - 1); ifsd_m = txq->ift_sds.ifsd_m; - ifsd_map = txq->ift_sds.ifsd_map; m = ifsd_m[pidx]; ifsd_m[pidx] = NULL; - bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[pidx]); + bus_dmamap_unload(txq->ift_desc_tag, txq->ift_sds.ifsd_map[pidx]); + if (txq->ift_sds.ifsd_tso_map != NULL) + bus_dmamap_unload(txq->ift_tso_desc_tag, + txq->ift_sds.ifsd_tso_map[pidx]); #if MEMORY_LOGGING txq->ift_dequeued++; #endif @@ -3131,11 +3196,13 @@ iflib_encap(iflib_txq_t txq, struct mbuf **m_headp) if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { desc_tag = txq->ift_tso_desc_tag; max_segs = scctx->isc_tx_tso_segments_max; + map = txq->ift_sds.ifsd_tso_map[pidx]; MPASS(desc_tag != NULL); MPASS(max_segs > 0); } else { desc_tag = txq->ift_desc_tag; max_segs = scctx->isc_tx_nsegments; + map = txq->ift_sds.ifsd_map[pidx]; } if ((sctx->isc_flags & IFLIB_NEED_ETHER_PAD) && __predict_false(m_head->m_pkthdr.len < scctx->isc_min_frame_size)) { @@ -3297,7 +3364,6 @@ iflib_tx_desc_free(iflib_txq_t txq, int n) { uint32_t qsize, cidx, mask, gen; struct mbuf *m, **ifsd_m; - bus_dmamap_t *ifsd_map; bool do_prefetch; cidx = txq->ift_cidx; @@ -3305,7 +3371,6 @@ iflib_tx_desc_free(iflib_txq_t txq, int n) qsize = txq->ift_size; mask = qsize-1; ifsd_m = txq->ift_sds.ifsd_m; - ifsd_map = txq->ift_sds.ifsd_map; do_prefetch = (txq->ift_ctx->ifc_flags & IFC_PREFETCH); while (n-- > 0) { @@ -3315,7 +3380,19 @@ iflib_tx_desc_free(iflib_txq_t txq, int n) } if ((m = ifsd_m[cidx]) != NULL) { prefetch(&ifsd_m[(cidx + CACHE_PTR_INCREMENT) & mask]); - bus_dmamap_unload(txq->ift_desc_tag, ifsd_map[cidx]); + if (m->m_pkthdr.csum_flags & CSUM_TSO) { + bus_dmamap_sync(txq->ift_tso_desc_tag, + txq->ift_sds.ifsd_tso_map[cidx], + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(txq->ift_tso_desc_tag, + txq->ift_sds.ifsd_tso_map[cidx]); + } else { + bus_dmamap_sync(txq->ift_desc_tag, + txq->ift_sds.ifsd_map[cidx], + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(txq->ift_desc_tag, + txq->ift_sds.ifsd_map[cidx]); + } /* XXX we don't support any drivers that batch packets yet */ MPASS(m->m_nextpkt == NULL); m_freem(m); @@ -3400,6 +3477,8 @@ iflib_txq_can_drain(struct ifmp_ring *r) iflib_txq_t txq = r->cookie; if_ctx_t ctx = txq->ift_ctx; + bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, + BUS_DMASYNC_POSTREAD); return ((TXQ_AVAIL(txq) > MAX_TX_DESC(ctx) + 2) || ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)); } @@ -3564,6 +3643,8 @@ _task_fn_tx(void *context) if (!(if_getdrvflags(ctx->ifc_ifp) & IFF_DRV_RUNNING)) return; if (if_getcapenable(ifp) & IFCAP_NETMAP) { + bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, + BUS_DMASYNC_POSTREAD); if (ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, false)) netmap_tx_irq(ifp, txq->ift_id); IFDI_TX_QUEUE_INTR_ENABLE(ctx, txq->ift_id); @@ -5817,6 +5898,8 @@ iflib_tx_credits_update(if_ctx_t ctx, iflib_txq_t txq) if (ctx->isc_txd_credits_update == NULL) return (0); + bus_dmamap_sync(txq->ift_ifdi->idi_tag, txq->ift_ifdi->idi_map, + BUS_DMASYNC_POSTREAD); if ((credits = ctx->isc_txd_credits_update(ctx->ifc_softc, txq->ift_id, true)) == 0) return (0);