From owner-svn-src-all@FreeBSD.ORG Fri Nov 30 23:45:56 2012 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id 4BB7CD41; Fri, 30 Nov 2012 23:45:56 +0000 (UTC) (envelope-from jfv@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id 32B738FC13; Fri, 30 Nov 2012 23:45:56 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id qAUNjuuP077697; Fri, 30 Nov 2012 23:45:56 GMT (envelope-from jfv@svn.freebsd.org) Received: (from jfv@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id qAUNjubV077695; Fri, 30 Nov 2012 23:45:56 GMT (envelope-from jfv@svn.freebsd.org) Message-Id: <201211302345.qAUNjubV077695@svn.freebsd.org> From: Jack F Vogel Date: Fri, 30 Nov 2012 23:45:55 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r243729 - head/sys/dev/ixgbe X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 30 Nov 2012 23:45:56 -0000 Author: jfv Date: Fri Nov 30 23:45:55 2012 New Revision: 243729 URL: http://svnweb.freebsd.org/changeset/base/243729 Log: Patch #8 Performance changes - this one improves locality, moving some counters and data to the ring struct from the adapter struct, also compressing some data in the move. Modified: head/sys/dev/ixgbe/ixgbe.c head/sys/dev/ixgbe/ixgbe.h Modified: head/sys/dev/ixgbe/ixgbe.c ============================================================================== --- head/sys/dev/ixgbe/ixgbe.c Fri Nov 30 23:28:01 2012 (r243728) +++ head/sys/dev/ixgbe/ixgbe.c Fri Nov 30 23:45:55 2012 (r243729) @@ -47,7 +47,7 @@ int ixgbe_display_debug_stat /********************************************************************* * Driver version *********************************************************************/ -char ixgbe_driver_version[] = "2.5.0 - 7"; +char ixgbe_driver_version[] = "2.5.0 - 8"; /********************************************************************* * PCI Device ID Table @@ -164,7 +164,7 @@ static int ixgbe_dma_malloc(struct adapt struct ixgbe_dma_alloc *, int); static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); static void ixgbe_add_process_limit(struct adapter *, const char *, - const char *, int *, int); + const char *, u16 *, u16); static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *); static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); @@ -1787,19 +1787,19 @@ ixgbe_xmit(struct tx_ring *txr, struct m *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); if (error == ENOMEM) { - adapter->no_tx_dma_setup++; + txr->no_tx_dma_setup++; return (error); } else if (error != 0) { - adapter->no_tx_dma_setup++; + txr->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); } } else if (error == ENOMEM) { - adapter->no_tx_dma_setup++; + txr->no_tx_dma_setup++; return (error); } else if (error != 0) { - adapter->no_tx_dma_setup++; + txr->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); @@ -1822,7 +1822,7 @@ ixgbe_xmit(struct tx_ring *txr, struct m cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; - ++adapter->tso_tx; + ++txr->tso_tx; } else return (ENXIO); } else if (ixgbe_tx_ctx_setup(txr, m_head)) @@ -1859,7 +1859,7 @@ ixgbe_xmit(struct tx_ring *txr, struct m txd->read.olinfo_status = htole32(olinfo_status); last = i; /* descriptor that will get completion IRQ */ - if (++i == adapter->num_tx_desc) + if (++i == txr->num_desc) i = 0; txbuf->m_head = NULL; @@ -2816,6 +2816,7 @@ ixgbe_allocate_queues(struct adapter *ad txr = &adapter->tx_rings[i]; txr->adapter = adapter; txr->me = i; + txr->num_desc = adapter->num_tx_desc; /* Initialize the TX side lock */ snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", @@ -2862,6 +2863,7 @@ ixgbe_allocate_queues(struct adapter *ad /* Set up some basics */ rxr->adapter = adapter; rxr->me = i; + rxr->num_desc = adapter->num_rx_desc; /* Initialize the RX side lock */ snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", @@ -3007,7 +3009,7 @@ ixgbe_setup_transmit_ring(struct tx_ring /* Free any existing tx buffers. */ txbuf = txr->tx_buffers; - for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { + for (i = 0; i < txr->num_desc; i++, txbuf++) { if (txbuf->m_head != NULL) { bus_dmamap_sync(txr->txtag, txbuf->map, BUS_DMASYNC_POSTWRITE); @@ -3226,7 +3228,6 @@ ixgbe_free_transmit_buffers(struct tx_ri static bool ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) { - struct adapter *adapter = txr->adapter; struct ixgbe_adv_tx_context_desc *TXD; struct ixgbe_tx_buf *tx_buffer; u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; @@ -3329,7 +3330,7 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, tx_buffer->eop_index = -1; /* We've consumed the first desc, adjust counters */ - if (++ctxd == adapter->num_tx_desc) + if (++ctxd == txr->num_desc) ctxd = 0; txr->next_avail_desc = ctxd; --txr->tx_avail; @@ -3347,7 +3348,6 @@ static bool ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen, u32 *olinfo_status) { - struct adapter *adapter = txr->adapter; struct ixgbe_adv_tx_context_desc *TXD; struct ixgbe_tx_buf *tx_buffer; u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; @@ -3450,7 +3450,7 @@ ixgbe_tso_setup(struct tx_ring *txr, str tx_buffer->m_head = NULL; tx_buffer->eop_index = -1; - if (++ctxd == adapter->num_tx_desc) + if (++ctxd == txr->num_desc) ctxd = 0; txr->tx_avail--; @@ -3595,7 +3595,7 @@ ixgbe_txeof(struct tx_ring *txr) } #endif /* DEV_NETMAP */ - if (txr->tx_avail == adapter->num_tx_desc) { + if (txr->tx_avail == txr->num_desc) { txr->queue_status = IXGBE_QUEUE_IDLE; return FALSE; } @@ -3683,7 +3683,7 @@ ixgbe_txeof(struct tx_ring *txr) if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG)) txr->queue_status = IXGBE_QUEUE_HUNG; - if (txr->tx_avail == adapter->num_tx_desc) { + if (txr->tx_avail == txr->num_desc) { txr->queue_status = IXGBE_QUEUE_IDLE; return (FALSE); } @@ -3712,20 +3712,20 @@ ixgbe_refresh_mbufs(struct rx_ring *rxr, i = j = rxr->next_to_refresh; /* Control the loop with one beyond */ - if (++j == adapter->num_rx_desc) + if (++j == rxr->num_desc) j = 0; while (j != limit) { rxbuf = &rxr->rx_buffers[i]; if (rxbuf->buf == NULL) { mp = m_getjcl(M_DONTWAIT, MT_DATA, - M_PKTHDR, adapter->rx_mbuf_sz); + M_PKTHDR, rxr->mbuf_sz); if (mp == NULL) goto update; } else mp = rxbuf->buf; - mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; + mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; /* If we're dealing with an mbuf that was copied rather * than replaced, there's no need to go through busdma. @@ -3755,7 +3755,7 @@ ixgbe_refresh_mbufs(struct rx_ring *rxr, /* Next is precalculated */ i = j; rxr->next_to_refresh = i; - if (++j == adapter->num_rx_desc) + if (++j == rxr->num_desc) j = 0; } update: @@ -3781,7 +3781,7 @@ ixgbe_allocate_receive_buffers(struct rx struct ixgbe_rx_buf *rxbuf; int i, bsize, error; - bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc; + bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc; if (!(rxr->rx_buffers = (struct ixgbe_rx_buf *) malloc(bsize, M_DEVBUF, M_NOWAIT | M_ZERO))) { @@ -3806,7 +3806,7 @@ ixgbe_allocate_receive_buffers(struct rx goto fail; } - for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { + for (i = 0; i < rxr->num_desc; i++, rxbuf++) { rxbuf = &rxr->rx_buffers[i]; error = bus_dmamap_create(rxr->tag, BUS_DMA_NOWAIT, &rxbuf->map); @@ -3875,11 +3875,11 @@ ixgbe_setup_hw_rsc(struct rx_ring *rxr) ** Limit the total number of descriptors that ** can be combined, so it does not exceed 64K */ - if (adapter->rx_mbuf_sz == MCLBYTES) + if (rxr->mbuf_sz == MCLBYTES) rscctrl |= IXGBE_RSCCTL_MAXDESC_16; - else if (adapter->rx_mbuf_sz == MJUMPAGESIZE) + else if (rxr->mbuf_sz == MJUMPAGESIZE) rscctrl |= IXGBE_RSCCTL_MAXDESC_8; - else if (adapter->rx_mbuf_sz == MJUM9BYTES) + else if (rxr->mbuf_sz == MJUM9BYTES) rscctrl |= IXGBE_RSCCTL_MAXDESC_4; else /* Using 16K cluster */ rscctrl |= IXGBE_RSCCTL_MAXDESC_1; @@ -3902,12 +3902,10 @@ ixgbe_setup_hw_rsc(struct rx_ring *rxr) static void ixgbe_free_receive_ring(struct rx_ring *rxr) { - struct adapter *adapter; struct ixgbe_rx_buf *rxbuf; int i; - adapter = rxr->adapter; - for (i = 0; i < adapter->num_rx_desc; i++) { + for (i = 0; i < rxr->num_desc; i++) { rxbuf = &rxr->rx_buffers[i]; if (rxbuf->buf != NULL) { bus_dmamap_sync(rxr->tag, rxbuf->map, @@ -3954,12 +3952,14 @@ ixgbe_setup_receive_ring(struct rx_ring rsize = roundup2(adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); bzero((void *)rxr->rx_base, rsize); + /* Cache the size */ + rxr->mbuf_sz = adapter->rx_mbuf_sz; /* Free current RX buffer structs and their mbufs */ ixgbe_free_receive_ring(rxr); /* Now replenish the mbufs */ - for (int j = 0; j != adapter->num_rx_desc; ++j) { + for (int j = 0; j != rxr->num_desc; ++j) { struct mbuf *mp; rxbuf = &rxr->rx_buffers[j]; @@ -3990,7 +3990,7 @@ ixgbe_setup_receive_ring(struct rx_ring goto fail; } mp = rxbuf->buf; - mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; + mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; /* Get the memory mapping */ error = bus_dmamap_load_mbuf_sg(rxr->tag, rxbuf->map, mp, seg, @@ -4359,7 +4359,7 @@ ixgbe_rxeof(struct ix_queue *que) struct lro_entry *queued; int i, nextp, processed = 0; u32 staterr = 0; - u32 count = rxr->process_limit; + u16 count = rxr->process_limit; union ixgbe_adv_rx_desc *cur; struct ixgbe_rx_buf *rbuf, *nbuf; @@ -4540,7 +4540,7 @@ next_desc: BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); /* Advance our pointers to the next descriptor. */ - if (++i == adapter->num_rx_desc) + if (++i == rxr->num_desc) i = 0; /* Now send to the stack or do LRO */ @@ -5286,15 +5286,9 @@ ixgbe_add_hw_stats(struct adapter *adapt SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed", CTLFLAG_RD, &adapter->mbuf_defrag_failed, "m_defrag() failed"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup", - CTLFLAG_RD, &adapter->no_tx_dma_setup, - "Driver tx dma failure in xmit"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events", CTLFLAG_RD, &adapter->watchdog_events, "Watchdog timeouts"); - SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx", - CTLFLAG_RD, &adapter->tso_tx, - "TSO"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", CTLFLAG_RD, &adapter->link_irq, "Link MSIX IRQ Handled"); @@ -5321,6 +5315,12 @@ ixgbe_add_hw_stats(struct adapter *adapt CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr), ixgbe_sysctl_tdt_handler, "IU", "Transmit Descriptor Tail"); + SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "tso_tx", + CTLFLAG_RD, &txr->tso_tx, + "TSO"); + SYSCTL_ADD_ULONG(ctx, queue_list, OID_AUTO, "no_tx_dma_setup", + CTLFLAG_RD, &txr->no_tx_dma_setup, + "Driver tx dma failure in xmit"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", CTLFLAG_RD, &txr->no_desc_avail, "Queue No Descriptor Available"); @@ -5552,7 +5552,7 @@ ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS) static void ixgbe_add_process_limit(struct adapter *adapter, const char *name, - const char *description, int *limit, int value) + const char *description, u16 *limit, u16 value) { *limit = value; SYSCTL_ADD_UINT(device_get_sysctl_ctx(adapter->dev), Modified: head/sys/dev/ixgbe/ixgbe.h ============================================================================== --- head/sys/dev/ixgbe/ixgbe.h Fri Nov 30 23:28:01 2012 (r243728) +++ head/sys/dev/ixgbe/ixgbe.h Fri Nov 30 23:45:55 2012 (r243729) @@ -291,19 +291,20 @@ struct tx_ring { struct adapter *adapter; struct mtx tx_mtx; u32 me; + int watchdog_time; + union ixgbe_adv_tx_desc *tx_base; + struct ixgbe_tx_buf *tx_buffers; + struct ixgbe_dma_alloc txdma; + volatile u16 tx_avail; + u16 next_avail_desc; + u16 next_to_clean; + u16 process_limit; + u16 num_desc; enum { IXGBE_QUEUE_IDLE, IXGBE_QUEUE_WORKING, IXGBE_QUEUE_HUNG, } queue_status; - u32 process_limit; - int watchdog_time; - union ixgbe_adv_tx_desc *tx_base; - struct ixgbe_dma_alloc txdma; - u32 next_avail_desc; - u32 next_to_clean; - struct ixgbe_tx_buf *tx_buffers; - volatile u16 tx_avail; u32 txd_cmd; bus_dma_tag_t txtag; char mtx_name[16]; @@ -318,6 +319,9 @@ struct tx_ring { u32 bytes; /* used for AIM */ u32 packets; /* Soft Stats */ + unsigned long tso_tx; + unsigned long no_tx_map_avail; + unsigned long no_tx_dma_setup; u64 no_desc_avail; u64 total_packets; }; @@ -337,9 +341,11 @@ struct rx_ring { bool hw_rsc; bool discard; bool vtag_strip; - u32 next_to_refresh; - u32 next_to_check; - u32 process_limit; + u16 next_to_refresh; + u16 next_to_check; + u16 num_desc; + u16 mbuf_sz; + u16 process_limit; char mtx_name[16]; struct ixgbe_rx_buf *rx_buffers; bus_dma_tag_t tag; @@ -437,15 +443,15 @@ struct adapter { * Allocated at run time, an array of rings. */ struct tx_ring *tx_rings; - int num_tx_desc; + u32 num_tx_desc; /* * Receive rings: * Allocated at run time, an array of rings. */ struct rx_ring *rx_rings; - int num_rx_desc; u64 que_mask; + u32 num_rx_desc; /* Multicast array memory */ u8 *mta; @@ -455,10 +461,7 @@ struct adapter { unsigned long mbuf_defrag_failed; unsigned long mbuf_header_failed; unsigned long mbuf_packet_failed; - unsigned long no_tx_map_avail; - unsigned long no_tx_dma_setup; unsigned long watchdog_events; - unsigned long tso_tx; unsigned long link_irq; struct ixgbe_hw_stats stats; @@ -522,12 +525,10 @@ drbr_needs_enqueue(struct ifnet *ifp, st static inline u16 ixgbe_rx_unrefreshed(struct rx_ring *rxr) { - struct adapter *adapter = rxr->adapter; - if (rxr->next_to_check > rxr->next_to_refresh) return (rxr->next_to_check - rxr->next_to_refresh - 1); else - return ((adapter->num_rx_desc + rxr->next_to_check) - + return ((rxr->num_desc + rxr->next_to_check) - rxr->next_to_refresh - 1); }