Date: Wed, 24 Jun 2009 18:27:07 +0000 (UTC) From: Jack F Vogel <jfv@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r194875 - in head/sys: dev/ixgbe modules/ixgbe Message-ID: <200906241827.n5OIR7iS023929@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jfv Date: Wed Jun 24 18:27:07 2009 New Revision: 194875 URL: http://svn.freebsd.org/changeset/base/194875 Log: Update for the Intel 10G driver, this adds support for newest hardware, adds multiqueue tx interface, infrastructure cleanup to allow up to 32 MSIX vectors on newer Nehalem systems. Bug fixes, etc. Modified: head/sys/dev/ixgbe/ixgbe.c head/sys/dev/ixgbe/ixgbe.h head/sys/dev/ixgbe/ixgbe_82598.c head/sys/dev/ixgbe/ixgbe_82599.c head/sys/dev/ixgbe/ixgbe_api.c head/sys/dev/ixgbe/ixgbe_api.h head/sys/dev/ixgbe/ixgbe_common.c head/sys/dev/ixgbe/ixgbe_osdep.h head/sys/dev/ixgbe/ixgbe_phy.c head/sys/dev/ixgbe/ixgbe_phy.h head/sys/dev/ixgbe/ixgbe_type.h head/sys/modules/ixgbe/Makefile Modified: head/sys/dev/ixgbe/ixgbe.c ============================================================================== --- head/sys/dev/ixgbe/ixgbe.c Wed Jun 24 18:25:41 2009 (r194874) +++ head/sys/dev/ixgbe/ixgbe.c Wed Jun 24 18:27:07 2009 (r194875) @@ -46,7 +46,7 @@ int ixgbe_display_debug_stat /********************************************************************* * Driver version *********************************************************************/ -char ixgbe_driver_version[] = "1.7.4"; +char ixgbe_driver_version[] = "1.8.7"; /********************************************************************* * PCI Device ID Table @@ -73,6 +73,7 @@ static ixgbe_vendor_info_t ixgbe_vendor_ {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0}, + {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0}, /* required last entry */ {0, 0, 0, 0, 0} }; @@ -94,6 +95,12 @@ static int ixgbe_detach(device_t); static int ixgbe_shutdown(device_t); static void ixgbe_start(struct ifnet *); static void ixgbe_start_locked(struct tx_ring *, struct ifnet *); +#if __FreeBSD_version >= 800000 +static int ixgbe_mq_start(struct ifnet *, struct mbuf *); +static int ixgbe_mq_start_locked(struct ifnet *, + struct tx_ring *, struct mbuf *); +static void ixgbe_qflush(struct ifnet *); +#endif static int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); static void ixgbe_watchdog(struct adapter *); static void ixgbe_init(void *); @@ -106,9 +113,7 @@ static int ixgbe_allocate_pci_resou static int ixgbe_allocate_msix(struct adapter *); static int ixgbe_allocate_legacy(struct adapter *); static int ixgbe_allocate_queues(struct adapter *); -#if __FreeBSD_version >= 602105 static int ixgbe_setup_msix(struct adapter *); -#endif static void ixgbe_free_pci_resources(struct adapter *); static void ixgbe_local_timer(void *); static int ixgbe_hardware_init(struct adapter *); @@ -151,22 +156,15 @@ static int ixgbe_dma_malloc(struct adapt static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); static void ixgbe_add_rx_process_limit(struct adapter *, const char *, const char *, int *, int); -static int ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); -static boolean_t ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); -static void ixgbe_set_ivar(struct adapter *, u16, u8, s8); +static bool ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); +static bool ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); +static void ixgbe_set_ivar(struct adapter *, u8, u8, s8); static void ixgbe_configure_ivars(struct adapter *); static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); -#ifdef IXGBE_HW_VLAN_SUPPORT +static void ixgbe_setup_vlan_hw_support(struct adapter *); static void ixgbe_register_vlan(void *, struct ifnet *, u16); static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); -#endif - -#ifdef IXGBE_TIMESYNC -/* Precision Time sync support */ -static int ixgbe_tsync_init(struct adapter *); -static void ixgbe_tsync_disable(struct adapter *); -#endif static void ixgbe_update_aim(struct rx_ring *); @@ -176,12 +174,10 @@ static bool ixgbe_sfp_probe(struct adapt /* Legacy (single vector interrupt handler */ static void ixgbe_legacy_irq(void *); -#if __FreeBSD_version >= 602105 /* The MSI/X Interrupt handlers */ static void ixgbe_msix_tx(void *); static void ixgbe_msix_rx(void *); static void ixgbe_msix_link(void *); -#endif /* Deferred interrupt tasklets */ static void ixgbe_handle_tx(void *, int); @@ -238,45 +234,25 @@ TUNABLE_INT("hw.ixgbe.bulk_latency", &ix static int ixgbe_rx_process_limit = 100; TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); -/* Flow control setting, default to off */ +/* Flow control setting, default to full */ static int ixgbe_flow_control = ixgbe_fc_full; TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); /* - * Should the driver do LRO on the RX end - * this can be toggled on the fly, but the - * interface must be reset (down/up) for it - * to take effect. - */ -static int ixgbe_enable_lro = 1; -TUNABLE_INT("hw.ixgbe.enable_lro", &ixgbe_enable_lro); - -/* * MSIX should be the default for best performance, * but this allows it to be forced off for testing. */ -#if __FreeBSD_version >= 602105 static int ixgbe_enable_msix = 1; -#else -static int ixgbe_enable_msix = 0; -#endif TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); /* - * Enable RX Header Split - * WARNING: disable this if bridging or forwarding!! + * Number of Queues, should normally + * be left at 0, it then autoconfigures to + * the number of cpus. Each queue is a pair + * of RX and TX rings with a dedicated interrupt */ -static int ixgbe_rx_hdr_split = 1; -TUNABLE_INT("hw.ixgbe.rx_hdr_split", &ixgbe_rx_hdr_split); - -/* - * Number of TX/RX Queues, with 0 setting - * it autoconfigures to the number of cpus. - */ -static int ixgbe_tx_queues = 1; -TUNABLE_INT("hw.ixgbe.tx_queues", &ixgbe_tx_queues); -static int ixgbe_rx_queues = 1; -TUNABLE_INT("hw.ixgbe.rx_queues", &ixgbe_rx_queues); +static int ixgbe_num_queues = 0; +TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues); /* Number of TX descriptors per ring */ static int ixgbe_txd = DEFAULT_TXD; @@ -290,6 +266,13 @@ TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); static int ixgbe_total_ports; /* +** Shadow VFTA table, this is needed because +** the real filter table gets cleared during +** a soft reset and we need to repopulate it. +*/ +static u32 ixgbe_shadow_vfta[IXGBE_VFTA_SIZE]; + +/* ** The number of scatter-gather segments ** differs for 82598 and 82599, default to ** the former. @@ -400,6 +383,9 @@ ixgbe_attach(device_t dev) case IXGBE_DEV_ID_82599_KX4 : adapter->optics = IFM_10G_CX4; ixgbe_num_segs = IXGBE_82599_SCATTER; + break; + case IXGBE_DEV_ID_82599_XAUI_LOM : + ixgbe_num_segs = IXGBE_82599_SCATTER; default: break; } @@ -422,11 +408,6 @@ ixgbe_attach(device_t dev) SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW, - &ixgbe_enable_lro, 1, "Large Receive Offload"); - - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, &ixgbe_enable_aim, 1, "Interrupt Moderation"); @@ -445,11 +426,6 @@ ixgbe_attach(device_t dev) OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW, &ixgbe_bulk_latency, 1, "Bulk Latency"); - SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), - SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), - OID_AUTO, "hdr_split", CTLTYPE_INT|CTLFLAG_RW, - &ixgbe_rx_hdr_split, 1, "RX Header Split"); - /* Set up the timer callout */ callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); @@ -476,10 +452,10 @@ ixgbe_attach(device_t dev) ** system mbuf allocation. Tuning nmbclusters ** can alleviate this. */ - if ((adapter->num_rx_queues > 1) && (nmbclusters > 0 )){ + if ((adapter->num_queues > 1) && (nmbclusters > 0 )){ int s; /* Calculate the total RX mbuf needs */ - s = (ixgbe_rxd * adapter->num_rx_queues) * ixgbe_total_ports; + s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports; if (s > nmbclusters) { device_printf(dev, "RX Descriptors exceed " "system mbuf max, using default instead!\n"); @@ -537,6 +513,22 @@ ixgbe_attach(device_t dev) /* Setup OS specific network interface */ ixgbe_setup_interface(dev, adapter); +#ifdef IXGBE_IEEE1588 + /* + ** Setup the timer: IEEE 1588 support + */ + adapter->cycles.read = ixgbe_read_clock; + adapter->cycles.mask = (u64)-1; + adapter->cycles.mult = 1; + adapter->cycles.shift = IXGBE_TSYNC_SHIFT; + IXGBE_WRITE_REG(&adapter->hw, IXGBE_TIMINCA, (1<<24) | + IXGBE_TSYNC_CYCLE_TIME * IXGBE_TSYNC_SHIFT); + IXGBE_WRITE_REG(&adapter->hw, IXGBE_SYSTIML, 0x00000000); + IXGBE_WRITE_REG(&adapter->hw, IXGBE_SYSTIMH, 0xFF800000); + + // JFV - this is not complete yet +#endif + /* Sysctl for limiting the amount of work done in the taskqueue */ ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", "max number of rx packets to process", &adapter->rx_process_limit, @@ -545,13 +537,11 @@ ixgbe_attach(device_t dev) /* Initialize statistics */ ixgbe_update_stats_counters(adapter); -#ifdef IXGBE_HW_VLAN_SUPPORT /* Register for VLAN events */ adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, ixgbe_register_vlan, 0, EVENTHANDLER_PRI_FIRST); adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, ixgbe_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); -#endif /* let hardware know driver is loaded */ ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); @@ -590,11 +580,7 @@ ixgbe_detach(device_t dev) INIT_DEBUGOUT("ixgbe_detach: begin"); /* Make sure VLANS are not using driver */ -#if __FreeBSD_version >= 700000 if (adapter->ifp->if_vlantrunk != NULL) { -#else - if (adapter->ifp->if_nvlans != 0) { -#endif device_printf(dev,"Vlan in use, detach first\n"); return (EBUSY); } @@ -603,14 +589,14 @@ ixgbe_detach(device_t dev) ixgbe_stop(adapter); IXGBE_CORE_UNLOCK(adapter); - for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { + for (int i = 0; i < adapter->num_queues; i++, txr++) { if (txr->tq) { taskqueue_drain(txr->tq, &txr->tx_task); taskqueue_free(txr->tq); } } - for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { + for (int i = 0; i < adapter->num_queues; i++, rxr++) { if (rxr->tq) { taskqueue_drain(rxr->tq, &rxr->rx_task); taskqueue_free(rxr->tq); @@ -630,13 +616,11 @@ ixgbe_detach(device_t dev) ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); -#ifdef IXGBE_HW_VLAN_SUPPORT /* Unregister VLAN events */ if (adapter->vlan_attach != NULL) EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); if (adapter->vlan_detach != NULL) EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); -#endif ether_ifdetach(adapter->ifp); callout_drain(&adapter->timer); @@ -697,19 +681,6 @@ ixgbe_start_locked(struct tx_ring *txr, IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); if (m_head == NULL) break; - /* - * Force a cleanup if number of TX descriptors - * available is below the threshold. If it fails - * to get above, then abort transmit. - */ - if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) { - ixgbe_txeof(txr); - /* Make sure things have improved */ - if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) { - txr->no_tx_desc_avail++; - break; - } - } if (ixgbe_xmit(txr, &m_head)) { if (m_head == NULL) @@ -728,35 +699,127 @@ ixgbe_start_locked(struct tx_ring *txr, return; } - +/* + * Legacy TX start - called by the stack, this + * always uses the first tx ring, and should + * not be used with multiqueue tx enabled. + */ static void ixgbe_start(struct ifnet *ifp) { struct adapter *adapter = ifp->if_softc; struct tx_ring *txr = adapter->tx_rings; - u32 queue = 0; - - /* - ** This is really just here for testing - ** TX multiqueue, ultimately what is - ** needed is the flow support in the stack - ** and appropriate logic here to deal with - ** it. -jfv - */ - if (adapter->num_tx_queues > 1) - queue = (curcpu % adapter->num_tx_queues); - - txr = &adapter->tx_rings[queue]; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - if (IXGBE_TX_TRYLOCK(txr) == 0) - return; + IXGBE_TX_LOCK(txr); ixgbe_start_locked(txr, ifp); IXGBE_TX_UNLOCK(txr); } return; } +#if __FreeBSD_version >= 800000 +/* +** Multiqueue Transmit driver +** +*/ +static int +ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) +{ + struct adapter *adapter = ifp->if_softc; + struct tx_ring *txr; + int i = 0, err = 0; + + /* Which queue to use */ + if ((m->m_flags & M_FLOWID) != 0) + i = m->m_pkthdr.flowid % adapter->num_queues; + txr = &adapter->tx_rings[i]; + + if (IXGBE_TX_TRYLOCK(txr)) { + err = ixgbe_mq_start_locked(ifp, txr, m); + IXGBE_TX_UNLOCK(txr); + } else + err = drbr_enqueue(ifp, txr->br, m); + + return (err); +} + +static int +ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) +{ + struct adapter *adapter = txr->adapter; + struct mbuf *next; + int err = 0; + + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { + err = drbr_enqueue(ifp, txr->br, m); + return (err); + } + + if (m == NULL) /* Called by tasklet */ + goto process; + + /* If nothing queued go right to xmit */ + if (drbr_empty(ifp, txr->br)) { + if (ixgbe_xmit(txr, &m)) { + if (m && (err = drbr_enqueue(ifp, txr->br, m)) != 0) + return (err); + } else { + /* Success, update stats */ + drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags); + /* Send a copy of the frame to the BPF listener */ + ETHER_BPF_MTAP(ifp, m); + /* Set the watchdog */ + txr->watchdog_timer = IXGBE_TX_TIMEOUT; + } + + } else if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) + return (err); + +process: + if (drbr_empty(ifp, txr->br)) + return (err); + + /* Process the queue */ + while (TRUE) { + if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) + break; + next = drbr_dequeue(ifp, txr->br); + if (next == NULL) + break; + if (ixgbe_xmit(txr, &next)) + break; + ETHER_BPF_MTAP(ifp, next); + /* Set the watchdog */ + txr->watchdog_timer = IXGBE_TX_TIMEOUT; + } + + if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) + ifp->if_drv_flags |= IFF_DRV_OACTIVE; + + return (err); +} + +/* +** Flush all ring buffers +*/ +static void +ixgbe_qflush(struct ifnet *ifp) +{ + struct adapter *adapter = ifp->if_softc; + struct tx_ring *txr = adapter->tx_rings; + struct mbuf *m; + + for (int i = 0; i < adapter->num_queues; i++, txr++) { + IXGBE_TX_LOCK(txr); + while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) + m_freem(m); + IXGBE_TX_UNLOCK(txr); + } + if_qflush(ifp); +} +#endif /* __FreeBSD_version >= 800000 */ + /********************************************************************* * Ioctl entry point * @@ -769,13 +832,16 @@ ixgbe_start(struct ifnet *ifp) static int ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) { - int error = 0; + struct adapter *adapter = ifp->if_softc; struct ifreq *ifr = (struct ifreq *) data; +#ifdef INET struct ifaddr *ifa = (struct ifaddr *) data; - struct adapter *adapter = ifp->if_softc; +#endif + int error = 0; switch (command) { case SIOCSIFADDR: +#ifdef INET IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); if (ifa->ifa_addr->sa_family == AF_INET) { ifp->if_flags |= IFF_UP; @@ -786,6 +852,7 @@ ixgbe_ioctl(struct ifnet * ifp, u_long c } arp_ifinit(ifp, ifa); } else +#endif ether_ioctl(ifp, command, data); break; case SIOCSIFMTU: @@ -843,75 +910,24 @@ ixgbe_ioctl(struct ifnet * ifp, u_long c ifp->if_capenable ^= IFCAP_HWCSUM; if (mask & IFCAP_TSO4) ifp->if_capenable ^= IFCAP_TSO4; + if (mask & IFCAP_LRO) + ifp->if_capenable ^= IFCAP_LRO; if (mask & IFCAP_VLAN_HWTAGGING) ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; if (ifp->if_drv_flags & IFF_DRV_RUNNING) ixgbe_init(adapter); -#if __FreeBSD_version >= 700000 VLAN_CAPABILITIES(ifp); -#endif break; } -#ifdef IXGBE_TIMESYNC + +#ifdef IXGBE_IEEE1588 /* ** IOCTL support for Precision Time (IEEE 1588) Support */ - case IXGBE_TIMESYNC_READTS: - { - u32 rx_ctl, tx_ctl; - struct ixgbe_tsync_read *tdata; - - tdata = (struct ixgbe_tsync_read *) ifr->ifr_data; - - if (tdata->read_current_time) { - getnanotime(&tdata->system_time); - tdata->network_time = IXGBE_READ_REG(&adapter->hw, - IXGBE_SYSTIML); - tdata->network_time |= - (u64)IXGBE_READ_REG(&adapter->hw, - IXGBE_SYSTIMH ) << 32; - } - - rx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCRXCTL); - tx_ctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TSYNCTXCTL); - - if (rx_ctl & 0x1) { - u32 tmp; - unsigned char *tmp_cp; - - tdata->rx_valid = 1; - tdata->rx_stamp = IXGBE_READ_REG(&adapter->hw, - IXGBE_RXSTMPL); - tdata->rx_stamp |= (u64)IXGBE_READ_REG(&adapter->hw, - IXGBE_RXSTMPH) << 32; - - tmp = IXGBE_READ_REG(&adapter->hw, IXGBE_RXSATRL); - tmp_cp = (unsigned char *) &tmp; - tdata->srcid[0] = tmp_cp[0]; - tdata->srcid[1] = tmp_cp[1]; - tdata->srcid[2] = tmp_cp[2]; - tdata->srcid[3] = tmp_cp[3]; - tmp = IXGBE_READ_REG(&adapter->hw, IXGBE_RXSATRH); - tmp_cp = (unsigned char *) &tmp; - tdata->srcid[4] = tmp_cp[0]; - tdata->srcid[5] = tmp_cp[1]; - tdata->seqid = tmp >> 16; - tdata->seqid = htons(tdata->seqid); - } else - tdata->rx_valid = 0; - - if (tx_ctl & 0x1) { - tdata->tx_valid = 1; - tdata->tx_stamp = IXGBE_READ_REG(&adapter->hw, - IXGBE_TXSTMPL); - tdata->tx_stamp |= (u64) IXGBE_READ_REG(&adapter->hw, - IXGBE_TXSTMPH) << 32; - } else - tdata->tx_valid = 0; - - return (0); - } -#endif /* IXGBE_TIMESYNC */ + case SIOCSHWTSTAMP: + error = ixgbe_hwtstamp_ioctl(adapter, ifp); + break; +#endif default: IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); @@ -947,7 +963,7 @@ ixgbe_watchdog(struct adapter *adapter) * Finally, anytime all descriptors are clean the timer is * set to 0. */ - for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { + for (int i = 0; i < adapter->num_queues; i++, txr++) { u32 head, tail; IXGBE_TX_LOCK(txr); @@ -976,7 +992,7 @@ ixgbe_watchdog(struct adapter *adapter) */ if (IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) { txr = adapter->tx_rings; /* reset pointer */ - for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { + for (int i = 0; i < adapter->num_queues; i++, txr++) { IXGBE_TX_LOCK(txr); txr->watchdog_timer = IXGBE_TX_TIMEOUT; IXGBE_TX_UNLOCK(txr); @@ -986,7 +1002,7 @@ ixgbe_watchdog(struct adapter *adapter) device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); - for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { + for (int i = 0; i < adapter->num_queues; i++, txr++) { device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", i, IXGBE_READ_REG(hw, IXGBE_TDH(i)), IXGBE_READ_REG(hw, IXGBE_TDT(i))); @@ -1041,16 +1057,6 @@ ixgbe_init_locked(struct adapter *adapte return; } -#ifndef IXGBE_HW_VLAN_SUPPORT - if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { - u32 ctrl; - - ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); - ctrl |= IXGBE_VLNCTRL_VME; - ctrl &= ~IXGBE_VLNCTRL_CFIEN; - IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); - } -#endif /* Prepare transmit descriptors and buffers */ if (ixgbe_setup_transmit_structures(adapter)) { device_printf(dev,"Could not setup transmit structures\n"); @@ -1096,7 +1102,7 @@ ixgbe_init_locked(struct adapter *adapte if (hw->device_id == IXGBE_DEV_ID_82598AT) gpie |= IXGBE_SDP1_GPIEN; - if (adapter->msix > 2) { + if (adapter->msix > 1) { /* Enable Enhanced MSIX mode */ gpie |= IXGBE_GPIE_MSIX_MODE; gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | @@ -1108,7 +1114,7 @@ ixgbe_init_locked(struct adapter *adapte ifp->if_hwassist = 0; if (ifp->if_capenable & IFCAP_TSO4) ifp->if_hwassist |= CSUM_TSO; - else if (ifp->if_capenable & IFCAP_TXCSUM) + if (ifp->if_capenable & IFCAP_TXCSUM) ifp->if_hwassist = (CSUM_TCP | CSUM_UDP); /* Set MTU size */ @@ -1121,7 +1127,7 @@ ixgbe_init_locked(struct adapter *adapte /* Now enable all the queues */ - for (int i = 0; i < adapter->num_tx_queues; i++) { + for (int i = 0; i < adapter->num_queues; i++) { txdctl = IXGBE_READ_REG(&adapter->hw, IXGBE_TXDCTL(i)); txdctl |= IXGBE_TXDCTL_ENABLE; /* Set WTHRESH to 8, burst writeback */ @@ -1129,7 +1135,7 @@ ixgbe_init_locked(struct adapter *adapte IXGBE_WRITE_REG(&adapter->hw, IXGBE_TXDCTL(i), txdctl); } - for (int i = 0; i < adapter->num_rx_queues; i++) { + for (int i = 0; i < adapter->num_queues; i++) { rxdctl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXDCTL(i)); /* PTHRESH set to 32 */ rxdctl |= 0x0020; @@ -1146,6 +1152,9 @@ ixgbe_init_locked(struct adapter *adapte IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); } + /* Set up VLAN offloads and filter */ + ixgbe_setup_vlan_hw_support(adapter); + /* Enable Receive engine */ rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); if (adapter->hw.mac.type == ixgbe_mac_82598EB) @@ -1185,13 +1194,6 @@ ixgbe_init_locked(struct adapter *adapte } else taskqueue_enqueue(adapter->tq, &adapter->link_task); - -#ifdef IXGBE_TIMESYNC - /* Initialize IEEE 1588 support */ - if (adapter->hw.mac.type == ixgbe_mac_82599EB) - ixgbe_tsync_init(adapter); -#endif - /* Now inform the stack we're ready */ ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; @@ -1212,9 +1214,67 @@ ixgbe_init(void *arg) /* -** MSIX Interrupt Tasklets +** +** MSIX Interrupt Handlers and Tasklets +** */ +static inline void +ixgbe_enable_queue(struct adapter *adapter, u32 vector) +{ + struct ixgbe_hw *hw = &adapter->hw; + u64 queue = (u64)(1 << vector); + u32 mask; + + if (hw->mac.type == ixgbe_mac_82598EB) { + mask = (IXGBE_EIMS_RTX_QUEUE & queue); + IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); + } else { + mask = (queue & 0xFFFFFFFF); + if (mask) + IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask); + mask = (queue >> 32); + if (mask) + IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask); + } +} + +static inline void +ixgbe_disable_queue(struct adapter *adapter, u32 vector) +{ + struct ixgbe_hw *hw = &adapter->hw; + u64 queue = (u64)(1 << vector); + u32 mask; + + if (hw->mac.type == ixgbe_mac_82598EB) { + mask = (IXGBE_EIMS_RTX_QUEUE & queue); + IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask); + } else { + mask = (queue & 0xFFFFFFFF); + if (mask) + IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask); + mask = (queue >> 32); + if (mask) + IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask); + } +} + +static inline void +ixgbe_rearm_rx_queues(struct adapter *adapter, u64 queues) +{ + u32 mask; + + if (adapter->hw.mac.type == ixgbe_mac_82598EB) { + mask = (IXGBE_EIMS_RTX_QUEUE & queues); + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask); + } else { + mask = (queues & 0xFFFFFFFF); + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask); + mask = (queues >> 32); + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask); + } +} + static void ixgbe_handle_rx(void *context, int pending) { @@ -1227,7 +1287,7 @@ ixgbe_handle_rx(void *context, int pendi more = ixgbe_rxeof(rxr, -1); } while (loop-- && more); /* Reenable this interrupt */ - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims); + ixgbe_enable_queue(adapter, rxr->msix); } static void @@ -1244,13 +1304,19 @@ ixgbe_handle_tx(void *context, int pendi more = ixgbe_txeof(txr); } while (loop-- && more); - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - ixgbe_start_locked(txr, ifp); + if (ifp->if_drv_flags & IFF_DRV_RUNNING) { +#if __FreeBSD_version >= 800000 + if (!drbr_empty(ifp, txr->br)) + ixgbe_mq_start_locked(ifp, txr, NULL); +#else + if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + ixgbe_start_locked(txr, ifp); +#endif + } IXGBE_TX_UNLOCK(txr); - /* Reenable this interrupt */ - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims); + ixgbe_enable_queue(adapter, txr->msix); } @@ -1315,7 +1381,6 @@ ixgbe_legacy_irq(void *arg) } -#if __FreeBSD_version >= 602105 /********************************************************************* * * MSI TX Interrupt Service routine @@ -1328,7 +1393,7 @@ ixgbe_msix_tx(void *arg) struct adapter *adapter = txr->adapter; bool more; - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, txr->eims); + ixgbe_disable_queue(adapter, txr->msix); IXGBE_TX_LOCK(txr); ++txr->tx_irq; @@ -1337,7 +1402,7 @@ ixgbe_msix_tx(void *arg) if (more) taskqueue_enqueue(txr->tq, &txr->tx_task); else /* Reenable this interrupt */ - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims); + ixgbe_enable_queue(adapter, txr->msix); return; } @@ -1354,7 +1419,7 @@ ixgbe_msix_rx(void *arg) struct adapter *adapter = rxr->adapter; bool more; - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, rxr->eims); + ixgbe_disable_queue(adapter, rxr->msix); ++rxr->rx_irq; more = ixgbe_rxeof(rxr, adapter->rx_process_limit); @@ -1366,7 +1431,7 @@ ixgbe_msix_rx(void *arg) if (more) taskqueue_enqueue(rxr->tq, &rxr->rx_task); else - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims); + ixgbe_enable_queue(adapter, rxr->msix); return; } @@ -1381,9 +1446,8 @@ ixgbe_msix_link(void *arg) ++adapter->link_irq; /* First get the cause */ - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, IXGBE_EIMS_OTHER); reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); - /* Clear with write */ + /* Clear interrupt with write */ IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); /* Link status change */ @@ -1418,7 +1482,6 @@ ixgbe_msix_link(void *arg) IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); return; } -#endif /* FreeBSD_version >= 602105 */ /* ** Routine to do adjust the RX EITR value based on traffic, @@ -1482,14 +1545,14 @@ ixgbe_init_moderation(struct adapter *ad } /* TX irq moderation rate is fixed */ - for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { + for (int i = 0; i < adapter->num_queues; i++, txr++) { IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(txr->msix), ixgbe_ave_latency); txr->watchdog_timer = FALSE; } /* RX moderation will be adapted over time, set default */ - for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { + for (int i = 0; i < adapter->num_queues; i++, rxr++) { IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(rxr->msix), ixgbe_low_latency); } @@ -1590,7 +1653,7 @@ ixgbe_xmit(struct tx_ring *txr, struct m u32 olinfo_status = 0, cmd_type_len; u32 paylen = 0; int i, j, error, nsegs; - int first, last = 0, offload = 0; + int first, last = 0; struct mbuf *m_head; bus_dma_segment_t segs[ixgbe_num_segs]; bus_dmamap_t map; @@ -1606,6 +1669,14 @@ ixgbe_xmit(struct tx_ring *txr, struct m if (m_head->m_flags & M_VLANTAG) cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; + /* Do a clean if descriptors are low */ + if (txr->tx_avail <= IXGBE_TX_CLEANUP_THRESHOLD) { + ixgbe_txeof(txr); + /* Now do we at least have a minimal? */ + if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) + return (ENOBUFS); + } + /* * Important to capture the first descriptor * used because it will contain the index of @@ -1679,14 +1750,15 @@ ixgbe_xmit(struct tx_ring *txr, struct m ++adapter->tso_tx; } else return (ENXIO); - } else /* Offloads other than TSO */ - offload = ixgbe_tx_ctx_setup(txr, m_head); - if (offload == TRUE) + } else if (ixgbe_tx_ctx_setup(txr, m_head)) olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; -#ifdef IXGBE_TIMESYNC - if (offload == IXGBE_TIMESTAMP) - cmd_type_len |= IXGBE_ADVTXD_TSTAMP; + +#ifdef IXGBE_IEEE1588 + /* This is changing soon to an mtag detection */ + if (we detect this mbuf has a TSTAMP mtag) + cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP; #endif + /* Record payload length */ if (paylen == 0) olinfo_status |= m_head->m_pkthdr.len << @@ -1881,7 +1953,7 @@ ixgbe_local_timer(void *arg) out: /* Trigger an RX interrupt on all queues */ - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, adapter->rx_mask); + ixgbe_rearm_rx_queues(adapter, adapter->rx_mask); callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); } @@ -1914,7 +1986,7 @@ ixgbe_update_link_status(struct adapter device_printf(dev,"Link is Down\n"); if_link_state_change(ifp, LINK_STATE_DOWN); adapter->link_active = FALSE; - for (int i = 0; i < adapter->num_tx_queues; + for (int i = 0; i < adapter->num_queues; i++, txr++) txr->watchdog_timer = FALSE; } @@ -1946,11 +2018,6 @@ ixgbe_stop(void *arg) /* Tell the stack that the interface is no longer active */ ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); -#ifdef IXGBE_TIMESYNC - /* Disable IEEE 1588 support */ - if (adapter->hw.mac.type == ixgbe_mac_82599EB) - ixgbe_tsync_disable(adapter); -#endif ixgbe_reset_hw(&adapter->hw); adapter->hw.adapter_stopped = FALSE; ixgbe_stop_adapter(&adapter->hw); @@ -1996,16 +2063,16 @@ ixgbe_allocate_legacy(struct adapter *ad device_t dev = adapter->dev; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; - int error; + int error, rid = 0; - /* Legacy RID at 0 */ - if (adapter->msix == 0) - adapter->rid[0] = 0; + /* MSI RID at 1 */ + if (adapter->msix == 1) + rid = 1; /* We allocate a single interrupt resource */ - adapter->res[0] = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); - if (adapter->res[0] == NULL) { + adapter->res = bus_alloc_resource_any(dev, + SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); + if (adapter->res == NULL) { device_printf(dev, "Unable to allocate bus resource: " "interrupt\n"); return (ENXIO); @@ -2026,13 +2093,18 @@ ixgbe_allocate_legacy(struct adapter *ad taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", device_get_nameunit(adapter->dev)); - if ((error = bus_setup_intr(dev, adapter->res[0], -#if __FreeBSD_version >= 700000 + /* Tasklets for Link, SFP and Multispeed Fiber */ + TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter); + TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter); + TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter); + adapter->tq = taskqueue_create_fast("ixgbe_link", M_NOWAIT, + taskqueue_thread_enqueue, &adapter->tq); + taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq", + device_get_nameunit(adapter->dev)); + + if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixgbe_legacy_irq, -#else - INTR_TYPE_NET | INTR_MPSAFE, ixgbe_legacy_irq, -#endif - adapter, &adapter->tag[0])) != 0) { + adapter, &adapter->tag)) != 0) { device_printf(dev, "Failed to register fast interrupt " "handler: %d\n", error); taskqueue_free(txr->tq); @@ -2046,7 +2118,6 @@ ixgbe_allocate_legacy(struct adapter *ad } -#if __FreeBSD_version >= 602105 /********************************************************************* * * Setup MSIX Interrupt resources and handlers @@ -2058,33 +2129,36 @@ ixgbe_allocate_msix(struct adapter *adap device_t dev = adapter->dev; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; - int error, vector = 0; + int error, rid, vector = 0; /* TX setup: the code is here for multi tx, there are other parts of the driver not ready for it */ - for (int i = 0; i < adapter->num_tx_queues; i++, vector++, txr++) { - adapter->res[vector] = bus_alloc_resource_any(dev, - SYS_RES_IRQ, &adapter->rid[vector], + for (int i = 0; i < adapter->num_queues; i++, vector++, txr++) { + rid = vector + 1; + txr->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); - if (!adapter->res[vector]) { + if (!txr->res) { device_printf(dev,"Unable to allocate" " bus resource: tx interrupt [%d]\n", vector); return (ENXIO); } /* Set the handler function */ - error = bus_setup_intr(dev, adapter->res[vector], - INTR_TYPE_NET | INTR_MPSAFE, -#if __FreeBSD_version > 700000 - NULL, -#endif - ixgbe_msix_tx, txr, &adapter->tag[vector]); + error = bus_setup_intr(dev, txr->res, *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200906241827.n5OIR7iS023929>