From owner-svn-src-all@FreeBSD.ORG Wed Nov 26 23:41:18 2008 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id C211B1065670; Wed, 26 Nov 2008 23:41:18 +0000 (UTC) (envelope-from jfv@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id A1F388FC0C; Wed, 26 Nov 2008 23:41:18 +0000 (UTC) (envelope-from jfv@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id mAQNfIqX069727; Wed, 26 Nov 2008 23:41:18 GMT (envelope-from jfv@svn.freebsd.org) Received: (from jfv@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id mAQNfIID069725; Wed, 26 Nov 2008 23:41:18 GMT (envelope-from jfv@svn.freebsd.org) Message-Id: <200811262341.mAQNfIID069725@svn.freebsd.org> From: Jack F Vogel Date: Wed, 26 Nov 2008 23:41:18 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r185352 - head/sys/dev/ixgbe X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 26 Nov 2008 23:41:18 -0000 Author: jfv Date: Wed Nov 26 23:41:18 2008 New Revision: 185352 URL: http://svn.freebsd.org/changeset/base/185352 Log: Updated ixgbe driver - version 1.6.2 -This version has header split, and as a result a number of aspects of the code have been improved/simplified. - Interrupt handling refined for performance - Many small bugs fixed along the way MFC after: ASAP - in time for 7.1 Modified: head/sys/dev/ixgbe/ixgbe.c head/sys/dev/ixgbe/ixgbe.h head/sys/dev/ixgbe/ixgbe_82598.c head/sys/dev/ixgbe/ixgbe_api.c head/sys/dev/ixgbe/ixgbe_api.h head/sys/dev/ixgbe/ixgbe_common.c head/sys/dev/ixgbe/ixgbe_common.h head/sys/dev/ixgbe/ixgbe_osdep.h head/sys/dev/ixgbe/ixgbe_phy.c head/sys/dev/ixgbe/ixgbe_phy.h head/sys/dev/ixgbe/ixgbe_type.h Modified: head/sys/dev/ixgbe/ixgbe.c ============================================================================== --- head/sys/dev/ixgbe/ixgbe.c Wed Nov 26 23:31:42 2008 (r185351) +++ head/sys/dev/ixgbe/ixgbe.c Wed Nov 26 23:41:18 2008 (r185352) @@ -36,9 +36,6 @@ #include "opt_device_polling.h" #endif -/* Undefine this if not using CURRENT */ -#define IXGBE_VLAN_EVENTS - #include "ixgbe.h" /********************************************************************* @@ -49,7 +46,7 @@ int ixgbe_display_debug_stat /********************************************************************* * Driver version *********************************************************************/ -char ixgbe_driver_version[] = "1.4.7"; +char ixgbe_driver_version[] = "1.6.2"; /********************************************************************* * PCI Device ID Table @@ -65,11 +62,15 @@ static ixgbe_vendor_info_t ixgbe_vendor_ { {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, - {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT_DUAL_PORT, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, + {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, + {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, + {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, + {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, + {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, /* required last entry */ {0, 0, 0, 0, 0} }; @@ -128,14 +129,14 @@ static void ixgbe_disable_intr(struc static void ixgbe_update_stats_counters(struct adapter *); static bool ixgbe_txeof(struct tx_ring *); static bool ixgbe_rxeof(struct rx_ring *, int); -static void ixgbe_rx_checksum(struct adapter *, u32, struct mbuf *); +static void ixgbe_rx_checksum(u32, struct mbuf *); static void ixgbe_set_promisc(struct adapter *); static void ixgbe_disable_promisc(struct adapter *); static void ixgbe_set_multi(struct adapter *); static void ixgbe_print_hw_stats(struct adapter *); static void ixgbe_print_debug_info(struct adapter *); static void ixgbe_update_link_status(struct adapter *); -static int ixgbe_get_buf(struct rx_ring *, int); +static int ixgbe_get_buf(struct rx_ring *, int, u8); static int ixgbe_xmit(struct tx_ring *, struct mbuf **); static int ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS); static int ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS); @@ -147,15 +148,20 @@ static void ixgbe_add_rx_process_limit(s const char *, int *, int); static boolean_t ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); static boolean_t ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); -static void ixgbe_set_ivar(struct adapter *, u16, u8); +static void ixgbe_set_ivar(struct adapter *, u16, u8, s8); static void ixgbe_configure_ivars(struct adapter *); static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); -#ifdef IXGBE_VLAN_EVENTS +#ifdef IXGBE_HW_VLAN_SUPPORT static void ixgbe_register_vlan(void *, struct ifnet *, u16); static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); #endif +static void ixgbe_update_aim(struct rx_ring *); + +/* Support for pluggable optic modules */ +static bool ixgbe_sfp_probe(struct adapter *); + /* Legacy (single vector interrupt handler */ static void ixgbe_legacy_irq(void *); @@ -168,9 +174,6 @@ static void ixgbe_msix_link(void *); static void ixgbe_handle_tx(void *context, int pending); static void ixgbe_handle_rx(void *context, int pending); -#ifndef NO_82598_A0_SUPPORT -static void desc_flip(void *); -#endif /********************************************************************* * FreeBSD Device Interface Entry Points @@ -199,12 +202,28 @@ MODULE_DEPEND(ixgbe, ether, 1, 1, 1); ** TUNEABLE PARAMETERS: */ +/* +** These parameters are used in Adaptive +** Interrupt Moderation. The value is set +** into EITR and controls the interrupt +** frequency. They can be modified but +** be careful in tuning them. +*/ +static int ixgbe_enable_aim = TRUE; +TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); +static int ixgbe_low_latency = IXGBE_LOW_LATENCY; +TUNABLE_INT("hw.ixgbe.low_latency", &ixgbe_low_latency); +static int ixgbe_ave_latency = IXGBE_LOW_LATENCY; +TUNABLE_INT("hw.ixgbe.ave_latency", &ixgbe_low_latency); +static int ixgbe_bulk_latency = IXGBE_BULK_LATENCY; +TUNABLE_INT("hw.ixgbe.bulk_latency", &ixgbe_bulk_latency); + /* How many packets rxeof tries to clean at a time */ static int ixgbe_rx_process_limit = 100; TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); /* Flow control setting, default to full */ -static int ixgbe_flow_control = 3; +static int ixgbe_flow_control = ixgbe_fc_none; TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); /* @@ -213,7 +232,7 @@ TUNABLE_INT("hw.ixgbe.flow_control", &ix * interface must be reset (down/up) for it * to take effect. */ -static int ixgbe_enable_lro = 0; +static int ixgbe_enable_lro = 1; TUNABLE_INT("hw.ixgbe.enable_lro", &ixgbe_enable_lro); /* @@ -224,12 +243,18 @@ static int ixgbe_enable_msix = 1; TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); /* + * Enable RX Header Split + */ +static int ixgbe_rx_hdr_split = 1; +TUNABLE_INT("hw.ixgbe.rx_hdr_split", &ixgbe_rx_hdr_split); + +/* * Number of TX/RX Queues, with 0 setting * it autoconfigures to the number of cpus. */ static int ixgbe_tx_queues = 1; TUNABLE_INT("hw.ixgbe.tx_queues", &ixgbe_tx_queues); -static int ixgbe_rx_queues = 4; +static int ixgbe_rx_queues = 1; TUNABLE_INT("hw.ixgbe.rx_queues", &ixgbe_rx_queues); /* Number of TX descriptors per ring */ @@ -243,9 +268,6 @@ TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); /* Total number of Interfaces - need for config sanity check */ static int ixgbe_total_ports; -/* Optics type of this interface */ -static int ixgbe_optics; - /********************************************************************* * Device identification routine * @@ -260,11 +282,11 @@ ixgbe_probe(device_t dev) { ixgbe_vendor_info_t *ent; - u_int16_t pci_vendor_id = 0; - u_int16_t pci_device_id = 0; - u_int16_t pci_subvendor_id = 0; - u_int16_t pci_subdevice_id = 0; - char adapter_name[128]; + u16 pci_vendor_id = 0; + u16 pci_device_id = 0; + u16 pci_subvendor_id = 0; + u16 pci_subdevice_id = 0; + char adapter_name[256]; INIT_DEBUGOUT("ixgbe_probe: begin"); @@ -289,41 +311,11 @@ ixgbe_probe(device_t dev) sprintf(adapter_name, "%s, Version - %s", ixgbe_strings[ent->index], ixgbe_driver_version); - switch (pci_device_id) { - case IXGBE_DEV_ID_82598AT_DUAL_PORT : - ixgbe_total_ports += 2; - break; - case IXGBE_DEV_ID_82598_CX4_DUAL_PORT : - ixgbe_optics = IFM_10G_CX4; - ixgbe_total_ports += 2; - break; - case IXGBE_DEV_ID_82598AF_DUAL_PORT : - ixgbe_optics = IFM_10G_SR; - ixgbe_total_ports += 2; - break; - case IXGBE_DEV_ID_82598AF_SINGLE_PORT : - ixgbe_optics = IFM_10G_SR; - ixgbe_total_ports += 1; - break; - case IXGBE_DEV_ID_82598EB_XF_LR : - ixgbe_optics = IFM_10G_LR; - ixgbe_total_ports += 1; - break; - case IXGBE_DEV_ID_82598EB_CX4 : - ixgbe_optics = IFM_10G_CX4; - ixgbe_total_ports += 1; - break; - case IXGBE_DEV_ID_82598AT : - ixgbe_total_ports += 1; - default: - break; - } device_set_desc_copy(dev, adapter_name); return (0); } ent++; } - return (ENXIO); } @@ -342,7 +334,8 @@ ixgbe_attach(device_t dev) { struct adapter *adapter; int error = 0; - u32 ctrl_ext; + u16 pci_device_id; + u32 ctrl_ext; INIT_DEBUGOUT("ixgbe_attach: begin"); @@ -353,6 +346,37 @@ ixgbe_attach(device_t dev) /* Core Lock Init*/ IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); + /* Keep track of number of ports and optics */ + pci_device_id = pci_get_device(dev); + switch (pci_device_id) { + case IXGBE_DEV_ID_82598_CX4_DUAL_PORT : + adapter->optics = IFM_10G_CX4; + ixgbe_total_ports += 2; + break; + case IXGBE_DEV_ID_82598AF_DUAL_PORT : + adapter->optics = IFM_10G_SR; + ixgbe_total_ports += 2; + break; + case IXGBE_DEV_ID_82598AF_SINGLE_PORT : + adapter->optics = IFM_10G_SR; + ixgbe_total_ports += 1; + break; + case IXGBE_DEV_ID_82598EB_XF_LR : + adapter->optics = IFM_10G_LR; + ixgbe_total_ports += 1; + break; + case IXGBE_DEV_ID_82598EB_CX4 : + adapter->optics = IFM_10G_CX4; + ixgbe_total_ports += 1; + break; + case IXGBE_DEV_ID_82598AT : + ixgbe_total_ports += 1; + case IXGBE_DEV_ID_82598_DA_DUAL_PORT : + ixgbe_total_ports += 2; + default: + break; + } + /* SYSCTL APIs */ SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), @@ -374,15 +398,37 @@ ixgbe_attach(device_t dev) OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW, &ixgbe_enable_lro, 1, "Large Receive Offload"); + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, + &ixgbe_enable_aim, 1, "Interrupt Moderation"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "low_latency", CTLTYPE_INT|CTLFLAG_RW, + &ixgbe_low_latency, 1, "Low Latency"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "ave_latency", CTLTYPE_INT|CTLFLAG_RW, + &ixgbe_ave_latency, 1, "Average Latency"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW, + &ixgbe_bulk_latency, 1, "Bulk Latency"); + + SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), + SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), + OID_AUTO, "hdr_split", CTLTYPE_INT|CTLFLAG_RW, + &ixgbe_rx_hdr_split, 1, "RX Header Split"); + /* Set up the timer callout */ callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); /* Determine hardware revision */ ixgbe_identify_hardware(adapter); - /* Indicate to RX setup to use Jumbo Clusters */ - adapter->bigbufs = TRUE; - /* Do base PCI setup - map BAR0 */ if (ixgbe_allocate_pci_resources(adapter)) { device_printf(dev, "Allocation of PCI resources failed\n"); @@ -428,7 +474,20 @@ ixgbe_attach(device_t dev) } /* Initialize the shared code */ - if (ixgbe_init_shared_code(&adapter->hw)) { + error = ixgbe_init_shared_code(&adapter->hw); + if (error == IXGBE_ERR_SFP_NOT_PRESENT) { + /* + ** No optics in this port, set up + ** so the timer routine will probe + ** for later insertion. + */ + adapter->sfp_probe = TRUE; + error = 0; + } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { + device_printf(dev,"Unsupported SFP+ module detected!\n"); + error = EIO; + goto err_late; + } else if (error) { device_printf(dev,"Unable to initialize the shared code\n"); error = EIO; goto err_late; @@ -459,14 +518,14 @@ ixgbe_attach(device_t dev) /* Initialize statistics */ ixgbe_update_stats_counters(adapter); -#ifdef IXGBE_VLAN_EVENTS +#ifdef IXGBE_HW_VLAN_SUPPORT /* Register for VLAN events */ adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, ixgbe_register_vlan, 0, EVENTHANDLER_PRI_FIRST); adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, ixgbe_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); #endif - + /* let hardware know driver is loaded */ ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; @@ -521,7 +580,6 @@ ixgbe_detach(device_t dev) if (txr->tq) { taskqueue_drain(txr->tq, &txr->tx_task); taskqueue_free(txr->tq); - txr->tq = NULL; } } @@ -529,22 +587,21 @@ ixgbe_detach(device_t dev) if (rxr->tq) { taskqueue_drain(rxr->tq, &rxr->rx_task); taskqueue_free(rxr->tq); - rxr->tq = NULL; } } -#ifdef IXGBE_VLAN_EVENTS + /* let hardware know driver is unloading */ + ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); + ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; + IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); + +#ifdef IXGBE_HW_VLAN_SUPPORT /* Unregister VLAN events */ if (adapter->vlan_attach != NULL) EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); if (adapter->vlan_detach != NULL) EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); -#endif - - /* let hardware know driver is unloading */ - ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); - ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; - IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); +#endif ether_ifdetach(adapter->ifp); callout_drain(&adapter->timer); @@ -848,10 +905,13 @@ ixgbe_watchdog(struct adapter *adapter) static void ixgbe_init_locked(struct adapter *adapter) { + struct rx_ring *rxr = adapter->rx_rings; + struct tx_ring *txr = adapter->tx_rings; struct ifnet *ifp = adapter->ifp; device_t dev = adapter->dev; struct ixgbe_hw *hw; - u32 txdctl, rxdctl, mhadd, gpie; + u32 k, txdctl, mhadd, gpie; + u32 rxdctl, rxctrl; INIT_DEBUGOUT("ixgbe_init: begin"); @@ -872,17 +932,16 @@ ixgbe_init_locked(struct adapter *adapte return; } -#ifndef IXGBE_VLAN_EVENTS - /* With events this is done when a vlan registers */ +#ifndef IXGBE_HW_VLAN_SUPPORT if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { - u32 ctrl; + u32 ctrl; + ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); ctrl |= IXGBE_VLNCTRL_VME; ctrl &= ~IXGBE_VLNCTRL_CFIEN; IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); } #endif - /* Prepare transmit descriptors and buffers */ if (ixgbe_setup_transmit_structures(adapter)) { device_printf(dev,"Could not setup transmit structures\n"); @@ -892,15 +951,24 @@ ixgbe_init_locked(struct adapter *adapte ixgbe_initialize_transmit_units(adapter); + /* TX irq moderation rate is fixed */ + for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { + IXGBE_WRITE_REG(&adapter->hw, + IXGBE_EITR(txr->msix), ixgbe_ave_latency); + txr->watchdog_timer = FALSE; + } + /* Setup Multicast table */ ixgbe_set_multi(adapter); /* - ** If we are resetting MTU smaller than 2K - ** drop to small RX buffers + ** Determine the correct mbuf pool + ** for doing jumbo/headersplit */ - if (adapter->max_frame_size <= MCLBYTES) - adapter->bigbufs = FALSE; + if (ifp->if_mtu > ETHERMTU) + adapter->rx_mbuf_sz = MJUMPAGESIZE; + else + adapter->rx_mbuf_sz = MCLBYTES; /* Prepare receive descriptors and buffers */ if (ixgbe_setup_receive_structures(adapter)) { @@ -912,10 +980,22 @@ ixgbe_init_locked(struct adapter *adapte /* Configure RX settings */ ixgbe_initialize_receive_units(adapter); + /* RX moderation will be adapted over time, set default */ + for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { + IXGBE_WRITE_REG(&adapter->hw, + IXGBE_EITR(rxr->msix), ixgbe_low_latency); + } + + /* Set Link moderation */ + IXGBE_WRITE_REG(&adapter->hw, + IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); + gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); + /* Enable Fan Failure Interrupt */ if (adapter->hw.phy.media_type == ixgbe_media_type_copper) gpie |= IXGBE_SDP1_GPIEN; + if (adapter->msix) { /* Enable Enhanced MSIX mode */ gpie |= IXGBE_GPIE_MSIX_MODE; @@ -955,12 +1035,29 @@ ixgbe_init_locked(struct adapter *adapte rxdctl |= 0x0020; rxdctl |= IXGBE_RXDCTL_ENABLE; IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXDCTL(i), rxdctl); + for (k = 0; k < 10; k++) { + if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & + IXGBE_RXDCTL_ENABLE) + break; + else + msec_delay(1); + } + wmb(); + IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); } + /* Enable Receive engine */ + rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); + if (adapter->hw.mac.type == ixgbe_mac_82598EB) + rxctrl |= IXGBE_RXCTRL_DMBYPS; + rxctrl |= IXGBE_RXCTRL_RXEN; + IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl); + callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); /* Set up MSI/X routing */ - ixgbe_configure_ivars(adapter); + if (ixgbe_enable_msix) + ixgbe_configure_ivars(adapter); ixgbe_enable_intr(adapter); @@ -984,7 +1081,7 @@ ixgbe_init(void *arg) /* -** Legacy Deferred Interrupt Handlers +** MSIX Interrupt Handlers */ static void @@ -992,11 +1089,14 @@ ixgbe_handle_rx(void *context, int pendi { struct rx_ring *rxr = context; struct adapter *adapter = rxr->adapter; - u32 loop = 0; + u32 loop = MAX_LOOP; + bool more; - while (loop++ < MAX_INTR) - if (ixgbe_rxeof(rxr, adapter->rx_process_limit) == 0) - break; + do { + more = ixgbe_rxeof(rxr, -1); + } while (loop-- && more); + /* Reenable this interrupt */ + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims); } static void @@ -1005,15 +1105,21 @@ ixgbe_handle_tx(void *context, int pendi struct tx_ring *txr = context; struct adapter *adapter = txr->adapter; struct ifnet *ifp = adapter->ifp; - u32 loop = 0; + u32 loop = MAX_LOOP; + bool more; - IXGBE_TX_LOCK(txr); - while (loop++ < MAX_INTR) - if (ixgbe_txeof(txr) == 0) - break; - if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) - ixgbe_start_locked(txr, ifp); - IXGBE_TX_UNLOCK(txr); + IXGBE_TX_LOCK(txr); + do { + more = ixgbe_txeof(txr); + } while (loop-- && more); + + if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) + ixgbe_start_locked(txr, ifp); + + IXGBE_TX_UNLOCK(txr); + + /* Reenable this interrupt */ + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims); } @@ -1026,34 +1132,38 @@ ixgbe_handle_tx(void *context, int pendi static void ixgbe_legacy_irq(void *arg) { - u32 reg_eicr; struct adapter *adapter = arg; + struct ixgbe_hw *hw = &adapter->hw; struct tx_ring *txr = adapter->tx_rings; struct rx_ring *rxr = adapter->rx_rings; - struct ixgbe_hw *hw; + u32 reg_eicr; - hw = &adapter->hw; - reg_eicr = IXGBE_READ_REG(&adapter->hw, IXGBE_EICR); - if (reg_eicr == 0) + + reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); + + if (reg_eicr == 0) { + ixgbe_enable_intr(adapter); return; + } - if (ixgbe_rxeof(rxr, adapter->rx_process_limit) != 0) + if (ixgbe_rxeof(rxr, adapter->rx_process_limit)) taskqueue_enqueue(rxr->tq, &rxr->rx_task); - if (ixgbe_txeof(txr) != 0) - taskqueue_enqueue(txr->tq, &txr->tx_task); + if (ixgbe_txeof(txr)) + taskqueue_enqueue(txr->tq, &txr->tx_task); /* Check for fan failure */ if ((hw->phy.media_type == ixgbe_media_type_copper) && (reg_eicr & IXGBE_EICR_GPI_SDP1)) { device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " "REPLACE IMMEDIATELY!!\n"); - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, - IXGBE_EICR_GPI_SDP1); + IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); } + /* Link status change */ if (reg_eicr & IXGBE_EICR_LSC) ixgbe_update_link_status(adapter); + ixgbe_enable_intr(adapter); return; } @@ -1067,25 +1177,25 @@ ixgbe_legacy_irq(void *arg) void ixgbe_msix_tx(void *arg) { - struct tx_ring *txr = arg; - struct adapter *adapter = txr->adapter; - u32 loop = 0; + struct tx_ring *txr = arg; + struct adapter *adapter = txr->adapter; + bool more; - ++txr->tx_irq; IXGBE_TX_LOCK(txr); - while (loop++ < MAX_INTR) - if (ixgbe_txeof(txr) == 0) - break; + ++txr->tx_irq; + more = ixgbe_txeof(txr); IXGBE_TX_UNLOCK(txr); - /* Reenable this interrupt */ - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims); - + if (more) + taskqueue_enqueue(txr->tq, &txr->tx_task); + else /* Reenable this interrupt */ + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims); return; } + /********************************************************************* * - * MSI RX Interrupt Service routine + * MSIX RX Interrupt Service routine * **********************************************************************/ @@ -1093,18 +1203,71 @@ static void ixgbe_msix_rx(void *arg) { struct rx_ring *rxr = arg; - struct adapter *adapter = rxr->adapter; - u32 loop = 0; + struct adapter *adapter = rxr->adapter; + bool more; ++rxr->rx_irq; - while (loop++ < MAX_INTR) - if (ixgbe_rxeof(rxr, adapter->rx_process_limit) == 0) - break; - /* Reenable this interrupt */ - IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims); + more = ixgbe_rxeof(rxr, -1); + if (more) + taskqueue_enqueue(rxr->tq, &rxr->rx_task); + else + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims); + /* Update interrupt rate */ + if (ixgbe_enable_aim == TRUE) + ixgbe_update_aim(rxr); return; } +/* +** Routine to do adjust the RX EITR value based on traffic, +** its a simple three state model, but seems to help. +** +** Note that the three EITR values are tuneable using +** sysctl in real time. The feature can be effectively +** nullified by setting them equal. +*/ +#define BULK_THRESHOLD 10000 +#define AVE_THRESHOLD 1600 + +static void +ixgbe_update_aim(struct rx_ring *rxr) +{ + struct adapter *adapter = rxr->adapter; + u32 olditr, newitr; + + /* Update interrupt moderation based on traffic */ + olditr = rxr->eitr_setting; + newitr = olditr; + + /* Idle, don't change setting */ + if (rxr->bytes == 0) + return; + + if (olditr == ixgbe_low_latency) { + if (rxr->bytes > AVE_THRESHOLD) + newitr = ixgbe_ave_latency; + } else if (olditr == ixgbe_ave_latency) { + if (rxr->bytes < AVE_THRESHOLD) + newitr = ixgbe_low_latency; + else if (rxr->bytes > BULK_THRESHOLD) + newitr = ixgbe_bulk_latency; + } else if (olditr == ixgbe_bulk_latency) { + if (rxr->bytes < BULK_THRESHOLD) + newitr = ixgbe_ave_latency; + } + + if (olditr != newitr) { + /* Change interrupt rate */ + rxr->eitr_setting = newitr; + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(rxr->me), + newitr | (newitr << 16)); + } + + rxr->bytes = 0; + return; +} + + static void ixgbe_msix_link(void *arg) { @@ -1164,7 +1327,7 @@ ixgbe_media_status(struct ifnet * ifp, s ifmr->ifm_active |= IFM_1000_T | IFM_FDX; break; case IXGBE_LINK_SPEED_10GB_FULL: - ifmr->ifm_active |= ixgbe_optics | IFM_FDX; + ifmr->ifm_active |= adapter->optics | IFM_FDX; break; } @@ -1220,7 +1383,7 @@ ixgbe_xmit(struct tx_ring *txr, struct m { struct adapter *adapter = txr->adapter; u32 olinfo_status = 0, cmd_type_len = 0; - u32 paylen; + u32 paylen = 0; int i, j, error, nsegs; int first, last = 0; struct mbuf *m_head; @@ -1230,7 +1393,6 @@ ixgbe_xmit(struct tx_ring *txr, struct m union ixgbe_adv_tx_desc *txd = NULL; m_head = *m_headp; - paylen = 0; /* Basic descriptor defines */ cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; @@ -1274,7 +1436,7 @@ ixgbe_xmit(struct tx_ring *txr, struct m m = m_defrag(*m_headp, M_DONTWAIT); if (m == NULL) { - adapter->mbuf_alloc_failed++; + adapter->mbuf_defrag_failed++; m_freem(*m_headp); *m_headp = NULL; return (ENOBUFS); @@ -1326,6 +1488,11 @@ ixgbe_xmit(struct tx_ring *txr, struct m } else if (ixgbe_tx_ctx_setup(txr, m_head)) olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; + /* Record payload length */ + if (paylen == 0) + olinfo_status |= m_head->m_pkthdr.len << + IXGBE_ADVTXD_PAYLEN_SHIFT; + i = txr->next_avail_tx_desc; for (j = 0; j < nsegs; j++) { bus_size_t seglen; @@ -1346,19 +1513,10 @@ ixgbe_xmit(struct tx_ring *txr, struct m i = 0; txbuf->m_head = NULL; - /* - ** we have to do this inside the loop right now - ** because of the hardware workaround. - */ - if (j == (nsegs -1)) /* Last descriptor gets EOP and RS */ - txd->read.cmd_type_len |= - htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); -#ifndef NO_82598_A0_SUPPORT - if (adapter->hw.revision_id == 0) - desc_flip(txd); -#endif } + txd->read.cmd_type_len |= + htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); txr->tx_avail -= nsegs; txr->next_avail_tx_desc = i; @@ -1375,8 +1533,8 @@ ixgbe_xmit(struct tx_ring *txr, struct m * Advance the Transmit Descriptor Tail (Tdt), this tells the * hardware that this frame is available to transmit. */ + ++txr->total_packets; IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); - ++txr->tx_packets; return (0); xmit_fail: @@ -1504,17 +1662,26 @@ ixgbe_local_timer(void *arg) mtx_assert(&adapter->core_mtx, MA_OWNED); + /* Check for pluggable optics */ + if (adapter->sfp_probe) + if (!ixgbe_sfp_probe(adapter)) + goto out; /* Nothing to do */ + ixgbe_update_link_status(adapter); ixgbe_update_stats_counters(adapter); if (ixgbe_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) { ixgbe_print_hw_stats(adapter); } /* - * Each second we check the watchdog + * Each tick we check the watchdog * to protect against hardware hangs. */ ixgbe_watchdog(adapter); +out: + /* Trigger an RX interrupt on all queues */ + IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, adapter->rx_mask); + callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); } @@ -1701,6 +1868,11 @@ ixgbe_allocate_msix(struct adapter *adap } txr->msix = vector; txr->eims = IXGBE_IVAR_TX_QUEUE(vector); + TASK_INIT(&txr->tx_task, 0, ixgbe_handle_tx, txr); + txr->tq = taskqueue_create_fast("ixgbe_txq", M_NOWAIT, + taskqueue_thread_enqueue, &txr->tq); + taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", + device_get_nameunit(adapter->dev)); } /* RX setup */ @@ -1725,6 +1897,13 @@ ixgbe_allocate_msix(struct adapter *adap } rxr->msix = vector; rxr->eims = IXGBE_IVAR_RX_QUEUE(vector); + /* used in local timer */ + adapter->rx_mask |= rxr->eims; + TASK_INIT(&rxr->rx_task, 0, ixgbe_handle_rx, rxr); + rxr->tq = taskqueue_create_fast("ixgbe_rxq", M_NOWAIT, + taskqueue_thread_enqueue, &rxr->tq); + taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", + device_get_nameunit(adapter->dev)); } /* Now for Link changes */ @@ -1759,11 +1938,20 @@ ixgbe_setup_msix(struct adapter *adapter device_t dev = adapter->dev; int rid, want, queues, msgs; + /* Override by tuneable */ + if (ixgbe_enable_msix == 0) + goto msi; + /* First try MSI/X */ - rid = PCIR_BAR(IXGBE_MSIX_BAR); + rid = PCIR_BAR(MSIX_82598_BAR); adapter->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!adapter->msix_mem) { + rid += 4; /* 82599 maps in higher BAR */ + adapter->msix_mem = bus_alloc_resource_any(dev, + SYS_RES_MEMORY, &rid, RF_ACTIVE); + } + if (!adapter->msix_mem) { /* May not be enabled */ device_printf(adapter->dev, "Unable to map MSIX table \n"); @@ -1773,7 +1961,7 @@ ixgbe_setup_msix(struct adapter *adapter msgs = pci_msix_count(dev); if (msgs == 0) { /* system has msix disabled */ bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(IXGBE_MSIX_BAR), adapter->msix_mem); + rid, adapter->msix_mem); adapter->msix_mem = NULL; goto msi; } @@ -1853,7 +2041,8 @@ ixgbe_allocate_pci_resources(struct adap static void ixgbe_free_pci_resources(struct adapter * adapter) { - device_t dev = adapter->dev; + device_t dev = adapter->dev; + int rid; /* * Legacy has this set to 0, but we need @@ -1862,6 +2051,8 @@ ixgbe_free_pci_resources(struct adapter if (adapter->msix == 0) adapter->msix = 1; + rid = PCIR_BAR(MSIX_82598_BAR); + /* * First release all the interrupt resources: * notice that since these are just kept @@ -1885,7 +2076,7 @@ ixgbe_free_pci_resources(struct adapter if (adapter->msix_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(IXGBE_MSIX_BAR), adapter->msix_mem); + rid, adapter->msix_mem); if (adapter->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, @@ -1920,7 +2111,7 @@ ixgbe_hardware_init(struct adapter *adap } /* Get Hardware Flow Control setting */ - adapter->hw.fc.type = ixgbe_fc_full; + adapter->hw.fc.requested_mode = ixgbe_fc_full; adapter->hw.fc.pause_time = IXGBE_FC_PAUSE; adapter->hw.fc.low_water = IXGBE_FC_LO; adapter->hw.fc.high_water = IXGBE_FC_HI; @@ -1977,8 +2168,7 @@ ixgbe_setup_interface(device_t dev, stru ifp->if_capenable = ifp->if_capabilities; - if ((hw->device_id == IXGBE_DEV_ID_82598AT) || - (hw->device_id == IXGBE_DEV_ID_82598AT_DUAL_PORT)) + if (hw->device_id == IXGBE_DEV_ID_82598AT) ixgbe_setup_link_speed(hw, (IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL), TRUE, TRUE); else @@ -1991,10 +2181,9 @@ ixgbe_setup_interface(device_t dev, stru */ ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, ixgbe_media_status); - ifmedia_add(&adapter->media, IFM_ETHER | ixgbe_optics | + ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics | IFM_FDX, 0, NULL); - if ((hw->device_id == IXGBE_DEV_ID_82598AT) || - (hw->device_id == IXGBE_DEV_ID_82598AT_DUAL_PORT)) { + if (hw->device_id == IXGBE_DEV_ID_82598AT) { ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); ifmedia_add(&adapter->media, @@ -2095,7 +2284,6 @@ ixgbe_allocate_queues(struct adapter *ad struct tx_ring *txr; struct rx_ring *rxr; int rsize, tsize, error = IXGBE_SUCCESS; - char name_string[16]; int txconf = 0, rxconf = 0; /* First allocate the TX ring struct memory */ @@ -2134,9 +2322,9 @@ ixgbe_allocate_queues(struct adapter *ad txr->me = i; /* Initialize the TX side lock */ - snprintf(name_string, sizeof(name_string), "%s:tx(%d)", + snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", device_get_nameunit(dev), txr->me); - mtx_init(&txr->tx_mtx, name_string, NULL, MTX_DEF); + mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); if (ixgbe_dma_malloc(adapter, tsize, &txr->txdma, BUS_DMA_NOWAIT)) { @@ -2169,10 +2357,10 @@ ixgbe_allocate_queues(struct adapter *ad rxr->adapter = adapter; rxr->me = i; - /* Initialize the TX side lock */ - snprintf(name_string, sizeof(name_string), "%s:rx(%d)", + /* Initialize the RX side lock */ + snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", device_get_nameunit(dev), rxr->me); - mtx_init(&rxr->rx_mtx, name_string, NULL, MTX_DEF); + mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); if (ixgbe_dma_malloc(adapter, rsize, &rxr->rxdma, BUS_DMA_NOWAIT)) { @@ -2554,11 +2742,6 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, TXD->seqnum_seed = htole32(0); TXD->mss_l4len_idx = htole32(0); -#ifndef NO_82598_A0_SUPPORT - if (adapter->hw.revision_id == 0) - desc_flip(TXD); -#endif - tx_buffer->m_head = NULL; /* We've consumed the first desc, adjust counters */ @@ -2652,11 +2835,6 @@ ixgbe_tso_setup(struct tx_ring *txr, str TXD->seqnum_seed = htole32(0); tx_buffer->m_head = NULL; -#ifndef NO_82598_A0_SUPPORT - if (adapter->hw.revision_id == 0) *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***