Date: Fri, 19 Jun 2009 23:34:32 +0000 (UTC) From: Kip Macy <kmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r194521 - in head/sys: conf dev/cxgb dev/cxgb/common dev/cxgb/sys modules/cxgb/cxgb Message-ID: <200906192334.n5JNYWji058504@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kmacy Date: Fri Jun 19 23:34:32 2009 New Revision: 194521 URL: http://svn.freebsd.org/changeset/base/194521 Log: Greatly simplify cxgb by removing almost all of the custom mbuf management logic - remove mbuf iovec - useful, but adds too much complexity when isolated to the driver - remove driver private caching - insufficient benefit over UMA to justify the added complexity and maintenance overhead - remove separate logic for managing multiple transmit queues, with the new drbr routines the control flow can be made to much more closely resemble legacy drivers - remove dedicated service threads, with per-cpu callouts one can get the same benefit much more simply by registering a callout 1 tick in the future if there are still buffered packets - remove embedded mbuf usage - Jeffr's changes will (I hope) soon be integrated greatly reducing the overhead of using kernel APIs for reference counting clusters - add hysteresis to descriptor coalescing logic - add coalesce threshold sysctls to allow users to decide at run-time between optimizing for forwarding / UDP or optimizing for TCP - add once per second watchdog to effectively close the very rare races occurring from coalescing - incorporate Navdeep's changes to the initialization path required to convert port and adapter locks back to ordinary mutexes (silencing BPF LOR complaints) - enable prefetches in get_packet and tx cleaning Reviewed by: navdeep@ MFC after: 2 weeks Deleted: head/sys/dev/cxgb/cxgb_multiq.c head/sys/dev/cxgb/sys/cxgb_support.c Modified: head/sys/conf/files head/sys/dev/cxgb/common/cxgb_t3_cpl.h head/sys/dev/cxgb/cxgb_adapter.h head/sys/dev/cxgb/cxgb_main.c head/sys/dev/cxgb/cxgb_osdep.h head/sys/dev/cxgb/cxgb_sge.c head/sys/dev/cxgb/sys/mvec.h head/sys/dev/cxgb/sys/uipc_mvec.c head/sys/modules/cxgb/cxgb/Makefile Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Fri Jun 19 23:30:09 2009 (r194520) +++ head/sys/conf/files Fri Jun 19 23:34:32 2009 (r194521) @@ -756,8 +756,6 @@ dev/cxgb/cxgb_offload.c optional cxgb p compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/cxgb_sge.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" -dev/cxgb/cxgb_multiq.c optional cxgb pci \ - compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_mc5.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/common/cxgb_vsc7323.c optional cxgb pci \ @@ -776,8 +774,6 @@ dev/cxgb/common/cxgb_tn1010.c optional c compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/sys/uipc_mvec.c optional cxgb pci \ compile-with "${NORMAL_C} -I$S/dev/cxgb" -dev/cxgb/sys/cxgb_support.c optional cxgb pci \ - compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cxgb/cxgb_t3fw.c optional cxgb cxgb_t3fw \ compile-with "${NORMAL_C} -I$S/dev/cxgb" dev/cy/cy.c optional cy Modified: head/sys/dev/cxgb/common/cxgb_t3_cpl.h ============================================================================== --- head/sys/dev/cxgb/common/cxgb_t3_cpl.h Fri Jun 19 23:30:09 2009 (r194520) +++ head/sys/dev/cxgb/common/cxgb_t3_cpl.h Fri Jun 19 23:34:32 2009 (r194521) @@ -237,10 +237,21 @@ struct rss_header { #ifndef CHELSIO_FW struct work_request_hdr { - __be32 wr_hi; - __be32 wr_lo; + union { + struct { + __be32 wr_hi; + __be32 wr_lo; + } ilp32; + struct { + __be64 wr_hilo; + } lp64; + } u; }; +#define wrh_hi u.ilp32.wr_hi +#define wrh_lo u.ilp32.wr_lo +#define wrh_hilo u.lp64.wr_hilo + /* wr_hi fields */ #define S_WR_SGE_CREDITS 0 #define M_WR_SGE_CREDITS 0xFF @@ -817,8 +828,7 @@ struct cpl_peer_close { }; struct tx_data_wr { - __be32 wr_hi; - __be32 wr_lo; + WR_HDR; __be32 len; __be32 flags; __be32 sndseq; @@ -936,8 +946,7 @@ struct cpl_rdma_ec_status { }; struct mngt_pktsched_wr { - __be32 wr_hi; - __be32 wr_lo; + WR_HDR; __u8 mngt_opcode; __u8 rsvd[7]; __u8 sched; Modified: head/sys/dev/cxgb/cxgb_adapter.h ============================================================================== --- head/sys/dev/cxgb/cxgb_adapter.h Fri Jun 19 23:30:09 2009 (r194520) +++ head/sys/dev/cxgb/cxgb_adapter.h Fri Jun 19 23:34:32 2009 (r194521) @@ -35,7 +35,6 @@ $FreeBSD$ #include <sys/lock.h> #include <sys/mutex.h> -#include <sys/sx.h> #include <sys/rman.h> #include <sys/mbuf.h> #include <sys/socket.h> @@ -63,8 +62,6 @@ $FreeBSD$ #include <netinet/tcp_lro.h> #endif -#define USE_SX - struct adapter; struct sge_qset; extern int cxgb_debug; @@ -82,22 +79,9 @@ extern int cxgb_debug; mtx_destroy((lock)); \ } while (0) -#define SX_INIT(lock, lockname) \ - do { \ - printf("initializing %s at %s:%d\n", lockname, __FILE__, __LINE__); \ - sx_init((lock), lockname); \ - } while (0) - -#define SX_DESTROY(lock) \ - do { \ - printf("destroying %s at %s:%d\n", (lock)->lock_object.lo_name, __FILE__, __LINE__); \ - sx_destroy((lock)); \ - } while (0) #else #define MTX_INIT mtx_init #define MTX_DESTROY mtx_destroy -#define SX_INIT sx_init -#define SX_DESTROY sx_destroy #endif enum { @@ -110,20 +94,17 @@ struct port_info { struct adapter *adapter; struct ifnet *ifp; int if_flags; + int flags; const struct port_type_info *port_type; struct cphy phy; struct cmac mac; struct link_config link_config; struct ifmedia media; -#ifdef USE_SX - struct sx lock; -#else struct mtx lock; -#endif - uint8_t port_id; - uint8_t tx_chan; - uint8_t txpkt_intf; - uint8_t first_qset; + uint32_t port_id; + uint32_t tx_chan; + uint32_t txpkt_intf; + uint32_t first_qset; uint32_t nqsets; int link_fault; @@ -135,19 +116,30 @@ struct port_info { #define PORT_NAME_LEN 32 char lockbuf[PORT_LOCK_NAME_LEN]; char namebuf[PORT_NAME_LEN]; -}; +} __aligned(L1_CACHE_BYTES); -enum { /* adapter flags */ +enum { + /* adapter flags */ FULL_INIT_DONE = (1 << 0), USING_MSI = (1 << 1), USING_MSIX = (1 << 2), QUEUES_BOUND = (1 << 3), - FW_UPTODATE = (1 << 4), - TPS_UPTODATE = (1 << 5), + FW_UPTODATE = (1 << 4), + TPS_UPTODATE = (1 << 5), CXGB_SHUTDOWN = (1 << 6), CXGB_OFLD_INIT = (1 << 7), - TP_PARITY_INIT = (1 << 8), + TP_PARITY_INIT = (1 << 8), + CXGB_BUSY = (1 << 9), + + /* port flags */ + DOOMED = (1 << 0), }; +#define IS_DOOMED(p) (p->flags & DOOMED) +#define SET_DOOMED(p) do {p->flags |= DOOMED;} while (0) +#define DOOMED(p) (p->flags & DOOMED) +#define IS_BUSY(sc) (sc->flags & CXGB_BUSY) +#define SET_BUSY(sc) do {sc->flags |= CXGB_BUSY;} while (0) +#define CLR_BUSY(sc) do {sc->flags &= ~CXGB_BUSY;} while (0) #define FL_Q_SIZE 4096 #define JUMBO_Q_SIZE 1024 @@ -205,10 +197,6 @@ struct sge_rspq { uint32_t rspq_dump_count; }; -#ifndef DISABLE_MBUF_IOVEC -#define rspq_mbuf rspq_mh.mh_head -#endif - struct rx_desc; struct rx_sw_desc; @@ -253,7 +241,6 @@ struct sge_txq { bus_addr_t phys_addr; struct task qresume_task; struct task qreclaim_task; - struct port_info *port; uint32_t cntxt_id; uint64_t stops; uint64_t restarts; @@ -261,26 +248,21 @@ struct sge_txq { bus_dmamap_t desc_map; bus_dma_tag_t entry_tag; struct mbuf_head sendq; - /* - * cleanq should really be an buf_ring to avoid extra - * mbuf touches - */ - struct mbuf_head cleanq; + struct buf_ring *txq_mr; struct ifaltq *txq_ifq; - struct mbuf *immpkt; - + struct callout txq_timer; + struct callout txq_watchdog; + uint64_t txq_coalesced; uint32_t txq_drops; uint32_t txq_skipped; - uint32_t txq_coalesced; uint32_t txq_enqueued; uint32_t txq_dump_start; uint32_t txq_dump_count; - unsigned long txq_frees; - struct mtx lock; + uint64_t txq_direct_packets; + uint64_t txq_direct_bytes; + uint64_t txq_frees; struct sg_ent txq_sgl[TX_MAX_SEGS / 2 + 1]; - #define TXQ_NAME_LEN 32 - char lockbuf[TXQ_NAME_LEN]; }; @@ -297,6 +279,8 @@ enum { #define QS_EXITING 0x1 #define QS_RUNNING 0x2 #define QS_BOUND 0x4 +#define QS_FLUSHING 0x8 +#define QS_TIMEOUT 0x10 struct sge_qset { struct sge_rspq rspq; @@ -309,10 +293,10 @@ struct sge_qset { uint64_t port_stats[SGE_PSTAT_MAX]; struct port_info *port; int idx; /* qset # */ - int qs_cpuid; int qs_flags; + int coalescing; struct cv qs_cv; - struct mtx qs_mtx; + struct mtx lock; #define QS_NAME_LEN 32 char namebuf[QS_NAME_LEN]; }; @@ -328,7 +312,7 @@ struct adapter { device_t dev; int flags; TAILQ_ENTRY(adapter) adapter_entry; - + /* PCI register resources */ int regs_rid; struct resource *regs_res; @@ -401,11 +385,7 @@ struct adapter { char port_types[MAX_NPORTS + 1]; uint32_t open_device_map; uint32_t registered_device_map; -#ifdef USE_SX - struct sx lock; -#else struct mtx lock; -#endif driver_intr_t *cxgb_intr; int msi_count; @@ -422,31 +402,17 @@ struct t3_rx_mode { struct port_info *port; }; - #define MDIO_LOCK(adapter) mtx_lock(&(adapter)->mdio_lock) #define MDIO_UNLOCK(adapter) mtx_unlock(&(adapter)->mdio_lock) #define ELMR_LOCK(adapter) mtx_lock(&(adapter)->elmer_lock) #define ELMR_UNLOCK(adapter) mtx_unlock(&(adapter)->elmer_lock) -#ifdef USE_SX -#define PORT_LOCK(port) sx_xlock(&(port)->lock); -#define PORT_UNLOCK(port) sx_xunlock(&(port)->lock); -#define PORT_LOCK_INIT(port, name) SX_INIT(&(port)->lock, name) -#define PORT_LOCK_DEINIT(port) SX_DESTROY(&(port)->lock) -#define PORT_LOCK_ASSERT_OWNED(port) sx_assert(&(port)->lock, SA_LOCKED) - -#define ADAPTER_LOCK(adap) sx_xlock(&(adap)->lock); -#define ADAPTER_UNLOCK(adap) sx_xunlock(&(adap)->lock); -#define ADAPTER_LOCK_INIT(adap, name) SX_INIT(&(adap)->lock, name) -#define ADAPTER_LOCK_DEINIT(adap) SX_DESTROY(&(adap)->lock) -#define ADAPTER_LOCK_ASSERT_NOTOWNED(adap) sx_assert(&(adap)->lock, SA_UNLOCKED) -#define ADAPTER_LOCK_ASSERT_OWNED(adap) sx_assert(&(adap)->lock, SA_LOCKED) -#else #define PORT_LOCK(port) mtx_lock(&(port)->lock); #define PORT_UNLOCK(port) mtx_unlock(&(port)->lock); #define PORT_LOCK_INIT(port, name) mtx_init(&(port)->lock, name, 0, MTX_DEF) #define PORT_LOCK_DEINIT(port) mtx_destroy(&(port)->lock) +#define PORT_LOCK_ASSERT_NOTOWNED(port) mtx_assert(&(port)->lock, MA_NOTOWNED) #define PORT_LOCK_ASSERT_OWNED(port) mtx_assert(&(port)->lock, MA_OWNED) #define ADAPTER_LOCK(adap) mtx_lock(&(adap)->lock); @@ -455,7 +421,6 @@ struct t3_rx_mode { #define ADAPTER_LOCK_DEINIT(adap) mtx_destroy(&(adap)->lock) #define ADAPTER_LOCK_ASSERT_NOTOWNED(adap) mtx_assert(&(adap)->lock, MA_NOTOWNED) #define ADAPTER_LOCK_ASSERT_OWNED(adap) mtx_assert(&(adap)->lock, MA_OWNED) -#endif static __inline uint32_t @@ -555,14 +520,11 @@ void t3_sge_stop(adapter_t *); void t3b_intr(void *data); void t3_intr_msi(void *data); void t3_intr_msix(void *data); -int t3_encap(struct sge_qset *, struct mbuf **, int); int t3_sge_init_adapter(adapter_t *); int t3_sge_reset_adapter(adapter_t *); int t3_sge_init_port(struct port_info *); -void t3_sge_deinit_sw(adapter_t *); -void t3_free_tx_desc(struct sge_txq *q, int n); -void t3_free_tx_desc_all(struct sge_txq *q); +void t3_free_tx_desc(struct sge_qset *qs, int n, int qid); void t3_rx_eth(struct adapter *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad); @@ -615,13 +577,8 @@ static inline int offload_running(adapte return isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); } -int cxgb_pcpu_enqueue_packet(struct ifnet *ifp, struct mbuf *m); -int cxgb_pcpu_transmit(struct ifnet *ifp, struct mbuf *m); -void cxgb_pcpu_shutdown_threads(struct adapter *sc); -void cxgb_pcpu_startup_threads(struct adapter *sc); - -int process_responses(adapter_t *adap, struct sge_qset *qs, int budget); -void t3_free_qset(adapter_t *sc, struct sge_qset *q); +void cxgb_tx_watchdog(void *arg); +int cxgb_transmit(struct ifnet *ifp, struct mbuf *m); +void cxgb_qflush(struct ifnet *ifp); void cxgb_start(struct ifnet *ifp); -void refill_fl_service(adapter_t *adap, struct sge_fl *fl); #endif Modified: head/sys/dev/cxgb/cxgb_main.c ============================================================================== --- head/sys/dev/cxgb/cxgb_main.c Fri Jun 19 23:30:09 2009 (r194520) +++ head/sys/dev/cxgb/cxgb_main.c Fri Jun 19 23:34:32 2009 (r194521) @@ -84,10 +84,12 @@ __FBSDID("$FreeBSD$"); static int cxgb_setup_interrupts(adapter_t *); static void cxgb_teardown_interrupts(adapter_t *); +static int cxgb_begin_op(struct port_info *, const char *); +static int cxgb_begin_detach(struct port_info *); +static int cxgb_end_op(struct port_info *); static void cxgb_init(void *); -static void cxgb_init_locked(struct port_info *); -static void cxgb_stop_locked(struct port_info *); -static void cxgb_set_rxmode(struct port_info *); +static int cxgb_init_synchronized(struct port_info *); +static int cxgb_uninit_synchronized(struct port_info *); static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t); static int cxgb_media_change(struct ifnet *); static int cxgb_ifm_type(int); @@ -96,7 +98,6 @@ static int setup_sge_qsets(adapter_t *); static void cxgb_async_intr(void *); static void cxgb_ext_intr_handler(void *, int); static void cxgb_tick_handler(void *, int); -static void cxgb_down_locked(struct adapter *sc); static void cxgb_tick(void *); static void setup_rss(adapter_t *sc); @@ -114,7 +115,6 @@ static int cxgb_get_regs_len(void); static int offload_open(struct port_info *pi); static void touch_bars(device_t dev); static int offload_close(struct t3cdev *tdev); -static void cxgb_link_start(struct port_info *p); int t3_detect_link_fault(adapter_t *adapter, int port_id); static device_method_t cxgb_controller_methods[] = { @@ -722,29 +722,47 @@ cxgb_free(struct adapter *sc) sc->flags |= CXGB_SHUTDOWN; ADAPTER_UNLOCK(sc); - cxgb_pcpu_shutdown_threads(sc); - - ADAPTER_LOCK(sc); - cxgb_down_locked(sc); - ADAPTER_UNLOCK(sc); - - t3_sge_deinit_sw(sc); /* - * Wait for last callout + * Make sure all child devices are gone. */ - - DELAY(hz*100); - bus_generic_detach(sc->dev); - for (i = 0; i < (sc)->params.nports; i++) { if (sc->portdev[i] && device_delete_child(sc->dev, sc->portdev[i]) != 0) device_printf(sc->dev, "failed to delete child port\n"); } - cxgb_teardown_interrupts(sc); + /* + * At this point, it is as if cxgb_port_detach has run on all ports, and + * cxgb_down has run on the adapter. All interrupts have been silenced, + * all open devices have been closed. + */ + KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)", + __func__, sc->open_device_map)); + for (i = 0; i < sc->params.nports; i++) { + KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!", + __func__, i)); + } + /* + * Finish off the adapter's callouts. + */ + callout_drain(&sc->cxgb_tick_ch); + callout_drain(&sc->sge_timer_ch); + + /* + * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The + * sysctls are cleaned up by the kernel linker. + */ + if (sc->flags & FULL_INIT_DONE) { + t3_free_sge_resources(sc); + sc->flags &= ~FULL_INIT_DONE; + } + + /* + * Release all interrupt resources. + */ + cxgb_teardown_interrupts(sc); #ifdef MSI_SUPPORTED if (sc->flags & (USING_MSI | USING_MSIX)) { device_printf(sc->dev, "releasing msi message(s)\n"); @@ -759,26 +777,26 @@ cxgb_free(struct adapter *sc) } #endif + /* + * Free the adapter's taskqueue. + */ if (sc->tq != NULL) { taskqueue_free(sc->tq); sc->tq = NULL; } if (is_offload(sc)) { + clrbit(&sc->registered_device_map, OFFLOAD_DEVMAP_BIT); cxgb_adapter_unofld(sc); - if (isset(&sc->open_device_map, OFFLOAD_DEVMAP_BIT)) - offload_close(&sc->tdev); - else - printf("cxgb_free: DEVMAP_BIT not set\n"); - } else - printf("not offloading set\n"); + } + #ifdef notyet if (sc->flags & CXGB_OFLD_INIT) cxgb_offload_deactivate(sc); #endif free(sc->filters, M_DEVBUF); t3_sge_free(sc); - + cxgb_offload_exit(); if (sc->udbs_res != NULL) @@ -1052,10 +1070,9 @@ cxgb_port_attach(device_t dev) } ether_ifattach(ifp, p->hw_addr); + ifp->if_transmit = cxgb_transmit; + ifp->if_qflush = cxgb_qflush; -#ifdef IFNET_MULTIQUEUE - ifp->if_transmit = cxgb_pcpu_transmit; -#endif /* * Only default to jumbo frames on 10GigE */ @@ -1112,15 +1129,8 @@ cxgb_port_attach(device_t dev) ifmedia_set(&p->media, IFM_ETHER | IFM_AUTO); } - /* Get the latest mac address, User can use a LAA */ - bcopy(IF_LLADDR(p->ifp), p->hw_addr, ETHER_ADDR_LEN); t3_sge_init_port(p); -#if defined(LINK_ATTACH) - cxgb_link_start(p); - t3_link_changed(sc, p->port_id); -#endif - return (err); } @@ -1130,46 +1140,38 @@ cxgb_port_attach(device_t dev) * removing the device from the view of the kernel, i.e. from all * interfaces lists etc. This routine is only called when the driver is * being unloaded, not when the link goes down. - * */ static int cxgb_port_detach(device_t dev) { struct port_info *p; struct adapter *sc; + int i; p = device_get_softc(dev); sc = p->adapter; + cxgb_begin_detach(p); + if (p->port_cdev != NULL) destroy_dev(p->port_cdev); - + + cxgb_uninit_synchronized(p); ether_ifdetach(p->ifp); - PORT_LOCK(p); - if (p->ifp->if_drv_flags & IFF_DRV_RUNNING) - cxgb_stop_locked(p); - PORT_UNLOCK(p); - - callout_drain(&sc->cxgb_tick_ch); - callout_drain(&sc->sge_timer_ch); - - if (sc->tq != NULL) { - printf("draining slow intr\n"); - - taskqueue_drain(sc->tq, &sc->slow_intr_task); - printf("draining ext intr\n"); - taskqueue_drain(sc->tq, &sc->ext_intr_task); - printf("draining tick task\n"); - taskqueue_drain(sc->tq, &sc->tick_task); + for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) { + struct sge_qset *qs = &sc->sge.qs[i]; + struct sge_txq *txq = &qs->txq[TXQ_ETH]; + + callout_drain(&txq->txq_watchdog); + callout_drain(&txq->txq_timer); } - /* - * the lock may be acquired in ifdetach - */ PORT_LOCK_DEINIT(p); if_free(p->ifp); - + p->ifp = NULL; + + cxgb_end_op(p); return (0); } @@ -1276,12 +1278,16 @@ t3_os_link_changed(adapter_t *adapter, i int duplex, int fc) { struct port_info *pi = &adapter->port[port_id]; + struct ifnet *ifp = pi->ifp; + + /* no race with detach, so ifp should always be good */ + KASSERT(ifp, ("%s: if detached.", __func__)); if (link_status) { - pi->ifp->if_baudrate = IF_Mbps(speed); - if_link_state_change(pi->ifp, LINK_STATE_UP); + ifp->if_baudrate = IF_Mbps(speed); + if_link_state_change(ifp, LINK_STATE_UP); } else - if_link_state_change(pi->ifp, LINK_STATE_DOWN); + if_link_state_change(ifp, LINK_STATE_DOWN); } /** @@ -1325,13 +1331,13 @@ t3_os_ext_intr_handler(adapter_t *sc) * interrupts in the meantime and let the task reenable them when * it's done. */ - ADAPTER_LOCK(sc); if (sc->slow_intr_mask) { + ADAPTER_LOCK(sc); sc->slow_intr_mask &= ~F_T3DBG; t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); taskqueue_enqueue(sc->tq, &sc->ext_intr_task); + ADAPTER_UNLOCK(sc); } - ADAPTER_UNLOCK(sc); } void @@ -1348,21 +1354,19 @@ t3_os_set_hw_addr(adapter_t *adapter, in bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN); } -/** - * link_start - enable a port - * @p: the port to enable - * - * Performs the MAC and PHY actions needed to enable a port. +/* + * Programs the XGMAC based on the settings in the ifnet. These settings + * include MTU, MAC address, mcast addresses, etc. */ static void -cxgb_link_start(struct port_info *p) +cxgb_update_mac_settings(struct port_info *p) { - struct ifnet *ifp; + struct ifnet *ifp = p->ifp; struct t3_rx_mode rm; struct cmac *mac = &p->mac; int mtu, hwtagging; - ifp = p->ifp; + PORT_LOCK_ASSERT_OWNED(p); bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN); @@ -1372,15 +1376,11 @@ cxgb_link_start(struct port_info *p) hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0; - t3_init_rx_mode(&rm, p); - if (!mac->multiport) - t3_mac_reset(mac); t3_mac_set_mtu(mac, mtu); t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging); t3_mac_set_address(mac, 0, p->hw_addr); + t3_init_rx_mode(&rm, p); t3_mac_set_rx_mode(mac, &rm); - t3_link_start(&p->phy, mac, &p->link_config); - t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX); } @@ -1415,7 +1415,7 @@ init_tp_parity(struct adapter *adap) req = mtod(m, struct cpl_smt_write_req *); m->m_len = m->m_pkthdr.len = sizeof(*req); memset(req, 0, sizeof(*req)); - req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i)); req->iff = i; t3_mgmt_tx(adap, m); @@ -1428,7 +1428,7 @@ init_tp_parity(struct adapter *adap) req = mtod(m, struct cpl_l2t_write_req *); m->m_len = m->m_pkthdr.len = sizeof(*req); memset(req, 0, sizeof(*req)); - req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i)); req->params = htonl(V_L2T_W_IDX(i)); t3_mgmt_tx(adap, m); @@ -1441,7 +1441,7 @@ init_tp_parity(struct adapter *adap) req = mtod(m, struct cpl_rte_write_req *); m->m_len = m->m_pkthdr.len = sizeof(*req); memset(req, 0, sizeof(*req)); - req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i)); req->l2t_idx = htonl(V_L2T_W_IDX(i)); t3_mgmt_tx(adap, m); @@ -1451,7 +1451,7 @@ init_tp_parity(struct adapter *adap) greq = mtod(m, struct cpl_set_tcb_field *); m->m_len = m->m_pkthdr.len = sizeof(*greq); memset(greq, 0, sizeof(*greq)); - greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0)); greq->mask = htobe64(1); t3_mgmt_tx(adap, m); @@ -1532,7 +1532,7 @@ write_smt_entry(struct adapter *adapter, req = mtod(m, struct cpl_smt_write_req *); m->m_pkthdr.len = m->m_len = sizeof(struct cpl_smt_write_req); - req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx)); req->mtu_idx = NMTUS - 1; /* should be 0 but there's a T3 bug */ req->iff = idx; @@ -1559,10 +1559,8 @@ init_smt(struct adapter *adapter) static void init_port_mtus(adapter_t *adapter) { - unsigned int mtus = adapter->port[0].ifp->if_mtu; + unsigned int mtus = ETHERMTU | (ETHERMTU << 16); - if (adapter->port[1].ifp) - mtus |= adapter->port[1].ifp->if_mtu << 16; t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus); } @@ -1576,7 +1574,7 @@ send_pktsched_cmd(struct adapter *adap, m = m_gethdr(M_DONTWAIT, MT_DATA); if (m) { req = mtod(m, struct mngt_pktsched_wr *); - req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); + req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT)); req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET; req->sched = sched; req->idx = qidx; @@ -1593,7 +1591,6 @@ bind_qsets(adapter_t *sc) { int i, j; - cxgb_pcpu_startup_threads(sc); for (i = 0; i < (sc)->params.nports; ++i) { const struct port_info *pi = adap2pinfo(sc, i); @@ -1717,14 +1714,20 @@ cxgb_up(struct adapter *sc) { int err = 0; + ADAPTER_LOCK_ASSERT_NOTOWNED(sc); + KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)", + __func__, sc->open_device_map)); + if ((sc->flags & FULL_INIT_DONE) == 0) { if ((sc->flags & FW_UPTODATE) == 0) if ((err = upgrade_fw(sc))) goto out; + if ((sc->flags & TPS_UPTODATE) == 0) if ((err = update_tpsram(sc))) goto out; + err = t3_init_hw(sc, 0); if (err) goto out; @@ -1756,78 +1759,53 @@ cxgb_up(struct adapter *sc) sc->flags |= TP_PARITY_INIT; if (sc->flags & TP_PARITY_INIT) { - t3_write_reg(sc, A_TP_INT_CAUSE, - F_CMCACHEPERR | F_ARPLUTPERR); + t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR); t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff); } - if (!(sc->flags & QUEUES_BOUND)) { bind_qsets(sc); sc->flags |= QUEUES_BOUND; } + + t3_sge_reset_adapter(sc); out: return (err); } - /* - * Bring down the interface but do not free any resources. + * Called when the last open device is closed. Does NOT undo all of cxgb_up's + * work. Specifically, the resources grabbed under FULL_INIT_DONE are released + * during controller_detach, not here. */ static void -cxgb_down_locked(struct adapter *sc) +cxgb_down(struct adapter *sc) { - + ADAPTER_LOCK_ASSERT_NOTOWNED(sc); + t3_sge_stop(sc); t3_intr_disable(sc); - - callout_stop(&sc->cxgb_tick_ch); - callout_stop(&sc->sge_timer_ch); } static int offload_open(struct port_info *pi) { - struct adapter *adapter = pi->adapter; - struct t3cdev *tdev = &adapter->tdev; + struct adapter *sc = pi->adapter; + struct t3cdev *tdev = &sc->tdev; - int adap_up = adapter->open_device_map & PORT_MASK; - int err = 0; + ADAPTER_LOCK_ASSERT_NOTOWNED(sc); - if (atomic_cmpset_int(&adapter->open_device_map, - (adapter->open_device_map & ~(1<<OFFLOAD_DEVMAP_BIT)), - (adapter->open_device_map | (1<<OFFLOAD_DEVMAP_BIT))) == 0) - return (0); + setbit(&sc->open_device_map, OFFLOAD_DEVMAP_BIT); - if (!isset(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT)) - printf("offload_open: DEVMAP_BIT did not get set 0x%x\n", - adapter->open_device_map); - ADAPTER_LOCK(pi->adapter); - if (!adap_up) - err = cxgb_up(adapter); - ADAPTER_UNLOCK(pi->adapter); - if (err) - return (err); - - t3_tp_set_offload_mode(adapter, 1); + t3_tp_set_offload_mode(sc, 1); tdev->lldev = pi->ifp; - - init_port_mtus(adapter); - t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd, - adapter->params.b_wnd, - adapter->params.rev == 0 ? - adapter->port[0].ifp->if_mtu : 0xffff); - init_smt(adapter); - /* Call back all registered clients */ + init_port_mtus(sc); + t3_load_mtus(sc, sc->params.mtus, sc->params.a_wnd, sc->params.b_wnd, + sc->params.rev == 0 ? sc->port[0].ifp->if_mtu : 0xffff); + init_smt(sc); cxgb_add_clients(tdev); - /* restore them in case the offload module has changed them */ - if (err) { - t3_tp_set_offload_mode(adapter, 0); - clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); - cxgb_set_dummy_ops(tdev); - } - return (err); + return (0); } static int @@ -1844,147 +1822,220 @@ offload_close(struct t3cdev *tdev) tdev->lldev = NULL; cxgb_set_dummy_ops(tdev); t3_tp_set_offload_mode(adapter, 0); + clrbit(&adapter->open_device_map, OFFLOAD_DEVMAP_BIT); - ADAPTER_LOCK(adapter); - if (!adapter->open_device_map) - cxgb_down_locked(adapter); + return (0); +} + +/* + * Begin a synchronized operation. If this call succeeds, it is guaranteed that + * no one will remove the port or its ifp from underneath the caller. Caller is + * also granted exclusive access to open_device_map. + * + * operation here means init, uninit, detach, and ioctl service. + * + * May fail. + * EINTR (ctrl-c pressed during ifconfig for example). + * ENXIO (port is about to detach - due to kldunload for example). + */ +int +cxgb_begin_op(struct port_info *p, const char *wmsg) +{ + int rc = 0; + struct adapter *sc = p->adapter; + + ADAPTER_LOCK(sc); + + while (!IS_DOOMED(p) && IS_BUSY(sc)) { + if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, wmsg, 0)) { + rc = EINTR; + goto done; + } + } + + if (IS_DOOMED(p)) + rc = ENXIO; + else if (!IS_BUSY(sc)) + SET_BUSY(sc); + else { + KASSERT(0, ("%s: port %d, p->flags = %x , sc->flags = %x", + __func__, p->port_id, p->flags, sc->flags)); + rc = EDOOFUS; + } + +done: + ADAPTER_UNLOCK(sc); + return (rc); +} - ADAPTER_UNLOCK(adapter); +/* + * End a synchronized operation. Read comment block above cxgb_begin_op. + */ +int +cxgb_end_op(struct port_info *p) +{ + struct adapter *sc = p->adapter; + + ADAPTER_LOCK(sc); + KASSERT(IS_BUSY(sc), ("%s: not busy.", __func__)); + CLR_BUSY(sc); + wakeup_one(&sc->flags); + ADAPTER_UNLOCK(sc); return (0); } +/* + * Prepare for port detachment. Detach is a special kind of synchronized + * operation. Also read comment before cxgb_begin_op. + */ +static int +cxgb_begin_detach(struct port_info *p) +{ + struct adapter *sc = p->adapter; + /* + * Inform those waiting for this port that it is going to be destroyed + * and they should not continue further. (They'll return with ENXIO). + */ + ADAPTER_LOCK(sc); + SET_DOOMED(p); + wakeup(&sc->flags); + ADAPTER_UNLOCK(sc); + + /* + * Wait for in-progress operations. + */ + ADAPTER_LOCK(sc); + while (IS_BUSY(sc)) { + mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0); + } + SET_BUSY(sc); + ADAPTER_UNLOCK(sc); + + return (0); +} + +/* + * if_init for cxgb ports. + */ static void cxgb_init(void *arg) { struct port_info *p = arg; - PORT_LOCK(p); - cxgb_init_locked(p); - PORT_UNLOCK(p); + if (cxgb_begin_op(p, "cxgbinit")) + return; + + cxgb_init_synchronized(p); + cxgb_end_op(p); } -static void -cxgb_init_locked(struct port_info *p) +static int +cxgb_init_synchronized(struct port_info *p) { - struct ifnet *ifp; - adapter_t *sc = p->adapter; - int err; - - PORT_LOCK_ASSERT_OWNED(p); - ifp = p->ifp; + struct adapter *sc = p->adapter; + struct ifnet *ifp = p->ifp; + struct cmac *mac = &p->mac; + int i, rc; - ADAPTER_LOCK(p->adapter); - if ((sc->open_device_map == 0) && (err = cxgb_up(sc))) { - ADAPTER_UNLOCK(p->adapter); - cxgb_stop_locked(p); - return; - } - if (p->adapter->open_device_map == 0) { - t3_intr_clear(sc); - } - setbit(&p->adapter->open_device_map, p->port_id); - ADAPTER_UNLOCK(p->adapter); + if (sc->open_device_map == 0) { + if ((rc = cxgb_up(sc)) != 0) + return (rc); - if (is_offload(sc) && !ofld_disable) { - err = offload_open(p); - if (err) + if (is_offload(sc) && !ofld_disable && offload_open(p)) log(LOG_WARNING, "Could not initialize offload capabilities\n"); } - device_printf(sc->dev, "enabling interrupts on port=%d\n", p->port_id); + PORT_LOCK(p); t3_port_intr_enable(sc, p->port_id); + if (!mac->multiport) + t3_mac_reset(mac); + cxgb_update_mac_settings(p); + t3_link_start(&p->phy, mac, &p->link_config); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200906192334.n5JNYWji058504>