Date: Wed, 31 Mar 2010 00:27:49 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r205950 - in head/sys/dev/cxgb: . common Message-ID: <201003310027.o2V0RoFn040323@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: np Date: Wed Mar 31 00:27:49 2010 New Revision: 205950 URL: http://svn.freebsd.org/changeset/base/205950 Log: Multiple fixes related to queue set sizing and resources: - Only the tunnelq (TXQ_ETH) requires a buf_ring, an ifq, and the watchdog/timer callouts. Do not allocate these for the other tx queues. - Use 16k jumbo clusters only on offload capable cards by default. - Do not allocate a full tx ring for the offload queue if the card is not offload capable. - Slightly better freelist size calculation. - Fix nmbjumbo4 typo, remove unneeded global variables. MFC after: 3 days Modified: head/sys/dev/cxgb/common/cxgb_common.h head/sys/dev/cxgb/common/cxgb_t3_hw.c head/sys/dev/cxgb/cxgb_adapter.h head/sys/dev/cxgb/cxgb_main.c head/sys/dev/cxgb/cxgb_sge.c Modified: head/sys/dev/cxgb/common/cxgb_common.h ============================================================================== --- head/sys/dev/cxgb/common/cxgb_common.h Wed Mar 31 00:26:35 2010 (r205949) +++ head/sys/dev/cxgb/common/cxgb_common.h Wed Mar 31 00:27:49 2010 (r205950) @@ -314,6 +314,7 @@ struct qset_params { / unsigned int rspq_size; /* # of entries in response queue */ unsigned int fl_size; /* # of entries in regular free list */ unsigned int jumbo_size; /* # of entries in jumbo free list */ + unsigned int jumbo_buf_size; /* buffer size of jumbo entry */ unsigned int txq_size[SGE_TXQ_PER_SET]; /* Tx queue sizes */ unsigned int cong_thres; /* FL congestion threshold */ unsigned int vector; /* Interrupt (line or vector) number */ Modified: head/sys/dev/cxgb/common/cxgb_t3_hw.c ============================================================================== --- head/sys/dev/cxgb/common/cxgb_t3_hw.c Wed Mar 31 00:26:35 2010 (r205949) +++ head/sys/dev/cxgb/common/cxgb_t3_hw.c Wed Mar 31 00:27:49 2010 (r205950) @@ -4467,8 +4467,6 @@ int __devinit t3_prep_adapter(adapter_t if (reset && t3_reset_adapter(adapter)) return -1; - t3_sge_prep(adapter, &adapter->params.sge); - if (adapter->params.vpd.mclk) { struct tp_params *p = &adapter->params.tp; @@ -4497,6 +4495,8 @@ int __devinit t3_prep_adapter(adapter_t t3_mc7_size(&adapter->pmtx) && t3_mc7_size(&adapter->cm); + t3_sge_prep(adapter, &adapter->params.sge); + if (is_offload(adapter)) { adapter->params.mc5.nservers = DEFAULT_NSERVERS; /* PR 6487. TOE and filtering are mutually exclusive */ Modified: head/sys/dev/cxgb/cxgb_adapter.h ============================================================================== --- head/sys/dev/cxgb/cxgb_adapter.h Wed Mar 31 00:26:35 2010 (r205949) +++ head/sys/dev/cxgb/cxgb_adapter.h Wed Mar 31 00:27:49 2010 (r205950) @@ -141,6 +141,8 @@ enum { #define JUMBO_Q_SIZE 1024 #define RSPQ_Q_SIZE 1024 #define TX_ETH_Q_SIZE 1024 +#define TX_OFLD_Q_SIZE 1024 +#define TX_CTRL_Q_SIZE 256 enum { TXQ_ETH = 0, TXQ_OFLD = 1, Modified: head/sys/dev/cxgb/cxgb_main.c ============================================================================== --- head/sys/dev/cxgb/cxgb_main.c Wed Mar 31 00:26:35 2010 (r205949) +++ head/sys/dev/cxgb/cxgb_main.c Wed Mar 31 00:27:49 2010 (r205950) @@ -218,9 +218,9 @@ TUNABLE_INT("hw.cxgb.force_fw_update", & SYSCTL_UINT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0, "update firmware even if up to date"); -int cxgb_use_16k_clusters = 1; +int cxgb_use_16k_clusters = -1; TUNABLE_INT("hw.cxgb.use_16k_clusters", &cxgb_use_16k_clusters); -SYSCTL_UINT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, +SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN, &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue "); /* Modified: head/sys/dev/cxgb/cxgb_sge.c ============================================================================== --- head/sys/dev/cxgb/cxgb_sge.c Wed Mar 31 00:26:35 2010 (r205949) +++ head/sys/dev/cxgb/cxgb_sge.c Wed Mar 31 00:27:49 2010 (r205950) @@ -119,13 +119,9 @@ SYSCTL_UINT(_hw_cxgb, OID_AUTO, tx_recla * we have an m_ext */ static int recycle_enable = 0; -int cxgb_ext_freed = 0; -int cxgb_ext_inited = 0; -int fl_q_size = 0; -int jumbo_q_size = 0; extern int cxgb_use_16k_clusters; -extern int nmbjumbo4; +extern int nmbjumbop; extern int nmbjumbo9; extern int nmbjumbo16; @@ -532,21 +528,30 @@ t3_sge_err_intr_handler(adapter_t *adapt void t3_sge_prep(adapter_t *adap, struct sge_params *p) { - int i, nqsets; + int i, nqsets, fl_q_size, jumbo_q_size, use_16k, jumbo_buf_size; - nqsets = min(SGE_QSETS, mp_ncpus*4); + nqsets = min(SGE_QSETS / adap->params.nports, mp_ncpus); + nqsets *= adap->params.nports; fl_q_size = min(nmbclusters/(3*nqsets), FL_Q_SIZE); while (!powerof2(fl_q_size)) fl_q_size--; + + use_16k = cxgb_use_16k_clusters != -1 ? cxgb_use_16k_clusters : + is_offload(adap); + #if __FreeBSD_version >= 700111 - if (cxgb_use_16k_clusters) + if (use_16k) { jumbo_q_size = min(nmbjumbo16/(3*nqsets), JUMBO_Q_SIZE); - else + jumbo_buf_size = MJUM16BYTES; + } else { jumbo_q_size = min(nmbjumbo9/(3*nqsets), JUMBO_Q_SIZE); + jumbo_buf_size = MJUM9BYTES; + } #else - jumbo_q_size = min(nmbjumbo4/(3*nqsets), JUMBO_Q_SIZE); + jumbo_q_size = min(nmbjumbop/(3*nqsets), JUMBO_Q_SIZE); + jumbo_buf_size = MJUMPAGESIZE; #endif while (!powerof2(jumbo_q_size)) jumbo_q_size--; @@ -555,8 +560,7 @@ t3_sge_prep(adapter_t *adap, struct sge_ device_printf(adap->dev, "Insufficient clusters and/or jumbo buffers.\n"); - /* XXX Does ETHER_ALIGN need to be accounted for here? */ - p->max_pkt_size = adap->sge.qs[0].fl[1].buf_size - sizeof(struct cpl_rx_data); + p->max_pkt_size = jumbo_buf_size - sizeof(struct cpl_rx_data); for (i = 0; i < SGE_QSETS; ++i) { struct qset_params *q = p->qset + i; @@ -574,9 +578,10 @@ t3_sge_prep(adapter_t *adap, struct sge_ q->rspq_size = RSPQ_Q_SIZE; q->fl_size = fl_q_size; q->jumbo_size = jumbo_q_size; + q->jumbo_buf_size = jumbo_buf_size; q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE; - q->txq_size[TXQ_OFLD] = 1024; - q->txq_size[TXQ_CTRL] = 256; + q->txq_size[TXQ_OFLD] = is_offload(adap) ? TX_OFLD_Q_SIZE : 16; + q->txq_size[TXQ_CTRL] = TX_CTRL_Q_SIZE; q->cong_thres = 0; } } @@ -2004,15 +2009,13 @@ t3_free_qset(adapter_t *sc, struct sge_q int i; reclaim_completed_tx(q, 0, TXQ_ETH); - for (i = 0; i < SGE_TXQ_PER_SET; i++) { - if (q->txq[i].txq_mr != NULL) - buf_ring_free(q->txq[i].txq_mr, M_DEVBUF); - if (q->txq[i].txq_ifq != NULL) { - ifq_delete(q->txq[i].txq_ifq); - free(q->txq[i].txq_ifq, M_DEVBUF); - } + if (q->txq[TXQ_ETH].txq_mr != NULL) + buf_ring_free(q->txq[TXQ_ETH].txq_mr, M_DEVBUF); + if (q->txq[TXQ_ETH].txq_ifq != NULL) { + ifq_delete(q->txq[TXQ_ETH].txq_ifq); + free(q->txq[TXQ_ETH].txq_ifq, M_DEVBUF); } - + for (i = 0; i < SGE_RXQ_PER_SET; ++i) { if (q->fl[i].desc) { mtx_lock_spin(&sc->sge.reg_lock); @@ -2549,25 +2552,22 @@ t3_sge_alloc_qset(adapter_t *sc, u_int i MTX_INIT(&q->lock, q->namebuf, NULL, MTX_DEF); q->port = pi; - for (i = 0; i < SGE_TXQ_PER_SET; i++) { - - if ((q->txq[i].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size, - M_DEVBUF, M_WAITOK, &q->lock)) == NULL) { - device_printf(sc->dev, "failed to allocate mbuf ring\n"); - goto err; - } - if ((q->txq[i].txq_ifq = - malloc(sizeof(struct ifaltq), M_DEVBUF, M_NOWAIT|M_ZERO)) - == NULL) { - device_printf(sc->dev, "failed to allocate ifq\n"); - goto err; - } - ifq_init(q->txq[i].txq_ifq, pi->ifp); - callout_init(&q->txq[i].txq_timer, 1); - callout_init(&q->txq[i].txq_watchdog, 1); - q->txq[i].txq_timer.c_cpu = id % mp_ncpus; - q->txq[i].txq_watchdog.c_cpu = id % mp_ncpus; + if ((q->txq[TXQ_ETH].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size, + M_DEVBUF, M_WAITOK, &q->lock)) == NULL) { + device_printf(sc->dev, "failed to allocate mbuf ring\n"); + goto err; } + if ((q->txq[TXQ_ETH].txq_ifq = malloc(sizeof(struct ifaltq), M_DEVBUF, + M_NOWAIT | M_ZERO)) == NULL) { + device_printf(sc->dev, "failed to allocate ifq\n"); + goto err; + } + ifq_init(q->txq[TXQ_ETH].txq_ifq, pi->ifp); + callout_init(&q->txq[TXQ_ETH].txq_timer, 1); + callout_init(&q->txq[TXQ_ETH].txq_watchdog, 1); + q->txq[TXQ_ETH].txq_timer.c_cpu = id % mp_ncpus; + q->txq[TXQ_ETH].txq_watchdog.c_cpu = id % mp_ncpus; + init_qset_cntxt(q, id); q->idx = id; if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), @@ -2632,21 +2632,22 @@ t3_sge_alloc_qset(adapter_t *sc, u_int i q->fl[0].buf_size = MCLBYTES; q->fl[0].zone = zone_pack; q->fl[0].type = EXT_PACKET; -#if __FreeBSD_version > 800000 - if (cxgb_use_16k_clusters) { - q->fl[1].buf_size = MJUM16BYTES; + + if (p->jumbo_buf_size == MJUM16BYTES) { q->fl[1].zone = zone_jumbo16; q->fl[1].type = EXT_JUMBO16; - } else { - q->fl[1].buf_size = MJUM9BYTES; + } else if (p->jumbo_buf_size == MJUM9BYTES) { q->fl[1].zone = zone_jumbo9; q->fl[1].type = EXT_JUMBO9; + } else if (p->jumbo_buf_size == MJUMPAGESIZE) { + q->fl[1].zone = zone_jumbop; + q->fl[1].type = EXT_JUMBOP; + } else { + KASSERT(0, ("can't deal with jumbo_buf_size %d.", p->jumbo_buf_size)); + ret = EDOOFUS; + goto err; } -#else - q->fl[1].buf_size = MJUMPAGESIZE; - q->fl[1].zone = zone_jumbop; - q->fl[1].type = EXT_JUMBOP; -#endif + q->fl[1].buf_size = p->jumbo_buf_size; /* Allocate and setup the lro_ctrl structure */ q->lro.enabled = !!(pi->ifp->if_capenable & IFCAP_LRO);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201003310027.o2V0RoFn040323>