From owner-p4-projects@FreeBSD.ORG Wed Sep 26 05:15:21 2007 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 4647F16A469; Wed, 26 Sep 2007 05:15:21 +0000 (UTC) Delivered-To: perforce@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id E714116A421 for ; Wed, 26 Sep 2007 05:15:20 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (repoman.freebsd.org [IPv6:2001:4f8:fff6::29]) by mx1.freebsd.org (Postfix) with ESMTP id D7E7413C469 for ; Wed, 26 Sep 2007 05:15:20 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.14.1/8.14.1) with ESMTP id l8Q5FKHw057768 for ; Wed, 26 Sep 2007 05:15:20 GMT (envelope-from kmacy@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.14.1/8.14.1/Submit) id l8Q5FK4I057765 for perforce@freebsd.org; Wed, 26 Sep 2007 05:15:20 GMT (envelope-from kmacy@freebsd.org) Date: Wed, 26 Sep 2007 05:15:20 GMT Message-Id: <200709260515.l8Q5FK4I057765@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to kmacy@freebsd.org using -f From: Kip Macy To: Perforce Change Reviews Cc: Subject: PERFORCE change 126815 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 26 Sep 2007 05:15:21 -0000 http://perforce.freebsd.org/chv.cgi?CH=126815 Change 126815 by kmacy@kmacy_home:ethng on 2007/09/26 05:14:21 add rudimentary cluster cache move buf_stack functions to cxgb_support.c Affected files ... .. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_osdep.h#10 edit .. //depot/projects/ethng/src/sys/dev/cxgb/cxgb_sge.c#24 edit .. //depot/projects/ethng/src/sys/dev/cxgb/sys/cxgb_support.c#2 edit .. //depot/projects/ethng/src/sys/dev/cxgb/sys/mvec.h#6 edit .. //depot/projects/ethng/src/sys/dev/cxgb/sys/uipc_mvec.c#6 edit Differences ... ==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_osdep.h#10 (text+ko) ==== @@ -117,37 +117,11 @@ return (m); } -struct buf_stack { - caddr_t *bs_stack; - volatile int bs_head; - int bs_size; -}; - -static __inline int -buf_push(struct buf_stack *bs, caddr_t buf) -{ - if (bs->bs_head + 1 >= bs->bs_size) - return (1); - - bs->bs_stack[++(bs->bs_head)] = buf; - return (0); -} - -static __inline caddr_t -buf_pop(struct buf_stack *bs) -{ - if (bs->bs_head < 0) - return (NULL); - - return (bs->bs_stack[(bs->bs_head)--]); -} - #define PANIC_IF(exp) do { \ if (exp) \ panic("BUG: %s", exp); \ } while (0) - #define m_get_priority(m) ((uintptr_t)(m)->m_pkthdr.rcvif) #define m_set_priority(m, pri) ((m)->m_pkthdr.rcvif = (struct ifnet *)((uintptr_t)pri)) ==== //depot/projects/ethng/src/sys/dev/cxgb/cxgb_sge.c#24 (text+ko) ==== @@ -550,7 +550,7 @@ /* * We only allocate a cluster, mbuf allocation happens after rx */ - if ((cl = m_cljget(NULL, M_DONTWAIT, q->zone)) == NULL) { + if ((cl = cxgb_cache_get(q->zone)) == NULL) { log(LOG_WARNING, "Failed to allocate cluster\n"); goto done; } @@ -830,6 +830,7 @@ callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc); mi_init(); + cxgb_cache_init(); return (0); } @@ -1113,7 +1114,6 @@ sgp->len[idx] = htobe32(segs[i].ds_len); sgp->addr[idx] = htobe64(segs[i].ds_addr); - idx ^= 1; } @@ -1162,8 +1162,6 @@ #endif } - - /** * write_wr_hdr_sgl - write a WR header and, optionally, SGL * @ndesc: number of Tx descriptors spanned by the SGL @@ -1181,7 +1179,6 @@ * and we just need to write the WR header. Otherwise we distribute the * SGL across the number of descriptors it spans. */ - static void write_wr_hdr_sgl(unsigned int ndesc, struct tx_desc *txd, struct txq_state *txqs, const struct sge_txq *txq, const struct sg_ent *sgl, unsigned int flits, @@ -1253,8 +1250,20 @@ static void dump_mi(struct mbuf_iovec *mi) { - DPRINTF("mi_flags=0x%08x mi_data=%p mi_len=%d mi_type=%d\n", + int i; + struct mbuf_vec *mv; + + printf("mi_flags=0x%08x mi_data=%p mi_len=%d mi_type=%d\n", mi->mi_flags, mi->mi_base + mi->mi_offset, mi->mi_len, mi->mi_type); + + if (mi->mi_type == EXT_CLIOVEC || + mi->mi_type == EXT_IOVEC) { + mv = mtomv((struct mbuf *)mi->mi_base); + mi = mv->mv_vec; + for (i = 0; i < mv->mv_count; i++, mi++) + dump_mi(mi); + + } } /* sizeof(*eh) + sizeof(*vhdr) + sizeof(*ip) + sizeof(*tcp) */ @@ -1340,7 +1349,6 @@ txsd->mi.mi_len = m0->m_pkthdr.len; } mi = &txsd->mi; - dump_mi(&txsd->mi); if (count > 1) { struct cpl_tx_pkt_batch *cpl_batch = (struct cpl_tx_pkt_batch *)txd; @@ -1378,7 +1386,12 @@ struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)txd; struct ip *ip; struct tcphdr *tcp; - char *pkthdr; + char *pkthdr, tmp[TCPPKTHDRSIZE]; + struct mbuf_vec *mv; + struct mbuf_iovec *tmpmi; + + mv = mtomv(m0); + tmpmi = mv->mv_vec; txd->flit[2] = 0; GET_VTAG_MI(cntrl, mi); @@ -1388,17 +1401,14 @@ hdr->len = htonl(mlen | 0x80000000); DPRINTF("tso buf len=%d\n", mlen); - if (__predict_false(mi->mi_len < TCPPKTHDRSIZE)) { - /* - * XXX - * - */ - pkthdr = NULL; + if (__predict_false(tmpmi->mi_len < TCPPKTHDRSIZE)) { + pkthdr = tmp; + dump_mi(mi); panic("discontig packet - fixxorz"); } else pkthdr = m0->m_data; - if (__predict_false(mi->mi_flags & M_VLANTAG)) { + if (__predict_false(m0->m_flags & M_VLANTAG)) { eth_type = CPL_ETH_II_VLAN; ip = (struct ip *)(pkthdr + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); @@ -1417,7 +1427,7 @@ } else { struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)txd; - GET_VTAG_MI(cntrl, mi); + GET_VTAG(cntrl, m0); cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT); cpl->cntrl = htonl(cntrl); mlen = m0->m_pkthdr.len; @@ -1484,7 +1494,7 @@ check_ring_tx_db(pi->adapter, txq); if ((m0->m_type == MT_DATA) && ((m0->m_flags & (M_EXT|M_NOFREE)) == M_EXT)) { - m0->m_flags = 0; + m0->m_flags &= ~M_EXT ; m_free(m0); } @@ -2267,7 +2277,7 @@ q->txq[i].txq_mr.mr_size = cxgb_txq_mbuf_ring_size; mtx_init(&q->txq[i].txq_mr.mr_lock, "txq mbuf ring", NULL, MTX_DEF); } - + init_qset_cntxt(q, id); if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), ==== //depot/projects/ethng/src/sys/dev/cxgb/sys/cxgb_support.c#2 (text+ko) ==== @@ -52,8 +52,53 @@ #include #endif -int -buf_init(struct buf_stack *bs, int size) +struct buf_stack { + caddr_t *bs_stack; + volatile int bs_head; + int bs_size; +}; + +static __inline int +buf_stack_push(struct buf_stack *bs, caddr_t buf) +{ + if (bs->bs_head + 1 >= bs->bs_size) + return (ENOSPC); + + bs->bs_stack[++(bs->bs_head)] = buf; + return (0); +} + +static __inline caddr_t +buf_stack_pop(struct buf_stack *bs) +{ + if (bs->bs_head < 0) + return (NULL); + + return (bs->bs_stack[(bs->bs_head)--]); +} + +/* + * Stack is more than half full + * we can free some elements to make room + */ +static __inline int +buf_stack_canfree(struct buf_stack *bs) +{ + return (bs->bs_head > (bs->bs_size>>1)); +} + +struct cxgb_cache_pcpu { + struct buf_stack ccp_jumbo_free; + struct buf_stack ccp_cluster_free; + uma_zone_t ccp_jumbo_zone; +}; + +struct cxgb_cache_system { + struct cxgb_cache_pcpu ccs_array[0]; +} *cxgb_caches; + +static int +buf_stack_init(struct buf_stack *bs, int size) { bs->bs_size = size; bs->bs_head = -1; @@ -63,3 +108,162 @@ return (0); } +static void +buf_stack_deinit(struct buf_stack *bs) +{ + if (bs->bs_stack != NULL) + free(bs->bs_stack, M_DEVBUF); +} + +static int +cxgb_cache_pcpu_init(struct cxgb_cache_pcpu *ccp) +{ + int err; + + if ((err = buf_stack_init(&ccp->ccp_jumbo_free, 2*JUMBO_Q_SIZE))) + return (err); + + if ((err = buf_stack_init(&ccp->ccp_cluster_free, 2*FL_Q_SIZE))) + return (err); + + if (jumbo_phys_contig) + ccp->ccp_jumbo_zone = zone_jumbo9; + else + ccp->ccp_jumbo_zone = zone_jumbop; + + return (0); +} + +static void +cxgb_cache_pcpu_deinit(struct cxgb_cache_pcpu *ccp) +{ + /* + * XXX free clusters + */ + + buf_stack_deinit(&ccp->ccp_jumbo_free); + buf_stack_deinit(&ccp->ccp_cluster_free); + +} + +static int inited = 0; + +int +cxgb_cache_init(void) +{ + int i, err; + + if (inited++ > 0) + return (0); + + if ((cxgb_caches = malloc(sizeof(struct cxgb_cache_pcpu)*mp_ncpus, M_DEVBUF, M_WAITOK|M_ZERO)) == NULL) + return (ENOMEM); + + for (i = 0; i < mp_ncpus; i++) + if ((err = cxgb_cache_pcpu_init(&cxgb_caches->ccs_array[i]))) + goto err; + + return (0); +err: + cxgb_cache_flush(); + + return (err); +} + +void +cxgb_cache_flush(void) +{ + int i; + + if (--inited > 0) + return; + + if (cxgb_caches == NULL) + return; + + for (i = 0; i < mp_ncpus; i++) + cxgb_cache_pcpu_deinit(&cxgb_caches->ccs_array[i]); + + free(cxgb_caches, M_DEVBUF); + cxgb_caches = NULL; +} + +caddr_t +cxgb_cache_get(uma_zone_t zone) +{ + caddr_t cl; + struct cxgb_cache_pcpu *ccp; + + critical_enter(); + ccp = &cxgb_caches->ccs_array[curcpu]; + if (zone == zone_clust) { + cl = buf_stack_pop(&ccp->ccp_cluster_free); + } else { + cl = buf_stack_pop(&ccp->ccp_jumbo_free); + } + critical_exit(); + + if (cl == NULL) + cl = uma_zalloc(zone, M_NOWAIT); + + return (cl); +} + +void +cxgb_cache_put(uma_zone_t zone, void *cl) +{ + struct cxgb_cache_pcpu *ccp; + int err = 0; + + critical_enter(); + ccp = &cxgb_caches->ccs_array[curcpu]; + if (zone == zone_clust) { + err = buf_stack_push(&ccp->ccp_cluster_free, cl); + } else if (zone == ccp->ccp_jumbo_zone){ + err = buf_stack_push(&ccp->ccp_jumbo_free, cl); + } + critical_exit(); + + if (err) + uma_zfree(zone, cl); +} + +void +cxgb_cache_rebalance(void) +{ + struct cxgb_cache_pcpu *ccp; + caddr_t vec[8]; + uma_zone_t zone; + int i, count; + + return; + + + critical_enter(); +restart: + ccp = &cxgb_caches->ccs_array[curcpu]; + zone = ccp->ccp_jumbo_zone; + for (i = 0; i < 8 && buf_stack_canfree(&ccp->ccp_jumbo_free); i++) + vec[i] = buf_stack_pop(&ccp->ccp_jumbo_free); + critical_exit(); + count = i; + for (i = 0; i < count; i++) + uma_zfree(zone, vec[i]); + + critical_enter(); + ccp = &cxgb_caches->ccs_array[curcpu]; + zone = zone_clust; + for (i = 0; i < 8 && buf_stack_canfree(&ccp->ccp_cluster_free); i++) + vec[i] = buf_stack_pop(&ccp->ccp_cluster_free); + critical_exit(); + count = i; + for (i = 0; i < count; i++) + uma_zfree(zone, vec[i]); + + critical_enter(); + ccp = &cxgb_caches->ccs_array[curcpu]; + if (buf_stack_canfree(&ccp->ccp_cluster_free) || buf_stack_canfree(&ccp->ccp_jumbo_free)) + goto restart; + critical_exit(); +} + ==== //depot/projects/ethng/src/sys/dev/cxgb/sys/mvec.h#6 (text+ko) ==== @@ -32,6 +32,16 @@ #ifndef _MVEC_H_ #define _MVEC_H_ +int cxgb_cache_init(void); + +void cxgb_cache_flush(void); + +caddr_t cxgb_cache_get(uma_zone_t zone); + +void cxgb_cache_put(uma_zone_t zone, void *cl); + +void cxgb_cache_rebalance(void); + #define mtomv(m) ((struct mbuf_vec *)((m)->m_pktdat)) #define M_IOVEC 0x100000 /* mbuf immediate data area is used for cluster ptrs */ @@ -164,8 +174,8 @@ case EXT_IOVEC: uma_zfree(zone_miovec, m); break; - case EXT_CLIOVEC: - uma_zfree(zone_clust, m); + case EXT_CLIOVEC: + cxgb_cache_put(zone_clust, m); break; default: panic("unexpected type %d\n", type); ==== //depot/projects/ethng/src/sys/dev/cxgb/sys/uipc_mvec.c#6 (text+ko) ==== @@ -202,7 +202,7 @@ return (ENOMEM); type = EXT_CLIOVEC; } else { - if ((m0 = uma_zalloc_arg(zone_miovec, NULL, M_DONTWAIT)) == NULL) + if ((m0 = uma_zalloc_arg(zone_miovec, NULL, M_NOWAIT)) == NULL) return (ENOMEM); type = EXT_IOVEC; } @@ -223,7 +223,8 @@ if (marray[i]->m_flags & M_EXT) { marray[i]->m_flags = 0; m_free(marray[i]); - } + + } } *nsegs = seg_count; *m = m0; @@ -301,13 +302,13 @@ m_free_fast((struct mbuf *)cl); break; case EXT_CLUSTER: - uma_zfree(zone_clust, cl); + cxgb_cache_put(zone_clust, cl); break; case EXT_JUMBOP: - uma_zfree(zone_jumbop, cl); + cxgb_cache_put(zone_jumbop, cl); break; case EXT_JUMBO9: - uma_zfree(zone_jumbo9, cl); + cxgb_cache_put(zone_jumbo9, cl); break; case EXT_JUMBO16: uma_zfree(zone_jumbo16, cl);