Date: Mon, 2 Apr 2007 01:41:13 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 117154 for review Message-ID: <200704020141.l321fDGT081316@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=117154 Change 117154 by kmacy@kmacy_vt-x:opentoe_init on 2007/04/02 01:40:58 convert receive processing over to using mbuf_iovec disable lro enabling until it can be re-written appropriately explode mbuf_iovec into traditional mbuf chain before calling if_input Affected files ... .. //depot/projects/opentoe/sys/dev/cxgb/cxgb_adapter.h#5 edit .. //depot/projects/opentoe/sys/dev/cxgb/cxgb_lro.c#2 edit .. //depot/projects/opentoe/sys/dev/cxgb/cxgb_osdep.h#4 edit .. //depot/projects/opentoe/sys/dev/cxgb/cxgb_sge.c#4 edit Differences ... ==== //depot/projects/opentoe/sys/dev/cxgb/cxgb_adapter.h#5 (text+ko) ==== @@ -115,7 +115,7 @@ }; struct sge_lro_session { - struct t3_mbuf_hdr mh; + struct mbuf *m; uint32_t seq; uint16_t ip_len; }; @@ -154,7 +154,7 @@ uint32_t cntxt_id; bus_dma_tag_t desc_tag; bus_dmamap_t desc_map; - struct t3_mbuf_hdr mh; + struct mbuf *m; struct mtx lock; }; @@ -403,7 +403,7 @@ int t3_sge_init_sw(adapter_t *); void t3_sge_deinit_sw(adapter_t *); -void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh, +void t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro); void t3_rx_eth(struct port_info *p, struct sge_rspq *rq, struct mbuf *m, int ethpad); void t3_sge_lro_flush_all(adapter_t *adap, struct sge_qset *qs); ==== //depot/projects/opentoe/sys/dev/cxgb/cxgb_lro.c#2 (text+ko) ==== @@ -65,22 +65,17 @@ #endif #ifdef DEBUG -#define MBUF_HEADER_CHECK(mh) do { \ - struct mbuf *head = mh->mh_head; \ - struct mbuf *tail = mh->mh_tail; \ - if (head->m_len == 0 || head->m_pkthdr.len == 0 \ - || (head->m_flags & M_PKTHDR) == 0) \ +#define MBUF_HEADER_CHECK(m) do { \ + if (m->m_len == 0 || m->m_pkthdr.len == 0 \ + || (m->m_flags & M_PKTHDR) == 0) \ panic("lro_flush_session - mbuf len=%d pktlen=%d flags=0x%x\n", \ - head->m_len, head->m_pkthdr.len, head->m_flags); \ - if ((head->m_flags & M_PKTHDR) == 0) \ + m->m_len, m->m_pkthdr.len, m->m_flags); \ + if (m->m_flags & M_PKTHDR) == 0) \ panic("first mbuf is not packet header - flags=0x%x\n", \ - head->m_flags); \ - if ((head == tail && head->m_len != head->m_pkthdr.len)) \ - panic("len=%d pktlen=%d mismatch\n", \ - head->m_len, head->m_pkthdr.len); \ - if (head->m_len < ETHER_HDR_LEN || head->m_pkthdr.len < ETHER_HDR_LEN) \ + m->m_flags); \ + if (m->m_len < ETHER_HDR_LEN || m->m_pkthdr.len < ETHER_HDR_LEN) \ panic("packet too small len=%d pktlen=%d\n", \ - head->m_len, head->m_pkthdr.len);\ + m->m_len, m->m_pkthdr.len);\ } while (0) #else #define MBUF_HEADER_CHECK(m) @@ -100,7 +95,7 @@ lro_match_session(struct sge_lro_session *s, struct ip *ih, struct tcphdr *th) { - struct ip *sih = (struct ip *)(s->mh.mh_head->m_data + IPH_OFFSET); + struct ip *sih = (struct ip *)(s->m->m_data + IPH_OFFSET); struct tcphdr *sth = (struct tcphdr *) (sih + 1); /* @@ -121,7 +116,7 @@ while (active < l->num_active) { s = lro_session(l, idx); - if (s->mh.mh_head) { + if (s->m) { if (lro_match_session(s, ih, th)) { l->last_s = s; return s; @@ -179,39 +174,37 @@ } static __inline void -lro_new_session_init(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) +lro_new_session_init(struct sge_lro_session *s, struct mbuf *m) { - struct ip *ih = (struct ip *)(mh->mh_head->m_data + IPH_OFFSET); + struct ip *ih = (struct ip *)(m->m_data + IPH_OFFSET); struct tcphdr *th = (struct tcphdr *) (ih + 1); int ip_len = ntohs(ih->ip_len); - DPRINTF("%s(s=%p, mh->mh_head=%p, mh->mh_tail=%p)\n", __FUNCTION__, - s, mh->mh_head, mh->mh_tail); + DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m); + + s->m = m; - *&(s->mh) = *mh; - - MBUF_HEADER_CHECK(mh); + MBUF_HEADER_CHECK(m); s->ip_len = ip_len; s->seq = ntohl(th->th_seq) + ip_len - sizeof(*ih) - (th->th_off << 2); } static void -lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct t3_mbuf_hdr *mh) +lro_flush_session(struct sge_qset *qs, struct sge_lro_session *s, struct mbuf *m) { struct sge_lro *l = &qs->lro; - struct t3_mbuf_hdr *smh = &s->mh; - struct ip *ih = (struct ip *)(smh->mh_head->m_data + IPH_OFFSET); + struct mbuf *sm = s->m; + struct ip *ih = (struct ip *)(sm->m_data + IPH_OFFSET); DPRINTF("%s(qs=%p, s=%p, ", __FUNCTION__, qs, s); - if (mh) - DPRINTF("mh->mh_head=%p, mh->mh_tail=%p)\n", - mh->mh_head, mh->mh_tail); + if (m) + DPRINTF("m=%p)\n", m); else - DPRINTF("mh=NULL)\n"); + DPRINTF("m=NULL)\n"); ih->ip_len = htons(s->ip_len); ih->ip_sum = 0; @@ -219,15 +212,14 @@ MBUF_HEADER_CHECK(smh); - smh->mh_head->m_flags |= M_LRO; - t3_rx_eth(qs->port, &qs->rspq, smh->mh_head, 2); + sm->m_flags |= M_LRO; + t3_rx_eth(qs->port, &qs->rspq, sm, 2); - if (mh) { - *smh = *mh; - lro_new_session_init(s, mh); + if (m) { + s->m = m; + lro_new_session_init(s, m); } else { - smh->mh_head = NULL; - smh->mh_tail = NULL; + s->m = NULL; l->num_active--; } @@ -235,23 +227,23 @@ } static __inline struct sge_lro_session * -lro_new_session(struct sge_qset *qs, struct t3_mbuf_hdr *mh, uint32_t rss_hash) +lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash) { struct sge_lro *l = &qs->lro; int idx = LRO_SESSION_IDX_HINT_HASH(rss_hash); struct sge_lro_session *s = lro_session(l, idx); - DPRINTF("%s(qs=%p, mh->mh_head=%p, mh->mh_tail=%p, rss_hash=0x%x)\n", __FUNCTION__, - qs, mh->mh_head, mh->mh_tail, rss_hash); + DPRINTF("%s(qs=%p, m=%p, rss_hash=0x%x)\n", __FUNCTION__, + qs, m, rss_hash); - if (__predict_true(!s->mh.mh_head)) + if (__predict_true(!s->m)) goto done; if (l->num_active > MAX_LRO_PER_QSET) panic("MAX_LRO_PER_QSET exceeded"); if (l->num_active == MAX_LRO_PER_QSET) { - lro_flush_session(qs, s, mh); + lro_flush_session(qs, s, m); qs->port_stats[SGE_PSTATS_LRO_X_STREAMS]++; return s; } @@ -259,24 +251,21 @@ while (1) { LRO_IDX_INC(idx); s = lro_session(l, idx); - if (!s->mh.mh_head) + if (!s->m) break; } done: - lro_new_session_init(s, mh); - + lro_new_session_init(s, m); l->num_active++; - return s; - + return s; } static __inline int -lro_update_session(struct sge_lro_session *s, struct t3_mbuf_hdr *mh) +lro_update_session(struct sge_lro_session *s, struct mbuf *m) { - struct mbuf *m = mh->mh_head; - struct t3_mbuf_hdr *smh = &s->mh; - struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(smh->mh_head->m_data + 2); + struct mbuf *sm = s->m; + struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(sm->m_data + 2); struct cpl_rx_pkt *ncpl = (struct cpl_rx_pkt *)(m->m_data + 2); struct ip *nih = (struct ip *)(m->m_data + IPH_OFFSET); struct tcphdr *th, *nth = (struct tcphdr *)(nih + 1); @@ -284,8 +273,7 @@ int plen, tcpiphlen, olen = (nth->th_off << 2) - sizeof (*nth); - DPRINTF("%s(s=%p, mh->mh_head=%p, mh->mh_tail=%p)\n", __FUNCTION__, - s, mh->mh_head, mh->mh_tail); + DPRINTF("%s(s=%p, m=%p)\n", __FUNCTION__, s, m); if (cpl->vlan_valid && cpl->vlan != ncpl->vlan) { return -1; } @@ -295,7 +283,7 @@ } MBUF_HEADER_CHECK(smh); - th = (struct tcphdr *)(smh->mh_head->m_data + IPH_OFFSET + sizeof (struct ip)); + th = (struct tcphdr *)(sm->m_data + IPH_OFFSET + sizeof (struct ip)); if (olen) { uint32_t *ptr = (uint32_t *)(th + 1); @@ -315,7 +303,13 @@ plen = ntohs(nih->ip_len) - tcpiphlen; s->seq += plen; s->ip_len += plen; - smh->mh_head->m_pkthdr.len += plen; + sm->m_pkthdr.len += plen; + + /* + * XXX FIX ME + * + * + */ #if 0 /* XXX this I *do not* understand */ @@ -323,8 +317,8 @@ skb_shinfo(s->skb)->gso_size = plen; #endif #if __FreeBSD_version > 700000 - if (plen > smh->mh_head->m_pkthdr.tso_segsz) - smh->mh_head->m_pkthdr.tso_segsz = plen; + if (plen > sm->m_pkthdr.tso_segsz) + sm->m_pkthdr.tso_segsz = plen; #endif DPRINTF("m_adj(%d)\n", (int)(IPH_OFFSET + tcpiphlen)); m_adj(m, IPH_OFFSET + tcpiphlen); @@ -333,9 +327,7 @@ skb_shinfo(s->skb)->frag_list = skb; #endif - mh->mh_head->m_flags &= ~M_PKTHDR; - smh->mh_tail->m_next = mh->mh_head; - smh->mh_tail = mh->mh_tail; + #if 0 /* @@ -354,10 +346,9 @@ } void -t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct t3_mbuf_hdr *mh, +t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m, int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro) { - struct mbuf *m = mh->mh_head; struct sge_qset *qs = rspq_to_qset(rq); struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(m->m_data + ethpad); struct ether_header *eh = (struct ether_header *)(cpl + 1); @@ -384,12 +375,12 @@ if (__predict_false(!can_lro_tcpsegment(th))) { goto no_lro; } else if (__predict_false(!s)) { - s = lro_new_session(qs, mh, rss_hash); + s = lro_new_session(qs, m, rss_hash); } else { - if (lro_update_session(s, mh)) { - lro_flush_session(qs, s, mh); + if (lro_update_session(s, m)) { + lro_flush_session(qs, s, m); } - if (__predict_false(s->mh.mh_head->m_pkthdr.len + pi->ifp->if_mtu > 65535)) { + if (__predict_false(s->m->m_pkthdr.len + pi->ifp->if_mtu > 65535)) { lro_flush_session(qs, s, NULL); } } @@ -417,7 +408,7 @@ s = lro_session(l, idx); while (active < num_active) { - if (s->mh.mh_head) { + if (s->m) { lro_flush_session(qs, s, NULL); active++; } ==== //depot/projects/opentoe/sys/dev/cxgb/cxgb_osdep.h#4 (text+ko) ==== @@ -52,11 +52,6 @@ struct sge_rspq; -struct t3_mbuf_hdr { - struct mbuf *mh_head; - struct mbuf *mh_tail; -}; - #if __FreeBSD_version > 700030 #define INTR_FILTERS #define FIRMWARE_LATEST ==== //depot/projects/opentoe/sys/dev/cxgb/cxgb_sge.c#4 (text+ko) ==== @@ -268,42 +268,36 @@ * Return a packet containing the immediate data of the given response. */ static __inline int -get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct t3_mbuf_hdr *mh) +get_imm_packet(adapter_t *sc, const struct rsp_desc *resp, struct mbuf *m) { - struct mbuf *m; - int len; + int len, idx; uint32_t flags = ntohl(resp->flags); uint8_t sopeop = G_RSPD_SOP_EOP(flags); - + struct mbuf_iovec *iov; + /* * would be a firmware bug */ if (sopeop == RSPQ_NSOP_NEOP || sopeop == RSPQ_SOP) - return (0); + panic("invalid hw response for sopeop\n"); - m = m_gethdr(M_NOWAIT, MT_DATA); len = G_RSPD_LEN(ntohl(resp->len_cq)); - - if (m) { + + switch (sopeop) { + case RSPQ_SOP_EOP: + m->m_len = m->m_pkthdr.len = len; + m->m_flags |= M_PKTHDR; + memcpy(m->m_data, resp->imm_data, IMMED_PKT_SIZE); MH_ALIGN(m, IMMED_PKT_SIZE); - memcpy(m->m_data, resp->imm_data, IMMED_PKT_SIZE); - m->m_len = len; - - switch (sopeop) { - case RSPQ_SOP_EOP: - mh->mh_head = mh->mh_tail = m; - m->m_pkthdr.len = len; - m->m_flags |= M_PKTHDR; - break; - case RSPQ_EOP: - m->m_flags &= ~M_PKTHDR; - mh->mh_head->m_pkthdr.len += len; - mh->mh_tail->m_next = m; - mh->mh_tail = m; - break; - } + break; + case RSPQ_EOP: + m->m_pkthdr.len += len; + iov = mtoiov(m); + idx = iov->mi_count + iov->mi_first - 1; + iov->mi_lens[idx] = len; + memcpy(iov->mi_bases[idx], resp->imm_data, IMMED_PKT_SIZE); + break; } - return (m != NULL); } @@ -862,7 +856,6 @@ err, m0->m_pkthdr.len, n); #endif } - if (err == EFBIG) { /* Too many segments, try to defrag */ @@ -1706,8 +1699,7 @@ #define NOMEM_INTR_DELAY 2500 static __inline void -deliver_partial_bundle(struct t3cdev *tdev, - struct sge_rspq *q) +deliver_partial_bundle(struct t3cdev *tdev, struct sge_rspq *q) { ; } @@ -1750,14 +1742,12 @@ panic("bad port index %d m->m_data=%p\n", cpl->iff, m->m_data); - m_adj(m, sizeof(*cpl) + ethpad); - - if ((ifp->if_capenable & IFCAP_RXCSUM) && !cpl->fragment && cpl->csum_valid && cpl->csum == 0xffff) { m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); rspq_to_qset(rq)->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++; - m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID|CSUM_DATA_VALID|CSUM_PSEUDO_HDR); + m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID| + CSUM_DATA_VALID|CSUM_PSEUDO_HDR); m->m_pkthdr.csum_data = 0xffff; } /* @@ -1771,6 +1761,35 @@ #endif m->m_pkthdr.rcvif = ifp; + if (m->m_flags & M_IOVEC) { + int i, offset, type; + struct mbuf_iovec *iov; + void *cl; + struct mbuf *m0, *head = NULL; + + iov = mtoiov(m); + for (i = iov->mi_count + iov->mi_first - 1; + i > iov->mi_first; i--) { + cl = iov->mi_bases[i]; + m0 = m_get(M_NOWAIT, MT_DATA); + m0->m_flags = 0; + type = mbuf_iovec_get_type(iov, i); + m_cljset(m0, (uint8_t *)cl, type); + m->m_data += iov->mi_offsets[i]; + m0->m_len = iov->mi_lens[i]; + m0->m_next = head; + head = m0; + } + cl = iov->mi_bases[0]; + offset = iov->mi_offsets[0]; + type = mbuf_iovec_get_type(iov, 0); + m->m_flags &= ~(M_IOVEC); + m_cljset(m, cl, type); + m->m_next = head; + } + + m_adj(m, sizeof(*cpl) + ethpad); + (*ifp->if_input)(ifp, m); } @@ -1791,11 +1810,9 @@ * be copied but there is no memory for the copy. */ -#include <vm/vm.h> -#include <vm/pmap.h> static int get_packet(adapter_t *adap, unsigned int drop_thres, struct sge_qset *qs, - struct t3_mbuf_hdr *mh, struct rsp_desc *r, struct mbuf *m) + struct mbuf *m, struct rsp_desc *r) { unsigned int len_cq = ntohl(r->len_cq); @@ -1804,6 +1821,7 @@ uint32_t len = G_RSPD_LEN(len_cq); uint32_t flags = ntohl(r->flags); uint8_t sopeop = G_RSPD_SOP_EOP(flags); + struct mbuf_iovec *iov; int ret = 0; prefetch(sd->cl); @@ -1812,47 +1830,37 @@ bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(fl->entry_tag, sd->map); - m_cljset(m, sd->cl, fl->buf_size); - m->m_len = len; - switch(sopeop) { case RSPQ_SOP_EOP: DBG(DBG_RX, ("get_packet: SOP-EOP m %p\n", m)); - mh->mh_head = mh->mh_tail = m; - m->m_pkthdr.len = len; + m_cljset(m, sd->cl, fl->type); + m->m_len = m->m_pkthdr.len = len; m->m_flags |= M_PKTHDR; ret = 1; + goto done; break; case RSPQ_NSOP_NEOP: DBG(DBG_RX, ("get_packet: NO_SOP-NO_EOP m %p\n", m)); - m->m_flags &= ~M_PKTHDR; - if (mh->mh_tail == NULL) { - if (cxgb_debug) - printf("discarding intermediate descriptor entry\n"); - m_freem(m); - break; - } - mh->mh_tail->m_next = m; - mh->mh_tail = m; - mh->mh_head->m_pkthdr.len += len; + m->m_pkthdr.len += len; ret = 0; break; case RSPQ_SOP: DBG(DBG_RX, ("get_packet: SOP m %p\n", m)); - m->m_pkthdr.len = len; - mh->mh_head = mh->mh_tail = m; - m->m_flags |= M_PKTHDR; + m->m_len = m->m_pkthdr.len = len; + m->m_flags |= (M_PKTHDR|M_IOVEC); + iov = mtoiov(m); + iov->mi_first = iov->mi_count = 0; ret = 0; break; case RSPQ_EOP: DBG(DBG_RX, ("get_packet: EOP m %p\n", m)); - m->m_flags &= ~M_PKTHDR; - mh->mh_head->m_pkthdr.len += len; - mh->mh_tail->m_next = m; - mh->mh_tail = m; + m->m_pkthdr.len += len; ret = 1; break; } + m_iovappend(m, sd->cl, fl->buf_size, len); + +done: if (++fl->cidx == fl->size) fl->cidx = 0; @@ -1978,28 +1986,37 @@ printf("async notification\n"); } else if (flags & F_RSPD_IMM_DATA_VALID) { + struct mbuf *m = NULL; + if (cxgb_debug) printf("IMM DATA VALID\n"); - - if(get_imm_packet(adap, r, &rspq->mh) == 0) { + if (rspq->m == NULL) { + rspq->m = m_gethdr(M_NOWAIT, MT_DATA); + } else { + m = m_gethdr(M_NOWAIT, MT_DATA); + if (m) + m_iovappend(rspq->m, m, MSIZE, 0); + } + if (rspq->m == NULL || m == NULL) { rspq->next_holdoff = NOMEM_INTR_DELAY; budget_left--; break; - } else { - eop = 1; - } - + } + get_imm_packet(adap, r, rspq->m); + eop = 1; rspq->imm_data++; } else if (r->len_cq) { int drop_thresh = eth ? SGE_RX_DROP_THRES : 0; - struct mbuf *m = m_gethdr(M_NOWAIT, MT_DATA); - if (m == NULL) { + if (rspq->m == NULL) { + rspq->m = m_gethdr(M_NOWAIT, MT_DATA); + rspq->m->m_flags = 0; + } else if (rspq->m == NULL) { log(LOG_WARNING, "failed to get mbuf for packet\n"); break; } ethpad = 2; - eop = get_packet(adap, drop_thresh, qs, &rspq->mh, r, m); + eop = get_packet(adap, drop_thresh, qs, rspq->m, r); } else { DPRINTF("pure response\n"); rspq->pure_rsps++; @@ -2024,14 +2041,14 @@ } if (eop) { - prefetch(rspq->mh.mh_head->m_data); - prefetch(rspq->mh.mh_head->m_data + L1_CACHE_BYTES); + prefetch(rspq->m->m_data); + prefetch(rspq->m->m_data + L1_CACHE_BYTES); if (eth) { - t3_rx_eth_lro(adap, rspq, &rspq->mh, ethpad, + t3_rx_eth_lro(adap, rspq, rspq->m, ethpad, rss_hash, rss_csum, lro); - rspq->mh.mh_tail = rspq->mh.mh_head = NULL; + rspq->m = NULL; } else { #ifdef notyet if (__predict_false(r->rss_hdr.opcode == CPL_TRACE_PKT)) @@ -2167,11 +2184,18 @@ mtx_unlock(&rspq->lock); } +/* + * broken by recent mbuf changes + */ static int t3_lro_enable(SYSCTL_HANDLER_ARGS) { adapter_t *sc; int i, j, enabled, err, nqsets = 0; + +#ifndef LRO_WORKING + return (0); +#endif sc = arg1; enabled = sc->sge.qs[0].lro.enabled; @@ -2252,13 +2276,11 @@ "firmware_version", CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); - SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW, sc, 0, t3_lro_enable, "I", "enable large receive offload"); - SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "intr_coal", CTLTYPE_INT|CTLFLAG_RW, sc,
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200704020141.l321fDGT081316>