Date: Fri, 2 Dec 2016 20:16:52 +0000 (UTC) From: John Baldwin <jhb@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r309440 - in stable/10/sys/dev/cxgbe: . tom Message-ID: <201612022016.uB2KGq3B099798@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jhb Date: Fri Dec 2 20:16:52 2016 New Revision: 309440 URL: https://svnweb.freebsd.org/changeset/base/309440 Log: MFC 292736: cxgbe(4): Updates to the base NIC driver and t4_tom to support the iSCSI offload driver. These changes come from projects/cxl_iscsi. Note that these changes make use of the mbufq API from 11.0, but that API is not present in 10.x in the same form. Borrow an implementation from the CAM CTL ha code that uses m_nextpkt to implement mbufq for use in 10. Modified: stable/10/sys/dev/cxgbe/adapter.h stable/10/sys/dev/cxgbe/offload.h stable/10/sys/dev/cxgbe/t4_main.c stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c stable/10/sys/dev/cxgbe/tom/t4_ddp.c stable/10/sys/dev/cxgbe/tom/t4_tom.c stable/10/sys/dev/cxgbe/tom/t4_tom.h Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/dev/cxgbe/adapter.h ============================================================================== --- stable/10/sys/dev/cxgbe/adapter.h Fri Dec 2 19:47:23 2016 (r309439) +++ stable/10/sys/dev/cxgbe/adapter.h Fri Dec 2 20:16:52 2016 (r309440) @@ -784,7 +784,7 @@ struct adapter { void *tom_softc; /* (struct tom_data *) */ struct tom_tunables tt; void *iwarp_softc; /* (struct c4iw_dev *) */ - void *iscsi_softc; + void *iscsi_ulp_softc; /* (struct cxgbei_data *) */ struct l2t_data *l2t; /* L2 table */ struct tid_info tids; Modified: stable/10/sys/dev/cxgbe/offload.h ============================================================================== --- stable/10/sys/dev/cxgbe/offload.h Fri Dec 2 19:47:23 2016 (r309439) +++ stable/10/sys/dev/cxgbe/offload.h Fri Dec 2 20:16:52 2016 (r309440) @@ -156,7 +156,7 @@ int t4_register_uld(struct uld_info *); int t4_unregister_uld(struct uld_info *); int t4_activate_uld(struct adapter *, int); int t4_deactivate_uld(struct adapter *, int); -void t4_iscsi_init(struct ifnet *, unsigned int, const unsigned int *); +void t4_iscsi_init(struct adapter *, u_int, const u_int *); int uld_active(struct adapter *, int); #endif #endif Modified: stable/10/sys/dev/cxgbe/t4_main.c ============================================================================== --- stable/10/sys/dev/cxgbe/t4_main.c Fri Dec 2 19:47:23 2016 (r309439) +++ stable/10/sys/dev/cxgbe/t4_main.c Fri Dec 2 20:16:52 2016 (r309440) @@ -8882,11 +8882,8 @@ t4_db_dropped(struct adapter *sc) #ifdef TCP_OFFLOAD void -t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, - const unsigned int *pgsz_order) +t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order) { - struct vi_info *vi = ifp->if_softc; - struct adapter *sc = vi->pi->adapter; t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | Modified: stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c ============================================================================== --- stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c Fri Dec 2 19:47:23 2016 (r309439) +++ stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c Fri Dec 2 20:16:52 2016 (r309440) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012 Chelsio Communications, Inc. + * Copyright (c) 2012, 2015 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar <np@FreeBSD.org> * @@ -71,31 +71,38 @@ VNET_DECLARE(int, tcp_autorcvbuf_inc); VNET_DECLARE(int, tcp_autorcvbuf_max); #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) -/* - * For ULP connections HW may add headers, e.g., for digests, that aren't part - * of the messages sent by the host but that are part of the TCP payload and - * therefore consume TCP sequence space. Tx connection parameters that - * operate in TCP sequence space are affected by the HW additions and need to - * compensate for them to accurately track TCP sequence numbers. This array - * contains the compensating extra lengths for ULP packets. It is indexed by - * a packet's ULP submode. - */ -const unsigned int t4_ulp_extra_len[] = {0, 4, 4, 8}; +static inline struct mbuf * +mbufq_dequeue(struct mbufq *q) +{ + struct mbuf *m; -/* - * Return the length of any HW additions that will be made to a Tx packet. - * Such additions can happen for some types of ULP packets. - */ -static inline unsigned int -ulp_extra_len(struct mbuf *m, int *ulp_mode) + m = q->head; + if (m) { + if (q->tail == m) + q->tail = NULL; + q->head = m->m_nextpkt; + m->m_nextpkt = NULL; + } + return (m); +} + +static inline void +mbufq_enqueue(struct mbufq *q, struct mbuf *m) { - struct m_tag *mtag; - if ((mtag = m_tag_find(m, CXGBE_ISCSI_MBUF_TAG, NULL)) == NULL) - return (0); - *ulp_mode = *((int *)(mtag + 1)); + m->m_nextpkt = NULL; + if (q->tail) + q->tail->m_nextpkt = m; + else + q->head = m; + q->tail = m; +} + +static inline struct mbuf * +mbufq_first(const struct mbufq *q) +{ - return (t4_ulp_extra_len[*ulp_mode & 3]); + return (q->head); } void @@ -386,13 +393,9 @@ t4_rcvd(struct toedev *tod, struct tcpcb KASSERT(toep->sb_cc >= sb->sb_cc, ("%s: sb %p has more data (%d) than last time (%d).", __func__, sb, sb->sb_cc, toep->sb_cc)); - if (toep->ulp_mode == ULP_MODE_ISCSI) { - toep->rx_credits += toep->sb_cc; - toep->sb_cc = 0; - } else { - toep->rx_credits += toep->sb_cc - sb->sb_cc; - toep->sb_cc = sb->sb_cc; - } + toep->rx_credits += toep->sb_cc - sb->sb_cc; + toep->sb_cc = sb->sb_cc; + if (toep->rx_credits > 0 && (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 || (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) || @@ -492,25 +495,16 @@ max_dsgl_nsegs(int tx_credits) static inline void write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, - unsigned int plen, uint8_t credits, int shove, int ulp_mode, int txalign) + unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign) { struct fw_ofld_tx_data_wr *txwr = dst; - unsigned int wr_ulp_mode; txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | V_FW_WR_IMMDLEN(immdlen)); txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | V_FW_WR_LEN16(credits)); - - /* for iscsi, the mode & submode setting is per-packet */ - if (toep->ulp_mode == ULP_MODE_ISCSI) - wr_ulp_mode = V_TX_ULP_MODE(ulp_mode >> 4) | - V_TX_ULP_SUBMODE(ulp_mode & 3); - else - wr_ulp_mode = V_TX_ULP_MODE(toep->ulp_mode); - - txwr->lsodisable_to_flags = htobe32(wr_ulp_mode | V_TX_URG(0) | /*XXX*/ - V_TX_SHOVE(shove)); + txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) | + V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove)); txwr->plen = htobe32(plen); if (txalign > 0) { @@ -618,6 +612,9 @@ t4_push_frames(struct adapter *sc, struc toep->ulp_mode == ULP_MODE_RDMA, ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); + if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) + return; + /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. @@ -802,56 +799,70 @@ t4_push_frames(struct adapter *sc, struc close_conn(sc, toep); } -/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */ +static inline void +rqdrop_locked(struct mbufq *q, int plen) +{ + struct mbuf *m; + + while (plen > 0) { + m = mbufq_dequeue(q); + + /* Too many credits. */ + MPASS(m != NULL); + M_ASSERTPKTHDR(m); + + /* Partial credits. */ + MPASS(plen >= m->m_pkthdr.len); + + plen -= m->m_pkthdr.len; + m_freem(m); + } +} + void -t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop) +t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop) { - struct mbuf *sndptr, *m = NULL; + struct mbuf *sndptr, *m; struct fw_ofld_tx_data_wr *txwr; struct wrqe *wr; - unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; + u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; + u_int adjusted_plen, ulp_submode; struct inpcb *inp = toep->inp; - struct tcpcb *tp; - struct socket *so; - struct sockbuf *sb; - int tx_credits, ulp_len = 0, ulp_mode = 0, qlen = 0; - int shove, compl; - struct ofld_tx_sdesc *txsd; + struct tcpcb *tp = intotcpcb(inp); + int tx_credits, shove; + struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; + struct mbufq *pduq = &toep->ulp_pduq; + static const u_int ulp_extra_len[] = {0, 4, 4, 8}; INP_WLOCK_ASSERT(inp); - if (toep->flags & TPF_ABORT_SHUTDOWN) - return; - - tp = intotcpcb(inp); - so = inp->inp_socket; - sb = &so->so_snd; - txsd = &toep->txsd[toep->txsd_pidx]; - KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); + KASSERT(toep->ulp_mode == ULP_MODE_ISCSI, + ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); + + if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) + return; /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. */ - if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) + if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { + KASSERT(drop == 0, + ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); return; + } - sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen); - if (!qlen) - return; + if (drop) + rqdrop_locked(&toep->ulp_pdu_reclaimq, drop); + + while ((sndptr = mbufq_first(pduq)) != NULL) { + M_ASSERTPKTHDR(sndptr); - do { tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); max_imm = max_imm_payload(tx_credits); max_nsegs = max_dsgl_nsegs(tx_credits); - if (drop) { - t4_cpl_iscsi_callback(toep->td, toep, &drop, - CPL_FW4_ACK); - drop = 0; - } - plen = 0; nsegs = 0; max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ @@ -861,7 +872,10 @@ t4_ulp_push_frames(struct adapter *sc, s nsegs += n; plen += m->m_len; - /* This mbuf sent us _over_ the nsegs limit, return */ + /* + * This mbuf would send us _over_ the nsegs limit. + * Suspend tx because the PDU can't be sent out. + */ if (plen > max_imm && nsegs > max_nsegs) { toep->flags |= TPF_TX_SUSPENDED; return; @@ -869,30 +883,35 @@ t4_ulp_push_frames(struct adapter *sc, s if (max_nsegs_1mbuf < n) max_nsegs_1mbuf = n; - - /* This mbuf put us right at the max_nsegs limit */ - if (plen > max_imm && nsegs == max_nsegs) { - toep->flags |= TPF_TX_SUSPENDED; - return; - } - } - - shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); - /* nothing to send */ - if (plen == 0) { - KASSERT(m == NULL, - ("%s: nothing to send, but m != NULL", __func__)); - break; } if (__predict_false(toep->flags & TPF_FIN_SENT)) panic("%s: excess tx.", __func__); - ulp_len = plen + ulp_extra_len(sndptr, &ulp_mode); + /* + * We have a PDU to send. All of it goes out in one WR so 'm' + * is NULL. A PDU's length is always a multiple of 4. + */ + MPASS(m == NULL); + MPASS((plen & 3) == 0); + MPASS(sndptr->m_pkthdr.len == plen); + + shove = !(tp->t_flags & TF_MORETOCOME); + ulp_submode = mbuf_ulp_submode(sndptr); + MPASS(ulp_submode < nitems(ulp_extra_len)); + + /* + * plen doesn't include header and data digests, which are + * generated and inserted in the right places by the TOE, but + * they do occupy TCP sequence space and need to be accounted + * for. + */ + adjusted_plen = plen + ulp_extra_len[ulp_submode]; if (plen <= max_imm) { /* Immediate data tx */ - wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16), + + wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), toep->ofld_txq); if (wr == NULL) { /* XXX: how will we recover from this? */ @@ -901,16 +920,17 @@ t4_ulp_push_frames(struct adapter *sc, s } txwr = wrtod(wr); credits = howmany(wr->wr_len, 16); - write_tx_wr(txwr, toep, plen, ulp_len, credits, shove, - ulp_mode, 0); + write_tx_wr(txwr, toep, plen, adjusted_plen, credits, + shove, ulp_submode, sc->tt.tx_align); m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); + nsegs = 0; } else { int wr_len; /* DSGL tx */ wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; - wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq); + wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); if (wr == NULL) { /* XXX: how will we recover from this? */ toep->flags |= TPF_TX_SUSPENDED; @@ -918,8 +938,8 @@ t4_ulp_push_frames(struct adapter *sc, s } txwr = wrtod(wr); credits = howmany(wr_len, 16); - write_tx_wr(txwr, toep, 0, ulp_len, credits, shove, - ulp_mode, 0); + write_tx_wr(txwr, toep, 0, adjusted_plen, credits, + shove, ulp_submode, sc->tt.tx_align); write_tx_sgl(txwr + 1, sndptr, m, nsegs, max_nsegs_1mbuf); if (wr_len & 0xf) { @@ -932,28 +952,26 @@ t4_ulp_push_frames(struct adapter *sc, s KASSERT(toep->tx_credits >= credits, ("%s: not enough credits", __func__)); + m = mbufq_dequeue(pduq); + MPASS(m == sndptr); + mbufq_enqueue(&toep->ulp_pdu_reclaimq, m); + toep->tx_credits -= credits; toep->tx_nocompl += credits; toep->plen_nocompl += plen; if (toep->tx_credits <= toep->tx_total * 3 / 8 && - toep->tx_nocompl >= toep->tx_total / 4) - compl = 1; - - if (compl) { + toep->tx_nocompl >= toep->tx_total / 4) { txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); toep->tx_nocompl = 0; toep->plen_nocompl = 0; } - tp->snd_nxt += ulp_len; - tp->snd_max += ulp_len; - /* goto next mbuf */ - sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen); + tp->snd_nxt += adjusted_plen; + tp->snd_max += adjusted_plen; toep->flags |= TPF_TX_DATA_SENT; - if (toep->tx_credits < MIN_OFLD_TX_CREDITS) { + if (toep->tx_credits < MIN_OFLD_TX_CREDITS) toep->flags |= TPF_TX_SUSPENDED; - } KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); txsd->plen = plen; @@ -966,10 +984,10 @@ t4_ulp_push_frames(struct adapter *sc, s toep->txsd_avail--; t4_l2t_send(sc, wr, toep->l2te); - } while (m != NULL); + } - /* Send a FIN if requested, but only if there's no more data to send */ - if (m == NULL && toep->flags & TPF_SEND_FIN) + /* Send a FIN if requested, but only if there are no more PDUs to send */ + if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN) close_conn(sc, toep); } @@ -987,7 +1005,10 @@ t4_tod_output(struct toedev *tod, struct ("%s: inp %p dropped.", __func__, inp)); KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); - t4_push_frames(sc, toep, 0); + if (toep->ulp_mode == ULP_MODE_ISCSI) + t4_push_pdus(sc, toep, 0); + else + t4_push_frames(sc, toep, 0); return (0); } @@ -1009,7 +1030,7 @@ t4_send_fin(struct toedev *tod, struct t toep->flags |= TPF_SEND_FIN; if (tp->t_state >= TCPS_ESTABLISHED) { if (toep->ulp_mode == ULP_MODE_ISCSI) - t4_ulp_push_frames(sc, toep, 0); + t4_push_pdus(sc, toep, 0); else t4_push_frames(sc, toep, 0); } @@ -1250,91 +1271,6 @@ abort_status_to_errno(struct tcpcb *tp, } } -int -cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *); -/* - * tom_cpl_iscsi_callback - - * iscsi and tom would share the following cpl messages, so when any of these - * message is received, after tom is done with processing it, the messages - * needs to be forwarded to iscsi for further processing: - * - CPL_SET_TCB_RPL - * - CPL_RX_DATA_DDP - */ -void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *, - unsigned int); - -struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *); -/* - * Check if the handler function is set for a given CPL - * return 0 if the function is NULL or cpl_not_handled, 1 otherwise. - */ -int -t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode) -{ - - MPASS(opcode < nitems(sc->cpl_handler)); - - return (sc->cpl_handler[opcode] && - sc->cpl_handler[opcode] != cpl_not_handled); -} - -/* - * set the tom_cpl_iscsi_callback function, this function should be used - * whenever both toe and iscsi need to process the same cpl msg. - */ -void -t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *, - void *, unsigned int)) -{ - - tom_cpl_iscsi_callback = fp; -} - -void -t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *, - unsigned int, int *qlen)) -{ - - tom_queue_iscsi_callback = fp; -} - -int -t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m, - unsigned int opcode) -{ - struct socket *so; - - if (opcode == CPL_FW4_ACK) - so = toep->inp->inp_socket; - else { - INP_WLOCK(toep->inp); - so = toep->inp->inp_socket; - INP_WUNLOCK(toep->inp); - } - - if (tom_cpl_iscsi_callback && so) { - if (toep->ulp_mode == ULP_MODE_ISCSI) { - tom_cpl_iscsi_callback(td, so, m, opcode); - return (0); - } - } - - return (1); -} - -struct mbuf * -t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep, - unsigned int cmd, int *qlen) -{ - - if (tom_queue_iscsi_callback && so) { - if (toep->ulp_mode == ULP_MODE_ISCSI) - return (tom_queue_iscsi_callback(so, cmd, qlen)); - } - - return (NULL); -} - /* * TCP RST from the peer, timeout, or some other such critical error. */ @@ -1739,21 +1675,34 @@ do_fw4_ack(struct sge_iq *iq, const stru toep->tx_credits >= toep->tx_total / 4) { toep->flags &= ~TPF_TX_SUSPENDED; if (toep->ulp_mode == ULP_MODE_ISCSI) - t4_ulp_push_frames(sc, toep, plen); + t4_push_pdus(sc, toep, plen); else t4_push_frames(sc, toep, plen); } else if (plen > 0) { struct sockbuf *sb = &so->so_snd; + int sbu; - if (toep->ulp_mode == ULP_MODE_ISCSI) - t4_cpl_iscsi_callback(toep->td, toep, &plen, - CPL_FW4_ACK); - else { - SOCKBUF_LOCK(sb); + SOCKBUF_LOCK(sb); + sbu = sb->sb_cc; + if (toep->ulp_mode == ULP_MODE_ISCSI) { + + if (__predict_false(sbu > 0)) { + /* + * The data trasmitted before the tid's ULP mode + * changed to ISCSI is still in so_snd. + * Incoming credits should account for so_snd + * first. + */ + sbdrop_locked(sb, min(sbu, plen)); + plen -= min(sbu, plen); + } + sowwakeup_locked(so); /* unlocks so_snd */ + rqdrop_locked(&toep->ulp_pdu_reclaimq, plen); + } else { sbdrop_locked(sb, plen); - sowwakeup_locked(so); - SOCKBUF_UNLOCK_ASSERT(sb); + sowwakeup_locked(so); /* unlocks so_snd */ } + SOCKBUF_UNLOCK_ASSERT(sb); } INP_WUNLOCK(inp); @@ -1777,14 +1726,21 @@ do_set_tcb_rpl(struct sge_iq *iq, const if (is_ftid(sc, tid)) return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */ - else { - struct toepcb *toep = lookup_tid(sc, tid); - t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL); - return (0); - } + /* + * TOM and/or other ULPs don't request replies for CPL_SET_TCB or + * CPL_SET_TCB_FIELD requests. This can easily change and when it does + * the dispatch code will go here. + */ +#ifdef INVARIANTS + panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__, + tid, iq); +#else + log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n", + __func__, tid, iq); +#endif - CXGBE_UNIMPLEMENTED(__func__); + return (0); } void Modified: stable/10/sys/dev/cxgbe/tom/t4_ddp.c ============================================================================== --- stable/10/sys/dev/cxgbe/tom/t4_ddp.c Fri Dec 2 19:47:23 2016 (r309439) +++ stable/10/sys/dev/cxgbe/tom/t4_ddp.c Fri Dec 2 20:16:52 2016 (r309440) @@ -488,7 +488,6 @@ do_rx_data_ddp(struct sge_iq *iq, const unsigned int tid = GET_TID(cpl); uint32_t vld; struct toepcb *toep = lookup_tid(sc, tid); - struct tom_data *td = toep->td; KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); @@ -500,16 +499,11 @@ do_rx_data_ddp(struct sge_iq *iq, const panic("%s: DDP error 0x%x (tid %d, toep %p)", __func__, vld, tid, toep); } + if (toep->ulp_mode == ULP_MODE_ISCSI) { - m = m_get(M_NOWAIT, MT_DATA); - if (m == NULL) - CXGBE_UNIMPLEMENTED("mbuf alloc failure"); - memcpy(mtod(m, unsigned char *), cpl, - sizeof(struct cpl_rx_data_ddp)); - if (!t4_cpl_iscsi_callback(td, toep, m, CPL_RX_DATA_DDP)) - return (0); - m_freem(m); - } + sc->cpl_handler[CPL_RX_ISCSI_DDP](iq, rss, m); + return (0); + } handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len)); Modified: stable/10/sys/dev/cxgbe/tom/t4_tom.c ============================================================================== --- stable/10/sys/dev/cxgbe/tom/t4_tom.c Fri Dec 2 19:47:23 2016 (r309439) +++ stable/10/sys/dev/cxgbe/tom/t4_tom.c Fri Dec 2 20:16:52 2016 (r309440) @@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/kernel.h> #include <sys/ktr.h> +#include <sys/limits.h> #include <sys/module.h> #include <sys/protosw.h> #include <sys/domain.h> @@ -103,6 +104,37 @@ static int in6_ifaddr_gen; static eventhandler_tag ifaddr_evhandler; static struct timeout_task clip_task; +static void +mbufq_init(struct mbufq *q, int limit) +{ + + q->head = q->tail = NULL; +} + +static void +mbufq_drain(struct mbufq *q) +{ + struct mbuf *m; + + while ((m = q->head) != NULL) { + q->head = m->m_nextpkt; + m_freem(m); + } + q->tail = NULL; +} + +static inline int +mbufq_len(const struct mbufq *q) +{ + struct mbuf *m; + int len; + + len = 0; + for (m = q->head; m != NULL; m = m->m_nextpkt) + len++; + return (len); +} + struct toepcb * alloc_toepcb(struct vi_info *vi, int txqid, int rxqid, int flags) { @@ -155,6 +187,8 @@ alloc_toepcb(struct vi_info *vi, int txq toep->ofld_txq = &sc->sge.ofld_txq[txqid]; toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid]; toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; + mbufq_init(&toep->ulp_pduq, INT_MAX); + mbufq_init(&toep->ulp_pdu_reclaimq, INT_MAX); toep->txsd_total = txsd_total; toep->txsd_avail = txsd_total; toep->txsd_pidx = 0; @@ -270,6 +304,14 @@ release_offload_resources(struct toepcb CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)", __func__, toep, tid, toep->l2te, toep->ce); + /* + * These queues should have been emptied at approximately the same time + * that a normal connection's socket's so_snd would have been purged or + * drained. Do _not_ clean up here. + */ + MPASS(mbufq_len(&toep->ulp_pduq) == 0); + MPASS(mbufq_len(&toep->ulp_pdu_reclaimq) == 0); + if (toep->ulp_mode == ULP_MODE_TCPDDP) release_ddp_resources(toep); @@ -377,6 +419,7 @@ final_cpl_received(struct toepcb *toep) toep->inp = NULL; toep->flags &= ~TPF_CPL_PENDING; + mbufq_drain(&toep->ulp_pdu_reclaimq); if (!(toep->flags & TPF_ATTACHED)) release_offload_resources(toep); Modified: stable/10/sys/dev/cxgbe/tom/t4_tom.h ============================================================================== --- stable/10/sys/dev/cxgbe/tom/t4_tom.h Fri Dec 2 19:47:23 2016 (r309439) +++ stable/10/sys/dev/cxgbe/tom/t4_tom.h Fri Dec 2 20:16:52 2016 (r309440) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012 Chelsio Communications, Inc. + * Copyright (c) 2012, 2015 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar <np@FreeBSD.org> * @@ -32,6 +32,15 @@ #define __T4_TOM_H__ #include <sys/vmem.h> +/* + * Inline version of mbufq for use on 10.x. Borrowed from + * sys/cam/ctl/ctl_ha.c. + */ +struct mbufq { + struct mbuf *head; + struct mbuf *tail; +}; + #define LISTEN_HASH_SIZE 32 /* @@ -115,6 +124,10 @@ struct toepcb { int rx_credits; /* rx credits (in bytes) to be returned to hw */ u_int ulp_mode; /* ULP mode */ + void *ulpcb; + void *ulpcb2; + struct mbufq ulp_pduq; /* PDUs waiting to be sent out. */ + struct mbufq ulp_pdu_reclaimq; u_int ddp_flags; struct ddp_buffer *db[2]; @@ -220,6 +233,28 @@ td_adapter(struct tom_data *td) return (td->tod.tod_softc); } +/* + * XXX: Don't define these for the iWARP driver on 10 due to differences + * in LinuxKPI. + */ +#ifndef _LINUX_TYPES_H_ +static inline void +set_mbuf_ulp_submode(struct mbuf *m, uint8_t ulp_submode) +{ + + M_ASSERTPKTHDR(m); + m->m_pkthdr.PH_per.eigth[0] = ulp_submode; +} + +static inline uint8_t +mbuf_ulp_submode(struct mbuf *m) +{ + + M_ASSERTPKTHDR(m); + return (m->m_pkthdr.PH_per.eigth[0]); +} +#endif + /* t4_tom.c */ struct toepcb *alloc_toepcb(struct vi_info *, int, int, int); void free_toepcb(struct toepcb *); @@ -275,6 +310,7 @@ int t4_send_rst(struct toedev *, struct void t4_set_tcb_field(struct adapter *, struct toepcb *, int, uint16_t, uint64_t, uint64_t); void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop); +void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop); /* t4_ddp.c */ void t4_init_ddp(struct adapter *, struct tom_data *); @@ -287,19 +323,4 @@ void handle_ddp_close(struct toepcb *, s uint32_t); void insert_ddp_data(struct toepcb *, uint32_t); -/* ULP related */ -#define CXGBE_ISCSI_MBUF_TAG 50 -int t4tom_cpl_handler_registered(struct adapter *, unsigned int); -void t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, - struct socket *, void *, unsigned int)); -void t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *, - unsigned int, int *)); -void t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int); -int t4_cpl_iscsi_callback(struct tom_data *, struct toepcb *, void *, uint32_t); -struct mbuf *t4_queue_iscsi_callback(struct socket *, struct toepcb *, uint32_t, - int *); -extern void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, - void *, unsigned int); -extern struct mbuf *(*tom_queue_iscsi_callback)(struct socket*, unsigned int, - int *); #endif
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201612022016.uB2KGq3B099798>