Date: Sat, 26 Dec 2015 00:26:03 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r292736 - in head/sys/dev/cxgbe: . tom Message-ID: <201512260026.tBQ0Q3sc000150@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: np Date: Sat Dec 26 00:26:02 2015 New Revision: 292736 URL: https://svnweb.freebsd.org/changeset/base/292736 Log: cxgbe(4): Updates to the base NIC driver and t4_tom to support the iSCSI offload driver. These changes come from projects/cxl_iscsi. Modified: head/sys/dev/cxgbe/adapter.h head/sys/dev/cxgbe/offload.h head/sys/dev/cxgbe/t4_main.c head/sys/dev/cxgbe/tom/t4_cpl_io.c head/sys/dev/cxgbe/tom/t4_ddp.c head/sys/dev/cxgbe/tom/t4_tom.c head/sys/dev/cxgbe/tom/t4_tom.h Modified: head/sys/dev/cxgbe/adapter.h ============================================================================== --- head/sys/dev/cxgbe/adapter.h Fri Dec 25 21:39:45 2015 (r292735) +++ head/sys/dev/cxgbe/adapter.h Sat Dec 26 00:26:02 2015 (r292736) @@ -766,7 +766,7 @@ struct adapter { void *tom_softc; /* (struct tom_data *) */ struct tom_tunables tt; void *iwarp_softc; /* (struct c4iw_dev *) */ - void *iscsi_softc; + void *iscsi_ulp_softc; /* (struct cxgbei_data *) */ #endif struct l2t_data *l2t; /* L2 table */ struct tid_info tids; Modified: head/sys/dev/cxgbe/offload.h ============================================================================== --- head/sys/dev/cxgbe/offload.h Fri Dec 25 21:39:45 2015 (r292735) +++ head/sys/dev/cxgbe/offload.h Sat Dec 26 00:26:02 2015 (r292736) @@ -156,7 +156,7 @@ int t4_register_uld(struct uld_info *); int t4_unregister_uld(struct uld_info *); int t4_activate_uld(struct adapter *, int); int t4_deactivate_uld(struct adapter *, int); -void t4_iscsi_init(struct ifnet *, unsigned int, const unsigned int *); +void t4_iscsi_init(struct adapter *, u_int, const u_int *); int uld_active(struct adapter *, int); #endif #endif Modified: head/sys/dev/cxgbe/t4_main.c ============================================================================== --- head/sys/dev/cxgbe/t4_main.c Fri Dec 25 21:39:45 2015 (r292735) +++ head/sys/dev/cxgbe/t4_main.c Sat Dec 26 00:26:02 2015 (r292736) @@ -8801,11 +8801,8 @@ t4_ioctl(struct cdev *dev, unsigned long #ifdef TCP_OFFLOAD void -t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, - const unsigned int *pgsz_order) +t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order) { - struct vi_info *vi = ifp->if_softc; - struct adapter *sc = vi->pi->adapter; t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | Modified: head/sys/dev/cxgbe/tom/t4_cpl_io.c ============================================================================== --- head/sys/dev/cxgbe/tom/t4_cpl_io.c Fri Dec 25 21:39:45 2015 (r292735) +++ head/sys/dev/cxgbe/tom/t4_cpl_io.c Sat Dec 26 00:26:02 2015 (r292736) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012 Chelsio Communications, Inc. + * Copyright (c) 2012, 2015 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar <np@FreeBSD.org> * @@ -71,33 +71,6 @@ VNET_DECLARE(int, tcp_autorcvbuf_inc); VNET_DECLARE(int, tcp_autorcvbuf_max); #define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) -/* - * For ULP connections HW may add headers, e.g., for digests, that aren't part - * of the messages sent by the host but that are part of the TCP payload and - * therefore consume TCP sequence space. Tx connection parameters that - * operate in TCP sequence space are affected by the HW additions and need to - * compensate for them to accurately track TCP sequence numbers. This array - * contains the compensating extra lengths for ULP packets. It is indexed by - * a packet's ULP submode. - */ -const unsigned int t4_ulp_extra_len[] = {0, 4, 4, 8}; - -/* - * Return the length of any HW additions that will be made to a Tx packet. - * Such additions can happen for some types of ULP packets. - */ -static inline unsigned int -ulp_extra_len(struct mbuf *m, int *ulp_mode) -{ - struct m_tag *mtag; - - if ((mtag = m_tag_find(m, CXGBE_ISCSI_MBUF_TAG, NULL)) == NULL) - return (0); - *ulp_mode = *((int *)(mtag + 1)); - - return (t4_ulp_extra_len[*ulp_mode & 3]); -} - void send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) { @@ -384,13 +357,10 @@ t4_rcvd(struct toedev *tod, struct tcpcb KASSERT(toep->sb_cc >= sbused(sb), ("%s: sb %p has more data (%d) than last time (%d).", __func__, sb, sbused(sb), toep->sb_cc)); - if (toep->ulp_mode == ULP_MODE_ISCSI) { - toep->rx_credits += toep->sb_cc; - toep->sb_cc = 0; - } else { - toep->rx_credits += toep->sb_cc - sbused(sb); - toep->sb_cc = sbused(sb); - } + + toep->rx_credits += toep->sb_cc - sbused(sb); + toep->sb_cc = sbused(sb); + if (toep->rx_credits > 0 && (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 || (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) || @@ -490,25 +460,16 @@ max_dsgl_nsegs(int tx_credits) static inline void write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, - unsigned int plen, uint8_t credits, int shove, int ulp_mode, int txalign) + unsigned int plen, uint8_t credits, int shove, int ulp_submode, int txalign) { struct fw_ofld_tx_data_wr *txwr = dst; - unsigned int wr_ulp_mode; txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | V_FW_WR_IMMDLEN(immdlen)); txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | V_FW_WR_LEN16(credits)); - - /* for iscsi, the mode & submode setting is per-packet */ - if (toep->ulp_mode == ULP_MODE_ISCSI) - wr_ulp_mode = V_TX_ULP_MODE(ulp_mode >> 4) | - V_TX_ULP_SUBMODE(ulp_mode & 3); - else - wr_ulp_mode = V_TX_ULP_MODE(toep->ulp_mode); - - txwr->lsodisable_to_flags = htobe32(wr_ulp_mode | V_TX_URG(0) | /*XXX*/ - V_TX_SHOVE(shove)); + txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(toep->ulp_mode) | + V_TX_ULP_SUBMODE(ulp_submode) | V_TX_URG(0) | V_TX_SHOVE(shove)); txwr->plen = htobe32(plen); if (txalign > 0) { @@ -616,6 +577,9 @@ t4_push_frames(struct adapter *sc, struc toep->ulp_mode == ULP_MODE_RDMA, ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); + if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) + return; + /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. @@ -802,56 +766,70 @@ t4_push_frames(struct adapter *sc, struc close_conn(sc, toep); } -/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */ +static inline void +rqdrop_locked(struct mbufq *q, int plen) +{ + struct mbuf *m; + + while (plen > 0) { + m = mbufq_dequeue(q); + + /* Too many credits. */ + MPASS(m != NULL); + M_ASSERTPKTHDR(m); + + /* Partial credits. */ + MPASS(plen >= m->m_pkthdr.len); + + plen -= m->m_pkthdr.len; + m_freem(m); + } +} + void -t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop) +t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop) { - struct mbuf *sndptr, *m = NULL; + struct mbuf *sndptr, *m; struct fw_ofld_tx_data_wr *txwr; struct wrqe *wr; - unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; + u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; + u_int adjusted_plen, ulp_submode; struct inpcb *inp = toep->inp; - struct tcpcb *tp; - struct socket *so; - struct sockbuf *sb; - int tx_credits, ulp_len = 0, ulp_mode = 0, qlen = 0; - int shove, compl; - struct ofld_tx_sdesc *txsd; + struct tcpcb *tp = intotcpcb(inp); + int tx_credits, shove; + struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx]; + struct mbufq *pduq = &toep->ulp_pduq; + static const u_int ulp_extra_len[] = {0, 4, 4, 8}; INP_WLOCK_ASSERT(inp); - if (toep->flags & TPF_ABORT_SHUTDOWN) - return; - - tp = intotcpcb(inp); - so = inp->inp_socket; - sb = &so->so_snd; - txsd = &toep->txsd[toep->txsd_pidx]; - KASSERT(toep->flags & TPF_FLOWC_WR_SENT, ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); + KASSERT(toep->ulp_mode == ULP_MODE_ISCSI, + ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep)); + + if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) + return; /* * This function doesn't resume by itself. Someone else must clear the * flag and call this function. */ - if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) + if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) { + KASSERT(drop == 0, + ("%s: drop (%d) != 0 but tx is suspended", __func__, drop)); return; + } - sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen); - if (!qlen) - return; + if (drop) + rqdrop_locked(&toep->ulp_pdu_reclaimq, drop); + + while ((sndptr = mbufq_first(pduq)) != NULL) { + M_ASSERTPKTHDR(sndptr); - do { tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); max_imm = max_imm_payload(tx_credits); max_nsegs = max_dsgl_nsegs(tx_credits); - if (drop) { - t4_cpl_iscsi_callback(toep->td, toep, &drop, - CPL_FW4_ACK); - drop = 0; - } - plen = 0; nsegs = 0; max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ @@ -861,7 +839,10 @@ t4_ulp_push_frames(struct adapter *sc, s nsegs += n; plen += m->m_len; - /* This mbuf sent us _over_ the nsegs limit, return */ + /* + * This mbuf would send us _over_ the nsegs limit. + * Suspend tx because the PDU can't be sent out. + */ if (plen > max_imm && nsegs > max_nsegs) { toep->flags |= TPF_TX_SUSPENDED; return; @@ -869,30 +850,35 @@ t4_ulp_push_frames(struct adapter *sc, s if (max_nsegs_1mbuf < n) max_nsegs_1mbuf = n; - - /* This mbuf put us right at the max_nsegs limit */ - if (plen > max_imm && nsegs == max_nsegs) { - toep->flags |= TPF_TX_SUSPENDED; - return; - } - } - - shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); - /* nothing to send */ - if (plen == 0) { - KASSERT(m == NULL, - ("%s: nothing to send, but m != NULL", __func__)); - break; } if (__predict_false(toep->flags & TPF_FIN_SENT)) panic("%s: excess tx.", __func__); - ulp_len = plen + ulp_extra_len(sndptr, &ulp_mode); + /* + * We have a PDU to send. All of it goes out in one WR so 'm' + * is NULL. A PDU's length is always a multiple of 4. + */ + MPASS(m == NULL); + MPASS((plen & 3) == 0); + MPASS(sndptr->m_pkthdr.len == plen); + + shove = !(tp->t_flags & TF_MORETOCOME); + ulp_submode = mbuf_ulp_submode(sndptr); + MPASS(ulp_submode < nitems(ulp_extra_len)); + + /* + * plen doesn't include header and data digests, which are + * generated and inserted in the right places by the TOE, but + * they do occupy TCP sequence space and need to be accounted + * for. + */ + adjusted_plen = plen + ulp_extra_len[ulp_submode]; if (plen <= max_imm) { /* Immediate data tx */ - wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16), + + wr = alloc_wrqe(roundup2(sizeof(*txwr) + plen, 16), toep->ofld_txq); if (wr == NULL) { /* XXX: how will we recover from this? */ @@ -901,16 +887,17 @@ t4_ulp_push_frames(struct adapter *sc, s } txwr = wrtod(wr); credits = howmany(wr->wr_len, 16); - write_tx_wr(txwr, toep, plen, ulp_len, credits, shove, - ulp_mode, 0); + write_tx_wr(txwr, toep, plen, adjusted_plen, credits, + shove, ulp_submode, sc->tt.tx_align); m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); + nsegs = 0; } else { int wr_len; /* DSGL tx */ wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; - wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq); + wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); if (wr == NULL) { /* XXX: how will we recover from this? */ toep->flags |= TPF_TX_SUSPENDED; @@ -918,8 +905,8 @@ t4_ulp_push_frames(struct adapter *sc, s } txwr = wrtod(wr); credits = howmany(wr_len, 16); - write_tx_wr(txwr, toep, 0, ulp_len, credits, shove, - ulp_mode, 0); + write_tx_wr(txwr, toep, 0, adjusted_plen, credits, + shove, ulp_submode, sc->tt.tx_align); write_tx_sgl(txwr + 1, sndptr, m, nsegs, max_nsegs_1mbuf); if (wr_len & 0xf) { @@ -932,28 +919,26 @@ t4_ulp_push_frames(struct adapter *sc, s KASSERT(toep->tx_credits >= credits, ("%s: not enough credits", __func__)); + m = mbufq_dequeue(pduq); + MPASS(m == sndptr); + mbufq_enqueue(&toep->ulp_pdu_reclaimq, m); + toep->tx_credits -= credits; toep->tx_nocompl += credits; toep->plen_nocompl += plen; if (toep->tx_credits <= toep->tx_total * 3 / 8 && - toep->tx_nocompl >= toep->tx_total / 4) - compl = 1; - - if (compl) { + toep->tx_nocompl >= toep->tx_total / 4) { txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); toep->tx_nocompl = 0; toep->plen_nocompl = 0; } - tp->snd_nxt += ulp_len; - tp->snd_max += ulp_len; - /* goto next mbuf */ - sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen); + tp->snd_nxt += adjusted_plen; + tp->snd_max += adjusted_plen; toep->flags |= TPF_TX_DATA_SENT; - if (toep->tx_credits < MIN_OFLD_TX_CREDITS) { + if (toep->tx_credits < MIN_OFLD_TX_CREDITS) toep->flags |= TPF_TX_SUSPENDED; - } KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); txsd->plen = plen; @@ -966,10 +951,10 @@ t4_ulp_push_frames(struct adapter *sc, s toep->txsd_avail--; t4_l2t_send(sc, wr, toep->l2te); - } while (m != NULL); + } - /* Send a FIN if requested, but only if there's no more data to send */ - if (m == NULL && toep->flags & TPF_SEND_FIN) + /* Send a FIN if requested, but only if there are no more PDUs to send */ + if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN) close_conn(sc, toep); } @@ -987,7 +972,10 @@ t4_tod_output(struct toedev *tod, struct ("%s: inp %p dropped.", __func__, inp)); KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); - t4_push_frames(sc, toep, 0); + if (toep->ulp_mode == ULP_MODE_ISCSI) + t4_push_pdus(sc, toep, 0); + else + t4_push_frames(sc, toep, 0); return (0); } @@ -1009,7 +997,7 @@ t4_send_fin(struct toedev *tod, struct t toep->flags |= TPF_SEND_FIN; if (tp->t_state >= TCPS_ESTABLISHED) { if (toep->ulp_mode == ULP_MODE_ISCSI) - t4_ulp_push_frames(sc, toep, 0); + t4_push_pdus(sc, toep, 0); else t4_push_frames(sc, toep, 0); } @@ -1250,91 +1238,6 @@ abort_status_to_errno(struct tcpcb *tp, } } -int -cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *); -/* - * tom_cpl_iscsi_callback - - * iscsi and tom would share the following cpl messages, so when any of these - * message is received, after tom is done with processing it, the messages - * needs to be forwarded to iscsi for further processing: - * - CPL_SET_TCB_RPL - * - CPL_RX_DATA_DDP - */ -void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *, - unsigned int); - -struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *); -/* - * Check if the handler function is set for a given CPL - * return 0 if the function is NULL or cpl_not_handled, 1 otherwise. - */ -int -t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode) -{ - - MPASS(opcode < nitems(sc->cpl_handler)); - - return (sc->cpl_handler[opcode] && - sc->cpl_handler[opcode] != cpl_not_handled); -} - -/* - * set the tom_cpl_iscsi_callback function, this function should be used - * whenever both toe and iscsi need to process the same cpl msg. - */ -void -t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *, - void *, unsigned int)) -{ - - tom_cpl_iscsi_callback = fp; -} - -void -t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *, - unsigned int, int *qlen)) -{ - - tom_queue_iscsi_callback = fp; -} - -int -t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m, - unsigned int opcode) -{ - struct socket *so; - - if (opcode == CPL_FW4_ACK) - so = toep->inp->inp_socket; - else { - INP_WLOCK(toep->inp); - so = toep->inp->inp_socket; - INP_WUNLOCK(toep->inp); - } - - if (tom_cpl_iscsi_callback && so) { - if (toep->ulp_mode == ULP_MODE_ISCSI) { - tom_cpl_iscsi_callback(td, so, m, opcode); - return (0); - } - } - - return (1); -} - -struct mbuf * -t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep, - unsigned int cmd, int *qlen) -{ - - if (tom_queue_iscsi_callback && so) { - if (toep->ulp_mode == ULP_MODE_ISCSI) - return (tom_queue_iscsi_callback(so, cmd, qlen)); - } - - return (NULL); -} - /* * TCP RST from the peer, timeout, or some other such critical error. */ @@ -1733,21 +1636,34 @@ do_fw4_ack(struct sge_iq *iq, const stru toep->tx_credits >= toep->tx_total / 4) { toep->flags &= ~TPF_TX_SUSPENDED; if (toep->ulp_mode == ULP_MODE_ISCSI) - t4_ulp_push_frames(sc, toep, plen); + t4_push_pdus(sc, toep, plen); else t4_push_frames(sc, toep, plen); } else if (plen > 0) { struct sockbuf *sb = &so->so_snd; + int sbu; - if (toep->ulp_mode == ULP_MODE_ISCSI) - t4_cpl_iscsi_callback(toep->td, toep, &plen, - CPL_FW4_ACK); - else { - SOCKBUF_LOCK(sb); + SOCKBUF_LOCK(sb); + sbu = sbused(sb); + if (toep->ulp_mode == ULP_MODE_ISCSI) { + + if (__predict_false(sbu > 0)) { + /* + * The data trasmitted before the tid's ULP mode + * changed to ISCSI is still in so_snd. + * Incoming credits should account for so_snd + * first. + */ + sbdrop_locked(sb, min(sbu, plen)); + plen -= min(sbu, plen); + } + sowwakeup_locked(so); /* unlocks so_snd */ + rqdrop_locked(&toep->ulp_pdu_reclaimq, plen); + } else { sbdrop_locked(sb, plen); - sowwakeup_locked(so); - SOCKBUF_UNLOCK_ASSERT(sb); + sowwakeup_locked(so); /* unlocks so_snd */ } + SOCKBUF_UNLOCK_ASSERT(sb); } INP_WUNLOCK(inp); @@ -1771,14 +1687,21 @@ do_set_tcb_rpl(struct sge_iq *iq, const if (is_ftid(sc, tid)) return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */ - else { - struct toepcb *toep = lookup_tid(sc, tid); - t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL); - return (0); - } + /* + * TOM and/or other ULPs don't request replies for CPL_SET_TCB or + * CPL_SET_TCB_FIELD requests. This can easily change and when it does + * the dispatch code will go here. + */ +#ifdef INVARIANTS + panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__, + tid, iq); +#else + log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n", + __func__, tid, iq); +#endif - CXGBE_UNIMPLEMENTED(__func__); + return (0); } void Modified: head/sys/dev/cxgbe/tom/t4_ddp.c ============================================================================== --- head/sys/dev/cxgbe/tom/t4_ddp.c Fri Dec 25 21:39:45 2015 (r292735) +++ head/sys/dev/cxgbe/tom/t4_ddp.c Sat Dec 26 00:26:02 2015 (r292736) @@ -513,7 +513,6 @@ do_rx_data_ddp(struct sge_iq *iq, const unsigned int tid = GET_TID(cpl); uint32_t vld; struct toepcb *toep = lookup_tid(sc, tid); - struct tom_data *td = toep->td; KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__)); @@ -525,16 +524,11 @@ do_rx_data_ddp(struct sge_iq *iq, const panic("%s: DDP error 0x%x (tid %d, toep %p)", __func__, vld, tid, toep); } + if (toep->ulp_mode == ULP_MODE_ISCSI) { - m = m_get(M_NOWAIT, MT_DATA); - if (m == NULL) - CXGBE_UNIMPLEMENTED("mbuf alloc failure"); - memcpy(mtod(m, unsigned char *), cpl, - sizeof(struct cpl_rx_data_ddp)); - if (!t4_cpl_iscsi_callback(td, toep, m, CPL_RX_DATA_DDP)) - return (0); - m_freem(m); - } + sc->cpl_handler[CPL_RX_ISCSI_DDP](iq, rss, m); + return (0); + } handle_ddp_data(toep, cpl->u.ddp_report, cpl->seq, be16toh(cpl->len)); Modified: head/sys/dev/cxgbe/tom/t4_tom.c ============================================================================== --- head/sys/dev/cxgbe/tom/t4_tom.c Fri Dec 25 21:39:45 2015 (r292735) +++ head/sys/dev/cxgbe/tom/t4_tom.c Sat Dec 26 00:26:02 2015 (r292736) @@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/ktr.h> #include <sys/lock.h> +#include <sys/limits.h> #include <sys/module.h> #include <sys/protosw.h> #include <sys/domain.h> @@ -158,6 +159,8 @@ alloc_toepcb(struct vi_info *vi, int txq toep->ofld_txq = &sc->sge.ofld_txq[txqid]; toep->ofld_rxq = &sc->sge.ofld_rxq[rxqid]; toep->ctrlq = &sc->sge.ctrlq[pi->port_id]; + mbufq_init(&toep->ulp_pduq, INT_MAX); + mbufq_init(&toep->ulp_pdu_reclaimq, INT_MAX); toep->txsd_total = txsd_total; toep->txsd_avail = txsd_total; toep->txsd_pidx = 0; @@ -273,6 +276,14 @@ release_offload_resources(struct toepcb CTR5(KTR_CXGBE, "%s: toep %p (tid %d, l2te %p, ce %p)", __func__, toep, tid, toep->l2te, toep->ce); + /* + * These queues should have been emptied at approximately the same time + * that a normal connection's socket's so_snd would have been purged or + * drained. Do _not_ clean up here. + */ + MPASS(mbufq_len(&toep->ulp_pduq) == 0); + MPASS(mbufq_len(&toep->ulp_pdu_reclaimq) == 0); + if (toep->ulp_mode == ULP_MODE_TCPDDP) release_ddp_resources(toep); @@ -380,6 +391,7 @@ final_cpl_received(struct toepcb *toep) toep->inp = NULL; toep->flags &= ~TPF_CPL_PENDING; + mbufq_drain(&toep->ulp_pdu_reclaimq); if (!(toep->flags & TPF_ATTACHED)) release_offload_resources(toep); Modified: head/sys/dev/cxgbe/tom/t4_tom.h ============================================================================== --- head/sys/dev/cxgbe/tom/t4_tom.h Fri Dec 25 21:39:45 2015 (r292735) +++ head/sys/dev/cxgbe/tom/t4_tom.h Sat Dec 26 00:26:02 2015 (r292736) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012 Chelsio Communications, Inc. + * Copyright (c) 2012, 2015 Chelsio Communications, Inc. * All rights reserved. * Written by: Navdeep Parhar <np@FreeBSD.org> * @@ -115,6 +115,10 @@ struct toepcb { int rx_credits; /* rx credits (in bytes) to be returned to hw */ u_int ulp_mode; /* ULP mode */ + void *ulpcb; + void *ulpcb2; + struct mbufq ulp_pduq; /* PDUs waiting to be sent out. */ + struct mbufq ulp_pdu_reclaimq; u_int ddp_flags; struct ddp_buffer *db[2]; @@ -220,6 +224,22 @@ td_adapter(struct tom_data *td) return (td->tod.tod_softc); } +static inline void +set_mbuf_ulp_submode(struct mbuf *m, uint8_t ulp_submode) +{ + + M_ASSERTPKTHDR(m); + m->m_pkthdr.PH_per.eight[0] = ulp_submode; +} + +static inline uint8_t +mbuf_ulp_submode(struct mbuf *m) +{ + + M_ASSERTPKTHDR(m); + return (m->m_pkthdr.PH_per.eight[0]); +} + /* t4_tom.c */ struct toepcb *alloc_toepcb(struct vi_info *, int, int, int); void free_toepcb(struct toepcb *); @@ -275,6 +295,7 @@ int t4_send_rst(struct toedev *, struct void t4_set_tcb_field(struct adapter *, struct toepcb *, int, uint16_t, uint64_t, uint64_t); void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop); +void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop); /* t4_ddp.c */ void t4_init_ddp(struct adapter *, struct tom_data *); @@ -287,19 +308,4 @@ void handle_ddp_close(struct toepcb *, s uint32_t); void insert_ddp_data(struct toepcb *, uint32_t); -/* ULP related */ -#define CXGBE_ISCSI_MBUF_TAG 50 -int t4tom_cpl_handler_registered(struct adapter *, unsigned int); -void t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, - struct socket *, void *, unsigned int)); -void t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *, - unsigned int, int *)); -void t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int); -int t4_cpl_iscsi_callback(struct tom_data *, struct toepcb *, void *, uint32_t); -struct mbuf *t4_queue_iscsi_callback(struct socket *, struct toepcb *, uint32_t, - int *); -extern void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, - void *, unsigned int); -extern struct mbuf *(*tom_queue_iscsi_callback)(struct socket*, unsigned int, - int *); #endif
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201512260026.tBQ0Q3sc000150>