Date: Sat, 26 May 2012 01:33:08 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r236035 - in user/np/toe_iwarp/sys: contrib/rdma/krping dev/cxgb dev/cxgb/ulp/iw_cxgb dev/cxgb/ulp/tom Message-ID: <201205260133.q4Q1X8VJ044612@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: np Date: Sat May 26 01:33:07 2012 New Revision: 236035 URL: http://svn.freebsd.org/changeset/base/236035 Log: Some bugfixes and cleanup of cxgb TOE and iWARP drivers. Enable TOE multiq. Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_listen.c user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_toepcb.h user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.c user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.h Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c ============================================================================== --- user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c Sat May 26 01:24:39 2012 (r236034) +++ user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c Sat May 26 01:33:07 2012 (r236035) @@ -1737,8 +1737,8 @@ int krping_doit(char *cmd) case 'm': cb->memlimit = optint; if (cb->memlimit < 1) { - log(LOG_ERR, "Invalid memory limit %lu\n" - ,cb->memlimit); + log(LOG_ERR, "Invalid memory limit %ju\n", + cb->memlimit); ret = EINVAL; } else DEBUG_LOG(PFX "memory limit %d\n", (int)optint); Modified: user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h ============================================================================== --- user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h Sat May 26 01:24:39 2012 (r236034) +++ user/np/toe_iwarp/sys/dev/cxgb/cxgb_offload.h Sat May 26 01:33:07 2012 (r236035) @@ -58,12 +58,6 @@ struct tom_tunables { enum { CPL_PRIORITY_DATA = 0, /* data messages */ CPL_PRIORITY_CONTROL = 1 /* offload control messages */ -#ifdef notyet - CPL_PRIORITY_SETUP = 1, /* connection setup messages */ - CPL_PRIORITY_TEARDOWN = 0, /* connection teardown messages */ - CPL_PRIORITY_LISTEN = 1, /* listen start/stop messages */ - CPL_PRIORITY_ACK = 1, /* RX ACK messages */ -#endif }; #define S_HDR_NDESC 0 Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c ============================================================================== --- user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c Sat May 26 01:24:39 2012 (r236034) +++ user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c Sat May 26 01:33:07 2012 (r236035) @@ -1033,10 +1033,9 @@ process_close_complete(struct iwch_ep *e IWCH_QP_ATTR_NEXT_STATE, &attrs, 1); } - if (ep->parent_ep) { - ((struct iwch_listen_ep *)ep->parent_ep)->child_ep = NULL; + if (ep->parent_ep) close_socket(&ep->com, 1); - } else + else close_socket(&ep->com, 0); close_complete_upcall(ep); __state_set(&ep->com, DEAD); @@ -1072,11 +1071,11 @@ terminate(struct sge_qset *qs, struct rs { struct adapter *sc = qs->adap; struct tom_data *td = sc->tom_softc; - struct cpl_rdma_ec_status *rep = (void *)r; - unsigned int tid = GET_TID(rep); - struct toepcb *toep = lookup_tid(&td->tid_maps, tid); + uint32_t hash = *((uint32_t *)r + 1); + unsigned int tid = ntohl(hash) >> 8 & 0xfffff; + struct toepcb *toep = lookup_tid(&td->tid_maps, tid); struct socket *so = toep->tp_inp->inp_socket; - struct iwch_ep *ep = so->so_rcv.sb_upcallarg; + struct iwch_ep *ep = so->so_rcv.sb_upcallarg; if (state_read(&ep->com) != FPDU_MODE) goto done; @@ -1418,11 +1417,6 @@ iwch_destroy_listen(struct iw_cm_id *cm_ CTR2(KTR_IW_CXGB, "%s ep %p", __FUNCTION__, ep); - /* lets clean any connected child if there are */ - if (ep->child_ep) { - if (ep->child_ep->com.so->so_state & SS_ISCONNECTED) - close_socket(&ep->child_ep->com, 1); - } state_set(&ep->com, DEAD); close_socket(&ep->com, 0); cm_id->rem_ref(cm_id); @@ -1607,8 +1601,6 @@ process_newconn(struct iwch_ep *parent_e child_ep->com.cm_id = NULL; child_ep->com.thread = parent_ep->com.thread; child_ep->parent_ep = parent_ep; - /* Save child reference in parent_ep which could be used for cleanup */ - ((struct iwch_listen_ep *)parent_ep)->child_ep = child_ep; free(remote, M_SONAME); get_ep(&parent_ep->com); Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h ============================================================================== --- user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h Sat May 26 01:24:39 2012 (r236034) +++ user/np/toe_iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h Sat May 26 01:33:07 2012 (r236035) @@ -165,7 +165,6 @@ struct iwch_listen_ep { struct iwch_ep_common com; unsigned int stid; int backlog; - struct iwch_ep *child_ep; }; struct iwch_ep { Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c ============================================================================== --- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c Sat May 26 01:24:39 2012 (r236034) +++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c Sat May 26 01:33:07 2012 (r236035) @@ -175,27 +175,9 @@ toepcb_detach(struct inpcb *inp) toep, inp, tp); tp->t_toe = NULL; + tp->t_flags &= ~TF_TOE; toep->tp_flags &= ~TP_ATTACHED; - /* - * Verify that the tcpcb really is going away. tcp_input shouldn't be - * able to find this inp, or should see it in TIME_WAIT. Otherwise - * there's a risk of tod_input with no way for the TOE driver to reach - * its toepcb (we just cleared it). - * - * The inp should be DROPPED already or about to go to TIME_WAIT (it - * can't be in TIME_WAIT already). There is no way to be certain that - * code that runs after this routine will actually call twstart as it - * should (while holding the inp's lock throughout), so we do the best - * we can by asserting that the current TCP state is something from - * which we can transition to TIME_WAIT. - */ - KASSERT(inp->inp_flags & INP_DROPPED || /* dropped already */ - (!(inp->inp_flags & INP_TIMEWAIT) && /* not in TIME_WAIT */ - (tp->t_state == TCPS_FIN_WAIT_1 || tp->t_state == TCPS_FIN_WAIT_2 || - tp->t_state == TCPS_CLOSING)), - ("%s: inp %p detached too early?", __func__, inp)); - if (toep->tp_flags & TP_CPL_DONE) t3_release_offload_resources(toep); } @@ -341,6 +323,28 @@ t3_process_tid_release_list(void *data, mtx_unlock(&td->tid_release_lock); } +static void +close_conn(struct adapter *sc, struct toepcb *toep) +{ + struct mbuf *m; + struct cpl_close_con_req *req; + + if (toep->tp_flags & TP_FIN_SENT) + return; + + m = M_GETHDR_OFLD(toep->tp_qset, CPL_PRIORITY_DATA, req); + if (m == NULL) + CXGB_UNIMPLEMENTED(); + + req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); + req->wr.wrh_lo = htonl(V_WR_TID(toep->tp_tid)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, toep->tp_tid)); + req->rsvd = 0; + + toep->tp_flags |= TP_FIN_SENT; + t3_offload_tx(sc, m); +} + static inline void make_tx_data_wr(struct socket *so, struct tx_data_wr *req, int len, struct mbuf *tail) @@ -432,11 +436,6 @@ t3_push_frames(struct socket *so, int re inp_lock_assert(tp->t_inpcb); - /* TCP state or socket state not suitable for sending data */ - if (tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_CLOSED || - (so_state_get(so) & (SS_ISDISCONNECTING | SS_ISDISCONNECTED))) - return (0); - snd = so_sockbuf_snd(so); SOCKBUF_LOCK(snd); @@ -557,16 +556,18 @@ t3_push_frames(struct socket *so, int re } out: SOCKBUF_UNLOCK(snd); + + if (sndptr == NULL && (toep->tp_flags & TP_SEND_FIN)) + close_conn(sc, toep); + return (total_bytes); } static int -send_rx_credits(struct toepcb *toep, int credits) +send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits) { struct mbuf *m; struct cpl_rx_data_ack *req; - struct toedev *tod = toep->tp_tod; - struct adapter *sc = tod->tod_softc; uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); m = M_GETHDR_OFLD(toep->tp_qset, CPL_PRIORITY_CONTROL, req); @@ -584,6 +585,7 @@ send_rx_credits(struct toepcb *toep, int void t3_rcvd(struct toedev *tod, struct tcpcb *tp) { + struct adapter *sc = tod->tod_softc; struct inpcb *inp = tp->t_inpcb; struct socket *so = inp->inp_socket; struct sockbuf *so_rcv = &so->so_rcv; @@ -603,7 +605,7 @@ t3_rcvd(struct toedev *tod, struct tcpcb if (must_send || toep->tp_rx_credits >= 15 * 1024) { int credits; - credits = send_rx_credits(toep, toep->tp_rx_credits); + credits = send_rx_credits(sc, toep, toep->tp_rx_credits); toep->tp_rx_credits -= credits; tp->rcv_wnd += credits; tp->rcv_adv += credits; @@ -628,13 +630,12 @@ do_rx_urg_notify(struct sge_qset *qs, st int t3_send_fin(struct toedev *tod, struct tcpcb *tp) { - struct mbuf *m; struct toepcb *toep = tp->t_toe; - struct adapter *sc = tod->tod_softc; - struct cpl_close_con_req *req; struct inpcb *inp = tp->t_inpcb; struct socket *so = inp_inpcbtosocket(inp); +#if defined(KTR) unsigned int tid = toep->tp_tid; +#endif INP_INFO_WLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(inp); @@ -642,23 +643,8 @@ t3_send_fin(struct toedev *tod, struct t CTR4(KTR_CXGB, "%s: tid %d, toep %p, flags %x", __func__, tid, toep, toep->tp_flags); - if (tp->t_state != TCPS_SYN_SENT) - t3_push_frames(so, 1); - - if (toep->tp_flags & TP_FIN_SENT) - return (0); - - m = M_GETHDR_OFLD(toep->tp_qset, CPL_PRIORITY_DATA, req); - if (m == NULL) - CXGB_UNIMPLEMENTED(); - - req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); - req->wr.wrh_lo = htonl(V_WR_TID(tid)); - OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); - req->rsvd = 0; - - toep->tp_flags |= TP_FIN_SENT; - t3_offload_tx(sc, m); + toep->tp_flags |= TP_SEND_FIN; + t3_push_frames(so, 1); return (0); } @@ -787,8 +773,12 @@ offload_socket(struct socket *so, struct INP_WLOCK_ASSERT(inp); /* Update socket */ + SOCKBUF_LOCK(&so->so_snd); so_sockbuf_snd(so)->sb_flags |= SB_NOCOALESCE; + SOCKBUF_UNLOCK(&so->so_snd); + SOCKBUF_LOCK(&so->so_rcv); so_sockbuf_rcv(so)->sb_flags |= SB_NOCOALESCE; + SOCKBUF_UNLOCK(&so->so_rcv); /* Update TCP PCB */ tp->tod = toep->tp_tod; @@ -921,26 +911,28 @@ do_act_open_rpl(struct sge_qset *qs, str unsigned int atid = G_TID(ntohl(rpl->atid)); struct toepcb *toep = lookup_atid(&td->tid_maps, atid); struct inpcb *inp = toep->tp_inp; - struct tcpcb *tp; + struct tcpcb *tp = intotcpcb(inp); + int s = rpl->status; - CTR3(KTR_CXGB, "%s: atid %u, status %u ", __func__, atid, rpl->status); + CTR3(KTR_CXGB, "%s: atid %u, status %u ", __func__, atid, s); free_atid(&td->tid_maps, atid); toep->tp_tid = -1; - if (act_open_has_tid(rpl->status)) + if (act_open_has_tid(s)) queue_tid_release(tod, GET_TID(rpl)); - INP_INFO_WLOCK(&V_tcbinfo); /* for tcp_drop */ - INP_WLOCK(inp); - if (!(inp->inp_flags & INP_DROPPED)) { - tp = intotcpcb(inp); - tp = tcp_drop(tp, act_open_rpl_status_to_errno(rpl->status)); - if (tp == NULL) - INP_WLOCK(inp); /* re-acquire */ + if (s == CPL_ERR_TCAM_FULL || s == CPL_ERR_CONN_EXIST) { + INP_WLOCK(inp); + toe_connect_failed(tod, tp, EAGAIN); + toepcb_release(toep); /* unlocks inp */ + } else { + INP_INFO_WLOCK(&V_tcbinfo); + INP_WLOCK(inp); + toe_connect_failed(tod, tp, act_open_rpl_status_to_errno(s)); + toepcb_release(toep); /* unlocks inp */ + INP_INFO_WUNLOCK(&V_tcbinfo); } - toepcb_release(toep); - INP_INFO_WUNLOCK(&V_tcbinfo); m_freem(m); return (0); @@ -983,11 +975,7 @@ t3_connect(struct toedev *tod, struct so if (atid < 0) goto failed; -#ifdef notyet qset = pi->first_qset + (arc4random() % pi->nqsets); -#else - qset = 0; -#endif m = M_GETHDR_OFLD(qset, CPL_PRIORITY_CONTROL, cpl); if (m == NULL) @@ -1241,7 +1229,7 @@ do_peer_close(struct sge_qset *qs, struc tp = intotcpcb(inp); CTR5(KTR_CXGB, "%s: tid %u (%s), toep_flags 0x%x, inp %p", __func__, - tid, tcpstates[tp->t_state], toep->tp_flags, toep->tp_inp); + tid, tp ? tcpstates[tp->t_state] : "no tp" , toep->tp_flags, inp); if (toep->tp_flags & TP_ABORT_RPL_PENDING) goto done; @@ -1303,16 +1291,15 @@ do_close_con_rpl(struct sge_qset *qs, st INP_WLOCK(inp); tp = intotcpcb(inp); - so = inp_inpcbtosocket(tp->t_inpcb); - CTR4(KTR_CXGB, "%s: tid %u (%s), toep_flags 0x%x", __func__, tid, - tcpstates[tp->t_state], toep->tp_flags); - - tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */ + tp ? tcpstates[tp->t_state] : "no tp", toep->tp_flags); if ((toep->tp_flags & TP_ABORT_RPL_PENDING)) goto done; + so = inp_inpcbtosocket(inp); + tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */ + switch (tp->t_state) { case TCPS_CLOSING: tcp_twstart(tp); @@ -1331,16 +1318,9 @@ release: goto release; case TCPS_FIN_WAIT_1: + if (so->so_rcv.sb_state & SBS_CANTRCVMORE) + soisdisconnected(so); tp->t_state = TCPS_FIN_WAIT_2; - if ((so_options_get(so) & SO_LINGER) && - so_linger_get(so) == 0) { - - if (tcp_close(tp)) - inp_wunlock(tp->t_inpcb); - INP_INFO_WUNLOCK(&V_tcbinfo); - m_freem(m); - return (0); - } break; default: log(LOG_ERR, Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_listen.c ============================================================================== --- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_listen.c Sat May 26 01:24:39 2012 (r236034) +++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_listen.c Sat May 26 01:33:07 2012 (r236035) @@ -413,6 +413,21 @@ pass_accept_req_to_protohdrs(const struc to->to_flags |= TOF_SACKPERM; } +static inline void +hold_synqe(struct synq_entry *synqe) +{ + + refcount_acquire(&synqe->refcnt); +} + +static inline void +release_synqe(struct synq_entry *synqe) +{ + + if (refcount_release(&synqe->refcnt)) + m_freem(synqe->m); +} + /* * Use the trailing space in the mbuf in which the PASS_ACCEPT_REQ arrived to * store some state temporarily. There will be enough room in the mbuf's @@ -428,7 +443,7 @@ mbuf_to_synq_entry(struct mbuf *m) int buflen; if (__predict_false(M_TRAILINGSPACE(m) < len)) { - panic("%s: no room for synq_entry (%ld, %d)\n", __func__, + panic("%s: no room for synq_entry (%td, %d)\n", __func__, M_TRAILINGSPACE(m), len); } @@ -538,47 +553,26 @@ do_pass_accept_req(struct sge_qset *qs, INP_INFO_WLOCK(&V_tcbinfo); - /* Check if the 4-tuple is already in use by the host stack. */ - inp = in_pcblookup(&V_tcbinfo, inc.inc_faddr, inc.inc_fport, - inc.inc_laddr, inc.inc_lport, INPLOOKUP_WLOCKPCB, ifp); - if (inp) { - INP_WLOCK_ASSERT(inp); - if ((inp->inp_flags & INP_TIMEWAIT) == 0) { - INP_WUNLOCK(inp); - INP_INFO_WUNLOCK(&V_tcbinfo); - REJECT_PASS_ACCEPT(); - } - - /* - * In TIME_WAIT, try to reuse the 4-tuple if possible. - * tcp_twcheck always unlocks the inp. - */ - if (!tcp_twcheck(inp, &to, &th, NULL, 0)) { - INP_UNLOCK_ASSERT(inp); - INP_INFO_WUNLOCK(&V_tcbinfo); - REJECT_PASS_ACCEPT(); /* let the kernel deal with it. */ - } - - INP_UNLOCK_ASSERT(inp); - INP_INFO_WLOCK_ASSERT(&V_tcbinfo); + /* Don't offload if the 4-tuple is already in use */ + if (toe_4tuple_check(&inc, &th, ifp) != 0) { + INP_INFO_WUNLOCK(&V_tcbinfo); + REJECT_PASS_ACCEPT(); } inp = lctx->inp; /* listening socket (not owned by the TOE) */ INP_WLOCK(inp); - if (inp->inp_flags & INP_DROPPED) { + if (__predict_false(inp->inp_flags & INP_DROPPED)) { /* * The listening socket has closed. The reply from the TOE to * our CPL_CLOSE_LISTSRV_REQ will ultimately release all * resources tied to this listen context. */ INP_WUNLOCK(inp); + INP_INFO_WUNLOCK(&V_tcbinfo); REJECT_PASS_ACCEPT(); } so = inp->inp_socket; - /* Will tell the hardware to respond with a SYN/ACK */ - CTR3(KTR_CXGB, "%s: stid %u, tid %u, OK", __func__, stid, tid); - /* Reuse the mbuf that delivered the CPL to us */ synqe = mbuf_to_synq_entry(m); synqe->flags = TP_IS_A_SYNQ_ENTRY; @@ -587,17 +581,18 @@ do_pass_accept_req(struct sge_qset *qs, synqe->tid = tid; synqe->e = e; synqe->opt0h = calc_opt0h(so, 0, 0, e); -#ifdef notyet synqe->qset = pi->first_qset + (arc4random() % pi->nqsets); -#else - synqe->qset = 0; -#endif SOCKBUF_LOCK(&so->so_rcv); synqe->rx_credits = min(select_rcv_wnd(so) >> 10, M_RCV_BUFSIZ); SOCKBUF_UNLOCK(&so->so_rcv); refcount_init(&synqe->refcnt, 1); atomic_store_rel_int(&synqe->reply, RPL_OK); + insert_tid(td, synqe, tid); + TAILQ_INSERT_TAIL(&lctx->synq, synqe, link); + hold_synqe(synqe); + hold_lctx(lctx); + /* syncache_add releases both pcbinfo and pcb locks */ toe_syncache_add(&inc, &to, &th, inp, tod, synqe); INP_UNLOCK_ASSERT(inp); @@ -610,28 +605,52 @@ do_pass_accept_req(struct sge_qset *qs, * The kernel is free to retry syncache_respond but we'll ignore it due * to RPL_DONT. */ - if (atomic_cmpset_int(&synqe->reply, RPL_OK, RPL_DONT)) + if (atomic_cmpset_int(&synqe->reply, RPL_OK, RPL_DONT)) { + + INP_WLOCK(inp); + if (__predict_false(inp->inp_flags & INP_DROPPED)) { + /* listener closed. synqe must have been aborted. */ + KASSERT(synqe->flags & TP_ABORT_SHUTDOWN, + ("%s: listener %p closed but synqe %p not aborted", + __func__, inp, synqe)); + + CTR5(KTR_CXGB, + "%s: stid %u, tid %u, lctx %p, synqe %p, ABORTED", + __func__, stid, tid, lctx, synqe); + INP_WUNLOCK(inp); + release_synqe(synqe); + return (__LINE__); + } + + KASSERT(!(synqe->flags & TP_ABORT_SHUTDOWN), + ("%s: synqe %p aborted, but listener %p not dropped.", + __func__, synqe, inp)); + + TAILQ_REMOVE(&lctx->synq, synqe, link); + release_synqe(synqe); /* removed from synq list */ + inp = release_lctx(td, lctx); + if (inp) + INP_WUNLOCK(inp); + + release_synqe(synqe); /* about to exit function */ REJECT_PASS_ACCEPT(); + } KASSERT(synqe->reply == RPL_DONE, ("%s: reply %d", __func__, synqe->reply)); - TAILQ_INSERT_HEAD(&lctx->synq, synqe, link); - hold_lctx(lctx); /* Each synq entry has a ref on its lctx */ - insert_tid(td, synqe, tid); - + CTR3(KTR_CXGB, "%s: stid %u, tid %u, OK", __func__, stid, tid); + release_synqe(synqe); return (0); reject: CTR4(KTR_CXGB, "%s: stid %u, tid %u, REJECT (%d)", __func__, stid, tid, reject_reason); - if (synqe == NULL || refcount_release(&synqe->refcnt)) + if (synqe == NULL) m_freem(m); - if (e) l2t_release(td->l2t, e); - queue_tid_release(tod, tid); return (0); @@ -688,10 +707,13 @@ do_pass_establish(struct sge_qset *qs, s CTR5(KTR_CXGB, "%s: stid %u, tid %u, lctx %p, inp_flags 0x%x", __func__, stid, tid, lctx, inp->inp_flags); + KASSERT(qs->idx == synqe->qset, + ("%s qset mismatch %d %d", __func__, qs->idx, synqe->qset)); + INP_INFO_WLOCK(&V_tcbinfo); /* for syncache_expand */ INP_WLOCK(inp); - if (inp->inp_flags & INP_DROPPED) { + if (__predict_false(inp->inp_flags & INP_DROPPED)) { /* * The listening socket has closed. The TOM must have aborted * all the embryonic connections (including this one) that were @@ -713,8 +735,8 @@ do_pass_establish(struct sge_qset *qs, s to.to_tsecr = synqe->ts; th.th_ack = synqe->iss + 1; - so = inp->inp_socket; - if (!toe_syncache_expand(&inc, &to, &th, &so) || so == NULL) { + toep = toepcb_alloc(tod); + if (toep == NULL) { reset: t3_send_reset_synqe(tod, synqe); INP_WUNLOCK(inp); @@ -722,36 +744,28 @@ reset: m_freem(m); return (0); } + toep->tp_qset = qs->idx; + toep->tp_l2t = synqe->e; + toep->tp_tid = tid; + toep->tp_rx_credits = synqe->rx_credits; - toep = toepcb_alloc(tod); - if (toep == NULL) + synqe->toep = toep; + synqe->cpl = cpl; + + so = inp->inp_socket; + if (!toe_syncache_expand(&inc, &to, &th, &so) || so == NULL) { + toepcb_free(toep); goto reset; + } /* Remove the synq entry and release its reference on the lctx */ TAILQ_REMOVE(&lctx->synq, synqe, link); inp = release_lctx(td, lctx); if (inp) INP_WUNLOCK(inp); - - KASSERT(qs->idx == synqe->qset, - ("%s qset mismatch %d %d", __func__, qs->idx, synqe->qset)); - - toep->tp_qset = qs->idx; - toep->tp_l2t = synqe->e; - toep->tp_tid = tid; - toep->tp_rx_credits = synqe->rx_credits; - update_tid(td, toep, tid); - - inp = sotoinpcb(so); /* Brand new socket, not the listening socket */ - INP_WLOCK(inp); - - offload_socket(so, toep); - make_established(so, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt); - - INP_WUNLOCK(inp); INP_INFO_WUNLOCK(&V_tcbinfo); + release_synqe(synqe); - m_freem(synqe->m); m_freem(m); return (0); } @@ -821,11 +835,7 @@ t3_listen_start(struct toedev *tod, stru if (listen_hash_find(td, inp) != NULL) goto done; /* already setup */ -#ifdef notyet lctx = alloc_lctx(td, inp, pi->first_qset); -#else - lctx = alloc_lctx(td, inp, 0); -#endif if (lctx == NULL) { log(LOG_ERR, "%s: listen request ignored, %s couldn't allocate lctx\n", @@ -905,7 +915,7 @@ t3_syncache_added(struct toedev *tod __u { struct synq_entry *synqe = arg; - refcount_acquire(&synqe->refcnt); + hold_synqe(synqe); } void @@ -913,8 +923,7 @@ t3_syncache_removed(struct toedev *tod _ { struct synq_entry *synqe = arg; - if (refcount_release(&synqe->refcnt)) - m_freem(synqe->m); + release_synqe(synqe); } /* XXX */ @@ -1025,7 +1034,7 @@ do_abort_req_synqe(struct sge_qset *qs, INP_WUNLOCK(inp); release_tid(tod, tid, qs->idx); l2t_release(td->l2t, synqe->e); - m_freem(synqe->m); + release_synqe(synqe); } send_abort_rpl(tod, tid, qs->idx); @@ -1064,7 +1073,7 @@ do_abort_rpl_synqe(struct sge_qset *qs, INP_WUNLOCK(inp); release_tid(tod, tid, qs->idx); l2t_release(td->l2t, synqe->e); - m_freem(synqe->m); + release_synqe(synqe); } } @@ -1084,7 +1093,6 @@ t3_send_reset_synqe(struct toedev *tod, struct inpcb *inp = lctx->inp; #endif - INP_INFO_WLOCK_ASSERT(&V_tcbinfo); INP_WLOCK_ASSERT(inp); CTR4(KTR_CXGB, "%s: tid %d, synqe %p (%x)", __func__, tid, synqe, @@ -1108,3 +1116,23 @@ t3_send_reset_synqe(struct toedev *tod, l2t_send(sc, m, synqe->e); } + +void +t3_offload_socket(struct toedev *tod, void *arg, struct socket *so) +{ + struct adapter *sc = tod->tod_softc; + struct tom_data *td = sc->tom_softc; + struct synq_entry *synqe = arg; +#ifdef INVARIANTS + struct inpcb *inp = sotoinpcb(so); +#endif + struct cpl_pass_establish *cpl = synqe->cpl; + struct toepcb *toep = synqe->toep; + + INP_INFO_LOCK_ASSERT(&V_tcbinfo); /* prevents bad race with accept() */ + INP_WLOCK_ASSERT(inp); + + offload_socket(so, toep); + make_established(so, cpl->snd_isn, cpl->rcv_isn, cpl->tcp_opt); + update_tid(td, toep, synqe->tid); +} Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_toepcb.h ============================================================================== --- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_toepcb.h Sat May 26 01:24:39 2012 (r236034) +++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_toepcb.h Sat May 26 01:33:07 2012 (r236035) @@ -43,6 +43,7 @@ #define TP_CPL_DONE (1 << 8) #define TP_IS_A_SYNQ_ENTRY (1 << 9) #define TP_ABORT_RPL_SENT (1 << 10) +#define TP_SEND_FIN (1 << 11) struct toepcb { TAILQ_ENTRY(toepcb) link; /* toep_list */ Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.c ============================================================================== --- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.c Sat May 26 01:24:39 2012 (r236034) +++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.c Sat May 26 01:33:07 2012 (r236035) @@ -260,6 +260,7 @@ t3_tom_activate(struct adapter *sc) tod->tod_syncache_added = t3_syncache_added; tod->tod_syncache_removed = t3_syncache_removed; tod->tod_syncache_respond = t3_syncache_respond; + tod->tod_offload_socket = t3_offload_socket; /* port MTUs */ mtus = sc->port[0].ifp->if_mtu; Modified: user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.h ============================================================================== --- user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.h Sat May 26 01:24:39 2012 (r236034) +++ user/np/toe_iwarp/sys/dev/cxgb/ulp/tom/cxgb_tom.h Sat May 26 01:33:07 2012 (r236035) @@ -109,6 +109,8 @@ struct synq_entry { int tid; struct mbuf *m; /* backpointer to containing mbuf */ struct listen_ctx *lctx; /* backpointer to listen ctx */ + struct cpl_pass_establish *cpl; + struct toepcb *toep; struct l2t_entry *e; uint32_t iss; uint32_t ts; @@ -273,4 +275,5 @@ void t3_syncache_removed(struct toedev * int t3_syncache_respond(struct toedev *, void *, struct mbuf *); int do_abort_req_synqe(struct sge_qset *, struct rsp_desc *, struct mbuf *); int do_abort_rpl_synqe(struct sge_qset *, struct rsp_desc *, struct mbuf *); +void t3_offload_socket(struct toedev *, void *, struct socket *); #endif
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201205260133.q4Q1X8VJ044612>