From owner-p4-projects@FreeBSD.ORG Tue Feb 12 21:03:50 2008 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 953C216A41A; Tue, 12 Feb 2008 21:03:50 +0000 (UTC) Delivered-To: perforce@FreeBSD.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 4154516A4C0 for ; Tue, 12 Feb 2008 21:03:50 +0000 (UTC) (envelope-from swise@FreeBSD.org) Received: from repoman.freebsd.org (repoman.freebsd.org [IPv6:2001:4f8:fff6::29]) by mx1.freebsd.org (Postfix) with ESMTP id EC83513C4DB for ; Tue, 12 Feb 2008 21:03:49 +0000 (UTC) (envelope-from swise@FreeBSD.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.14.1/8.14.1) with ESMTP id m1CL3n9q046990 for ; Tue, 12 Feb 2008 21:03:49 GMT (envelope-from swise@FreeBSD.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.14.1/8.14.1/Submit) id m1CL3njR046987 for perforce@freebsd.org; Tue, 12 Feb 2008 21:03:49 GMT (envelope-from swise@FreeBSD.org) Date: Tue, 12 Feb 2008 21:03:49 GMT Message-Id: <200802122103.m1CL3njR046987@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to swise@FreeBSD.org using -f From: Steve Wise To: Perforce Change Reviews Cc: Subject: PERFORCE change 135280 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 12 Feb 2008 21:03:51 -0000 http://perforce.freebsd.org/chv.cgi?CH=135280 Change 135280 by swise@swise:vic10:iwarp on 2008/02/12 21:03:35 krping server side works over toe socket. - added more PDBG() stuff - only do a shutdown() to initiate a normal close, and when we get the ec_status() we either close the socket or abort the connection based on the ec_status. - connection aborts don't need any upcalls after doing a soclose(linger==0). - fixed a bug in iwch_modify_qp() - changed ep lock to a spinlock to handle upcall int context Affected files ... .. //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c#3 edit .. //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_qp.c#2 edit Differences ... ==== //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.c#3 (text+ko) ==== @@ -223,9 +223,9 @@ { enum iwch_ep_state state; - mtx_lock(&epc->lock); + mtx_lock_spin(&epc->lock); state = epc->state; - mtx_unlock(&epc->lock); + mtx_unlock_spin(&epc->lock); return state; } @@ -239,10 +239,10 @@ state_set(struct iwch_ep_common *epc, enum iwch_ep_state new) { - mtx_lock(&epc->lock); + mtx_lock_spin(&epc->lock); PDBG("%s - %s -> %s\n", __FUNCTION__, states[epc->state], states[new]); __state_set(epc, new); - mtx_unlock(&epc->lock); + mtx_unlock_spin(&epc->lock); return; } @@ -255,7 +255,7 @@ if (epc) { memset(epc, 0, size); refcount_init(&epc->refcount, 1); - mtx_init(&epc->lock, "iwch_epc lock", NULL, MTX_DEF); + mtx_init(&epc->lock, "iwch_epc lock", NULL, MTX_SPIN|MTX_DUPOK); cv_init(&epc->waitq, "iwch_epc cv"); } PDBG("%s alloc ep %p\n", __FUNCTION__, epc); @@ -339,6 +339,7 @@ static void close_socket(struct iwch_ep_common *epc) { + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, epc, epc->so, states[epc->state]); SOCK_LOCK(epc->so); epc->so->so_upcall = NULL; epc->so->so_upcallarg = NULL; @@ -349,12 +350,20 @@ } static void +shutdown_socket(struct iwch_ep_common *epc) +{ + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, epc, epc->so, states[epc->state]); + soshutdown(epc->so, SHUT_WR); +} + +static void abort_socket(struct iwch_ep *ep) { struct sockopt sopt; int err; struct linger l; + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); l.l_onoff = 1; l.l_linger = 0; @@ -368,7 +377,6 @@ err = sosetopt(ep->com.so, &sopt); if (err) printf("%s can't set linger to 0, no RST! err %d\n", __FUNCTION__, err); - close_socket(&ep->com); } static void @@ -477,19 +485,11 @@ } static void -abort_connection(struct iwch_ep *ep) -{ - PDBG("%s ep %p\n", __FILE__, ep); - state_set(&ep->com, ABORTING); - abort_socket(ep); -} - -static void close_complete_upcall(struct iwch_ep *ep) { struct iw_cm_event event; - PDBG("%s ep %p\n", __FUNCTION__, ep); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_CLOSE; if (ep->com.cm_id) { @@ -503,11 +503,23 @@ } static void +abort_connection(struct iwch_ep *ep) +{ + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); + state_set(&ep->com, ABORTING); + abort_socket(ep); + close_socket(&ep->com); + close_complete_upcall(ep); + state_set(&ep->com, DEAD); + put_ep(&ep->com); +} + +static void peer_close_upcall(struct iwch_ep *ep) { struct iw_cm_event event; - PDBG("%s ep %p\n", __FUNCTION__, ep); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_DISCONNECT; if (ep->com.cm_id) { @@ -522,7 +534,7 @@ { struct iw_cm_event event; - PDBG("%s ep %p\n", __FUNCTION__, ep); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_CLOSE; event.status = ECONNRESET; @@ -541,7 +553,7 @@ { struct iw_cm_event event; - PDBG("%s ep %p status %d\n", __FUNCTION__, ep, status); + PDBG("%s ep %p so %p state %s status %d\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state], status); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_CONNECT_REPLY; event.status = status; @@ -569,7 +581,7 @@ { struct iw_cm_event event; - PDBG("%s ep %p tid %d\n", __FUNCTION__, ep, ep->hwtid); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_CONNECT_REQUEST; event.local_addr = ep->com.local_addr; @@ -590,7 +602,7 @@ { struct iw_cm_event event; - PDBG("%s ep %p\n", __FUNCTION__, ep); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_ESTABLISHED; if (ep->com.cm_id) { @@ -612,7 +624,7 @@ struct uio uio; int len; - PDBG("%s ep %p\n", __FUNCTION__, ep); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); /* * Stop mpa timer. If it expired, then the state has @@ -765,7 +777,7 @@ struct uio uio; int len; - PDBG("%s ep %p so %p\n", __FUNCTION__, ep, ep->com.so); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); /* * Stop mpa timer. If it expired, then the state has @@ -905,9 +917,9 @@ int disconnect = 1; int release = 0; - PDBG("%s ep %p\n", __FUNCTION__, ep); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); - mtx_lock(&ep->com.lock); + mtx_lock_spin(&ep->com.lock); switch (ep->com.state) { case MPA_REQ_WAIT: __state_set(&ep->com, CLOSING); @@ -962,7 +974,7 @@ default: BUG_ON(1); } - mtx_unlock(&ep->com.lock); + mtx_unlock_spin(&ep->com.lock); if (disconnect) iwch_ep_disconnect(ep, 0, M_NOWAIT); if (release) @@ -978,7 +990,7 @@ int state; state = state_read(&ep->com); - PDBG("%s ep %p state %u\n", __FUNCTION__, ep, state); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); switch (state) { case MPA_REQ_WAIT: stop_ep_timer(ep); @@ -1041,11 +1053,11 @@ struct iwch_qp_attributes attrs; int release = 0; - PDBG("%s ep %p\n", __FUNCTION__, ep); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); BUG_ON(!ep); /* The cm_id may be null if we failed to connect */ - mtx_lock(&ep->com.lock); + mtx_lock_spin(&ep->com.lock); switch (ep->com.state) { case CLOSING: __state_set(&ep->com, MORIBUND); @@ -1070,7 +1082,7 @@ BUG_ON(1); break; } - mtx_unlock(&ep->com.lock); + mtx_unlock_spin(&ep->com.lock); if (release) put_ep(&ep->com); return; @@ -1091,7 +1103,9 @@ static int terminate(struct t3cdev *tdev, struct mbuf *m, void *ctx) { - struct iwch_ep *ep = ctx; + struct toepcb *toep = (struct toepcb *)ctx; + struct socket *so = toeptoso(toep); + struct iwch_ep *ep = so->so_upcallarg; PDBG("%s ep %p\n", __FUNCTION__, ep); m_adj(m, sizeof(struct cpl_rdma_terminate)); @@ -1105,23 +1119,40 @@ static int ec_status(struct t3cdev *tdev, struct mbuf *m, void *ctx) { + struct toepcb *toep = (struct toepcb *)ctx; + struct socket *so = toeptoso(toep); struct cpl_rdma_ec_status *rep = cplhdr(m); - struct iwch_ep *ep = ctx; + struct iwch_ep *ep = so->so_upcallarg; + struct iwch_qp_attributes attrs; + int release; - PDBG("%s ep %p tid %u status %d\n", __FUNCTION__, ep, ep->hwtid, - rep->status); + PDBG("%s ep %p so %p state %s ec_status %d\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state], rep->status); + mtx_lock_spin(&ep->com.lock); + stop_ep_timer(ep); if (rep->status) { - struct iwch_qp_attributes attrs; log(LOG_ERR, "%s BAD CLOSE - Aborting tid %u\n", __FUNCTION__, ep->hwtid); - stop_ep_timer(ep); attrs.next_state = IWCH_QP_STATE_ERROR; iwch_modify_qp(ep->com.qp->rhp, ep->com.qp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 1); - abort_connection(ep); + } else { + if ((ep->com.cm_id) && (ep->com.qp)) { + attrs.next_state = IWCH_QP_STATE_IDLE; + iwch_modify_qp(ep->com.qp->rhp, + ep->com.qp, + IWCH_QP_ATTR_NEXT_STATE, + &attrs, 1); + } + close_complete_upcall(ep); + close_socket(&ep->com); + __state_set(&ep->com, DEAD); + release = 1; } + mtx_unlock_spin(&ep->com.lock); + if (release) + put_ep(&ep->com); return CPL_RET_BUF_DONE; } @@ -1131,9 +1162,8 @@ struct iwch_ep *ep = (struct iwch_ep *)arg; struct iwch_qp_attributes attrs; - mtx_lock(&ep->com.lock); - PDBG("%s ep %p tid %u state %d\n", __FUNCTION__, ep, ep->hwtid, - ep->com.state); + mtx_lock_spin(&ep->com.lock); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); switch (ep->com.state) { case MPA_REQ_SENT: connect_reply_upcall(ep, -ETIMEDOUT); @@ -1153,7 +1183,7 @@ panic("unknown state: %d\n", ep->com.state); } __state_set(&ep->com, CLOSING); - mtx_unlock(&ep->com.lock); + mtx_unlock_spin(&ep->com.lock); abort_connection(ep); put_ep(&ep->com); } @@ -1163,7 +1193,7 @@ { int err; struct iwch_ep *ep = to_ep(cm_id); - PDBG("%s ep %p tid %u\n", __FUNCTION__, ep, ep->hwtid); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); if (state_read(&ep->com) == DEAD) { put_ep(&ep->com); @@ -1189,7 +1219,7 @@ struct iwch_dev *h = to_iwch_dev(cm_id->device); struct iwch_qp *qp = get_qhp(h, conn_param->qpn); - PDBG("%s ep %p so %p\n", __FUNCTION__, ep, ep->com.so); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); if (state_read(&ep->com) == DEAD) return (-ECONNRESET); @@ -1411,7 +1441,10 @@ { int close = 0; - mtx_lock(&ep->com.lock); + mtx_lock_spin(&ep->com.lock); + + BUG_ON(!ep); + BUG_ON(!ep->com.so); PDBG("%s ep %p so %p state %s, abrupt %d\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state], abrupt); @@ -1450,12 +1483,12 @@ break; } out: - mtx_unlock(&ep->com.lock); + mtx_unlock_spin(&ep->com.lock); if (close) { if (abrupt) - abort_socket(ep); + abort_connection(ep); else - close_socket(&ep->com); + shutdown_socket(&ep->com); } return 0; } @@ -1465,7 +1498,7 @@ { struct sockaddr_in *local, *remote; - PDBG("%s ep %p so %p\n", __FUNCTION__, ep, ep->com.so); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); switch (state_read(&ep->com)) { case MPA_REQ_SENT: @@ -1490,11 +1523,11 @@ free(remote, M_SONAME); process_mpa_request(ep); break; - case MPA_REP_SENT: default: - printf("%s Unexpected streaming data." - " ep %p state %d so %p\n", - __FUNCTION__, ep, state_read(&ep->com), ep->com.so); + if (ep->com.so->so_rcv.sb_cc) + printf("%s Unexpected streaming data." + " ep %p state %d so %p\n", + __FUNCTION__, ep, state_read(&ep->com), ep->com.so); break; } return; @@ -1503,7 +1536,7 @@ static void process_connected(struct iwch_ep *ep) { - PDBG("%s ep %p so %p\n", __FUNCTION__, ep, ep->com.so); + PDBG("%s ep %p so %p state %s\n", __FUNCTION__, ep, ep->com.so, states[ep->com.state]); if ((ep->com.so->so_state & SS_ISCONNECTED) && !ep->com.so->so_error) { send_mpa_req(ep); } else { @@ -1615,7 +1648,7 @@ process_close_complete(ep); return; } - + /* rx data */ process_data(ep); return; ==== //depot/projects/iwarp/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_qp.c#2 (text+ko) ==== @@ -936,6 +936,7 @@ wakeup(qhp); break; case IWCH_QP_STATE_ERROR: + disconnect=1; goto err; default: ret = -EINVAL; @@ -996,7 +997,7 @@ * an abnormal close (RTS/CLOSING->ERROR). */ if (disconnect) - iwch_ep_disconnect(ep, abort, M_WAITOK); + iwch_ep_disconnect(ep, abort, M_NOWAIT); /* * If free is 1, then we've disassociated the EP from the QP