From owner-p4-projects@FreeBSD.ORG Thu Feb 7 04:36:56 2008 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 209CF16A468; Thu, 7 Feb 2008 04:36:56 +0000 (UTC) Delivered-To: perforce@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id C1B9D16A420 for ; Thu, 7 Feb 2008 04:36:55 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (repoman.freebsd.org [IPv6:2001:4f8:fff6::29]) by mx1.freebsd.org (Postfix) with ESMTP id 962E713C455 for ; Thu, 7 Feb 2008 04:36:55 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.14.1/8.14.1) with ESMTP id m174atjI002746 for ; Thu, 7 Feb 2008 04:36:55 GMT (envelope-from kmacy@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.14.1/8.14.1/Submit) id m174atu0002740 for perforce@freebsd.org; Thu, 7 Feb 2008 04:36:55 GMT (envelope-from kmacy@freebsd.org) Date: Thu, 7 Feb 2008 04:36:55 GMT Message-Id: <200802070436.m174atu0002740@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to kmacy@freebsd.org using -f From: Kip Macy To: Perforce Change Reviews Cc: Subject: PERFORCE change 134954 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 07 Feb 2008 04:36:56 -0000 http://perforce.freebsd.org/chv.cgi?CH=134954 Change 134954 by kmacy@kmacy:storage:toehead on 2008/02/07 04:36:06 add inline SBAPPEND for validating mbufs and socket buffers move back to sbappend acquire socket buffer lock earlier to serialize access to DDP state Affected files ... .. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#15 edit Differences ... ==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#15 (text+ko) ==== @@ -152,6 +152,38 @@ #endif +static inline void +SBAPPEND(struct sockbuf *sb, struct mbuf *n) +{ + struct mbuf * m; + + m = sb->sb_mb; + while (m) { + KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || + !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n", + !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len)); + KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x", + m->m_next, m->m_nextpkt, m->m_flags)); + m = m->m_next; + } + m = n; + while (m) { + KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || + !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n", + !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len)); + KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x", + m->m_next, m->m_nextpkt, m->m_flags)); + m = m->m_next; + } + sbappend_locked(sb, n); + m = sb->sb_mb; + while (m) { + KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x", + m->m_next, m->m_nextpkt, m->m_flags)); + m = m->m_next; + } +} + static inline int is_t3a(const struct toedev *dev) { @@ -221,6 +253,7 @@ m->m_pkthdr.len = m->m_len = sizeof(*req); req = mtod(m, struct cpl_tid_release *); req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wr_lo = 0; OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); } @@ -470,6 +503,7 @@ req = mtod(m, struct cpl_rx_data_ack *); req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wr_lo = 0; OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid)); req->credit_dack = htonl(dack | V_RX_CREDITS(credits)); m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toep)); @@ -492,6 +526,7 @@ req = mtod(m, struct cpl_rx_data_ack *); req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wr_lo = 0; m->m_pkthdr.len = m->m_len = sizeof(*req); OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid)); @@ -743,6 +778,7 @@ req = mtod(m, struct cpl_set_tcb_field *); m->m_pkthdr.len = m->m_len = sizeof(*req); req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wr_lo = 0; OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tp_tid)); req->reply = V_NO_REPLY(no_reply); req->cpu_idx = 0; @@ -907,8 +943,10 @@ req = mtod(m, struct cpl_get_tcb *); m->m_pkthdr.len = m->m_len = sizeof(*req); req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wr_lo = 0; OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, toep->tp_tid)); req->cpuno = htons(toep->tp_qset); + req->rsvd = 0; if (sototcpcb(so)->t_state == TCPS_SYN_SENT) mbufq_tail(&toep->out_of_order_queue, m); // defer else @@ -1214,6 +1252,7 @@ m->m_pkthdr.len = m->m_len = sizeof(*req); req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); + req->wr.wr_lo = 0; OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid)); req->local_port = inp->inp_lport; req->peer_port = inp->inp_fport; @@ -1623,6 +1662,9 @@ uint64_t t; __be64 *tcb; + so = toeptoso(toep); + SOCKBUF_LOCK(&so->so_rcv); + /* Note that we only accout for CPL_GET_TCB issued by the DDP code. We * really need a cookie in order to dispatch the RPLs. */ @@ -1638,7 +1680,9 @@ m_freem(m); if (__predict_true((so->so_state & SS_NOFDREF) == 0)) - sorwakeup(so); + sorwakeup_locked(so); + else + SOCKBUF_UNLOCK(&so->so_rcv); return; } @@ -1692,9 +1736,8 @@ #endif m->m_cur_offset = bsp->cur_offset; bsp->cur_offset = ddp_offset; - m->m_pkthdr.len = ddp_offset - m->m_cur_offset; - so = toeptoso(toep); - + m->m_len = m->m_pkthdr.len = ddp_offset - m->m_cur_offset; + if (__predict_false(so_no_receive(so) && m->m_pkthdr.len)) { handle_excess_rx(toep, m); return; @@ -1736,10 +1779,13 @@ q->kbuf_posted--; bsp->flags |= DDP_BF_NODATA; } + SOCKBUF_UNLOCK(&so->so_rcv); + m_free(m); return; } } else { + SOCKBUF_UNLOCK(&so->so_rcv); /* This reply is for a CPL_GET_TCB_RPL to cancel the UBUF DDP, * but it got here way late and nobody cares anymore. */ @@ -1747,6 +1793,7 @@ return; } + KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); tp = toep->tp_tp; m->m_ddp_gl = (unsigned char *)bsp->gl; m->m_flags |= M_DDP; @@ -1762,8 +1809,7 @@ "tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u lskb->len %u", m->m_seq, q->cur_buf, m->m_pkthdr.len); #endif - SOCKBUF_LOCK(&so->so_rcv); - sbappendstream_locked(&so->so_rcv, m); + SBAPPEND(&so->so_rcv, m); if (__predict_true((so->so_state & SS_NOFDREF) == 0)) sorwakeup_locked(so); else @@ -1793,6 +1839,7 @@ handle_ddp_data(struct toepcb *toep, struct mbuf *m) { struct tcpcb *tp = toep->tp_tp; + struct socket *so = toeptoso(toep); struct ddp_state *q; struct ddp_buf_state *bsp; struct cpl_rx_data *hdr = cplhdr(m); @@ -1802,10 +1849,11 @@ return; TRACE_ENTER; + SOCKBUF_LOCK(&so->so_rcv); q = &toep->tp_ddp_state; bsp = &q->buf_state[q->cur_buf]; m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt; - + KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); printf("rcv_nxt=0x%x tp->rcv_next=0x%x len=%d\n", rcv_nxt, tp->rcv_nxt, m->m_pkthdr.len); @@ -1834,6 +1882,7 @@ * mode. */ q->ubuf_ddp_ready = 0; + SOCKBUF_UNLOCK(&so->so_rcv); TRACE_EXIT; } @@ -1910,7 +1959,7 @@ if (sb_notify(&so->so_rcv)) DPRINTF("rx_data so=%p flags=0x%x len=%d\n", so, so->so_rcv.sb_flags, m->m_pkthdr.len); - sbappendstream_locked(&so->so_rcv, m); + SBAPPEND(&so->so_rcv, m); #ifdef notyet /* @@ -2009,6 +2058,9 @@ ("length received exceeds ddp pages: len=%d dgl_length=%d", m->m_len, bsp->gl->dgl_length)); + KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); + KASSERT(m->m_next == NULL, ("m_len=%p", m->m_next)); + /* * Figure out where the new data was placed in the buffer and store it * in when. Assumes the buffer offset starts at 0, consumer needs to @@ -2045,7 +2097,7 @@ tp->t_rcvtime = ticks; SOCKBUF_LOCK(&so->so_rcv); - sbappendstream_locked(&so->so_rcv, m); + SBAPPEND(&so->so_rcv, m); if ((so->so_state & SS_NOFDREF) == 0) sorwakeup_locked(so); @@ -2072,7 +2124,7 @@ if (__predict_false(ntohl(hdr->ddpvld_status) & DDP_ERR)) { log(LOG_ERR, "RX_DATA_DDP for TID %u reported error 0x%x\n", GET_TID(hdr), G_DDP_VALID(ntohl(hdr->ddpvld_status))); - return CPL_RET_BUF_DONE; + return (CPL_RET_BUF_DONE); } #if 0 skb->h.th = tcphdr_skb->h.th; @@ -2107,9 +2159,10 @@ buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1; bsp = &q->buf_state[buf_idx]; + SOCKBUF_LOCK(&so->so_rcv); when = bsp->cur_offset; m->m_len = m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when; - + KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); #ifdef T3_TRACE T3_TRACE5(TIDTB(sk), "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u " @@ -2145,12 +2198,12 @@ tp->rcv_nxt += m->m_len; tp->t_rcvtime = ticks; - SOCKBUF_LOCK(&so->so_rcv); - sbappendstream_locked(&so->so_rcv, m); + SBAPPEND(&so->so_rcv, m); if ((so->so_state & SS_NOFDREF) == 0) sorwakeup_locked(so); - SOCKBUF_UNLOCK(&so->so_rcv); + else + SOCKBUF_UNLOCK(&so->so_rcv); TRACE_EXIT; } @@ -2229,8 +2282,10 @@ } q = &toep->tp_ddp_state; + SOCKBUF_LOCK(&so->so_rcv); bsp = &q->buf_state[q->cur_buf]; m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt; + KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); m->m_ddp_gl = (unsigned char *)bsp->gl; m->m_flags |= M_DDP; m->m_cur_offset = bsp->cur_offset; @@ -2242,9 +2297,11 @@ if (!(bsp->flags & DDP_BF_NOFLIP)) q->cur_buf ^= 1; tp->t_rcvtime = ticks; - sbappendstream(&so->so_rcv, m); + SBAPPEND(&so->so_rcv, m); if (__predict_true((so->so_state & SS_NOFDREF) == 0)) - sorwakeup(so); + sorwakeup_locked(so); + else + SOCKBUF_UNLOCK(&so->so_rcv); return (1); } @@ -3799,6 +3856,7 @@ req = mtod(m, struct ulp_mem_io *); m->m_pkthdr.len = m->m_len = sizeof(*req) + PPOD_SIZE; req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); + req->wr.wr_lo = 0; req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(ppod_addr >> 5) | V_ULPTX_CMD(ULP_MEM_WRITE)); req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE / 32) | @@ -3896,13 +3954,16 @@ struct cpl_get_tcb *getreq; struct ddp_state *p = &toep->tp_ddp_state; + SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv); wrlen = sizeof(*wr) + sizeof(*req) + 2 * sizeof(*lock) + sizeof(*getreq); m = m_gethdr_nofail(wrlen); m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); wr = mtod(m, struct work_request_hdr *); + bzero(wr, wrlen); + wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); - m->m_pkthdr.len = m->m_len = sizeof(wrlen); + m->m_pkthdr.len = m->m_len = wrlen; lock = (struct cpl_barrier *)(wr + 1); mk_cpl_barrier_ulp(lock); @@ -3932,7 +3993,7 @@ /* Keep track of the number of oustanding CPL_GET_TCB requests */ p->get_tcb_count++; - + #ifdef T3_TRACE T3_TRACE1(TIDTB(so), "t3_cancel_ddpbuf: bufidx %u", bufidx); @@ -3967,17 +4028,19 @@ struct cpl_set_tcb_field *req; struct ddp_state *p = &toep->tp_ddp_state; + SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv); wrlen = sizeof(*wr) + 3 * sizeof(*req) + sizeof(*getreq); m = m_gethdr_nofail(wrlen); m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); wr = mtod(m, struct work_request_hdr *); m->m_pkthdr.len = m->m_len = wrlen; + bzero(wr, wrlen); + /* Set the ATOMIC flag to make sure that TP processes the following * CPLs in an atomic manner and no wire segments can be interleaved. */ wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); - req = (struct cpl_set_tcb_field *)(wr + 1); mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_TAG, V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG) | @@ -4038,12 +4101,15 @@ struct work_request_hdr *wr; struct cpl_set_tcb_field *req; + SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv); wrlen = sizeof(*wr) + sizeof(*req) + (len0 ? sizeof(*req) : 0) + (len1 ? sizeof(*req) : 0) + (modulate ? sizeof(struct cpl_rx_data_ack) : 0); m = m_gethdr_nofail(wrlen); m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); wr = mtod(m, struct work_request_hdr *); + bzero(wr, wrlen); + wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); m->m_pkthdr.len = m->m_len = wrlen; @@ -4118,7 +4184,6 @@ tcphdr_skb->h.raw = tcphdr_skb->data; memset(tcphdr_skb->data, 0, tcphdr_skb->len); #endif - t3tom_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); t3tom_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl);