From owner-p4-projects@FreeBSD.ORG Fri Jan 25 01:18:13 2008 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 5D0D816A420; Fri, 25 Jan 2008 01:18:13 +0000 (UTC) Delivered-To: perforce@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 1D86B16A41A for ; Fri, 25 Jan 2008 01:18:13 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (repoman.freebsd.org [IPv6:2001:4f8:fff6::29]) by mx1.freebsd.org (Postfix) with ESMTP id 08E0313C448 for ; Fri, 25 Jan 2008 01:18:13 +0000 (UTC) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.14.1/8.14.1) with ESMTP id m0P1IDBB047296 for ; Fri, 25 Jan 2008 01:18:13 GMT (envelope-from kmacy@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.14.1/8.14.1/Submit) id m0P1IC1V047293 for perforce@freebsd.org; Fri, 25 Jan 2008 01:18:12 GMT (envelope-from kmacy@freebsd.org) Date: Fri, 25 Jan 2008 01:18:12 GMT Message-Id: <200801250118.m0P1IC1V047293@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to kmacy@freebsd.org using -f From: Kip Macy To: Perforce Change Reviews Cc: Subject: PERFORCE change 134046 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 25 Jan 2008 01:18:13 -0000 http://perforce.freebsd.org/chv.cgi?CH=134046 Change 134046 by kmacy@kmacy:storage:toehead on 2008/01/25 01:17:18 import more DDP cpl_io infrastructure Affected files ... .. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#6 edit .. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c#5 edit .. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_defs.h#2 edit .. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h#5 edit Differences ... ==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#6 (text+ko) ==== @@ -189,36 +189,35 @@ * it is sent directly. */ static inline void -send_or_defer(struct socket *so, struct tcpcb *tp, struct mbuf *m, int through_l2t) +send_or_defer(struct toepcb *toep, struct mbuf *m, int through_l2t) { - struct toepcb *toep = tp->t_toe; + struct tcpcb *tp = toep->tp_tp; - if (__predict_false(tp->t_state == TCPS_SYN_SENT)) { INP_LOCK(tp->t_inpcb); mbufq_tail(&toep->out_of_order_queue, m); // defer INP_UNLOCK(tp->t_inpcb); } else if (through_l2t) - l2t_send(T3C_DEV(so), m, toep->tp_l2t); // send through L2T + l2t_send(TOEP_T3C_DEV(toep), m, toep->tp_l2t); // send through L2T else - cxgb_ofld_send(T3C_DEV(so), m); // send directly + cxgb_ofld_send(TOEP_T3C_DEV(toep), m); // send directly } static inline unsigned int -mkprio(unsigned int cntrl, const struct socket *so) +mkprio(unsigned int cntrl, const struct toepcb *toep) { - return cntrl; + return (cntrl); } /* * Populate a TID_RELEASE WR. The skb must be already propely sized. */ static inline void -mk_tid_release(struct mbuf *m, const struct socket *so, unsigned int tid) +mk_tid_release(struct mbuf *m, const struct toepcb *toep, unsigned int tid) { struct cpl_tid_release *req; - m_set_priority(m, mkprio(CPL_PRIORITY_SETUP, so)); + m_set_priority(m, mkprio(CPL_PRIORITY_SETUP, toep)); m->m_pkthdr.len = m->m_len = sizeof(*req); req = mtod(m, struct cpl_tid_release *); req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); @@ -357,7 +356,7 @@ toep->tp_wr_unacked += mbuf_wrs[count]; make_tx_data_wr(so, m0, bytes, tail); - m_set_priority(m0, mkprio(CPL_PRIORITY_DATA, so)); + m_set_priority(m0, mkprio(CPL_PRIORITY_DATA, toep)); m_set_sgl(m0, segs); m_set_sgllen(m0, count); /* @@ -473,12 +472,67 @@ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid)); req->credit_dack = htonl(dack | V_RX_CREDITS(credits)); - m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toeptoso(toep))); + m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toep)); cxgb_ofld_send(TOM_DATA(tdev)->cdev, m); return (credits); } /* + * Send RX_DATA_ACK CPL message to request a modulation timer to be scheduled. + * This is only used in DDP mode, so we take the opportunity to also set the + * DACK mode and flush any Rx credits. + */ +void +t3_send_rx_modulate(struct toepcb *toep) +{ + struct mbuf *m; + struct cpl_rx_data_ack *req; + + m = m_gethdr_nofail(sizeof(*req)); + +#ifdef notyet + req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req)); + req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); +#else + req = mtod(m, struct cpl_rx_data_ack *); +#endif + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid)); + req->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE | + V_RX_DACK_MODE(1) | + V_RX_CREDITS(toep->tp_copied_seq - toep->tp_rcv_wup)); + m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); + cxgb_ofld_send(TOEP_T3C_DEV(toep), m); + toep->tp_rcv_wup = toep->tp_copied_seq; +} + +/* + * Handle receipt of an urgent pointer. + */ +static void +handle_urg_ptr(struct socket *so, uint32_t urg_seq) +{ +#ifdef notyet + struct tcpcb *tp = sototcpcb(so); + + urg_seq--; /* initially points past the urgent data, per BSD */ + + if (tp->urg_data && !after(urg_seq, tp->urg_seq)) + return; /* duplicate pointer */ + sk_send_sigurg(sk); + if (tp->urg_seq == tp->copied_seq && tp->urg_data && + !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) { + struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); + + tp->copied_seq++; + if (skb && tp->copied_seq - TCP_SKB_CB(skb)->seq >= skb->len) + tom_eat_skb(sk, skb, 0); + } + tp->urg_data = TCP_URG_NOTYET; + tp->urg_seq = urg_seq; +#endif +} + +/* * Returns true if a socket cannot accept new Rx data. */ static inline int @@ -488,6 +542,35 @@ } /* + * Process an urgent data notification. + */ +static void +rx_urg_notify(struct toepcb *toep, struct mbuf *m) +{ + struct cpl_rx_urg_notify *hdr = cplhdr(m); + struct socket *so = toeptoso(toep); + + VALIDATE_SOCK(so); + + if (!so_no_receive(so)) + handle_urg_ptr(so, ntohl(hdr->seq)); + + m_freem(m); +} + +/* + * Handler for RX_URG_NOTIFY CPL messages. + */ +static int +do_rx_urg_notify(struct t3cdev *cdev, struct mbuf *m, void *ctx) +{ + struct toepcb *toep = (struct toepcb *)ctx; + + rx_urg_notify(toep, m); + return (0); +} + +/* * Set of states for which we should return RX credits. */ #define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2) @@ -643,12 +726,10 @@ static void -__set_tcb_field(struct socket *so, struct mbuf *m, uint16_t word, +__set_tcb_field(struct toepcb *toep, struct mbuf *m, uint16_t word, uint64_t mask, uint64_t val, int no_reply) { struct cpl_set_tcb_field *req; - struct tcpcb *tp = sototcpcb(so); - struct toepcb *toep = tp->t_toe; req = mtod(m, struct cpl_set_tcb_field *); m->m_pkthdr.len = m->m_len = sizeof(*req); @@ -660,8 +741,8 @@ req->mask = htobe64(mask); req->val = htobe64(val); - m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, so)); - send_or_defer(so, tp, m, 0); + m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); + send_or_defer(toep, m, 0); } static void @@ -679,7 +760,7 @@ m = m_gethdr_nofail(sizeof(struct cpl_set_tcb_field)); - __set_tcb_field(so, m, word, mask, val, 1); + __set_tcb_field(toep, m, word, mask, val, 1); } /* @@ -759,7 +840,6 @@ } - void t3_set_ddp_tag(struct socket *so, int buf_idx, unsigned int tag_color) { @@ -814,7 +894,7 @@ return (ENOMEM); INP_LOCK_ASSERT(tp->t_inpcb); - m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, so)); + m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); req = mtod(m, struct cpl_get_tcb *); m->m_pkthdr.len = m->m_len = sizeof(*req); req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); @@ -1097,7 +1177,7 @@ struct toepcb *toep = tp->t_toe; struct toedev *tdev = TOE_DEV(so); - m_set_priority((struct mbuf *)m, mkprio(CPL_PRIORITY_SETUP, so)); + m_set_priority((struct mbuf *)m, mkprio(CPL_PRIORITY_SETUP, toep)); req = mtod(m, struct cpl_act_open_req *); m->m_pkthdr.len = m->m_len = sizeof(*req); @@ -1346,7 +1426,7 @@ mode |= CPL_ABORT_POST_CLOSE_REQ; m = m_gethdr_nofail(sizeof(*req)); - m_set_priority(m, mkprio(CPL_PRIORITY_DATA, so)); + m_set_priority(m, mkprio(CPL_PRIORITY_DATA, toep)); set_arp_failure_handler(m, abort_arp_failure); req = mtod(m, struct cpl_abort_req *); @@ -1901,10 +1981,9 @@ tp->t_rcvtime = ticks; sbappendstream_locked(&so->so_rcv, m); -#ifdef notyet - if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_data_ready(sk, 0); -#endif + + if ((so->so_state & SS_NOFDREF) == 0) + sorwakeup_locked(so); } #define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\ @@ -1979,9 +2058,7 @@ tp->rcv_nxt, bsp->cur_offset, ddp_report, G_DDP_OFFSET(ddp_report)); #endif -#if 0 - skb->mac.raw = (unsigned char *)bsp->gl; -#endif + m->m_ddp_gl = (unsigned char *)bsp->gl; m->m_pkthdr.csum_flags = (bsp->flags & DDP_BF_NOCOPY) | 1; if (bsp->flags & DDP_BF_NOCOPY) bsp->flags &= ~DDP_BF_NOCOPY; @@ -1989,8 +2066,8 @@ tp->rcv_nxt += m->m_len; tp->t_rcvtime = ticks; - sbappendstream_locked(&so->so_rcv, m) - ; + sbappendstream_locked(&so->so_rcv, m); + if ((so->so_state & SS_NOFDREF) == 0) sorwakeup_locked(so); @@ -2037,6 +2114,61 @@ } /* + * For TCP DDP a PEER_CLOSE may also be an implicit RX_DDP_COMPLETE. This + * function deals with the data that may be reported along with the FIN. + * Returns -1 if no further processing of the PEER_CLOSE is needed, >= 0 to + * perform normal FIN-related processing. In the latter case 1 indicates that + * there was an implicit RX_DDP_COMPLETE and the skb should not be freed, 0 the + * skb can be freed. + */ +static int +handle_peer_close_data(struct socket *so, struct mbuf *m) +{ + struct tcpcb *tp = sototcpcb(so); + struct toepcb *toep = tp->t_toe; + struct ddp_state *q; + struct ddp_buf_state *bsp; + struct cpl_peer_close *req = cplhdr(m); + unsigned int rcv_nxt = ntohl(req->rcv_nxt) - 1; /* exclude FIN */ + + if (tp->rcv_nxt == rcv_nxt) /* no data */ + return 0; + + if (__predict_false(so_no_receive(so))) { + handle_excess_rx(toep, m); + + /* + * Although we discard the data we want to process the FIN so + * that PEER_CLOSE + data behaves the same as RX_DATA_DDP + + * PEER_CLOSE without data. In particular this PEER_CLOSE + * may be what will close the connection. We return 1 because + * handle_excess_rx() already freed the packet. + */ + return (1); + } + + q = &toep->tp_ddp_state; + bsp = &q->buf_state[q->cur_buf]; + m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt; + m->m_ddp_gl = (unsigned char *)bsp->gl; + m->m_cur_offset = bsp->cur_offset; + m->m_ddp_flags = + DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1; + m->m_seq = tp->rcv_nxt; + tp->rcv_nxt = rcv_nxt; + bsp->cur_offset += m->m_pkthdr.len; + if (!(bsp->flags & DDP_BF_NOFLIP)) + q->cur_buf ^= 1; + tp->t_rcvtime = ticks; +#ifdef notyet + __skb_queue_tail(&sk->sk_receive_queue, skb); +#endif + if (__predict_true((so->so_state & SS_NOFDREF) == 0)) + sorwakeup(so); + return (1); +} + +/* * Handle a peer FIN. */ static void @@ -2058,15 +2190,11 @@ goto out; } -#ifdef notyet - if (ULP_MODE(tp) == ULP_MODE_TCPDDP) { - keep = handle_peer_close_data(so, skb); + if (toep->tp_ulp_mode == ULP_MODE_TCPDDP) { + keep = handle_peer_close_data(so, m); if (keep < 0) return; } - sk->sk_shutdown |= RCV_SHUTDOWN; - sock_set_flag(so, SOCK_DONE); -#endif INP_INFO_WLOCK(&tcbinfo); INP_LOCK(tp->t_inpcb); if (TCPS_HAVERCVDFIN(tp->t_state) == 0) @@ -2113,8 +2241,6 @@ sowwakeup(so); wakeup(&so->so_timeo); #ifdef notyet - sk->sk_state_change(sk); - /* Do not send POLL_HUP for half duplex close. */ if ((sk->sk_shutdown & SEND_SHUTDOWN) || sk->sk_state == TCP_CLOSE) @@ -2525,10 +2651,8 @@ (is_t3a(TOE_DEV(so)) && (toep->tp_flags & TP_CLOSE_CON_REQUESTED))) { so->so_error = abort_status_to_errno(so, req->status, &rst_status); -#if 0 - if (!sock_flag(sk, SOCK_DEAD)) - sk->sk_error_report(sk); -#endif + if (__predict_true((so->so_state & SS_NOFDREF) == 0)) + sorwakeup(so); /* * SYN_RECV needs special processing. If abort_syn_rcv() * returns 0 is has taken care of the abort. @@ -2937,8 +3061,7 @@ DPRINTF("opt0l_status=%08x\n", rpl->opt0l_status); - m_set_priority(reply_mbuf, mkprio(CPL_PRIORITY_SETUP, so)); - + m_set_priority(reply_mbuf, mkprio(CPL_PRIORITY_SETUP, newtoep)); #ifdef DEBUG_PRINT { int i; @@ -2954,24 +3077,22 @@ l2t_send(cdev, reply_mbuf, e); m_free(m); -#ifdef notyet /* * XXX this call path has to be converted to not depend on sockets */ if (newtoep->tp_ulp_mode) - __set_tcb_field(newso, ddp_mbuf, W_TCB_RX_DDP_FLAGS, + __set_tcb_field(newtoep, ddp_mbuf, W_TCB_RX_DDP_FLAGS, V_TF_DDP_OFF(1) | TP_DDP_TIMER_WORKAROUND_MASK, V_TF_DDP_OFF(1) | TP_DDP_TIMER_WORKAROUND_VAL, 1); -#endif return; reject: if (tdev->tod_ttid == TOE_ID_CHELSIO_T3) mk_pass_accept_rpl(reply_mbuf, m); else - mk_tid_release(reply_mbuf, NULL, tid); + mk_tid_release(reply_mbuf, newtoep, tid); cxgb_ofld_send(cdev, reply_mbuf); m_free(m); out: @@ -3184,10 +3305,9 @@ tp = sototcpcb(so); INP_LOCK(tp->t_inpcb); -#ifdef notyet - so->so_snd.sb_flags |= SB_TOE; - so->so_rcv.sb_flags |= SB_TOE; -#endif + + so->so_snd.sb_flags |= SB_NOCOALESCE; + so->so_rcv.sb_flags |= SB_NOCOALESCE; toep->tp_tp = tp; toep->tp_flags = 0; tp->t_toe = toep; @@ -3495,6 +3615,15 @@ return 0; } +/* + * Handler for TRACE_PKT CPL messages. Just sink these packets. + */ +static int +do_trace_pkt(struct t3cdev *dev, struct mbuf *m, void *ctx) +{ + m_freem(m); + return 0; +} /* * Reset a connection that is on a listener's SYN queue or accept queue, @@ -3574,7 +3703,7 @@ for (i = 0; i < nppods; ++i) { m = m_gethdr_nofail(sizeof(*req) + PPOD_SIZE); - m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, so)); + m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); req = mtod(m, struct ulp_mem_io *); m->m_pkthdr.len = m->m_len = sizeof(*req) + PPOD_SIZE; req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); @@ -3596,14 +3725,281 @@ htobe64(VM_PAGE_TO_PHYS(gl->dgl_pages[pidx])) : 0; } else p->pp_vld_tid = 0; /* mark sentinel page pods invalid */ - send_or_defer(so, tp, m, 0); + send_or_defer(toep, m, 0); ppod_addr += PPOD_SIZE; } return (0); } +/* + * Build a CPL_BARRIER message as payload of a ULP_TX_PKT command. + */ +static inline void +mk_cpl_barrier_ulp(struct cpl_barrier *b) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)b; + + txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); + txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*b) / 8)); + b->opcode = CPL_BARRIER; +} + +/* + * Build a CPL_GET_TCB message as payload of a ULP_TX_PKT command. + */ +static inline void +mk_get_tcb_ulp(struct cpl_get_tcb *req, unsigned int tid, unsigned int cpuno) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; + + txpkt = (struct ulp_txpkt *)req; + txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); + txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, tid)); + req->cpuno = htons(cpuno); +} + +/* + * Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command. + */ +static inline void +mk_set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid, + unsigned int word, uint64_t mask, uint64_t val) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; + + txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); + txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); + req->reply = V_NO_REPLY(1); + req->cpu_idx = 0; + req->word = htons(word); + req->mask = htobe64(mask); + req->val = htobe64(val); +} + +/* + * Build a CPL_RX_DATA_ACK message as payload of a ULP_TX_PKT command. + */ +static void +mk_rx_data_ack_ulp(struct cpl_rx_data_ack *ack, unsigned int tid, unsigned int credits) +{ + struct ulp_txpkt *txpkt = (struct ulp_txpkt *)ack; + + txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); + txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*ack) / 8)); + OPCODE_TID(ack) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid)); + ack->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE | + V_RX_DACK_MODE(1) | V_RX_CREDITS(credits)); +} void +t3_cancel_ddpbuf(struct toepcb *toep, unsigned int bufidx) +{ + unsigned int wrlen; + struct mbuf *m; + struct work_request_hdr *wr; + struct cpl_barrier *lock; + struct cpl_set_tcb_field *req; + struct cpl_get_tcb *getreq; + struct ddp_state *p = &toep->tp_ddp_state; + + wrlen = sizeof(*wr) + sizeof(*req) + 2 * sizeof(*lock) + + sizeof(*getreq); + m = m_gethdr_nofail(wrlen); + m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); +#ifdef notyet + wr = (struct work_request_hdr *)__skb_put(skb, wrlen); + wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); +#else + wr = mtod(m, struct work_request_hdr *); +#endif + lock = (struct cpl_barrier *)(wr + 1); + mk_cpl_barrier_ulp(lock); + + req = (struct cpl_set_tcb_field *)(lock + 1); + + /* Hmmm, not sure if this actually a good thing: reactivating + * the other buffer might be an issue if it has been completed + * already. However, that is unlikely, since the fact that the UBUF + * is not completed indicates that there is no oustanding data. + */ + if (bufidx == 0) + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, + V_TF_DDP_ACTIVE_BUF(1) | + V_TF_DDP_BUF0_VALID(1), + V_TF_DDP_ACTIVE_BUF(1)); + else + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, + V_TF_DDP_ACTIVE_BUF(1) | + V_TF_DDP_BUF1_VALID(1), 0); + + getreq = (struct cpl_get_tcb *)(req + 1); + mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset); + + mk_cpl_barrier_ulp((struct cpl_barrier *)(getreq + 1)); + + /* Keep track of the number of oustanding CPL_GET_TCB requests + */ + p->get_tcb_count++; + +#ifdef T3_TRACE + T3_TRACE1(TIDTB(so), + "t3_cancel_ddpbuf: bufidx %u", bufidx); +#endif + cxgb_ofld_send(TOEP_T3C_DEV(toep), m); +} + +/** + * t3_overlay_ddpbuf - overlay an existing DDP buffer with a new one + * @sk: the socket associated with the buffers + * @bufidx: index of HW DDP buffer (0 or 1) + * @tag0: new tag for HW buffer 0 + * @tag1: new tag for HW buffer 1 + * @len: new length for HW buf @bufidx + * + * Sends a compound WR to overlay a new DDP buffer on top of an existing + * buffer by changing the buffer tag and length and setting the valid and + * active flag accordingly. The caller must ensure the new buffer is at + * least as big as the existing one. Since we typically reprogram both HW + * buffers this function sets both tags for convenience. Read the TCB to + * determine how made data was written into the buffer before the overlay + * took place. + */ +void +t3_overlay_ddpbuf(struct toepcb *toep, unsigned int bufidx, unsigned int tag0, + unsigned int tag1, unsigned int len) +{ + unsigned int wrlen; + struct mbuf *m; + struct work_request_hdr *wr; + struct cpl_get_tcb *getreq; + struct cpl_set_tcb_field *req; + struct ddp_state *p = &toep->tp_ddp_state; + + wrlen = sizeof(*wr) + 3 * sizeof(*req) + sizeof(*getreq); + m = m_gethdr_nofail(wrlen); + m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); +#ifdef notyet + wr = (struct work_request_hdr *)__skb_put(skb, wrlen); +#else + wr = mtod(m, struct work_request_hdr *); +#endif + /* Set the ATOMIC flag to make sure that TP processes the following + * CPLs in an atomic manner and no wire segments can be interleaved. + */ + wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); + + req = (struct cpl_set_tcb_field *)(wr + 1); + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_TAG, + V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG) | + V_TCB_RX_DDP_BUF1_TAG(M_TCB_RX_DDP_BUF1_TAG) << 32, + V_TCB_RX_DDP_BUF0_TAG(tag0) | + V_TCB_RX_DDP_BUF1_TAG((uint64_t)tag1) << 32); + req++; + if (bufidx == 0) { + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_LEN, + V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), + V_TCB_RX_DDP_BUF0_LEN((uint64_t)len)); + req++; + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, + V_TF_DDP_PUSH_DISABLE_0(1) | + V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(1), + V_TF_DDP_PUSH_DISABLE_0(0) | + V_TF_DDP_BUF0_VALID(1)); + } else { + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_LEN, + V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN), + V_TCB_RX_DDP_BUF1_LEN((uint64_t)len)); + req++; + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, + V_TF_DDP_PUSH_DISABLE_1(1) | + V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1), + V_TF_DDP_PUSH_DISABLE_1(0) | + V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1)); + } + + getreq = (struct cpl_get_tcb *)(req + 1); + mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset); + + /* Keep track of the number of oustanding CPL_GET_TCB requests + */ + p->get_tcb_count++; + +#ifdef T3_TRACE + T3_TRACE4(TIDTB(sk), + "t3_overlay_ddpbuf: bufidx %u tag0 %u tag1 %u " + "len %d", + bufidx, tag0, tag1, len); +#endif + cxgb_ofld_send(TOEP_T3C_DEV(toep), m); +} + +/* + * Sends a compound WR containing all the CPL messages needed to program the + * two HW DDP buffers, namely optionally setting up the length and offset of + * each buffer, programming the DDP flags, and optionally sending RX_DATA_ACK. + */ +void +t3_setup_ddpbufs(struct toepcb *toep, unsigned int len0, unsigned int offset0, + unsigned int len1, unsigned int offset1, + uint64_t ddp_flags, uint64_t flag_mask, int modulate) +{ + unsigned int wrlen; + struct mbuf *m; + struct work_request_hdr *wr; + struct cpl_set_tcb_field *req; + + wrlen = sizeof(*wr) + sizeof(*req) + (len0 ? sizeof(*req) : 0) + + (len1 ? sizeof(*req) : 0) + + (modulate ? sizeof(struct cpl_rx_data_ack) : 0); + m = m_gethdr_nofail(wrlen); + m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); +#ifdef notyet + wr = (struct work_request_hdr *)__skb_put(skb, wrlen); + wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); +#else + wr = mtod(m, struct work_request_hdr *); +#endif + req = (struct cpl_set_tcb_field *)(wr + 1); + if (len0) { /* program buffer 0 offset and length */ + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_OFFSET, + V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) | + V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), + V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)offset0) | + V_TCB_RX_DDP_BUF0_LEN((uint64_t)len0)); + req++; + } + if (len1) { /* program buffer 1 offset and length */ + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_OFFSET, + V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) | + V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN) << 32, + V_TCB_RX_DDP_BUF1_OFFSET((uint64_t)offset1) | + V_TCB_RX_DDP_BUF1_LEN((uint64_t)len1) << 32); + req++; + } + + mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, flag_mask, + ddp_flags); + + if (modulate) { + mk_rx_data_ack_ulp((struct cpl_rx_data_ack *)(req + 1), toep->tp_tid, + toep->tp_copied_seq - toep->tp_rcv_wup); + toep->tp_rcv_wup = toep->tp_copied_seq; + } + +#ifdef T3_TRACE + T3_TRACE5(TIDTB(sk), + "t3_setup_ddpbufs: len0 %u len1 %u ddp_flags 0x%08x%08x " + "modulate %d", + len0, len1, ddp_flags >> 32, ddp_flags & 0xffffffff, + modulate); +#endif + + cxgb_ofld_send(TOEP_T3C_DEV(toep), m); +} + +void t3_init_wr_tab(unsigned int wr_len) { int i; @@ -3650,10 +4046,8 @@ t3tom_register_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl); t3tom_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp); t3tom_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete); -#ifdef notyet t3tom_register_cpl_handler(CPL_RX_URG_NOTIFY, do_rx_urg_notify); t3tom_register_cpl_handler(CPL_TRACE_PKT, do_trace_pkt); -#endif t3tom_register_cpl_handler(CPL_GET_TCB_RPL, do_get_tcb_rpl); return (0); } ==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c#5 (text+ko) ==== @@ -810,14 +810,14 @@ p->cur_buf = bufidx; p->kbuf_idx = bufidx; if (!bufidx) - t3_setup_ddpbufs(so, 0, 0, 0, 0, + t3_setup_ddpbufs(toep, 0, 0, 0, 0, V_TF_DDP_PSH_NO_INVALIDATE(p->kbuf_noinval) | V_TF_DDP_BUF0_VALID(1), V_TF_DDP_PSH_NO_INVALIDATE(1) | V_TF_DDP_OFF(1) | V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(activate), modulate); else - t3_setup_ddpbufs(so, 0, 0, 0, 0, + t3_setup_ddpbufs(toep, 0, 0, 0, 0, V_TF_DDP_PSH_NO_INVALIDATE(p->kbuf_noinval) | V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(activate), @@ -956,15 +956,15 @@ } if (ubuf_idx == 0) { - t3_overlay_ddpbuf(so, 0, p->ubuf_tag << 6, p->kbuf_tag[1] << 6, + t3_overlay_ddpbuf(toep, 0, p->ubuf_tag << 6, p->kbuf_tag[1] << 6, len); - t3_setup_ddpbufs(so, 0, 0, p->kbuf[1]->dgl_length, 0, + t3_setup_ddpbufs(toep, 0, 0, p->kbuf[1]->dgl_length, 0, flags, OVERLAY_MASK | flags, 1); } else { - t3_overlay_ddpbuf(so, 1, p->kbuf_tag[0] << 6, p->ubuf_tag << 6, + t3_overlay_ddpbuf(toep, 1, p->kbuf_tag[0] << 6, p->ubuf_tag << 6, len); - t3_setup_ddpbufs(so, p->kbuf[0]->dgl_length, 0, 0, 0, + t3_setup_ddpbufs(toep, p->kbuf[0]->dgl_length, 0, 0, 0, flags, OVERLAY_MASK | flags, 1); } ==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_defs.h#2 (text+ko) ==== @@ -40,6 +40,7 @@ #define toeptoso(toep) ((toep)->tp_tp->t_inpcb->inp_socket) #define sototoep(so) (sototcpcb((so))->t_toe) +struct toepcb; struct listen_ctx; typedef void (*defer_handler_t)(struct toedev *dev, struct mbuf *m); @@ -54,6 +55,7 @@ int t3_init_cpl_io(void); void t3_init_wr_tab(unsigned int wr_len); uint32_t t3_send_rx_credits(struct tcpcb *tp, uint32_t credits, uint32_t dack, int nofail); +void t3_send_rx_modulate(struct toepcb *toep); void t3_cleanup_rbuf(struct tcpcb *tp); void t3_init_socket_ops(void); ==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h#5 (text+ko) ==== @@ -165,10 +165,10 @@ int t3_enter_ddp(struct socket *so, unsigned int kbuf_size, unsigned int waitall); void t3_cleanup_ddp(struct socket *so); void t3_release_ddp_resources(struct toepcb *toep); -void t3_cancel_ddpbuf(struct socket *so, unsigned int bufidx); -void t3_overlay_ddpbuf(struct socket *so, unsigned int bufidx, unsigned int tag0, +void t3_cancel_ddpbuf(struct toepcb *, unsigned int bufidx); +void t3_overlay_ddpbuf(struct toepcb *, unsigned int bufidx, unsigned int tag0, unsigned int tag1, unsigned int len); -void t3_setup_ddpbufs(struct socket *so, unsigned int len0, unsigned int offset0, +void t3_setup_ddpbufs(struct toepcb *, unsigned int len0, unsigned int offset0, unsigned int len1, unsigned int offset1, uint64_t ddp_flags, uint64_t flag_mask, int modulate); #endif /* T3_DDP_H */