Date: Sun, 28 Apr 2019 18:50:25 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org Subject: svn commit: r346852 - in stable/11/sys/dev/cxgbe: . tom Message-ID: <201904281850.x3SIoPn9092257@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: np Date: Sun Apr 28 18:50:25 2019 New Revision: 346852 URL: https://svnweb.freebsd.org/changeset/base/346852 Log: MFC r333114: cxgbe(4): Use opaque cookies or tid range-checks to determine the intended recipient of a CPL when it can't be determined solely from the opcode. Retire the per-queue handlers for such CPLs in favor of the new scheme. Sponsored by: Chelsio Communications Modified: stable/11/sys/dev/cxgbe/adapter.h stable/11/sys/dev/cxgbe/t4_main.c stable/11/sys/dev/cxgbe/t4_sge.c stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c stable/11/sys/dev/cxgbe/tom/t4_ddp.c stable/11/sys/dev/cxgbe/tom/t4_tom.c stable/11/sys/dev/cxgbe/tom/t4_tom.h Directory Properties: stable/11/ (props changed) Modified: stable/11/sys/dev/cxgbe/adapter.h ============================================================================== --- stable/11/sys/dev/cxgbe/adapter.h Sun Apr 28 18:45:44 2019 (r346851) +++ stable/11/sys/dev/cxgbe/adapter.h Sun Apr 28 18:50:25 2019 (r346852) @@ -366,6 +366,19 @@ enum { NM_BUSY = 2, }; +enum { + CPL_COOKIE_RESERVED = 0, + CPL_COOKIE_FILTER, + CPL_COOKIE_DDP0, + CPL_COOKIE_DDP1, + CPL_COOKIE_TOM, + CPL_COOKIE_AVAILABLE1, + CPL_COOKIE_AVAILABLE2, + CPL_COOKIE_AVAILABLE3, + + NUM_CPL_COOKIES = 8 /* Limited by M_COOKIE. Do not increase. */ +}; + struct sge_iq; struct rss_header; typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *, @@ -380,8 +393,6 @@ struct sge_iq { uint32_t flags; volatile int state; struct adapter *adapter; - cpl_handler_t set_tcb_rpl; - cpl_handler_t l2t_write_rpl; struct iq_desc *desc; /* KVA of descriptor ring */ int8_t intr_pktc_idx; /* packet count threshold index */ uint8_t gen; /* generation bit */ @@ -1158,9 +1169,10 @@ int parse_pkt(struct adapter *, struct mbuf **); void *start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *); void commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *); int tnl_cong(struct port_info *, int); -int t4_register_an_handler(an_handler_t); -int t4_register_fw_msg_handler(int, fw_msg_handler_t); -int t4_register_cpl_handler(int, cpl_handler_t); +void t4_register_an_handler(an_handler_t); +void t4_register_fw_msg_handler(int, fw_msg_handler_t); +void t4_register_cpl_handler(int, cpl_handler_t); +void t4_register_shared_cpl_handler(int, cpl_handler_t, int); /* t4_tracer.c */ struct t4_tracer; Modified: stable/11/sys/dev/cxgbe/t4_main.c ============================================================================== --- stable/11/sys/dev/cxgbe/t4_main.c Sun Apr 28 18:45:44 2019 (r346851) +++ stable/11/sys/dev/cxgbe/t4_main.c Sun Apr 28 18:50:25 2019 (r346852) @@ -694,8 +694,6 @@ static int del_filter(struct adapter *, struct t4_filt static void clear_filter(struct filter_entry *); static int set_filter_wr(struct adapter *, int); static int del_filter_wr(struct adapter *, int); -static int set_tcb_rpl(struct sge_iq *, const struct rss_header *, - struct mbuf *); static int get_sge_context(struct adapter *, struct t4_sge_context *); static int load_fw(struct adapter *, struct t4_data *); static int load_cfg(struct adapter *, struct t4_data *); @@ -9703,22 +9701,6 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_head } static int -set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) -{ - - MPASS(iq->set_tcb_rpl != NULL); - return (iq->set_tcb_rpl(iq, rss, m)); -} - -static int -l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) -{ - - MPASS(iq->l2t_write_rpl != NULL); - return (iq->l2t_write_rpl(iq, rss, m)); -} - -static int get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) { int rc; @@ -11007,8 +10989,10 @@ mod_event(module_t mod, int cmd, void *arg) sx_xlock(&mlu); if (loaded++ == 0) { t4_sge_modload(); - t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl); - t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl); + t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, + t4_filter_rpl, CPL_COOKIE_FILTER); + t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, + do_l2t_write_rpl, CPL_COOKIE_FILTER); t4_register_cpl_handler(CPL_TRACE_PKT, t4_trace_pkt); t4_register_cpl_handler(CPL_T5_TRACE_PKT, t5_trace_pkt); sx_init(&t4_list_lock, "T4/T5 adapters"); Modified: stable/11/sys/dev/cxgbe/t4_sge.c ============================================================================== --- stable/11/sys/dev/cxgbe/t4_sge.c Sun Apr 28 18:45:44 2019 (r346851) +++ stable/11/sys/dev/cxgbe/t4_sge.c Sun Apr 28 18:50:25 2019 (r346852) @@ -296,98 +296,122 @@ static counter_u64_t extfree_rels; an_handler_t t4_an_handler; fw_msg_handler_t t4_fw_msg_handler[NUM_FW6_TYPES]; cpl_handler_t t4_cpl_handler[NUM_CPL_CMDS]; +cpl_handler_t set_tcb_rpl_handlers[NUM_CPL_COOKIES]; +cpl_handler_t l2t_write_rpl_handlers[NUM_CPL_COOKIES]; - -static int -an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) +void +t4_register_an_handler(an_handler_t h) { + uintptr_t *loc; -#ifdef INVARIANTS - panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); -#else - log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", - __func__, iq, ctrl); -#endif - return (EDOOFUS); + MPASS(h == NULL || t4_an_handler == NULL); + + loc = (uintptr_t *)&t4_an_handler; + atomic_store_rel_ptr(loc, (uintptr_t)h); } -int -t4_register_an_handler(an_handler_t h) +void +t4_register_fw_msg_handler(int type, fw_msg_handler_t h) { - uintptr_t *loc, new; + uintptr_t *loc; - new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; - loc = (uintptr_t *) &t4_an_handler; - atomic_store_rel_ptr(loc, new); + MPASS(type < nitems(t4_fw_msg_handler)); + MPASS(h == NULL || t4_fw_msg_handler[type] == NULL); + /* + * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL + * handler dispatch table. Reject any attempt to install a handler for + * this subtype. + */ + MPASS(type != FW_TYPE_RSSCPL); + MPASS(type != FW6_TYPE_RSSCPL); - return (0); + loc = (uintptr_t *)&t4_fw_msg_handler[type]; + atomic_store_rel_ptr(loc, (uintptr_t)h); } -static int -fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) +void +t4_register_cpl_handler(int opcode, cpl_handler_t h) { - const struct cpl_fw6_msg *cpl = - __containerof(rpl, struct cpl_fw6_msg, data[0]); + uintptr_t *loc; -#ifdef INVARIANTS - panic("%s: fw_msg type %d", __func__, cpl->type); -#else - log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); -#endif - return (EDOOFUS); + MPASS(opcode < nitems(t4_cpl_handler)); + MPASS(h == NULL || t4_cpl_handler[opcode] == NULL); + + loc = (uintptr_t *)&t4_cpl_handler[opcode]; + atomic_store_rel_ptr(loc, (uintptr_t)h); } -int -t4_register_fw_msg_handler(int type, fw_msg_handler_t h) +static int +set_tcb_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, + struct mbuf *m) { - uintptr_t *loc, new; + const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); + u_int tid; + int cookie; - if (type >= nitems(t4_fw_msg_handler)) - return (EINVAL); + MPASS(m == NULL); - /* - * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL - * handler dispatch table. Reject any attempt to install a handler for - * this subtype. - */ - if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) - return (EINVAL); + tid = GET_TID(cpl); + if (is_ftid(iq->adapter, tid)) { + /* + * The return code for filter-write is put in the CPL cookie so + * we have to rely on the hardware tid (is_ftid) to determine + * that this is a response to a filter. + */ + cookie = CPL_COOKIE_FILTER; + } else { + cookie = G_COOKIE(cpl->cookie); + } + MPASS(cookie > CPL_COOKIE_RESERVED); + MPASS(cookie < nitems(set_tcb_rpl_handlers)); - new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; - loc = (uintptr_t *) &t4_fw_msg_handler[type]; - atomic_store_rel_ptr(loc, new); - - return (0); + return (set_tcb_rpl_handlers[cookie](iq, rss, m)); } static int -cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) +l2t_write_rpl_handler(struct sge_iq *iq, const struct rss_header *rss, + struct mbuf *m) { + const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1); + unsigned int cookie; -#ifdef INVARIANTS - panic("%s: opcode 0x%02x on iq %p with payload %p", - __func__, rss->opcode, iq, m); -#else - log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", - __func__, rss->opcode, iq, m); - m_freem(m); -#endif - return (EDOOFUS); + MPASS(m == NULL); + + cookie = GET_TID(rpl) & F_SYNC_WR ? CPL_COOKIE_TOM : CPL_COOKIE_FILTER; + return (l2t_write_rpl_handlers[cookie](iq, rss, m)); } -int -t4_register_cpl_handler(int opcode, cpl_handler_t h) +static void +t4_init_shared_cpl_handlers(void) { - uintptr_t *loc, new; - if (opcode >= nitems(t4_cpl_handler)) - return (EINVAL); + t4_register_cpl_handler(CPL_SET_TCB_RPL, set_tcb_rpl_handler); + t4_register_cpl_handler(CPL_L2T_WRITE_RPL, l2t_write_rpl_handler); +} - new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; - loc = (uintptr_t *) &t4_cpl_handler[opcode]; - atomic_store_rel_ptr(loc, new); +void +t4_register_shared_cpl_handler(int opcode, cpl_handler_t h, int cookie) +{ + uintptr_t *loc; - return (0); + MPASS(opcode < nitems(t4_cpl_handler)); + MPASS(cookie > CPL_COOKIE_RESERVED); + MPASS(cookie < NUM_CPL_COOKIES); + MPASS(t4_cpl_handler[opcode] != NULL); + + switch (opcode) { + case CPL_SET_TCB_RPL: + loc = (uintptr_t *)&set_tcb_rpl_handlers[cookie]; + break; + case CPL_L2T_WRITE_RPL: + loc = (uintptr_t *)&l2t_write_rpl_handlers[cookie]; + break; + default: + MPASS(0); + return; + } + MPASS(h == NULL || *loc == (uintptr_t)NULL); + atomic_store_rel_ptr(loc, (uintptr_t)h); } /* @@ -396,7 +420,6 @@ t4_register_cpl_handler(int opcode, cpl_handler_t h) void t4_sge_modload(void) { - int i; if (fl_pktshift < 0 || fl_pktshift > 7) { printf("Invalid hw.cxgbe.fl_pktshift value (%d)," @@ -436,12 +459,7 @@ t4_sge_modload(void) counter_u64_zero(extfree_refs); counter_u64_zero(extfree_rels); - t4_an_handler = an_not_handled; - for (i = 0; i < nitems(t4_fw_msg_handler); i++) - t4_fw_msg_handler[i] = fw_msg_not_handled; - for (i = 0; i < nitems(t4_cpl_handler); i++) - t4_cpl_handler[i] = cpl_not_handled; - + t4_init_shared_cpl_handlers(); t4_register_cpl_handler(CPL_FW4_MSG, handle_fw_msg); t4_register_cpl_handler(CPL_FW6_MSG, handle_fw_msg); t4_register_cpl_handler(CPL_SGE_EGR_UPDATE, handle_sge_egr_update); @@ -2903,11 +2921,8 @@ alloc_fwq(struct adapter *sc) init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE); if (sc->flags & IS_VF) intr_idx = 0; - else { + else intr_idx = sc->intr_count > 1 ? 1 : 0; - fwq->set_tcb_rpl = t4_filter_rpl; - fwq->l2t_write_rpl = do_l2t_write_rpl; - } rc = alloc_iq_fl(&sc->port[0]->vi[0], fwq, NULL, intr_idx, -1); if (rc != 0) { device_printf(sc->dev, Modified: stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c ============================================================================== --- stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c Sun Apr 28 18:45:44 2019 (r346851) +++ stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c Sun Apr 28 18:50:25 2019 (r346852) @@ -1912,44 +1912,6 @@ do_fw4_ack(struct sge_iq *iq, const struct rss_header return (0); } -int -do_set_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) -{ - struct adapter *sc = iq->adapter; - const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); - unsigned int tid = GET_TID(cpl); - struct toepcb *toep; -#ifdef INVARIANTS - unsigned int opcode = G_CPL_OPCODE(be32toh(OPCODE_TID(cpl))); -#endif - - KASSERT(opcode == CPL_SET_TCB_RPL, - ("%s: unexpected opcode 0x%x", __func__, opcode)); - KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); - MPASS(iq != &sc->sge.fwq); - - toep = lookup_tid(sc, tid); - if (toep->ulp_mode == ULP_MODE_TCPDDP) { - handle_ddp_tcb_rpl(toep, cpl); - return (0); - } - - /* - * TOM and/or other ULPs don't request replies for CPL_SET_TCB or - * CPL_SET_TCB_FIELD requests. This can easily change and when it does - * the dispatch code will go here. - */ -#ifdef INVARIANTS - panic("%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p", __func__, - tid, iq); -#else - log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n", - __func__, tid, iq); -#endif - - return (0); -} - void t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep, uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie) @@ -1959,6 +1921,9 @@ t4_set_tcb_field(struct adapter *sc, struct sge_wrq *w struct ofld_tx_sdesc *txsd; MPASS((cookie & ~M_COOKIE) == 0); + if (reply) { + MPASS(cookie != CPL_COOKIE_RESERVED); + } wr = alloc_wrqe(sizeof(*req), wrq); if (wr == NULL) { Modified: stable/11/sys/dev/cxgbe/tom/t4_ddp.c ============================================================================== --- stable/11/sys/dev/cxgbe/tom/t4_ddp.c Sun Apr 28 18:45:44 2019 (r346851) +++ stable/11/sys/dev/cxgbe/tom/t4_ddp.c Sun Apr 28 18:50:25 2019 (r346852) @@ -631,11 +631,17 @@ enum { DDP_BUF1_INVALIDATED }; -void -handle_ddp_tcb_rpl(struct toepcb *toep, const struct cpl_set_tcb_rpl *cpl) +CTASSERT(DDP_BUF0_INVALIDATED == CPL_COOKIE_DDP0); + +static int +do_ddp_tcb_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) { + struct adapter *sc = iq->adapter; + const struct cpl_set_tcb_rpl *cpl = (const void *)(rss + 1); + unsigned int tid = GET_TID(cpl); unsigned int db_idx; - struct inpcb *inp = toep->inp; + struct toepcb *toep; + struct inpcb *inp; struct ddp_buffer *db; struct kaiocb *job; long copied; @@ -643,6 +649,8 @@ handle_ddp_tcb_rpl(struct toepcb *toep, const struct c if (cpl->status != CPL_ERR_NONE) panic("XXX: tcp_rpl failed: %d", cpl->status); + toep = lookup_tid(sc, tid); + inp = toep->inp; switch (cpl->cookie) { case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(DDP_BUF0_INVALIDATED): case V_WORD(W_TCB_RX_DDP_FLAGS) | V_COOKIE(DDP_BUF1_INVALIDATED): @@ -650,6 +658,7 @@ handle_ddp_tcb_rpl(struct toepcb *toep, const struct c * XXX: This duplicates a lot of code with handle_ddp_data(). */ db_idx = G_COOKIE(cpl->cookie) - DDP_BUF0_INVALIDATED; + MPASS(db_idx < nitems(toep->ddp.db)); INP_WLOCK(inp); DDP_LOCK(toep); db = &toep->ddp.db[db_idx]; @@ -694,6 +703,8 @@ handle_ddp_tcb_rpl(struct toepcb *toep, const struct c panic("XXX: unknown tcb_rpl offset %#x, cookie %#x", G_WORD(cpl->cookie), G_COOKIE(cpl->cookie)); } + + return (0); } void @@ -1946,6 +1957,10 @@ void t4_ddp_mod_load(void) { + t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl, + CPL_COOKIE_DDP0); + t4_register_shared_cpl_handler(CPL_SET_TCB_RPL, do_ddp_tcb_rpl, + CPL_COOKIE_DDP1); t4_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp); t4_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete); TAILQ_INIT(&ddp_orphan_pagesets); Modified: stable/11/sys/dev/cxgbe/tom/t4_tom.c ============================================================================== --- stable/11/sys/dev/cxgbe/tom/t4_tom.c Sun Apr 28 18:45:44 2019 (r346851) +++ stable/11/sys/dev/cxgbe/tom/t4_tom.c Sun Apr 28 18:50:25 2019 (r346852) @@ -1326,8 +1326,7 @@ t4_tom_activate(struct adapter *sc) struct tom_data *td; struct toedev *tod; struct vi_info *vi; - struct sge_ofld_rxq *ofld_rxq; - int i, j, rc, v; + int i, rc, v; ASSERT_SYNCHRONIZED_OP(sc); @@ -1392,10 +1391,6 @@ t4_tom_activate(struct adapter *sc) for_each_port(sc, i) { for_each_vi(sc->port[i], v, vi) { TOEDEV(vi->ifp) = &td->tod; - for_each_ofld_rxq(vi, j, ofld_rxq) { - ofld_rxq->iq.set_tcb_rpl = do_set_tcb_rpl; - ofld_rxq->iq.l2t_write_rpl = do_l2t_write_rpl2; - } } } @@ -1498,6 +1493,8 @@ t4_tom_mod_load(void) struct protosw *tcp_protosw, *tcp6_protosw; /* CPL handlers */ + t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl2, + CPL_COOKIE_TOM); t4_init_connect_cpl_handlers(); t4_init_listen_cpl_handlers(); t4_init_cpl_io_handlers(); @@ -1562,6 +1559,7 @@ t4_tom_mod_unload(void) t4_uninit_connect_cpl_handlers(); t4_uninit_listen_cpl_handlers(); t4_uninit_cpl_io_handlers(); + t4_register_shared_cpl_handler(CPL_L2T_WRITE_RPL, NULL, CPL_COOKIE_TOM); return (0); } Modified: stable/11/sys/dev/cxgbe/tom/t4_tom.h ============================================================================== --- stable/11/sys/dev/cxgbe/tom/t4_tom.h Sun Apr 28 18:45:44 2019 (r346851) +++ stable/11/sys/dev/cxgbe/tom/t4_tom.h Sun Apr 28 18:50:25 2019 (r346852) @@ -386,7 +386,6 @@ void t4_set_tcb_field(struct adapter *, struct sge_wrq uint16_t, uint64_t, uint64_t, int, int); void t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop); void t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop); -int do_set_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); /* t4_ddp.c */ int t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int, @@ -412,7 +411,6 @@ void ddp_queue_toep(struct toepcb *); void release_ddp_resources(struct toepcb *toep); void handle_ddp_close(struct toepcb *, struct tcpcb *, uint32_t); void handle_ddp_indicate(struct toepcb *); -void handle_ddp_tcb_rpl(struct toepcb *, const struct cpl_set_tcb_rpl *); void insert_ddp_data(struct toepcb *, uint32_t); const struct offload_settings *lookup_offload_policy(struct adapter *, int, struct mbuf *, uint16_t, struct inpcb *);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201904281850.x3SIoPn9092257>