From owner-svn-src-head@freebsd.org Tue Oct 22 20:41:55 2019 Return-Path: Delivered-To: svn-src-head@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 48005164D89; Tue, 22 Oct 2019 20:41:55 +0000 (UTC) (envelope-from jhb@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 46yQR71yZKz4G59; Tue, 22 Oct 2019 20:41:55 +0000 (UTC) (envelope-from jhb@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 2627419D29; Tue, 22 Oct 2019 20:41:55 +0000 (UTC) (envelope-from jhb@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id x9MKftnH016548; Tue, 22 Oct 2019 20:41:55 GMT (envelope-from jhb@FreeBSD.org) Received: (from jhb@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id x9MKfsuC016543; Tue, 22 Oct 2019 20:41:54 GMT (envelope-from jhb@FreeBSD.org) Message-Id: <201910222041.x9MKfsuC016543@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: jhb set sender to jhb@FreeBSD.org using -f From: John Baldwin Date: Tue, 22 Oct 2019 20:41:54 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r353900 - head/sys/dev/cxgbe X-SVN-Group: head X-SVN-Commit-Author: jhb X-SVN-Commit-Paths: head/sys/dev/cxgbe X-SVN-Commit-Revision: 353900 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 22 Oct 2019 20:41:55 -0000 Author: jhb Date: Tue Oct 22 20:41:54 2019 New Revision: 353900 URL: https://svnweb.freebsd.org/changeset/base/353900 Log: Split Chelsio send tags into a generic base tag and a ratelimit tag. NIC KTLS will add a new TLS send tag type in cxgbe(4) that is a distinct tag from a ratelimit tag. To support this, refactor cxgbe_snd_tag to be a simple send tag with a type and convert the existing ratelimit tag to a new cxgbe_rate_tag structure. Reviewed by: np Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D22072 Modified: head/sys/dev/cxgbe/adapter.h head/sys/dev/cxgbe/offload.h head/sys/dev/cxgbe/t4_main.c head/sys/dev/cxgbe/t4_sched.c head/sys/dev/cxgbe/t4_sge.c Modified: head/sys/dev/cxgbe/adapter.h ============================================================================== --- head/sys/dev/cxgbe/adapter.h Tue Oct 22 20:01:47 2019 (r353899) +++ head/sys/dev/cxgbe/adapter.h Tue Oct 22 20:41:54 2019 (r353900) @@ -1155,6 +1155,7 @@ int update_mac_settings(struct ifnet *, int); int adapter_full_init(struct adapter *); int adapter_full_uninit(struct adapter *); uint64_t cxgbe_get_counter(struct ifnet *, ift_counter); +void cxgbe_snd_tag_init(struct cxgbe_snd_tag *, struct ifnet *, int); int vi_full_init(struct vi_info *); int vi_full_uninit(struct vi_info *); void vi_sysctls(struct vi_info *); @@ -1212,7 +1213,7 @@ void t4_register_cpl_handler(int, cpl_handler_t); void t4_register_shared_cpl_handler(int, cpl_handler_t, int); #ifdef RATELIMIT int ethofld_transmit(struct ifnet *, struct mbuf *); -void send_etid_flush_wr(struct cxgbe_snd_tag *); +void send_etid_flush_wr(struct cxgbe_rate_tag *); #endif /* t4_tracer.c */ @@ -1238,13 +1239,13 @@ int sysctl_tc_params(SYSCTL_HANDLER_ARGS); #ifdef RATELIMIT void t4_init_etid_table(struct adapter *); void t4_free_etid_table(struct adapter *); -struct cxgbe_snd_tag *lookup_etid(struct adapter *, int); -int cxgbe_snd_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *, +struct cxgbe_rate_tag *lookup_etid(struct adapter *, int); +int cxgbe_rate_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *, struct m_snd_tag **); -int cxgbe_snd_tag_modify(struct m_snd_tag *, union if_snd_tag_modify_params *); -int cxgbe_snd_tag_query(struct m_snd_tag *, union if_snd_tag_query_params *); -void cxgbe_snd_tag_free(struct m_snd_tag *); -void cxgbe_snd_tag_free_locked(struct cxgbe_snd_tag *); +int cxgbe_rate_tag_modify(struct m_snd_tag *, union if_snd_tag_modify_params *); +int cxgbe_rate_tag_query(struct m_snd_tag *, union if_snd_tag_query_params *); +void cxgbe_rate_tag_free(struct m_snd_tag *); +void cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *); void cxgbe_ratelimit_query(struct ifnet *, struct if_ratelimit_query_results *); #endif Modified: head/sys/dev/cxgbe/offload.h ============================================================================== --- head/sys/dev/cxgbe/offload.h Tue Oct 22 20:01:47 2019 (r353899) +++ head/sys/dev/cxgbe/offload.h Tue Oct 22 20:41:54 2019 (r353900) @@ -79,7 +79,7 @@ union aopen_entry { union aopen_entry *next; }; -/* cxgbe_snd_tag flags */ +/* cxgbe_rate_tag flags */ enum { EO_FLOWC_PENDING = (1 << 0), /* flowc needs to be sent */ EO_FLOWC_RPL_PENDING = (1 << 1), /* flowc credits due back */ @@ -89,6 +89,11 @@ enum { struct cxgbe_snd_tag { struct m_snd_tag com; + int type; +}; + +struct cxgbe_rate_tag { + struct cxgbe_snd_tag com; struct adapter *adapter; u_int flags; struct mtx lock; @@ -114,8 +119,14 @@ mst_to_cst(struct m_snd_tag *t) return (__containerof(t, struct cxgbe_snd_tag, com)); } +static inline struct cxgbe_rate_tag * +mst_to_crt(struct m_snd_tag *t) +{ + return ((struct cxgbe_rate_tag *)mst_to_cst(t)); +} + union etid_entry { - struct cxgbe_snd_tag *cst; + struct cxgbe_rate_tag *cst; union etid_entry *next; }; Modified: head/sys/dev/cxgbe/t4_main.c ============================================================================== --- head/sys/dev/cxgbe/t4_main.c Tue Oct 22 20:01:47 2019 (r353899) +++ head/sys/dev/cxgbe/t4_main.c Tue Oct 22 20:41:54 2019 (r353900) @@ -230,6 +230,15 @@ static void cxgbe_init(void *); static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); static int cxgbe_transmit(struct ifnet *, struct mbuf *); static void cxgbe_qflush(struct ifnet *); +#ifdef RATELIMIT +static int cxgbe_snd_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *, + struct m_snd_tag **); +static int cxgbe_snd_tag_modify(struct m_snd_tag *, + union if_snd_tag_modify_params *); +static int cxgbe_snd_tag_query(struct m_snd_tag *, + union if_snd_tag_query_params *); +static void cxgbe_snd_tag_free(struct m_snd_tag *); +#endif MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); @@ -2044,11 +2053,18 @@ cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; struct sge_txq *txq; +#ifdef RATELIMIT + struct cxgbe_snd_tag *cst; +#endif void *items[1]; int rc; M_ASSERTPKTHDR(m); MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ +#ifdef RATELIMIT + if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) + MPASS(m->m_pkthdr.snd_tag->ifp == ifp); +#endif if (__predict_false(pi->link_cfg.link_ok == false)) { m_freem(m); @@ -2063,8 +2079,9 @@ cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) } #ifdef RATELIMIT if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) { - MPASS(m->m_pkthdr.snd_tag->ifp == ifp); - return (ethofld_transmit(ifp, m)); + cst = mst_to_cst(m->m_pkthdr.snd_tag); + if (cst->type == IF_SND_TAG_TYPE_RATE_LIMIT) + return (ethofld_transmit(ifp, m)); } #endif @@ -2221,6 +2238,87 @@ cxgbe_get_counter(struct ifnet *ifp, ift_counter c) return (if_get_counter_default(ifp, c)); } } + +#ifdef RATELIMIT +void +cxgbe_snd_tag_init(struct cxgbe_snd_tag *cst, struct ifnet *ifp, int type) +{ + + m_snd_tag_init(&cst->com, ifp); + cst->type = type; +} + +static int +cxgbe_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, + struct m_snd_tag **pt) +{ + int error; + + switch (params->hdr.type) { +#ifdef RATELIMIT + case IF_SND_TAG_TYPE_RATE_LIMIT: + error = cxgbe_rate_tag_alloc(ifp, params, pt); + break; +#endif + default: + error = EOPNOTSUPP; + } + if (error == 0) + MPASS(mst_to_cst(*pt)->type == params->hdr.type); + return (error); +} + +static int +cxgbe_snd_tag_modify(struct m_snd_tag *mst, + union if_snd_tag_modify_params *params) +{ + struct cxgbe_snd_tag *cst; + + cst = mst_to_cst(mst); + switch (cst->type) { +#ifdef RATELIMIT + case IF_SND_TAG_TYPE_RATE_LIMIT: + return (cxgbe_rate_tag_modify(mst, params)); +#endif + default: + return (EOPNOTSUPP); + } +} + +static int +cxgbe_snd_tag_query(struct m_snd_tag *mst, + union if_snd_tag_query_params *params) +{ + struct cxgbe_snd_tag *cst; + + cst = mst_to_cst(mst); + switch (cst->type) { +#ifdef RATELIMIT + case IF_SND_TAG_TYPE_RATE_LIMIT: + return (cxgbe_rate_tag_query(mst, params)); +#endif + default: + return (EOPNOTSUPP); + } +} + +static void +cxgbe_snd_tag_free(struct m_snd_tag *mst) +{ + struct cxgbe_snd_tag *cst; + + cst = mst_to_cst(mst); + switch (cst->type) { +#ifdef RATELIMIT + case IF_SND_TAG_TYPE_RATE_LIMIT: + cxgbe_rate_tag_free(mst); + return; +#endif + default: + panic("shouldn't get here"); + } +} +#endif /* * The kernel picks a media from the list we had provided but we still validate Modified: head/sys/dev/cxgbe/t4_sched.c ============================================================================== --- head/sys/dev/cxgbe/t4_sched.c Tue Oct 22 20:01:47 2019 (r353899) +++ head/sys/dev/cxgbe/t4_sched.c Tue Oct 22 20:41:54 2019 (r353900) @@ -711,11 +711,11 @@ t4_free_etid_table(struct adapter *sc) } /* etid services */ -static int alloc_etid(struct adapter *, struct cxgbe_snd_tag *); +static int alloc_etid(struct adapter *, struct cxgbe_rate_tag *); static void free_etid(struct adapter *, int); static int -alloc_etid(struct adapter *sc, struct cxgbe_snd_tag *cst) +alloc_etid(struct adapter *sc, struct cxgbe_rate_tag *cst) { struct tid_info *t = &sc->tids; int etid = -1; @@ -733,7 +733,7 @@ alloc_etid(struct adapter *sc, struct cxgbe_snd_tag *c return (etid); } -struct cxgbe_snd_tag * +struct cxgbe_rate_tag * lookup_etid(struct adapter *sc, int etid) { struct tid_info *t = &sc->tids; @@ -755,17 +755,16 @@ free_etid(struct adapter *sc, int etid) } int -cxgbe_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, +cxgbe_rate_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, struct m_snd_tag **pt) { int rc, schedcl; struct vi_info *vi = ifp->if_softc; struct port_info *pi = vi->pi; struct adapter *sc = pi->adapter; - struct cxgbe_snd_tag *cst; + struct cxgbe_rate_tag *cst; - if (params->hdr.type != IF_SND_TAG_TYPE_RATE_LIMIT) - return (ENOTSUP); + MPASS(params->hdr.type == IF_SND_TAG_TYPE_RATE_LIMIT); rc = t4_reserve_cl_rl_kbps(sc, pi->port_id, (params->rate_limit.max_rate * 8ULL / 1000), &schedcl); @@ -789,7 +788,7 @@ failed: mtx_init(&cst->lock, "cst_lock", NULL, MTX_DEF); mbufq_init(&cst->pending_tx, INT_MAX); mbufq_init(&cst->pending_fwack, INT_MAX); - m_snd_tag_init(&cst->com, ifp); + cxgbe_snd_tag_init(&cst->com, ifp, IF_SND_TAG_TYPE_RATE_LIMIT); cst->flags |= EO_FLOWC_PENDING | EO_SND_TAG_REF; cst->adapter = sc; cst->port_id = pi->port_id; @@ -806,7 +805,7 @@ failed: * Queues will be selected later when the connection flowid is available. */ - *pt = &cst->com; + *pt = &cst->com.com; return (0); } @@ -814,11 +813,11 @@ failed: * Change in parameters, no change in ifp. */ int -cxgbe_snd_tag_modify(struct m_snd_tag *mst, +cxgbe_rate_tag_modify(struct m_snd_tag *mst, union if_snd_tag_modify_params *params) { int rc, schedcl; - struct cxgbe_snd_tag *cst = mst_to_cst(mst); + struct cxgbe_rate_tag *cst = mst_to_crt(mst); struct adapter *sc = cst->adapter; /* XXX: is schedcl -1 ok here? */ @@ -840,10 +839,10 @@ cxgbe_snd_tag_modify(struct m_snd_tag *mst, } int -cxgbe_snd_tag_query(struct m_snd_tag *mst, +cxgbe_rate_tag_query(struct m_snd_tag *mst, union if_snd_tag_query_params *params) { - struct cxgbe_snd_tag *cst = mst_to_cst(mst); + struct cxgbe_rate_tag *cst = mst_to_crt(mst); params->rate_limit.max_rate = cst->max_rate; @@ -858,7 +857,7 @@ cxgbe_snd_tag_query(struct m_snd_tag *mst, * Unlocks cst and frees it. */ void -cxgbe_snd_tag_free_locked(struct cxgbe_snd_tag *cst) +cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *cst) { struct adapter *sc = cst->adapter; @@ -879,9 +878,9 @@ cxgbe_snd_tag_free_locked(struct cxgbe_snd_tag *cst) } void -cxgbe_snd_tag_free(struct m_snd_tag *mst) +cxgbe_rate_tag_free(struct m_snd_tag *mst) { - struct cxgbe_snd_tag *cst = mst_to_cst(mst); + struct cxgbe_rate_tag *cst = mst_to_crt(mst); mtx_lock(&cst->lock); @@ -896,7 +895,7 @@ cxgbe_snd_tag_free(struct m_snd_tag *mst) * credits for the etid otherwise. */ if (cst->tx_credits == cst->tx_total) { - cxgbe_snd_tag_free_locked(cst); + cxgbe_rate_tag_free_locked(cst); return; /* cst is gone. */ } send_etid_flush_wr(cst); Modified: head/sys/dev/cxgbe/t4_sge.c ============================================================================== --- head/sys/dev/cxgbe/t4_sge.c Tue Oct 22 20:01:47 2019 (r353899) +++ head/sys/dev/cxgbe/t4_sge.c Tue Oct 22 20:41:54 2019 (r353900) @@ -2307,10 +2307,10 @@ set_mbuf_eo_tsclk_tsoff(struct mbuf *m, uint8_t tsclk_ } static inline int -needs_eo(struct mbuf *m) +needs_eo(struct cxgbe_snd_tag *cst) { - return (m->m_pkthdr.csum_flags & CSUM_SND_TAG); + return (cst != NULL && cst->type == IF_SND_TAG_TYPE_RATE_LIMIT); } #endif @@ -2542,6 +2542,9 @@ parse_pkt(struct adapter *sc, struct mbuf **mp) #if defined(INET) || defined(INET6) struct tcphdr *tcp; #endif +#ifdef RATELIMIT + struct cxgbe_snd_tag *cst; +#endif uint16_t eh_type; uint8_t cflags; @@ -2562,6 +2565,12 @@ restart: M_ASSERTPKTHDR(m0); MPASS(m0->m_pkthdr.len > 0); nsegs = count_mbuf_nsegs(m0, 0, &cflags); +#ifdef RATELIMIT + if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG) + cst = mst_to_cst(m0->m_pkthdr.snd_tag); + else + cst = NULL; +#endif if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) { if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) { rc = EFBIG; @@ -2595,16 +2604,17 @@ restart: * checksumming is enabled. needs_l4_csum happens to check for all the * right things. */ - if (__predict_false(needs_eo(m0) && !needs_l4_csum(m0))) { + if (__predict_false(needs_eo(cst) && !needs_l4_csum(m0))) { m_snd_tag_rele(m0->m_pkthdr.snd_tag); m0->m_pkthdr.snd_tag = NULL; m0->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; + cst = NULL; } #endif if (!needs_tso(m0) && #ifdef RATELIMIT - !needs_eo(m0) && + !needs_eo(cst) && #endif !(sc->flags & IS_VF && (needs_l3_csum(m0) || needs_l4_csum(m0)))) return (0); @@ -2666,7 +2676,7 @@ restart: #endif } #ifdef RATELIMIT - if (needs_eo(m0)) { + if (needs_eo(cst)) { u_int immhdrs; /* EO WRs have the headers in the WR and not the GL. */ @@ -5723,7 +5733,7 @@ done: #define ETID_FLOWC_LEN16 (howmany(ETID_FLOWC_LEN, 16)) static int -send_etid_flowc_wr(struct cxgbe_snd_tag *cst, struct port_info *pi, +send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi, struct vi_info *vi) { struct wrq_cookie cookie; @@ -5769,7 +5779,7 @@ send_etid_flowc_wr(struct cxgbe_snd_tag *cst, struct p #define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16)) void -send_etid_flush_wr(struct cxgbe_snd_tag *cst) +send_etid_flush_wr(struct cxgbe_rate_tag *cst) { struct fw_flowc_wr *flowc; struct wrq_cookie cookie; @@ -5795,7 +5805,7 @@ send_etid_flush_wr(struct cxgbe_snd_tag *cst) } static void -write_ethofld_wr(struct cxgbe_snd_tag *cst, struct fw_eth_tx_eo_wr *wr, +write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr, struct mbuf *m0, int compl) { struct cpl_tx_pkt_core *cpl; @@ -5944,7 +5954,7 @@ write_ethofld_wr(struct cxgbe_snd_tag *cst, struct fw_ } static void -ethofld_tx(struct cxgbe_snd_tag *cst) +ethofld_tx(struct cxgbe_rate_tag *cst) { struct mbuf *m; struct wrq_cookie cookie; @@ -5977,7 +5987,7 @@ ethofld_tx(struct cxgbe_snd_tag *cst) cst->tx_credits -= next_credits; cst->tx_nocompl += next_credits; compl = cst->ncompl == 0 || cst->tx_nocompl >= cst->tx_total / 2; - ETHER_BPF_MTAP(cst->com.ifp, m); + ETHER_BPF_MTAP(cst->com.com.ifp, m); write_ethofld_wr(cst, wr, m, compl); commit_wrq_wr(cst->eo_txq, wr, &cookie); if (compl) { @@ -5989,7 +5999,7 @@ ethofld_tx(struct cxgbe_snd_tag *cst) /* * Drop the mbuf's reference on the tag now rather * than waiting until m_freem(). This ensures that - * cxgbe_snd_tag_free gets called when the inp drops + * cxgbe_rate_tag_free gets called when the inp drops * its reference on the tag and there are no more * mbufs in the pending_tx queue and can flush any * pending requests. Otherwise if the last mbuf @@ -5998,7 +6008,7 @@ ethofld_tx(struct cxgbe_snd_tag *cst) */ m->m_pkthdr.snd_tag = NULL; m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG; - m_snd_tag_rele(&cst->com); + m_snd_tag_rele(&cst->com.com); mbufq_enqueue(&cst->pending_fwack, m); } @@ -6007,13 +6017,13 @@ ethofld_tx(struct cxgbe_snd_tag *cst) int ethofld_transmit(struct ifnet *ifp, struct mbuf *m0) { - struct cxgbe_snd_tag *cst; + struct cxgbe_rate_tag *cst; int rc; MPASS(m0->m_nextpkt == NULL); MPASS(m0->m_pkthdr.csum_flags & CSUM_SND_TAG); MPASS(m0->m_pkthdr.snd_tag != NULL); - cst = mst_to_cst(m0->m_pkthdr.snd_tag); + cst = mst_to_crt(m0->m_pkthdr.snd_tag); mtx_lock(&cst->lock); MPASS(cst->flags & EO_SND_TAG_REF); @@ -6052,10 +6062,10 @@ ethofld_transmit(struct ifnet *ifp, struct mbuf *m0) * ethofld_tx() in case we are sending the final mbuf after * the inp was freed. */ - m_snd_tag_ref(&cst->com); + m_snd_tag_ref(&cst->com.com); ethofld_tx(cst); mtx_unlock(&cst->lock); - m_snd_tag_rele(&cst->com); + m_snd_tag_rele(&cst->com.com); return (0); done: @@ -6072,7 +6082,7 @@ ethofld_fw4_ack(struct sge_iq *iq, const struct rss_he const struct cpl_fw4_ack *cpl = (const void *)(rss + 1); struct mbuf *m; u_int etid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl))); - struct cxgbe_snd_tag *cst; + struct cxgbe_rate_tag *cst; uint8_t credits = cpl->credits; cst = lookup_etid(sc, etid); @@ -6104,7 +6114,7 @@ ethofld_fw4_ack(struct sge_iq *iq, const struct rss_he cst->flags &= ~EO_FLUSH_RPL_PENDING; cst->tx_credits += cpl->credits; - cxgbe_snd_tag_free_locked(cst); + cxgbe_rate_tag_free_locked(cst); return (0); /* cst is gone. */ } KASSERT(m != NULL, @@ -6126,12 +6136,12 @@ ethofld_fw4_ack(struct sge_iq *iq, const struct rss_he * As with ethofld_transmit(), hold an extra reference * so that the tag is stable across ethold_tx(). */ - m_snd_tag_ref(&cst->com); + m_snd_tag_ref(&cst->com.com); m = mbufq_first(&cst->pending_tx); if (m != NULL && cst->tx_credits >= mbuf_eo_len16(m)) ethofld_tx(cst); mtx_unlock(&cst->lock); - m_snd_tag_rele(&cst->com); + m_snd_tag_rele(&cst->com.com); } else { /* * There shouldn't be any pending packets if the tag