Date: Mon, 29 Sep 2025 14:37:41 GMT From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 6d29d22a9e0c - main - cxgbe(4): Use hw_port and tx_chan correctly all over the driver Message-ID: <202509291437.58TEbfEd017523@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by np: URL: https://cgit.FreeBSD.org/src/commit/?id=6d29d22a9e0c33d1e3e70b8c458982c59b76eb29 commit 6d29d22a9e0c33d1e3e70b8c458982c59b76eb29 Author: Navdeep Parhar <np@FreeBSD.org> AuthorDate: 2025-09-29 05:45:01 +0000 Commit: Navdeep Parhar <np@FreeBSD.org> CommitDate: 2025-09-29 14:25:58 +0000 cxgbe(4): Use hw_port and tx_chan correctly all over the driver The port's tx channel and its hardware index are different things that happen to have the same value on NICs using T4 through T6. This is going to change so use the correct spelling all over the driver. Specifically, the firmware expects the hw port index in all of these: * PCIe channel for queues * TX interface for TX CPLs * PORTID in FW_PORT commands * PORTID in t4_alloc_vi * flowid in all the firmware flowc WRs * mbox commands that deal with the tx scheduler Also, create a port_id lookup table for hw_port, just like the one for tx_chan. Use it to lookup the port softc in the port_info handler. MFC after: 3 days Sponsored by: Chelsio Communications --- sys/dev/cxgbe/adapter.h | 6 ++++-- sys/dev/cxgbe/common/t4_hw.c | 16 ++++++---------- sys/dev/cxgbe/t4_main.c | 14 ++++++++------ sys/dev/cxgbe/t4_netmap.c | 6 +++--- sys/dev/cxgbe/t4_sched.c | 6 +++--- sys/dev/cxgbe/t4_sge.c | 22 ++++++++++++---------- sys/dev/cxgbe/tom/t4_cpl_io.c | 5 +++-- sys/dev/cxgbe/tom/t4_listen.c | 5 +++-- 8 files changed, 42 insertions(+), 38 deletions(-) diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index 248a8e112add..5abe0de7664f 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -477,6 +477,7 @@ struct sge_eq { uint8_t doorbells; uint8_t port_id; /* port_id of the port associated with the eq */ uint8_t tx_chan; /* tx channel used by the eq */ + uint8_t hw_port; /* hw port used by the eq */ struct mtx eq_lock; struct tx_desc *desc; /* KVA of descriptor ring */ @@ -937,7 +938,8 @@ struct adapter { struct taskqueue *tq[MAX_NPORTS]; /* General purpose taskqueues */ struct port_info *port[MAX_NPORTS]; - uint8_t chan_map[MAX_NCHAN]; /* channel -> port */ + uint8_t chan_map[MAX_NCHAN]; /* tx_chan -> port_id */ + uint8_t port_map[MAX_NPORTS]; /* hw_port -> port_id */ CXGBE_LIST_HEAD(, clip_entry) *clip_table; TAILQ_HEAD(, clip_entry) clip_pending; /* these need hw update. */ @@ -961,7 +963,7 @@ struct adapter { uint8_t doorbells; int offload_map; /* port_id's with IFCAP_TOE enabled */ - int bt_map; /* tx_chan's with BASE-T */ + int bt_map; /* hw_port's that are BASE-T */ int active_ulds; /* ULDs activated on this adapter */ int flags; int debug_flags; diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c index 01890b9598b8..f90ad3010626 100644 --- a/sys/dev/cxgbe/common/t4_hw.c +++ b/sys/dev/cxgbe/common/t4_hw.c @@ -8964,7 +8964,7 @@ static void handle_port_info(struct port_info *pi, const struct fw_port_cmd *p, memset(&cmd, 0, sizeof(cmd)); cmd.op_to_portid = cpu_to_be32(V_FW_CMD_OP(FW_PORT_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_READ | - V_FW_PORT_CMD_PORTID(pi->tx_chan)); + V_FW_PORT_CMD_PORTID(pi->hw_port)); action = sc->params.port_caps32 ? FW_PORT_ACTION_GET_PORT_INFO32 : FW_PORT_ACTION_GET_PORT_INFO; cmd.action_to_len16 = cpu_to_be32(V_FW_PORT_CMD_ACTION(action) | @@ -8996,16 +8996,12 @@ int t4_handle_fw_rpl(struct adapter *adap, const __be64 *rpl) (action == FW_PORT_ACTION_GET_PORT_INFO || action == FW_PORT_ACTION_GET_PORT_INFO32)) { /* link/module state change message */ - int i; - int chan = G_FW_PORT_CMD_PORTID(be32_to_cpu(p->op_to_portid)); - struct port_info *pi = NULL; - - for_each_port(adap, i) { - pi = adap2pinfo(adap, i); - if (pi->tx_chan == chan) - break; - } + int hw_port = G_FW_PORT_CMD_PORTID(be32_to_cpu(p->op_to_portid)); + int port_id = adap->port_map[hw_port]; + struct port_info *pi; + MPASS(port_id >= 0 && port_id < adap->params.nports); + pi = adap->port[port_id]; PORT_LOCK(pi); handle_port_info(pi, p, action, &mod_changed, &link_changed); PORT_UNLOCK(pi); diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index 9756a6945384..9f6162133d05 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -1277,6 +1277,7 @@ t4_attach(device_t dev) goto done; /* error message displayed already */ memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); + memset(sc->port_map, 0xff, sizeof(sc->port_map)); /* Prepare the adapter for operation. */ buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK); @@ -1407,14 +1408,15 @@ t4_attach(device_t dev) } if (is_bt(pi->port_type)) - setbit(&sc->bt_map, pi->tx_chan); + setbit(&sc->bt_map, pi->hw_port); else - MPASS(!isset(&sc->bt_map, pi->tx_chan)); + MPASS(!isset(&sc->bt_map, pi->hw_port)); snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", device_get_nameunit(dev), i); mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); sc->chan_map[pi->tx_chan] = i; + sc->port_map[pi->hw_port] = i; /* * The MPS counter for FCS errors doesn't work correctly on the @@ -3684,7 +3686,7 @@ alloc_extra_vi(struct adapter *sc, struct port_info *pi, struct vi_info *vi) ("%s: VI %s doesn't have a MAC func", __func__, device_get_nameunit(vi->dev))); func = vi_mac_funcs[index]; - rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, + rc = t4_alloc_vi_func(sc, sc->mbox, pi->hw_port, sc->pf, 0, 1, vi->hw_addr, &vi->rss_size, &vi->vfvld, &vi->vin, func, 0); if (rc < 0) { CH_ERR(vi, "failed to allocate virtual interface %d" @@ -6220,7 +6222,7 @@ apply_link_config(struct port_info *pi) MPASS(lc->pcaps & FW_PORT_CAP32_FEC_BASER_RS); #endif if (!(sc->flags & IS_VF)) { - rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); + rc = -t4_link_l1cfg(sc, sc->mbox, pi->hw_port, lc); if (rc != 0) { device_printf(pi->dev, "l1cfg failed: %d\n", rc); return (rc); @@ -8452,14 +8454,14 @@ sysctl_tx_vm_wr(SYSCTL_HANDLER_ARGS) vi->flags |= TX_USES_VM_WR; if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_VM_TSO); ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | - V_TXPKT_INTF(pi->tx_chan)); + V_TXPKT_INTF(pi->hw_port)); if (!(sc->flags & IS_VF)) npkt--; } else { vi->flags &= ~TX_USES_VM_WR; if_sethwtsomaxsegcount(vi->ifp, TX_SGL_SEGS_TSO); ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | - V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | + V_TXPKT_INTF(pi->hw_port) | V_TXPKT_PF(sc->pf) | V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); } for_each_txq(vi, i, txq) { diff --git a/sys/dev/cxgbe/t4_netmap.c b/sys/dev/cxgbe/t4_netmap.c index e53fb5054316..ddd7d974aa48 100644 --- a/sys/dev/cxgbe/t4_netmap.c +++ b/sys/dev/cxgbe/t4_netmap.c @@ -232,7 +232,7 @@ alloc_nm_txq(struct vi_info *vi, struct sge_nm_txq *nm_txq, int iqidx, int idx) nm_txq->nid = idx; nm_txq->iqidx = iqidx; nm_txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT) | - V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | + V_TXPKT_INTF(pi->hw_port) | V_TXPKT_PF(sc->pf) | V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); if (sc->params.fw_vers >= FW_VERSION32(1, 24, 11, 0)) nm_txq->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS2_WR)); @@ -314,7 +314,7 @@ alloc_nm_rxq_hwq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq) V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | V_FW_IQ_CMD_VIID(vi->viid) | V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); - c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | + c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->hw_port) | F_FW_IQ_CMD_IQGTSMODE | V_FW_IQ_CMD_IQINTCNTTHRESH(0) | V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4)); @@ -429,7 +429,7 @@ alloc_nm_txq_hwq(struct vi_info *vi, struct sge_nm_txq *nm_txq) F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid)); c.fetchszm_to_iqid = htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | - V_FW_EQ_ETH_CMD_PCIECHN(vi->pi->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | + V_FW_EQ_ETH_CMD_PCIECHN(vi->pi->hw_port) | F_FW_EQ_ETH_CMD_FETCHRO | V_FW_EQ_ETH_CMD_IQID(sc->sge.nm_rxq[nm_txq->iqidx].iq_cntxt_id)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? diff --git a/sys/dev/cxgbe/t4_sched.c b/sys/dev/cxgbe/t4_sched.c index 2186c8aa2ac0..65c2720d692c 100644 --- a/sys/dev/cxgbe/t4_sched.c +++ b/sys/dev/cxgbe/t4_sched.c @@ -272,7 +272,7 @@ update_tx_sched(void *context, int pending) } rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, FW_SCHED_PARAMS_LEVEL_CL_RL, tc->mode, tc->rateunit, - tc->ratemode, pi->tx_chan, j, 0, tc->maxrate, 0, + tc->ratemode, pi->hw_port, j, 0, tc->maxrate, 0, tc->pktsize, tc->burstsize, 1); end_synchronized_op(sc, 0); @@ -291,7 +291,7 @@ update_tx_sched(void *context, int pending) "params: mode %d, rateunit %d, ratemode %d, " "channel %d, minrate %d, maxrate %d, pktsize %d, " "burstsize %d\n", j, rc, tc->mode, tc->rateunit, - tc->ratemode, pi->tx_chan, 0, tc->maxrate, + tc->ratemode, pi->hw_port, 0, tc->maxrate, tc->pktsize, tc->burstsize); } } @@ -839,7 +839,7 @@ failed: cst->tx_total = cst->tx_credits; cst->plen = 0; cst->ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | - V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | + V_TXPKT_INTF(pi->hw_port) | V_TXPKT_PF(sc->pf) | V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); /* diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index 86454bc4fe10..4998f27603db 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -3414,6 +3414,7 @@ init_eq(struct adapter *sc, struct sge_eq *eq, int eqtype, int qsize, eq->type = eqtype; eq->port_id = port_id; eq->tx_chan = sc->port[port_id]->tx_chan; + eq->hw_port = sc->port[port_id]->hw_port; eq->iq = iq; eq->sidx = qsize - sc->params.sge.spg_len / EQ_ESIZE; strlcpy(eq->lockname, name, sizeof(eq->lockname)); @@ -3577,7 +3578,7 @@ alloc_iq_fl_hwq(struct vi_info *vi, struct sge_iq *iq, struct sge_fl *fl) V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) | V_FW_IQ_CMD_VIID(vi->viid) | V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT)); - c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | + c.iqdroprss_to_iqesize = htobe16(V_FW_IQ_CMD_IQPCIECH(pi->hw_port) | F_FW_IQ_CMD_IQGTSMODE | V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) | V_FW_IQ_CMD_IQESIZE(ilog2(IQ_ESIZE) - 4)); @@ -4270,7 +4271,7 @@ ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) c.physeqid_pkd = htobe32(0); c.fetchszm_to_iqid = htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | - V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) | + V_FW_EQ_CTRL_CMD_PCIECHN(eq->hw_port) | F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_CTRL_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? @@ -4282,8 +4283,8 @@ ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq) rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c); if (rc != 0) { - CH_ERR(sc, "failed to create hw ctrlq for tx_chan %d: %d\n", - eq->tx_chan, rc); + CH_ERR(sc, "failed to create hw ctrlq for port %d: %d\n", + eq->port_id, rc); return (rc); } @@ -4316,7 +4317,7 @@ eth_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(vi->viid)); c.fetchszm_to_iqid = htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) | - V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO | + V_FW_EQ_ETH_CMD_PCIECHN(eq->hw_port) | F_FW_EQ_ETH_CMD_FETCHRO | V_FW_EQ_ETH_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? @@ -4360,7 +4361,7 @@ ofld_eq_alloc(struct adapter *sc, struct vi_info *vi, struct sge_eq *eq) F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c)); c.fetchszm_to_iqid = htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) | - V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) | + V_FW_EQ_OFLD_CMD_PCIECHN(eq->hw_port) | F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid)); c.dcaen_to_eqsize = htobe32(V_FW_EQ_OFLD_CMD_FBMIN(chip_id(sc) <= CHELSIO_T5 ? @@ -4678,10 +4679,10 @@ failed: if (vi->flags & TX_USES_VM_WR) txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | - V_TXPKT_INTF(pi->tx_chan)); + V_TXPKT_INTF(pi->hw_port)); else txq->cpl_ctrl0 = htobe32(V_TXPKT_OPCODE(CPL_TX_PKT_XT) | - V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(sc->pf) | + V_TXPKT_INTF(pi->hw_port) | V_TXPKT_PF(sc->pf) | V_TXPKT_VF(vi->vin) | V_TXPKT_VF_VLD(vi->vfvld)); txq->tc_idx = -1; @@ -6583,10 +6584,11 @@ send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi, V_FW_WR_FLOWID(cst->etid)); flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; flowc->mnemval[0].val = htobe32(pfvf); + /* Firmware expects hw port and will translate to channel itself. */ flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; - flowc->mnemval[1].val = htobe32(pi->tx_chan); + flowc->mnemval[1].val = htobe32(pi->hw_port); flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; - flowc->mnemval[2].val = htobe32(pi->tx_chan); + flowc->mnemval[2].val = htobe32(pi->hw_port); flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; flowc->mnemval[3].val = htobe32(cst->iqid); flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_EOSTATE; diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index be20ea42474e..402aa29c05cb 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -127,8 +127,9 @@ send_flowc_wr(struct toepcb *toep, struct tcpcb *tp) paramidx = 0; FLOWC_PARAM(PFNVFN, pfvf); - FLOWC_PARAM(CH, pi->tx_chan); - FLOWC_PARAM(PORT, pi->tx_chan); + /* Firmware expects hw port and will translate to channel itself. */ + FLOWC_PARAM(CH, pi->hw_port); + FLOWC_PARAM(PORT, pi->hw_port); FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id); FLOWC_PARAM(SNDBUF, toep->params.sndbuf); if (tp) { diff --git a/sys/dev/cxgbe/tom/t4_listen.c b/sys/dev/cxgbe/tom/t4_listen.c index 06c495dcafc3..ad835eafdc56 100644 --- a/sys/dev/cxgbe/tom/t4_listen.c +++ b/sys/dev/cxgbe/tom/t4_listen.c @@ -508,10 +508,11 @@ send_flowc_wr_synqe(struct adapter *sc, struct synq_entry *synqe) V_FW_WR_FLOWID(synqe->tid)); flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; flowc->mnemval[0].val = htobe32(pfvf); + /* Firmware expects hw port and will translate to channel itself. */ flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; - flowc->mnemval[1].val = htobe32(pi->tx_chan); + flowc->mnemval[1].val = htobe32(pi->hw_port); flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; - flowc->mnemval[2].val = htobe32(pi->tx_chan); + flowc->mnemval[2].val = htobe32(pi->hw_port); flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; flowc->mnemval[3].val = htobe32(ofld_rxq->iq.abs_id); flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202509291437.58TEbfEd017523>