Date: Thu, 9 Aug 2018 14:19:47 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r337538 - in head/sys/dev/cxgbe: . common tom Message-ID: <201808091419.w79EJlJi058448@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: np Date: Thu Aug 9 14:19:47 2018 New Revision: 337538 URL: https://svnweb.freebsd.org/changeset/base/337538 Log: cxgbe(4): Add support for high priority filters on T6+. They have their own region in the TCAM starting with T6, unlike previous chips where they were in the same region as normal filters. These filters "hit" before anything else in the LE's lookup. The exact order is: a) High priority filters b) TOE's active region (TCAM and/or hash) c) Servers (TOE hw listeners) d) Normal filters MFC after: 1 week Sponsored by: Chelsio Communications Modified: head/sys/dev/cxgbe/common/common.h head/sys/dev/cxgbe/offload.h head/sys/dev/cxgbe/t4_filter.c head/sys/dev/cxgbe/t4_main.c head/sys/dev/cxgbe/t4_sge.c head/sys/dev/cxgbe/tom/t4_tom.c Modified: head/sys/dev/cxgbe/common/common.h ============================================================================== --- head/sys/dev/cxgbe/common/common.h Thu Aug 9 14:04:51 2018 (r337537) +++ head/sys/dev/cxgbe/common/common.h Thu Aug 9 14:19:47 2018 (r337538) @@ -441,6 +441,13 @@ static inline int is_ftid(const struct adapter *sc, u_ tid <= sc->tids.ftid_end); } +static inline int is_hpftid(const struct adapter *sc, u_int tid) +{ + + return (sc->tids.nhpftids > 0 && tid >= sc->tids.hpftid_base && + tid <= sc->tids.hpftid_end); +} + static inline int is_etid(const struct adapter *sc, u_int tid) { Modified: head/sys/dev/cxgbe/offload.h ============================================================================== --- head/sys/dev/cxgbe/offload.h Thu Aug 9 14:04:51 2018 (r337537) +++ head/sys/dev/cxgbe/offload.h Thu Aug 9 14:19:47 2018 (r337538) @@ -133,7 +133,12 @@ struct tid_info { u_int ftid_base; u_int ftid_end; + u_int nhpftids; + u_int hpftid_base; + u_int hpftid_end; + u_int ntids; + u_int tid_base; u_int netids; u_int etid_base; @@ -150,10 +155,13 @@ struct tid_info { union aopen_entry *afree; u_int atids_in_use; + /* High priority filters and normal filters share the lock and cv. */ struct mtx ftid_lock __aligned(CACHE_LINE_SIZE); struct cv ftid_cv; struct filter_entry *ftid_tab; + struct filter_entry *hpftid_tab; u_int ftids_in_use; + u_int hpftids_in_use; /* * hashfilter and TOE are mutually exclusive and both use ntids and Modified: head/sys/dev/cxgbe/t4_filter.c ============================================================================== --- head/sys/dev/cxgbe/t4_filter.c Thu Aug 9 14:04:51 2018 (r337537) +++ head/sys/dev/cxgbe/t4_filter.c Thu Aug 9 14:19:47 2018 (r337538) @@ -64,12 +64,20 @@ struct filter_entry { }; static void free_filter_resources(struct filter_entry *); +static int get_tcamfilter(struct adapter *, struct t4_filter *); static int get_hashfilter(struct adapter *, struct t4_filter *); static int set_hashfilter(struct adapter *, struct t4_filter *, uint64_t, struct l2t_entry *, struct smt_entry *); static int del_hashfilter(struct adapter *, struct t4_filter *); static int configure_hashfilter_tcb(struct adapter *, struct filter_entry *); +static inline bool +separate_hpfilter_region(struct adapter *sc) +{ + + return (chip_id(sc) >= CHELSIO_T6); +} + static int alloc_hftid_tab(struct tid_info *t, int flags) { @@ -302,7 +310,7 @@ set_filter_mode(struct adapter *sc, uint32_t mode) if (rc) return (rc); - if (sc->tids.ftids_in_use > 0) { + if (sc->tids.ftids_in_use > 0 || sc->tids.hpftids_in_use > 0) { rc = EBUSY; goto done; } @@ -343,39 +351,10 @@ get_filter_hits(struct adapter *sc, uint32_t tid) int get_filter(struct adapter *sc, struct t4_filter *t) { - int i, nfilters = sc->tids.nftids; - struct filter_entry *f; - if (t->fs.hash) return (get_hashfilter(sc, t)); - - if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || - t->idx >= nfilters) { - t->idx = 0xffffffff; - return (0); - } - - mtx_lock(&sc->tids.ftid_lock); - f = &sc->tids.ftid_tab[t->idx]; - for (i = t->idx; i < nfilters; i++, f++) { - if (f->valid) { - MPASS(f->tid == sc->tids.ftid_base + i); - t->idx = i; - t->l2tidx = f->l2te ? f->l2te->idx : 0; - t->smtidx = f->smt ? f->smt->idx : 0; - if (f->fs.hitcnts) - t->hits = get_filter_hits(sc, f->tid); - else - t->hits = UINT64_MAX; - t->fs = f->fs; - - goto done; - } - } - t->idx = 0xffffffff; -done: - mtx_unlock(&sc->tids.ftid_lock); - return (0); + else + return (get_tcamfilter(sc, t)); } static int @@ -387,15 +366,23 @@ set_tcamfilter(struct adapter *sc, struct t4_filter *t u_int vnic_vld, vnic_vld_mask; struct wrq_cookie cookie; int i, rc, busy, locked; + u_int tid; const int ntids = t->fs.type ? 4 : 1; MPASS(!t->fs.hash); - MPASS(t->idx < sc->tids.nftids); /* Already validated against fconf, iconf */ MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0); MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0); - f = &sc->tids.ftid_tab[t->idx]; + if (separate_hpfilter_region(sc) && t->fs.prio) { + MPASS(t->idx < sc->tids.nhpftids); + f = &sc->tids.hpftid_tab[t->idx]; + tid = sc->tids.hpftid_base + t->idx; + } else { + MPASS(t->idx < sc->tids.nftids); + f = &sc->tids.ftid_tab[t->idx]; + tid = sc->tids.ftid_base + t->idx; + } rc = busy = locked = 0; mtx_lock(&sc->tids.ftid_lock); for (i = 0; i < ntids; i++) { @@ -418,7 +405,10 @@ set_tcamfilter(struct adapter *sc, struct t4_filter *t rc = ENOMEM; else { f->pending = 1; - sc->tids.ftids_in_use++; + if (separate_hpfilter_region(sc) && t->fs.prio) + sc->tids.hpftids_in_use++; + else + sc->tids.ftids_in_use++; } } mtx_unlock(&sc->tids.ftid_lock); @@ -434,7 +424,7 @@ set_tcamfilter(struct adapter *sc, struct t4_filter *t * Can't fail now. A set-filter WR will definitely be sent. */ - f->tid = sc->tids.ftid_base + t->idx; + f->tid = tid; f->fs = t->fs; f->l2te = l2te; f->smt = smt; @@ -649,10 +639,17 @@ set_filter(struct adapter *sc, struct t4_filter *t) if (rc != 0) return (rc); } else { - if (ti->nftids == 0) - return (ENOTSUP); - if (t->idx >= ti->nftids) - return (EINVAL); + if (separate_hpfilter_region(sc) && t->fs.prio) { + if (ti->nhpftids == 0) + return (ENOTSUP); + if (t->idx >= ti->nhpftids) + return (EINVAL); + } else { + if (ti->nftids == 0) + return (ENOTSUP); + if (t->idx >= ti->nftids) + return (EINVAL); + } /* IPv6 filter idx must be 4 aligned */ if (t->fs.type == 1 && ((t->idx & 0x3) || t->idx + 4 >= ti->nftids)) @@ -702,17 +699,37 @@ set_filter(struct adapter *sc, struct t4_filter *t) if (rc != 0) goto done; } + } else if (separate_hpfilter_region(sc) && t->fs.prio && + __predict_false(ti->hpftid_tab == NULL)) { + MPASS(ti->nhpftids != 0); + KASSERT(ti->hpftids_in_use == 0, + ("%s: no memory allocated but hpftids_in_use is %u", + __func__, ti->hpftids_in_use)); + ti->hpftid_tab = malloc(sizeof(struct filter_entry) * + ti->nhpftids, M_CXGBE, M_NOWAIT | M_ZERO); + if (ti->hpftid_tab == NULL) { + rc = ENOMEM; + goto done; + } + if (!mtx_initialized(&sc->tids.ftid_lock)) { + mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF); + cv_init(&ti->ftid_cv, "t4fcv"); + } } else if (__predict_false(ti->ftid_tab == NULL)) { + MPASS(ti->nftids != 0); KASSERT(ti->ftids_in_use == 0, - ("%s: no memory allocated but ftids_in_use > 0", __func__)); + ("%s: no memory allocated but ftids_in_use is %u", + __func__, ti->ftids_in_use)); ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids, M_CXGBE, M_NOWAIT | M_ZERO); if (ti->ftid_tab == NULL) { rc = ENOMEM; goto done; } - mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF); - cv_init(&ti->ftid_cv, "t4fcv"); + if (!mtx_initialized(&sc->tids.ftid_lock)) { + mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF); + cv_init(&ti->ftid_cv, "t4fcv"); + } } done: end_synchronized_op(sc, 0); @@ -768,16 +785,32 @@ del_tcamfilter(struct adapter *sc, struct t4_filter *t struct filter_entry *f; struct fw_filter_wr *fwr; struct wrq_cookie cookie; - int rc; + int rc, nfilters; +#ifdef INVARIANTS + u_int tid_base; +#endif - MPASS(sc->tids.ftid_tab != NULL); - MPASS(sc->tids.nftids > 0); - - if (t->idx >= sc->tids.nftids) - return (EINVAL); - mtx_lock(&sc->tids.ftid_lock); - f = &sc->tids.ftid_tab[t->idx]; + if (separate_hpfilter_region(sc) && t->fs.prio) { + nfilters = sc->tids.nhpftids; + f = sc->tids.hpftid_tab; +#ifdef INVARIANTS + tid_base = sc->tids.hpftid_base; +#endif + } else { + nfilters = sc->tids.nftids; + f = sc->tids.ftid_tab; +#ifdef INVARIANTS + tid_base = sc->tids.ftid_base; +#endif + } + MPASS(f != NULL); /* Caller checked this. */ + if (t->idx >= nfilters) { + rc = EINVAL; + goto done; + } + f += t->idx; + if (f->locked) { rc = EPERM; goto done; @@ -790,7 +823,7 @@ del_tcamfilter(struct adapter *sc, struct t4_filter *t rc = EINVAL; goto done; } - MPASS(f->tid == sc->tids.ftid_base + t->idx); + MPASS(f->tid == tid_base + t->idx); fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); if (fwr == NULL) { rc = ENOMEM; @@ -833,6 +866,9 @@ del_filter(struct adapter *sc, struct t4_filter *t) if (t->fs.hash) { if (sc->tids.hftid_tab != NULL) return (del_hashfilter(sc, t)); + } else if (separate_hpfilter_region(sc) && t->fs.prio) { + if (sc->tids.hpftid_tab != NULL) + return (del_tcamfilter(sc, t)); } else { if (sc->tids.ftid_tab != NULL) return (del_tcamfilter(sc, t)); @@ -899,21 +935,28 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_head struct adapter *sc = iq->adapter; const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); u_int tid = GET_TID(rpl); - u_int rc, cleanup, idx; + u_int rc, idx; struct filter_entry *f; KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, rss->opcode)); - MPASS(is_ftid(sc, tid)); - cleanup = 0; - idx = tid - sc->tids.ftid_base; - f = &sc->tids.ftid_tab[idx]; + + if (is_hpftid(sc, tid)) { + idx = tid - sc->tids.hpftid_base; + f = &sc->tids.hpftid_tab[idx]; + } else if (is_ftid(sc, tid)) { + idx = tid - sc->tids.ftid_base; + f = &sc->tids.ftid_tab[idx]; + } else + panic("%s: FW reply for invalid TID %d.", __func__, tid); + + MPASS(f->tid == tid); rc = G_COOKIE(rpl->cookie); mtx_lock(&sc->tids.ftid_lock); KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.", - __func__, rc, idx)); + __func__, rc, tid)); switch(rc) { case FW_FILTER_WR_FLT_ADDED: /* set-filter succeeded */ @@ -936,7 +979,10 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_head /* set-filter failed due to lack of SMT space. */ MPASS(f->valid == 0); free_filter_resources(f); - sc->tids.ftids_in_use--; + if (separate_hpfilter_region(sc) && f->fs.prio) + sc->tids.hpftids_in_use--; + else + sc->tids.ftids_in_use--; break; case FW_FILTER_WR_SUCCESS: case FW_FILTER_WR_EINVAL: @@ -1077,10 +1123,68 @@ t4_del_hashfilter_rpl(struct sge_iq *iq, const struct } static int +get_tcamfilter(struct adapter *sc, struct t4_filter *t) +{ + int i, nfilters; + struct filter_entry *f; + u_int in_use; +#ifdef INVARIANTS + u_int tid_base; +#endif + + MPASS(!t->fs.hash); + + if (separate_hpfilter_region(sc) && t->fs.prio) { + nfilters = sc->tids.nhpftids; + f = sc->tids.hpftid_tab; + in_use = sc->tids.hpftids_in_use; +#ifdef INVARIANTS + tid_base = sc->tids.hpftid_base; +#endif + } else { + nfilters = sc->tids.nftids; + f = sc->tids.ftid_tab; + in_use = sc->tids.ftids_in_use; +#ifdef INVARIANTS + tid_base = sc->tids.ftid_base; +#endif + } + + if (in_use == 0 || f == NULL || t->idx >= nfilters) { + t->idx = 0xffffffff; + return (0); + } + + f += t->idx; + mtx_lock(&sc->tids.ftid_lock); + for (i = t->idx; i < nfilters; i++, f++) { + if (f->valid) { + MPASS(f->tid == tid_base + i); + t->idx = i; + t->l2tidx = f->l2te ? f->l2te->idx : 0; + t->smtidx = f->smt ? f->smt->idx : 0; + if (f->fs.hitcnts) + t->hits = get_filter_hits(sc, f->tid); + else + t->hits = UINT64_MAX; + t->fs = f->fs; + + goto done; + } + } + t->idx = 0xffffffff; +done: + mtx_unlock(&sc->tids.ftid_lock); + return (0); +} + +static int get_hashfilter(struct adapter *sc, struct t4_filter *t) { int i, nfilters = sc->tids.ntids; struct filter_entry *f; + + MPASS(t->fs.hash); if (sc->tids.tids_in_use == 0 || sc->tids.hftid_tab == NULL || t->idx >= nfilters) { Modified: head/sys/dev/cxgbe/t4_main.c ============================================================================== --- head/sys/dev/cxgbe/t4_main.c Thu Aug 9 14:04:51 2018 (r337537) +++ head/sys/dev/cxgbe/t4_main.c Thu Aug 9 14:19:47 2018 (r337538) @@ -1404,6 +1404,7 @@ t4_detach_common(device_t dev) free(sc->sge.iqmap, M_CXGBE); free(sc->sge.eqmap, M_CXGBE); free(sc->tids.ftid_tab, M_CXGBE); + free(sc->tids.hpftid_tab, M_CXGBE); if (sc->tids.hftid_tab) free_hftid_tab(&sc->tids); free(sc->tids.atid_tab, M_CXGBE); @@ -3853,6 +3854,47 @@ get_params__post_init(struct adapter *sc) __func__, sc->vres.l2t.size, L2T_SIZE)); sc->params.core_vdd = val[6]; + if (chip_id(sc) >= CHELSIO_T6) { + +#ifdef INVARIANTS + if (sc->params.fw_vers >= + (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) | + V_FW_HDR_FW_VER_MICRO(1) | V_FW_HDR_FW_VER_BUILD(0))) { + /* + * Note that the code to enable the region should run + * before t4_fw_initialize and not here. This is just a + * reminder to add said code. + */ + device_printf(sc->dev, + "hpfilter region not enabled.\n"); + } +#endif + + sc->tids.tid_base = t4_read_reg(sc, + A_LE_DB_ACTIVE_TABLE_START_INDEX); + + param[0] = FW_PARAM_PFVF(HPFILTER_START); + param[1] = FW_PARAM_PFVF(HPFILTER_END); + rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); + if (rc != 0) { + device_printf(sc->dev, + "failed to query hpfilter parameters: %d.\n", rc); + return (rc); + } + if ((int)val[1] > (int)val[0]) { + sc->tids.hpftid_base = val[0]; + sc->tids.hpftid_end = val[1]; + sc->tids.nhpftids = val[1] - val[0] + 1; + + /* + * These should go off if the layout changes and the + * driver needs to catch up. + */ + MPASS(sc->tids.hpftid_base == 0); + MPASS(sc->tids.tid_base == sc->tids.nhpftids); + } + } + /* * MPSBGMAP is queried separately because only recent firmwares support * it as a parameter and we don't want the compound query above to fail @@ -3916,6 +3958,12 @@ get_params__post_init(struct adapter *sc) return (rc); } sc->tids.ntids = val[0]; + if (sc->params.fw_vers < + (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) | + V_FW_HDR_FW_VER_MICRO(5) | V_FW_HDR_FW_VER_BUILD(0))) { + MPASS(sc->tids.ntids >= sc->tids.nhpftids); + sc->tids.ntids -= sc->tids.nhpftids; + } sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); sc->params.hash_filter = 1; } @@ -8127,6 +8175,11 @@ sysctl_tids(SYSCTL_HANDLER_ARGS) t->atids_in_use); } + if (t->nhpftids) { + sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n", + t->hpftid_base, t->hpftid_end, t->hpftids_in_use); + } + if (t->ntids) { sbuf_printf(sb, "TID range: "); if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { @@ -8141,10 +8194,10 @@ sysctl_tids(SYSCTL_HANDLER_ARGS) } if (b) - sbuf_printf(sb, "0-%u, ", b - 1); + sbuf_printf(sb, "%u-%u, ", t->tid_base, b - 1); sbuf_printf(sb, "%u-%u", hb, t->ntids - 1); } else - sbuf_printf(sb, "0-%u", t->ntids - 1); + sbuf_printf(sb, "%u-%u", t->tid_base, t->ntids - 1); sbuf_printf(sb, ", in use: %u\n", atomic_load_acq_int(&t->tids_in_use)); } @@ -8155,8 +8208,8 @@ sysctl_tids(SYSCTL_HANDLER_ARGS) } if (t->nftids) { - sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, - t->ftid_base + t->nftids - 1); + sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base, + t->ftid_end, t->ftids_in_use); } if (t->netids) { Modified: head/sys/dev/cxgbe/t4_sge.c ============================================================================== --- head/sys/dev/cxgbe/t4_sge.c Thu Aug 9 14:04:51 2018 (r337537) +++ head/sys/dev/cxgbe/t4_sge.c Thu Aug 9 14:19:47 2018 (r337538) @@ -367,7 +367,7 @@ set_tcb_rpl_handler(struct sge_iq *iq, const struct rs MPASS(m == NULL); tid = GET_TID(cpl); - if (is_ftid(iq->adapter, tid)) { + if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) { /* * The return code for filter-write is put in the CPL cookie so * we have to rely on the hardware tid (is_ftid) to determine Modified: head/sys/dev/cxgbe/tom/t4_tom.c ============================================================================== --- head/sys/dev/cxgbe/tom/t4_tom.c Thu Aug 9 14:04:51 2018 (r337537) +++ head/sys/dev/cxgbe/tom/t4_tom.c Thu Aug 9 14:19:47 2018 (r337538) @@ -510,7 +510,10 @@ insert_tid(struct adapter *sc, int tid, void *ctx, int { struct tid_info *t = &sc->tids; - t->tid_tab[tid] = ctx; + MPASS(tid >= t->tid_base); + MPASS(tid - t->tid_base < t->ntids); + + t->tid_tab[tid - t->tid_base] = ctx; atomic_add_int(&t->tids_in_use, ntids); } @@ -519,7 +522,7 @@ lookup_tid(struct adapter *sc, int tid) { struct tid_info *t = &sc->tids; - return (t->tid_tab[tid]); + return (t->tid_tab[tid - t->tid_base]); } void @@ -527,7 +530,7 @@ update_tid(struct adapter *sc, int tid, void *ctx) { struct tid_info *t = &sc->tids; - t->tid_tab[tid] = ctx; + t->tid_tab[tid - t->tid_base] = ctx; } void @@ -535,7 +538,7 @@ remove_tid(struct adapter *sc, int tid, int ntids) { struct tid_info *t = &sc->tids; - t->tid_tab[tid] = NULL; + t->tid_tab[tid - t->tid_base] = NULL; atomic_subtract_int(&t->tids_in_use, ntids); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201808091419.w79EJlJi058448>