Skip site navigation (1)Skip section navigation (2) 
Date:      Tue, 03 Sep 2019 14:05:47 -0000
From:      Navdeep Parhar <np@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org
Subject:   svn commit: r345595 - in stable/12: sys/dev/cxgbe usr.sbin/cxgbetool
Message-ID:  <201903271937.x2RJboW1078265@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: np
Date: Wed Mar 27 19:37:50 2019
New Revision: 345595
URL: https://svnweb.freebsd.org/changeset/base/345595

Log:
  MFC r339717, r339749, and r339809.
  
  r339717:
  cxgbe(4): Allow "pass" filters to distribute matching traffic using a
  subset of a VI's RSS indirection table.
  
  This makes it possible to make groups out of rx queues and steer
  different kinds of traffic to different groups.  For example, an
  interface with 8 rx queues could have all non-TCP traffic delivered to
  queues 0-3 and all TCP traffic to queues 4-7.
  
  Note that it is already possible for filters to steer traffic to a
  particular queue or to distribute it using the full indirection table
  (much like normal rx does).
  
  Sponsored by:	Chelsio Communications
  
  r339749:
  cxgbe(4): Add a knob to split the rx queues for a netmap enabled
  interface into two groups.  Filters can be used to match traffic
  and distribute it across a group.
  
  hw.cxgbe.nm_split_rss
  
  Sponsored by:	Chelsio Communications
  
  r339809:
  cxgbetool(8): Add a subaction (tcbrss <n>) that can be used with "pass"
  action to distribute traffic using the half of the VI's RSS indirection
  table.
  
  The value specified should either be the start of the VI's RSS slice
  (available at dev.<ifname>.<inst>.rss_base since r339700) or the
  midpoint (rss_base + rss_size/2).  The traffic that hits the filter will
  use the first or second half of the indirection table respectively.
  The indirection table can be populated in different ways to achieve
  different kinds of traffic/load distributions.  For example, r339749
  allows a netmap interface to have half the rx queues in the first half
  of the table and the rest in the other.
  
  Sponsored by:	Chelsio Communications

Modified:
  stable/12/sys/dev/cxgbe/t4_filter.c
  stable/12/sys/dev/cxgbe/t4_ioctl.h
  stable/12/sys/dev/cxgbe/t4_main.c
  stable/12/sys/dev/cxgbe/t4_netmap.c
  stable/12/usr.sbin/cxgbetool/cxgbetool.c
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/dev/cxgbe/t4_filter.c
==============================================================================
--- stable/12/sys/dev/cxgbe/t4_filter.c	Wed Mar 27 19:34:19 2019	(r345594)
+++ stable/12/sys/dev/cxgbe/t4_filter.c	Wed Mar 27 19:37:50 2019	(r345595)
@@ -864,8 +864,8 @@ set_filter(struct adapter *sc, struct t4_filter *t)
 	if (t->fs.val.iport >= sc->params.nports)
 		return (EINVAL);
 
-	/* Can't specify an iq if not steering to it */
-	if (!t->fs.dirsteer && t->fs.iq)
+	/* Can't specify an iqid/rss_info if not steering. */
+	if (!t->fs.dirsteer && !t->fs.dirsteerhash && !t->fs.maskhash && t->fs.iq)
 		return (EINVAL);
 
 	/* Validate against the global filter mode and ingress config */

Modified: stable/12/sys/dev/cxgbe/t4_ioctl.h
==============================================================================
--- stable/12/sys/dev/cxgbe/t4_ioctl.h	Wed Mar 27 19:34:19 2019	(r345594)
+++ stable/12/sys/dev/cxgbe/t4_ioctl.h	Wed Mar 27 19:37:50 2019	(r345595)
@@ -209,7 +209,7 @@ struct t4_filter_specification {
 	uint32_t rpttid:1;	/* report TID in RSS hash field */
 	uint32_t dirsteer:1;	/* 0 => RSS, 1 => steer to iq */
 	uint32_t iq:10;		/* ingress queue */
-	uint32_t maskhash:1;	/* dirsteer=0: store RSS hash in TCB */
+	uint32_t maskhash:1;	/* dirsteer=0: steer to an RSS sub-region */
 	uint32_t dirsteerhash:1;/* dirsteer=1: 0 => TCB contains RSS hash */
 				/*             1 => TCB contains IQ ID */
 

Modified: stable/12/sys/dev/cxgbe/t4_main.c
==============================================================================
--- stable/12/sys/dev/cxgbe/t4_main.c	Wed Mar 27 19:34:19 2019	(r345594)
+++ stable/12/sys/dev/cxgbe/t4_main.c	Wed Mar 27 19:37:50 2019	(r345595)
@@ -4286,6 +4286,11 @@ set_params__post_init(struct adapter *sc)
 	if (t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val) == 0)
 		sc->params.port_caps32 = 1;
 
+	/* Let filter + maskhash steer to a part of the VI's RSS region. */
+	val = 1 << (G_MASKSIZE(t4_read_reg(sc, A_TP_RSS_CONFIG_TNL)) - 1);
+	t4_set_reg_field(sc, A_TP_RSS_CONFIG_TNL, V_MASKFILTER(M_MASKFILTER),
+	    V_MASKFILTER(val - 1));
+
 #ifdef TCP_OFFLOAD
 	/*
 	 * Override the TOE timers with user provided tunables.  This is not the

Modified: stable/12/sys/dev/cxgbe/t4_netmap.c
==============================================================================
--- stable/12/sys/dev/cxgbe/t4_netmap.c	Wed Mar 27 19:34:19 2019	(r345594)
+++ stable/12/sys/dev/cxgbe/t4_netmap.c	Wed Mar 27 19:37:50 2019	(r345595)
@@ -101,6 +101,15 @@ int lazy_tx_credit_flush = 1;
 SYSCTL_INT(_hw_cxgbe, OID_AUTO, lazy_tx_credit_flush, CTLFLAG_RWTUN,
     &lazy_tx_credit_flush, 0, "lazy credit flush for netmap tx queues.");
 
+/*
+ * Split the netmap rx queues into two groups that populate separate halves of
+ * the RSS indirection table.  This allows filters with hashmask to steer to a
+ * particular group of queues.
+ */
+static int nm_split_rss = 0;
+SYSCTL_INT(_hw_cxgbe, OID_AUTO, nm_split_rss, CTLFLAG_RWTUN,
+    &nm_split_rss, 0, "Split the netmap rx queues into two groups.");
+
 static int
 alloc_nm_rxq_hwq(struct vi_info *vi, struct sge_nm_rxq *nm_rxq, int cong)
 {
@@ -333,7 +342,7 @@ cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi
 	struct netmap_kring *kring;
 	struct sge_nm_rxq *nm_rxq;
 	struct sge_nm_txq *nm_txq;
-	int rc, i, j, hwidx;
+	int rc, i, j, hwidx, defq, nrssq;
 	struct hw_buf_info *hwb;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
@@ -403,20 +412,69 @@ cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi
 		vi->nm_rss = malloc(vi->rss_size * sizeof(uint16_t), M_CXGBE,
 		    M_ZERO | M_WAITOK);
 	}
-	for (i = 0; i < vi->rss_size;) {
-		for_each_nm_rxq(vi, j, nm_rxq) {
-			vi->nm_rss[i++] = nm_rxq->iq_abs_id;
-			if (i == vi->rss_size)
-				break;
+
+	MPASS(vi->nnmrxq > 0);
+	if (nm_split_rss == 0 || vi->nnmrxq == 1) {
+		for (i = 0; i < vi->rss_size;) {
+			for_each_nm_rxq(vi, j, nm_rxq) {
+				vi->nm_rss[i++] = nm_rxq->iq_abs_id;
+				if (i == vi->rss_size)
+					break;
+			}
 		}
+		defq = vi->nm_rss[0];
+	} else {
+		/* We have multiple queues and we want to split the table. */
+		MPASS(nm_split_rss != 0);
+		MPASS(vi->nnmrxq > 1);
+
+		nm_rxq = &sc->sge.nm_rxq[vi->first_nm_rxq];
+		nrssq = vi->nnmrxq;
+		if (vi->nnmrxq & 1) {
+			/*
+			 * Odd number of queues. The first rxq is designated the
+			 * default queue, the rest are split evenly.
+			 */
+			defq = nm_rxq->iq_abs_id;
+			nm_rxq++;
+			nrssq--;
+		} else {
+			/*
+			 * Even number of queues split into two halves.  The
+			 * first rxq in one of the halves is designated the
+			 * default queue.
+			 */
+#if 1
+			/* First rxq in the first half. */
+			defq = nm_rxq->iq_abs_id;
+#else
+			/* First rxq in the second half. */
+			defq = nm_rxq[vi->nnmrxq / 2].iq_abs_id;
+#endif
+		}
+
+		i = 0;
+		while (i < vi->rss_size / 2) {
+			for (j = 0; j < nrssq / 2; j++) {
+				vi->nm_rss[i++] = nm_rxq[j].iq_abs_id;
+				if (i == vi->rss_size / 2)
+					break;
+			}
+		}
+		while (i < vi->rss_size) {
+			for (j = nrssq / 2; j < nrssq; j++) {
+				vi->nm_rss[i++] = nm_rxq[j].iq_abs_id;
+				if (i == vi->rss_size)
+					break;
+			}
+		}
 	}
 	rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size,
 	    vi->nm_rss, vi->rss_size);
 	if (rc != 0)
 		if_printf(ifp, "netmap rss_config failed: %d\n", rc);
 
-	rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen,
-	    vi->nm_rss[0], 0, 0);
+	rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, vi->hashen, defq, 0, 0);
 	if (rc != 0)
 		if_printf(ifp, "netmap rss hash/defaultq config failed: %d\n", rc);
 

Modified: stable/12/usr.sbin/cxgbetool/cxgbetool.c
==============================================================================
--- stable/12/usr.sbin/cxgbetool/cxgbetool.c	Wed Mar 27 19:34:19 2019	(r345594)
+++ stable/12/usr.sbin/cxgbetool/cxgbetool.c	Wed Mar 27 19:37:50 2019	(r345595)
@@ -924,7 +924,7 @@ do_show_one_filter_info(struct t4_filter *t, uint32_t 
 		if (t->fs.dirsteer == 0) {
 			printf("RSS");
 			if (t->fs.maskhash)
-				printf("(TCB=hash)");
+				printf("(region %d)", t->fs.iq << 1);
 		} else {
 			printf("%d", t->fs.iq);
 			if (t->fs.dirsteerhash == 0)
@@ -1263,11 +1263,19 @@ set_filter(uint32_t idx, int argc, const char *argv[],
 		} else if (!parse_val("rpttid", args, &val)) {
 			t.fs.rpttid = 1;
 		} else if (!parse_val("queue", args, &val)) {
-			t.fs.dirsteer = 1;
-			t.fs.iq = val;
+			t.fs.dirsteer = 1;	/* direct steer */
+			t.fs.iq = val;		/* to the iq with this cntxt_id */
 		} else if (!parse_val("tcbhash", args, &val)) {
-			t.fs.maskhash = 1;
-			t.fs.dirsteerhash = 1;
+			t.fs.dirsteerhash = 1;	/* direct steer */
+			/* XXX: use (val << 1) as the rss_hash? */
+			t.fs.iq = val;
+		} else if (!parse_val("tcbrss", args, &val)) {
+			t.fs.maskhash = 1;	/* steer to RSS region */
+			/*
+			 * val = start idx of the region but the internal TCB
+			 * field is 10b only and is left shifted by 1 before use.
+			 */
+			t.fs.iq = val >> 1;
 		} else if (!parse_val("eport", args, &val)) {
 			t.fs.eport = val;
 		} else if (!parse_val("swapmac", args, &val)) {

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201903271937.x2RJboW1078265>