Skip site navigation (1)Skip section navigation (2)
Date:      Mon, 26 Mar 2018 20:27:08 +0000 (UTC)
From:      Hans Petter Selasky <hselasky@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org
Subject:   svn commit: r331577 - in stable/11/sys/dev/mlx5: . mlx5_core mlx5_en
Message-ID:  <201803262027.w2QKR8op016507@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: hselasky
Date: Mon Mar 26 20:27:08 2018
New Revision: 331577
URL: https://svnweb.freebsd.org/changeset/base/331577

Log:
  MFC r330607:
  Implement rate limit per traffic class in mlx5core.
  
  Add support for rate limiting traffic class via sysctl.
  
  Submitted by:	Slava Shwartsman <slavash@mellanox.com>
  Sponsored by:	Mellanox Technologies

Modified:
  stable/11/sys/dev/mlx5/mlx5_core/mlx5_port.c
  stable/11/sys/dev/mlx5/mlx5_en/en.h
  stable/11/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
  stable/11/sys/dev/mlx5/mlx5_ifc.h
  stable/11/sys/dev/mlx5/port.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/dev/mlx5/mlx5_core/mlx5_port.c
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_core/mlx5_port.c	Mon Mar 26 20:25:24 2018	(r331576)
+++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_port.c	Mon Mar 26 20:27:08 2018	(r331577)
@@ -860,6 +860,89 @@ int mlx5_query_port_cong_params(struct mlx5_core_dev *
 					  out, out_size);
 }
 
+static int mlx5_query_port_qetcr_reg(struct mlx5_core_dev *mdev, u32 *out,
+				     int outlen)
+{
+	u32 in[MLX5_ST_SZ_DW(qtct_reg)];
+
+	if (!MLX5_CAP_GEN(mdev, ets))
+		return -ENOTSUPP;
+
+	memset(in, 0, sizeof(in));
+	return mlx5_core_access_reg(mdev, in, sizeof(in), out, outlen,
+				    MLX5_REG_QETCR, 0, 0);
+}
+
+int mlx5_max_tc(struct mlx5_core_dev *mdev)
+{
+	u8 num_tc = MLX5_CAP_GEN(mdev, max_tc) ? : 8;
+
+	return num_tc - 1;
+}
+EXPORT_SYMBOL_GPL(mlx5_max_tc);
+
+static int mlx5_set_port_qetcr_reg(struct mlx5_core_dev *mdev, u32 *in,
+				   int inlen)
+{
+	u32 out[MLX5_ST_SZ_DW(qtct_reg)];
+
+	if (!MLX5_CAP_GEN(mdev, ets))
+		return -ENOTSUPP;
+
+	return mlx5_core_access_reg(mdev, in, inlen, out, sizeof(out),
+				    MLX5_REG_QETCR, 0, 1);
+}
+
+int mlx5_query_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+				   u8 *max_bw_value,
+				   u8 *max_bw_units)
+{
+	u32 out[MLX5_ST_SZ_DW(qetc_reg)];
+	void *ets_tcn_conf;
+	int err;
+	int i;
+
+	err = mlx5_query_port_qetcr_reg(mdev, out, sizeof(out));
+	if (err)
+		return err;
+
+	for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+		ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, out, tc_configuration[i]);
+
+		max_bw_value[i] = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf,
+					   max_bw_value);
+		max_bw_units[i] = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf,
+					   max_bw_units);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_port_tc_rate_limit);
+
+int mlx5_modify_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+				   const u8 *max_bw_value,
+				   const u8 *max_bw_units)
+{
+	u32 in[MLX5_ST_SZ_DW(qetc_reg)] = {};
+	void *ets_tcn_conf;
+	int i;
+
+	MLX5_SET(qetc_reg, in, port_number, 1);
+
+	for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+		ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, in, tc_configuration[i]);
+
+		MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, r, 1);
+		MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, max_bw_units,
+			 max_bw_units[i]);
+		MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, max_bw_value,
+			 max_bw_value[i]);
+	}
+
+	return mlx5_set_port_qetcr_reg(mdev, in, sizeof(in));
+}
+EXPORT_SYMBOL_GPL(mlx5_modify_port_tc_rate_limit);
+
 int mlx5_modify_port_cong_params(struct mlx5_core_dev *mdev,
 				 void *in, int in_size)
 {

Modified: stable/11/sys/dev/mlx5/mlx5_en/en.h
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_en/en.h	Mon Mar 26 20:25:24 2018	(r331576)
+++ stable/11/sys/dev/mlx5/mlx5_en/en.h	Mon Mar 26 20:27:08 2018	(r331577)
@@ -70,6 +70,8 @@
 #include <dev/mlx5/mlx5_core/transobj.h>
 #include <dev/mlx5/mlx5_core/mlx5_core.h>
 
+#define	IEEE_8021QAZ_MAX_TCS	8
+
 #define	MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE                0x7
 #define	MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE                0xa
 #define	MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE                0xe
@@ -114,6 +116,9 @@
   (MLX5E_MAX_TX_HEADER - sizeof(struct mlx5e_tx_wqe) + \
   sizeof(((struct mlx5e_tx_wqe *)0)->eth.inline_hdr_start))	/* bytes */
 
+#define	MLX5E_100MB (100000)
+#define	MLX5E_1GB   (1000000)
+
 MALLOC_DECLARE(M_MLX5EN);
 
 struct mlx5_core_dev;
@@ -417,11 +422,13 @@ struct mlx5e_params {
   m(+1, u64 mc_local_lb, "mc_local_lb", "0: Local multicast loopback enabled 1: Disabled") \
   m(+1, u64 uc_local_lb, "uc_local_lb", "0: Local unicast loopback enabled 1: Disabled")
 
+
 #define	MLX5E_PARAMS_NUM (0 MLX5E_PARAMS(MLX5E_STATS_COUNT))
 
 struct mlx5e_params_ethtool {
 	u64	arg [0];
 	MLX5E_PARAMS(MLX5E_STATS_VAR)
+	u64	max_bw_value[IEEE_8021QAZ_MAX_TCS];
 };
 
 /* EEPROM Standards for plug in modules */
@@ -629,6 +636,12 @@ enum {
 	MLX5E_STATE_OPENED,
 };
 
+enum {
+	MLX5_BW_NO_LIMIT   = 0,
+	MLX5_100_MBPS_UNIT = 3,
+	MLX5_GBPS_UNIT     = 4,
+};
+
 struct mlx5e_vlan_db {
 	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
 	struct mlx5_flow_rule	*active_vlans_ft_rule[VLAN_N_VID];
@@ -731,6 +744,19 @@ struct mlx5e_eeprom {
 	int	page_valid;
 	u32	*data;
 };
+
+/*
+ * This structure contains rate limit extension to the IEEE 802.1Qaz ETS
+ * managed object.
+ * Values are 64 bits long and specified in Kbps to enable usage over both
+ * slow and very fast networks.
+ *
+ * @tc_maxrate: maximal tc tx bandwidth indexed by traffic class
+ */
+struct ieee_maxrate {
+	__u64	tc_maxrate[IEEE_8021QAZ_MAX_TCS];
+};
+
 
 #define	MLX5E_FLD_MAX(typ, fld) ((1ULL << __mlx5_bit_sz(typ, fld)) - 1ULL)
 

Modified: stable/11/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c	Mon Mar 26 20:25:24 2018	(r331576)
+++ stable/11/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c	Mon Mar 26 20:27:08 2018	(r331577)
@@ -84,6 +84,97 @@ mlx5e_ethtool_sync_tx_completion_fact(struct mlx5e_pri
 		priv->params_ethtool.tx_completion_fact = max;
 }
 
+static int
+mlx5e_getmaxrate(struct mlx5e_priv *priv)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u8 max_bw_unit[IEEE_8021QAZ_MAX_TCS];
+	u8 max_bw_value[IEEE_8021QAZ_MAX_TCS];
+	int err;
+	int i;
+
+	PRIV_LOCK(priv);
+	err = -mlx5_query_port_tc_rate_limit(mdev, max_bw_value, max_bw_unit);
+	if (err)
+		goto done;
+
+	for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+		switch (max_bw_unit[i]) {
+		case MLX5_100_MBPS_UNIT:
+			priv->params_ethtool.max_bw_value[i] = max_bw_value[i] * MLX5E_100MB;
+			break;
+		case MLX5_GBPS_UNIT:
+			priv->params_ethtool.max_bw_value[i] = max_bw_value[i] * MLX5E_1GB;
+			break;
+		case MLX5_BW_NO_LIMIT:
+			priv->params_ethtool.max_bw_value[i] = 0;
+			break;
+		default:
+			priv->params_ethtool.max_bw_value[i] = -1;
+			WARN_ONCE(true, "non-supported BW unit");
+			break;
+		}
+	}
+done:
+	PRIV_UNLOCK(priv);
+	return (err);
+}
+
+static int
+mlx5e_tc_maxrate_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct mlx5e_priv *priv = arg1;
+	int prio_index = arg2;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u8 max_bw_unit[IEEE_8021QAZ_MAX_TCS];
+	u8 max_bw_value[IEEE_8021QAZ_MAX_TCS];
+	int i, err;
+	u64 bw_val;
+	u64 result = priv->params_ethtool.max_bw_value[prio_index];
+	const u64 upper_limit_mbps = 255 * MLX5E_100MB;
+	const u64 upper_limit_gbps = 255 * MLX5E_1GB;
+
+	PRIV_LOCK(priv);
+	err = sysctl_handle_64(oidp, &result, 0, req);
+	if (err || !req->newptr ||
+	    result == priv->params_ethtool.max_bw_value[prio_index])
+		goto done;
+
+	if (result % MLX5E_100MB) {
+		err = ERANGE;
+		goto done;
+	}
+
+	memset(max_bw_value, 0, sizeof(max_bw_value));
+	memset(max_bw_unit, 0, sizeof(max_bw_unit));
+
+	for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+		bw_val = (i == prio_index) ? result : priv->params_ethtool.max_bw_value[i];
+
+		if (!bw_val) {
+			max_bw_unit[i] = MLX5_BW_NO_LIMIT;
+		} else if (bw_val > upper_limit_gbps) {
+			result = 0;
+			max_bw_unit[i] = MLX5_BW_NO_LIMIT;
+		} else if (bw_val <= upper_limit_mbps) {
+			max_bw_value[i] = howmany(bw_val, MLX5E_100MB);
+			max_bw_unit[i]  = MLX5_100_MBPS_UNIT;
+		} else {
+			max_bw_value[i] = howmany(bw_val, MLX5E_1GB);
+			max_bw_unit[i]  = MLX5_GBPS_UNIT;
+		}
+	}
+
+	err = -mlx5_modify_port_tc_rate_limit(mdev, max_bw_value, max_bw_unit);
+	if (err)
+		goto done;
+
+	priv->params_ethtool.max_bw_value[prio_index] = result;
+done:
+	PRIV_UNLOCK(priv);
+	return (err);
+}
+
 #define	MLX5_PARAM_OFFSET(n)				\
     __offsetof(struct mlx5e_priv, params_ethtool.n)
 
@@ -734,9 +825,11 @@ mlx5e_create_diagnostics(struct mlx5e_priv *priv)
 void
 mlx5e_create_ethtool(struct mlx5e_priv *priv)
 {
-	struct sysctl_oid *node;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct sysctl_oid *node, *qos_node;
 	const char *pnameunit;
 	unsigned x;
+	int i;
 
 	/* set some defaults */
 	priv->params_ethtool.tx_queue_size_max = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE;
@@ -830,4 +923,25 @@ mlx5e_create_ethtool(struct mlx5e_priv *priv)
 
 	/* Diagnostics support */
 	mlx5e_create_diagnostics(priv);
+
+	/* create qos node */
+	qos_node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
+	    SYSCTL_CHILDREN(node), OID_AUTO,
+	    "qos", CTLFLAG_RW, NULL, "Quality Of Service configuration");
+	if (node == NULL)
+		return;
+
+	/* Prioriry rate limit support */
+	if (mlx5e_getmaxrate(priv))
+		return;
+
+	for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+		char name[32];
+		snprintf(name, sizeof(name), "tc_%d_max_rate", i);
+		SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(qos_node),
+				OID_AUTO, name, CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
+				priv, i, mlx5e_tc_maxrate_handler, "QU",
+				"Max rate for priority, specified in kilobits, where kilo=1000, \
+				max_rate must be divisible by 100000");
+	}
 }

Modified: stable/11/sys/dev/mlx5/mlx5_ifc.h
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_ifc.h	Mon Mar 26 20:25:24 2018	(r331576)
+++ stable/11/sys/dev/mlx5/mlx5_ifc.h	Mon Mar 26 20:27:08 2018	(r331577)
@@ -9096,6 +9096,15 @@ struct mlx5_ifc_ets_global_config_reg_bits {
 	u8         max_bw_value[0x8];
 };
 
+struct mlx5_ifc_qetc_reg_bits {
+	u8                                         reserved_at_0[0x8];
+	u8                                         port_number[0x8];
+	u8                                         reserved_at_10[0x30];
+
+	struct mlx5_ifc_ets_tcn_config_reg_bits    tc_configuration[0x8];
+	struct mlx5_ifc_ets_global_config_reg_bits global_configuration;
+};
+
 struct mlx5_ifc_nodnic_mac_filters_bits {
 	struct mlx5_ifc_mac_address_layout_bits mac_filter0;
 

Modified: stable/11/sys/dev/mlx5/port.h
==============================================================================
--- stable/11/sys/dev/mlx5/port.h	Mon Mar 26 20:25:24 2018	(r331576)
+++ stable/11/sys/dev/mlx5/port.h	Mon Mar 26 20:27:08 2018	(r331577)
@@ -144,4 +144,11 @@ int mlx5_query_eeprom(struct mlx5_core_dev *dev, int i
 		      int device_addr, int size, int module_num, u32 *data,
 		      int *size_read);
 
+int mlx5_max_tc(struct mlx5_core_dev *mdev);
+int mlx5_query_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+				   u8 *max_bw_value,
+				   u8 *max_bw_units);
+int mlx5_modify_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+				   const u8 *max_bw_value,
+				   const u8 *max_bw_units);
 #endif /* __MLX5_PORT_H__ */



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201803262027.w2QKR8op016507>