Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 10 Feb 2017 15:28:19 +0000 (UTC)
From:      Hans Petter Selasky <hselasky@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r313556 - in head/sys/dev/mlx4: . mlx4_core
Message-ID:  <201702101528.v1AFSJUm022561@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: hselasky
Date: Fri Feb 10 15:28:18 2017
New Revision: 313556
URL: https://svnweb.freebsd.org/changeset/base/313556

Log:
  Change mlx4 QP allocation scheme.
  
  When using Blue-Flame, BF, the QPN overrides the VLAN, CV, and SV
  fields in the WQE. Thus, BF may only be used for QPNs with bits 6,7
  unset.
  
  The current ethernet driver code reserves a TX QP range with 256b
  alignment.
  
  This is wrong because if there are more than 64 TX QPs in use, QPNs >=
  base + 65 will have bits 6/7 set.
  
  This problem is not specific for the Ethernet driver, any entity that
  tries to reserve more than 64 BF-enabled QPs should fail. Also, using
  ranges is not necessary here and is wasteful.
  
  The new mechanism introduced here will support reservation for "Eth
  QPs eligible for BF" for all drivers: bare-metal, multi-PF, and VFs
  (when hypervisors support WC in VMs). The flow we use is:
  
  1. In mlx4_en, allocate Tx QPs one by one instead of a range allocation,
     and request "BF enabled QPs" if BF is supported for the function
  
  2. In the ALLOC_RES FW command, change param1 to:
  a. param1[23:0]  - number of QPs
  b. param1[31-24] - flags controlling QPs reservation
  
  Bit 31 refers to Eth blueflame supported QPs. Those QPs must have bits
  6 and 7 unset in order to be used in Ethernet.
  
  Bits 24-30 of the flags are currently reserved.
  
  When a function tries to allocate a QP, it states the required
  attributes for this QP. Those attributes are considered "best-effort".
  If an attribute, such as Ethernet BF enabled QP, is a must-have
  attribute, the function has to check that attribute is supported
  before trying to do the allocation.
  
  In a lower layer of the code, mlx4_qp_reserve_range masks out the bits
  which are unsupported. If SRIOV is used, the PF validates those
  attributes and masks out unsupported attributes as well. In order to
  notify VFs which attributes are supported, the VF uses QUERY_FUNC_CAP
  command. This command's mailbox is filled by the PF, which notifies
  which QP allocation attributes it supports.
  
  Obtained from:		Linux (dual BSD/GPLv2 licensed)
  Submitted by:		Dexuan Cui @ microsoft . com
  Differential Revision:	https://reviews.freebsd.org/D8868
  MFC after:		2 weeks
  Sponsored by:		Mellanox Technologies

Modified:
  head/sys/dev/mlx4/device.h
  head/sys/dev/mlx4/mlx4_core/fw.h
  head/sys/dev/mlx4/mlx4_core/mlx4_fw.c
  head/sys/dev/mlx4/mlx4_core/mlx4_main.c
  head/sys/dev/mlx4/mlx4_core/mlx4_qp.c
  head/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c

Modified: head/sys/dev/mlx4/device.h
==============================================================================
--- head/sys/dev/mlx4/device.h	Fri Feb 10 15:22:21 2017	(r313555)
+++ head/sys/dev/mlx4/device.h	Fri Feb 10 15:28:18 2017	(r313556)
@@ -219,6 +219,23 @@ enum {
 };
 
 enum {
+	MLX4_QUERY_FUNC_FLAGS_BF_RES_QP	= 1LL << 0
+};
+
+/* bit enums for an 8-bit flags field indicating special use
+ * QPs which require special handling in qp_reserve_range.
+ * Currently, this only includes QPs used by the ETH interface,
+ * where we expect to use blueflame.  These QPs must not have
+ * bits 6 and 7 set in their qp number.
+ *
+ * This enum may use only bits 0..7.
+ */
+enum {
+	MLX4_RESERVE_ETH_BF_QP		= 1 << 7,
+};
+
+
+enum {
 	MLX4_DEV_CAP_64B_EQE_ENABLED	= 1LL << 0,
 	MLX4_DEV_CAP_64B_CQE_ENABLED	= 1LL << 1
 };
@@ -533,6 +550,7 @@ struct mlx4_caps {
 	u32			max_basic_counters;
 	u32			max_extended_counters;
 	u8			def_counter_index[MLX4_MAX_PORTS + 1];
+	u8			alloc_res_qp_mask;
 };
 
 struct mlx4_buf_list {

Modified: head/sys/dev/mlx4/mlx4_core/fw.h
==============================================================================
--- head/sys/dev/mlx4/mlx4_core/fw.h	Fri Feb 10 15:22:21 2017	(r313555)
+++ head/sys/dev/mlx4/mlx4_core/fw.h	Fri Feb 10 15:28:18 2017	(r313556)
@@ -145,6 +145,7 @@ struct mlx4_func_cap {
 	u8	physical_port;
 	u8	port_flags;
 	u8	def_counter_index;
+	u8	extra_flags;
 };
 
 struct mlx4_func {

Modified: head/sys/dev/mlx4/mlx4_core/mlx4_fw.c
==============================================================================
--- head/sys/dev/mlx4/mlx4_core/mlx4_fw.c	Fri Feb 10 15:22:21 2017	(r313555)
+++ head/sys/dev/mlx4/mlx4_core/mlx4_fw.c	Fri Feb 10 15:28:18 2017	(r313556)
@@ -265,10 +265,15 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct m
 #define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET		0x64
 #define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET		0x68
 
+#define QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET	0x6c
+
 #define QUERY_FUNC_CAP_FMR_FLAG			0x80
 #define QUERY_FUNC_CAP_FLAG_RDMA		0x40
 #define QUERY_FUNC_CAP_FLAG_ETH			0x80
 #define QUERY_FUNC_CAP_FLAG_QUOTAS		0x10
+#define QUERY_FUNC_CAP_FLAG_VALID_MAILBOX	0x04
+
+#define QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG	(1UL << 31)
 
 /* when opcode modifier = 1 */
 #define QUERY_FUNC_CAP_PHYS_PORT_OFFSET		0x3
@@ -322,7 +327,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct m
 	} else if (vhcr->op_modifier == 0) {
 		/* enable rdma and ethernet interfaces, and new quota locations */
 		field = (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA |
-			 QUERY_FUNC_CAP_FLAG_QUOTAS);
+			 QUERY_FUNC_CAP_FLAG_QUOTAS | QUERY_FUNC_CAP_FLAG_VALID_MAILBOX);
 		MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS_OFFSET);
 
 		field = dev->caps.num_ports;
@@ -382,6 +387,8 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct m
 		MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET);
 		MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP);
 
+		size = QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG;
+		MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET);
 	} else
 		err = -EINVAL;
 
@@ -474,6 +481,17 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev 
 		MLX4_GET(size, outbox, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET);
 		func_cap->reserved_eq = size & 0xFFFFFF;
 
+		func_cap->extra_flags = 0;
+
+		/* Mailbox data from 0x6c and onward should only be treated if
+		 * QUERY_FUNC_CAP_FLAG_VALID_MAILBOX is set in func_cap->flags
+		 */
+		if (func_cap->flags & QUERY_FUNC_CAP_FLAG_VALID_MAILBOX) {
+			MLX4_GET(size, outbox, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET);
+			if (size & QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG)
+				func_cap->extra_flags |= MLX4_QUERY_FUNC_FLAGS_BF_RES_QP;
+		}
+
 		goto out;
 	}
 

Modified: head/sys/dev/mlx4/mlx4_core/mlx4_main.c
==============================================================================
--- head/sys/dev/mlx4/mlx4_core/mlx4_main.c	Fri Feb 10 15:22:21 2017	(r313555)
+++ head/sys/dev/mlx4/mlx4_core/mlx4_main.c	Fri Feb 10 15:28:18 2017	(r313556)
@@ -849,6 +849,11 @@ static int mlx4_dev_cap(struct mlx4_dev 
 	if (!mlx4_is_slave(dev)) {
 		for (i = 0; i < dev->caps.num_ports; ++i)
 			dev->caps.def_counter_index[i] = i << 1;
+
+		dev->caps.alloc_res_qp_mask =
+			(dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0);
+	} else {
+		dev->caps.alloc_res_qp_mask = 0;
 	}
 
 	return 0;
@@ -1114,6 +1119,10 @@ static int mlx4_slave_cap(struct mlx4_de
 
 	slave_adjust_steering_mode(dev, &dev_cap, &hca_param);
 
+	if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_BF_RES_QP &&
+	    dev->caps.bf_reg_size)
+		dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP;
+
 	return 0;
 
 err_mem:

Modified: head/sys/dev/mlx4/mlx4_core/mlx4_qp.c
==============================================================================
--- head/sys/dev/mlx4/mlx4_core/mlx4_qp.c	Fri Feb 10 15:22:21 2017	(r313555)
+++ head/sys/dev/mlx4/mlx4_core/mlx4_qp.c	Fri Feb 10 15:28:18 2017	(r313556)
@@ -242,6 +242,9 @@ int mlx4_qp_reserve_range(struct mlx4_de
 	u64 out_param;
 	int err;
 
+	/* Turn off all unsupported QP allocation flags */
+	flags &= dev->caps.alloc_res_qp_mask;
+
 	if (mlx4_is_mfunc(dev)) {
 		set_param_l(&in_param, (((u32) flags) << 24) | (u32) cnt);
 		set_param_h(&in_param, align);

Modified: head/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c
==============================================================================
--- head/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c	Fri Feb 10 15:22:21 2017	(r313555)
+++ head/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c	Fri Feb 10 15:28:18 2017	(r313556)
@@ -1544,7 +1544,10 @@ static int qp_alloc_res(struct mlx4_dev 
 	switch (op) {
 	case RES_OP_RESERVE:
 		count = get_param_l(&in_param) & 0xffffff;
-		flags = get_param_l(&in_param) >> 24;
+		/* Turn off all unsupported QP allocation flags that the
+		 * slave tries to set.
+		 */
+		flags = (get_param_l(&in_param) >> 24) & dev->caps.alloc_res_qp_mask;
 		align = get_param_h(&in_param);
 		err = mlx4_grant_resource(dev, slave, RES_QP, count, 0);
 		if (err)



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201702101528.v1AFSJUm022561>