Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 16 May 2019 17:28:30 +0000 (UTC)
From:      Hans Petter Selasky <hselasky@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org
Subject:   svn commit: r347818 - in stable/11/sys/dev/mlx5: . mlx5_core
Message-ID:  <201905161728.x4GHSUMN005218@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: hselasky
Date: Thu May 16 17:28:30 2019
New Revision: 347818
URL: https://svnweb.freebsd.org/changeset/base/347818

Log:
  MFC r347268:
  Add Fast teardown support to mlx5core.
  
  Today mlx5 devices support two teardown modes:
  1- Regular teardown
  2- Force teardown
  
  This change introduces the enhanced version of the "Force teardown" that
  allows SW to perform teardown in a faster way without the need to reclaim
  all the pages.
  
  Fast teardown provides the following advantages:
  1- Fix a FW race condition that could cause command timeout
  2- Avoid moving to polling mode
  3- Close the vport to prevent PCI ACK to be sent without been
     scattered to memory
  
  Linux commit:
  fcd29ad17c6ff885dfae58f557e9323941e63ba2
  
  Sponsored by:	Mellanox Technologies

Modified:
  stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h
  stable/11/sys/dev/mlx5/mlx5_core/mlx5_fw.c
  stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c
  stable/11/sys/dev/mlx5/mlx5_ifc.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h	Thu May 16 17:25:41 2019	(r347817)
+++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_core.h	Thu May 16 17:28:30 2019	(r347818)
@@ -79,6 +79,7 @@ int mlx5_query_qcam_reg(struct mlx5_core_dev *mdev, u3
 int mlx5_cmd_init_hca(struct mlx5_core_dev *dev);
 int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev);
 int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev);
+int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev);
 void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
 		     unsigned long param);
 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force);

Modified: stable/11/sys/dev/mlx5/mlx5_core/mlx5_fw.c
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_core/mlx5_fw.c	Thu May 16 17:25:41 2019	(r347817)
+++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_fw.c	Thu May 16 17:28:30 2019	(r347818)
@@ -249,12 +249,59 @@ int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *
 	if (ret)
 		return ret;
 
-	force_state = MLX5_GET(teardown_hca_out, out, force_state);
+	force_state = MLX5_GET(teardown_hca_out, out, state);
 	if (force_state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL)  {
 		mlx5_core_err(dev, "teardown with force mode failed\n");
 		return -EIO;
 	}
 
+	return 0;
+}
+
+#define	MLX5_FAST_TEARDOWN_WAIT_MS 3000
+int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev)
+{
+	int end, delay_ms = MLX5_FAST_TEARDOWN_WAIT_MS;
+	u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {};
+	u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {};
+	int state;
+	int ret;
+
+	if (!MLX5_CAP_GEN(dev, fast_teardown)) {
+		mlx5_core_dbg(dev, "fast teardown is not supported in the firmware\n");
+		return -EOPNOTSUPP;
+	}
+
+	MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA);
+	MLX5_SET(teardown_hca_in, in, profile,
+		 MLX5_TEARDOWN_HCA_IN_PROFILE_PREPARE_FAST_TEARDOWN);
+
+	ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+	if (ret)
+		return ret;
+
+	state = MLX5_GET(teardown_hca_out, out, state);
+	if (state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) {
+		mlx5_core_warn(dev, "teardown with fast mode failed\n");
+		return -EIO;
+	}
+
+	mlx5_set_nic_state(dev, MLX5_NIC_IFC_DISABLED);
+
+	/* Loop until device state turns to disable */
+	end = jiffies + msecs_to_jiffies(delay_ms);
+	do {
+		if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
+			break;
+
+		pause("W", 1);
+	} while (!time_after(jiffies, end));
+
+	if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) {
+		dev_err(&dev->pdev->dev, "NIC IFC still %d after %ums.\n",
+			mlx5_get_nic_state(dev), delay_ms);
+		return -EIO;
+	}
 	return 0;
 }
 

Modified: stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c	Thu May 16 17:25:41 2019	(r347817)
+++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c	Thu May 16 17:28:30 2019	(r347818)
@@ -1397,12 +1397,17 @@ static const struct pci_error_handlers mlx5_err_handle
 
 static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)
 {
+	bool fast_teardown, force_teardown;
 	int err;
 
-	if (!MLX5_CAP_GEN(dev, force_teardown)) {
-		mlx5_core_dbg(dev, "force teardown is not supported in the firmware\n");
+	fast_teardown = MLX5_CAP_GEN(dev, fast_teardown);
+	force_teardown = MLX5_CAP_GEN(dev, force_teardown);
+
+	mlx5_core_dbg(dev, "force teardown firmware support=%d\n", force_teardown);
+	mlx5_core_dbg(dev, "fast teardown firmware support=%d\n", fast_teardown);
+
+	if (!fast_teardown && !force_teardown)
 		return -EOPNOTSUPP;
-	}
 
 	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
 		mlx5_core_dbg(dev, "Device in internal error state, giving up\n");
@@ -1415,14 +1420,19 @@ static int mlx5_try_fast_unload(struct mlx5_core_dev *
 	mlx5_drain_health_wq(dev);
 	mlx5_stop_health_poll(dev, false);
 
+	err = mlx5_cmd_fast_teardown_hca(dev);
+	if (!err)
+		goto done;
+
 	err = mlx5_cmd_force_teardown_hca(dev);
-	if (err) {
-		mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", err);
-		return err;
-	}
+	if (!err)
+		goto done;
 
+	mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", err);
+	mlx5_start_health_poll(dev);
+	return err;
+done:
 	mlx5_enter_error_state(dev, true);
-
 	return 0;
 }
 

Modified: stable/11/sys/dev/mlx5/mlx5_ifc.h
==============================================================================
--- stable/11/sys/dev/mlx5/mlx5_ifc.h	Thu May 16 17:25:41 2019	(r347817)
+++ stable/11/sys/dev/mlx5/mlx5_ifc.h	Thu May 16 17:28:30 2019	(r347818)
@@ -1054,7 +1054,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         relaxed_ordering_write[1];
 	u8         reserved_6[0x1];
 	u8         log_max_mkey[0x6];
-	u8         reserved_7[0xc];
+	u8         reserved_7[0xb];
+	u8         fast_teardown[0x1];
 	u8         log_max_eq[0x4];
 
 	u8         max_indirection[0x8];
@@ -3289,12 +3290,13 @@ struct mlx5_ifc_teardown_hca_out_bits {
 
 	u8         reserved_1[0x3f];
 
-	u8	   force_state[0x1];
+	u8	   state[0x1];
 };
 
 enum {
 	MLX5_TEARDOWN_HCA_IN_PROFILE_GRACEFUL_CLOSE  = 0x0,
 	MLX5_TEARDOWN_HCA_IN_PROFILE_FORCE_CLOSE     = 0x1,
+	MLX5_TEARDOWN_HCA_IN_PROFILE_PREPARE_FAST_TEARDOWN = 0x2,
 };
 
 struct mlx5_ifc_teardown_hca_in_bits {



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201905161728.x4GHSUMN005218>