Date: Tue, 3 Apr 2018 09:28:11 +0000 (UTC) From: Hans Petter Selasky <hselasky@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org Subject: svn commit: r331915 - stable/11/sys/dev/mlx5/mlx5_core Message-ID: <201804030928.w339SBWm040616@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: hselasky Date: Tue Apr 3 09:28:11 2018 New Revision: 331915 URL: https://svnweb.freebsd.org/changeset/base/331915 Log: MFC r331822: Reorganize health recovery in mlx5core. - Move the semaphore locking and unlocking to the same function. - Flags are no longer needed if the reset and crdump will be done in the same function. Submitted by: slavash@ Sponsored by: Mellanox Technologies Modified: stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c Directory Properties: stable/11/ (props changed) Modified: stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c ============================================================================== --- stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c Tue Apr 3 09:27:05 2018 (r331914) +++ stable/11/sys/dev/mlx5/mlx5_core/mlx5_health.c Tue Apr 3 09:28:11 2018 (r331915) @@ -48,8 +48,6 @@ enum { enum { MLX5_DROP_NEW_HEALTH_WORK, MLX5_DROP_NEW_RECOVERY_WORK, - MLX5_SKIP_SW_RESET, - MLX5_SW_RESET_SEM_LOCKED, }; enum { @@ -167,7 +165,6 @@ static void reset_fw_if_needed(struct mlx5_core_dev *d { bool supported = (ioread32be(&dev->iseg->initializing) >> MLX5_FW_RESET_SUPPORTED_OFFSET) & 1; - struct mlx5_core_health *health = &dev->priv.health; u32 cmdq_addr, fatal_error; if (!supported) @@ -181,8 +178,7 @@ static void reset_fw_if_needed(struct mlx5_core_dev *d fatal_error = check_fatal_sensors(dev); if (fatal_error == MLX5_SENSOR_PCI_COMM_ERR || fatal_error == MLX5_SENSOR_NIC_DISABLED || - fatal_error == MLX5_SENSOR_NIC_SW_RESET || - test_bit(MLX5_SKIP_SW_RESET, &health->flags)) { + fatal_error == MLX5_SENSOR_NIC_SW_RESET) { mlx5_core_warn(dev, "Not issuing FW reset. Either it's already done or won't help.\n"); return; } @@ -197,26 +193,65 @@ static void reset_fw_if_needed(struct mlx5_core_dev *d &dev->iseg->cmdq_addr_l_sz); } +#define MLX5_CRDUMP_WAIT_MS 60000 +#define MLX5_FW_RESET_WAIT_MS 1000 +#define MLX5_NIC_STATE_POLL_MS 5 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) { + unsigned long end, delay_ms = MLX5_CRDUMP_WAIT_MS; + u32 fatal_error; + int lock = -EBUSY; + mutex_lock(&dev->intf_state_mutex); if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { goto unlock; return; } - if (!force) - mlx5_core_err(dev, "internal state error detected\n"); - if (check_fatal_sensors(dev) || force) { - reset_fw_if_needed(dev); + fatal_error = check_fatal_sensors(dev); + + if (fatal_error || force) { + if (!force) + mlx5_core_err(dev, "internal state error detected\n"); dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; mlx5_trigger_cmd_completions(dev); } - mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0); - if (!force) - mlx5_core_err(dev, "system error event triggered\n"); + if (force) + goto err_state_done; + if (fatal_error == MLX5_SENSOR_FW_SYND_RFR) { + if (mlx5_core_is_pf(dev)) + lock = lock_sem_sw_reset(dev, LOCK); + + if (lock != -EBUSY) { + reset_fw_if_needed(dev); + delay_ms = MLX5_FW_RESET_WAIT_MS; + } + } + + /* Recover from SW reset */ + end = jiffies + msecs_to_jiffies(delay_ms); + do { + if (sensor_nic_disabled(dev)) + break; + + msleep(MLX5_NIC_STATE_POLL_MS); + } while (!time_after(jiffies, end)); + + if (!sensor_nic_disabled(dev)) { + dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n", + get_nic_mode(dev), delay_ms); + } + + /* Release FW semaphore if you are the lock owner */ + if (!lock) + lock_sem_sw_reset(dev, UNLOCK); + + mlx5_core_err(dev, "system error event triggered\n"); + +err_state_done: + mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0); unlock: mutex_unlock(&dev->intf_state_mutex); } @@ -265,7 +300,6 @@ static void health_recover(struct work_struct *work) if (sensor_pci_no_comm(dev)) { dev_err(&dev->pdev->dev, "health recovery flow aborted, PCI reads still not working\n"); recover = false; - goto clear_sem; } nic_mode = get_nic_mode(dev); @@ -281,14 +315,6 @@ static void health_recover(struct work_struct *work) recover = false; } -clear_sem: - if (test_and_clear_bit(MLX5_SW_RESET_SEM_LOCKED, &health->flags)) { - mlx5_core_dbg(dev, "Unlocking FW reset semaphore\n"); - lock_sem_sw_reset(dev, UNLOCK); - } - - test_and_clear_bit(MLX5_SKIP_SW_RESET, &health->flags); - if (recover) { dev_err(&dev->pdev->dev, "starting health recovery flow\n"); mlx5_recover_device(dev); @@ -312,28 +338,10 @@ static void health_care(struct work_struct *work) struct mlx5_core_dev *dev; struct mlx5_priv *priv; unsigned long flags; - int ret; health = container_of(work, struct mlx5_core_health, work); priv = container_of(health, struct mlx5_priv, health); dev = container_of(priv, struct mlx5_core_dev, priv); - - if (mlx5_core_is_pf(dev)) { - ret = lock_sem_sw_reset(dev, LOCK); - if (!ret) { - mlx5_core_warn(dev, "Locked FW reset semaphore\n"); - set_bit(MLX5_SW_RESET_SEM_LOCKED, &health->flags); - } - else if (ret == -EBUSY) { - /* sw reset will be skipped only in case we detect the - * semaphore was already taken. In case of an error - * while taking the semaphore we prefer to issue a - * reset since longer cr-dump time and multiple resets - * are better than a stuck fw. - */ - set_bit(MLX5_SKIP_SW_RESET, &health->flags); - } - } mlx5_core_warn(dev, "handling bad device here\n"); mlx5_handle_bad_state(dev);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201804030928.w339SBWm040616>