Date: Fri, 23 Mar 2018 17:33:14 +0000 (UTC) From: Hans Petter Selasky <hselasky@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r331443 - in head/sys/dev/mlx5: . mlx5_core Message-ID: <201803231733.w2NHXEo9035304@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: hselasky Date: Fri Mar 23 17:33:14 2018 New Revision: 331443 URL: https://svnweb.freebsd.org/changeset/base/331443 Log: Improve support for health recovery in mlx5core. This patch accumulates the following Linux commits: - 04c0c1ab38e95105d950db5b84e727637e149ce7 net/mlx5: PCI error recovery health care simulation - 0179720d6be2096b8d0a4d143254ff9e77747daa net/mlx5: Introduce trigger_health_work function - 3fece5d676939f42f434c63dfe1bd42d7d94e6f0 net/mlx5: Continue health polling until it is explicitly stopped Submitted by: Matthew Finlay <matt@mellanox.com> MFC after: 1 week Sponsored by: Mellanox Technologies Modified: head/sys/dev/mlx5/driver.h head/sys/dev/mlx5/mlx5_core/mlx5_core.h head/sys/dev/mlx5/mlx5_core/mlx5_health.c head/sys/dev/mlx5/mlx5_core/mlx5_main.c Modified: head/sys/dev/mlx5/driver.h ============================================================================== --- head/sys/dev/mlx5/driver.h Fri Mar 23 17:28:04 2018 (r331442) +++ head/sys/dev/mlx5/driver.h Fri Mar 23 17:33:14 2018 (r331443) @@ -498,6 +498,7 @@ struct mlx5_core_health { struct workqueue_struct *wq; unsigned long flags; struct work_struct work; + struct delayed_work recover_work; }; #define MLX5_CQ_LINEAR_ARRAY_SIZE 1024 @@ -887,6 +888,7 @@ int mlx5_health_init(struct mlx5_core_dev *dev); void mlx5_start_health_poll(struct mlx5_core_dev *dev); void mlx5_stop_health_poll(struct mlx5_core_dev *dev); void mlx5_drain_health_wq(struct mlx5_core_dev *dev); +void mlx5_trigger_health_work(struct mlx5_core_dev *dev); #define mlx5_buf_alloc_node(dev, size, direct, buf, node) \ mlx5_buf_alloc(dev, size, direct, buf) Modified: head/sys/dev/mlx5/mlx5_core/mlx5_core.h ============================================================================== --- head/sys/dev/mlx5/mlx5_core/mlx5_core.h Fri Mar 23 17:28:04 2018 (r331442) +++ head/sys/dev/mlx5/mlx5_core/mlx5_core.h Fri Mar 23 17:33:14 2018 (r331443) @@ -74,6 +74,7 @@ void mlx5_core_event(struct mlx5_core_dev *dev, enum m unsigned long param); void mlx5_enter_error_state(struct mlx5_core_dev *dev); void mlx5_disable_device(struct mlx5_core_dev *dev); +void mlx5_recover_device(struct mlx5_core_dev *dev); void mlx5e_init(void); void mlx5e_cleanup(void); Modified: head/sys/dev/mlx5/mlx5_core/mlx5_health.c ============================================================================== --- head/sys/dev/mlx5/mlx5_core/mlx5_health.c Fri Mar 23 17:28:04 2018 (r331442) +++ head/sys/dev/mlx5/mlx5_core/mlx5_health.c Fri Mar 23 17:33:14 2018 (r331443) @@ -40,14 +40,15 @@ enum { MLX5_NIC_IFC_FULL = 0, MLX5_NIC_IFC_DISABLED = 1, - MLX5_NIC_IFC_NO_DRAM_NIC = 2 + MLX5_NIC_IFC_NO_DRAM_NIC = 2, + MLX5_NIC_IFC_INVALID = 3, }; enum { MLX5_DROP_NEW_HEALTH_WORK, }; -static u8 get_nic_interface(struct mlx5_core_dev *dev) +static u8 get_nic_state(struct mlx5_core_dev *dev) { return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3; } @@ -80,7 +81,7 @@ static int in_fatal(struct mlx5_core_dev *dev) struct mlx5_core_health *health = &dev->priv.health; struct mlx5_health_buffer __iomem *h = health->health; - if (get_nic_interface(dev) == MLX5_NIC_IFC_DISABLED) + if (get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) return 1; if (ioread32be(&h->fw_ver) == 0xffffffff) @@ -112,9 +113,9 @@ unlock: static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) { - u8 nic_interface = get_nic_interface(dev); + u8 nic_state = get_nic_state(dev); - switch (nic_interface) { + switch (nic_state) { case MLX5_NIC_IFC_FULL: mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n"); break; @@ -128,23 +129,58 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev break; default: mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n", - nic_interface); + nic_state); } mlx5_disable_device(dev); } +static void health_recover(struct work_struct *work) +{ + struct mlx5_core_health *health; + struct delayed_work *dwork; + struct mlx5_core_dev *dev; + struct mlx5_priv *priv; + u8 nic_state; + + dwork = container_of(work, struct delayed_work, work); + health = container_of(dwork, struct mlx5_core_health, recover_work); + priv = container_of(health, struct mlx5_priv, health); + dev = container_of(priv, struct mlx5_core_dev, priv); + + nic_state = get_nic_state(dev); + if (nic_state == MLX5_NIC_IFC_INVALID) { + dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n"); + return; + } + + dev_err(&dev->pdev->dev, "starting health recovery flow\n"); + mlx5_recover_device(dev); +} + +/* How much time to wait until health resetting the driver (in msecs) */ +#define MLX5_RECOVERY_DELAY_MSECS 60000 static void health_care(struct work_struct *work) { + unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS); struct mlx5_core_health *health; struct mlx5_core_dev *dev; struct mlx5_priv *priv; + unsigned long flags; health = container_of(work, struct mlx5_core_health, work); priv = container_of(health, struct mlx5_priv, health); dev = container_of(priv, struct mlx5_core_dev, priv); mlx5_core_warn(dev, "handling bad device here\n"); mlx5_handle_bad_state(dev); + + spin_lock_irqsave(&health->wq_lock, flags); + if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) + schedule_delayed_work(&health->recover_work, recover_delay); + else + dev_err(&dev->pdev->dev, + "new health works are not permitted at this stage\n"); + spin_unlock_irqrestore(&health->wq_lock, flags); } static int get_next_poll_jiffies(void) @@ -158,6 +194,20 @@ static int get_next_poll_jiffies(void) return next; } +void mlx5_trigger_health_work(struct mlx5_core_dev *dev) +{ + struct mlx5_core_health *health = &dev->priv.health; + unsigned long flags; + + spin_lock_irqsave(&health->wq_lock, flags); + if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) + queue_work(health->wq, &health->work); + else + dev_err(&dev->pdev->dev, + "new health works are not permitted at this stage\n"); + spin_unlock_irqrestore(&health->wq_lock, flags); +} + static const char *hsynd_str(u8 synd) { switch (synd) { @@ -224,10 +274,8 @@ static void poll_health(unsigned long data) if (dev->state != MLX5_DEVICE_STATE_UP) return; - if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { - mod_timer(&health->timer, get_next_poll_jiffies()); - return; - } + if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) + goto out; count = ioread32be(health->health_counter); if (count == health->prev) @@ -239,21 +287,16 @@ static void poll_health(unsigned long data) if (health->miss_counter == MAX_MISSES) { mlx5_core_err(dev, "device's health compromised - reached miss count\n"); print_health_info(dev); - } else { - mod_timer(&health->timer, get_next_poll_jiffies()); } if (in_fatal(dev) && !health->sick) { health->sick = true; print_health_info(dev); - spin_lock(&health->wq_lock); - if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) - queue_work(health->wq, &health->work); - else - dev_err(&dev->pdev->dev, - "new health works are not permitted at this stage\n"); - spin_unlock(&health->wq_lock); + mlx5_trigger_health_work(dev); } + +out: + mod_timer(&health->timer, get_next_poll_jiffies()); } void mlx5_start_health_poll(struct mlx5_core_dev *dev) @@ -281,10 +324,12 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev) void mlx5_drain_health_wq(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; + unsigned long flags; - spin_lock(&health->wq_lock); + spin_lock_irqsave(&health->wq_lock, flags); set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); - spin_unlock(&health->wq_lock); + spin_unlock_irqrestore(&health->wq_lock, flags); + cancel_delayed_work_sync(&health->recover_work); cancel_work_sync(&health->work); } @@ -316,6 +361,7 @@ int mlx5_health_init(struct mlx5_core_dev *dev) spin_lock_init(&health->wq_lock); INIT_WORK(&health->work, health_care); + INIT_DELAYED_WORK(&health->recover_work, health_recover); return 0; } Modified: head/sys/dev/mlx5/mlx5_core/mlx5_main.c ============================================================================== --- head/sys/dev/mlx5/mlx5_core/mlx5_main.c Fri Mar 23 17:28:04 2018 (r331442) +++ head/sys/dev/mlx5/mlx5_core/mlx5_main.c Fri Mar 23 17:33:14 2018 (r331443) @@ -1257,11 +1257,6 @@ static pci_ers_result_t mlx5_pci_slot_reset(struct pci return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; } -void mlx5_disable_device(struct mlx5_core_dev *dev) -{ - mlx5_pci_err_detected(dev->pdev, 0); -} - /* wait for the device to show vital signs. For now we check * that we can read the device ID and that the health buffer * shows a non zero value which is different than 0xffffffff @@ -1376,6 +1371,18 @@ static const struct pci_device_id mlx5_core_pci_table[ }; MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); + +void mlx5_disable_device(struct mlx5_core_dev *dev) +{ + mlx5_pci_err_detected(dev->pdev, 0); +} + +void mlx5_recover_device(struct mlx5_core_dev *dev) +{ + mlx5_pci_disable_device(dev); + if (mlx5_pci_slot_reset(dev->pdev) == PCI_ERS_RESULT_RECOVERED) + mlx5_pci_resume(dev->pdev); +} struct pci_driver mlx5_core_driver = { .name = DRIVER_NAME,
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201803231733.w2NHXEo9035304>