Date: Tue, 18 Mar 2014 18:10:35 +0000 (UTC) From: Jim Harris <jimharris@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r263311 - head/sys/dev/nvme Message-ID: <201403181810.s2IIAZLF075078@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jimharris Date: Tue Mar 18 18:10:35 2014 New Revision: 263311 URL: http://svnweb.freebsd.org/changeset/base/263311 Log: nvme: Allocate all MSI resources up front so that we can fall back to INTx if necessary. Sponsored by: Intel MFC after: 3 days Modified: head/sys/dev/nvme/nvme_ctrlr.c head/sys/dev/nvme/nvme_private.h head/sys/dev/nvme/nvme_qpair.c Modified: head/sys/dev/nvme/nvme_ctrlr.c ============================================================================== --- head/sys/dev/nvme/nvme_ctrlr.c Tue Mar 18 18:09:08 2014 (r263310) +++ head/sys/dev/nvme/nvme_ctrlr.c Tue Mar 18 18:10:35 2014 (r263311) @@ -1075,8 +1075,8 @@ nvme_ctrlr_construct(struct nvme_control { union cap_lo_register cap_lo; union cap_hi_register cap_hi; - int num_vectors, per_cpu_io_queues, status = 0; - int timeout_period; + int i, num_vectors, per_cpu_io_queues, rid; + int status, timeout_period; ctrlr->dev = dev; @@ -1149,8 +1149,45 @@ nvme_ctrlr_construct(struct nvme_control goto intx; } - if (pci_alloc_msix(dev, &num_vectors) != 0) + if (pci_alloc_msix(dev, &num_vectors) != 0) { ctrlr->msix_enabled = 0; + goto intx; + } + + /* + * On earlier FreeBSD releases, there are reports that + * pci_alloc_msix() can return successfully with all vectors + * requested, but a subsequent bus_alloc_resource_any() + * for one of those vectors fails. This issue occurs more + * readily with multiple devices using per-CPU vectors. + * To workaround this issue, try to allocate the resources now, + * and fall back to INTx if we cannot allocate all of them. + * This issue cannot be reproduced on more recent versions of + * FreeBSD which have increased the maximum number of MSI-X + * vectors, but adding the workaround makes it easier for + * vendors wishing to import this driver into kernels based on + * older versions of FreeBSD. + */ + for (i = 0; i < num_vectors; i++) { + rid = i + 1; + ctrlr->msi_res[i] = bus_alloc_resource_any(ctrlr->dev, + SYS_RES_IRQ, &rid, RF_ACTIVE); + + if (ctrlr->msi_res[i] == NULL) { + ctrlr->msix_enabled = 0; + while (i > 0) { + i--; + bus_release_resource(ctrlr->dev, + SYS_RES_IRQ, + rman_get_rid(ctrlr->msi_res[i]), + ctrlr->msi_res[i]); + } + pci_release_msi(dev); + nvme_printf(ctrlr, "could not obtain all MSI-X " + "resources, reverting to intx\n"); + break; + } + } intx: Modified: head/sys/dev/nvme/nvme_private.h ============================================================================== --- head/sys/dev/nvme/nvme_private.h Tue Mar 18 18:09:08 2014 (r263310) +++ head/sys/dev/nvme/nvme_private.h Tue Mar 18 18:10:35 2014 (r263311) @@ -289,6 +289,8 @@ struct nvme_controller { struct task fail_req_task; struct taskqueue *taskqueue; + struct resource *msi_res[MAXCPU + 1]; + /* For shared legacy interrupt. */ int rid; struct resource *res; Modified: head/sys/dev/nvme/nvme_qpair.c ============================================================================== --- head/sys/dev/nvme/nvme_qpair.c Tue Mar 18 18:09:08 2014 (r263310) +++ head/sys/dev/nvme/nvme_qpair.c Tue Mar 18 18:10:35 2014 (r263311) @@ -1,5 +1,5 @@ /*- - * Copyright (C) 2012-2013 Intel Corporation + * Copyright (C) 2012-2014 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -487,9 +487,7 @@ nvme_qpair_construct(struct nvme_qpair * * the queue's vector to get the corresponding rid to use. */ qpair->rid = vector + 1; - - qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ, - &qpair->rid, RF_ACTIVE); + qpair->res = ctrlr->msi_res[vector]; bus_setup_intr(ctrlr->dev, qpair->res, INTR_TYPE_MISC | INTR_MPSAFE, NULL,
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201403181810.s2IIAZLF075078>