Skip site navigation (1)Skip section navigation (2) 
Date:      Wed, 7 May 2014 16:48:44 +0000 (UTC)
From:      Jim Harris <jimharris@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-9@freebsd.org
Subject:   svn commit: r265566 - stable/9/sys/dev/nvme
Message-ID:  <201405071648.s47GmifO070542@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: jimharris
Date: Wed May  7 16:48:43 2014
New Revision: 265566
URL: http://svnweb.freebsd.org/changeset/base/265566

Log:
  MFC r263311:
  
  nvme: Allocate all MSI resources up front so that we can fall back to
  INTx if necessary.

Modified:
  stable/9/sys/dev/nvme/nvme_ctrlr.c
  stable/9/sys/dev/nvme/nvme_private.h
  stable/9/sys/dev/nvme/nvme_qpair.c
Directory Properties:
  stable/9/sys/   (props changed)
  stable/9/sys/dev/   (props changed)

Modified: stable/9/sys/dev/nvme/nvme_ctrlr.c
==============================================================================
--- stable/9/sys/dev/nvme/nvme_ctrlr.c	Wed May  7 16:47:58 2014	(r265565)
+++ stable/9/sys/dev/nvme/nvme_ctrlr.c	Wed May  7 16:48:43 2014	(r265566)
@@ -1075,8 +1075,8 @@ nvme_ctrlr_construct(struct nvme_control
 {
 	union cap_lo_register	cap_lo;
 	union cap_hi_register	cap_hi;
-	int			num_vectors, per_cpu_io_queues, status = 0;
-	int			timeout_period;
+	int			i, num_vectors, per_cpu_io_queues, rid;
+	int			status, timeout_period;
 
 	ctrlr->dev = dev;
 
@@ -1149,8 +1149,45 @@ nvme_ctrlr_construct(struct nvme_control
 		goto intx;
 	}
 
-	if (pci_alloc_msix(dev, &num_vectors) != 0)
+	if (pci_alloc_msix(dev, &num_vectors) != 0) {
 		ctrlr->msix_enabled = 0;
+		goto intx;
+	}
+
+	/*
+	 * On earlier FreeBSD releases, there are reports that
+	 *  pci_alloc_msix() can return successfully with all vectors
+	 *  requested, but a subsequent bus_alloc_resource_any()
+	 *  for one of those vectors fails.  This issue occurs more
+	 *  readily with multiple devices using per-CPU vectors.
+	 * To workaround this issue, try to allocate the resources now,
+	 *  and fall back to INTx if we cannot allocate all of them.
+	 *  This issue cannot be reproduced on more recent versions of
+	 *  FreeBSD which have increased the maximum number of MSI-X
+	 *  vectors, but adding the workaround makes it easier for
+	 *  vendors wishing to import this driver into kernels based on
+	 *  older versions of FreeBSD.
+	 */
+	for (i = 0; i < num_vectors; i++) {
+		rid = i + 1;
+		ctrlr->msi_res[i] = bus_alloc_resource_any(ctrlr->dev,
+		    SYS_RES_IRQ, &rid, RF_ACTIVE);
+
+		if (ctrlr->msi_res[i] == NULL) {
+			ctrlr->msix_enabled = 0;
+			while (i > 0) {
+				i--;
+				bus_release_resource(ctrlr->dev,
+				    SYS_RES_IRQ,
+				    rman_get_rid(ctrlr->msi_res[i]),
+				    ctrlr->msi_res[i]);
+			}
+			pci_release_msi(dev);
+			nvme_printf(ctrlr, "could not obtain all MSI-X "
+			    "resources, reverting to intx\n");
+			break;
+		}
+	}
 
 intx:
 

Modified: stable/9/sys/dev/nvme/nvme_private.h
==============================================================================
--- stable/9/sys/dev/nvme/nvme_private.h	Wed May  7 16:47:58 2014	(r265565)
+++ stable/9/sys/dev/nvme/nvme_private.h	Wed May  7 16:48:43 2014	(r265566)
@@ -289,6 +289,8 @@ struct nvme_controller {
 	struct task		fail_req_task;
 	struct taskqueue	*taskqueue;
 
+	struct resource		*msi_res[MAXCPU + 1];
+
 	/* For shared legacy interrupt. */
 	int			rid;
 	struct resource		*res;

Modified: stable/9/sys/dev/nvme/nvme_qpair.c
==============================================================================
--- stable/9/sys/dev/nvme/nvme_qpair.c	Wed May  7 16:47:58 2014	(r265565)
+++ stable/9/sys/dev/nvme/nvme_qpair.c	Wed May  7 16:48:43 2014	(r265566)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (C) 2012-2013 Intel Corporation
+ * Copyright (C) 2012-2014 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -487,9 +487,7 @@ nvme_qpair_construct(struct nvme_qpair *
 		 *  the queue's vector to get the corresponding rid to use.
 		 */
 		qpair->rid = vector + 1;
-
-		qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ,
-		    &qpair->rid, RF_ACTIVE);
+		qpair->res = ctrlr->msi_res[vector];
 
 		bus_setup_intr(ctrlr->dev, qpair->res,
 		    INTR_TYPE_MISC | INTR_MPSAFE, NULL,

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201405071648.s47GmifO070542>