Date: Wed, 4 Sep 2019 20:08:36 +0000 (UTC) From: Warner Losh <imp@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r351828 - head/sys/dev/nvme Message-ID: <201909042008.x84K8af9048476@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: imp Date: Wed Sep 4 20:08:36 2019 New Revision: 351828 URL: https://svnweb.freebsd.org/changeset/base/351828 Log: Support doorbell strides != 0. The NVMe standard (1.4) states >>> 8.6 Doorbell Stride for Software Emulation >>> The doorbell stride,...is useful in software emulation of an NVM >>> Express controller. ... For hardware implementations of the NVM >>> Express interface, the expected doorbell stride value is 0h. However, hardware in the wild exists with a doorbell stride of 1 (meaning 8 byte separation). This change supports that hardware, as well as software emulators as envisioned in Section 8.6. Since this is the fast path, care has been taken to make this computation efficient. The bit of math to compute an offset for each is replaced by a memory load from cache of a pre-computed value. MFC After: 3 days Reviewed by: scottl@ Differential Revision: https://reviews.freebsd.org/D21514 Modified: head/sys/dev/nvme/nvme_ctrlr.c head/sys/dev/nvme/nvme_private.h head/sys/dev/nvme/nvme_qpair.c Modified: head/sys/dev/nvme/nvme_ctrlr.c ============================================================================== --- head/sys/dev/nvme/nvme_ctrlr.c Wed Sep 4 19:32:50 2019 (r351827) +++ head/sys/dev/nvme/nvme_ctrlr.c Wed Sep 4 20:08:36 2019 (r351828) @@ -90,19 +90,25 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller struct nvme_qpair *qpair; uint32_t cap_lo; uint16_t mqes; - int i, error, num_entries, num_trackers; + int i, error, num_entries, num_trackers, max_entries; - num_entries = NVME_IO_ENTRIES; - TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries); - /* - * NVMe spec sets a hard limit of 64K max entries, but - * devices may specify a smaller limit, so we need to check - * the MQES field in the capabilities register. + * NVMe spec sets a hard limit of 64K max entries, but devices may + * specify a smaller limit, so we need to check the MQES field in the + * capabilities register. We have to cap the number of entries to the + * current stride allows for in BAR 0/1, otherwise the remainder entries + * are inaccessable. MQES should reflect this, and this is just a + * fail-safe. */ + max_entries = + (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) / + (1 << (ctrlr->dstrd + 1)); + num_entries = NVME_IO_ENTRIES; + TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries); cap_lo = nvme_mmio_read_4(ctrlr, cap_lo); mqes = NVME_CAP_LO_MQES(cap_lo); num_entries = min(num_entries, mqes + 1); + num_entries = min(num_entries, max_entries); num_trackers = NVME_IO_TRACKERS; TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers); @@ -110,9 +116,9 @@ nvme_ctrlr_construct_io_qpairs(struct nvme_controller num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS); num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS); /* - * No need to have more trackers than entries in the submit queue. - * Note also that for a queue size of N, we can only have (N-1) - * commands outstanding, hence the "-1" here. + * No need to have more trackers than entries in the submit queue. Note + * also that for a queue size of N, we can only have (N-1) commands + * outstanding, hence the "-1" here. */ num_trackers = min(num_trackers, (num_entries-1)); @@ -1120,7 +1126,6 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, de uint32_t cap_lo; uint32_t cap_hi; uint32_t to; - uint8_t dstrd; uint8_t mpsmin; int status, timeout_period; @@ -1128,14 +1133,8 @@ nvme_ctrlr_construct(struct nvme_controller *ctrlr, de mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF); - /* - * Software emulators may set the doorbell stride to something - * other than zero, but this driver is not set up to handle that. - */ cap_hi = nvme_mmio_read_4(ctrlr, cap_hi); - dstrd = NVME_CAP_HI_DSTRD(cap_hi); - if (dstrd != 0) - return (ENXIO); + ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2; mpsmin = NVME_CAP_HI_MPSMIN(cap_hi); ctrlr->min_page_size = 1 << (12 + mpsmin); Modified: head/sys/dev/nvme/nvme_private.h ============================================================================== --- head/sys/dev/nvme/nvme_private.h Wed Sep 4 19:32:50 2019 (r351827) +++ head/sys/dev/nvme/nvme_private.h Wed Sep 4 20:08:36 2019 (r351828) @@ -297,6 +297,9 @@ struct nvme_controller { /** timeout period in seconds */ uint32_t timeout_period; + /** doorbell stride */ + uint32_t dstrd; + struct nvme_qpair adminq; struct nvme_qpair *ioq; Modified: head/sys/dev/nvme/nvme_qpair.c ============================================================================== --- head/sys/dev/nvme/nvme_qpair.c Wed Sep 4 19:32:50 2019 (r351827) +++ head/sys/dev/nvme/nvme_qpair.c Wed Sep 4 20:08:36 2019 (r351828) @@ -622,8 +622,8 @@ nvme_qpair_process_completions(struct nvme_qpair *qpai qpair->phase = !qpair->phase; /* 3 */ } - nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl, - qpair->cq_head); + bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, + qpair->cq_hdbl_off, qpair->cq_head); } return (done != 0); } @@ -731,8 +731,15 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_ qpair->cpl_bus_addr = queuemem_phys + cmdsz; prpmem_phys = queuemem_phys + cmdsz + cplsz; - qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl); - qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl); + /* + * Calcuate the stride of the doorbell register. Many emulators set this + * value to correspond to a cache line. However, some hardware has set + * it to various small values. + */ + qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[0]) + + (id << (ctrlr->dstrd + 1)); + qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[0]) + + (id << (ctrlr->dstrd + 1)) + (1 << ctrlr->dstrd); TAILQ_INIT(&qpair->free_tr); TAILQ_INIT(&qpair->outstanding_tr); @@ -950,9 +957,8 @@ nvme_qpair_submit_tracker(struct nvme_qpair *qpair, st wmb(); #endif - nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl, - qpair->sq_tail); - + bus_space_write_4(qpair->ctrlr->bus_tag, qpair->ctrlr->bus_handle, + qpair->sq_tdbl_off, qpair->sq_tail); qpair->num_cmds++; }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201909042008.x84K8af9048476>