Date: Thu, 18 Oct 2012 00:44:40 +0000 (UTC) From: Jim Harris <jimharris@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r241664 - head/sys/dev/nvme Message-ID: <201210180044.q9I0ieh7002017@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jimharris Date: Thu Oct 18 00:44:39 2012 New Revision: 241664 URL: http://svn.freebsd.org/changeset/base/241664 Log: Preallocate a limited number of nvme_tracker objects per qpair, rather than dynamically creating them at runtime. Sponsored by: Intel Modified: head/sys/dev/nvme/nvme_ctrlr.c head/sys/dev/nvme/nvme_private.h head/sys/dev/nvme/nvme_qpair.c head/sys/dev/nvme/nvme_sysctl.c Modified: head/sys/dev/nvme/nvme_ctrlr.c ============================================================================== --- head/sys/dev/nvme/nvme_ctrlr.c Thu Oct 18 00:43:25 2012 (r241663) +++ head/sys/dev/nvme/nvme_ctrlr.c Thu Oct 18 00:44:39 2012 (r241664) @@ -217,7 +217,13 @@ nvme_ctrlr_construct_admin_qpair(struct * The admin queue's max xfer size is treated differently than the * max I/O xfer size. 16KB is sufficient here - maybe even less? */ - nvme_qpair_construct(qpair, 0, 0, num_entries, 16*1024, ctrlr); + nvme_qpair_construct(qpair, + 0, /* qpair ID */ + 0, /* vector */ + num_entries, + NVME_ADMIN_TRACKERS, + 16*1024, /* max xfer size */ + ctrlr); } static int @@ -225,13 +231,11 @@ nvme_ctrlr_construct_io_qpairs(struct nv { struct nvme_qpair *qpair; union cap_lo_register cap_lo; - int i, num_entries; + int i, num_entries, num_trackers; num_entries = NVME_IO_ENTRIES; TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries); - num_entries = max(num_entries, NVME_MIN_IO_ENTRIES); - /* * NVMe spec sets a hard limit of 64K max entries, but * devices may specify a smaller limit, so we need to check @@ -240,6 +244,18 @@ nvme_ctrlr_construct_io_qpairs(struct nv cap_lo.raw = nvme_mmio_read_4(ctrlr, cap_lo); num_entries = min(num_entries, cap_lo.bits.mqes+1); + num_trackers = NVME_IO_TRACKERS; + TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers); + + num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS); + num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS); + /* + * No need to have more trackers than entries in the submit queue. + * Note also that for a queue size of N, we can only have (N-1) + * commands outstanding, hence the "-1" here. + */ + num_trackers = min(num_trackers, (num_entries-1)); + ctrlr->max_xfer_size = NVME_MAX_XFER_SIZE; TUNABLE_INT_FETCH("hw.nvme.max_xfer_size", &ctrlr->max_xfer_size); /* @@ -270,6 +286,7 @@ nvme_ctrlr_construct_io_qpairs(struct nv i+1, /* qpair ID */ ctrlr->msix_enabled ? i+1 : 0, /* vector */ num_entries, + num_trackers, ctrlr->max_xfer_size, ctrlr); Modified: head/sys/dev/nvme/nvme_private.h ============================================================================== --- head/sys/dev/nvme/nvme_private.h Thu Oct 18 00:43:25 2012 (r241663) +++ head/sys/dev/nvme/nvme_private.h Thu Oct 18 00:44:39 2012 (r241664) @@ -68,14 +68,25 @@ MALLOC_DECLARE(M_NVME); */ #define NVME_MAX_XFER_SIZE NVME_MAX_PRP_LIST_ENTRIES * PAGE_SIZE +#define NVME_ADMIN_TRACKERS (16) #define NVME_ADMIN_ENTRIES (128) /* min and max are defined in admin queue attributes section of spec */ #define NVME_MIN_ADMIN_ENTRIES (2) #define NVME_MAX_ADMIN_ENTRIES (4096) -#define NVME_IO_ENTRIES (1024) -/* min is a reasonable value picked for the nvme(4) driver */ -#define NVME_MIN_IO_ENTRIES (128) +/* + * NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion + * queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we + * will allow outstanding on an I/O qpair at any time. The only advantage in + * having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping + * the contents of the submission and completion queues, it will show a longer + * history of data. + */ +#define NVME_IO_ENTRIES (256) +#define NVME_IO_TRACKERS (128) +#define NVME_MIN_IO_TRACKERS (16) +#define NVME_MAX_IO_TRACKERS (1024) + /* * NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES * for each controller. @@ -134,6 +145,7 @@ struct nvme_qpair { uint32_t max_xfer_size; uint32_t num_entries; + uint32_t num_trackers; uint32_t sq_tdbl_off; uint32_t cq_hdbl_off; @@ -155,8 +167,6 @@ struct nvme_qpair { bus_dmamap_t cpl_dma_map; uint64_t cpl_bus_addr; - uint32_t num_tr; - SLIST_HEAD(, nvme_tracker) free_tr; struct nvme_tracker **act_tr; @@ -348,12 +358,11 @@ void nvme_ctrlr_submit_io_request(struct void nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, uint16_t vector, uint32_t num_entries, - uint32_t max_xfer_size, + uint32_t num_trackers, uint32_t max_xfer_size, struct nvme_controller *ctrlr); void nvme_qpair_submit_cmd(struct nvme_qpair *qpair, struct nvme_tracker *tr); void nvme_qpair_process_completions(struct nvme_qpair *qpair); -struct nvme_tracker * nvme_qpair_allocate_tracker(struct nvme_qpair *qpair); void nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req); Modified: head/sys/dev/nvme/nvme_qpair.c ============================================================================== --- head/sys/dev/nvme/nvme_qpair.c Thu Oct 18 00:43:25 2012 (r241663) +++ head/sys/dev/nvme/nvme_qpair.c Thu Oct 18 00:44:39 2012 (r241664) @@ -74,42 +74,20 @@ nvme_completion_check_retry(const struct } } -struct nvme_tracker * -nvme_qpair_allocate_tracker(struct nvme_qpair *qpair) +static void +nvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr, + uint16_t cid) { - struct nvme_tracker *tr; - - tr = SLIST_FIRST(&qpair->free_tr); - if (tr == NULL) { - /* - * We can't support more trackers than we have entries in - * our queue, because it would generate invalid indices - * into the qpair's active tracker array. - */ - if (qpair->num_tr == qpair->num_entries) { - return (NULL); - } - - tr = malloc(sizeof(struct nvme_tracker), M_NVME, - M_ZERO | M_NOWAIT); - - if (tr == NULL) { - return (NULL); - } - bus_dmamap_create(qpair->dma_tag, 0, &tr->payload_dma_map); - bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map); + bus_dmamap_create(qpair->dma_tag, 0, &tr->payload_dma_map); + bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map); - bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp, - sizeof(tr->prp), nvme_single_map, &tr->prp_bus_addr, 0); - - callout_init_mtx(&tr->timer, &qpair->lock, 0); - tr->cid = qpair->num_tr++; - tr->qpair = qpair; - } else - SLIST_REMOVE_HEAD(&qpair->free_tr, slist); + bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp, + sizeof(tr->prp), nvme_single_map, &tr->prp_bus_addr, 0); - return (tr); + callout_init_mtx(&tr->timer, &qpair->lock, 0); + tr->cid = cid; + tr->qpair = qpair; } void @@ -163,6 +141,10 @@ nvme_qpair_process_completions(struct nv tr->payload_dma_map); nvme_free_request(req); + + if (SLIST_EMPTY(&qpair->free_tr)) + wakeup(qpair); + SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist); } @@ -188,9 +170,11 @@ nvme_qpair_msix_handler(void *arg) void nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, - uint16_t vector, uint32_t num_entries, uint32_t max_xfer_size, - struct nvme_controller *ctrlr) + uint16_t vector, uint32_t num_entries, uint32_t num_trackers, + uint32_t max_xfer_size, struct nvme_controller *ctrlr) { + struct nvme_tracker *tr; + uint32_t i; qpair->id = id; qpair->vector = vector; @@ -233,7 +217,7 @@ nvme_qpair_construct(struct nvme_qpair * qpair->num_cmds = 0; qpair->num_intr_handler_calls = 0; - qpair->num_tr = 0; + qpair->num_trackers = num_trackers; qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0; /* TODO: error checking on contigmalloc, bus_dmamap_load calls */ @@ -259,6 +243,18 @@ nvme_qpair_construct(struct nvme_qpair * SLIST_INIT(&qpair->free_tr); + for (i = 0; i < num_trackers; i++) { + tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_NOWAIT); + + if (tr == NULL) { + printf("warning: nvme tracker malloc failed\n"); + break; + } + + nvme_qpair_construct_tracker(qpair, tr, i); + SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist); + } + qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * qpair->num_entries, M_NVME, M_ZERO | M_NOWAIT); } @@ -379,19 +375,6 @@ nvme_qpair_submit_cmd(struct nvme_qpair req->cmd.cid = tr->cid; qpair->act_tr[tr->cid] = tr; - /* - * TODO: rather than spin until entries free up, put this tracker - * on a queue, and submit from the interrupt handler when - * entries free up. - */ - if ((qpair->sq_tail+1) % qpair->num_entries == qpair->sq_head) { - do { - mtx_unlock(&qpair->lock); - DELAY(5); - mtx_lock(&qpair->lock); - } while ((qpair->sq_tail+1) % qpair->num_entries == qpair->sq_head); - } - callout_reset(&tr->timer, NVME_TIMEOUT_IN_SEC * hz, nvme_timeout, tr); /* Copy the command from the tracker to the submission queue. */ @@ -415,7 +398,15 @@ nvme_qpair_submit_request(struct nvme_qp mtx_lock(&qpair->lock); - tr = nvme_qpair_allocate_tracker(qpair); + tr = SLIST_FIRST(&qpair->free_tr); + + while (tr == NULL) { + msleep(qpair, &qpair->lock, PRIBIO, "qpair_tr", 0); + printf("msleep\n"); + tr = SLIST_FIRST(&qpair->free_tr); + } + + SLIST_REMOVE_HEAD(&qpair->free_tr, slist); tr->req = req; if (req->uio == NULL) { Modified: head/sys/dev/nvme/nvme_sysctl.c ============================================================================== --- head/sys/dev/nvme/nvme_sysctl.c Thu Oct 18 00:43:25 2012 (r241663) +++ head/sys/dev/nvme/nvme_sysctl.c Thu Oct 18 00:44:39 2012 (r241664) @@ -192,9 +192,9 @@ nvme_sysctl_initialize_queue(struct nvme SYSCTL_ADD_UINT(ctrlr_ctx, que_list, OID_AUTO, "num_entries", CTLFLAG_RD, &qpair->num_entries, 0, "Number of entries in hardware queue"); - SYSCTL_ADD_UINT(ctrlr_ctx, que_list, OID_AUTO, "num_tr", - CTLFLAG_RD, &qpair->num_tr, 0, - "Number of trackers allocated"); + SYSCTL_ADD_UINT(ctrlr_ctx, que_list, OID_AUTO, "num_trackers", + CTLFLAG_RD, &qpair->num_trackers, 0, + "Number of trackers pre-allocated for this queue pair"); SYSCTL_ADD_UINT(ctrlr_ctx, que_list, OID_AUTO, "sq_head", CTLFLAG_RD, &qpair->sq_head, 0, "Current head of submission queue (as observed by driver)");
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201210180044.q9I0ieh7002017>