From owner-svn-src-head@FreeBSD.ORG Thu Oct 18 00:38:29 2012 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id 10EBDD8E; Thu, 18 Oct 2012 00:38:29 +0000 (UTC) (envelope-from jimharris@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id E7B598FC17; Thu, 18 Oct 2012 00:38:28 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id q9I0cSZO000763; Thu, 18 Oct 2012 00:38:28 GMT (envelope-from jimharris@svn.freebsd.org) Received: (from jimharris@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id q9I0cSDT000756; Thu, 18 Oct 2012 00:38:28 GMT (envelope-from jimharris@svn.freebsd.org) Message-Id: <201210180038.q9I0cSDT000756@svn.freebsd.org> From: Jim Harris Date: Thu, 18 Oct 2012 00:38:28 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r241659 - head/sys/dev/nvme X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 18 Oct 2012 00:38:29 -0000 Author: jimharris Date: Thu Oct 18 00:38:28 2012 New Revision: 241659 URL: http://svn.freebsd.org/changeset/base/241659 Log: Add struct nvme_request object which contains all of the parameters passed from an NVMe consumer. This allows us to mostly build NVMe command buffers without holding the qpair lock, and also allows for future queueing of nvme_request objects in cases where the submission queue is full and no nvme_tracker objects are available. Sponsored by: Intel Modified: head/sys/dev/nvme/nvme.c head/sys/dev/nvme/nvme_ctrlr_cmd.c head/sys/dev/nvme/nvme_ns_cmd.c head/sys/dev/nvme/nvme_private.h head/sys/dev/nvme/nvme_qpair.c head/sys/dev/nvme/nvme_uio.c Modified: head/sys/dev/nvme/nvme.c ============================================================================== --- head/sys/dev/nvme/nvme.c Thu Oct 18 00:37:11 2012 (r241658) +++ head/sys/dev/nvme/nvme.c Thu Oct 18 00:38:28 2012 (r241659) @@ -32,6 +32,8 @@ __FBSDID("$FreeBSD$"); #include #include +#include + #include #include @@ -44,6 +46,8 @@ struct nvme_consumer { struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS]; +uma_zone_t nvme_request_zone; + MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations"); static int nvme_probe(device_t); @@ -110,6 +114,23 @@ nvme_probe (device_t device) } static void +nvme_init(void) +{ + nvme_request_zone = uma_zcreate("nvme_request", + sizeof(struct nvme_request), NULL, NULL, NULL, NULL, 0, 0); +} + +SYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL); + +static void +nvme_uninit(void) +{ + uma_zdestroy(nvme_request_zone); +} + +SYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL); + +static void nvme_load(void) { } @@ -224,13 +245,13 @@ nvme_payload_map(void *arg, bus_dma_segm * we can safely just transfer each segment to its * associated PRP entry. */ - tr->cmd.prp1 = seg[0].ds_addr; + tr->req->cmd.prp1 = seg[0].ds_addr; if (nseg == 2) { - tr->cmd.prp2 = seg[1].ds_addr; + tr->req->cmd.prp2 = seg[1].ds_addr; } else if (nseg > 2) { cur_nseg = 1; - tr->cmd.prp2 = (uint64_t)tr->prp_bus_addr; + tr->req->cmd.prp2 = (uint64_t)tr->prp_bus_addr; while (cur_nseg < nseg) { tr->prp[cur_nseg-1] = (uint64_t)seg[cur_nseg].ds_addr; @@ -243,7 +264,7 @@ nvme_payload_map(void *arg, bus_dma_segm struct nvme_tracker * nvme_allocate_tracker(struct nvme_controller *ctrlr, boolean_t is_admin, - nvme_cb_fn_t cb_fn, void *cb_arg, uint32_t payload_size, void *payload) + struct nvme_request *req) { struct nvme_tracker *tr; struct nvme_qpair *qpair; @@ -262,12 +283,8 @@ nvme_allocate_tracker(struct nvme_contro if (tr == NULL) return (NULL); - memset(&tr->cmd, 0, sizeof(tr->cmd)); - tr->qpair = qpair; - tr->cb_fn = cb_fn; - tr->cb_arg = cb_arg; - tr->payload_size = payload_size; + tr->req = req; return (tr); } Modified: head/sys/dev/nvme/nvme_ctrlr_cmd.c ============================================================================== --- head/sys/dev/nvme/nvme_ctrlr_cmd.c Thu Oct 18 00:37:11 2012 (r241658) +++ head/sys/dev/nvme/nvme_ctrlr_cmd.c Thu Oct 18 00:38:28 2012 (r241659) @@ -33,14 +33,17 @@ void nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, void *payload, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, - sizeof(struct nvme_controller_data), payload); + req = nvme_allocate_request(payload, + sizeof(struct nvme_controller_data), cb_fn, cb_arg); - cmd = &tr->cmd; + tr = nvme_allocate_tracker(ctrlr, TRUE, req); + + cmd = &req->cmd; cmd->opc = NVME_OPC_IDENTIFY; /* @@ -50,7 +53,7 @@ nvme_ctrlr_cmd_identify_controller(struc cmd->cdw10 = 1; err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - tr->payload_size, nvme_payload_map, tr, 0); + req->payload_size, nvme_payload_map, tr, 0); KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); } @@ -59,14 +62,17 @@ void nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr, uint16_t nsid, void *payload, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, - sizeof(struct nvme_namespace_data), payload); + req = nvme_allocate_request(payload, + sizeof(struct nvme_namespace_data), cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_IDENTIFY; /* @@ -75,7 +81,7 @@ nvme_ctrlr_cmd_identify_namespace(struct cmd->nsid = nsid; err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - tr->payload_size, nvme_payload_map, tr, 0); + req->payload_size, nvme_payload_map, tr, 0); KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); } @@ -85,12 +91,15 @@ nvme_ctrlr_cmd_create_io_cq(struct nvme_ struct nvme_qpair *io_que, uint16_t vector, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, 0, NULL); + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_CREATE_IO_CQ; /* @@ -109,12 +118,15 @@ void nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr, struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, 0, NULL); + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - cmd = &tr->cmd; + tr = nvme_allocate_tracker(ctrlr, TRUE, req); + + cmd = &req->cmd; cmd->opc = NVME_OPC_CREATE_IO_SQ; /* @@ -133,12 +145,15 @@ void nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr, struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, 0, NULL); + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_DELETE_IO_CQ; /* @@ -154,12 +169,15 @@ void nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr, struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, 0, NULL); + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - cmd = &tr->cmd; + tr = nvme_allocate_tracker(ctrlr, TRUE, req); + + cmd = &req->cmd; cmd->opc = NVME_OPC_DELETE_IO_SQ; /* @@ -176,14 +194,16 @@ nvme_ctrlr_cmd_set_feature(struct nvme_c uint32_t cdw11, void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, - payload_size, payload); + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_SET_FEATURES; cmd->cdw10 = feature; cmd->cdw11 = cdw11; @@ -202,14 +222,16 @@ nvme_ctrlr_cmd_get_feature(struct nvme_c uint32_t cdw11, void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, - payload_size, payload); + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_GET_FEATURES; cmd->cdw10 = feature; cmd->cdw11 = cdw11; @@ -276,12 +298,15 @@ void nvme_ctrlr_cmd_asynchronous_event_request(struct nvme_controller *ctrlr, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, 0, NULL); + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - cmd = &tr->cmd; + tr = nvme_allocate_tracker(ctrlr, TRUE, req); + + cmd = &req->cmd; cmd->opc = NVME_OPC_ASYNC_EVENT_REQUEST; nvme_qpair_submit_cmd(tr->qpair, tr); @@ -292,14 +317,16 @@ nvme_ctrlr_cmd_get_health_information_pa uint32_t nsid, struct nvme_health_information_page *payload, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err; - tr = nvme_allocate_tracker(ctrlr, TRUE, cb_fn, cb_arg, - sizeof(*payload), payload); + req = nvme_allocate_request(payload, sizeof(*payload), cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_GET_LOG_PAGE; cmd->nsid = nsid; cmd->cdw10 = ((sizeof(*payload)/sizeof(uint32_t)) - 1) << 16; Modified: head/sys/dev/nvme/nvme_ns_cmd.c ============================================================================== --- head/sys/dev/nvme/nvme_ns_cmd.c Thu Oct 18 00:37:11 2012 (r241658) +++ head/sys/dev/nvme/nvme_ns_cmd.c Thu Oct 18 00:38:28 2012 (r241659) @@ -33,17 +33,19 @@ int nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err; - tr = nvme_allocate_tracker(ns->ctrlr, FALSE, cb_fn, cb_arg, - lba_count*512, payload); + req = nvme_allocate_request(payload, lba_count*512, cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ns->ctrlr, FALSE, req); if (tr == NULL) return (ENOMEM); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_READ; cmd->nsid = ns->id; @@ -52,7 +54,7 @@ nvme_ns_cmd_read(struct nvme_namespace * cmd->cdw12 = lba_count-1; err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - tr->payload_size, nvme_payload_map, tr, 0); + req->payload_size, nvme_payload_map, tr, 0); KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); @@ -63,17 +65,19 @@ int nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err; - tr = nvme_allocate_tracker(ns->ctrlr, FALSE, cb_fn, cb_arg, - lba_count*512, payload); + req = nvme_allocate_request(payload, lba_count*512, cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ns->ctrlr, FALSE, req); if (tr == NULL) return (ENOMEM); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_WRITE; cmd->nsid = ns->id; @@ -82,7 +86,7 @@ nvme_ns_cmd_write(struct nvme_namespace cmd->cdw12 = lba_count-1; err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - tr->payload_size, nvme_payload_map, tr, 0); + req->payload_size, nvme_payload_map, tr, 0); KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); @@ -93,17 +97,20 @@ int nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload, uint8_t num_ranges, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err; - tr = nvme_allocate_tracker(ns->ctrlr, FALSE, cb_fn, cb_arg, - num_ranges * sizeof(struct nvme_dsm_range), payload); + req = nvme_allocate_request(payload, + num_ranges * sizeof(struct nvme_dsm_range), cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ns->ctrlr, FALSE, req); if (tr == NULL) return (ENOMEM); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_DATASET_MANAGEMENT; cmd->nsid = ns->id; @@ -112,7 +119,7 @@ nvme_ns_cmd_deallocate(struct nvme_names cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE; err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - tr->payload_size, nvme_payload_map, tr, 0); + req->payload_size, nvme_payload_map, tr, 0); KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); @@ -122,15 +129,18 @@ nvme_ns_cmd_deallocate(struct nvme_names int nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn, void *cb_arg) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; - tr = nvme_allocate_tracker(ns->ctrlr, FALSE, cb_fn, cb_arg, 0, NULL); + req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); + + tr = nvme_allocate_tracker(ns->ctrlr, FALSE, req); if (tr == NULL) return (ENOMEM); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_FLUSH; cmd->nsid = ns->id; Modified: head/sys/dev/nvme/nvme_private.h ============================================================================== --- head/sys/dev/nvme/nvme_private.h Thu Oct 18 00:37:11 2012 (r241658) +++ head/sys/dev/nvme/nvme_private.h Thu Oct 18 00:38:28 2012 (r241659) @@ -38,6 +38,8 @@ #include #include +#include + #include #include "nvme.h" @@ -92,16 +94,25 @@ MALLOC_DECLARE(M_NVME); #define CACHE_LINE_SIZE (64) #endif +extern uma_zone_t nvme_request_zone; + +struct nvme_request { + + struct nvme_command cmd; + void *payload; + uint32_t payload_size; + nvme_cb_fn_t cb_fn; + void *cb_arg; + SLIST_ENTRY(nvme_request) slist; +}; + struct nvme_tracker { SLIST_ENTRY(nvme_tracker) slist; + struct nvme_request *req; struct nvme_qpair *qpair; - struct nvme_command cmd; struct callout timer; bus_dmamap_t payload_dma_map; - nvme_cb_fn_t cb_fn; - void *cb_arg; - uint32_t payload_size; uint16_t cid; uint64_t prp[NVME_MAX_PRP_LIST_ENTRIES]; @@ -322,9 +333,7 @@ void nvme_ctrlr_cmd_asynchronous_event_r struct nvme_tracker * nvme_allocate_tracker(struct nvme_controller *ctrlr, boolean_t is_admin, - nvme_cb_fn_t cb_fn, void *cb_arg, - uint32_t payload_size, - void *payload); + struct nvme_request *request); void nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error); @@ -364,4 +373,24 @@ nvme_single_map(void *arg, bus_dma_segme *bus_addr = seg[0].ds_addr; } +static __inline struct nvme_request * +nvme_allocate_request(void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn, + void *cb_arg) +{ + struct nvme_request *req; + + req = uma_zalloc(nvme_request_zone, M_NOWAIT | M_ZERO); + if (req == NULL) + return (NULL); + + req->payload = payload; + req->payload_size = payload_size; + req->cb_fn = cb_fn; + req->cb_arg = cb_arg; + + return (req); +} + +#define nvme_free_request(req) uma_zfree(nvme_request_zone, req) + #endif /* __NVME_PRIVATE_H__ */ Modified: head/sys/dev/nvme/nvme_qpair.c ============================================================================== --- head/sys/dev/nvme/nvme_qpair.c Thu Oct 18 00:37:11 2012 (r241658) +++ head/sys/dev/nvme/nvme_qpair.c Thu Oct 18 00:38:28 2012 (r241659) @@ -119,6 +119,7 @@ void nvme_qpair_process_completions(struct nvme_qpair *qpair) { struct nvme_tracker *tr; + struct nvme_request *req; struct nvme_completion *cpl; boolean_t retry, error; @@ -131,6 +132,8 @@ nvme_qpair_process_completions(struct nv break; tr = qpair->act_tr[cpl->cid]; + req = tr->req; + KASSERT(tr, ("completion queue has entries but no active trackers\n")); @@ -139,7 +142,7 @@ nvme_qpair_process_completions(struct nv if (error) { nvme_dump_completion(cpl); - nvme_dump_command(&tr->cmd); + nvme_dump_command(&tr->req->cmd); } qpair->act_tr[cpl->cid] = NULL; @@ -147,8 +150,8 @@ nvme_qpair_process_completions(struct nv KASSERT(cpl->cid == tr->cmd.cid, ("cpl cid does not match cmd cid\n")); - if (tr->cb_fn && !retry) - tr->cb_fn(tr->cb_arg, cpl); + if (req->cb_fn && !retry) + req->cb_fn(req->cb_arg, cpl); qpair->sq_head = cpl->sqhd; @@ -159,10 +162,11 @@ nvme_qpair_process_completions(struct nv /* nvme_qpair_submit_cmd() will release the lock. */ nvme_qpair_submit_cmd(qpair, tr); else { - if (tr->payload_size > 0) + if (req->payload_size > 0) bus_dmamap_unload(qpair->dma_tag, tr->payload_dma_map); + nvme_free_request(req); SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist); mtx_unlock(&qpair->lock); @@ -373,8 +377,10 @@ nvme_timeout(void *arg) void nvme_qpair_submit_cmd(struct nvme_qpair *qpair, struct nvme_tracker *tr) { + struct nvme_request *req; - tr->cmd.cid = tr->cid; + req = tr->req; + req->cmd.cid = tr->cid; qpair->act_tr[tr->cid] = tr; /* @@ -393,7 +399,7 @@ nvme_qpair_submit_cmd(struct nvme_qpair callout_reset(&tr->timer, NVME_TIMEOUT_IN_SEC * hz, nvme_timeout, tr); /* Copy the command from the tracker to the submission queue. */ - memcpy(&qpair->cmd[qpair->sq_tail], &tr->cmd, sizeof(tr->cmd)); + memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd)); if (++qpair->sq_tail == qpair->num_entries) qpair->sq_tail = 0; Modified: head/sys/dev/nvme/nvme_uio.c ============================================================================== --- head/sys/dev/nvme/nvme_uio.c Thu Oct 18 00:37:11 2012 (r241658) +++ head/sys/dev/nvme/nvme_uio.c Thu Oct 18 00:38:28 2012 (r241659) @@ -48,7 +48,8 @@ nvme_uio_done(void *arg, const struct nv } static struct nvme_tracker * -nvme_allocate_tracker_uio(struct nvme_controller *ctrlr, struct uio *uio) +nvme_allocate_tracker_uio(struct nvme_controller *ctrlr, struct uio *uio, + struct nvme_request *req) { struct nvme_tracker *tr; struct nvme_qpair *qpair; @@ -63,11 +64,8 @@ nvme_allocate_tracker_uio(struct nvme_co if (tr == NULL) return (NULL); - memset(&tr->cmd, 0, sizeof(tr->cmd)); - tr->qpair = qpair; - tr->cb_fn = nvme_uio_done; - tr->cb_arg = uio; + tr->req = req; return (tr); } @@ -82,27 +80,30 @@ nvme_payload_map_uio(void *arg, bus_dma_ static int nvme_read_uio(struct nvme_namespace *ns, struct uio *uio) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err, i; uint64_t lba, iosize = 0; - tr = nvme_allocate_tracker_uio(ns->ctrlr, uio); + for (i = 0; i < uio->uio_iovcnt; i++) { + iosize += uio->uio_iov[i].iov_len; + } + + req = nvme_allocate_request(NULL, iosize, nvme_uio_done, uio); + + tr = nvme_allocate_tracker_uio(ns->ctrlr, uio, req); if (tr == NULL) return (ENOMEM); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_READ; cmd->nsid = ns->id; lba = uio->uio_offset / nvme_ns_get_sector_size(ns); *(uint64_t *)&cmd->cdw10 = lba; - for (i = 0; i < uio->uio_iovcnt; i++) { - iosize += uio->uio_iov[i].iov_len; - } - cmd->cdw12 = (iosize / nvme_ns_get_sector_size(ns))-1; err = bus_dmamap_load_uio(tr->qpair->dma_tag, tr->payload_dma_map, uio, @@ -116,27 +117,30 @@ nvme_read_uio(struct nvme_namespace *ns, static int nvme_write_uio(struct nvme_namespace *ns, struct uio *uio) { + struct nvme_request *req; struct nvme_tracker *tr; struct nvme_command *cmd; int err, i; uint64_t lba, iosize = 0; - tr = nvme_allocate_tracker_uio(ns->ctrlr, uio); + for (i = 0; i < uio->uio_iovcnt; i++) { + iosize += uio->uio_iov[i].iov_len; + } + + req = nvme_allocate_request(NULL, iosize, nvme_uio_done, uio); + + tr = nvme_allocate_tracker_uio(ns->ctrlr, uio, req); if (tr == NULL) return (ENOMEM); - cmd = &tr->cmd; + cmd = &req->cmd; cmd->opc = NVME_OPC_WRITE; cmd->nsid = ns->id; lba = uio->uio_offset / nvme_ns_get_sector_size(ns); *(uint64_t *)&cmd->cdw10 = lba; - for (i = 0; i < uio->uio_iovcnt; i++) { - iosize += uio->uio_iov[i].iov_len; - } - cmd->cdw12 = (iosize / nvme_ns_get_sector_size(ns))-1; err = bus_dmamap_load_uio(tr->qpair->dma_tag, tr->payload_dma_map, uio,