From owner-svn-src-head@FreeBSD.ORG Thu Oct 18 00:39:30 2012 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id 5196BF05; Thu, 18 Oct 2012 00:39:30 +0000 (UTC) (envelope-from jimharris@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 34E748FC08; Thu, 18 Oct 2012 00:39:30 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id q9I0dUcK000958; Thu, 18 Oct 2012 00:39:30 GMT (envelope-from jimharris@svn.freebsd.org) Received: (from jimharris@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id q9I0dU62000950; Thu, 18 Oct 2012 00:39:30 GMT (envelope-from jimharris@svn.freebsd.org) Message-Id: <201210180039.q9I0dU62000950@svn.freebsd.org> From: Jim Harris Date: Thu, 18 Oct 2012 00:39:30 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r241660 - head/sys/dev/nvme X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 18 Oct 2012 00:39:30 -0000 Author: jimharris Date: Thu Oct 18 00:39:29 2012 New Revision: 241660 URL: http://svn.freebsd.org/changeset/base/241660 Log: Add nvme_ctrlr_submit_[admin|io]_request functions which consolidates code for allocating nvme_tracker objects and making calls into bus_dmamap_load for commands which have payloads. Sponsored by: Intel Modified: head/sys/dev/nvme/nvme.c head/sys/dev/nvme/nvme_ctrlr.c head/sys/dev/nvme/nvme_ctrlr_cmd.c head/sys/dev/nvme/nvme_ns_cmd.c head/sys/dev/nvme/nvme_private.h head/sys/dev/nvme/nvme_qpair.c Modified: head/sys/dev/nvme/nvme.c ============================================================================== --- head/sys/dev/nvme/nvme.c Thu Oct 18 00:38:28 2012 (r241659) +++ head/sys/dev/nvme/nvme.c Thu Oct 18 00:39:29 2012 (r241660) @@ -262,33 +262,6 @@ nvme_payload_map(void *arg, bus_dma_segm nvme_qpair_submit_cmd(qpair, tr); } -struct nvme_tracker * -nvme_allocate_tracker(struct nvme_controller *ctrlr, boolean_t is_admin, - struct nvme_request *req) -{ - struct nvme_tracker *tr; - struct nvme_qpair *qpair; - - if (is_admin) { - qpair = &ctrlr->adminq; - } else { - if (ctrlr->per_cpu_io_queues) - qpair = &ctrlr->ioq[curcpu]; - else - qpair = &ctrlr->ioq[0]; - } - - tr = nvme_qpair_allocate_tracker(qpair); - - if (tr == NULL) - return (NULL); - - tr->qpair = qpair; - tr->req = req; - - return (tr); -} - static int nvme_attach(device_t dev) { Modified: head/sys/dev/nvme/nvme_ctrlr.c ============================================================================== --- head/sys/dev/nvme/nvme_ctrlr.c Thu Oct 18 00:38:28 2012 (r241659) +++ head/sys/dev/nvme/nvme_ctrlr.c Thu Oct 18 00:39:29 2012 (r241660) @@ -791,3 +791,54 @@ intx: return (0); } + +void +nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, + struct nvme_request *req) +{ + struct nvme_qpair *qpair; + struct nvme_tracker *tr; + int err; + + qpair = &ctrlr->adminq; + + tr = nvme_qpair_allocate_tracker(qpair); + + tr->req = req; + + if (req->payload_size > 0) { + err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, + req->payload, req->payload_size, + nvme_payload_map, tr, 0); + if (err != 0) + panic("bus_dmamap_load returned non-zero!\n"); + } else + nvme_qpair_submit_cmd(tr->qpair, tr); +} + +void +nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, + struct nvme_request *req) +{ + struct nvme_qpair *qpair; + struct nvme_tracker *tr; + int err; + + if (ctrlr->per_cpu_io_queues) + qpair = &ctrlr->ioq[curcpu]; + else + qpair = &ctrlr->ioq[0]; + + tr = nvme_qpair_allocate_tracker(qpair); + + tr->req = req; + + if (req->payload_size > 0) { + err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, + req->payload, req->payload_size, + nvme_payload_map, tr, 0); + if (err != 0) + panic("bus_dmamap_load returned non-zero!\n"); + } else + nvme_qpair_submit_cmd(tr->qpair, tr); +} Modified: head/sys/dev/nvme/nvme_ctrlr_cmd.c ============================================================================== --- head/sys/dev/nvme/nvme_ctrlr_cmd.c Thu Oct 18 00:38:28 2012 (r241659) +++ head/sys/dev/nvme/nvme_ctrlr_cmd.c Thu Oct 18 00:39:29 2012 (r241660) @@ -34,15 +34,11 @@ nvme_ctrlr_cmd_identify_controller(struc nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; - int err; req = nvme_allocate_request(payload, sizeof(struct nvme_controller_data), cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_IDENTIFY; @@ -52,10 +48,7 @@ nvme_ctrlr_cmd_identify_controller(struc */ cmd->cdw10 = 1; - err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - req->payload_size, nvme_payload_map, tr, 0); - - KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); + nvme_ctrlr_submit_admin_request(ctrlr, req); } void @@ -63,15 +56,11 @@ nvme_ctrlr_cmd_identify_namespace(struct void *payload, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; - int err; req = nvme_allocate_request(payload, sizeof(struct nvme_namespace_data), cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_IDENTIFY; @@ -80,10 +69,7 @@ nvme_ctrlr_cmd_identify_namespace(struct */ cmd->nsid = nsid; - err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - req->payload_size, nvme_payload_map, tr, 0); - - KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); + nvme_ctrlr_submit_admin_request(ctrlr, req); } void @@ -92,13 +78,10 @@ nvme_ctrlr_cmd_create_io_cq(struct nvme_ void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_CREATE_IO_CQ; @@ -111,7 +94,7 @@ nvme_ctrlr_cmd_create_io_cq(struct nvme_ cmd->cdw11 = (vector << 16) | 0x3; cmd->prp1 = io_que->cpl_bus_addr; - nvme_qpair_submit_cmd(tr->qpair, tr); + nvme_ctrlr_submit_admin_request(ctrlr, req); } void @@ -119,13 +102,10 @@ nvme_ctrlr_cmd_create_io_sq(struct nvme_ struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_CREATE_IO_SQ; @@ -138,7 +118,7 @@ nvme_ctrlr_cmd_create_io_sq(struct nvme_ cmd->cdw11 = (io_que->id << 16) | 0x1; cmd->prp1 = io_que->cmd_bus_addr; - nvme_qpair_submit_cmd(tr->qpair, tr); + nvme_ctrlr_submit_admin_request(ctrlr, req); } void @@ -146,13 +126,10 @@ nvme_ctrlr_cmd_delete_io_cq(struct nvme_ struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_DELETE_IO_CQ; @@ -162,7 +139,7 @@ nvme_ctrlr_cmd_delete_io_cq(struct nvme_ */ cmd->cdw10 = io_que->id; - nvme_qpair_submit_cmd(tr->qpair, tr); + nvme_ctrlr_submit_admin_request(ctrlr, req); } void @@ -170,13 +147,10 @@ nvme_ctrlr_cmd_delete_io_sq(struct nvme_ struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_DELETE_IO_SQ; @@ -186,7 +160,7 @@ nvme_ctrlr_cmd_delete_io_sq(struct nvme_ */ cmd->cdw10 = io_que->id; - nvme_qpair_submit_cmd(tr->qpair, tr); + nvme_ctrlr_submit_admin_request(ctrlr, req); } void @@ -195,26 +169,16 @@ nvme_ctrlr_cmd_set_feature(struct nvme_c nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; - int err; req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_SET_FEATURES; cmd->cdw10 = feature; cmd->cdw11 = cdw11; - if (payload_size > 0) { - err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, - payload, payload_size, nvme_payload_map, tr, 0); - - KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); - } else - nvme_qpair_submit_cmd(tr->qpair, tr); + nvme_ctrlr_submit_admin_request(ctrlr, req); } void @@ -223,26 +187,16 @@ nvme_ctrlr_cmd_get_feature(struct nvme_c nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; - int err; req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_GET_FEATURES; cmd->cdw10 = feature; cmd->cdw11 = cdw11; - if (payload_size > 0) { - err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, - payload, payload_size, nvme_payload_map, tr, 0); - - KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); - } else - nvme_qpair_submit_cmd(tr->qpair, tr); + nvme_ctrlr_submit_admin_request(ctrlr, req); } void @@ -299,17 +253,14 @@ nvme_ctrlr_cmd_asynchronous_event_reques nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_ASYNC_EVENT_REQUEST; - nvme_qpair_submit_cmd(tr->qpair, tr); + nvme_ctrlr_submit_admin_request(ctrlr, req); } void @@ -318,22 +269,15 @@ nvme_ctrlr_cmd_get_health_information_pa nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; - int err; req = nvme_allocate_request(payload, sizeof(*payload), cb_fn, cb_arg); - tr = nvme_allocate_tracker(ctrlr, TRUE, req); - cmd = &req->cmd; cmd->opc = NVME_OPC_GET_LOG_PAGE; cmd->nsid = nsid; cmd->cdw10 = ((sizeof(*payload)/sizeof(uint32_t)) - 1) << 16; cmd->cdw10 |= NVME_LOG_HEALTH_INFORMATION; - err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - sizeof(*payload), nvme_payload_map, tr, 0); - - KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); + nvme_ctrlr_submit_admin_request(ctrlr, req); } Modified: head/sys/dev/nvme/nvme_ns_cmd.c ============================================================================== --- head/sys/dev/nvme/nvme_ns_cmd.c Thu Oct 18 00:38:28 2012 (r241659) +++ head/sys/dev/nvme/nvme_ns_cmd.c Thu Oct 18 00:39:29 2012 (r241660) @@ -34,17 +34,12 @@ nvme_ns_cmd_read(struct nvme_namespace * uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; - int err; req = nvme_allocate_request(payload, lba_count*512, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ns->ctrlr, FALSE, req); - - if (tr == NULL) + if (req == NULL) return (ENOMEM); - cmd = &req->cmd; cmd->opc = NVME_OPC_READ; cmd->nsid = ns->id; @@ -53,10 +48,7 @@ nvme_ns_cmd_read(struct nvme_namespace * *(uint64_t *)&cmd->cdw10 = lba; cmd->cdw12 = lba_count-1; - err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - req->payload_size, nvme_payload_map, tr, 0); - - KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); + nvme_ctrlr_submit_io_request(ns->ctrlr, req); return (0); } @@ -66,15 +58,11 @@ nvme_ns_cmd_write(struct nvme_namespace uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; - int err; req = nvme_allocate_request(payload, lba_count*512, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ns->ctrlr, FALSE, req); - - if (tr == NULL) + if (req == NULL) return (ENOMEM); cmd = &req->cmd; @@ -85,10 +73,7 @@ nvme_ns_cmd_write(struct nvme_namespace *(uint64_t *)&cmd->cdw10 = lba; cmd->cdw12 = lba_count-1; - err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - req->payload_size, nvme_payload_map, tr, 0); - - KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); + nvme_ctrlr_submit_io_request(ns->ctrlr, req); return (0); } @@ -98,16 +83,12 @@ nvme_ns_cmd_deallocate(struct nvme_names uint8_t num_ranges, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; - int err; req = nvme_allocate_request(payload, num_ranges * sizeof(struct nvme_dsm_range), cb_fn, cb_arg); - tr = nvme_allocate_tracker(ns->ctrlr, FALSE, req); - - if (tr == NULL) + if (req == NULL) return (ENOMEM); cmd = &req->cmd; @@ -118,10 +99,7 @@ nvme_ns_cmd_deallocate(struct nvme_names cmd->cdw10 = num_ranges; cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE; - err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map, payload, - req->payload_size, nvme_payload_map, tr, 0); - - KASSERT(err == 0, ("bus_dmamap_load returned non-zero!\n")); + nvme_ctrlr_submit_io_request(ns->ctrlr, req); return (0); } @@ -130,21 +108,18 @@ int nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn, void *cb_arg) { struct nvme_request *req; - struct nvme_tracker *tr; struct nvme_command *cmd; req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg); - tr = nvme_allocate_tracker(ns->ctrlr, FALSE, req); - - if (tr == NULL) + if (req == NULL) return (ENOMEM); cmd = &req->cmd; cmd->opc = NVME_OPC_FLUSH; cmd->nsid = ns->id; - nvme_qpair_submit_cmd(tr->qpair, tr); + nvme_ctrlr_submit_io_request(ns->ctrlr, req); return (0); } Modified: head/sys/dev/nvme/nvme_private.h ============================================================================== --- head/sys/dev/nvme/nvme_private.h Thu Oct 18 00:38:28 2012 (r241659) +++ head/sys/dev/nvme/nvme_private.h Thu Oct 18 00:39:29 2012 (r241660) @@ -331,9 +331,6 @@ void nvme_ctrlr_cmd_asynchronous_event_r nvme_cb_fn_t cb_fn, void *cb_arg); -struct nvme_tracker * nvme_allocate_tracker(struct nvme_controller *ctrlr, - boolean_t is_admin, - struct nvme_request *request); void nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error); @@ -341,6 +338,10 @@ int nvme_ctrlr_construct(struct nvme_con int nvme_ctrlr_reset(struct nvme_controller *ctrlr); /* ctrlr defined as void * to allow use with config_intrhook. */ void nvme_ctrlr_start(void *ctrlr_arg); +void nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr, + struct nvme_request *req); +void nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr, + struct nvme_request *req); void nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, uint16_t vector, uint32_t num_entries, Modified: head/sys/dev/nvme/nvme_qpair.c ============================================================================== --- head/sys/dev/nvme/nvme_qpair.c Thu Oct 18 00:38:28 2012 (r241659) +++ head/sys/dev/nvme/nvme_qpair.c Thu Oct 18 00:39:29 2012 (r241660) @@ -109,6 +109,7 @@ nvme_qpair_allocate_tracker(struct nvme_ callout_init_mtx(&tr->timer, &qpair->lock, 0); tr->cid = qpair->num_tr++; + tr->qpair = qpair; } else SLIST_REMOVE_HEAD(&qpair->free_tr, slist);