Date: Wed, 14 Aug 2013 06:36:32 +0000 (UTC) From: Alexander Motin <mav@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r254318 - projects/camlock/sys/cam Message-ID: <201308140636.r7E6aWEe030673@svn.freebsd.org>
index | next in thread | raw e-mail
Author: mav Date: Wed Aug 14 06:36:32 2013 New Revision: 254318 URL: http://svnweb.freebsd.org/changeset/base/254318 Log: Take new approach to multiple completion threads. Create number of global threads and spread the load between them using hash of the device address. Modified: projects/camlock/sys/cam/cam_xpt.c Modified: projects/camlock/sys/cam/cam_xpt.c ============================================================================== --- projects/camlock/sys/cam/cam_xpt.c Wed Aug 14 06:27:58 2013 (r254317) +++ projects/camlock/sys/cam/cam_xpt.c Wed Aug 14 06:36:32 2013 (r254318) @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include <sys/fcntl.h> #include <sys/interrupt.h> #include <sys/sbuf.h> +#include <sys/smp.h> #include <sys/taskqueue.h> #include <sys/lock.h> @@ -156,14 +157,18 @@ TUNABLE_INT("kern.cam.boot_delay", &xsof SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN, &xsoftc.boot_delay, 0, "Bus registration wait time"); -/* Queues for our software interrupt handler */ -typedef TAILQ_HEAD(cam_isrq, ccb_hdr) cam_isrq_t; -typedef TAILQ_HEAD(cam_simq, cam_sim) cam_simq_t; -static cam_simq_t cam_simq; -static struct mtx cam_simq_lock; +struct cam_doneq { + struct mtx_padalign cam_doneq_mtx; + TAILQ_HEAD(, ccb_hdr) cam_doneq; +}; -/* Pointers to software interrupt handlers */ -static void *cambio_ih; +static struct cam_doneq cam_doneqs[MAXCPU]; +static int cam_num_doneqs; +static struct proc *cam_proc; + +TUNABLE_INT("kern.cam.num_doneqs", &cam_num_doneqs); +SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN, + &cam_num_doneqs, 0, "Number of completion queues/threads"); struct cam_periph *xpt_periph; @@ -249,8 +254,9 @@ static int xpt_schedule_dev(struct camq static xpt_devicefunc_t xptpassannouncefunc; static void xptaction(struct cam_sim *sim, union ccb *work_ccb); static void xptpoll(struct cam_sim *sim); -static void camisr(void *); static void camisr_runqueue(struct cam_sim *); +static void xpt_done_process(struct ccb_hdr *ccb_h); +static void xpt_done_td(void *); static dev_match_ret xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns, struct cam_eb *bus); static dev_match_ret xptdevicematch(struct dev_match_pattern *patterns, @@ -849,14 +855,13 @@ xpt_init(void *dummy) struct cam_path *path; struct cam_devq *devq; cam_status status; + int error, i; TAILQ_INIT(&xsoftc.xpt_busses); - TAILQ_INIT(&cam_simq); TAILQ_INIT(&xsoftc.ccb_scanq); STAILQ_INIT(&xsoftc.highpowerq); xsoftc.num_highpower = CAM_MAX_HIGHPOWER; - mtx_init(&cam_simq_lock, "CAM SIMQ lock", NULL, MTX_DEF); mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF); mtx_init(&xsoftc.xpt_topo_lock, "XPT topology lock", NULL, MTX_DEF); xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK, @@ -914,8 +919,26 @@ xpt_init(void *dummy) path, NULL, 0, xpt_sim); xpt_free_path(path); mtx_unlock(&xsoftc.xpt_lock); - /* Install our software interrupt handlers */ - swi_add(NULL, "cambio", camisr, NULL, SWI_CAMBIO, INTR_MPSAFE, &cambio_ih); + if (cam_num_doneqs < 1) + cam_num_doneqs = 1 + mp_ncpus / 6; + else if (cam_num_doneqs > MAXCPU) + cam_num_doneqs = MAXCPU; + for (i = 0; i < cam_num_doneqs; i++) { + mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL, + MTX_DEF); + TAILQ_INIT(&cam_doneqs[i].cam_doneq); + error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i], + &cam_proc, NULL, 0, 0, "cam", "doneq%d", i); + if (error != 0) { + cam_num_doneqs = i; + break; + } + } + if (cam_num_doneqs < 1) { + printf("xpt_init: Cannot init completion queues " + "- failing attach\n"); + return (ENOMEM); + } /* * Register a callback for when interrupts are enabled. */ @@ -4309,7 +4332,8 @@ void xpt_done(union ccb *done_ccb) { struct cam_sim *sim; - int first; + struct cam_doneq *queue; + int run, hash; CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n")); if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) != 0) { @@ -4319,21 +4343,24 @@ xpt_done(union ccb *done_ccb) */ sim = done_ccb->ccb_h.path->bus->sim; mtx_lock(&sim->sim_doneq_mtx); - TAILQ_INSERT_TAIL(&sim->sim_doneq, &done_ccb->ccb_h, + if (sim->sim_doneq_flags & CAM_SIM_DQ_BATCH) { + TAILQ_INSERT_TAIL(&sim->sim_doneq, &done_ccb->ccb_h, + sim_links.tqe); + mtx_unlock(&sim->sim_doneq_mtx); + return; + } + mtx_unlock(&sim->sim_doneq_mtx); + hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id + + done_ccb->ccb_h.target_lun) % cam_num_doneqs; + queue = &cam_doneqs[hash]; + mtx_lock(&queue->cam_doneq_mtx); + run = TAILQ_EMPTY(&queue->cam_doneq); + TAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.tqe); done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX; - if ((sim->sim_doneq_flags & (CAM_SIM_DQ_ONQ | - CAM_SIM_DQ_POLLED | CAM_SIM_DQ_BATCH)) == 0) { - mtx_lock(&cam_simq_lock); - first = TAILQ_EMPTY(&cam_simq); - TAILQ_INSERT_TAIL(&cam_simq, sim, links); - mtx_unlock(&cam_simq_lock); - sim->sim_doneq_flags |= CAM_SIM_DQ_ONQ; - } else - first = 0; - mtx_unlock(&sim->sim_doneq_mtx); - if (first) - swi_sched(cambio_ih, 0); + mtx_unlock(&queue->cam_doneq_mtx); + if (run) + wakeup(&queue->cam_doneq); } } @@ -4351,22 +4378,21 @@ xpt_batch_start(struct cam_sim *sim) void xpt_batch_done(struct cam_sim *sim) { - int runq; + struct ccb_hdr *ccb_h; + CAM_SIM_UNLOCK(sim); mtx_lock(&sim->sim_doneq_mtx); KASSERT((sim->sim_doneq_flags & CAM_SIM_DQ_BATCH) != 0, ("Batch flag was not set")); sim->sim_doneq_flags &= ~CAM_SIM_DQ_BATCH; - runq = ((sim->sim_doneq_flags & (CAM_SIM_DQ_ONQ | - CAM_SIM_DQ_POLLED)) == 0); - if (runq) - sim->sim_doneq_flags |= CAM_SIM_DQ_ONQ; - mtx_unlock(&sim->sim_doneq_mtx); - if (runq) { - CAM_SIM_UNLOCK(sim); - camisr_runqueue(sim); - CAM_SIM_LOCK(sim); + while ((ccb_h = TAILQ_FIRST(&sim->sim_doneq)) != NULL) { + TAILQ_REMOVE(&sim->sim_doneq, ccb_h, sim_links.tqe); + mtx_unlock(&sim->sim_doneq_mtx); + xpt_done_process(ccb_h); + mtx_lock(&sim->sim_doneq_mtx); } + mtx_unlock(&sim->sim_doneq_mtx); + CAM_SIM_LOCK(sim); } union ccb * @@ -4840,7 +4866,8 @@ xpt_config(void *arg) callout_reset(&xsoftc.boot_callout, hz * xsoftc.boot_delay / 1000, xpt_boot_delay, NULL); /* Fire up rescan thread. */ - if (kproc_create(xpt_scanner_thread, NULL, NULL, 0, 0, "xpt_thrd")) { + if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0, + "cam", "scanner")) { printf("xpt_config: failed to create rescan thread.\n"); } } @@ -5065,133 +5092,140 @@ xpt_path_mtx(struct cam_path *path) } static void -camisr(void *dummy) +xpt_done_process(struct ccb_hdr *ccb_h) { - cam_simq_t queue; struct cam_sim *sim; + struct mtx *mtx; - mtx_lock(&cam_simq_lock); - TAILQ_INIT(&queue); - while (!TAILQ_EMPTY(&cam_simq)) { - TAILQ_CONCAT(&queue, &cam_simq, links); - mtx_unlock(&cam_simq_lock); - - while ((sim = TAILQ_FIRST(&queue)) != NULL) { - TAILQ_REMOVE(&queue, sim, links); - camisr_runqueue(sim); - } - mtx_lock(&cam_simq_lock); - } - mtx_unlock(&cam_simq_lock); -} + if (ccb_h->flags & CAM_HIGH_POWER) { + struct highpowerlist *hphead; + struct cam_ed *device; -static void -camisr_runqueue(struct cam_sim *sim) -{ - struct ccb_hdr *ccb_h; - struct mtx *mtx; + mtx_lock(&xsoftc.xpt_lock); + hphead = &xsoftc.highpowerq; - mtx_lock(&sim->sim_doneq_mtx); - while ((ccb_h = TAILQ_FIRST(&sim->sim_doneq)) != NULL) { - TAILQ_REMOVE(&sim->sim_doneq, ccb_h, sim_links.tqe); - mtx_unlock(&sim->sim_doneq_mtx); - ccb_h->pinfo.index = CAM_UNQUEUED_INDEX; + device = STAILQ_FIRST(hphead); - CAM_DEBUG(ccb_h->path, CAM_DEBUG_TRACE, - ("camisr\n")); + /* + * Increment the count since this command is done. + */ + xsoftc.num_highpower++; - if (ccb_h->flags & CAM_HIGH_POWER) { - struct highpowerlist *hphead; - struct cam_ed *device; + /* + * Any high powered commands queued up? + */ + if (device != NULL) { - mtx_lock(&xsoftc.xpt_lock); - hphead = &xsoftc.highpowerq; + STAILQ_REMOVE_HEAD(hphead, highpowerq_entry); + mtx_unlock(&xsoftc.xpt_lock); - device = STAILQ_FIRST(hphead); + mtx_lock(&device->sim->devq->send_mtx); + xpt_release_devq_device(device, + /*count*/1, /*runqueue*/TRUE); + mtx_unlock(&device->sim->devq->send_mtx); + } else + mtx_unlock(&xsoftc.xpt_lock); + } - /* - * Increment the count since this command is done. - */ - xsoftc.num_highpower++; + sim = ccb_h->path->bus->sim; + mtx = xpt_path_mtx(ccb_h->path); + mtx_lock(mtx); - /* - * Any high powered commands queued up? - */ - if (device != NULL) { + if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) { + struct cam_ed *dev; - STAILQ_REMOVE_HEAD(hphead, highpowerq_entry); - mtx_unlock(&xsoftc.xpt_lock); + mtx_lock(&sim->devq->send_mtx); + sim->devq->send_active--; + sim->devq->send_openings++; + + dev = ccb_h->path->device; + cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h); + + if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0 + && (dev->ccbq.dev_active == 0))) { + dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY; + xpt_release_devq_device(dev, /*count*/1, + /*run_queue*/FALSE); + } - mtx_lock(&device->sim->devq->send_mtx); - xpt_release_devq_device(device, - /*count*/1, /*runqueue*/TRUE); - mtx_unlock(&device->sim->devq->send_mtx); - } else - mtx_unlock(&xsoftc.xpt_lock); + if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0 + && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) { + dev->flags &= ~CAM_DEV_REL_ON_COMPLETE; + xpt_release_devq_device(dev, /*count*/1, + /*run_queue*/FALSE); } - mtx = xpt_path_mtx(ccb_h->path); - mtx_lock(mtx); + if (!device_is_queued(dev)) + (void)xpt_schedule_devq(sim->devq, dev); + mtx_unlock(&sim->devq->send_mtx); - if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) { - struct cam_ed *dev; + if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 + && (--dev->tag_delay_count == 0)) + xpt_start_tags(ccb_h->path); + } - mtx_lock(&sim->devq->send_mtx); - sim->devq->send_active--; - sim->devq->send_openings++; - - dev = ccb_h->path->device; - cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h); - - if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0 - && (dev->ccbq.dev_active == 0))) { - dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY; - xpt_release_devq_device(dev, /*count*/1, - /*run_queue*/FALSE); - } + if (ccb_h->status & CAM_RELEASE_SIMQ) { + xpt_release_simq(sim, /*run_queue*/FALSE); + ccb_h->status &= ~CAM_RELEASE_SIMQ; + } - if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0 - && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) { - dev->flags &= ~CAM_DEV_REL_ON_COMPLETE; - xpt_release_devq_device(dev, /*count*/1, - /*run_queue*/FALSE); - } + if ((ccb_h->flags & CAM_DEV_QFRZDIS) + && (ccb_h->status & CAM_DEV_QFRZN)) { + xpt_release_devq(ccb_h->path, /*count*/1, + /*run_queue*/FALSE); + ccb_h->status &= ~CAM_DEV_QFRZN; + } - if (!device_is_queued(dev)) - (void)xpt_schedule_devq(sim->devq, dev); - mtx_unlock(&sim->devq->send_mtx); + if (ccb_h->flags & CAM_UNLOCKED) { + mtx_unlock(mtx); + mtx = NULL; + } - if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 - && (--dev->tag_delay_count == 0)) - xpt_start_tags(ccb_h->path); - } + /* Call the peripheral driver's callback */ + (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h); + if (mtx != NULL) + mtx_unlock(mtx); - if (ccb_h->status & CAM_RELEASE_SIMQ) { - xpt_release_simq(sim, /*run_queue*/FALSE); - ccb_h->status &= ~CAM_RELEASE_SIMQ; - } + mtx_lock(&sim->devq->send_mtx); + xpt_run_devq(sim->devq); + mtx_unlock(&sim->devq->send_mtx); +} - if ((ccb_h->flags & CAM_DEV_QFRZDIS) - && (ccb_h->status & CAM_DEV_QFRZN)) { - xpt_release_devq(ccb_h->path, /*count*/1, - /*run_queue*/FALSE); - ccb_h->status &= ~CAM_DEV_QFRZN; - } +void +xpt_done_td(void *arg) +{ + struct cam_doneq *queue = arg; + struct ccb_hdr *ccb_h; - if (ccb_h->flags & CAM_UNLOCKED) { - mtx_unlock(mtx); - mtx = NULL; + mtx_lock(&queue->cam_doneq_mtx); + while (1) { + if ((ccb_h = TAILQ_FIRST(&queue->cam_doneq)) == NULL) { + msleep(&queue->cam_doneq, &queue->cam_doneq_mtx, + PRIBIO, "-", 0); + continue; } + TAILQ_REMOVE(&queue->cam_doneq, ccb_h, sim_links.tqe); + mtx_unlock(&queue->cam_doneq_mtx); + xpt_done_process(ccb_h); + mtx_lock(&queue->cam_doneq_mtx); + } +} + +static void +camisr_runqueue(struct cam_sim *sim) +{ + struct ccb_hdr *ccb_h; + + mtx_lock(&sim->sim_doneq_mtx); + while ((ccb_h = TAILQ_FIRST(&sim->sim_doneq)) != NULL) { + TAILQ_REMOVE(&sim->sim_doneq, ccb_h, sim_links.tqe); + mtx_unlock(&sim->sim_doneq_mtx); + ccb_h->pinfo.index = CAM_UNQUEUED_INDEX; + + xpt_done_process(ccb_h); - /* Call the peripheral driver's callback */ - (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h); - if (mtx != NULL) - mtx_unlock(mtx); mtx_lock(&sim->sim_doneq_mtx); } sim->sim_doneq_flags &= ~CAM_SIM_DQ_ONQ; mtx_unlock(&sim->sim_doneq_mtx); - mtx_lock(&sim->devq->send_mtx); - xpt_run_devq(sim->devq); - mtx_unlock(&sim->devq->send_mtx); }help
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201308140636.r7E6aWEe030673>