From owner-freebsd-hackers@FreeBSD.ORG Tue Apr 3 21:27:44 2012 Return-Path: Delivered-To: freebsd-hackers@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id EEA641065670 for ; Tue, 3 Apr 2012 21:27:44 +0000 (UTC) (envelope-from jrytoung@gmail.com) Received: from mail-wi0-f172.google.com (mail-wi0-f172.google.com [209.85.212.172]) by mx1.freebsd.org (Postfix) with ESMTP id 79A248FC08 for ; Tue, 3 Apr 2012 21:27:44 +0000 (UTC) Received: by wibhj6 with SMTP id hj6so3313615wib.13 for ; Tue, 03 Apr 2012 14:27:43 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20120113; h=mime-version:in-reply-to:references:date:message-id:subject:from:to :cc:content-type; bh=SIK+bZ3SUelstjS/Pqr6V3fNYBxMlx4RKxdwrT36OpA=; b=LdjkOk1mXkz2WMLyMVb5Rtp3Hj7+jqJcABH8/6hlFCirbj7jJJvnNOlM1X2v/MNb15 Gv+BWAlEx2I4Jr2Mn6CGRcifg0uX8475a4PyfByMr0Qcz3mmXsYVe9lY1y2A6QldAc5R oirMHTvO4hmxcThIf5M0CVAQO/HPyHNU2UIjx/YYqNYtYoL24U+CZiqz5bmzi9IvO+Px PxoCCWBkZnHB8zQlipsofpVZ9CPy6Ey8c8uMTr3IpodRzAnNCk1xLtZkdYCn4N+wZnNr 7pJssh50YeDl6qDcLgKpJLPYz+2Y/9nT4V6SeA5EVrzkQoutLlpEscXhEgr3BApQdsPW +3uA== MIME-Version: 1.0 Received: by 10.180.24.66 with SMTP id s2mr40243820wif.7.1333488463331; Tue, 03 Apr 2012 14:27:43 -0700 (PDT) Received: by 10.216.27.148 with HTTP; Tue, 3 Apr 2012 14:27:43 -0700 (PDT) In-Reply-To: <20120403193124.46ad9de9@ernst.jennejohn.org> References: <20120403193124.46ad9de9@ernst.jennejohn.org> Date: Tue, 3 Apr 2012 14:27:43 -0700 Message-ID: From: Jerry Toung To: gljennjohn@googlemail.com Content-Type: text/plain; charset=ISO-8859-1 Cc: freebsd-hackers Subject: Re: CAM disk I/O starvation X-BeenThere: freebsd-hackers@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: Technical Discussions relating to FreeBSD List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 03 Apr 2012 21:27:45 -0000 On 4/3/12, Gary Jennejohn wrote: > It would be interesting to see your patch. I always run HEAD but maybe > I could use it as a base for my own mods/tests. > Here is the patch diff -rup cam/cam_sim.c cam/cam_sim.c --- cam/cam_sim.c 2010-06-13 19:09:06.000000000 -0700 +++ cam/cam_sim.c 2012-03-19 13:05:10.000000000 -0700 @@ -87,6 +87,7 @@ cam_sim_alloc(sim_action_func sim_action sim->refcount = 1; sim->devq = queue; sim->max_ccbs = 8; /* Reserve for management purposes. */ + sim->dev_count = 0; sim->mtx = mtx; if (mtx == &Giant) { sim->flags |= 0; diff -rup cam/cam_sim.h cam/cam_sim.h --- cam/cam_sim.h 2010-06-13 19:09:06.000000000 -0700 +++ cam/cam_sim.h 2012-03-19 15:34:17.000000000 -0700 @@ -118,6 +118,8 @@ struct cam_sim { u_int max_ccbs; /* Current count of allocated ccbs */ u_int ccb_count; + /* Number of peripheral drivers mapped to this sim */ + u_int dev_count; }; diff -rup cam/cam_xpt.c cam/cam_xpt.c --- cam/cam_xpt.c 2010-06-13 19:09:06.000000000 -0700 +++ cam/cam_xpt.c 2012-03-29 11:41:51.000000000 -0700 @@ -303,7 +303,7 @@ xpt_schedule_dev_allocq(struct cam_eb *b int retval; if ((dev->drvq.entries > 0) && - (dev->ccbq.devq_openings > 0) && + (dev->runs_token < dev->ccbq.queue.array_size) && (cam_ccbq_frozen(&dev->ccbq, CAM_PRIORITY_TO_RL( CAMQ_GET_PRIO(&dev->drvq))) == 0)) { /* @@ -327,7 +327,7 @@ xpt_schedule_dev_sendq(struct cam_eb *bu int retval; if ((dev->ccbq.queue.entries > 0) && - (dev->ccbq.dev_openings > 0) && + (dev->runs_token < dev->ccbq.queue.array_size) && (cam_ccbq_frozen_top(&dev->ccbq) == 0)) { /* * The priority of a device waiting for controller @@ -973,6 +973,9 @@ xpt_add_periph(struct cam_periph *periph struct cam_ed *device; int32_t status; struct periph_list *periph_head; + struct cam_eb *bus; + struct cam_et *target; + struct cam_ed *devptr; mtx_assert(periph->sim->mtx, MA_OWNED); @@ -991,6 +994,8 @@ xpt_add_periph(struct cam_periph *periph status = camq_resize(&device->drvq, device->drvq.array_size + 1); + if (periph->periph_name != NULL && strncmp(periph->periph_name, "da",2) ==0 ) + device->sim->dev_count++; device->generation++; SLIST_INSERT_HEAD(periph_head, periph, periph_links); @@ -998,6 +1003,24 @@ xpt_add_periph(struct cam_periph *periph mtx_lock(&xsoftc.xpt_topo_lock); xsoftc.xpt_generation++; + + if (device != NULL && device->sim->dev_count > 1 && + (device->sim->max_dev_openings > device->sim->dev_count)) { + TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) { + if (bus->sim != device->sim) + continue; + TAILQ_FOREACH(target, &bus->et_entries, links) { + TAILQ_FOREACH(devptr, &target->ed_entries, links) { + /* + * The number of openings/tags supported by the sim (i.e controller) + * is evenly distributed between all devices that share this sim. + */ + cam_ccbq_resize(&devptr->ccbq, + (devptr->sim->max_dev_openings/devptr->sim->dev_count)); + } + } + } + } mtx_unlock(&xsoftc.xpt_topo_lock); return (status); @@ -3072,6 +3095,11 @@ xpt_run_dev_allocq(struct cam_eb *bus) } /* We may have more work. Attempt to reschedule. */ + device->runs_token++; + if (device->runs_token >= device->ccbq.queue.array_size) { + device->runs_token = 0; + break; + } xpt_schedule_dev_allocq(bus, device); } devq->alloc_queue.qfrozen_cnt[0]--; @@ -3139,7 +3167,6 @@ xpt_run_dev_sendq(struct cam_eb *bus) devq->send_openings--; devq->send_active++; - xpt_schedule_dev_sendq(bus, device); if (work_ccb && (work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0){ /* @@ -3170,6 +3197,13 @@ xpt_run_dev_sendq(struct cam_eb *bus) */ sim = work_ccb->ccb_h.path->bus->sim; (*(sim->sim_action))(sim, work_ccb); + + device->runs_token++; + if (device->runs_token >= device->ccbq.queue.array_size) { + device->runs_token = 0; + break; + } + xpt_schedule_dev_sendq(bus, device); } devq->send_queue.qfrozen_cnt[0]--; } @@ -4285,6 +4319,7 @@ xpt_alloc_device(struct cam_eb *bus, str device->tag_delay_count = 0; device->tag_saved_openings = 0; device->refcount = 1; + device->runs_token = 0; callout_init_mtx(&device->callout, bus->sim->mtx, 0); /* diff -rup cam/cam_xpt_internal.h cam/cam_xpt_internal.h --- cam/cam_xpt_internal.h 2010-06-13 19:09:06.000000000 -0700 +++ cam/cam_xpt_internal.h 2012-03-21 13:57:45.000000000 -0700 @@ -118,6 +118,7 @@ struct cam_ed { #define CAM_TAG_DELAY_COUNT 5 u_int32_t tag_saved_openings; u_int32_t refcount; + u_int32_t runs_token; struct callout callout; }; diff -rup cam/scsi/scsi_da.c cam/scsi/scsi_da.c --- cam/scsi/scsi_da.c 2010-06-13 19:09:06.000000000 -0700 +++ cam/scsi/scsi_da.c 2012-03-21 14:16:00.000000000 -0700 @@ -56,7 +56,13 @@ __FBSDID("$FreeBSD: src/sys/cam/scsi/scs #include #include #include +#include #include +#include +#include +#include +#include +#include #include @@ -1102,6 +1108,26 @@ dasysctlinit(void *context, int pending) &softc->minimum_cmd_size, 0, dacmdsizesysctl, "I", "Minimum CDB size"); + SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "outstanding_cmds", CTLTYPE_INT | CTLFLAG_RD, + &softc->outstanding_cmds, 0, "Outstanding CCB Cmds"); + + SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "ccbq_devq_openings", CTLTYPE_INT | CTLFLAG_RD, + &periph->path->device->ccbq.devq_openings, 0, "CCBQ Dev Openings"); + + SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "ccbq_array_size", CTLTYPE_INT | CTLFLAG_RW, + &periph->path->device->ccbq.queue.array_size, 0, "CCBQ Array Size"); + + SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "sim_ccb_count", CTLTYPE_INT | CTLFLAG_RD, + &periph->sim->ccb_count, 0, "SIM CCB COUNT"); + + SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "sim_devq_alloc_openings", CTLTYPE_INT | CTLFLAG_RD, + &periph->sim->devq->alloc_openings, 0, "SIM Devq Alloc Openings"); + cam_periph_release(periph); }