Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 3 Apr 2012 14:27:43 -0700
From:      Jerry Toung <jrytoung@gmail.com>
To:        gljennjohn@googlemail.com
Cc:        freebsd-hackers <freebsd-hackers@freebsd.org>
Subject:   Re: CAM disk I/O starvation
Message-ID:  <CADC0LVm1HY2Dz%2BVk_GK35szRS6ySviLhMiL1TSRBOnPwQnBwRg@mail.gmail.com>
In-Reply-To: <20120403193124.46ad9de9@ernst.jennejohn.org>
References:  <CADC0LV=-e%2B7PshRQdc69e2-Vktf6XFpVrqiMpx=QL4m_%2B9hSnw@mail.gmail.com> <20120403193124.46ad9de9@ernst.jennejohn.org>

next in thread | previous in thread | raw e-mail | index | archive | help
On 4/3/12, Gary Jennejohn <gljennjohn@googlemail.com> wrote:

> It would be interesting to see your patch.  I always run HEAD but maybe
> I could use it as a base for my own mods/tests.
>

Here is the patch

diff -rup cam/cam_sim.c cam/cam_sim.c
--- cam/cam_sim.c       2010-06-13 19:09:06.000000000 -0700
+++ cam/cam_sim.c       2012-03-19 13:05:10.000000000 -0700
@@ -87,6 +87,7 @@ cam_sim_alloc(sim_action_func sim_action
        sim->refcount = 1;
        sim->devq = queue;
        sim->max_ccbs = 8;      /* Reserve for management purposes. */
+       sim->dev_count = 0;
        sim->mtx = mtx;
        if (mtx == &Giant) {
                sim->flags |= 0;
diff -rup cam/cam_sim.h cam/cam_sim.h
--- cam/cam_sim.h       2010-06-13 19:09:06.000000000 -0700
+++ cam/cam_sim.h       2012-03-19 15:34:17.000000000 -0700
@@ -118,6 +118,8 @@ struct cam_sim {
        u_int                   max_ccbs;
        /* Current count of allocated ccbs */
        u_int                   ccb_count;
+       /* Number of peripheral drivers mapped to this sim */
+       u_int                   dev_count;

 };

diff -rup cam/cam_xpt.c cam/cam_xpt.c
--- cam/cam_xpt.c       2010-06-13 19:09:06.000000000 -0700
+++ cam/cam_xpt.c       2012-03-29 11:41:51.000000000 -0700
@@ -303,7 +303,7 @@ xpt_schedule_dev_allocq(struct cam_eb *b
        int retval;

        if ((dev->drvq.entries > 0) &&
-           (dev->ccbq.devq_openings > 0) &&
+           (dev->runs_token < dev->ccbq.queue.array_size) &&
            (cam_ccbq_frozen(&dev->ccbq, CAM_PRIORITY_TO_RL(
                CAMQ_GET_PRIO(&dev->drvq))) == 0)) {
                /*
@@ -327,7 +327,7 @@ xpt_schedule_dev_sendq(struct cam_eb *bu
        int     retval;

        if ((dev->ccbq.queue.entries > 0) &&
-           (dev->ccbq.dev_openings > 0) &&
+           (dev->runs_token < dev->ccbq.queue.array_size) &&
            (cam_ccbq_frozen_top(&dev->ccbq) == 0)) {
                /*
                 * The priority of a device waiting for controller
@@ -973,6 +973,9 @@ xpt_add_periph(struct cam_periph *periph
        struct cam_ed *device;
        int32_t  status;
        struct periph_list *periph_head;
+       struct cam_eb *bus;
+       struct cam_et *target;
+       struct cam_ed *devptr;

        mtx_assert(periph->sim->mtx, MA_OWNED);

@@ -991,6 +994,8 @@ xpt_add_periph(struct cam_periph *periph
                status = camq_resize(&device->drvq,
                                     device->drvq.array_size + 1);

+               if (periph->periph_name != NULL &&
strncmp(periph->periph_name, "da",2) ==0 )
+                       device->sim->dev_count++;
                device->generation++;

                SLIST_INSERT_HEAD(periph_head, periph, periph_links);
@@ -998,6 +1003,24 @@ xpt_add_periph(struct cam_periph *periph

        mtx_lock(&xsoftc.xpt_topo_lock);
        xsoftc.xpt_generation++;
+
+       if (device != NULL && device->sim->dev_count > 1 &&
+            (device->sim->max_dev_openings > device->sim->dev_count)) {
+               TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
+                       if (bus->sim != device->sim)
+                               continue;
+                       TAILQ_FOREACH(target, &bus->et_entries, links) {
+                               TAILQ_FOREACH(devptr,
&target->ed_entries, links) {
+                               /*
+                                * The number of openings/tags
supported by the sim (i.e controller)
+                                * is evenly distributed between all
devices that share this sim.
+                                */
+                                       cam_ccbq_resize(&devptr->ccbq,
+
(devptr->sim->max_dev_openings/devptr->sim->dev_count));
+                                }
+                        }
+                }
+        }
        mtx_unlock(&xsoftc.xpt_topo_lock);

        return (status);
@@ -3072,6 +3095,11 @@ xpt_run_dev_allocq(struct cam_eb *bus)
                }

                /* We may have more work. Attempt to reschedule. */
+               device->runs_token++;
+               if (device->runs_token >= device->ccbq.queue.array_size) {
+                       device->runs_token = 0;
+                       break;
+               }
                xpt_schedule_dev_allocq(bus, device);
        }
        devq->alloc_queue.qfrozen_cnt[0]--;
@@ -3139,7 +3167,6 @@ xpt_run_dev_sendq(struct cam_eb *bus)
                devq->send_openings--;
                devq->send_active++;

-               xpt_schedule_dev_sendq(bus, device);

                if (work_ccb && (work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0){
                        /*
@@ -3170,6 +3197,13 @@ xpt_run_dev_sendq(struct cam_eb *bus)
                 */
                sim = work_ccb->ccb_h.path->bus->sim;
                (*(sim->sim_action))(sim, work_ccb);
+
+               device->runs_token++;
+               if (device->runs_token >= device->ccbq.queue.array_size) {
+                       device->runs_token = 0;
+                       break;
+               }
+               xpt_schedule_dev_sendq(bus, device);
        }
        devq->send_queue.qfrozen_cnt[0]--;
 }
@@ -4285,6 +4319,7 @@ xpt_alloc_device(struct cam_eb *bus, str
                device->tag_delay_count = 0;
                device->tag_saved_openings = 0;
                device->refcount = 1;
+               device->runs_token = 0;
                callout_init_mtx(&device->callout, bus->sim->mtx, 0);

                /*
diff -rup cam/cam_xpt_internal.h cam/cam_xpt_internal.h
--- cam/cam_xpt_internal.h      2010-06-13 19:09:06.000000000 -0700
+++ cam/cam_xpt_internal.h      2012-03-21 13:57:45.000000000 -0700
@@ -118,6 +118,7 @@ struct cam_ed {
 #define        CAM_TAG_DELAY_COUNT             5
        u_int32_t        tag_saved_openings;
        u_int32_t        refcount;
+       u_int32_t        runs_token;
        struct callout   callout;
 };

diff -rup cam/scsi/scsi_da.c cam/scsi/scsi_da.c
--- cam/scsi/scsi_da.c  2010-06-13 19:09:06.000000000 -0700
+++ cam/scsi/scsi_da.c  2012-03-21 14:16:00.000000000 -0700
@@ -56,7 +56,13 @@ __FBSDID("$FreeBSD: src/sys/cam/scsi/scs
 #include <cam/cam_ccb.h>
 #include <cam/cam_periph.h>
 #include <cam/cam_xpt_periph.h>
+#include <cam/cam_queue.h>
 #include <cam/cam_sim.h>
+#include <cam/cam_xpt.h>
+#include <cam/cam_xpt_sim.h>
+#include <cam/cam_xpt_periph.h>
+#include <cam/cam_xpt_internal.h>
+#include <cam/cam_debug.h>

 #include <cam/scsi/scsi_message.h>

@@ -1102,6 +1108,26 @@ dasysctlinit(void *context, int pending)
                &softc->minimum_cmd_size, 0, dacmdsizesysctl, "I",
                "Minimum CDB size");

+       SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+               OID_AUTO, "outstanding_cmds", CTLTYPE_INT | CTLFLAG_RD,
+               &softc->outstanding_cmds, 0, "Outstanding CCB Cmds");
+
+       SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+               OID_AUTO, "ccbq_devq_openings", CTLTYPE_INT | CTLFLAG_RD,
+               &periph->path->device->ccbq.devq_openings, 0, "CCBQ
Dev Openings");
+
+       SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+               OID_AUTO, "ccbq_array_size", CTLTYPE_INT | CTLFLAG_RW,
+               &periph->path->device->ccbq.queue.array_size, 0, "CCBQ
Array Size");
+
+       SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+               OID_AUTO, "sim_ccb_count", CTLTYPE_INT | CTLFLAG_RD,
+               &periph->sim->ccb_count, 0, "SIM CCB COUNT");
+
+       SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+               OID_AUTO, "sim_devq_alloc_openings", CTLTYPE_INT | CTLFLAG_RD,
+               &periph->sim->devq->alloc_openings, 0, "SIM Devq Alloc
Openings");
+
        cam_periph_release(periph);
 }



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?CADC0LVm1HY2Dz%2BVk_GK35szRS6ySviLhMiL1TSRBOnPwQnBwRg>