Date: Tue, 3 Apr 2012 14:27:43 -0700 From: Jerry Toung <jrytoung@gmail.com> To: gljennjohn@googlemail.com Cc: freebsd-hackers <freebsd-hackers@freebsd.org> Subject: Re: CAM disk I/O starvation Message-ID: <CADC0LVm1HY2Dz%2BVk_GK35szRS6ySviLhMiL1TSRBOnPwQnBwRg@mail.gmail.com> In-Reply-To: <20120403193124.46ad9de9@ernst.jennejohn.org> References: <CADC0LV=-e%2B7PshRQdc69e2-Vktf6XFpVrqiMpx=QL4m_%2B9hSnw@mail.gmail.com> <20120403193124.46ad9de9@ernst.jennejohn.org>
next in thread | previous in thread | raw e-mail | index | archive | help
On 4/3/12, Gary Jennejohn <gljennjohn@googlemail.com> wrote:
> It would be interesting to see your patch. I always run HEAD but maybe
> I could use it as a base for my own mods/tests.
>
Here is the patch
diff -rup cam/cam_sim.c cam/cam_sim.c
--- cam/cam_sim.c 2010-06-13 19:09:06.000000000 -0700
+++ cam/cam_sim.c 2012-03-19 13:05:10.000000000 -0700
@@ -87,6 +87,7 @@ cam_sim_alloc(sim_action_func sim_action
sim->refcount = 1;
sim->devq = queue;
sim->max_ccbs = 8; /* Reserve for management purposes. */
+ sim->dev_count = 0;
sim->mtx = mtx;
if (mtx == &Giant) {
sim->flags |= 0;
diff -rup cam/cam_sim.h cam/cam_sim.h
--- cam/cam_sim.h 2010-06-13 19:09:06.000000000 -0700
+++ cam/cam_sim.h 2012-03-19 15:34:17.000000000 -0700
@@ -118,6 +118,8 @@ struct cam_sim {
u_int max_ccbs;
/* Current count of allocated ccbs */
u_int ccb_count;
+ /* Number of peripheral drivers mapped to this sim */
+ u_int dev_count;
};
diff -rup cam/cam_xpt.c cam/cam_xpt.c
--- cam/cam_xpt.c 2010-06-13 19:09:06.000000000 -0700
+++ cam/cam_xpt.c 2012-03-29 11:41:51.000000000 -0700
@@ -303,7 +303,7 @@ xpt_schedule_dev_allocq(struct cam_eb *b
int retval;
if ((dev->drvq.entries > 0) &&
- (dev->ccbq.devq_openings > 0) &&
+ (dev->runs_token < dev->ccbq.queue.array_size) &&
(cam_ccbq_frozen(&dev->ccbq, CAM_PRIORITY_TO_RL(
CAMQ_GET_PRIO(&dev->drvq))) == 0)) {
/*
@@ -327,7 +327,7 @@ xpt_schedule_dev_sendq(struct cam_eb *bu
int retval;
if ((dev->ccbq.queue.entries > 0) &&
- (dev->ccbq.dev_openings > 0) &&
+ (dev->runs_token < dev->ccbq.queue.array_size) &&
(cam_ccbq_frozen_top(&dev->ccbq) == 0)) {
/*
* The priority of a device waiting for controller
@@ -973,6 +973,9 @@ xpt_add_periph(struct cam_periph *periph
struct cam_ed *device;
int32_t status;
struct periph_list *periph_head;
+ struct cam_eb *bus;
+ struct cam_et *target;
+ struct cam_ed *devptr;
mtx_assert(periph->sim->mtx, MA_OWNED);
@@ -991,6 +994,8 @@ xpt_add_periph(struct cam_periph *periph
status = camq_resize(&device->drvq,
device->drvq.array_size + 1);
+ if (periph->periph_name != NULL &&
strncmp(periph->periph_name, "da",2) ==0 )
+ device->sim->dev_count++;
device->generation++;
SLIST_INSERT_HEAD(periph_head, periph, periph_links);
@@ -998,6 +1003,24 @@ xpt_add_periph(struct cam_periph *periph
mtx_lock(&xsoftc.xpt_topo_lock);
xsoftc.xpt_generation++;
+
+ if (device != NULL && device->sim->dev_count > 1 &&
+ (device->sim->max_dev_openings > device->sim->dev_count)) {
+ TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
+ if (bus->sim != device->sim)
+ continue;
+ TAILQ_FOREACH(target, &bus->et_entries, links) {
+ TAILQ_FOREACH(devptr,
&target->ed_entries, links) {
+ /*
+ * The number of openings/tags
supported by the sim (i.e controller)
+ * is evenly distributed between all
devices that share this sim.
+ */
+ cam_ccbq_resize(&devptr->ccbq,
+
(devptr->sim->max_dev_openings/devptr->sim->dev_count));
+ }
+ }
+ }
+ }
mtx_unlock(&xsoftc.xpt_topo_lock);
return (status);
@@ -3072,6 +3095,11 @@ xpt_run_dev_allocq(struct cam_eb *bus)
}
/* We may have more work. Attempt to reschedule. */
+ device->runs_token++;
+ if (device->runs_token >= device->ccbq.queue.array_size) {
+ device->runs_token = 0;
+ break;
+ }
xpt_schedule_dev_allocq(bus, device);
}
devq->alloc_queue.qfrozen_cnt[0]--;
@@ -3139,7 +3167,6 @@ xpt_run_dev_sendq(struct cam_eb *bus)
devq->send_openings--;
devq->send_active++;
- xpt_schedule_dev_sendq(bus, device);
if (work_ccb && (work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0){
/*
@@ -3170,6 +3197,13 @@ xpt_run_dev_sendq(struct cam_eb *bus)
*/
sim = work_ccb->ccb_h.path->bus->sim;
(*(sim->sim_action))(sim, work_ccb);
+
+ device->runs_token++;
+ if (device->runs_token >= device->ccbq.queue.array_size) {
+ device->runs_token = 0;
+ break;
+ }
+ xpt_schedule_dev_sendq(bus, device);
}
devq->send_queue.qfrozen_cnt[0]--;
}
@@ -4285,6 +4319,7 @@ xpt_alloc_device(struct cam_eb *bus, str
device->tag_delay_count = 0;
device->tag_saved_openings = 0;
device->refcount = 1;
+ device->runs_token = 0;
callout_init_mtx(&device->callout, bus->sim->mtx, 0);
/*
diff -rup cam/cam_xpt_internal.h cam/cam_xpt_internal.h
--- cam/cam_xpt_internal.h 2010-06-13 19:09:06.000000000 -0700
+++ cam/cam_xpt_internal.h 2012-03-21 13:57:45.000000000 -0700
@@ -118,6 +118,7 @@ struct cam_ed {
#define CAM_TAG_DELAY_COUNT 5
u_int32_t tag_saved_openings;
u_int32_t refcount;
+ u_int32_t runs_token;
struct callout callout;
};
diff -rup cam/scsi/scsi_da.c cam/scsi/scsi_da.c
--- cam/scsi/scsi_da.c 2010-06-13 19:09:06.000000000 -0700
+++ cam/scsi/scsi_da.c 2012-03-21 14:16:00.000000000 -0700
@@ -56,7 +56,13 @@ __FBSDID("$FreeBSD: src/sys/cam/scsi/scs
#include <cam/cam_ccb.h>
#include <cam/cam_periph.h>
#include <cam/cam_xpt_periph.h>
+#include <cam/cam_queue.h>
#include <cam/cam_sim.h>
+#include <cam/cam_xpt.h>
+#include <cam/cam_xpt_sim.h>
+#include <cam/cam_xpt_periph.h>
+#include <cam/cam_xpt_internal.h>
+#include <cam/cam_debug.h>
#include <cam/scsi/scsi_message.h>
@@ -1102,6 +1108,26 @@ dasysctlinit(void *context, int pending)
&softc->minimum_cmd_size, 0, dacmdsizesysctl, "I",
"Minimum CDB size");
+ SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+ OID_AUTO, "outstanding_cmds", CTLTYPE_INT | CTLFLAG_RD,
+ &softc->outstanding_cmds, 0, "Outstanding CCB Cmds");
+
+ SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+ OID_AUTO, "ccbq_devq_openings", CTLTYPE_INT | CTLFLAG_RD,
+ &periph->path->device->ccbq.devq_openings, 0, "CCBQ
Dev Openings");
+
+ SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+ OID_AUTO, "ccbq_array_size", CTLTYPE_INT | CTLFLAG_RW,
+ &periph->path->device->ccbq.queue.array_size, 0, "CCBQ
Array Size");
+
+ SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+ OID_AUTO, "sim_ccb_count", CTLTYPE_INT | CTLFLAG_RD,
+ &periph->sim->ccb_count, 0, "SIM CCB COUNT");
+
+ SYSCTL_ADD_INT(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+ OID_AUTO, "sim_devq_alloc_openings", CTLTYPE_INT | CTLFLAG_RD,
+ &periph->sim->devq->alloc_openings, 0, "SIM Devq Alloc
Openings");
+
cam_periph_release(periph);
}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?CADC0LVm1HY2Dz%2BVk_GK35szRS6ySviLhMiL1TSRBOnPwQnBwRg>