Date: Fri, 6 Mar 2015 00:24:08 +0000 (UTC) From: Warner Losh <imp@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r279681 - in projects/iosched/sys: cam cam/ata cam/scsi conf Message-ID: <201503060024.t260O8aa077256@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: imp Date: Fri Mar 6 00:24:07 2015 New Revision: 279681 URL: https://svnweb.freebsd.org/changeset/base/279681 Log: Create a simple scheduling library for cam, and update da and ada to use this library. As this is common code between the two drivers, that's been merged, there are no functional changes. System stability is now the same with or without these patches. Added: projects/iosched/sys/cam/cam_iosched.c projects/iosched/sys/cam/cam_iosched.h Modified: projects/iosched/sys/cam/ata/ata_da.c projects/iosched/sys/cam/scsi/scsi_da.c projects/iosched/sys/conf/files Modified: projects/iosched/sys/cam/ata/ata_da.c ============================================================================== --- projects/iosched/sys/cam/ata/ata_da.c Fri Mar 6 00:23:52 2015 (r279680) +++ projects/iosched/sys/cam/ata/ata_da.c Fri Mar 6 00:24:07 2015 (r279681) @@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$"); #include <cam/cam_periph.h> #include <cam/cam_xpt_periph.h> #include <cam/cam_sim.h> +#include <cam/cam_iosched.h> #include <cam/ata/ata_all.h> @@ -68,6 +69,8 @@ __FBSDID("$FreeBSD$"); #define ATA_MAX_28BIT_LBA 268435455UL +extern int iosched_debug; + typedef enum { ADA_STATE_RAHEAD, ADA_STATE_WCACHE, @@ -147,17 +150,14 @@ struct trim_request { }; struct ada_softc { - struct bio_queue_head bio_queue; - struct bio_queue_head trim_queue; + struct cam_iosched_softc *cam_iosched; int outstanding_cmds; /* Number of active commands */ int refcount; /* Active xpt_action() calls */ ada_state state; ada_flags flags; ada_quirks quirks; - int sort_io_queue; ada_delete_methods delete_method; int trim_max_ranges; - int trim_running; int read_ahead; int write_cache; #ifdef ADA_TEST_FAILURE @@ -576,8 +576,6 @@ static void adaresume(void *arg); softc->read_ahead : ada_read_ahead) #define ADA_WC (softc->write_cache >= 0 ? \ softc->write_cache : ada_write_cache) -#define ADA_SIO (softc->sort_io_queue >= 0 ? \ - softc->sort_io_queue : cam_sort_io_queues) /* * Most platforms map firmware geometry to actual, but some don't. If @@ -736,11 +734,7 @@ adaschedule(struct cam_periph *periph) if (softc->state != ADA_STATE_NORMAL) return; - /* Check if we have more work to do. */ - if (bioq_first(&softc->bio_queue) || - (!softc->trim_running && bioq_first(&softc->trim_queue))) { - xpt_schedule(periph, CAM_PRIORITY_NORMAL); - } + cam_iosched_schedule(softc->cam_iosched, periph); } /* @@ -778,13 +772,8 @@ adastrategy(struct bio *bp) ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT)), ("BIO_DELETE but no supported TRIM method.")); - bioq_disksort(&softc->trim_queue, bp); - } else { - if (ADA_SIO) - bioq_disksort(&softc->bio_queue, bp); - else - bioq_insert_tail(&softc->bio_queue, bp); } + cam_iosched_queue_work(softc->cam_iosched, bp); /* * Schedule ourselves for performing the work. @@ -945,8 +934,7 @@ adaoninvalidate(struct cam_periph *perip * XXX Handle any transactions queued to the card * with XPT_ABORT_CCB. */ - bioq_flush(&softc->bio_queue, NULL, ENXIO); - bioq_flush(&softc->trim_queue, NULL, ENXIO); + cam_iosched_flush(softc->cam_iosched, NULL, ENXIO); disk_gone(softc->disk); } @@ -1166,10 +1154,6 @@ adasysctlinit(void *context, int pending SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE, &softc->write_cache, 0, "Enable disk write cache."); - SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), - OID_AUTO, "sort_io_queue", CTLFLAG_RW | CTLFLAG_MPSAFE, - &softc->sort_io_queue, 0, - "Sort IO queue to try and optimise disk access patterns"); #ifdef ADA_TEST_FAILURE /* * Add a 'door bell' sysctl which allows one to set it from userland @@ -1189,6 +1173,9 @@ adasysctlinit(void *context, int pending &softc->periodic_read_error, 0, "Force a read error every N reads (don't set too low)."); #endif + cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx, + softc->sysctl_tree); + cam_periph_release(periph); } @@ -1272,8 +1259,11 @@ adaregister(struct cam_periph *periph, v return(CAM_REQ_CMP_ERR); } - bioq_init(&softc->bio_queue); - bioq_init(&softc->trim_queue); + if (cam_iosched_init(&softc->cam_iosched) != 0) { + printf("adaregister: Unable to probe new device. " + "Unable to allocate iosched memory\n"); + return(CAM_REQ_CMP_ERR); + } if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) && (cgd->inq_flags & SID_DMA)) @@ -1345,10 +1335,8 @@ adaregister(struct cam_periph *periph, v "kern.cam.ada.%d.write_cache", periph->unit_number); TUNABLE_INT_FETCH(announce_buf, &softc->write_cache); /* Disable queue sorting for non-rotational media by default. */ - if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) - softc->sort_io_queue = 0; - else - softc->sort_io_queue = -1; + cam_iosched_set_sort_queue(softc->cam_iosched, + cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING); adagetparams(periph, cgd); softc->disk = disk_alloc(); softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate; @@ -1536,8 +1524,6 @@ ada_dsmtrim_req_create(struct ada_softc uint64_t lba = bp->bio_pblkno; int count = bp->bio_bcount / softc->params.secsize; - bioq_remove(&softc->trim_queue, bp); - /* Try to extend the previous range. */ if (lba == lastlba) { c = min(count, ATA_DSM_RANGE_MAX - lastcount); @@ -1573,11 +1559,15 @@ ada_dsmtrim_req_create(struct ada_softc } lastlba = lba; TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue); - bp = bioq_first(&softc->trim_queue); - if (bp == NULL || - bp->bio_bcount / softc->params.secsize > - (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) + + bp = cam_iosched_next_trim(softc->cam_iosched); + if (bp == NULL) + break; + if (bp->bio_bcount / softc->params.secsize > + (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) { + cam_iosched_put_back_trim(softc->cam_iosched, bp); break; + } } while (1); return (ranges); @@ -1637,7 +1627,6 @@ ada_cfaerase(struct ada_softc *softc, st bzero(req, sizeof(*req)); TAILQ_INIT(&req->bps); - bioq_remove(&softc->trim_queue, bp); TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue); cam_fill_ataio(ataio, @@ -1668,37 +1657,15 @@ adastart(struct cam_periph *periph, unio struct bio *bp; u_int8_t tag_code; - /* Run TRIM if not running yet. */ - if (!softc->trim_running && - (bp = bioq_first(&softc->trim_queue)) != 0) { - switch (softc->delete_method) { - case ADA_DELETE_NCQ_DSM_TRIM: - ada_ncq_dsmtrim(softc, bp, ataio); - break; - case ADA_DELETE_DSM_TRIM: - ada_dsmtrim(softc, bp, ataio); - break; - case ADA_DELETE_CFA_ERASE: - ada_cfaerase(softc, bp, ataio); - break; - default: - panic("adastart: BIO_DELETE without method, not possible."); - } - softc->trim_running = 1; - start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM; - start_ccb->ccb_h.flags |= CAM_UNLOCKED; - goto out; - } - /* Run regular command. */ - bp = bioq_first(&softc->bio_queue); + bp = cam_iosched_next_bio(softc->cam_iosched); if (bp == NULL) { xpt_release_ccb(start_ccb); break; } - bioq_remove(&softc->bio_queue, bp); - if ((bp->bio_flags & BIO_ORDERED) != 0 - || (softc->flags & ADA_FLAG_NEED_OTAG) != 0) { + if ((bp->bio_flags & BIO_ORDERED) != 0 || + (bp->bio_cmd != BIO_DELETE && + (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) { softc->flags &= ~ADA_FLAG_NEED_OTAG; softc->flags |= ADA_FLAG_WAS_OTAG; tag_code = 0; @@ -1819,6 +1786,24 @@ adastart(struct cam_periph *periph, unio } break; } + case BIO_DELETE: + switch (softc->delete_method) { + case ADA_DELETE_NCQ_DSM_TRIM: + ada_ncq_dsmtrim(softc, bp, ataio); + break; + case ADA_DELETE_DSM_TRIM: + ada_dsmtrim(softc, bp, ataio); + break; + case ADA_DELETE_CFA_ERASE: + ada_cfaerase(softc, bp, ataio); + break; + default: + panic("adastart: BIO_DELETE without method, not possible."); + } + start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM; + start_ccb->ccb_h.flags |= CAM_UNLOCKED; + cam_iosched_submit_trim(softc->cam_iosched); + goto out; case BIO_FLUSH: cam_fill_ataio(ataio, 1, @@ -1899,7 +1884,7 @@ adadone(struct cam_periph *periph, union case ADA_CCB_TRIM: { struct bio *bp; - int error; + int error, need_sched; cam_periph_lock(periph); bp = (struct bio *)done_ccb->ccb_h.ccb_bp; @@ -1950,6 +1935,8 @@ adadone(struct cam_periph *periph, union softc->outstanding_cmds--; if (softc->outstanding_cmds == 0) softc->flags |= ADA_FLAG_WAS_OTAG; + + need_sched = cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb); xpt_release_ccb(done_ccb); if (state == ADA_CCB_TRIM) { TAILQ_HEAD(, bio) queue; @@ -1961,13 +1948,13 @@ adadone(struct cam_periph *periph, union * Normally, the xpt_release_ccb() above would make sure * that when we have more work to do, that work would * get kicked off. However, we specifically keep - * trim_running set to 0 before the call above to allow + * trim running set to 0 before the call above to allow * other I/O to progress when many BIO_DELETE requests - * are pushed down. We set trim_running to 0 and call + * are pushed down. We set trim running to 0 and call * daschedule again so that we don't stall if there are * no other I/Os pending apart from BIO_DELETEs. */ - softc->trim_running = 0; + cam_iosched_trim_done(softc->cam_iosched); adaschedule(periph); cam_periph_unlock(periph); while ((bp1 = TAILQ_FIRST(&queue)) != NULL) { @@ -1981,6 +1968,8 @@ adadone(struct cam_periph *periph, union biodone(bp1); } } else { + if (need_sched) + adaschedule(periph); cam_periph_unlock(periph); biodone(bp); } Added: projects/iosched/sys/cam/cam_iosched.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/iosched/sys/cam/cam_iosched.c Fri Mar 6 00:24:07 2015 (r279681) @@ -0,0 +1,326 @@ +/*- + * CAM IO Scheduler Interface + * + * Copyright (c) 2015 Netflix, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification, immediately at the beginning of the file. + * 2. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> + +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/bio.h> +#include <sys/lock.h> +#include <sys/malloc.h> +#include <sys/mutex.h> +#include <sys/sysctl.h> + +#include <cam/cam.h> +#include <cam/cam_ccb.h> +#include <cam/cam_periph.h> +#include <cam/cam_xpt_periph.h> +#include <cam/cam_iosched.h> + +static MALLOC_DEFINE(M_CAMSCHED, "CAM I/O Scheduler", "CAM I/O Scheduler buffers"); + +/* + * Default I/O scheduler for FreeBSD. This implementation is just a thin-vineer + * over the bioq_* interface, with notions of separate calls for normal I/O and + * for trims. + */ + +struct cam_iosched_softc +{ + struct bio_queue_head bio_queue; + struct bio_queue_head trim_queue; + /* scheduler flags < 16, user flags >= 16 */ + uint32_t flags; + int sort_io_queue; +}; + + /* Trim or similar currently pending completion */ +#define CAM_IOSCHED_FLAG_TRIM_ACTIVE 1 + + /* Periph drivers set these flags to indicate work */ +#define CAM_IOSCHED_FLAG_WORK_FLAGS ((0xffffu) << 16) + +static inline int +cam_iosched_has_flagged_work(struct cam_iosched_softc *isc) +{ + return !!(isc->flags & CAM_IOSCHED_FLAG_WORK_FLAGS); +} + +static inline int +cam_iosched_has_io(struct cam_iosched_softc *isc) +{ + return bioq_first(&(isc)->bio_queue) != NULL; +} + +static inline int +cam_iosched_has_more_trim(struct cam_iosched_softc *isc) +{ + return !(isc->flags & CAM_IOSCHED_FLAG_TRIM_ACTIVE) && + bioq_first(&isc->trim_queue); +} + +#define cam_iosched_sort_queue(isc) ((isc)->sort_io_queue >= 0 ? \ + (isc)->sort_io_queue : cam_sort_io_queues) + + +static inline int +cam_iosched_has_work(struct cam_iosched_softc *isc) +{ + return cam_iosched_has_io(isc) || + cam_iosched_has_more_trim(isc) || + cam_iosched_has_flagged_work(isc); +} + +/* + * Allocate the iosched structure. This also insulates callers from knowing + * sizeof struct cam_iosched_softc. + */ +int +cam_iosched_init(struct cam_iosched_softc **iscp) +{ + + *iscp = malloc(sizeof(**iscp), M_CAMSCHED, M_NOWAIT | M_ZERO); + if (*iscp == NULL) + return ENOMEM; + (*iscp)->sort_io_queue = -1; + bioq_init(&(*iscp)->bio_queue); + bioq_init(&(*iscp)->trim_queue); + + return 0; +} + +/* + * Reclaim all used resources. This assumes that other folks have + * drained the requests in the hardware. Maybe an unwise assumption. + */ +void +cam_iosched_fini(struct cam_iosched_softc *isc) +{ + cam_iosched_flush(isc, NULL, ENXIO); + free(isc, M_CAMSCHED); +} + +/* + * After we're sure we're attaching a device, go ahead and add + * hooks for any sysctl we may wish to honor. + */ +void cam_iosched_sysctl_init(struct cam_iosched_softc *isc, + struct sysctl_ctx_list *ctx, struct sysctl_oid *node) +{ + SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(node), + OID_AUTO, "sort_io_queue", CTLFLAG_RW | CTLFLAG_MPSAFE, + &isc->sort_io_queue, 0, + "Sort IO queue to try and optimise disk access patterns"); +} + +/* + * Flush outstanding I/O. Consumers of this library don't know all the + * queues we may keep, so this allows all I/O to be flushed in one + * convenient call. + */ +void +cam_iosched_flush(struct cam_iosched_softc *isc, struct devstat *stp, int err) +{ + bioq_flush(&isc->bio_queue, stp, err); + bioq_flush(&isc->trim_queue, stp, err); +} + +/* + * Put back a trim that you weren't able to actually schedule this time. + */ +void +cam_iosched_put_back_trim(struct cam_iosched_softc *isc, struct bio *bp) +{ + bioq_insert_head(&isc->trim_queue, bp); +} + +/* + * gets the next trim from the trim queue. + * + * Assumes we're called with the periph lock held. It removes this + * trim from the queue and the device must explicitly reinstert it + * should the need arise. + */ +struct bio * +cam_iosched_next_trim(struct cam_iosched_softc *isc) +{ + struct bio *bp; + + bp = bioq_first(&isc->trim_queue); + if (bp == NULL) + return NULL; + bioq_remove(&isc->trim_queue, bp); + return bp; +} + +/* + * gets the an available trim from the trim queue, if there's no trim + * already pending. It removes this trim from the queue and the device + * must explicitly reinstert it should the need arise. + * + * Assumes we're called with the periph lock held. + */ +struct bio * +cam_iosched_get_trim(struct cam_iosched_softc *isc) +{ + + if (!cam_iosched_has_more_trim(isc)) + return NULL; + + return cam_iosched_next_trim(isc); +} + +/* + * Determine what the next bit of work to do is for the periph. The + * default implementation looks to see if we have trims to do, but no + * trims outstanding. If so, we do that. Otherwise we see if we have + * other work. If we do, then we do that. Otherwise why were we called? + */ +struct bio * +cam_iosched_next_bio(struct cam_iosched_softc *isc) +{ + struct bio *bp; + + /* + * First, see if we have a trim that can be scheduled. We can only send one + * at a time down, so this takes that into account. + */ + if ((bp = cam_iosched_get_trim(isc)) != NULL) + return bp; + + /* + * next, see if there's a normal I/O waiting. If so return that. + */ + if ((bp = bioq_first(&isc->bio_queue)) != NULL) { + bioq_remove(&isc->bio_queue, bp); + return bp; + } + + /* + * Otherwise, we got nada, so return that + */ + return NULL; +} + +/* + * Driver has been given some work to do by the block layer. Tell the + * scheduler about it and have it queue the work up. The scheduler module + * will then return the currently most useful bit of work later, possibly + * deferring work for various reasons. + */ +void +cam_iosched_queue_work(struct cam_iosched_softc *isc, struct bio *bp) +{ + + /* + * Put all trims on the trim queue sorted, since we know + * that the collapsing code requires this. Otherwise put + * the work on the bio queue. + */ + if (bp->bio_cmd == BIO_DELETE) + bioq_disksort(&isc->trim_queue, bp); + else if (cam_iosched_sort_queue(isc)) + bioq_disksort(&isc->bio_queue, bp); + else + bioq_insert_tail(&isc->bio_queue, bp); +} + +/* + * If we have work, get it scheduled. Called with the periph lock held. + */ +void +cam_iosched_schedule(struct cam_iosched_softc *isc, struct cam_periph *periph) +{ + + if (cam_iosched_has_work(isc)) + xpt_schedule(periph, CAM_PRIORITY_NORMAL); +} + +/* + * Complete a trim request + */ +void +cam_iosched_trim_done(struct cam_iosched_softc *isc) +{ + + isc->flags &= ~CAM_IOSCHED_FLAG_TRIM_ACTIVE; +} + +/* + * Complete a bio. Called before we release the ccb with xpt_release_ccb so we + * might use notes in the ccb for statistics. + */ +int +cam_iosched_bio_complete(struct cam_iosched_softc *isc, struct bio *bp, union ccb *done_ccb) +{ + return 0; +} + +/* + * Tell the io scheduler that you've pushed a trim down into the sim. + * xxx better place for this? + */ +void +cam_iosched_submit_trim(struct cam_iosched_softc *isc) +{ + + isc->flags |= CAM_IOSCHED_FLAG_TRIM_ACTIVE; +} + +/* + * Change the sorting policy hint for I/O transactions for this device. + */ +void +cam_iosched_set_sort_queue(struct cam_iosched_softc *isc, int val) +{ + + isc->sort_io_queue = val; +} + +int +cam_iosched_has_work_flags(struct cam_iosched_softc *isc, uint32_t flags) +{ + return isc->flags & flags; +} + +void +cam_iosched_set_work_flags(struct cam_iosched_softc *isc, uint32_t flags) +{ + isc->flags |= flags; +} + +void +cam_iosched_clr_work_flags(struct cam_iosched_softc *isc, uint32_t flags) +{ + isc->flags &= ~flags; +} Added: projects/iosched/sys/cam/cam_iosched.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/iosched/sys/cam/cam_iosched.h Fri Mar 6 00:24:07 2015 (r279681) @@ -0,0 +1,64 @@ +/*- + * CAM IO Scheduler Interface + * + * Copyright (c) 2015 Netflix, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification, immediately at the beginning of the file. + * 2. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _CAM_CAM_IOSCHED_H +#define _CAM_CAM_IOSCHED_H + +/* No user-servicable parts in here. */ +#ifdef _KERNEL + +/* Forward declare all structs to keep interface thin */ +struct cam_iosched_softc; +struct sysctl_ctx_list; +struct sysctl_oid; +union ccb; +struct bio; + +int cam_iosched_init(struct cam_iosched_softc **); +void cam_iosched_fini(struct cam_iosched_softc *); +void cam_iosched_sysctl_init(struct cam_iosched_softc *, struct sysctl_ctx_list *, struct sysctl_oid *); +struct bio *cam_iosched_next_trim(struct cam_iosched_softc *isc); +struct bio *cam_iosched_get_trim(struct cam_iosched_softc *isc); +struct bio *cam_iosched_next_bio(struct cam_iosched_softc *isc); +void cam_iosched_queue_work(struct cam_iosched_softc *isc, struct bio *bp); +void cam_iosched_flush(struct cam_iosched_softc *isc, struct devstat *stp, int err); +void cam_iosched_schedule(struct cam_iosched_softc *isc, struct cam_periph *periph); +void cam_iosched_finish_trim(struct cam_iosched_softc *isc); +void cam_iosched_submit_trim(struct cam_iosched_softc *isc); +void cam_iosched_put_back_trim(struct cam_iosched_softc *isc, struct bio *bp); +void cam_iosched_set_sort_queue(struct cam_iosched_softc *isc, int val); +int cam_iosched_has_work_flags(struct cam_iosched_softc *isc, uint32_t flags); +void cam_iosched_set_work_flags(struct cam_iosched_softc *isc, uint32_t flags); +void cam_iosched_clr_work_flags(struct cam_iosched_softc *isc, uint32_t flags); +void cam_iosched_trim_done(struct cam_iosched_softc *isc); +int cam_iosched_bio_complete(struct cam_iosched_softc *isc, struct bio *bp, union ccb *done_ccb); + +#endif +#endif Modified: projects/iosched/sys/cam/scsi/scsi_da.c ============================================================================== --- projects/iosched/sys/cam/scsi/scsi_da.c Fri Mar 6 00:23:52 2015 (r279680) +++ projects/iosched/sys/cam/scsi/scsi_da.c Fri Mar 6 00:24:07 2015 (r279681) @@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$"); #include <cam/cam_periph.h> #include <cam/cam_xpt_periph.h> #include <cam/cam_sim.h> +#include <cam/cam_iosched.h> #include <cam/scsi/scsi_message.h> @@ -199,21 +200,19 @@ struct disk_params { #define ATA_TRIM_MAX_RANGES ((UNMAP_BUF_SIZE / \ (ATA_DSM_RANGE_SIZE * ATA_DSM_BLK_SIZE)) * ATA_DSM_BLK_SIZE) +#define DA_WORK_TUR (1 << 16) + struct da_softc { - struct bio_queue_head bio_queue; - struct bio_queue_head delete_queue; + struct cam_iosched_softc *cam_iosched; struct bio_queue_head delete_run_queue; LIST_HEAD(, ccb_hdr) pending_ccbs; - int tur; /* TEST UNIT READY should be sent */ int refcount; /* Active xpt_action() calls */ da_state state; da_flags flags; da_quirks quirks; - int sort_io_queue; int minimum_cmd_size; int error_inject; int trim_max_ranges; - int delete_running; int delete_available; /* Delete methods possibly available */ u_int maxio; uint32_t unmap_max_ranges; @@ -1228,9 +1227,6 @@ static timeout_t damediapoll; #define DA_DEFAULT_SEND_ORDERED 1 #endif -#define DA_SIO (softc->sort_io_queue >= 0 ? \ - softc->sort_io_queue : cam_sort_io_queues) - static int da_poll_period = DA_DEFAULT_POLL_PERIOD; static int da_retry_count = DA_DEFAULT_RETRY; static int da_default_timeout = DA_DEFAULT_TIMEOUT; @@ -1390,12 +1386,7 @@ daschedule(struct cam_periph *periph) if (softc->state != DA_STATE_NORMAL) return; - /* Check if we have more work to do. */ - if (bioq_first(&softc->bio_queue) || - (!softc->delete_running && bioq_first(&softc->delete_queue)) || - softc->tur) { - xpt_schedule(periph, CAM_PRIORITY_NORMAL); - } + cam_iosched_schedule(softc->cam_iosched, periph); } /* @@ -1428,13 +1419,7 @@ dastrategy(struct bio *bp) /* * Place it in the queue of disk activities for this disk */ - if (bp->bio_cmd == BIO_DELETE) { - bioq_disksort(&softc->delete_queue, bp); - } else if (DA_SIO) { - bioq_disksort(&softc->bio_queue, bp); - } else { - bioq_insert_tail(&softc->bio_queue, bp); - } + cam_iosched_queue_work(softc->cam_iosched, bp); /* * Schedule ourselves for performing the work. @@ -1595,8 +1580,7 @@ daoninvalidate(struct cam_periph *periph * XXX Handle any transactions queued to the card * with XPT_ABORT_CCB. */ - bioq_flush(&softc->bio_queue, NULL, ENXIO); - bioq_flush(&softc->delete_queue, NULL, ENXIO); + cam_iosched_flush(softc->cam_iosched, NULL, ENXIO); /* * Tell GEOM that we've gone away, we'll get a callback when it is @@ -1721,9 +1705,9 @@ daasync(void *callback_arg, u_int32_t co } case AC_SCSI_AEN: softc = (struct da_softc *)periph->softc; - if (!softc->tur) { + if (!cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR)) { if (cam_periph_acquire(periph) == CAM_REQ_CMP) { - softc->tur = 1; + cam_iosched_set_work_flags(softc->cam_iosched, DA_WORK_TUR); daschedule(periph); } } @@ -1797,9 +1781,6 @@ dasysctlinit(void *context, int pending) OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW, &softc->minimum_cmd_size, 0, dacmdsizesysctl, "I", "Minimum CDB size"); - SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), - OID_AUTO, "sort_io_queue", CTLFLAG_RW, &softc->sort_io_queue, 0, - "Sort IO queue to try and optimise disk access patterns"); SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), @@ -1835,6 +1816,10 @@ dasysctlinit(void *context, int pending) &softc->wwpn, "World Wide Port Name"); } } + + cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx, + softc->sysctl_tree); + cam_periph_release(periph); } @@ -1897,7 +1882,6 @@ static void dadeletemethodset(struct da_softc *softc, da_delete_methods delete_method) { - softc->delete_method = delete_method; softc->disk->d_delmaxsize = dadeletemaxsize(softc, delete_method); softc->delete_func = da_delete_functions[delete_method]; @@ -2065,14 +2049,18 @@ daregister(struct cam_periph *periph, vo if (softc == NULL) { printf("daregister: Unable to probe new device. " - "Unable to allocate softc\n"); + "Unable to allocate softc\n"); return(CAM_REQ_CMP_ERR); } + if (cam_iosched_init(&softc->cam_iosched) != 0) { + printf("daregister: Unable to probe new device. " + "Unable to allocate iosched memory\n"); + return(CAM_REQ_CMP_ERR); + } + LIST_INIT(&softc->pending_ccbs); softc->state = DA_STATE_PROBE_RC; - bioq_init(&softc->bio_queue); - bioq_init(&softc->delete_queue); bioq_init(&softc->delete_run_queue); if (SID_IS_REMOVABLE(&cgd->inq_data)) softc->flags |= DA_FLAG_PACK_REMOVABLE; @@ -2080,7 +2068,6 @@ daregister(struct cam_periph *periph, vo softc->unmap_max_lba = UNMAP_RANGE_MAX; softc->ws_max_blks = WS16_MAX_BLKS; softc->trim_max_ranges = ATA_TRIM_MAX_RANGES; - softc->sort_io_queue = -1; periph->softc = softc; @@ -2267,23 +2254,11 @@ skipstate: struct bio *bp; uint8_t tag_code; - /* Run BIO_DELETE if not running yet. */ - if (!softc->delete_running && - (bp = bioq_first(&softc->delete_queue)) != NULL) { - if (softc->delete_func != NULL) { - softc->delete_func(periph, start_ccb, bp); - goto out; - } else { - bioq_flush(&softc->delete_queue, NULL, 0); - /* FALLTHROUGH */ - } - } - - /* Run regular command. */ - bp = bioq_takefirst(&softc->bio_queue); +more: + bp = cam_iosched_next_bio(softc->cam_iosched); if (bp == NULL) { - if (softc->tur) { - softc->tur = 0; + if (cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR)) { + cam_iosched_clr_work_flags(softc->cam_iosched, DA_WORK_TUR); scsi_test_unit_ready(&start_ccb->csio, /*retries*/ da_retry_count, dadone, @@ -2297,9 +2272,21 @@ skipstate: xpt_release_ccb(start_ccb); break; } - if (softc->tur) { - softc->tur = 0; - cam_periph_release_locked(periph); + + if (bp->bio_cmd == BIO_DELETE) { + if (softc->delete_func != NULL) { + softc->delete_func(periph, start_ccb, bp); + goto out; + } else { + /* Not sure this is possible, but failsafe by lying and saying "sure, done." */ + biofinish(bp, NULL, 0); + goto more; + } + } + + if (cam_iosched_has_work_flags(softc->cam_iosched, DA_WORK_TUR)) { + cam_iosched_clr_work_flags(softc->cam_iosched, DA_WORK_TUR); + cam_periph_release_locked(periph); /* XXX is this still valid? I think so but unverified */ } if ((bp->bio_flags & BIO_ORDERED) != 0 || @@ -2604,11 +2591,19 @@ da_delete_unmap(struct cam_periph *perip * fewer LBA's than requested. */ - softc->delete_running = 1; bzero(softc->unmap_buf, sizeof(softc->unmap_buf)); bp1 = bp; do { - bioq_remove(&softc->delete_queue, bp1); + /* + * Note: ada and da are different in how they store the + * pending bp's in a trim. ada stores all of them in the + * trim_req.bps. da stores all but the first one in the + * delete_run_queue. ada then completes all the bps in + * its adadone() loop. da completes all the bps in the + * delete_run_queue in dadone, and relies on the biodone + * after to complete. This should be reconciled since there's + * no real reason to do it differently. XXX + */ if (bp1 != bp) bioq_insert_tail(&softc->delete_run_queue, bp1); lba = bp1->bio_pblkno; @@ -2648,11 +2643,15 @@ da_delete_unmap(struct cam_periph *perip lastcount = c; } lastlba = lba; - bp1 = bioq_first(&softc->delete_queue); - if (bp1 == NULL || ranges >= softc->unmap_max_ranges || + bp1 = cam_iosched_next_trim(softc->cam_iosched); + if (bp1 == NULL) + break; + if (ranges >= softc->unmap_max_ranges || totalcount + bp1->bio_bcount / - softc->params.secsize > softc->unmap_max_lba) + softc->params.secsize > softc->unmap_max_lba) { + cam_iosched_put_back_trim(softc->cam_iosched, bp1); break; + } } while (1); scsi_ulto2b(ranges * 16 + 6, &buf[0]); scsi_ulto2b(ranges * 16, &buf[2]); @@ -2668,6 +2667,7 @@ da_delete_unmap(struct cam_periph *perip da_default_timeout * 1000); ccb->ccb_h.ccb_state = DA_CCB_DELETE; ccb->ccb_h.flags |= CAM_UNLOCKED; + cam_iosched_submit_trim(softc->cam_iosched); } static void @@ -2682,12 +2682,10 @@ da_delete_trim(struct cam_periph *periph uint32_t lastcount = 0, c, requestcount; int ranges = 0, off, block_count; - softc->delete_running = 1; bzero(softc->unmap_buf, sizeof(softc->unmap_buf)); bp1 = bp; do { - bioq_remove(&softc->delete_queue, bp1); - if (bp1 != bp) + if (bp1 != bp)//XXX imp XXX bioq_insert_tail(&softc->delete_run_queue, bp1); lba = bp1->bio_pblkno; count = bp1->bio_bcount / softc->params.secsize; @@ -2731,10 +2729,14 @@ da_delete_trim(struct cam_periph *periph } } lastlba = lba; - bp1 = bioq_first(&softc->delete_queue); - if (bp1 == NULL || bp1->bio_bcount / softc->params.secsize > - (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) + bp1 = cam_iosched_next_trim(softc->cam_iosched); + if (bp1 == NULL) + break; + if (bp1->bio_bcount / softc->params.secsize > + (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) { + cam_iosched_put_back_trim(softc->cam_iosched, bp1); break; + } } while (1); block_count = (ranges + ATA_DSM_BLK_RANGES - 1) / ATA_DSM_BLK_RANGES; @@ -2749,6 +2751,7 @@ da_delete_trim(struct cam_periph *periph da_default_timeout * 1000); ccb->ccb_h.ccb_state = DA_CCB_DELETE; ccb->ccb_h.flags |= CAM_UNLOCKED; + cam_iosched_submit_trim(softc->cam_iosched); } /* @@ -2767,13 +2770,11 @@ da_delete_ws(struct cam_periph *periph, softc = (struct da_softc *)periph->softc; ws_max_blks = softc->disk->d_delmaxsize / softc->params.secsize; - softc->delete_running = 1; lba = bp->bio_pblkno; count = 0; bp1 = bp; do { - bioq_remove(&softc->delete_queue, bp1); - if (bp1 != bp) + if (bp1 != bp)//XXX imp XXX bioq_insert_tail(&softc->delete_run_queue, bp1); count += bp1->bio_bcount / softc->params.secsize; if (count > ws_max_blks) { @@ -2784,11 +2785,15 @@ da_delete_ws(struct cam_periph *periph, count = omin(count, ws_max_blks); break; } - bp1 = bioq_first(&softc->delete_queue); - if (bp1 == NULL || lba + count != bp1->bio_pblkno || + bp1 = cam_iosched_next_trim(softc->cam_iosched); + if (bp1 == NULL) + break; + if (lba + count != bp1->bio_pblkno || count + bp1->bio_bcount / *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201503060024.t260O8aa077256>