From owner-svn-src-all@freebsd.org Mon Oct 5 08:59:20 2015 Return-Path: Delivered-To: svn-src-all@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 0B392A10338; Mon, 5 Oct 2015 08:59:20 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id F0812D40; Mon, 5 Oct 2015 08:59:19 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.70]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id t958xJqP019046; Mon, 5 Oct 2015 08:59:19 GMT (envelope-from mav@FreeBSD.org) Received: (from mav@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id t958xJ2S019044; Mon, 5 Oct 2015 08:59:19 GMT (envelope-from mav@FreeBSD.org) Message-Id: <201510050859.t958xJ2S019044@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mav set sender to mav@FreeBSD.org using -f From: Alexander Motin Date: Mon, 5 Oct 2015 08:59:19 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r288734 - stable/10/sys/cam/ctl X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 05 Oct 2015 08:59:20 -0000 Author: mav Date: Mon Oct 5 08:59:18 2015 New Revision: 288734 URL: https://svnweb.freebsd.org/changeset/base/288734 Log: MFC r287670: Close races between device close and request processing. All requests arriving for processing after OFFLINE flag set are rejected with BUSY status. Races around OFFLINE flag setting are closed by calling taskqueue_drain_all(). Modified: stable/10/sys/cam/ctl/ctl_backend_block.c stable/10/sys/cam/ctl/ctl_backend_ramdisk.c Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/cam/ctl/ctl_backend_block.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_backend_block.c Mon Oct 5 08:58:25 2015 (r288733) +++ stable/10/sys/cam/ctl/ctl_backend_block.c Mon Oct 5 08:59:18 2015 (r288734) @@ -1618,33 +1618,32 @@ ctl_be_block_dispatch(struct ctl_be_bloc static void ctl_be_block_worker(void *context, int pending) { - struct ctl_be_block_lun *be_lun; - struct ctl_be_block_softc *softc; + struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)context; + struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun; union ctl_io *io; - - be_lun = (struct ctl_be_block_lun *)context; - softc = be_lun->softc; + struct ctl_be_block_io *beio; DPRINTF("entered\n"); - - mtx_lock(&be_lun->queue_lock); + /* + * Fetch and process I/Os from all queues. If we detect LUN + * CTL_LUN_FLAG_OFFLINE status here -- it is result of a race, + * so make response maximally opaque to not confuse initiator. + */ for (;;) { + mtx_lock(&be_lun->queue_lock); io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue); if (io != NULL) { - struct ctl_be_block_io *beio; - DPRINTF("datamove queue\n"); - STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr, ctl_io_hdr, links); - mtx_unlock(&be_lun->queue_lock); - beio = (struct ctl_be_block_io *)PRIV(io)->ptr; - + if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) { + ctl_set_busy(&io->scsiio); + ctl_complete_beio(beio); + return; + } be_lun->dispatch(be_lun, beio); - - mtx_lock(&be_lun->queue_lock); continue; } io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue); @@ -1653,8 +1652,12 @@ ctl_be_block_worker(void *context, int p STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr, ctl_io_hdr, links); mtx_unlock(&be_lun->queue_lock); + if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) { + ctl_set_busy(&io->scsiio); + ctl_config_write_done(io); + return; + } ctl_be_block_cw_dispatch(be_lun, io); - mtx_lock(&be_lun->queue_lock); continue; } io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue); @@ -1663,25 +1666,26 @@ ctl_be_block_worker(void *context, int p STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr, ctl_io_hdr, links); mtx_unlock(&be_lun->queue_lock); + if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) { + ctl_set_busy(&io->scsiio); + ctl_config_read_done(io); + return; + } ctl_be_block_cr_dispatch(be_lun, io); - mtx_lock(&be_lun->queue_lock); continue; } io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue); if (io != NULL) { DPRINTF("input queue\n"); - STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr, ctl_io_hdr, links); mtx_unlock(&be_lun->queue_lock); - - /* - * We must drop the lock, since this routine and - * its children may sleep. - */ + if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) { + ctl_set_busy(&io->scsiio); + ctl_data_submit_done(io); + return; + } ctl_be_block_dispatch(be_lun, io); - - mtx_lock(&be_lun->queue_lock); continue; } @@ -1689,9 +1693,9 @@ ctl_be_block_worker(void *context, int p * If we get here, there is no work left in the queues, so * just break out and let the task queue go to sleep. */ + mtx_unlock(&be_lun->queue_lock); break; } - mtx_unlock(&be_lun->queue_lock); } /* @@ -2457,6 +2461,7 @@ ctl_be_block_rm(struct ctl_be_block_soft { struct ctl_lun_rm_params *params; struct ctl_be_block_lun *be_lun; + struct ctl_be_lun *cbe_lun; int retval; params = &req->reqdata.rm; @@ -2474,18 +2479,24 @@ ctl_be_block_rm(struct ctl_be_block_soft params->lun_id); goto bailout_error; } + cbe_lun = &be_lun->cbe_lun; - retval = ctl_disable_lun(&be_lun->cbe_lun); - + retval = ctl_disable_lun(cbe_lun); if (retval != 0) { snprintf(req->error_str, sizeof(req->error_str), "error %d returned from ctl_disable_lun() for " "LUN %d", retval, params->lun_id); goto bailout_error; + } + if (be_lun->vn != NULL) { + cbe_lun->flags |= CTL_LUN_FLAG_OFFLINE; + ctl_lun_offline(cbe_lun); + taskqueue_drain_all(be_lun->io_taskqueue); + ctl_be_block_close(be_lun); } - retval = ctl_invalidate_lun(&be_lun->cbe_lun); + retval = ctl_invalidate_lun(cbe_lun); if (retval != 0) { snprintf(req->error_str, sizeof(req->error_str), "error %d returned from ctl_invalidate_lun() for " @@ -2494,15 +2505,12 @@ ctl_be_block_rm(struct ctl_be_block_soft } mtx_lock(&softc->lock); - be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING; - while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0); if (retval == EINTR) break; } - be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING; if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) { @@ -2517,18 +2525,15 @@ ctl_be_block_rm(struct ctl_be_block_soft softc->num_luns--; mtx_unlock(&softc->lock); - taskqueue_drain(be_lun->io_taskqueue, &be_lun->io_task); - + taskqueue_drain_all(be_lun->io_taskqueue); taskqueue_free(be_lun->io_taskqueue); - ctl_be_block_close(be_lun); - if (be_lun->disk_stats != NULL) devstat_remove_entry(be_lun->disk_stats); uma_zdestroy(be_lun->lun_zone); - ctl_free_opts(&be_lun->cbe_lun.options); + ctl_free_opts(&cbe_lun->options); free(be_lun->dev_path, M_CTLBLK); mtx_destroy(&be_lun->queue_lock); mtx_destroy(&be_lun->io_lock); @@ -2693,7 +2698,7 @@ ctl_be_block_modify(struct ctl_be_block_ if (be_lun->vn != NULL) { cbe_lun->flags |= CTL_LUN_FLAG_OFFLINE; ctl_lun_offline(cbe_lun); - pause("CTL LUN offline", hz / 8); // XXX + taskqueue_drain_all(be_lun->io_taskqueue); error = ctl_be_block_close(be_lun); } else error = 0; Modified: stable/10/sys/cam/ctl/ctl_backend_ramdisk.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_backend_ramdisk.c Mon Oct 5 08:58:25 2015 (r288733) +++ stable/10/sys/cam/ctl/ctl_backend_ramdisk.c Mon Oct 5 08:59:18 2015 (r288734) @@ -507,7 +507,7 @@ ctl_backend_ramdisk_rm(struct ctl_be_ram mtx_unlock(&softc->lock); if (retval == 0) { - taskqueue_drain(be_lun->io_taskqueue, &be_lun->io_task); + taskqueue_drain_all(be_lun->io_taskqueue); taskqueue_free(be_lun->io_taskqueue); ctl_free_opts(&be_lun->cbe_lun.options); mtx_destroy(&be_lun->queue_lock);