Date: Fri, 5 Jan 2007 23:04:34 GMT From: Matt Jacob <mjacob@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 112575 for review Message-ID: <200701052304.l05N4Yc1060092@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=112575 Change 112575 by mjacob@mjexp_6 on 2007/01/05 23:03:51 (temp) checkpoint of multipath stuff in a RELENG_6 context Affected files ... .. //depot/projects/mjexp_6/sys/cam/cam_xpt.c#3 edit .. //depot/projects/mjexp_6/sys/cam/cam_xpt.h#2 edit .. //depot/projects/mjexp_6/sys/dev/isp/isp.c#3 edit .. //depot/projects/mjexp_6/sys/dev/isp/isp_freebsd.c#3 edit .. //depot/projects/mjexp_6/sys/dev/isp/isp_pci.c#3 edit .. //depot/projects/mjexp_6/sys/dev/mpt/mpt.c#3 edit .. //depot/projects/mjexp_6/sys/dev/mpt/mpt.h#3 edit .. //depot/projects/mjexp_6/sys/dev/mpt/mpt_cam.c#3 edit .. //depot/projects/mjexp_6/sys/dev/mpt/mpt_cam.h#2 edit .. //depot/projects/mjexp_6/sys/dev/mpt/mpt_pci.c#2 edit .. //depot/projects/mjexp_6/sys/geom/multipath/g_multipath.c#5 edit .. //depot/projects/mjexp_6/sys/geom/multipath/g_multipath.h#2 edit Differences ... ==== //depot/projects/mjexp_6/sys/cam/cam_xpt.c#3 (text+ko) ==== @@ -46,6 +46,7 @@ #include <sys/lock.h> #include <sys/mutex.h> #include <sys/sysctl.h> +#include <sys/kthread.h> #ifdef PC98 #include <pc98/pc98/pc98_machdep.h> /* geometry translation */ @@ -1412,10 +1413,50 @@ return 0; } +/* thread to handle bus rescans */ +static TAILQ_HEAD(, ccb_hdr) ccb_scanq; +static void +xpt_scanner_thread(void *dummy) +{ + mtx_lock(&Giant); + for (;;) { + union ccb *ccb; + tsleep(&ccb_scanq, PRIBIO, "ccb_scanq", 0); + while ((ccb = (union ccb *)TAILQ_FIRST(&ccb_scanq)) != NULL) { + TAILQ_REMOVE(&ccb_scanq, &ccb->ccb_h, sim_links.tqe); + ccb->ccb_h.func_code = XPT_SCAN_BUS; + ccb->ccb_h.cbfcnp = xptdone; + xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, 5); + cam_periph_runccb(ccb, NULL, 0, 0, NULL); + xpt_free_path(ccb->ccb_h.path); + xpt_free_ccb(ccb); + } + } +} + +void +xpt_rescan(union ccb *ccb) +{ + struct ccb_hdr *hdr; + GIANT_REQUIRED; + /* + * Don't make duplicate entries for the same paths. + */ + TAILQ_FOREACH(hdr, &ccb_scanq, sim_links.tqe) { + if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) { + xpt_print(ccb->ccb_h.path, "rescan already queued\n"); + xpt_free_path(ccb->ccb_h.path); + xpt_free_ccb(ccb); + return; + } + } + TAILQ_INSERT_TAIL(&ccb_scanq, &ccb->ccb_h, sim_links.tqe); + wakeup(&ccb_scanq); +} + /* Functions accessed by the peripheral drivers */ static void -xpt_init(dummy) - void *dummy; +xpt_init(void *dummy) { struct cam_sim *xpt_sim; struct cam_path *path; @@ -1425,6 +1466,7 @@ TAILQ_INIT(&xpt_busses); TAILQ_INIT(&cam_bioq); SLIST_INIT(&ccb_freeq); + TAILQ_INIT(&ccb_scanq); STAILQ_INIT(&highpowerq); mtx_init(&cam_bioq_lock, "CAM BIOQ lock", NULL, MTX_DEF); @@ -1490,6 +1532,10 @@ "- failing attach\n"); } + /* fire up rescan thread */ + if (kthread_create(xpt_scanner_thread, NULL, NULL, 0, 0, "xpt_thrd")) { + printf("xpt_init: failed to create rescan thread\n"); + } /* Install our software interrupt handlers */ swi_add(NULL, "cambio", camisr, &cam_bioq, SWI_CAMBIO, 0, &cambio_ih); } ==== //depot/projects/mjexp_6/sys/cam/cam_xpt.h#2 (text+ko) ==== @@ -72,6 +72,7 @@ struct cam_periph *xpt_path_periph(struct cam_path *path); void xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg); +void xpt_rescan(union ccb *ccb); #endif /* _KERNEL */ #endif /* _CAM_CAM_XPT_H */ ==== //depot/projects/mjexp_6/sys/dev/isp/isp.c#3 (text+ko) ==== @@ -1634,12 +1634,14 @@ /* * We end up with these Loop IDs for F-Port topologies */ - if (icbp->icb_hardaddr != 0xff && icbp->icb_hardaddr != 0x800) { - isp_prt(isp, ISP_LOGERR, - "bad hard address %u- resetting to zero", - icbp->icb_hardaddr); + if (icbp->icb_hardaddr != 0xff && + icbp->icb_hardaddr != 0x800 && + icbp->icb_hardaddr != 0xffff) { + isp_prt(isp, ISP_LOGERR, + "bad hard address %u- resetting to zero", + icbp->icb_hardaddr); + icbp->icb_hardaddr = 0; } - icbp->icb_hardaddr = 0; } /* @@ -1888,7 +1890,9 @@ /* * We end up with these Loop IDs for F-Port topologies */ - if (icbp->icb_hardaddr != 0xff && icbp->icb_hardaddr != 0x800) { + if (icbp->icb_hardaddr != 0xff && + icbp->icb_hardaddr != 0x800 && + icbp->icb_hardaddr != 0xffff) { isp_prt(isp, ISP_LOGERR, "bad hard address %u- resetting to zero", icbp->icb_hardaddr); @@ -2151,8 +2155,8 @@ mbs.param[3] = DMA_WD0(FCPARAM(isp)->isp_scdma); mbs.param[6] = DMA_WD3(FCPARAM(isp)->isp_scdma); mbs.param[7] = DMA_WD2(FCPARAM(isp)->isp_scdma); + mbs.timeout = 500000; mbs.logval = MBLOGALL; - mbs.timeout = 250000; MEMORYBARRIER(isp, SYNC_SFORDEV, 0, QENTRY_LEN); isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { @@ -2585,6 +2589,7 @@ } if (check_for_fabric && isp_getpdb(isp, loopid, &pdb, 1) == 0) { + int r; if (IS_2100(isp)) { fcp->isp_topo = TOPO_FL_PORT; } @@ -2609,9 +2614,14 @@ lp->new_portid = lp->portid; lp->new_roles = lp->roles; if (IS_24XX(isp)) { - (void) isp_register_fc4_type_24xx(isp); + r = isp_register_fc4_type_24xx(isp); } else { - (void) isp_register_fc4_type(isp); + r = isp_register_fc4_type(isp); + } + if (r) { + isp_prt(isp, ISP_LOGSANCFG, + "isp_fclink_test: register fc4 type failed"); + return (-1); } } else { not_on_fabric: @@ -2625,6 +2635,7 @@ mbs.param[1] = MBGSD_GET_RATE; /* mbs.param[2] undefined if we're just getting rate */ mbs.logval = MBLOGALL; + mbs.timeout = 3000000; isp_mboxcmd(isp, &mbs); if (mbs.param[0] == MBOX_COMMAND_COMPLETE) { if (mbs.param[1] == MBGSD_FOURGB) { @@ -3138,7 +3149,7 @@ mbs.param[6] = DMA_WD3(fcp->isp_scdma); mbs.param[7] = DMA_WD2(fcp->isp_scdma); mbs.logval = MBLOGALL; - mbs.timeout = 1000000; + mbs.timeout = 10000000; isp_mboxcmd(isp, &mbs); if (mbs.param[0] != MBOX_COMMAND_COMPLETE) { if (mbs.param[0] == MBOX_INVALID_COMMAND) { @@ -3224,6 +3235,7 @@ mbs.param[3] = DMA_WD0(fcp->isp_scdma + CTXOFF); mbs.param[6] = DMA_WD3(fcp->isp_scdma + CTXOFF); mbs.param[7] = DMA_WD2(fcp->isp_scdma + CTXOFF); + mbs.timeout = 500000; mbs.logval = MBLOGALL; MEMORYBARRIER(isp, SYNC_SFORDEV, XTXOFF, 2 * QENTRY_LEN); isp_mboxcmd(isp, &mbs); @@ -3825,7 +3837,7 @@ mbs.param[6] = DMA_WD3(fcp->isp_scdma); mbs.param[7] = DMA_WD2(fcp->isp_scdma); mbs.logval = MBLOGALL; - mbs.timeout = 1000000; + mbs.timeout = 10000000; MEMORYBARRIER(isp, SYNC_SFORDEV, 0, SNS_RFT_ID_REQ_SIZE); isp_mboxcmd(isp, &mbs); FC_SCRATCH_RELEASE(isp); @@ -3902,6 +3914,7 @@ mbs.param[3] = DMA_WD0(fcp->isp_scdma + CTXOFF); mbs.param[6] = DMA_WD3(fcp->isp_scdma + CTXOFF); mbs.param[7] = DMA_WD2(fcp->isp_scdma + CTXOFF); + mbs.timeout = 500000; mbs.logval = MBLOGALL; MEMORYBARRIER(isp, SYNC_SFORDEV, XTXOFF, 2 * QENTRY_LEN); isp_mboxcmd(isp, &mbs); ==== //depot/projects/mjexp_6/sys/dev/isp/isp_freebsd.c#3 (text+ko) ==== @@ -42,6 +42,7 @@ #include <sys/sysctl.h> #endif #include <cam/cam_periph.h> +#include <cam/cam_xpt_periph.h> #if !defined(CAM_NEW_TRAN_CODE) && __FreeBSD_version >= 700025 #define CAM_NEW_TRAN_CODE 1 @@ -2128,21 +2129,44 @@ #if __FreeBSD_version >= 500000 -#define isp_make_here(isp, tgt) isp_announce(isp, tgt, AC_FOUND_DEVICE) -#define isp_make_gone(isp, tgt) isp_announce(isp, tgt, AC_LOST_DEVICE) - /* - * Support function for Announcement + * Support functions for Found/Lost */ static void -isp_announce(ispsoftc_t *isp, int tgt, int action) +isp_make_here(ispsoftc_t *isp, int tgt) +{ + union ccb *ccb; + ISPLOCK_2_CAMLOCK(mpt); + /* + * Allocate a CCB, create a wildcard path for this bus, + * and schedule a rescan. + */ + ccb = xpt_alloc_ccb_nowait(); + if (ccb == NULL) { + isp_prt(isp, ISP_LOGWARN, "unable to alloc CCB for rescan"); + CAMLOCK_2_ISPLOCK(mpt); + return; + } + if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, + cam_sim_path(isp->isp_sim), tgt, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { + CAMLOCK_2_ISPLOCK(mpt); + isp_prt(isp, ISP_LOGWARN, "unable to create path for rescan"); + xpt_free_ccb(ccb); + return; + } + xpt_rescan(ccb); + CAMLOCK_2_ISPLOCK(mpt); +} + +static void +isp_make_gone(ispsoftc_t *isp, int tgt) { - struct cam_path *tmppath; + struct cam_path *tp; ISPLOCK_2_CAMLOCK(isp); - if (xpt_create_path(&tmppath, NULL, cam_sim_path(isp->isp_sim), tgt, + if (xpt_create_path(&tp, NULL, cam_sim_path(isp->isp_sim), tgt, CAM_LUN_WILDCARD) == CAM_REQ_CMP) { - xpt_async(action, tmppath, NULL); - xpt_free_path(tmppath); + xpt_async(AC_LOST_DEVICE, tp, NULL); + xpt_free_path(tp); } CAMLOCK_2_ISPLOCK(isp); } ==== //depot/projects/mjexp_6/sys/dev/isp/isp_pci.c#3 (text+ko) ==== @@ -1378,8 +1378,9 @@ if (hccr & HCCR_PAUSE) { ISP_WRITE(isp, HCCR, HCCR_RESET); isp_prt(isp, ISP_LOGERR, - "RISC paused at interrupt (%x->%x\n", hccr, + "RISC paused at interrupt (%x->%x)", hccr, ISP_READ(isp, HCCR)); + ISP_WRITE(isp, BIU_ICR, 0); } else { isp_prt(isp, ISP_LOGERR, "unknown interrupt 0x%x\n", r2hisr); ==== //depot/projects/mjexp_6/sys/dev/mpt/mpt.c#3 (text+ko) ==== @@ -2000,7 +2000,12 @@ SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "role", CTLFLAG_RD, &mpt->role, 0, "HBA role"); +#ifdef MPT_TEST_MULTIPATH + SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, + "failure_id", CTLFLAG_RW, &mpt->failure_id, -1, + "Next Target to Fail"); #endif +#endif } int @@ -2120,6 +2125,9 @@ } STAILQ_INIT(&mpt->trt_wildcard.atios); STAILQ_INIT(&mpt->trt_wildcard.inots); +#ifdef MPT_TEST_MULTIPATH + mpt->failure_id = -1; +#endif mpt->scsi_tgt_handler_id = MPT_HANDLER_ID_NONE; mpt_sysctl_attach(mpt); mpt_lprt(mpt, MPT_PRT_DEBUG, "doorbell req = %s\n", ==== //depot/projects/mjexp_6/sys/dev/mpt/mpt.h#3 (text+ko) ==== @@ -159,6 +159,7 @@ #define MPT_U64_2_SCALAR(y) ((((uint64_t)y.High) << 32) | (y.Low)) /****************************** Misc Definitions ******************************/ +/* #define MPT_TEST_MULTIPATH 1 */ #define MPT_OK (0) #define MPT_FAIL (0x10000) @@ -514,7 +515,7 @@ uint32_t : 8, unit : 8, - : 1, + ready : 1, fw_uploaded : 1, msi_enable : 1, twildcard : 1, @@ -535,6 +536,9 @@ u_int role; /* role: none, ini, target, both */ u_int verbose; +#ifdef MPT_TEST_MULTIPATH + int failure_id; +#endif /* * IOC Facts ==== //depot/projects/mjexp_6/sys/dev/mpt/mpt_cam.c#3 (text+ko) ==== @@ -110,7 +110,7 @@ #include <sys/callout.h> #include <sys/kthread.h> -#if __FreeBSD_version >= 700000 +#if __FreeBSD_version >= 700025 #ifndef CAM_NEW_TRAN_CODE #define CAM_NEW_TRAN_CODE 1 #endif @@ -842,6 +842,7 @@ } MPT_UNLOCK(mpt); } + mpt->ready = 1; } void @@ -849,6 +850,7 @@ { mpt_handler_t handler; + mpt->ready = 0; mpt_terminate_recovery_thread(mpt); handler.reply_handler = mpt_scsi_reply_handler; @@ -2081,13 +2083,44 @@ break; case MPI_EVENT_RESCAN: + { + union ccb *ccb; + uint32_t pathid; /* * In general this means a device has been added to the loop. */ mpt_prt(mpt, "Rescan Port: %d\n", (data0 >> 8) & 0xff); -/* xpt_async(AC_FOUND_DEVICE, path, NULL); */ + if (mpt->ready == 0) { + break; + } + if (mpt->phydisk_sim) { + pathid = cam_sim_path(mpt->phydisk_sim);; + } else { + pathid = cam_sim_path(mpt->sim); + } + MPTLOCK_2_CAMLOCK(mpt); + /* + * Allocate a CCB, create a wildcard path for this bus, + * and schedule a rescan. + */ + ccb = xpt_alloc_ccb_nowait(); + if (ccb == NULL) { + mpt_prt(mpt, "unable to alloc CCB for rescan\n"); + CAMLOCK_2_MPTLOCK(mpt); + break; + } + + if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, pathid, + CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { + CAMLOCK_2_MPTLOCK(mpt); + mpt_prt(mpt, "unable to create path for rescan\n"); + xpt_free_ccb(ccb); + break; + } + xpt_rescan(ccb); + CAMLOCK_2_MPTLOCK(mpt); break; - + } case MPI_EVENT_LINK_STATUS_CHANGE: mpt_prt(mpt, "Port %d: LinkState: %s\n", (data1 >> 8) & 0xff, @@ -2907,6 +2940,13 @@ mpt_set_ccb_status(ccb, CAM_REQ_INVALID); break; } +#ifdef MPT_TEST_MULTIPATH + if (mpt->failure_id == ccb->ccb_h.target_id) { + ccb->ccb_h.status &= ~CAM_SIM_QUEUED; + mpt_set_ccb_status(ccb, CAM_SEL_TIMEOUT); + break; + } +#endif ccb->csio.scsi_status = SCSI_STATUS_OK; mpt_start(sim, ccb); return; ==== //depot/projects/mjexp_6/sys/dev/mpt/mpt_cam.h#2 (text+ko) ==== @@ -101,6 +101,8 @@ #include <cam/cam_ccb.h> #include <cam/cam_sim.h> #include <cam/cam_xpt.h> +#include <cam/cam_periph.h> +#include <cam/cam_xpt_periph.h> #include <cam/cam_xpt_sim.h> #include <cam/cam_debug.h> #include <cam/scsi/scsi_all.h> ==== //depot/projects/mjexp_6/sys/dev/mpt/mpt_pci.c#2 (text+ko) ==== @@ -105,6 +105,12 @@ #include <dev/mpt/mpt_cam.h> #include <dev/mpt/mpt_raid.h> +#if __FreeBSD_version < 700000 +#define pci_msi_count(x) 0 +#define pci_msi_enable(x) 0 +#define pci_alloc_msi(x, y) 1 +#define pci_release_msi(x) do { ; } while (0) +#endif #ifndef PCI_VENDOR_LSI #define PCI_VENDOR_LSI 0x1000 @@ -327,6 +333,7 @@ } mpt->do_cfg_role = 1; } + mpt->msi_enable = 0; } #else static void @@ -351,6 +358,13 @@ mpt->cfg_role = tval; mpt->do_cfg_role = 1; } + + tval = 0; + mpt->msi_enable = 0; + if (resource_int_value(device_get_name(mpt->dev), + device_get_unit(mpt->dev), "msi_enable", &tval) == 0 && tval == 1) { + mpt->msi_enable = 1; + } } #endif @@ -513,6 +527,14 @@ /* Get a handle to the interrupt */ iqd = 0; + if (mpt->msi_enable && pci_msi_count(dev) == 1) { + mpt->pci_msi_count = 1; + if (pci_alloc_msi(dev, &mpt->pci_msi_count) == 0) { + iqd = 1; + } else { + mpt->pci_msi_count = 0; + } + } mpt->pci_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &iqd, RF_ACTIVE | RF_SHAREABLE); if (mpt->pci_irq == NULL) { @@ -609,10 +631,16 @@ } if (mpt->pci_irq) { - bus_release_resource(mpt->dev, SYS_RES_IRQ, 0, mpt->pci_irq); + bus_release_resource(mpt->dev, SYS_RES_IRQ, + mpt->pci_msi_count ? 1 : 0, mpt->pci_irq); mpt->pci_irq = 0; } + if (mpt->pci_msi_count) { + pci_release_msi(mpt->dev); + mpt->pci_msi_count = 0; + } + if (mpt->pci_pio_reg) { bus_release_resource(mpt->dev, SYS_RES_IOPORT, mpt->pci_pio_rid, mpt->pci_pio_reg); @@ -709,8 +737,8 @@ * Align at byte boundaries, * Limit to 32-bit addressing for request/reply queues. */ - if (mpt_dma_tag_create(mpt, /*parent*/NULL, /*alignment*/1, - /*boundary*/0, /*lowaddr*/BUS_SPACE_MAXADDR, + if (mpt_dma_tag_create(mpt, /*parent*/bus_get_dma_tag(mpt->dev), + /*alignment*/1, /*boundary*/0, /*lowaddr*/BUS_SPACE_MAXADDR, /*highaddr*/BUS_SPACE_MAXADDR, /*filter*/NULL, /*filterarg*/NULL, /*maxsize*/BUS_SPACE_MAXSIZE_32BIT, /*nsegments*/BUS_SPACE_MAXSIZE_32BIT, ==== //depot/projects/mjexp_6/sys/geom/multipath/g_multipath.c#5 (text+ko) ==== @@ -74,6 +74,7 @@ static void g_multipath_orphan(struct g_consumer *cp) { +printf("%s called on %s\n", __FUNCTION__, cp->provider? cp->provider->name : "none"); g_topology_assert(); g_multipath_destroy(cp->geom); } @@ -104,6 +105,17 @@ } static void +g_mpd(void *arg, int flags __unused) +{ + struct g_consumer *cp; + g_topology_assert(); + cp = arg; + g_access(cp, -cp->acr, -cp->acw, -cp->ace); + g_detach(cp); + g_destroy_consumer(cp); +} + +static void g_multipath_done(struct bio *bp) { struct bio *pbp = bp->bio_parent; @@ -131,6 +143,8 @@ if (dofail) { struct g_consumer *cp = bp->bio_from; + struct g_consumer *lcp; + struct g_provider *pp = cp->provider; /* * If we had a failure, we have to check first to see @@ -139,31 +153,44 @@ * a number of failures). If so, we then switch consumers * to the next available consumer. */ + g_topology_lock(); if (cp == sc->cp_active) { - printf("i/o failure is causing detach of %s from %s\n", - cp->provider->name, gp->name); -/* - * XXX: The following two lines are probably wrong due to inflights - */ - g_detach(cp); - g_destroy_consumer(cp); - sc->cp_active = LIST_FIRST(&gp->consumer); + printf("GEOM_MULTIPATH: I/O failure terminates use of " + "%s in %s\n", cp->provider->name, gp->name); + cp->index = 1; + sc->cp_active = NULL; + LIST_FOREACH(lcp, &gp->consumer, consumer) { + if (lcp->index == 0) { + sc->cp_active = lcp; + break; + } + } + if (sc->cp_active == NULL) { + printf("GEOM_MULTIPATH: out of providers\n"); + g_topology_unlock(); + goto out; + } + printf("GEOM_MULTIPATH: switching to provider %s\n", + sc->cp_active->provider->name); + } + g_topology_unlock(); + if (cp->nend == cp->nstart && pp->nend == pp->nstart) { + printf("GEOM_MULTIPATH: old provider %s is now quiet\n", + pp->name); + g_post_event(g_mpd, cp, M_NOWAIT, NULL); } /* * If we can fruitfully restart the I/O, do so. */ if (sc->cp_active) { - printf("switching to provider %s\n", - sc->cp_active->provider->name); g_destroy_bio(bp); pbp->bio_children--; g_multipath_start(pbp); return; - } else { - printf("out of providers to try\n"); } } +out: g_std_done(bp); } @@ -275,7 +302,7 @@ return (error); } cp->private = sc; - cp->index = sc->index++; + cp->index = 0; /* * Set access permissions on new consumer to match other consumers @@ -375,7 +402,6 @@ g_destroy_consumer(cp); g_destroy_geom(gp); if (error != 0) { - printf("%s had error %d reading metadata\n", pp->name, error); return (NULL); } gp = NULL; ==== //depot/projects/mjexp_6/sys/geom/multipath/g_multipath.h#2 (text+ko) ==== @@ -47,7 +47,6 @@ struct g_consumer * cp_active; char sc_name[16]; char sc_uuid[40]; - int index; }; #endif /* _KERNEL */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200701052304.l05N4Yc1060092>