From owner-svn-src-stable@FreeBSD.ORG Fri Dec 5 07:23:28 2014 Return-Path: Delivered-To: svn-src-stable@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id B20728F0; Fri, 5 Dec 2014 07:23:28 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 9D926D97; Fri, 5 Dec 2014 07:23:28 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id sB57NSmY037504; Fri, 5 Dec 2014 07:23:28 GMT (envelope-from mav@FreeBSD.org) Received: (from mav@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id sB57NP66037487; Fri, 5 Dec 2014 07:23:25 GMT (envelope-from mav@FreeBSD.org) Message-Id: <201412050723.sB57NP66037487@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: mav set sender to mav@FreeBSD.org using -f From: Alexander Motin Date: Fri, 5 Dec 2014 07:23:25 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r275493 - stable/10/sys/cam/ctl X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-stable@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: SVN commit messages for all the -stable branches of the src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 05 Dec 2014 07:23:28 -0000 Author: mav Date: Fri Dec 5 07:23:25 2014 New Revision: 275493 URL: https://svnweb.freebsd.org/changeset/base/275493 Log: MFC r274785: Partially reconstruct Active/Standby clusting. In this mode one head is in Active state, supporting all commands, while another is in Standby state, supporting only minimal LUN discovery subset. It is still incomplete since Standby state requires reservation support, which is impossible to do right without having interlink between heads. But it allows to run some basic experiments. Modified: stable/10/sys/cam/ctl/ctl.c stable/10/sys/cam/ctl/ctl_cmd_table.c stable/10/sys/cam/ctl/ctl_frontend.c stable/10/sys/cam/ctl/ctl_frontend.h stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c stable/10/sys/cam/ctl/ctl_frontend_internal.c stable/10/sys/cam/ctl/ctl_frontend_iscsi.c stable/10/sys/cam/ctl/ctl_ha.h stable/10/sys/cam/ctl/ctl_private.h stable/10/sys/cam/ctl/ctl_tpc_local.c stable/10/sys/cam/ctl/scsi_ctl.c Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/cam/ctl/ctl.c ============================================================================== --- stable/10/sys/cam/ctl/ctl.c Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl.c Fri Dec 5 07:23:25 2014 (r275493) @@ -357,7 +357,6 @@ static struct ctl_logical_block_provisio static int rcv_sync_msg; static int persis_offset; static uint8_t ctl_pause_rtr; -static int ctl_is_single = 1; SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer"); static int worker_threads = -1; @@ -972,12 +971,42 @@ ctl_copy_sense_data(union ctl_ha_msg *sr } static int +ctl_ha_state_sysctl(SYSCTL_HANDLER_ARGS) +{ + struct ctl_softc *softc = (struct ctl_softc *)arg1; + struct ctl_lun *lun; + int error, value, i; + + if (softc->flags & CTL_FLAG_ACTIVE_SHELF) + value = 0; + else + value = 1; + + error = sysctl_handle_int(oidp, &value, 0, req); + if ((error != 0) || (req->newptr == NULL)) + return (error); + + mtx_lock(&softc->ctl_lock); + if (value == 0) + softc->flags |= CTL_FLAG_ACTIVE_SHELF; + else + softc->flags &= ~CTL_FLAG_ACTIVE_SHELF; + STAILQ_FOREACH(lun, &softc->lun_list, links) { + mtx_lock(&lun->lun_lock); + for (i = 0; i < CTL_MAX_INITIATORS; i++) + lun->pending_ua[i] |= CTL_UA_ASYM_ACC_CHANGE; + mtx_unlock(&lun->lun_lock); + } + mtx_unlock(&softc->ctl_lock); + return (0); +} + +static int ctl_init(void) { struct ctl_softc *softc; struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool; struct ctl_port *port; - uint8_t sc_id =0; int i, error, retval; //int isc_retval; @@ -1035,16 +1064,17 @@ ctl_init(void) * In Copan's HA scheme, the "master" and "slave" roles are * figured out through the slot the controller is in. Although it * is an active/active system, someone has to be in charge. - */ -#ifdef NEEDTOPORT - scmicro_rw(SCMICRO_GET_SHELF_ID, &sc_id); -#endif - - if (sc_id == 0) { - softc->flags |= CTL_FLAG_MASTER_SHELF; - persis_offset = 0; + */ + SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "ha_id", CTLFLAG_RDTUN, &softc->ha_id, 0, + "HA head ID (0 - no HA)"); + if (softc->ha_id == 0) { + softc->flags |= CTL_FLAG_ACTIVE_SHELF; + softc->is_single = 1; + softc->port_offset = 0; } else - persis_offset = CTL_MAX_INITIATORS; + softc->port_offset = (softc->ha_id - 1) * CTL_MAX_PORTS; + persis_offset = softc->port_offset * CTL_MAX_INIT_PER_PORT; /* * XXX KDM need to figure out where we want to get our target ID @@ -1157,12 +1187,15 @@ ctl_init(void) port->max_targets = 15; port->max_target_id = 15; - if (ctl_port_register(&softc->ioctl_info.port, - (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) { + if (ctl_port_register(&softc->ioctl_info.port) != 0) { printf("ctl: ioctl front end registration failed, will " "continue anyway\n"); } + SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree), + OID_AUTO, "ha_state", CTLTYPE_INT | CTLFLAG_RWTUN, + softc, 0, ctl_ha_state_sysctl, "I", "HA state for this head"); + #ifdef CTL_IO_DELAY if (sizeof(struct callout) > CTL_TIMER_BYTES) { printf("sizeof(struct callout) %zd > CTL_TIMER_BYTES %zd\n", @@ -1264,10 +1297,10 @@ ctl_close(struct cdev *dev, int flags, i int ctl_port_enable(ctl_port_type port_type) { - struct ctl_softc *softc; + struct ctl_softc *softc = control_softc; struct ctl_port *port; - if (ctl_is_single == 0) { + if (softc->is_single == 0) { union ctl_ha_msg msg_info; int isc_retval; @@ -1292,8 +1325,6 @@ ctl_port_enable(ctl_port_type port_type) #endif } - softc = control_softc; - STAILQ_FOREACH(port, &softc->port_list, links) { if (port_type & port->port_type) { @@ -7439,8 +7470,8 @@ ctl_report_tagret_port_groups(struct ctl { struct scsi_maintenance_in *cdb; int retval; - int alloc_len, ext, total_len = 0, g, p, pc, pg; - int num_target_port_groups, num_target_ports, single; + int alloc_len, ext, total_len = 0, g, p, pc, pg, gs, os; + int num_target_port_groups, num_target_ports; struct ctl_lun *lun; struct ctl_softc *softc; struct ctl_port *port; @@ -7474,8 +7505,7 @@ ctl_report_tagret_port_groups(struct ctl return(retval); } - single = ctl_is_single; - if (single) + if (softc->is_single) num_target_port_groups = 1; else num_target_port_groups = NUM_TARGET_PORT_GROUPS; @@ -7531,18 +7561,26 @@ ctl_report_tagret_port_groups(struct ctl tpg_desc = &rtg_ptr->groups[0]; } - pg = ctsio->io_hdr.nexus.targ_port / CTL_MAX_PORTS; mtx_lock(&softc->ctl_lock); + pg = softc->port_offset / CTL_MAX_PORTS; + if (softc->flags & CTL_FLAG_ACTIVE_SHELF) { + if (softc->ha_mode == CTL_HA_MODE_ACT_STBY) { + gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED; + os = TPG_ASYMMETRIC_ACCESS_STANDBY; + } else if (lun->flags & CTL_LUN_PRIMARY_SC) { + gs = TPG_ASYMMETRIC_ACCESS_OPTIMIZED; + os = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED; + } else { + gs = TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED; + os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED; + } + } else { + gs = TPG_ASYMMETRIC_ACCESS_STANDBY; + os = TPG_ASYMMETRIC_ACCESS_OPTIMIZED; + } for (g = 0; g < num_target_port_groups; g++) { - if (g == pg) - tpg_desc->pref_state = TPG_PRIMARY | - TPG_ASYMMETRIC_ACCESS_OPTIMIZED; - else - tpg_desc->pref_state = - TPG_ASYMMETRIC_ACCESS_NONOPTIMIZED; - tpg_desc->support = TPG_AO_SUP; - if (!single) - tpg_desc->support |= TPG_AN_SUP; + tpg_desc->pref_state = (g == pg) ? gs : os; + tpg_desc->support = TPG_AO_SUP | TPG_AN_SUP | TPG_S_SUP; scsi_ulto2b(g + 1, tpg_desc->target_port_group); tpg_desc->status = TPG_IMPLICIT; pc = 0; @@ -10181,12 +10219,11 @@ ctl_inquiry_evpd_scsi_ports(struct ctl_s struct ctl_lun *lun; struct ctl_port *port; int data_len, num_target_ports, iid_len, id_len, g, pg, p; - int num_target_port_groups, single; + int num_target_port_groups; lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr; - single = ctl_is_single; - if (single) + if (softc->is_single) num_target_port_groups = 1; else num_target_port_groups = NUM_TARGET_PORT_GROUPS; @@ -10246,10 +10283,7 @@ ctl_inquiry_evpd_scsi_ports(struct ctl_s pd = &sp->design[0]; mtx_lock(&softc->ctl_lock); - if (softc->flags & CTL_FLAG_MASTER_SHELF) - pg = 0; - else - pg = 1; + pg = softc->port_offset / CTL_MAX_PORTS; for (g = 0; g < num_target_port_groups; g++) { STAILQ_FOREACH(port, &softc->port_list, links) { if ((port->status & CTL_PORT_STATUS_ONLINE) == 0) @@ -11329,15 +11363,12 @@ ctl_scsiio_lun_check(struct ctl_softc *c * If this shelf is a secondary shelf controller, we have to reject * any media access commands. */ -#if 0 - /* No longer needed for HA */ - if (((ctl_softc->flags & CTL_FLAG_MASTER_SHELF) == 0) - && ((entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0)) { + if ((ctl_softc->flags & CTL_FLAG_ACTIVE_SHELF) == 0 && + (entry->flags & CTL_CMD_FLAG_OK_ON_SECONDARY) == 0) { ctl_set_lun_standby(ctsio); retval = 1; goto bailout; } -#endif if (entry->pattern & CTL_LUN_PAT_WRITE) { if (lun->flags & CTL_LUN_READONLY) { @@ -14392,7 +14423,7 @@ ctl_isc_start(struct ctl_ha_component *c // UNKNOWN->HA or UNKNOWN->SINGLE (bootstrap) if (c->state == CTL_HA_STATE_UNKNOWN ) { - ctl_is_single = 0; + control_softc->is_single = 0; if (ctl_ha_msg_create(CTL_HA_CHAN_CTL, ctl_isc_event_handler) != CTL_HA_STATUS_SUCCESS) { printf("ctl_isc_start: ctl_ha_msg_create failed.\n"); @@ -14402,14 +14433,14 @@ ctl_isc_start(struct ctl_ha_component *c && CTL_HA_STATE_IS_SINGLE(state)){ // HA->SINGLE transition ctl_failover(); - ctl_is_single = 1; + control_softc->is_single = 1; } else { printf("ctl_isc_start:Invalid state transition %X->%X\n", c->state, state); ret = CTL_HA_COMP_STATUS_ERROR; } if (CTL_HA_STATE_IS_SINGLE(state)) - ctl_is_single = 1; + control_softc->is_single = 1; c->state = state; c->status = ret; Modified: stable/10/sys/cam/ctl/ctl_cmd_table.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_cmd_table.c Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl_cmd_table.c Fri Dec 5 07:23:25 2014 (r275493) @@ -800,12 +800,18 @@ const struct ctl_cmd_entry ctl_cmd_table /* 3B WRITE BUFFER */ {ctl_write_buffer, CTL_SERIDX_MD_SEL, CTL_CMD_FLAG_OK_ON_BOTH | + CTL_CMD_FLAG_OK_ON_STOPPED | + CTL_CMD_FLAG_OK_ON_INOPERABLE | + CTL_CMD_FLAG_OK_ON_OFFLINE | CTL_FLAG_DATA_OUT, CTL_LUN_PAT_NONE, 10, {0x1f, 0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x07}}, /* 3C READ BUFFER */ {ctl_read_buffer, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_BOTH | + CTL_CMD_FLAG_OK_ON_STOPPED | + CTL_CMD_FLAG_OK_ON_INOPERABLE | + CTL_CMD_FLAG_OK_ON_OFFLINE | CTL_FLAG_DATA_IN | CTL_CMD_FLAG_ALLOW_ON_PR_WRESV, CTL_LUN_PAT_NONE, Modified: stable/10/sys/cam/ctl/ctl_frontend.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_frontend.c Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl_frontend.c Fri Dec 5 07:23:25 2014 (r275493) @@ -136,7 +136,7 @@ ctl_frontend_find(char *frontend_name) } int -ctl_port_register(struct ctl_port *port, int master_shelf) +ctl_port_register(struct ctl_port *port) { struct ctl_io_pool *pool; int port_num; @@ -193,7 +193,7 @@ error: STAILQ_INIT(&port->options); mtx_lock(&control_softc->ctl_lock); - port->targ_port = port_num + (master_shelf != 0 ? 0 : CTL_MAX_PORTS); + port->targ_port = port_num + control_softc->port_offset; STAILQ_INSERT_TAIL(&port->frontend->port_list, port, fe_links); STAILQ_INSERT_TAIL(&control_softc->port_list, port, links); control_softc->ctl_ports[port_num] = port; Modified: stable/10/sys/cam/ctl/ctl_frontend.h ============================================================================== --- stable/10/sys/cam/ctl/ctl_frontend.h Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl_frontend.h Fri Dec 5 07:23:25 2014 (r275493) @@ -278,7 +278,7 @@ struct ctl_frontend * ctl_frontend_find( * This may block until resources are allocated. Called at FETD module load * time. Returns 0 for success, non-zero for failure. */ -int ctl_port_register(struct ctl_port *port, int master_SC); +int ctl_port_register(struct ctl_port *port); /* * Called at FETD module unload time. Modified: stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c Fri Dec 5 07:23:25 2014 (r275493) @@ -164,7 +164,7 @@ cfcs_init(void) port->max_targets = 1; port->max_target_id = 15; - retval = ctl_port_register(port, /*master_SC*/ 1); + retval = ctl_port_register(port); if (retval != 0) { printf("%s: ctl_port_register() failed with error %d!\n", __func__, retval); Modified: stable/10/sys/cam/ctl/ctl_frontend_internal.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_frontend_internal.c Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl_frontend_internal.c Fri Dec 5 07:23:25 2014 (r275493) @@ -244,8 +244,6 @@ cfi_init(void) memset(softc, 0, sizeof(*softc)); mtx_init(&softc->lock, "CTL frontend mutex", NULL, MTX_DEF); - softc->flags |= CTL_FLAG_MASTER_SHELF; - STAILQ_INIT(&softc->lun_list); STAILQ_INIT(&softc->metatask_list); sprintf(softc->fe_name, "kernel"); @@ -264,7 +262,7 @@ cfi_init(void) port->max_targets = 15; port->max_target_id = 15; - if (ctl_port_register(port, (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) + if (ctl_port_register(port) != 0) { printf("%s: internal frontend registration failed\n", __func__); return (0); Modified: stable/10/sys/cam/ctl/ctl_frontend_iscsi.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_frontend_iscsi.c Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl_frontend_iscsi.c Fri Dec 5 07:23:25 2014 (r275493) @@ -2054,7 +2054,7 @@ cfiscsi_ioctl_port_create(struct ctl_req desc->length = idlen; strlcpy(desc->identifier, target, idlen); - retval = ctl_port_register(port, /*master_SC*/ 1); + retval = ctl_port_register(port); if (retval != 0) { ctl_free_opts(&port->options); cfiscsi_target_release(ct); Modified: stable/10/sys/cam/ctl/ctl_ha.h ============================================================================== --- stable/10/sys/cam/ctl/ctl_ha.h Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl_ha.h Fri Dec 5 07:23:25 2014 (r275493) @@ -38,6 +38,8 @@ /* * CTL High Availability Modes: * + * CTL_HA_MODE_ACT_STBY: One side is in Active state and processing commands, + * the other side is in Standby state, returning errors. * CTL_HA_MODE_SER_ONLY: Commands are serialized to the other side. Write * mirroring and read re-direction are assumed to * happen in the back end. @@ -46,6 +48,7 @@ */ typedef enum { + CTL_HA_MODE_ACT_STBY, CTL_HA_MODE_SER_ONLY, CTL_HA_MODE_XFER } ctl_ha_mode; Modified: stable/10/sys/cam/ctl/ctl_private.h ============================================================================== --- stable/10/sys/cam/ctl/ctl_private.h Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl_private.h Fri Dec 5 07:23:25 2014 (r275493) @@ -442,7 +442,7 @@ struct ctl_lun { typedef enum { CTL_FLAG_REAL_SYNC = 0x02, - CTL_FLAG_MASTER_SHELF = 0x04 + CTL_FLAG_ACTIVE_SHELF = 0x04 } ctl_gen_flags; #define CTL_MAX_THREADS 16 @@ -467,6 +467,10 @@ struct ctl_softc { int num_luns; ctl_gen_flags flags; ctl_ha_mode ha_mode; + int ha_id; + int ha_state; + int is_single; + int port_offset; int inquiry_pq_no_lun; struct sysctl_ctx_list sysctl_ctx; struct sysctl_oid *sysctl_tree; Modified: stable/10/sys/cam/ctl/ctl_tpc_local.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_tpc_local.c Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/ctl_tpc_local.c Fri Dec 5 07:23:25 2014 (r275493) @@ -87,7 +87,6 @@ CTL_FRONTEND_DECLARE(ctltpc, tpcl_fronte static int tpcl_init(void) { - struct ctl_softc *softc = control_softc; struct tpcl_softc *tsoftc = &tpcl_softc; struct ctl_port *port; struct scsi_transportid_spi *tid; @@ -112,7 +111,7 @@ tpcl_init(void) port->max_target_id = 0; port->max_initiators = 1; - if (ctl_port_register(port, (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0) + if (ctl_port_register(port) != 0) { printf("%s: tpc frontend registration failed\n", __func__); return (0); Modified: stable/10/sys/cam/ctl/scsi_ctl.c ============================================================================== --- stable/10/sys/cam/ctl/scsi_ctl.c Fri Dec 5 00:32:33 2014 (r275492) +++ stable/10/sys/cam/ctl/scsi_ctl.c Fri Dec 5 07:23:25 2014 (r275493) @@ -425,7 +425,7 @@ ctlfeasync(void *callback_arg, uint32_t printf("%s: calling ctl_port_register() for %s%d\n", __func__, cpi->dev_name, cpi->unit_number); #endif - retval = ctl_port_register(port, /*master_SC*/ 1); + retval = ctl_port_register(port); if (retval != 0) { printf("%s: ctl_port_register() failed with " "error %d!\n", __func__, retval);