From owner-svn-src-all@freebsd.org Thu Feb 15 14:34:19 2018 Return-Path: Delivered-To: svn-src-all@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 020E9F22DA7; Thu, 15 Feb 2018 14:34:19 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 9B1FD7B41A; Thu, 15 Feb 2018 14:34:18 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 92D6920050; Thu, 15 Feb 2018 14:34:18 +0000 (UTC) (envelope-from avg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id w1FEYIvO059339; Thu, 15 Feb 2018 14:34:18 GMT (envelope-from avg@FreeBSD.org) Received: (from avg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id w1FEYIbB059337; Thu, 15 Feb 2018 14:34:18 GMT (envelope-from avg@FreeBSD.org) Message-Id: <201802151434.w1FEYIbB059337@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: avg set sender to avg@FreeBSD.org using -f From: Andriy Gapon Date: Thu, 15 Feb 2018 14:34:18 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r329313 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys X-SVN-Commit-Author: avg X-SVN-Commit-Paths: in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Commit-Revision: 329313 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.25 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 15 Feb 2018 14:34:19 -0000 Author: avg Date: Thu Feb 15 14:34:18 2018 New Revision: 329313 URL: https://svnweb.freebsd.org/changeset/base/329313 Log: 8857 zio_remove_child() panic due to already destroyed parent zio illumos/illumos-gate@d6e1c446d7897003fd9fd36ef5aa7da350b7f6af https://github.com/illumos/illumos-gate/commit/d6e1c446d7897003fd9fd36ef5aa7da350b7f6af https://www.illumos.org/issues/8857 I had an OS panic on one of our servers: ffffff01809128c0 vpanic() ffffff01809128e0 mutex_panic+0x58(fffffffffb94c904, ffffff597dde7f80) ffffff0180912950 mutex_vector_enter+0x347(ffffff597dde7f80) ffffff01809129b0 zio_remove_child+0x50(ffffff597dde7c58, ffffff32bd901ac0, ffffff3373370908) ffffff0180912a40 zio_done+0x390(ffffff32bd901ac0) ffffff0180912a70 zio_execute+0x78(ffffff32bd901ac0) ffffff0180912b30 taskq_thread+0x2d0(ffffff33bae44140) ffffff0180912b40 thread_start+8() It panicked here: http://src.illumos.org/source/xref/illumos-gate/usr/src/uts/common/fs/zfs/ zio.c#430 pio->io_lock is DEAD, thus a panic. Further analysis shows the "pio" (parent zio of "cio") has already been destroyed. Reviewed by: Matthew Ahrens Reviewed by: Andriy Gapon Reviewed by: Youzhong Yang Approved by: Dan McDonald Author: George Wilson Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h Thu Feb 15 11:41:38 2018 (r329312) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h Thu Feb 15 14:34:18 2018 (r329313) @@ -208,6 +208,9 @@ enum zio_flag { (((zio)->io_flags & ZIO_FLAG_VDEV_INHERIT) | \ ZIO_FLAG_CANFAIL) +#define ZIO_CHILD_BIT(x) (1 << (x)) +#define ZIO_CHILD_BIT_IS_SET(val, x) ((val) & (1 << (x))) + enum zio_child { ZIO_CHILD_VDEV = 0, ZIO_CHILD_GANG, @@ -215,6 +218,14 @@ enum zio_child { ZIO_CHILD_LOGICAL, ZIO_CHILD_TYPES }; + +#define ZIO_CHILD_VDEV_BIT ZIO_CHILD_BIT(ZIO_CHILD_VDEV) +#define ZIO_CHILD_GANG_BIT ZIO_CHILD_BIT(ZIO_CHILD_GANG) +#define ZIO_CHILD_DDT_BIT ZIO_CHILD_BIT(ZIO_CHILD_DDT) +#define ZIO_CHILD_LOGICAL_BIT ZIO_CHILD_BIT(ZIO_CHILD_LOGICAL) +#define ZIO_CHILD_ALL_BITS \ + (ZIO_CHILD_VDEV_BIT | ZIO_CHILD_GANG_BIT | \ + ZIO_CHILD_DDT_BIT | ZIO_CHILD_LOGICAL_BIT) enum zio_wait_type { ZIO_WAIT_READY = 0, Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Thu Feb 15 11:41:38 2018 (r329312) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zio.c Thu Feb 15 14:34:18 2018 (r329313) @@ -442,21 +442,26 @@ zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *z } static boolean_t -zio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait) +zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait) { - uint64_t *countp = &zio->io_children[child][wait]; boolean_t waiting = B_FALSE; mutex_enter(&zio->io_lock); ASSERT(zio->io_stall == NULL); - if (*countp != 0) { - zio->io_stage >>= 1; - ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN); - zio->io_stall = countp; - waiting = B_TRUE; + for (int c = 0; c < ZIO_CHILD_TYPES; c++) { + if (!(ZIO_CHILD_BIT_IS_SET(childbits, c))) + continue; + + uint64_t *countp = &zio->io_children[c][wait]; + if (*countp != 0) { + zio->io_stage >>= 1; + ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN); + zio->io_stall = countp; + waiting = B_TRUE; + break; + } } mutex_exit(&zio->io_lock); - return (waiting); } @@ -1237,9 +1242,10 @@ zio_write_compress(zio_t *zio) * If our children haven't all reached the ready stage, * wait for them and then repeat this pipeline stage. */ - if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || - zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY)) + if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT | + ZIO_CHILD_GANG_BIT, ZIO_WAIT_READY)) { return (ZIO_PIPELINE_STOP); + } if (!IO_IS_ALLOCATING(zio)) return (ZIO_PIPELINE_CONTINUE); @@ -2080,8 +2086,9 @@ zio_gang_issue(zio_t *zio) { blkptr_t *bp = zio->io_bp; - if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE)) + if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT, ZIO_WAIT_DONE)) { return (ZIO_PIPELINE_STOP); + } ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); ASSERT(zio->io_child_type > ZIO_CHILD_GANG); @@ -2402,8 +2409,9 @@ zio_ddt_read_done(zio_t *zio) { blkptr_t *bp = zio->io_bp; - if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE)) + if (zio_wait_for_children(zio, ZIO_CHILD_DDT_BIT, ZIO_WAIT_DONE)) { return (ZIO_PIPELINE_STOP); + } ASSERT(BP_GET_DEDUP(bp)); ASSERT(BP_GET_PSIZE(bp) == zio->io_size); @@ -3109,8 +3117,9 @@ zio_vdev_io_done(zio_t *zio) vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops; boolean_t unexpected_error = B_FALSE; - if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) + if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) { return (ZIO_PIPELINE_STOP); + } ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE); @@ -3176,8 +3185,9 @@ zio_vdev_io_assess(zio_t *zio) { vdev_t *vd = zio->io_vd; - if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE)) + if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) { return (ZIO_PIPELINE_STOP); + } if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) spa_config_exit(zio->io_spa, SCL_ZIO, zio); @@ -3392,9 +3402,10 @@ zio_ready(zio_t *zio) zio_t *pio, *pio_next; zio_link_t *zl = NULL; - if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) || - zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY)) + if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT | ZIO_CHILD_DDT_BIT, + ZIO_WAIT_READY)) { return (ZIO_PIPELINE_STOP); + } if (zio->io_ready) { ASSERT(IO_IS_ALLOCATING(zio)); @@ -3534,11 +3545,9 @@ zio_done(zio_t *zio) * If our children haven't all completed, * wait for them and then repeat this pipeline stage. */ - if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) || - zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) || - zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) || - zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE)) + if (zio_wait_for_children(zio, ZIO_CHILD_ALL_BITS, ZIO_WAIT_DONE)) { return (ZIO_PIPELINE_STOP); + } /* * If the allocation throttle is enabled, then update the accounting.