From owner-freebsd-fs@FreeBSD.ORG Mon Dec 3 22:41:37 2012 Return-Path: Delivered-To: fs@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id BFF50E85; Mon, 3 Dec 2012 22:41:37 +0000 (UTC) (envelope-from kostikbel@gmail.com) Received: from kib.kiev.ua (kib.kiev.ua [IPv6:2001:470:d5e7:1::1]) by mx1.freebsd.org (Postfix) with ESMTP id 2CCF88FC14; Mon, 3 Dec 2012 22:41:36 +0000 (UTC) Received: from tom.home (localhost [127.0.0.1]) by kib.kiev.ua (8.14.5/8.14.5) with ESMTP id qB3MfWXd084116; Tue, 4 Dec 2012 00:41:32 +0200 (EET) (envelope-from kostikbel@gmail.com) DKIM-Filter: OpenDKIM Filter v2.7.1 kib.kiev.ua qB3MfWXd084116 Received: (from kostik@localhost) by tom.home (8.14.5/8.14.5/Submit) id qB3MfWmZ084115; Tue, 4 Dec 2012 00:41:32 +0200 (EET) (envelope-from kostikbel@gmail.com) X-Authentication-Warning: tom.home: kostik set sender to kostikbel@gmail.com using -f Date: Tue, 4 Dec 2012 00:41:32 +0200 From: Konstantin Belousov To: sig6247 Subject: Re: clang compiled kernel panic when mounting zfs root on i386 Message-ID: <20121203224132.GJ3013@kib.kiev.ua> References: <50b37d46.8584440a.735c.ffffb4e6@mx.google.com> <20121126171658.GD3013@kib.kiev.ua> <20121127071243.D1255@besplex.bde.org> <20121129232944.GQ3013@kib.kiev.ua> <50b8a9c5.e64dec0a.1d88.133a@mx.google.com> <20121130164715.GW3013@kib.kiev.ua> <50b9cf0c.0fd9650a.5bbf.ffffb9b3@mx.google.com> MIME-Version: 1.0 Content-Type: multipart/signed; micalg=pgp-sha1; protocol="application/pgp-signature"; boundary="dOlf0zsnhJ4/ZPgF" Content-Disposition: inline In-Reply-To: <50b9cf0c.0fd9650a.5bbf.ffffb9b3@mx.google.com> User-Agent: Mutt/1.5.21 (2010-09-15) X-Spam-Status: No, score=-2.0 required=5.0 tests=ALL_TRUSTED,BAYES_00, DKIM_ADSP_CUSTOM_MED,FREEMAIL_FROM,NML_ADSP_CUSTOM_MED autolearn=no version=3.3.2 X-Spam-Checker-Version: SpamAssassin 3.3.2 (2011-06-06) on tom.home Cc: freebsd-current@freebsd.org, dim@freebsd.org, fs@freebsd.org X-BeenThere: freebsd-fs@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: Filesystems List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 03 Dec 2012 22:41:37 -0000 --dOlf0zsnhJ4/ZPgF Content-Type: text/plain; charset=us-ascii Content-Disposition: inline Content-Transfer-Encoding: quoted-printable On Sat, Dec 01, 2012 at 01:34:04AM -0800, sig6247 wrote: > On Fri, 30 Nov 2012 18:47:15 +0200, Konstantin Belousov wrote: >=20 > > Hm, this is not very useful. Although the panic is again caused by the = stack > > overflow, most likely (please also include the output of the "show thre= ad" > > from ddb), it is at different place, and probably at the leaf function. > > > > Can you try some more times, so that we could see 'big' backtrace ? >=20 > Sure. Thanks. >=20 > WARNING: WITNESS option enabled, expect reduced performance. > Trying to mount root from zfs:zroot []... >=20 > Fatal double fault: > eip =3D 0xc0add15d > esp =3D 0xc86bffc8 > ebp =3D 0xc86c003c > cpuid =3D 1; apic id =3D 01 > panic: double fault > cpuid =3D 1 > KDB: enter: panic > [ thread pid 1 tid 100002 ] > Stopped at kdb_enter+0x3d: movl $0,kdb_why > db> bt > Tracing pid 1 tid 100002 td 0xc89efbc0 > kdb_enter(c1065960,c1065960,c10b903b,c139f438,2243cdbd,...) at kdb_enter+= 0x3d/frame 0xc139f3f0 > panic(c10b903b,1,1,1,c86c003c,...) at panic+0x14b/frame 0xc139f42c > dblfault_handler() at dblfault_handler+0xab/frame 0xc139f42c > --- trap 0x17, eip =3D 0xc0add15d, esp =3D 0xc86bffc8, ebp =3D 0xc86c003c= --- > witness_checkorder(c1fd7508,9,c109ee8c,7fa,0,...) at witness_checkorder+0= x37d/frame 0xc86c003c > __mtx_lock_flags(c1fd7518,0,c109ee8c,7fa,c135e998,...) at __mtx_lock_flag= s+0x87/frame 0xc86c007 > 0 > uma_zalloc_arg(c1fd66c0,0,1,4d3,c86c0110,...) at uma_zalloc_arg+0x605/fra= me 0xc86c00c8 > vm_map_insert(c1fd508c,c13e0ca0,bd3a000,0,cbc39000,...) at vm_map_insert+= 0x499/frame 0xc86c0130 >=20 > kmem_back(c1fd508c,cbc39000,1000,3,c86c01d4,...) at kmem_back+0x76/frame = 0xc86c018c > kmem_malloc(c1fd508c,1000,3) at kmem_malloc+0x250/frame 0xc86c01c0 > page_alloc(c1fd1d80,1000,c86c020b,3,c1fd1d80,...) at page_alloc+0x27/fram= e 0xc86c01d4 > keg_alloc_slab(103,4,c109ee8c,870,cbb95f6c,...) at keg_alloc_slab+0xc3/fr= ame 0xc86c0218 > keg_fetch_slab(103,c1fd1d80,cbb95f6c,c1fc8230,c86c02c0,...) at keg_fetch_= slab+0xe2/frame 0xc86c > 0250 > zone_fetch_slab(c1fd1d80,c1fd0480,103,826,0,...) at zone_fetch_slab+0x43/= frame 0xc86c0268 > uma_zalloc_arg(c1fd1d80,0,102,3,2,...) at uma_zalloc_arg+0x3f2/frame 0xc8= 6c02c0 > malloc(4c,c1826100,102,c86c0388,c173909a,...) at malloc+0xe9/frame 0xc86c= 02e8 > zfs_kmem_alloc(4c,102,cb7d8820,c89efbc0,cb7d8820,...) at zfs_kmem_alloc+0= x20/frame 0xc86c02fc > vdev_mirror_io_start(cba232e0,10,cba232e0,1,0,...) at vdev_mirror_io_star= t+0x14a/frame 0xc86c03 > 88 > zio_vdev_io_start(cba232e0,c89efbc0,0,cba232e0,c86c0600,...) at zio_vdev_= io_start+0x228/frame 0 > xc86c03e4 > zio_execute(cba232e0,cb7d8000,cbbec640,cbbe2000,600,...) at zio_execute+0= x106/frame 0xc86c0418 > spa_load_verify_cb(cb7d8000,0,cbbec640,cba6bd20,c86c0600,...) at spa_load= _verify_cb+0x89/frame=20 > 0xc86c0458 > traverse_visitbp(cba6bd20,cbbec640,c86c0600,c86c0ba0,0,...) at traverse_v= isitbp+0x29f/frame 0xc > 86c05e0 > traverse_dnode(cba6bd20,0,0,23,0,...) at traverse_dnode+0x92/frame 0xc86c= 0638 > traverse_visitbp(cba6bd98,cbbf0080,c86c0890,cba6bdd4,c16ca7e0,...) at tra= verse_visitbp+0xe47/fr > ame 0xc86c07c0 > traverse_visitbp(cba6bdd4,cbbe2840,c86c0968,c86c0ba0,0,...) at traverse_v= isitbp+0xf32/frame 0xc > 86c0948 > traverse_dnode(cba6bdd4,0,0,0,0,...) at traverse_dnode+0x92/frame 0xc86c0= 9a0 > traverse_visitbp(0,cb7d8398,c86c0b50,2,cbbdc214,...) at traverse_visitbp+= 0x96d/frame 0xc86c0b28 >=20 > traverse_impl(0,0,cb7d8398,74,0,...) at traverse_impl+0x268/frame 0xc86c0= be0 > traverse_pool(cb7d8000,74,0,d,c1723830,...) at traverse_pool+0x79/frame 0= xc86c0c88 > spa_load(0,1,c86c0ec4,1e,0,...) at spa_load+0x1dde/frame 0xc86c0df0 > spa_load(0,0,c13d9d14,1,3,...) at spa_load+0x11a5/frame 0xc86c0f58 > spa_load_best(0,ffffffff,ffffffff,1,c0add175,...) at spa_load_best+0x71/f= rame 0xc86c0fb0 > spa_open_common(c17dce4e,0,0,c86c1190,c16f1a1c,...) at spa_open_common+0x= 11a/frame 0xc86c100c > spa_open(c86c1078,c86c1074,c17dce4e,c135e998,c1fd7798,...) at spa_open+0x= 27/frame 0xc86c1020 > dsl_dir_open_spa(0,c89770b0,c17dd1e1,c86c11f8,c86c11f4,...) at dsl_dir_op= en_spa+0x6c/frame 0xc8 > 6c1190 > dsl_dataset_hold(c89770b0,cb7d3800,c86c1240,cb7d3800,cb7d3800,...) at dsl= _dataset_hold+0x3a/fra > me 0xc86c120c > dsl_dataset_own(c89770b0,0,cb7d3800,c86c1240,c1824e30,...) at dsl_dataset= _own+0x21/frame 0xc86c > 1228 > dmu_objset_own(c89770b0,2,1,cb7d3800,c86c1290,...) at dmu_objset_own+0x2a= /frame 0xc86c1250 > zfsvfs_create(c89770b0,c86c13ac,c17ea09b,681,0,...) at zfsvfs_create+0x4c= /frame 0xc86c12a8 > zfs_mount(cb99b540,c17f0160,cb98b100,c89cae80,0,...) at zfs_mount+0x42c/f= rame 0xc86c14e0 > vfs_donmount(c89efbc0,4000,0,c86c1790,cb98b180,...) at vfs_donmount+0xc6d= /frame 0xc86c1778 > kernel_mount(c8977490,4000,0,0,1,...) at kernel_mount+0x6b/frame 0xc86c17= b8 > parse_mount(cb96e0e0,c1195498,0,1,0,...) at parse_mount+0x606/frame 0xc86= c19d8 > vfs_mountroot(c13da634,4,c105ceba,2bb,0,...) at vfs_mountroot+0x6cf/frame= 0xc86c1c60 > start_init(0,c86c1d08,c105f7c4,3db,0,...) at start_init+0x6a/frame 0xc86c= 1ccc > fork_exit(c0a429e0,0,c86c1d08) at fork_exit+0x7f/frame 0xc86c1cf4 > fork_trampoline() at fork_trampoline+0x8/frame 0xc86c1cf4 > --- trap 0, eip =3D 0, esp =3D 0xc86c1d40, ebp =3D 0 --- > db> show thread > Thread 100002 at 0xc89efbc0: > proc (pid 1): 0xc89edb40 > name: kernel > stack: 0xc86c0000-0xc86c1fff > flags: 0x4 pflags: 0x10000 > state: RUNNING (CPU 1) > priority: 84 > container lock: sched lock 1 (0xc1220000) > db> Please try the patch below. It might give an immediate relief, but still there are many offenders in the backtrace. diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c index 83948f2..147926e 100644 --- a/sys/kern/vfs_mountroot.c +++ b/sys/kern/vfs_mountroot.c @@ -672,10 +672,11 @@ parse_mount_dev_present(const char *dev) return (error !=3D 0) ? 0 : 1; } =20 +#define ERRMSGL 255 static int parse_mount(char **conf) { - char errmsg[255]; + char *errmsg; struct mntarg *ma; char *dev, *fs, *opts, *tok; int delay, error, timeout; @@ -707,7 +708,7 @@ parse_mount(char **conf) printf("Trying to mount root from %s:%s [%s]...\n", fs, dev, (opts !=3D NULL) ? opts : ""); =20 - bzero(errmsg, sizeof(errmsg)); + errmsg =3D malloc(ERRMSGL, M_TEMP, M_WAITOK | M_ZERO); =20 if (vfs_byname(fs) =3D=3D NULL) { strlcpy(errmsg, "unknown file system", sizeof(errmsg)); @@ -734,7 +735,7 @@ parse_mount(char **conf) ma =3D mount_arg(ma, "fstype", fs, -1); ma =3D mount_arg(ma, "fspath", "/", -1); ma =3D mount_arg(ma, "from", dev, -1); - ma =3D mount_arg(ma, "errmsg", errmsg, sizeof(errmsg)); + ma =3D mount_arg(ma, "errmsg", errmsg, ERRMSGL); ma =3D mount_arg(ma, "ro", NULL, 0); ma =3D parse_mountroot_options(ma, opts); error =3D kernel_mount(ma, MNT_ROOTFS); @@ -748,11 +749,13 @@ parse_mount(char **conf) printf(".\n"); } free(fs, M_TEMP); + free(errmsg, M_TEMP); if (opts !=3D NULL) free(opts, M_TEMP); /* kernel_mount can return -1 on error. */ return ((error < 0) ? EDOOFUS : error); } +#undef ERRMSGL =20 static int vfs_mountroot_parse(struct sbuf *sb, struct mount *mpdevfs) --dOlf0zsnhJ4/ZPgF Content-Type: application/pgp-signature -----BEGIN PGP SIGNATURE----- Version: GnuPG v2.0.19 (FreeBSD) iQIcBAEBAgAGBQJQvSqbAAoJEJDCuSvBvK1BReQP/RXbi0ki7FLFns4TAlqSmnKL 40xkEdhbtAAQpSsLyf0JX3tgZCPhmr7evS1klFdM0CAYd3OlFl8VA8n63EmBZDp1 zo2BxQjREBtaw6tSiCvA8SCFgX0PvDCjkcdY7/a77e4zMh8+8LYs62HquJsYwV4g Jd9qsS2/8thsAegP8OxcU3sgHkC1EzGPbbwTyz0bNx+XddDDHe6kSITPLt1VdQuK NHG9H3QaO6Wi/H0YxeBCgN+wKifwMq2YzBLBdInMCXVaWEERduLyChUVvFc7k73x 62VAjUj+mltD16HefhGGp6Dy2Tv2gTdUACN6w5XFh/TKazwGQzqyXQx2HuEAyArv yCf55AZuHw8DJ3tca1MF1CSiz6aKJ4ru8b288bBL0ztf+wgyLSnh2dKzyF7gOXEV MUu+ZzV0iyxSjtiHzaVMcmX8SznM0YQQUO0ghhoHQCl5jfCZn1f1vfjTaG0YJig9 xiXPyzpE5Xu1M9mKQIzTIkzoA38kj/E60jsYBEeWmDUwNo30vOmQXtwfS9CaaAXN g7gt2eN2b+q7I0Y3DCSOGKkZNlW6it9rmqfLKqfHPFAXUMNi5NGl2hp2fcPTCGci JESomTPNMMRpOaU0XWeiP6YtBYpEweSamzDST7wchMXeYSjEncOkW6grLR2AWzgN QBl/tmfaXYGODTdbR9cT =kQfc -----END PGP SIGNATURE----- --dOlf0zsnhJ4/ZPgF--