From owner-freebsd-current Sat Dec 9 20:11:22 2000 From owner-freebsd-current@FreeBSD.ORG Sat Dec 9 20:11:15 2000 Return-Path: Delivered-To: freebsd-current@freebsd.org Received: from mailgate.rz.uni-karlsruhe.de (mailgate.rz.uni-karlsruhe.de [129.13.64.97]) by hub.freebsd.org (Postfix) with ESMTP id 0325437B400; Sat, 9 Dec 2000 20:11:14 -0800 (PST) Received: from rzstud1.rz.uni-karlsruhe.de (rzstud1.rz.uni-karlsruhe.de [129.13.197.1]) by mailgate.rz.uni-karlsruhe.de with esmtp (Exim 3.16 #1) id 144wKQ-00044R-00; Sun, 10 Dec 2000 03:35:06 +0100 Received: from un1i by rzstud1.rz.uni-karlsruhe.de with local (Exim 2.12 #1) id 144wL4-0002Sr-00; Sun, 10 Dec 2000 03:35:46 +0100 Date: Sun, 10 Dec 2000 03:35:46 +0100 From: Philipp Mergenthaler To: dillon@FreeBSD.ORG Cc: freebsd-current@freebsd.org Subject: panic: vm_pageout_flush: partially dirty page Message-ID: <20001210033546.A13896@rz.uni-karlsruhe.de> Mime-Version: 1.0 Content-Type: text/plain; charset=iso-8859-1 Content-Disposition: inline Content-Transfer-Encoding: quoted-printable User-Agent: Mutt/1.2.5i Sender: owner-freebsd-current@FreeBSD.ORG Precedence: bulk X-Loop: FreeBSD.ORG Hi, ever since this commit: ... dillon 2000/11/18 15:06:27 PST =20 Modified files: sys/kern vfs_bio.c vfs_cluster.c vfs_subr.c vfs_vnops.c sys/sys buf.h vnode.h sys/ufs/ffs ffs_inode.c ffs_softdep.c sys/ufs/ufs ufs_readwrite.c sys/vm swap_pager.c vm_page.c vm_page.h vm_pageout.c Log: Implement a low-memory deadlock solution. =20 =2E.. I can very reliable reproduce this panic. I have INN running here (inn-2.3.0 straight from /usr/ports) and feed it articles with suck. Actually suck is run twice in a row (for different news servers) and as soon as the second run starts feeding articles to innd, the panic occurs. (In a few cases the panic occured a bit later, maybe 30 seconds.) I've appended some output from gdb and put a crash dump (96 MB / 17MB gzipped) and a debug kernel on http://ltilx150.etec.uni-karlsruhe.de/p/ (This is with sources from today.) Some additional observations: -The INN is compiled with mmap(). The history file has about 11 MBytes, the active file has just 23 lines :-). -The output from "trace" in ddb has one line more than gdb's "backtrace": [...] sync_fsyc(c76e1f7c) at sync_fsync + 0xcf sched_sync at sched_sync + 0x13a fork_trampoline at fork_trampoline + 0x1c -Apart from this situation, I haven't seen this (or any other) panic. Please let me know if I should provide additional information. Bye, Philipp --=20 http://www.uni-karlsruhe.de/~un1i/ (,.) \\\00 ) \=3D ) cc_|\_,^ gdb -k kernel.73.debug vmcore.73 GNU gdb 4.18 Copyright 1998 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain condition= s. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "i386-unknown-freebsd"... IdlePTD 4055040 initial pcb at 32e6c0 panicstr: from debugger panic messages: --- panic: vm_pageout_flush page 0xc04eacd0 index 0/1: partially dirty page panic: from debugger Uptime: 4m2s dumping to dev da0s1b, offset 26624 dump 96 95 94 93 92 91 90 89 88 87 86 85 84 83 82 81 80 79 78 77 76 75 74 7= 3 72 71 70 69 68 67 66 65 64 63 62 61 60 59 58 57 56 55 54 53 52 51 50 49 4= 8 47 46 45 44 43 42 41 40 39 38 37 36 35 34 33 32 31 30 29 28 27 26 25 24 2= 3 22 21 20 19 18 17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1=20 --- #0 dumpsys () at /usr/src/sys/kern/kern_shutdown.c:477 477 if (dumping++) { (kgdb) bt #0 dumpsys () at /usr/src/sys/kern/kern_shutdown.c:477 #1 0xc0178080 in boot (howto=3D260) at /usr/src/sys/kern/kern_shutdown.c:3= 20 #2 0xc01784d9 in panic (fmt=3D0xc02b70d4 "from debugger") at /usr/src/sys/kern/kern_shutdown.c:570 #3 0xc0132c01 in db_panic (addr=3D-1071064268, have_addr=3D0, count=3D-1,= =20 modif=3D0xc70e1c4c "") at /usr/src/sys/ddb/db_command.c:433 #4 0xc0132ba1 in db_command (last_cmdp=3D0xc02f81d4, cmd_table=3D0xc02f803= 4,=20 aux_cmd_tablep=3D0xc031acb8) at /usr/src/sys/ddb/db_command.c:333 #5 0xc0132c66 in db_command_loop () at /usr/src/sys/ddb/db_command.c:455 #6 0xc0134e2b in db_trap (type=3D3, code=3D0) at /usr/src/sys/ddb/db_trap.= c:71 #7 0xc028d8c6 in kdb_trap (type=3D3, code=3D0, regs=3D0xc70e1d4c) at /usr/src/sys/i386/i386/db_interface.c:163 #8 0xc029989c in trap (frame=3D{tf_fs =3D 16, tf_es =3D 16, tf_ds =3D 16, = tf_edi =3D 0,=20 tf_esi =3D 256, tf_ebp =3D -955376232, tf_isp =3D -955376264,=20 tf_ebx =3D 2097666, tf_edx =3D -1072980320, tf_ecx =3D 32, tf_eax =3D= 18,=20 tf_trapno =3D 3, tf_err =3D 0, tf_eip =3D -1071064268, tf_cs =3D 8,= =20 tf_eflags =3D 2097222, tf_esp =3D -1070695009, tf_ss =3D -1070848829}) at /usr/src/sys/i386/i386/trap.c:589 #9 0xc028db34 in Debugger (msg=3D0xc02c24c3 "panic") at machine/cpufunc.h:= 60 #10 0xc01784d0 in panic ( fmt=3D0xc02e2580 "vm_pageout_flush page %p index %d/%d: partially dirty= page") at /usr/src/sys/kern/kern_shutdown.c:568 #11 0xc027798f in vm_pageout_flush (mc=3D0xc70e1df4, count=3D1, flags=3D0) at /usr/src/sys/vm/vm_pageout.c:378 #12 0xc0274902 in vm_object_page_clean (object=3D0xc78754e0, start=3D0, end= =3D0,=20 flags=3D4) at /usr/src/sys/vm/vm_object.c:655 #13 0xc01ae696 in vfs_msync (mp=3D0xc0aa1200, flags=3D2) at /usr/src/sys/kern/vfs_subr.c:2597 #14 0xc01aea73 in sync_fsync (ap=3D0xc70e1f7c) at /usr/src/sys/kern/vfs_subr.c:2866 #15 0xc01ac9e6 in sched_sync () at vnode_if.h:423 (kgdb) frame 15 #15 0xc01ac9e6 in sched_sync () at vnode_if.h:423 423 rc =3D VCALL(vp, VOFFSET(vop_fsync), &a); (kgdb) print a $1 =3D {a_desc =3D 0xc02f2a40, a_vp =3D 0xc781bd00, a_cred =3D 0xc0699e00,= =20 a_waitfor =3D 3, a_p =3D 0xc6a0cfe0} (kgdb) print vp $2 =3D (struct vnode *) 0xc781bd00 (kgdb) print *((struct vnode *) 0xc781bd00) $3 =3D {v_flag =3D 2097152, v_usecount =3D 1, v_writecount =3D 0, v_holdcnt= =3D 0,=20 v_id =3D 92, v_mount =3D 0xc0aa1200, v_op =3D 0xc0a56600, v_freelist =3D { tqe_next =3D 0x0, tqe_prev =3D 0x0}, v_mntvnodes =3D {le_next =3D 0x0,= =20 le_prev =3D 0xc7819524}, v_cleanblkhd =3D {tqh_first =3D 0x0,=20 tqh_last =3D 0xc781bd2c}, v_dirtyblkhd =3D {tqh_first =3D 0x0,=20 tqh_last =3D 0xc781bd34}, v_synclist =3D {le_next =3D 0x0,=20 le_prev =3D 0xc0a4a798}, v_numoutput =3D 0, v_type =3D VNON, v_un =3D { vu_mountedhere =3D 0x0, vu_socket =3D 0x0, vu_spec =3D {vu_specinfo =3D= 0x0,=20 vu_specnext =3D {sle_next =3D 0x0}}, vu_fifoinfo =3D 0x0}, v_lease = =3D 0x0,=20 v_lastw =3D 0, v_cstart =3D 0, v_lasta =3D 0, v_clen =3D 0, v_object =3D = 0x0,=20 v_interlock =3D {mtx_lock =3D 8, mtx_recurse =3D 0, mtx_saveintr =3D 0,= =20 mtx_description =3D 0xc02c82e5 "vnode interlock", mtx_blocked =3D { tqh_first =3D 0x0, tqh_last =3D 0xc781bd7c}, mtx_contested =3D {le_ne= xt =3D 0x0,=20 le_prev =3D 0x0}, mtx_next =3D 0xc781bc6c, mtx_prev =3D 0xc781be6c}, = v_lock =3D { lk_interlock =3D 0xc0692e28, lk_flags =3D 16777216, lk_sharecount =3D 0= ,=20 lk_waitcount =3D 0, lk_exclusivecount =3D 0, lk_prio =3D 20,=20 lk_wmesg =3D 0xc02c82f5 "vnlock", lk_timo =3D 0, lk_lockholder =3D -1},= =20 v_vnlock =3D 0x0, v_tag =3D VT_VFS, v_data =3D 0x0, v_cache_src =3D { lh_first =3D 0x0}, v_cache_dst =3D {tqh_first =3D 0x0, tqh_last =3D 0xc= 781bdc4},=20 v_dd =3D 0xc781bd00, v_ddid =3D 0, v_pollinfo =3D {vpi_lock =3D {lock_dat= a =3D 0},=20 vpi_selinfo =3D {si_pid =3D 0, si_note =3D {slh_first =3D 0x0}, si_flag= s =3D 0},=20 vpi_events =3D 0, vpi_revents =3D 0}, v_vxproc =3D 0x0} (kgdb) down #14 0xc01aea73 in sync_fsync (ap=3D0xc70e1f7c) at /usr/src/sys/kern/vfs_subr.c:2866 2866 vfs_msync(mp, MNT_NOWAIT); (kgdb) print mp $4 =3D (struct mount *) 0xc0aa1200 (kgdb) print *((struct mount *) 0xc0aa1200) $5 =3D {mnt_list =3D {tqe_next =3D 0xc0ad9400, tqe_prev =3D 0xc0aa1a00},=20 mnt_op =3D 0xc030df20, mnt_vfc =3D 0xc030df60, mnt_vnodecovered =3D 0xc70= d8200,=20 mnt_syncer =3D 0xc781bd00, mnt_vnodelist =3D {lh_first =3D 0xc7828e00},= =20 mnt_lock =3D {lk_interlock =3D 0xc0692e00, lk_flags =3D 17825792,=20 lk_sharecount =3D 1, lk_waitcount =3D 0, lk_exclusivecount =3D 0, lk_pr= io =3D 20,=20 lk_wmesg =3D 0xc02c88a1 "vfslock", lk_timo =3D 0, lk_lockholder =3D -1}= ,=20 mnt_writeopcount =3D 1, mnt_flag =3D 4096, mnt_kern_flag =3D 0,=20 mnt_maxsymlinklen =3D 60, mnt_stat =3D {f_spare2 =3D 0, f_bsize =3D 1024,= =20 f_iosize =3D 8192, f_blocks =3D 1817777, f_bfree =3D 201403, f_bavail = =3D 55981,=20 f_files =3D 445438, f_ffree =3D 309929, f_fsid =3D {val =3D {901998103,= =20 703157984}}, f_owner =3D 0, f_type =3D 6, f_flags =3D 4096,=20 f_syncwrites =3D 110, f_asyncwrites =3D 256,=20 f_fstypename =3D "ufs", '\000' ,=20 f_mntonname =3D "/usr", '\000' , f_syncreads =3D 727,= =20 f_asyncreads =3D 213, f_spares1 =3D 0,=20 f_mntfromname =3D "/dev/da0s1f", '\000' , f_spares2 = =3D 0,=20 f_spare =3D {0, 0}}, mnt_data =3D 0xc0aa1000, mnt_time =3D 0,=20 mnt_iosize_max =3D 65536} (kgdb) down #13 0xc01ae696 in vfs_msync (mp=3D0xc0aa1200, flags=3D2) at /usr/src/sys/kern/vfs_subr.c:2597 2597 vm_object_page_clean(obj, 0, 0, flags =3D=3D MNT_WAIT ? OBJPC_SYN= C : OBJPC_NOSYNC); (kgdb) print obj $6 =3D (struct vm_object *) 0xc78754e0 (kgdb) print *((struct vm_object *) 0xc78754e0) $7 =3D {object_list =3D {tqe_next =3D 0xc787fa20, tqe_prev =3D 0xc7875420},= =20 shadow_head =3D {tqh_first =3D 0x0, tqh_last =3D 0xc78754e8}, shadow_list= =3D { tqe_next =3D 0x0, tqe_prev =3D 0xc787fc10}, memq =3D {tqh_first =3D 0xc= 04eacd0,=20 tqh_last =3D 0xc04eacdc}, generation =3D 31, size =3D 1, ref_count =3D = 1,=20 shadow_count =3D 0, hash_rand =3D -767679, type =3D 2 '\002', flags =3D 5= 12,=20 pg_color =3D 13, paging_in_progress =3D 0, resident_page_count =3D 1,=20 backing_object =3D 0x0, backing_object_offset =3D 0, pager_object_list = =3D { tqe_next =3D 0x0, tqe_prev =3D 0x0}, handle =3D 0xc70d6900, un_pager = =3D {vnp =3D { vnp_size =3D 932}, devp =3D {devp_pglist =3D {tqh_first =3D 0x3a4,=20 tqh_last =3D 0x0}}, swp =3D {swp_bcount =3D 932}}} (kgdb) down #12 0xc0274902 in vm_object_page_clean (object=3D0xc78754e0, start=3D0, end= =3D0,=20 flags=3D4) at /usr/src/sys/vm/vm_object.c:655 655 vm_pageout_flush(ma, runlen, pagerflags); (kgdb) print runlen $8 =3D 1 (kgdb) print pagerflags $9 =3D 0 (kgdb) print ma $10 =3D (vm_page_t (*)[0]) 0xc70e1df4 (kgdb) print *(struct vm_page *)ma[0] $14 =3D {pageq =3D {tqe_next =3D 0xc04eacd0, tqe_prev =3D 0xc70e1e28},=20 hnext =3D 0xc01a4bc0, listq =3D {tqe_next =3D 0x0, tqe_prev =3D 0xc28590e= 4},=20 object =3D 0xc70e1e28, pindex =3D 3222948849, phys_addr =3D 0, md =3D { pv_list_count =3D -1031434144, pv_list =3D {tqh_first =3D 0x1, tqh_last= =3D 0x0}},=20 queue =3D 37092, flags =3D 49797, pc =3D 0, wire_count =3D 0, hold_count = =3D 7780,=20 act_count =3D 14 '\016', busy =3D 199 '=C7', valid =3D 97 'a', dirty =3D = 64 '@'} (kgdb) print *((struct vm_page *)ma[1]) $15 =3D {pageq =3D {tqe_next =3D 0x28c04eac, tqe_prev =3D 0xc0c70e1e},=20 hnext =3D 0xc01a4b, listq =3D {tqe_next =3D 0xe4000000, tqe_prev =3D 0x28= c28590},=20 object =3D 0xf1c70e1e, pindex =3D 12589643, phys_addr =3D 1610612736, md = =3D { pv_list_count =3D 29525392, pv_list =3D {tqh_first =3D 0x0,=20 tqh_last =3D 0xe4000000}}, queue =3D 34192, flags =3D 194, pc =3D 0,= =20 wire_count =3D 25600, hold_count =3D 3614, act_count =3D 199 '=C7', busy = =3D 97 'a',=20 valid =3D 64 '@', dirty =3D 26 '\032'} (kgdb) down #11 0xc027798f in vm_pageout_flush (mc=3D0xc70e1df4, count=3D1, flags=3D0) at /usr/src/sys/vm/vm_pageout.c:378 378 KASSERT(mc[i]->valid =3D=3D VM_PAGE_BITS_ALL && mc[i]->dirty =3D=3D V= M_PAGE_BITS_ALL, ("vm_pageout_flush page %p index %d/%d: partially dirty pa= ge", mc[i], i, count)); (kgdb) print i $16 =3D 0 (kgdb) print count $17 =3D 1 (kgdb) print mc[0] $18 =3D 0xc04eacd0 (kgdb) print *mc[0] $19 =3D {pageq =3D {tqe_next =3D 0xc04baa6c, tqe_prev =3D 0xc0500af8}, hnex= t =3D 0x0,=20 listq =3D {tqe_next =3D 0x0, tqe_prev =3D 0xc78754f8}, object =3D 0xc7875= 4e0,=20 pindex =3D 0, phys_addr =3D 39374848, md =3D {pv_list_count =3D 1, pv_lis= t =3D { tqh_first =3D 0xc0635e40, tqh_last =3D 0xc0635e48}}, queue =3D 33,=20 flags =3D 160, pc =3D 13, wire_count =3D 0, hold_count =3D 0, act_count = =3D 5 '\005',=20 busy =3D 0 '\000', valid =3D 255 '=FF', dirty =3D 252 '=FC'} (kgdb) quit To Unsubscribe: send mail to majordomo@FreeBSD.org with "unsubscribe freebsd-current" in the body of the message