From owner-freebsd-bugs@FreeBSD.ORG Tue Jan 17 11:50:10 2006 Return-Path: X-Original-To: freebsd-bugs@hub.freebsd.org Delivered-To: freebsd-bugs@hub.freebsd.org Received: from mx1.FreeBSD.org (mx1.freebsd.org [216.136.204.125]) by hub.freebsd.org (Postfix) with ESMTP id 2506E16A41F for ; Tue, 17 Jan 2006 11:50:10 +0000 (GMT) (envelope-from gnats@FreeBSD.org) Received: from freefall.freebsd.org (freefall.freebsd.org [216.136.204.21]) by mx1.FreeBSD.org (Postfix) with ESMTP id 5B1C643D45 for ; Tue, 17 Jan 2006 11:50:09 +0000 (GMT) (envelope-from gnats@FreeBSD.org) Received: from freefall.freebsd.org (gnats@localhost [127.0.0.1]) by freefall.freebsd.org (8.13.4/8.13.4) with ESMTP id k0HBo9aO095094 for ; Tue, 17 Jan 2006 11:50:09 GMT (envelope-from gnats@freefall.freebsd.org) Received: (from gnats@localhost) by freefall.freebsd.org (8.13.4/8.13.4/Submit) id k0HBo9Pa095093; Tue, 17 Jan 2006 11:50:09 GMT (envelope-from gnats) Date: Tue, 17 Jan 2006 11:50:09 GMT Message-Id: <200601171150.k0HBo9Pa095093@freefall.freebsd.org> To: freebsd-bugs@FreeBSD.org From: Henri Hennebert Cc: Subject: Re: kern/91631: 6.0-RELEASE freeze repetitively X-BeenThere: freebsd-bugs@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list Reply-To: Henri Hennebert List-Id: Bug reports List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 17 Jan 2006 11:50:10 -0000 The following reply was made to PR kern/91631; it has been noted by GNATS. From: Henri Hennebert To: bug-followup@FreeBSD.org, hlh@ciger.be Cc: Subject: Re: kern/91631: 6.0-RELEASE freeze repetitively Date: Tue, 17 Jan 2006 12:42:16 +0100 One more freeze, this time I run a kernel with WITNESS. WITNESS trigger the following error (in message.log) Jan 16 20:03:45 morzine kernel: lock order reversal Jan 16 20:03:45 morzine kernel: 1st 0xc07cc440 UMA lock (UMA lock) @ /usr/src/sys/vm/uma_core.c:1494 Jan 16 20:03:45 morzine kernel: 2nd 0xc1062144 system map (system map) @ /usr/src/sys/vm/vm_map.c:2317 Jan 16 20:03:45 morzine kernel: KDB: stack backtrace: Jan 16 20:03:45 morzine kernel: kdb_backtrace (c0708103,c1062144,c0707cca,c0707cca,c071e26c) at kdb_backtrace+0x2e Jan 16 20:03:45 morzine kernel: witness_checkorder (c1062144,9,c071e26c,90d,246) at witness_checkorder+0x5e4 Jan 16 20:03:45 morzine kernel: _mtx_lock_flags (c1062144,0,c071e26c,90d,c6303000) at _mtx_lock_flags+0x32 Jan 16 20:03:45 morzine kernel: _vm_map_lock (c10620c0,c071e26c,90d,0,c5fd1b80) at _vm_map_lock+0x37 Jan 16 20:03:45 morzine kernel: vm_map_remove (c10620c0,c6303000,c6304000,e69d4bd8,c06767eb) at vm_map_remove+0x30 Jan 16 20:03:45 morzine kernel: kmem_free (c10620c0,c6303000,1000,e69d4c10,c0675fcf) at kmem_free+0x32 Jan 16 20:03:45 morzine kernel: page_free(c6303000,1000,2,0,2) at page_free+0x3b Jan 16 20:03:45 morzine kernel: zone_drain(c372c3c0,0,c071d8f6,5d6,0) at zone_drain+0x2df Jan 16 20:03:45 morzine kernel: zone_foreach (c0675cf0,e69d4ce0,c068bc96,c34d7a0c,0) at zone_foreach+0x55 Jan 16 20:03:45 morzine kernel: uma_reclaim (c34d7a0c,0,c071f160,2d1,c050250d) at uma_reclaim+0x17 Jan 16 20:03:45 morzine kernel: vm_pageout_scan(0,0,c071f160,604,1388) at vm_pageout_scan+0xf6 Jan 16 20:03:45 morzine kernel: vm_pageout(0,e69d4d38,c070343f,30d,0) at vm_pageout+0x31b Jan 16 20:03:45 morzine kernel: fork_exit(c068cd00,0,e69d4d38) at fork_exit+0x78 Jan 16 20:03:45 morzine kernel: fork_trampoline() at fork_trampoline+0x8 Jan 16 20:03:45 morzine kernel: --- trap 0x1, eip = 0, esp = 0xe69d4d6c, ebp = 0 --- The freeze only happen hours later (Jan 17 10:17:00) this time it is uvscan called by amavisd which is running: KDB: enter: Line break on console [thread pid 61 tid 100052 ] Stopped at kdb_enter+0x30: leave db> bt Tracing pid 61 tid 100052 td 0xc355e300 kdb_enter(c07216f1,e69e0a94,5024ad,c07cf3f9,8) at kdb_enter+0x30 siointr1(c3660800,0,c07214f7,56e,e69e0ad4) at siointr1+0xe7 siointr(c3660800,c07958c0,2,4,c355e300) at siointr+0x35 intr_execute_handlers(c352b090,e69e0aec,e69e0b34,c06c1753,34) at intr_execute_handlers+0x98 lapic_handle_intr(34) at lapic_handle_intr+0x3a Xapic_isr1() at Xapic_isr1+0x33 --- interrupt, eip = 0xc06cb2c7, esp = 0xe69e0b30, ebp = 0xe69e0b34 --- spinlock_exit(c076bc40,8,c0707e12,351) at spinlock_exit+0x27 _mtx_unlock_spin_flags(c076bc40,0,c0707e12,351,c4c826dc) at _mtx_unlock_spin_flags+0x4d witness_checkorder(c37f6844,9,c071d127,2f8,2e5) at witness_checkorder +0x325 _mtx_lock_flags(c37f6844,0,c071d127,2f8,c37f6844) at _mtx_lock_flags +0x32 qsync(c37f6800,0,c071c747,45f,c37f6844) at qsync+0x1b4 ffs_sync(c37f6800,3,c355e300,c355e300,c37f6800) at ffs_sync+0x2ae sync_fsync(e69e0ca0,e69e0cbc,c056c462,c0746f60,e69e0ca0) at sync_fsync +0x176 VOP_FSYNC_APV(c0746f60,e69e0ca0,c355e300,612,0) at VOP_FSYNC_APV+0x3e sync_vnode(c3913720,c355e300,c070bf48,649,0) at sync_vnode+0x152 sched_sync(0,e69e0d38,c070343f,30d,244489dc) at sched_sync+0x26f fork_exit(c056c510,0,e69e0d38) at fork_exit+0x78 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xe69e0d6c, ebp = 0 --- db> show allpcpu Current CPU: 0 cpuid = 0 curthread = 0xc355e300: pid 61 "syncer" curpcb = 0xe69e0d90 fpcurthread = none idlethread = 0xc34da900: pid 12 "idle: cpu0" APIC ID = 3 currentldt = 0x50 spin locks held: exclusive spin mutex sio r = 0 (0xc07cf340) locked @ /usr/src/sys/dev/sio/sio.c:1390 cpuid = 1 curthread = 0xc3b86c00: pid 56078 "uvscan" curpcb = 0xe8f18d90 fpcurthread = none idlethread = 0xc34da780: pid 11 "idle: cpu1" APIC ID = 0 currentldt = 0x50 spin locks held: db> show lockedvnods Locked vnodes 0xc3913660: tag syncer, type VNON usecount 1, writecount 0, refcount 2 mountedhere 0 flags () lock type syncer: EXCL (count 1) by thread 0xc355e300 (pid 61) 0xc5389550: tag ufs, type VDIR usecount 1, writecount 0, refcount 3 mountedhere 0 flags () lock type ufs: EXCL (count 1) by thread 0xc3ce4d80 (pid 56098) ino 8, on dev da1s1h 0xc49cf990: tag ufs, type VREG usecount 0, writecount 0, refcount 1 mountedhere 0 flags (VI_DOOMED) lock type ufs: EXCL (count 1) by thread 0xc3ce4d80 (pid 56098) ino 93, on dev da1s1h db> show alllocks Process 61 (syncer) thread 0xc355e300 (100052) exclusive sleep mutex Giant r = 0 (0xc07647e0) locked @ /usr/src/sys/kern/vfs_subr.c:1597 exclusive spin mutex sio r = 0 (0xc07cf340) locked @ /usr/src/sys/dev/sio/sio.c:1390 db> bt 56098 Tracing pid 56098 tid 100275 td 0xc3ce4d80 sched_switch(c3ce4d80,0,1,c05330a7,5213f3ca) at sched_switch+0x160 mi_switch(1,0,c0707ad9,253,c076b2f8) at mi_switch+0x1d9 turnstile_wait(c07647e0,c355e300,c07647e0,ec,c0705a4e) at turnstile_wait +0x269 _mtx_lock_sleep(c07647e0,c3ce4d80,0,c0705a4e,ec) at _mtx_lock_sleep +0x101 _mtx_lock_flags(c07647e0,0,c0705a4e,ec,0) at _mtx_lock_flags+0x68 msleep(c39ec278,c0764454,50,c070a95a,0) at msleep+0x414 acquire(e8fad998,40,60000,b6,c3ce4d80) at acquire+0x8e lockmgr(c39ec278,3002,c39ec29c,c3ce4d80,e8fad9c0) at lockmgr+0x466 vop_stdlock(e8fada10,c39ec29c,c07523c0,e8fada10,e8fad9d0) at vop_stdlock +0x2f VOP_LOCK_APV(c0752ac0,e8fada10,e8fad9e8,c06e1554,e8fada10) at VOP_LOCK_APV+0x54 ffs_lock(e8fada10,1002,1002,c39ec220,e8fada2c) at ffs_lock+0x19 VOP_LOCK_APV(c07523c0,e8fada10,c070c405,31f,c057ad21) at VOP_LOCK_APV +0x54 vn_lock(c39ec220,1002,c3ce4d80,c05328b0,c070b9df) at vn_lock+0xec dqsync(c49cf990,c3f83a00,0,e8fadaec,c0669664) at dqsync+0x86 dqrele(c49cf990,c3f83a00,c3ce4d80,0,c3916d00) at dqrele+0x35 ufs_reclaim(e8fadb18,e8fadb44,c056dc4f,c07523c0,e8fadb18) at ufs_reclaim +0xa4 VOP_RECLAIM_APV(c07523c0,e8fadb18,c3ce4d80,0,0) at VOP_RECLAIM_APV+0x3e vgonel(c49cf990,0,c070bf48,923,c7aa9210) at vgonel+0x19f vrecycle(c49cf990,c3ce4d80,1,c00,0) at vrecycle+0x4d ufs_inactive(e8fadbbc,e8fadbd4,c056d466,c07523c0,e8fadbbc) at ufs_inactive+0x225 VOP_INACTIVE_APV(c07523c0,e8fadbbc,c070bf48,887,c075f120) at VOP_INACTIVE_APV+0x3e vinactive(c49cf990,c3ce4d80,c070bf48,819,3d7) at vinactive+0x66 vput(c49cf990,ffffffdf,2,c3ce4d80,c37f6800) at vput+0x14e kern_unlink(c3ce4d80,806f6e0,0,e8fadd30,c06d6490) at kern_unlink+0x1ce unlink(c3ce4d80,e8fadd04,4,c3ce4d80,1) at unlink+0x22 syscall(3b,3b,3b,806f6e0,8178004) at syscall+0x2c0 Xint0x80_syscall() at Xint0x80_syscall+0x1f --- syscall (10, FreeBSD ELF32, unlink), eip = 0x38257d3f, esp = 0xbfbfec9c, ebp = 0xbfbfece8 --- db> reset Some help would be really appreciated ... Henri