Date: Mon, 18 Oct 2004 18:13:00 -0400 (EDT) From: Robert Watson <rwatson@FreeBSD.org> To: current@FreeBSD.org Subject: uma_zfree: Freeing to non free bucket index. Message-ID: <Pine.NEB.3.96L.1041018180742.47572G-100000@fledge.watson.org>
next in thread | raw e-mail | index | archive | help
I've not seen this UMA failure before -- saw it under a high web load on
an SMP Xeon here. Some debugging details from DDB below. I have a
workable core; a few kgdb output blips are below the DDB output.=20
Robert N M Watson FreeBSD Core Team, TrustedBSD Projects
robert@fledge.watson.org Principal Research Scientist, McAfee Research
Heavy web service load on hippy.rv.nailabs.com with GENERIC kernel and
accept lock patches.=20
FreeBSD/i386 (hippy.rv.nailabs.com) (ttyd0)
login: panic: uma_zfree: Freeing to non free bucket index.
cpuid =3D 2
KDB: enter: panic
[thread 100014]
Stopped at kdb_enter+0x2b: nop
db> trace
kdb_enter(c07fc72c) at kdb_enter+0x2b
panic(c0815e8e,1,2,c22583c0,c2821100) at panic+0x127
uma_zfree_arg(c101fc60,c2821100,0) at uma_zfree_arg+0xa5
mb_free_ext(c2821100) at mb_free_ext+0x39
m_freem(c2821100,0,0,1,1) at m_freem+0x21
tcp_input(c2821100,14,c2821100,0,0) at tcp_input+0x2d1c
ip_input(c2821100) at ip_input+0x50d
netisr_processqueue(c08eae58) at netisr_processqueue+0x6e
swi_net(0) at swi_net+0xbe
ithread_loop(c2260c00,e3384d48,c2260c00,c05f7d50,0) at ithread_loop+0x124
fork_exit(c05f7d50,c2260c00,e3384d48) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip =3D 0, esp =3D 0xe3384d7c, ebp =3D 0 ---
db> show locks
exclusive sleep mutex UMA pcpu r =3D 0 (0xc08f8548) locked @ vm/uma_core.c:=
2215
exclusive sleep mutex inp (tcpinp) r =3D 0 (0xc2b4d2ac) locked @ netinet/tc=
p_input.c:743
exclusive sleep mutex tcp r =3D 0 (0xc08ec02c) locked @ netinet/tcp_input.c=
:617
db> show pcpu
cpuid =3D 2
curthread =3D 0xc2268600: pid 38 "swi1: net"
curpcb =3D 0xe3384da0
fpcurthread =3D none
idlethread =3D 0xc2262780: pid 12 "idle: cpu2"
APIC ID =3D 2
currentldt =3D 0x28
spin locks held:
db> ps
pid proc uarea uid ppid pgrp flag stat wmesg wchan cmd
619 c2b1ce00 ef357000 0 507 507 0000100 [SLPQ kqread 0xc27fb300][S=
LP] httpd
618 c2b1cc00 ef356000 0 507 507 0000100 [SLPQ kqread 0xc2aacd00][S=
LP] httpd
617 c2931e00 ef240000 0 507 507 0000100 [SLPQ kqread 0xc2aad500][S=
LP] httpd
616 c2b22600 ef35b000 0 507 507 0000100 [SLPQ kqread 0xc27fb600][S=
LP] httpd
615 c2931800 ef23d000 80 507 507 0000100 [Can run] httpd
614 c2931a00 ef23e000 80 507 507 0000100 [SLPQ accept 0xc2800916][S=
LP] httpd
613 c2735000 ef16e000 80 507 507 0000100 [SLPQ sbwait 0xc2acac64][S=
LP] httpd
589 c2afe200 ef301000 80 507 507 0000100 [SLPQ sbwait 0xc2b3cda8][S=
LP] httpd
588 c2afe400 ef302000 80 507 507 0000100 [SLPQ sbwait 0xc2b3c388][S=
LP] httpd
587 c2afe600 ef303000 80 507 507 0000100 [SLPQ sbwait 0xc2aca9dc][S=
LP] httpd
586 c26eec00 ecf80000 80 507 507 0000100 [SLPQ sbwait 0xc2b44610][S=
LP] httpd
585 c2735c00 ef194000 80 507 507 0000100 [SLPQ sbwait 0xc2b23b20][S=
LP] httpd
584 c26eea00 ecf7f000 80 507 507 0000100 [SLPQ accept 0xc2800916][S=
LP] httpd
583 c2795a00 ef1b5000 80 507 507 0000100 [SLPQ sbwait 0xc2aca754][S=
LP] httpd
582 c2795400 ef1b2000 80 507 507 0000100 [SLPQ sbwait 0xc2b23da8][S=
LP] httpd
581 c2797000 ef1b8000 80 507 507 0000100 [Can run] httpd
580 c273a800 ef19a000 80 507 507 0000100 [Can run] httpd
579 c2795000 ef1b0000 80 507 507 0000100 [SLPQ sbwait 0xc2b23100][S=
LP] httpd
578 c273ae00 ef19d000 80 507 507 0000100 [Can run] httpd
577 c2797400 ef1f9000 80 507 507 0000100 [SLPQ sbwait 0xc2b3cc64][S=
LP] httpd
576 c273aa00 ef19b000 80 507 507 0000100 [SLPQ sbwait 0xc2acada8][S=
LP] httpd
575 c2795e00 ef1b7000 80 507 507 0000100 [SLPQ sbwait 0xc2b234cc][S=
LP] httpd
574 c26ed000 ecf36000 80 507 507 0000100 [Can run] httpd
573 c2797200 ef1b9000 80 507 507 0000100 [SLPQ accept 0xc2800916][S=
LP] httpd
572 c2795800 ef1b4000 80 507 507 0000100 [Can run] httpd
571 c273ac00 ef19c000 80 507 507 0000100 [SLPQ sbwait 0xc2ac94cc][S=
LP] httpd
570 c2930600 ef216000 80 507 507 0000100 [Can run] httpd
569 c2930400 ef215000 0 1 569 0004002 [SLPQ ttyin 0xc24ab010][SL=
P] getty
568 c2797c00 ef1fd000 0 1 568 0004002 [SLPQ ttyin 0xc24c9410][SL=
P] getty
567 c2930e00 ef21a000 0 1 567 0004002 [SLPQ ttyin 0xc24ca410][SL=
P] getty
566 c26ed600 ecf39000 0 1 566 0004002 [SLPQ ttyin 0xc24ca010][SL=
P] getty
565 c2797a00 ef1fc000 0 1 565 0004002 [SLPQ ttyin 0xc24c8c10][SL=
P] getty
564 c2931600 ef23c000 0 1 564 0004002 [SLPQ ttyin 0xc24c8810][SL=
P] getty
563 c273a600 ef199000 0 1 563 0004002 [SLPQ ttyin 0xc24c0c10][SL=
P] getty
562 c2797e00 ef1fe000 0 1 562 0004002 [SLPQ ttyin 0xc24c8010][SL=
P] getty
561 c2797600 ef1fa000 0 1 561 0004002 [SLPQ ttyin 0xc24c8410][SL=
P] getty
558 c2930c00 ef219000 88 511 65 000c182 (threaded) mysqld
thread 0xc279bc00 ksegrp 0xc2739a10 [SLPQ kserel 0xc2739a50][SLP]
thread 0xc2932000 ksegrp 0xc27394d0 [SLPQ kserel 0xc2739510][SLP]
thread 0xc279b600 ksegrp 0xc2739a10 [SLPQ kserel 0xc2739a50][SLP]
thread 0xc279b900 ksegrp 0xc2739a10 [SLPQ select 0xc08e9ee4][SLP]
thread 0xc2af8600 ksegrp 0xc2739a10 [SLPQ kserel 0xc2739a50][SLP]
thread 0xc2af8300 ksegrp 0xc2739a10 [SLPQ kserel 0xc2739a50][SLP]
thread 0xc2af8000 ksegrp 0xc2739540 [SLPQ sigwait 0xef2c0c2c][SLP]
thread 0xc2932300 ksegrp 0xc27395b0 [SLPQ ksesigwait 0xc2930d3c][SLP]
511 c26ed200 ecf37000 0 1 65 0004002 [SLPQ wait 0xc26ed200][SLP=
] sh
507 c2735400 ef190000 0 1 507 0000000 [SLPQ select 0xc08e9ee4][S=
LP] httpd
489 c2735800 ef192000 0 1 489 0000000 [SLPQ nanslp 0xc08bfccc][S=
LP] cron
476 c2931400 ef23b000 25 1 476 0000100 [SLPQ pause 0xc2931438][SL=
P] sendmail
472 c2735e00 ef195000 0 1 472 0000100 [SLPQ select 0xc08e9ee4][S=
LP] sendmail
467 c273a200 ef197000 0 1 467 0000100 [SLPQ select 0xc08e9ee4][S=
LP] sshd
441 c26ed400 ecf38000 0 1 441 0000000 [SLPQ select 0xc08e9ee4][S=
LP] lpd
424 c2797800 ef1fb000 0 1 424 0000000 [SLPQ select 0xc08e9ee4][S=
LP] usbd
400 c2931000 ef21b000 0 1 400 0000000 [SLPQ select 0xc08e9ee4][S=
LP] rpc.statd
394 c2795c00 ef1b6000 0 390 390 0000000 [SLPQ - 0xc26c1a00][SLP] n=
fsd
393 c2795200 ef1b1000 0 390 390 0000000 [SLPQ - 0xc26dbc00][SLP] n=
fsd
392 c273a000 ef196000 0 390 390 0000000 [SLPQ - 0xc26dc400][SLP] n=
fsd
391 c26eee00 ecf81000 0 390 390 0000000 [SLPQ - 0xc26cd200][SLP] n=
fsd
390 c2930000 ef1ae000 0 1 390 0000000 [SLPQ select 0xc08e9ee4][S=
LP] nfsd
388 c2735a00 ef193000 0 1 388 0000000 [SLPQ select 0xc08e9ee4][S=
LP] mountd
322 c2930200 ef1af000 0 1 322 0000000 [SLPQ select 0xc08e9ee4][S=
LP] ypbind
309 c2931200 ef21c000 0 1 309 0000000 [SLPQ select 0xc08e9ee4][S=
LP] rpcbind
294 c2378e00 e4e81000 0 1 294 0000000 [SLPQ select 0xc08e9ee4][S=
LP] syslogd
271 c273a400 ef198000 0 1 271 0000000 [SLPQ select 0xc08e9ee4][S=
LP] devd
242 c2795600 ef1b3000 0 1 242 0000000 [SLPQ select 0xc08e9ee4][S=
LP] dhclient
64 c26ed800 ecf3a000 0 0 0 0000204 [SLPQ - 0xe4e4fd14][SLP] s=
chedcpu
63 c26eda00 ecf3b000 0 0 0 0000204 [SLPQ - 0xc08f192c][SLP] n=
fsiod 3
62 c26edc00 ecf3c000 0 0 0 0000204 [SLPQ - 0xc08f1928][SLP] n=
fsiod 2
61 c26ede00 ecf3d000 0 0 0 0000204 [SLPQ - 0xc08f1924][SLP] n=
fsiod 1
60 c26ee000 ecf3e000 0 0 0 0000204 [SLPQ - 0xc08f1920][SLP] n=
fsiod 0
59 c26ee200 ecf3f000 0 0 0 0000204 [SLPQ vlruwt 0xc26ee200][S=
LP] vnlru
58 c26ee400 ecf7c000 0 0 0 0000204 [SLPQ syncer 0xc08bfa4c][S=
LP] syncer
57 c26ee600 ecf7d000 0 0 0 0000204 [SLPQ psleep 0xc08ea4ac][S=
LP] bufdaemon
56 c26ee800 ecf7e000 0 0 0 000020c [SLPQ pgzero 0xc08f8270][S=
LP] pagezero
55 c22d0400 e4e38000 0 0 0 0000204 [SLPQ psleep 0xc08f82c4][S=
LP] vmdaemon
54 c22d0600 e4e39000 0 0 0 0000204 [SLPQ psleep 0xc08f8280][S=
LP] pagedaemon
53 c22d0800 e4e3a000 0 0 0 0000204 [RUNQ] swi0: sio
52 c22d0a00 e4e3b000 0 0 0 0000204 [SLPQ - 0xc23ac83c][SLP] f=
dc0
51 c22d0c00 e4e3c000 0 0 0 0000204 [SLPQ usbevt 0xc249e210][S=
LP] usb1
50 c22d0e00 e4e3d000 0 0 0 0000204 [SLPQ usbtsk 0xc08b7bb8][S=
LP] usbtask
49 c2378000 e4e3e000 0 0 0 0000204 [SLPQ usbevt 0xc249a210][S=
LP] usb0
48 c2378200 e4e3f000 0 0 0 0000204 [SLPQ idle 0xc2376600][SLP=
] aic_recovery0
47 c2378400 e4e40000 0 0 0 0000204 [SLPQ idle 0xc2376600][SLP=
] aic_recovery0
9 c2378600 e4e7d000 0 0 0 0000204 [SLPQ actask 0xc0a23a2c][S=
LP] acpi_task2
8 c2378800 e4e7e000 0 0 0 0000204 [SLPQ actask 0xc0a23a2c][S=
LP] acpi_task1
7 c2378a00 e4e7f000 0 0 0 0000204 [SLPQ actask 0xc0a23a2c][S=
LP] acpi_task0
46 c2378c00 e4e80000 0 0 0 0000204 [IWAIT] swi6:+
45 c22c3c00 e4e0e000 0 0 0 0000204 [IWAIT] swi6: task queue
44 c22c3e00 e4e0f000 0 0 0 0000204 [IWAIT] swi6: acpitaskq
6 c22cc000 e4e10000 0 0 0 0000204 [SLPQ - 0xc22f5640][SLP] k=
queue taskq
43 c22cc200 e4e11000 0 0 0 0000204 [IWAIT] swi2: cambio
42 c22cc400 e4e12000 0 0 0 0000204 [IWAIT] swi5:+
5 c22cc600 e4e13000 0 0 0 0000204 [SLPQ - 0xc22f5840][SLP] t=
hread taskq
41 c22cc800 e4e14000 0 0 0 0000204 [SLPQ - 0xc08b5900][SLP] y=
arrow
4 c22cca00 e4e33000 0 0 0 0000204 [SLPQ - 0xc08ba568][SLP] g=
_down
3 c22ccc00 e4e34000 0 0 0 0000204 [SLPQ - 0xc08ba564][SLP] g=
_up
2 c22cce00 e4e35000 0 0 0 0000204 [SLPQ - 0xc08ba55c][SLP] g=
_event
40 c22d0000 e4e36000 0 0 0 0000204 [IWAIT] swi3: vm
39 c22d0200 e4e37000 0 0 0 000020c [RUNQ] swi4: clock sio
38 c22b3600 e4de5000 0 0 0 0000204 [CPU 2] swi1: net
37 c22b3800 e4de6000 0 0 0 0000204 [IWAIT] irq0: clk
36 c22b3a00 e4de7000 0 0 0 0000204 [CPU 0] irq23: xl0 uhci1
35 c22b3c00 e4de8000 0 0 0 0000204 [IWAIT] irq22: ahc0
34 c22b3e00 e4de9000 0 0 0 0000204 [IWAIT] irq21:
33 c22c3000 e4e08000 0 0 0 0000204 [IWAIT] irq20: em0
32 c22c3200 e4e09000 0 0 0 0000204 [IWAIT] irq19: uhci0
31 c22c3400 e4e0a000 0 0 0 0000204 [IWAIT] irq18:
30 c22c3600 e4e0b000 0 0 0 0000204 [IWAIT] irq17:
29 c22c3800 e4e0c000 0 0 0 0000204 [IWAIT] irq16: fwohci0
28 c22c3a00 e4e0d000 0 0 0 0000204 [IWAIT] irq15: ata1
27 c226b200 e339c000 0 0 0 0000204 [IWAIT] irq14: ata0
26 c226b400 e339d000 0 0 0 0000204 [IWAIT] irq13:
25 c226b600 e339e000 0 0 0 0000204 [IWAIT] irq12:
24 c226b800 e33bd000 0 0 0 0000204 [IWAIT] irq11:
23 c226ba00 e33be000 0 0 0 0000204 [IWAIT] irq10:
22 c226bc00 e33bf000 0 0 0 0000204 [IWAIT] irq9: acpi0
21 c226be00 e33c0000 0 0 0 0000204 [IWAIT] irq8: rtc
20 c22b3000 e4de2000 0 0 0 0000204 [IWAIT] irq7: ppc0
19 c22b3200 e4de3000 0 0 0 0000204 [IWAIT] irq6: fdc0
18 c22b3400 e4de4000 0 0 0 0000204 [IWAIT] irq5:
17 c2261000 e3357000 0 0 0 0000204 [IWAIT] irq4: sio0
16 c2261200 e3394000 0 0 0 0000204 [IWAIT] irq3: sio1
15 c2261400 e3395000 0 0 0 0000204 [IWAIT] irq1: atkbd0
14 c2261600 e3396000 0 0 0 000020c [Can run] idle: cpu0
13 c2261800 e3397000 0 0 0 000020c [CPU 1] idle: cpu1
12 c2261a00 e3398000 0 0 0 000020c [Can run] idle: cpu2
11 c2261c00 e3399000 0 0 0 000020c [CPU 3] idle: cpu3
1 c2261e00 e339a000 0 0 1 0004200 [SLPQ wait 0xc2261e00][SLP=
] init
10 c226b000 e339b000 0 0 0 0000204 [SLPQ ktrace 0xc08bdc58][S=
LP] ktrace
0 c08ba6c0 c0c1f000 0 0 0 0000200 [SLPQ sched 0xc08ba6c0][SL=
P] swapper
db> trace 615
sched_switch(c2932900,0,1) at sched_switch+0x16f
mi_switch(1,0) at mi_switch+0x264
sleepq_switch(c2b3c9dc,0,ef231bac,c060f686,c2b3c9dc) at sleepq_switch+0xe0
sleepq_wait_sig(c2b3c9dc,0,100,c0802936,34a) at sleepq_wait_sig+0xc
msleep(c2b3c9dc,c2b3c9ac,158,c0802bbc,0) at msleep+0x2da
sbwait(c2b3c994,c2b3c944,c2b3c944,c2b3c9ac,0) at sbwait+0x4e
sosend(c2b3c8dc,0,ef231c88,0,0) at sosend+0x33c
soo_write(c271a550,ef231c88,c2adf800,0,c2932900) at soo_write+0x46
dofilewrite(c2932900,c271a550,3,bfbfcb50,2000) at dofilewrite+0xa8
write(c2932900,ef231d14,3,5,296) at write+0x39
syscall(2f,2f,2f,2000,809a044) at syscall+0x227
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip =3D 0x2812558b, esp =3D 0xbfbfca=
4c, ebp =3D 0xbfbfca68 ---
db> show locks 615
db> trace 581
sched_switch(c237a780,c2268300,6) at sched_switch+0x16f
mi_switch(6,c2268300,c2268450,c2268300,e4e70cc8) at mi_switch+0x264
maybe_preempt(c2268300) at maybe_preempt+0x156
sched_add(c2268300,4,c2260d00,c2268300,c22b3a00) at sched_add+0x153
setrunqueue(c2268300,4) at setrunqueue+0xab
ithread_schedule(c2260d00,17,c237a780,2819c5ec,80e2300) at ithread_schedule=
+0xb3
intr_execute_handlers(c225a658,e4e70d44,17,bfbfcba8,c0780c83) at intr_execu=
te_handlers+0xf5
lapic_handle_intr(47) at lapic_handle_intr+0x2e
Xapic_isr2() at Xapic_isr2+0x33
--- interrupt, eip =3D 0x2818ead2, esp =3D 0xbfbfcb74, ebp =3D 0xbfbfcba8 -=
--
db> show locks 581
db> trace 580
sched_switch(c26f0780,0,1) at sched_switch+0x16f
mi_switch(1,0) at mi_switch+0x264
turnstile_wait(c08ec02c,c26ef780,c08ec02c,2,c07fbabd,21e) at turnstile_wait=
+0x2f8
_mtx_lock_sleep(c08ec02c,c26f0780,0,c08091ed,26f) at _mtx_lock_sleep+0x142
_mtx_lock_flags(c08ec02c,0,c08091ed,26f,bfbfcbd0) at _mtx_lock_flags+0x85
tcp_usr_send(c2b44ca8,4,c2c3bc00,0,0) at tcp_usr_send+0x2c
sosend(c2b44ca8,0,ecf6fc88,c2c3bc00,0) at sosend+0x5e7
soo_write(c2719110,ecf6fc88,c2adf880,0,c26f0780) at soo_write+0x46
dofilewrite(c26f0780,c2719110,3,bfbfcbd0,2000) at dofilewrite+0xa8
write(c26f0780,ecf6fd14,3,a,292) at write+0x39
syscall(2f,2f,2f,2000,809a044) at syscall+0x227
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip =3D 0x2812558b, esp =3D 0xbfbfca=
cc, ebp =3D 0xbfbfcae8 ---
db> show locks 580
db> trace 578
sched_switch(c26f0300,c2268300,6) at sched_switch+0x16f
mi_switch(6,c2268300,c2268450,c2268300,ecf66cc8) at mi_switch+0x264
maybe_preempt(c2268300) at maybe_preempt+0x156
sched_add(c2268300,4,c2260d00,c2268300,c22b3a00) at sched_add+0x153
setrunqueue(c2268300,4) at setrunqueue+0xab
ithread_schedule(c2260d00,17,c26f0300,282085bc,80bf034) at ithread_schedule=
+0xb3
intr_execute_handlers(c225a658,ecf66d44,17,bfbfec98,c0780c83) at intr_execu=
te_handlers+0xf5
lapic_handle_intr(47) at lapic_handle_intr+0x2e
Xapic_isr2() at Xapic_isr2+0x33
--- interrupt, eip =3D 0x28200047, esp =3D 0xbfbfe870, ebp =3D 0xbfbfec98 -=
--
db> show locks 578
db> trace 574
sched_switch(c2379c00,c2268300,6) at sched_switch+0x16f
mi_switch(6,c2268300,c2268450,c2268300,e4e5baa4) at mi_switch+0x264
maybe_preempt(c2268300) at maybe_preempt+0x156
sched_add(c2268300,4,c2260d00,c2268300,c22b3a00) at sched_add+0x153
setrunqueue(c2268300,4) at setrunqueue+0xab
ithread_schedule(c2260d00,17,c2379c00,c2268600,c08ec02c) at ithread_schedul=
e+0xb3
intr_execute_handlers(c225a658,e4e5bb20,17,e4e5bb70,c0780c83) at intr_execu=
te_handlers+0xf5
lapic_handle_intr(47) at lapic_handle_intr+0x2e
Xapic_isr2() at Xapic_isr2+0x33
--- interrupt, eip =3D 0xc06022d8, esp =3D 0xe4e5bb64, ebp =3D 0xe4e5bb70 -=
--
_mtx_lock_sleep(c08ec02c,c2379c00,0,c08091ed,26f) at _mtx_lock_sleep+0xf4
_mtx_lock_flags(c08ec02c,0,c08091ed,26f,bfbfd3d0) at _mtx_lock_flags+0x85
tcp_usr_send(c2acaa20,4,c2c20b00,0,0) at tcp_usr_send+0x2c
sosend(c2acaa20,0,e4e5bc88,c2c20b00,0) at sosend+0x5e7
soo_write(c271a50c,e4e5bc88,c2ac6d80,0,c2379c00) at soo_write+0x46
dofilewrite(c2379c00,c271a50c,3,bfbfcbd0,2000) at dofilewrite+0xa8
write(c2379c00,e4e5bd14,3,13,292) at write+0x39
syscall(2f,809002f,bfbf002f,2000,809a044) at syscall+0x227
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip =3D 0x2812558b, esp =3D 0xbfbfca=
cc, ebp =3D 0xbfbfcae8 ---
db> show locks 574
db> trace 572
sched_switch(c237ad80,0,2) at sched_switch+0x16f
mi_switch(2,0,c237ad80,b4,c08be1e0,0,c07ff747,f4) at mi_switch+0x264
ast(e4e7cd48) at ast+0x2d9
doreti_ast() at doreti_ast+0x17
db> trace 570
sched_switch(c2798480,0,1) at sched_switch+0x16f
mi_switch(1,0) at mi_switch+0x264
turnstile_wait(c08ec02c,c26ef780,c08ec02c,2,c07fbabd,21e) at turnstile_wait=
+0x2f8
_mtx_lock_sleep(c08ec02c,c2798480,0,c08091ed,26f) at _mtx_lock_sleep+0x142
_mtx_lock_flags(c08ec02c,0,c08091ed,26f,bfbfd3d0) at _mtx_lock_flags+0x85
tcp_usr_send(c2b44144,0,c2c20600,0,0) at tcp_usr_send+0x2c
sosend(c2b44144,0,ef1c5c88,c2c20600,0) at sosend+0x5e7
soo_write(c2b07110,ef1c5c88,c2ac6c80,0,c2798480) at soo_write+0x46
dofilewrite(c2798480,c2b07110,3,bfbfcbd0,2000) at dofilewrite+0xa8
write(c2798480,ef1c5d14,3,15,292) at write+0x39
syscall(2f,2819002f,bfbf002f,2000,809a044) at syscall+0x227
Xint0x80_syscall() at Xint0x80_syscall+0x1f
--- syscall (4, FreeBSD ELF32, write), eip =3D 0x2812558b, esp =3D 0xbfbfca=
cc, ebp =3D 0xbfbfcae8 ---
db> show locks 572
db> trace 53
sched_switch(c22cd180,0,1) at sched_switch+0x16f
mi_switch(1,0) at mi_switch+0x264
ithread_loop(c24a1e80,e4e1ad48,c24a1e80,c05f7d50,0) at ithread_loop+0x22d
fork_exit(c05f7d50,c24a1e80,e4e1ad48) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip =3D 0, esp =3D 0xe4e1ad7c, ebp =3D 0 ---
db> show locks 53
db> trace 38
kdb_enter(c07fc72c) at kdb_enter+0x2b
panic(c0815e8e,1,2,c22583c0,c2821100) at panic+0x127
uma_zfree_arg(c101fc60,c2821100,0) at uma_zfree_arg+0xa5
mb_free_ext(c2821100) at mb_free_ext+0x39
m_freem(c2821100,0,0,1,1) at m_freem+0x21
tcp_input(c2821100,14,c2821100,0,0) at tcp_input+0x2d1c
ip_input(c2821100) at ip_input+0x50d
netisr_processqueue(c08eae58) at netisr_processqueue+0x6e
swi_net(0) at swi_net+0xbe
ithread_loop(c2260c00,e3384d48,c2260c00,c05f7d50,0) at ithread_loop+0x124
fork_exit(c05f7d50,c2260c00,e3384d48) at fork_exit+0xa4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip =3D 0, esp =3D 0xe3384d7c, ebp =3D 0 ---
db> show locks 38
exclusive sleep mutex UMA pcpu r =3D 0 (0xc08f8548) locked @ vm/uma_core.c:=
2215
exclusive sleep mutex inp (tcpinp) r =3D 0 (0xc2b4d2ac) locked @ netinet/tc=
p_input.c:743
exclusive sleep mutex tcp r =3D 0 (0xc08ec02c) locked @ netinet/tcp_input.c=
:617
db> trace 36
sched_switch(c0780fc1,c090e5a0,e3370018,c2260010,10) at sched_switch+0x16f
*** error reading from address e3370014 ***
(kgdb) bt
#0 doadump () at pcpu.h:159
#1 0xc04601ba in db_fncall (dummy1=3D0, dummy2=3D0, dummy3=3D-1064327584,=
=20
dummy4=3D0xe33849d0 "=ECI8=E3$!`=C0`=A6\217=C0`=A6\217=C0=ECI8=E3=F8\00=
3")
at ../../../ddb/db_command.c:531
#2 0xc045ffc8 in db_command (last_cmdp=3D0xc08a1744, cmd_table=3D0x0,=20
aux_cmd_tablep=3D0xc082161c, aux_cmd_tablep_end=3D0xc0821638)
at ../../../ddb/db_command.c:349
#3 0xc0460090 in db_command_loop () at ../../../ddb/db_command.c:455
#4 0xc0461bf5 in db_trap (type=3D3, code=3D0) at ../../../ddb/db_main.c:22=
1
#5 0xc0620368 in kdb_trap (type=3D3, code=3D0, tf=3D0xe3384b14)
at ../../../kern/subr_kdb.c:419
#6 0xc0792120 in trap (frame=3D
{tf_fs =3D -482869224, tf_es =3D -1067319280, tf_ds =3D -1065418736,
tf_edi =3D -1065263474, tf_esi =3D 1, tf_ebp =3D -482849964, tf_isp =3D
-482849984, tf_ebx =3D -482849920, tf_edx =3D 0, tf_ecx =3D -1056882688, tf=
_eax
=3D 18, tf_trapno =3D 3, tf_err =3D 0, tf_eip =3D -1067319089, tf_cs =3D 8,
tf_eflags =3D 658, tf_esp =3D -482849932, tf_ss =3D -1067409941}) at
=2E./../../i386/i386/trap.c:576
#7 0xc078087a in calltrap () at ../../../i386/i386/exception.s:140
#8 0xe3380018 in ?? ()
#9 0xc0620010 in kdb_alt_break (key=3D0, state=3D0x0)
at ../../../kern/subr_kdb.c:179
#10 0xc0609deb in panic (
fmt=3D0xc0815e8e "uma_zfree: Freeing to non free bucket index.")
---Type <return> to continue, or q <return> to quit---
at ../../../kern/kern_shutdown.c:525
#11 0xc075b841 in uma_zfree_arg (zone=3D0xc101fc60, item=3D0xc2821100,
udata=3D0x0)
at ../../../vm/uma_core.c:2228
#12 0xc063d50d in mb_free_ext (m=3D0xc2821100) at uma.h:302
#13 0xc063d425 in m_freem (mb=3D0x0) at mbuf.h:397
#14 0xc0693fa8 in tcp_input (m=3D0xc2821100, off0=3D686)
at ../../../netinet/tcp_input.c:2435
#15 0xc068bb29 in ip_input (m=3D0xc2821100) at
=2E./../../netinet/ip_input.c:739
#16 0xc067457a in netisr_processqueue (ni=3D0xc08eae58)
at ../../../net/netisr.c:235
#17 0xc0674922 in swi_net (dummy=3D0x0) at ../../../net/netisr.c:348
#18 0xc05f7e74 in ithread_loop (arg=3D0xc2260c00)
at ../../../kern/kern_intr.c:547
#19 0xc05f7284 in fork_exit (callout=3D0xc05f7d50 <ithread_loop>,=20
arg=3D0xc2260c00, frame=3D0xe3384d48) at ../../../kern/kern_fork.c:807
#20 0xc07808dc in fork_trampoline () at ../../../i386/i386/exception.s:209
(kgdb) frame 11
#11 0xc075b841 in uma_zfree_arg (zone=3D0xc101fc60, item=3D0xc2821100,
udata=3D0x0)
at ../../../vm/uma_core.c:2228
2228 KASSERT(bucket->ub_bucket[bucket->ub_cnt]
=3D=3D NULL,
(kgdb) print bucket
$2 =3D 0xc2b38624
(kgdb) print *bucket
$3 =3D {ub_link =3D {le_next =3D 0x0, le_prev =3D 0xc101fc78}, ub_cnt =3D 7=
8,=20
ub_entries =3D 128, ub_bucket =3D 0xc2b38630}
(kgdb) print bucket->ub_bucket[bucket->ub_cnt]
$4 =3D (void *) 0xc2ca5900
(kgdb) inspect *zone
$5 =3D {uz_name =3D 0xc07e455f "Packet", uz_lock =3D 0xc22583c8,=20
uz_keg =3D 0xc22583c0, uz_link =3D {le_next =3D 0x0, le_prev =3D 0xc101f9=
ac},=20
uz_full_bucket =3D {lh_first =3D 0xc280ca3c}, uz_free_bucket =3D {
lh_first =3D 0x0}, uz_ctor =3D 0xc0601310 <mb_ctor_pack>,=20
uz_dtor =3D 0xc060121c <mb_dtor_pack>, uz_init =3D 0xc06012a8
<mb_init_pack>,=20
uz_fini =3D 0xc06012e4 <mb_fini_pack>, uz_allocs =3D 16842, uz_fills =3D =
0,=20
uz_count =3D 128, uz_cpu =3D {{uc_freebucket =3D 0xc2988418,=20
uc_allocbucket =3D 0xc286ba3c, uc_allocs =3D 133}}}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?Pine.NEB.3.96L.1041018180742.47572G-100000>