From owner-freebsd-net@FreeBSD.ORG Fri Mar 4 06:54:42 2011 Return-Path: Delivered-To: freebsd-net@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 5DE621065670 for ; Fri, 4 Mar 2011 06:54:42 +0000 (UTC) (envelope-from pluknet@gmail.com) Received: from mail-qw0-f54.google.com (mail-qw0-f54.google.com [209.85.216.54]) by mx1.freebsd.org (Postfix) with ESMTP id 148B58FC08 for ; Fri, 4 Mar 2011 06:54:41 +0000 (UTC) Received: by qwj8 with SMTP id 8so1592800qwj.13 for ; Thu, 03 Mar 2011 22:54:41 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=gamma; h=domainkey-signature:mime-version:date:message-id:subject:from:to :content-type; bh=t3SX6EWcZ5IzC5FzYQKrXQmhthYJTY9PIyOrdWVcBh0=; b=rR51e+XynZbzRc6EljaIpRMmrmh1s6fu/uTFbYrLuHx1zXuQx80Nlq8j4DCyZmNDB+ WfPAOhnzKOoDXB+wkoDV3cZYNQyRK7TUap+3azIchGZBGi4hj7H2yYlsdD+d5xYqt8pg 64wjos2aAswX6iYJj5/z0w76ZYkln0xLdph+8= DomainKey-Signature: a=rsa-sha1; c=nofws; d=gmail.com; s=gamma; h=mime-version:date:message-id:subject:from:to:content-type; b=hux77gvc9Od2R0MewiUuSr1BZMu7iw5c4iV/SxpWCkPiR9jLhXLAevpgZbDFxzMgr2 m/WVIua6xH7C2b+qmsdQ2wXF9St+kL87RUEJLiBoIlKkl9GFErksg2D564i0hULbYJNM u9228LE78K7Ytq73NSB77rCKxxx8jra1QeJ2Y= MIME-Version: 1.0 Received: by 10.229.77.142 with SMTP id g14mr173404qck.55.1299221681017; Thu, 03 Mar 2011 22:54:41 -0800 (PST) Received: by 10.229.84.129 with HTTP; Thu, 3 Mar 2011 22:54:40 -0800 (PST) Date: Fri, 4 Mar 2011 09:54:40 +0300 Message-ID: From: Sergey Kandaurov To: FreeBSD Net Content-Type: text/plain; charset=ISO-8859-1 Subject: arpintr()->in_lltable_lookup() 8.1 bce(4) crash X-BeenThere: freebsd-net@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: Networking and TCP/IP with FreeBSD List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 04 Mar 2011 06:54:42 -0000 Hi. That's the second crash on 8.1. A previous one was month ago on another box. They are almost identical. Kernel can't dump core on these disk controllers. Any hints are appreciated. kernel trap 12 with interrupts disabled Fatal trap 12: page fault while in kernel mode cpuid = 0; apic id = 00 fault virtual address = 0xc fault code = supervisor read data, page not present instruction pointer = 0x20:0xffffffff805f30d2 stack pointer = 0x28:0xffffff82b1554890 frame pointer = 0x28:0xffffff82b15548c0 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, long 1, def32 0, gran 1 processor eflags = resume, IOPL = 0 current process = 12 (irq256: bce0) db> bt Tracing pid 12 tid 100038 td 0xffffff00029a17c0 propagate_priority() at propagate_priority+0x72 turnstile_wait() at turnstile_wait+0x1aa _rw_wlock_hard() at _rw_wlock_hard+0xfa in_lltable_lookup() at in_lltable_lookup+0x12b arpintr() at arpintr+0x9d6 netisr_dispatch_src() at netisr_dispatch_src+0x7e ether_demux() at ether_demux+0x14d ether_input() at ether_input+0x17b bce_intr() at bce_intr+0x3b0 intr_event_execute_handlers() at intr_event_execute_handlers+0xfd ithread_loop() at ithread_loop+0x8e fork_exit() at fork_exit+0x118 fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffff82b1554d30, rbp = 0 --- #################################### kernel trap 12 with interrupts disabled Fatal trap 12: page fault while in kernel mode cpuid = 2; apic id = 02 fault virtual address = 0xc fault code = supervisor read data, page not present instruction pointer = 0x20:0xffffffff805f30c2 stack pointer = 0x28:0xffffff82b155d830 frame pointer = 0x28:0xffffff82b155d860 code segment = base 0x0, limit 0xfffff, type 0x1b = DPL 0, pres 1, long 1, def32 0, gran 1 processor eflags = resume, IOPL = 0 current process = 12 (irq257: bce1) db> bt Tracing pid 12 tid 100039 td 0xffffff00029a23e0 propagate_priority() at propagate_priority+0x72 turnstile_wait() at turnstile_wait+0x1aa _rw_wlock_hard() at _rw_wlock_hard+0xfa in_lltable_lookup() at in_lltable_lookup+0x12b arpintr() at arpintr+0x9d6 netisr_dispatch_src() at netisr_dispatch_src+0x7e ether_demux() at ether_demux+0x14d ether_input() at ether_input+0x17b ether_demux() at ether_demux+0x6f ether_input() at ether_input+0x17b bce_intr() at bce_intr+0x3b0 intr_event_execute_handlers() at intr_event_execute_handlers+0xfd ithread_loop() at ithread_loop+0x8e fork_exit() at fork_exit+0x118 fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffff82b155dd30, rbp = 0 --- That's what was on another CPUs atm, if that matters: db> show proc 12 Process 12 (intr) at 0xffffff00026e7000: state: NORMAL uid: 0 gids: 0 parent: pid 0 at 0xffffffff80c7e3e0 ABI: null threads: 24 100065 I [irq1: atkbd0] 100064 I [swi0: uart uart] 100063 I [irq15: ata1] 100062 I [irq14: ata0] 100045 I [irq22: uhci1 uhci3] 100040 I [irq23: uhci0 uhci2+] 100039 L *lle 0xffffff032af863c0 [irq257: bce1] 100038 I [irq256: bce0] 100036 I [irq17: aac0] 100035 I [irq9: acpi0] 100034 I [swi5: +] 100032 I [swi2: cambio] 100028 I [swi6: task queue] 100027 I [swi6: Giant taskq] 100020 I [swi3: vm] 100019 Run CPU 1 [swi4: clock] 100018 Run CPU 5 [swi4: clock] 100017 Run CPU 6 [swi4: clock] 100016 I [swi4: clock] 100015 Run CPU 3 [swi4: clock] 100014 I [swi4: clock] 100013 CanRun [swi4: clock] 100012 I [swi4: clock] 100011 I [swi1: netisr 0] db> show allpcpu Current CPU: 2 cpuid = 0 dynamic pcpu = 0x2a3f80 curthread = 0xffffff0061aca000: pid 52989 "httpd" curpcb = 0xffffff82b7e2fd40 fpcurthread = none idlethread = 0xffffff00026ea7c0: pid 11 "idle: cpu0" curpmap = 0 tssp = 0xffffffff80cf0080 commontssp = 0xffffffff80cf0080 rsp0 = 0xffffff82b7e2fd40 gs32p = 0xffffffff80ceeeb8 ldt = 0xffffffff80ceeef8 tss = 0xffffffff80ceeee8 cpuid = 1 dynamic pcpu = 0xffffff807f418f80 curthread = 0xffffff00027003e0: pid 12 "swi4: clock" curpcb = 0xffffff80000ecd40 fpcurthread = none idlethread = 0xffffff00026eaba0: pid 11 "idle: cpu1" curpmap = 0 tssp = 0xffffffff80cf00e8 commontssp = 0xffffffff80cf00e8 rsp0 = 0xffffff80000ecd40 gs32p = 0xffffffff80ceef20 ldt = 0xffffffff80ceef60 tss = 0xffffffff80ceef50 cpuid = 2 dynamic pcpu = 0xffffff807f41ff80 curthread = 0xffffff00029a23e0: pid 12 "irq257: bce1" curpcb = 0xffffff82b155dd40 fpcurthread = none idlethread = 0xffffff00026f6000: pid 11 "idle: cpu2" curpmap = 0 tssp = 0xffffffff80cf0150 commontssp = 0xffffffff80cf0150 rsp0 = 0xffffff82b155dd40 gs32p = 0xffffffff80ceef88 ldt = 0xffffffff80ceefc8 tss = 0xffffffff80ceefb8 cpuid = 3 dynamic pcpu = 0xffffff807f426f80 curthread = 0xffffff00026fb7c0: pid 12 "swi4: clock" curpcb = 0xffffff80000d8d40 fpcurthread = none idlethread = 0xffffff00026f63e0: pid 11 "idle: cpu3" curpmap = 0 tssp = 0xffffffff80cf01b8 commontssp = 0xffffffff80cf01b8 rsp0 = 0xffffff80000d8d40 gs32p = 0xffffffff80ceeff0 ldt = 0xffffffff80cef030 tss = 0xffffffff80cef020 cpuid = 4 dynamic pcpu = 0xffffff807f42df80 curthread = 0xffffff014bbbbba0: pid 53746 "head" curpcb = 0xffffff82b92c0d40 fpcurthread = none idlethread = 0xffffff00026f67c0: pid 11 "idle: cpu4" curpmap = 0 tssp = 0xffffffff80cf0220 commontssp = 0xffffffff80cf0220 rsp0 = 0xffffff82b92c0d40 gs32p = 0xffffffff80cef058 ldt = 0xffffffff80cef098 tss = 0xffffffff80cef088 cpuid = 5 dynamic pcpu = 0xffffff807f434f80 curthread = 0xffffff00026f6ba0: pid 12 "swi4: clock" curpcb = 0xffffff80000e7d40 fpcurthread = none idlethread = 0xffffff00026e9000: pid 11 "idle: cpu5" curpmap = 0 tssp = 0xffffffff80cf0288 commontssp = 0xffffffff80cf0288 rsp0 = 0xffffff80000e7d40 gs32p = 0xffffffff80cef0c0 ldt = 0xffffffff80cef100 tss = 0xffffffff80cef0f0 cpuid = 6 dynamic pcpu = 0xffffff807f43bf80 curthread = 0xffffff00026fb000: pid 12 "swi4: clock" curpcb = 0xffffff80000e2d40 fpcurthread = none idlethread = 0xffffff00026e93e0: pid 11 "idle: cpu6" curpmap = 0 tssp = 0xffffffff80cf02f0 commontssp = 0xffffffff80cf02f0 rsp0 = 0xffffff80000e2d40 gs32p = 0xffffffff80cef128 ldt = 0xffffffff80cef168 tss = 0xffffffff80cef158 cpuid = 7 dynamic pcpu = 0xffffff807f442f80 curthread = 0xffffff008976eba0: pid 45168 "httpd" curpcb = 0xffffff82b71b4d40 fpcurthread = 0xffffff008976eba0: pid 45168 "httpd" idlethread = 0xffffff00026e97c0: pid 11 "idle: cpu7" curpmap = 0 tssp = 0xffffffff80cf0358 commontssp = 0xffffffff80cf0358 rsp0 = 0xffffff82b71b4d40 gs32p = 0xffffffff80cef190 ldt = 0xffffffff80cef1d0 tss = 0xffffffff80cef1c0 db> bt 52989 Tracing pid 52989 tid 103210 td 0xffffff0061aca000 cpustop_handler() at cpustop_handler+0x40 ipi_nmi_handler() at ipi_nmi_handler+0x30 trap() at trap+0x175 nmi_calltrap() at nmi_calltrap+0x8 --- trap 0x13, rip = 0xffffffff808c45c2, rsp = 0xffffffff80cf7e40, rbp = 0xffffff82b7e2f8d0 --- DELAY() at DELAY+0x62 _thread_lock_flags() at _thread_lock_flags+0xb0 sleepq_add() at sleepq_add+0x8c _sleep() at _sleep+0x159 soreceive_generic() at soreceive_generic+0xeba dofileread() at dofileread+0xa1 kern_readv() at kern_readv+0x60 read() at read+0x55 ia32_syscall() at ia32_syscall+0x1eb Xint0x80_syscall() at Xint0x80_syscall+0x95 --- syscall (3, FreeBSD ELF32, read), rip = 0x281b834f, rsp = 0xffffb89c, rbp = 0xffffb8b8 --- db> bt 53746 Tracing pid 53746 tid 104263 td 0xffffff014bbbbba0 cpustop_handler() at cpustop_handler+0x40 ipi_nmi_handler() at ipi_nmi_handler+0x30 trap() at trap+0x175 nmi_calltrap() at nmi_calltrap+0x8 --- trap 0x13, rip = 0xffffffff808c45c4, rsp = 0xffffff8000067fe0, rbp = 0xffffff82b92c05e0 --- DELAY() at DELAY+0x64 _thread_lock_flags() at _thread_lock_flags+0xb0 intr_event_schedule_thread() at intr_event_schedule_thread+0x72 callout_tick() at callout_tick+0xf8 hardclock_cpu() at hardclock_cpu+0xd2 lapic_handle_timer() at lapic_handle_timer+0xff Xtimerint() at Xtimerint+0x8c --- interrupt, rip = 0xffffffff808be298, rsp = 0xffffff82b92c07e0, rbp = 0xffffff82b92c0840 --- pmap_remove() at pmap_remove+0x2a8 vm_map_delete() at vm_map_delete+0xf4 vm_map_fixed() at vm_map_fixed+0x78 vm_mmap() at vm_mmap+0x51d mmap() at mmap+0x219 syscall() at syscall+0x1e7 Xfast_syscall() at Xfast_syscall+0xe1 --- syscall (477, FreeBSD ELF64, mmap), rip = 0x80050c86c, rsp = 0x7fffffffe278, rbp = 0x7fffffffe370 --- db> bt 45168 Tracing pid 45168 tid 102571 td 0xffffff008976eba0 cpustop_handler() at cpustop_handler+0x40 ipi_nmi_handler() at ipi_nmi_handler+0x30 trap() at trap+0x175 nmi_calltrap() at nmi_calltrap+0x8 --- trap 0x13, rip = 0xffffffff805a3d8d, rsp = 0xffffff800007cfe0, rbp = 0xffffff82b71b4860 --- _mtx_lock_sleep() at _mtx_lock_sleep+0xdd pmap_enter() at pmap_enter+0x2f7 vm_fault() at vm_fault+0x14dc trap_pfault() at trap_pfault+0x132 trap() at trap+0x4dc calltrap() at calltrap+0x8 --- trap 0xc, rip = 0x281b7b96, rsp = 0xfffe2424, rbp = 0xfffe24e8 --- db> bt 11 Tracing pid 11 tid 100010 td 0xffffff00026ea7c0 sched_switch() at sched_switch+0xea mi_switch() at mi_switch+0x16f sched_preempt() at sched_preempt+0xb5 ipi_bitmap_handler() at ipi_bitmap_handler+0x70 Xipi_intr_bitmap_handler() at Xipi_intr_bitmap_handler+0x9b --- interrupt, rip = 0xffffffff805a41f4, rsp = 0xffffff80000bfb60, rbp = 0xffffff80000bfba0 --- _thread_lock_flags() at _thread_lock_flags+0x64 sched_idletd() at sched_idletd+0xde fork_exit() at fork_exit+0x118 fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffff80000bfd30, rbp = 0 --- db> bt 12 Tracing pid 12 tid 100065 td 0xffffff0009001ba0 fork_trampoline() at fork_trampoline -- wbr, pluknet