Date: Fri, 04 Jul 2014 13:47:08 GMT From: "Christer Eriksson" <dev@vlsi.se> To: <freebsd-scsi@freebsd.org> Subject: Kernel panic: Page fault when loading kernel native iSCSI target (FreeBSD 10.0-STABLE #0 r268091) Message-ID: <1404481628956.6459.61486@webmail3>
next in thread | raw e-mail | index | archive | help
Hello All,
We are getting kernel panics while reading and writing to an iSCSI target. =
It is the kernel implementation of iSCSI and we are running the initiators =
in Windows 2012R2 with load on two 10 GE links. The problem is repeatable, =
but occurs what appears to be within a random period from when the load is =
initiated. No obvious useful info in dmesg or syslog.
Backtrace from the kernel dump below.
I will try to collect additional information upon request.
Best Regards
Christer Eriksson
INFO -------------------------------------------------------------
Dump header from device /dev/ada1s1
Architecture: amd64
Architecture Version: 2
Dump Length: 2995580928B (2856 MB)
Blocksize: 512
Dumptime: Fri Jul 4 14:50:34 2014
Hostname: TestArray1.
Magic: FreeBSD Kernel Dump
Version String: FreeBSD 10.0-STABLE #0 r268091: Tue Jul 1 15:40:42 CEST 201=
4
root@TestArray1.:/usr/obj/usr/src/sys/GENERIC
Panic String: page fault
Dump Parity: 455418556
Bounds: 3
Dump Status: good
KGDB -------------------------------------------------------------
#kgdb /usr/obj/usr/src/sys/GENERIC/kernel.debug /var/crash/vmcore.3
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you ar=
e
welcome to change it and/or distribute copies of it under certain condition=
s.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...
Unread portion of the kernel message buffer:
Fatal trap 12: page fault while in kernel mode
cpuid =3D 6; apic id =3D 06
fault virtual address=3D 0x0
fault code=3D supervisor write data, page not present
instruction pointer=3D 0x20:0xffffffff80ce2766
stack pointer =3D 0x28:0xfffffe1049ba38f0
frame pointer =3D 0x28:0xfffffe1049ba3940
code segment=3D base 0x0, limit 0xfffff, type 0x1b
=3D DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags=3D interrupt enabled, resume, IOPL =3D 0
current process=3D 0 (cfiscsirx)
trap number=3D 12
panic: page fault
cpuid =3D 6
KDB: stack backtrace:
#0 0xffffffff8092a270 at kdb_backtrace+0x60
#1 0xffffffff808ef7c5 at panic+0x155
#2 0xffffffff80ce4a5f at trap_fatal+0x38f
#3 0xffffffff80ce4d78 at trap_pfault+0x308
#4 0xffffffff80ce4430 at trap+0x4a0
#5 0xffffffff80ccae32 at calltrap+0x8
#6 0xffffffff81c304c0 at cfiscsi_handle_data_segment+0xf0
#7 0xffffffff81c30eda at cfiscsi_receive_callback+0x5ea
#8 0xffffffff81c4f5bb at icl_receive_thread+0x11b
#9 0xffffffff808c037a at fork_exit+0x9a
#10 0xffffffff80ccb36e at fork_trampoline+0xe
Uptime: 9m51s
Dumping 2856 out of 65476 MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..=
91%
Reading symbols from /boot/kernel/zfs.ko.symbols...done.
Loaded symbols for /boot/kernel/zfs.ko.symbols
Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.
Loaded symbols for /boot/kernel/opensolaris.ko.symbols
Reading symbols from /boot/kernel/ums.ko.symbols...done.
Loaded symbols for /boot/kernel/ums.ko.symbols
Reading symbols from /boot/kernel/uhid.ko.symbols...done.
Loaded symbols for /boot/kernel/uhid.ko.symbols
Reading symbols from /boot/kernel/ctl.ko.symbols...done.
Loaded symbols for /boot/kernel/ctl.ko.symbols
Reading symbols from /boot/kernel/iscsi.ko.symbols...done.
Loaded symbols for /boot/kernel/iscsi.ko.symbols
Reading symbols from /boot/kernel/dtraceall.ko.symbols...done.
Loaded symbols for /boot/kernel/dtraceall.ko.symbols
Reading symbols from /boot/kernel/cyclic.ko.symbols...done.
Loaded symbols for /boot/kernel/cyclic.ko.symbols
Reading symbols from /boot/kernel/dtrace.ko.symbols...done.
Loaded symbols for /boot/kernel/dtrace.ko.symbols
Reading symbols from /boot/kernel/dtmalloc.ko.symbols...done.
Loaded symbols for /boot/kernel/dtmalloc.ko.symbols
Reading symbols from /boot/kernel/dtnfscl.ko.symbols...done.
Loaded symbols for /boot/kernel/dtnfscl.ko.symbols
Reading symbols from /boot/kernel/fbt.ko.symbols...done.
Loaded symbols for /boot/kernel/fbt.ko.symbols
Reading symbols from /boot/kernel/fasttrap.ko.symbols...done.
Loaded symbols for /boot/kernel/fasttrap.ko.symbols
Reading symbols from /boot/kernel/lockstat.ko.symbols...done.
Loaded symbols for /boot/kernel/lockstat.ko.symbols
Reading symbols from /boot/kernel/sdt.ko.symbols...done.
Loaded symbols for /boot/kernel/sdt.ko.symbols
Reading symbols from /boot/kernel/systrace.ko.symbols...done.
Loaded symbols for /boot/kernel/systrace.ko.symbols
Reading symbols from /boot/kernel/systrace_freebsd32.ko.symbols...done.
Loaded symbols for /boot/kernel/systrace_freebsd32.ko.symbols
Reading symbols from /boot/kernel/profile.ko.symbols...done.
Loaded symbols for /boot/kernel/profile.ko.symbols
#0 doadump (textdump=3D<value optimized out>) at pcpu.h:219
219pcpu.h: No such file or directory.
in pcpu.h
(kgdb) bt
#0 doadump (textdump=3D<value optimized out>) at pcpu.h:219
#1 0xffffffff808ef442 in kern_reboot (howto=3D260) at /usr/src/sys/kern/ker=
n_shutdown.c:452
#2 0xffffffff808ef804 in panic (fmt=3D<value optimized out>) at /usr/src/sy=
s/kern/kern_shutdown.c:759
#3 0xffffffff80ce4a5f in trap_fatal (frame=3D<value optimized out>, eva=3D<=
value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:881
#4 0xffffffff80ce4d78 in trap_pfault (frame=3D0xfffffe1049ba3840, usermode=
=3D<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:692
#5 0xffffffff80ce4430 in trap (frame=3D0xfffffe1049ba3840) at /usr/src/sys/=
amd64/amd64/trap.c:456
#6 0xffffffff80ccae32 in calltrap () at /usr/src/sys/amd64/amd64/exception.=
S:232
#7 0xffffffff80ce2766 in bcopy () at /usr/src/sys/amd64/amd64/support.S:112=
#8 0xffffffff8095be82 in m_copydata (m=3D<value optimized out>, off=3D<valu=
e optimized out>, len=3D<value optimized out>, cp=3D<value optimized out>)
at /usr/src/sys/kern/uipc_mbuf.c:887
#9 0xffffffff81c304c0 in cfiscsi_handle_data_segment (request=3D0xfffff8024=
8460eb0, cdw=3D0xfffff80248484540)
at /usr/src/sys/modules/ctl/../../cam/ctl/ctl_frontend_iscsi.c:782
#10 0xffffffff81c30eda in cfiscsi_receive_callback (request=3D0xfffff802484=
60eb0) at /usr/src/sys/modules/ctl/../../cam/ctl/ctl_frontend_iscsi.c:916
#11 0xffffffff81c4f5bb in icl_receive_thread (arg=3D0xfffff80248a16980) at =
/usr/src/sys/modules/iscsi/../../dev/iscsi/icl.c:730
#12 0xffffffff808c037a in fork_exit (callout=3D0xffffffff81c4f4a0 <icl_rece=
ive_thread>, arg=3D0xfffff80248a16980, frame=3D0xfffffe1049ba3ac0)
at /usr/src/sys/kern/kern_fork.c:995
#13 0xffffffff80ccb36e in fork_trampoline () at /usr/src/sys/amd64/amd64/ex=
ception.S:606
#14 0x0000000000000000 in ?? ()
Current language: auto; currently minimal
(kgdb) bt full
#0 doadump (textdump=3D<value optimized out>) at pcpu.h:219
No locals.
#1 0xffffffff808ef442 in kern_reboot (howto=3D260) at /usr/src/sys/kern/ker=
n_shutdown.c:452
No locals.
#2 0xffffffff808ef804 in panic (fmt=3D<value optimized out>) at /usr/src/sy=
s/kern/kern_shutdown.c:759
ap =3D {{gp_offset =3D 16, fp_offset =3D 48, overflow_arg_area =3D 0xfffffe=
1049ba3530, reg_save_area =3D 0xfffffe1049ba34b0}}
#3 0xffffffff80ce4a5f in trap_fatal (frame=3D<value optimized out>, eva=3D<=
value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:881
softseg =3D {ssd_base =3D 0, ssd_limit =3D 1048575, ssd_type =3D 27, ssd_dp=
l =3D 0, ssd_p =3D 1, ssd_long =3D 1, ssd_def32 =3D 0, ssd_gran =3D 1}
#4 0xffffffff80ce4d78 in trap_pfault (frame=3D0xfffffe1049ba3840, usermode=
=3D<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:692
rv =3D Cannot access memory at address 0x0
(kgdb) list *0xffffffff80ce2766
0xffffffff80ce2766 is at /usr/src/sys/amd64/amd64/support.S:113.
108cmpq%rcx,%rax/* overlapping && src < dst? */
109jb1f
110
111shrq$3,%rcx/* copy by 64-bit words */
112cld/* nope, copy forwards */
113rep
114movsq
115movq%rdx,%rcx
116andq$7,%rcx/* any bytes left? */
117rep
(kgdb) up
#1 0xffffffff808ef442 in kern_reboot (howto=3D260) at /usr/src/sys/kern/ker=
n_shutdown.c:452
452doadump(TRUE);
(kgdb) up
#2 0xffffffff808ef804 in panic (fmt=3D<value optimized out>) at /usr/src/sy=
s/kern/kern_shutdown.c:759
759kern_reboot(bootopt);
(kgdb) up
#3 0xffffffff80ce4a5f in trap_fatal (frame=3D<value optimized out>, eva=3D<=
value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:881
881panic("%s", trap_msg[type]);
(kgdb) up
#4 0xffffffff80ce4d78 in trap_pfault (frame=3D0xfffffe1049ba3840, usermode=
=3D<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:692
692trap_fatal(frame, eva);
(kgdb) up
#5 0xffffffff80ce4430 in trap (frame=3D0xfffffe1049ba3840) at /usr/src/sys/=
amd64/amd64/trap.c:456
456(void) trap_pfault(frame, FALSE);
(kgdb) up
#6 0xffffffff80ccae32 in calltrap () at /usr/src/sys/amd64/amd64/exception.=
S:232
232calltrap
Current language: auto; currently asm
(kgdb) up
#7 0xffffffff80ce2766 in bcopy () at /usr/src/sys/amd64/amd64/support.S:112=
112cld/* nope, copy forwards */
(kgdb) up
#8 0xffffffff8095be82 in m_copydata (m=3D<value optimized out>, off=3D<valu=
e optimized out>, len=3D<value optimized out>, cp=3D<value optimized out>)
at /usr/src/sys/kern/uipc_mbuf.c:887
887bcopy(mtod(m, caddr_t) + off, cp, count);
Current language: auto; currently minimal
(kgdb) up
#9 0xffffffff81c304c0 in cfiscsi_handle_data_segment (request=3D0xfffff8024=
8460eb0, cdw=3D0xfffff80248484540)
at /usr/src/sys/modules/ctl/../../cam/ctl/ctl_frontend_iscsi.c:782
782icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
(kgdb) up
#10 0xffffffff81c30eda in cfiscsi_receive_callback (request=3D0xfffff802484=
60eb0) at /usr/src/sys/modules/ctl/../../cam/ctl/ctl_frontend_iscsi.c:916
916done =3D cfiscsi_handle_data_segment(request, cdw);
(kgdb) list
911
912io =3D cdw->cdw_ctl_io;
913KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) !=3D CTL_FLAG_DATA_IN,
914 ("CTL_FLAG_DATA_IN"));
915
916done =3D cfiscsi_handle_data_segment(request, cdw);
917if (done) {
918CFISCSI_SESSION_LOCK(cs);
919TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
920CFISCSI_SESSION_UNLOCK(cs);
(kgdb) up
#11 0xffffffff81c4f5bb in icl_receive_thread (arg=3D0xfffff80248a16980) at =
/usr/src/sys/modules/iscsi/../../dev/iscsi/icl.c:730
730(ic->ic_receive)(response);
(kgdb) list
725icl_pdu_free(response);
726icl_conn_fail(ic);
727return;
728}
729
730(ic->ic_receive)(response);
731}
732}
733
734static void
(kgdb) up
#12 0xffffffff808c037a in fork_exit (callout=3D0xffffffff81c4f4a0 <icl_rece=
ive_thread>, arg=3D0xfffff80248a16980, frame=3D0xfffffe1049ba3ac0)
at /usr/src/sys/kern/kern_fork.c:995
995callout(arg, frame);
(kgdb) list
990 * cpu_set_fork_handler intercepts this function call to
991 * have this call a non-return function to stay in kernel mode.
992 * initproc has its own fork handler, but it does return.
993 */
994KASSERT(callout !=3D NULL, ("NULL callout in fork_exit"));
995callout(arg, frame);
996
997/*
998 * Check if a kernel thread misbehaved and returned from its main
999 * function.
(kgdb) up
#13 0xffffffff80ccb36e in fork_trampoline () at /usr/src/sys/amd64/amd64/ex=
ception.S:606
606callfork_exit
Current language: auto; currently asm
(kgdb) list
601
602ENTRY(fork_trampoline)
603movq%r12,%rdi/* function */
604movq%rbx,%rsi/* arg1 */
605movq%rsp,%rdx/* trapframe pointer */
606callfork_exit
607MEXITCOUNT
608jmpdoreti/* Handle any ASTs */
609
610/*
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?1404481628956.6459.61486>