Date: Tue, 22 Dec 2020 09:05:01 -0500 From: mike tancsa <mike@sentex.net> To: FreeBSD-STABLE Mailing List <freebsd-stable@freebsd.org> Subject: Re: zfs panic RELENG_12 Message-ID: <5b3415cb-2176-895e-9d22-4f4f0f359d85@sentex.net> In-Reply-To: <878824fe-dde2-b551-4685-e8bd27371275@sentex.net> References: <878824fe-dde2-b551-4685-e8bd27371275@sentex.net>
next in thread | previous in thread | raw e-mail | index | archive | help
Hmmm, another one. Not sure if this is hardware as it seems different ?
Fatal trap 12: page fault while in kernel mode
cpuid = 11; apic id = 0b
fault virtual address = 0x0
fault code = supervisor write data, page not present
instruction pointer = 0x20:0xffffffff80ca0826
stack pointer = 0x28:0xfffffe00bc0f8540
frame pointer = 0x28:0xfffffe00bc0f8590
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 33 (dom0)
trap number = 12
panic: page fault
cpuid = 11
time = 1608641071
KDB: stack backtrace:
#0 0xffffffff80a3fe85 at kdb_backtrace+0x65
#1 0xffffffff809f406b at vpanic+0x17b
#2 0xffffffff809f3ee3 at panic+0x43
#3 0xffffffff80e3fe71 at trap_fatal+0x391
#4 0xffffffff80e3fecf at trap_pfault+0x4f
#5 0xffffffff80e3f516 at trap+0x286
#6 0xffffffff80e19318 at calltrap+0x8
#7 0xffffffff80ca47d4 at bucket_cache_drain+0x134
#8 0xffffffff80c9e302 at zone_drain_wait+0xa2
#9 0xffffffff80ca2bbd at uma_reclaim_locked+0x6d
#10 0xffffffff80ca2af4 at uma_reclaim+0x34
#11 0xffffffff80cc5321 at vm_pageout_worker+0x421
#12 0xffffffff80cc4ee3 at vm_pageout+0x193
#13 0xffffffff809b55be at fork_exit+0x7e
#14 0xffffffff80e1a34e at fork_trampoline+0xe
Uptime: 5d20h37m16s
Dumping 16057 out of 65398
MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%
__curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
55 __asm("movq %%gs:%P1,%0" : "=r" (td) : "n"
(offsetof(struct pcpu,
(kgdb) bt
#0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
#1 doadump (textdump=<optimized out>) at
/usr/src/sys/kern/kern_shutdown.c:371
#2 0xffffffff809f3c85 in kern_reboot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:451
#3 0xffffffff809f40c3 in vpanic (fmt=<optimized out>, ap=<optimized
out>) at /usr/src/sys/kern/kern_shutdown.c:880
#4 0xffffffff809f3ee3 in panic (fmt=<unavailable>) at
/usr/src/sys/kern/kern_shutdown.c:807
#5 0xffffffff80e3fe71 in trap_fatal (frame=0xfffffe00bc0f8480, eva=0)
at /usr/src/sys/amd64/amd64/trap.c:921
#6 0xffffffff80e3fecf in trap_pfault (frame=0xfffffe00bc0f8480,
usermode=<optimized out>, signo=<optimized out>, ucode=<optimized out>)
at /usr/src/sys/amd64/amd64/trap.c:739
#7 0xffffffff80e3f516 in trap (frame=0xfffffe00bc0f8480) at
/usr/src/sys/amd64/amd64/trap.c:405
#8 <signal handler called>
#9 0xffffffff80ca0826 in slab_free_item (keg=0xfffff800037fa380,
slab=0xfffff80de4656fb0, item=<optimized out>) at
/usr/src/sys/vm/uma_core.c:3357
#10 zone_release (zone=<optimized out>, bucket=0xfffff801c7fd5218,
cnt=<optimized out>) at /usr/src/sys/vm/uma_core.c:3404
#11 0xffffffff80ca47d4 in bucket_drain (zone=0xfffff800037da000,
bucket=0xfffff801c7fd5200) at /usr/src/sys/vm/uma_core.c:758
#12 bucket_cache_drain (zone=0xfffff800037da000) at
/usr/src/sys/vm/uma_core.c:915
#13 0xffffffff80c9e302 in zone_drain_wait (zone=0xfffff800037da000,
waitok=1) at /usr/src/sys/vm/uma_core.c:1037
#14 0xffffffff80ca2bbd in zone_drain (zone=0xfffff800037da000) at
/usr/src/sys/vm/uma_core.c:1056
#15 zone_foreach (zfunc=<optimized out>) at /usr/src/sys/vm/uma_core.c:1985
#16 uma_reclaim_locked (kmem_danger=<optimized out>) at
/usr/src/sys/vm/uma_core.c:3737
#17 0xffffffff80ca2af4 in uma_reclaim () at /usr/src/sys/vm/uma_core.c:3757
#18 0xffffffff80cc5321 in vm_pageout_lowmem () at
/usr/src/sys/vm/vm_pageout.c:1890
#19 vm_pageout_worker (arg=<optimized out>) at
/usr/src/sys/vm/vm_pageout.c:1966
#20 0xffffffff80cc4ee3 in vm_pageout () at /usr/src/sys/vm/vm_pageout.c:2126
#21 0xffffffff809b55be in fork_exit (callout=0xffffffff80cc4d50
<vm_pageout>, arg=0x0, frame=0xfffffe00bc0f8b00) at
/usr/src/sys/kern/kern_fork.c:1080
#22 <signal handler called>
(kgdb) bt full
#0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
td = <optimized out>
#1 doadump (textdump=<optimized out>) at
/usr/src/sys/kern/kern_shutdown.c:371
error = <error reading variable error (Cannot access memory at
address 0x0)>
coredump = <optimized out>
#2 0xffffffff809f3c85 in kern_reboot (howto=260) at
/usr/src/sys/kern/kern_shutdown.c:451
once = <error reading variable once (Cannot access memory at
address 0x0)>
#3 0xffffffff809f40c3 in vpanic (fmt=<optimized out>, ap=<optimized
out>) at /usr/src/sys/kern/kern_shutdown.c:880
buf = "page fault", '\000' <repeats 245 times>
other_cpus = {__bits = {2047, 0, 0, 0}}
td = 0xfffff80004964740
newpanic = <error reading variable newpanic (Cannot access
memory at address 0x0)>
bootopt = <unavailable>
#4 0xffffffff809f3ee3 in panic (fmt=<unavailable>) at
/usr/src/sys/kern/kern_shutdown.c:807
ap = {{gp_offset = 16, fp_offset = 48, overflow_arg_area =
0xfffffe00bc0f82c0, reg_save_area = 0xfffffe00bc0f8260}}
#5 0xffffffff80e3fe71 in trap_fatal (frame=0xfffffe00bc0f8480, eva=0)
at /usr/src/sys/amd64/amd64/trap.c:921
softseg = {ssd_base = 0, ssd_limit = 1048575, ssd_type = 27,
ssd_dpl = 0, ssd_p = 1, ssd_long = 1, ssd_def32 = 0, ssd_gran = 1}
code = <optimized out>
type = <optimized out>
ss = 40
handled = <optimized out>
#6 0xffffffff80e3fecf in trap_pfault (frame=0xfffffe00bc0f8480,
usermode=<optimized out>, signo=<optimized out>, ucode=<optimized out>)
at /usr/src/sys/amd64/amd64/trap.c:739
td = 0xfffff80004964740
p = <optimized out>
eva = 0
map = <optimized out>
ftype = <optimized out>
rv = <optimized out>
#7 0xffffffff80e3f516 in trap (frame=0xfffffe00bc0f8480) at
/usr/src/sys/amd64/amd64/trap.c:405
ksi = {ksi_link = {tqe_next = 0x5fe1ea2f, tqe_prev = 0x1},
ksi_info = {si_signo = 76957504, si_errno = -2048, si_code =
-1891515752, si_pid = 506236, si_uid = 0, si_status = 0, si_addr = 0x46,
si_value = {sival_int = 0, sival_ptr = 0x0,
sigval_int = 0, sigval_ptr = 0x0}, _reason = {_fault =
{_trapno = -1891515752}, _timer = {_timerid = -1891515752, _overrun =
506236}, _mesgq = {_mqd = -1891515752}, _poll = {_band =
2174269467509400}, __spare__ = {
__spare1__ = 2174269467509400, __spare2__ =
{-1139833872, -512, -2131811593, -1, 1, 0, 0}}}}, ksi_flags =
2126206336, ksi_sigq = 0xfffffe00002e4af8}
td = 0xfffff80004964740
dr6 = <error reading variable dr6 (Cannot access memory at
address 0x0)>
p = <optimized out>
type = 12
addr = <optimized out>
signo = <optimized out>
ucode = <optimized out>
pf = <optimized out>
#8 <signal handler called>
No locals.
#9 0xffffffff80ca0826 in slab_free_item (keg=0xfffff800037fa380,
slab=0xfffff80de4656fb0, item=<optimized out>) at
/usr/src/sys/vm/uma_core.c:3357
dom = <optimized out>
freei = <optimized out>
#10 zone_release (zone=<optimized out>, bucket=0xfffff801c7fd5218,
cnt=<optimized out>) at /usr/src/sys/vm/uma_core.c:3404
clearfull = 0
keg = 0xfffff800037fa380
i = 18
item = <optimized out>
mem = <optimized out>
slab = 0xfffff80de4656fb0
#11 0xffffffff80ca47d4 in bucket_drain (zone=0xfffff800037da000,
bucket=0xfffff801c7fd5200) at /usr/src/sys/vm/uma_core.c:758
i = <optimized out>
#12 bucket_cache_drain (zone=0xfffff800037da000) at
/usr/src/sys/vm/uma_core.c:915
i = 0
bucket = 0xfffff801c7fd5200
zdom = <optimized out>
#13 0xffffffff80c9e302 in zone_drain_wait (zone=0xfffff800037da000,
waitok=1) at /usr/src/sys/vm/uma_core.c:1037
No locals.
#14 0xffffffff80ca2bbd in zone_drain (zone=0xfffff800037da000) at
/usr/src/sys/vm/uma_core.c:1056
--Type <RET> for more, q to quit, c to continue without paging--
No locals.
#15 zone_foreach (zfunc=<optimized out>) at /usr/src/sys/vm/uma_core.c:1985
keg = 0xfffff800037fa380
zone = 0xfffff800037da000
#16 uma_reclaim_locked (kmem_danger=<optimized out>) at
/usr/src/sys/vm/uma_core.c:3737
No locals.
#17 0xffffffff80ca2af4 in uma_reclaim () at /usr/src/sys/vm/uma_core.c:3757
No locals.
#18 0xffffffff80cc5321 in vm_pageout_lowmem () at
/usr/src/sys/vm/vm_pageout.c:1890
last = <optimized out>
_el = <optimized out>
_ep = <optimized out>
_t = <optimized out>
_v = <optimized out>
_tid = <optimized out>
_v = <optimized out>
_v = <optimized out>
#19 vm_pageout_worker (arg=<optimized out>) at
/usr/src/sys/vm/vm_pageout.c:1966
domain = <optimized out>
target_met = true
shortage = 107
vmd = <optimized out>
ofree = 348095
addl_shortage = <optimized out>
#20 0xffffffff80cc4ee3 in vm_pageout () at /usr/src/sys/vm/vm_pageout.c:2126
td = <optimized out>
p = 0xfffff800049f8530
i = <optimized out>
first = 0
error = <optimized out>
#21 0xffffffff809b55be in fork_exit (callout=0xffffffff80cc4d50
<vm_pageout>, arg=0x0, frame=0xfffffe00bc0f8b00) at
/usr/src/sys/kern/kern_fork.c:1080
td = 0xfffff80004964740
p = 0xfffff800049f8530
dtd = <optimized out>
#22 <signal handler called>
No locals.
(kgdb)
On 12/15/2020 4:39 PM, mike tancsa wrote:
> Was doing a backup via zfs send | zfs recv when the box panic'd. Its a
> not so old RELENG_12 box from last week. Any ideas if this is a hardware
> issue or a bug ? Its r368493 from last Wednesday. I dont see an ECC
> errors logged, so dont think its hardware.
>
> Reading symbols from /boot/kernel/kernel...
> Reading symbols from /usr/lib/debug//boot/kernel/kernel.debug...
>
> Unread portion of the kernel message buffer:
>
>
> Fatal trap 12: page fault while in kernel mode
> cpuid = 1; apic id = 01
> fault virtual address = 0x0
> fault code = supervisor read data, page not present
> instruction pointer = 0x20:0xffffffff823a554b
> stack pointer = 0x28:0xfffffe0343231000
> frame pointer = 0x28:0xfffffe03432310c0
> code segment = base 0x0, limit 0xfffff, type 0x1b
> = DPL 0, pres 1, long 1, def32 0, gran 1
> processor eflags = interrupt enabled, resume, IOPL = 0
> current process = 87427 (zfs)
> trap number = 12
> panic: page fault
> cpuid = 1
> time = 1608065221
> KDB: stack backtrace:
> #0 0xffffffff80a3fa05 at kdb_backtrace+0x65
> #1 0xffffffff809f3beb at vpanic+0x17b
> #2 0xffffffff809f3a63 at panic+0x43
> #3 0xffffffff80e400d1 at trap_fatal+0x391
> #4 0xffffffff80e4012f at trap_pfault+0x4f
> #5 0xffffffff80e3f776 at trap+0x286
> #6 0xffffffff80e19568 at calltrap+0x8
> #7 0xffffffff82393a5e at dmu_object_info+0x1e
> #8 0xffffffff823983a5 at dmu_recv_stream+0x7b5
> #9 0xffffffff8244b706 at zfs_ioc_recv+0xac6
> #10 0xffffffff8244dd3d at zfsdev_ioctl+0x62d
> #11 0xffffffff808a35e0 at devfs_ioctl+0xb0
> #12 0xffffffff80f3becb at VOP_IOCTL_APV+0x7b
> #13 0xffffffff80ad1b0a at vn_ioctl+0x16a
> #14 0xffffffff808a3bce at devfs_ioctl_f+0x1e
> #15 0xffffffff80a5d807 at kern_ioctl+0x2b7
> #16 0xffffffff80a5d4aa at sys_ioctl+0xfa
> #17 0xffffffff80e40c87 at amd64_syscall+0x387
> Uptime: 3d14h59m52s
> Dumping 17213 out of 65366
> MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%
>
> __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
> 55 __asm("movq %%gs:%P1,%0" : "=r" (td) : "n"
> (offsetof(struct pcpu,
> (kgdb) #0 __curthread () at /usr/src/sys/amd64/include/pcpu_aux.h:55
> #1 doadump (textdump=<optimized out>)
> at /usr/src/sys/kern/kern_shutdown.c:371
> #2 0xffffffff809f3805 in kern_reboot (howto=260)
> at /usr/src/sys/kern/kern_shutdown.c:451
> #3 0xffffffff809f3c43 in vpanic (fmt=<optimized out>, ap=<optimized out>)
> at /usr/src/sys/kern/kern_shutdown.c:880
> #4 0xffffffff809f3a63 in panic (fmt=<unavailable>)
> at /usr/src/sys/kern/kern_shutdown.c:807
> #5 0xffffffff80e400d1 in trap_fatal (frame=0xfffffe0343230f40, eva=0)
> at /usr/src/sys/amd64/amd64/trap.c:921
> #6 0xffffffff80e4012f in trap_pfault (frame=0xfffffe0343230f40,
> usermode=<optimized out>, signo=<optimized out>, ucode=<optimized out>)
> at /usr/src/sys/amd64/amd64/trap.c:739
> #7 0xffffffff80e3f776 in trap (frame=0xfffffe0343230f40)
> at /usr/src/sys/amd64/amd64/trap.c:405
> #8 <signal handler called>
> #9 0xffffffff823a554b in dnode_hold_impl (os=0xfffff805e1d2b800,
> object=<optimized out>, flag=<optimized out>, slots=<optimized out>,
> tag=<optimized out>, dnp=0xfffffe03432310d8)
> at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dnode.c:1370
> #10 0xffffffff82393a5e in dmu_object_info (os=0xfffff80777890070,
> object=18446744071600721588, doi=0xfffffe03432312e0)
> at /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu.c:2615
> #11 0xffffffff823983a5 in receive_read_record (ra=<optimized out>)
> at
> /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c:2821
> #12 dmu_recv_stream (drc=0xfffffe0343231430, fp=<optimized out>,
> voffp=<optimized out>, cleanup_fd=8, action_handlep=<optimized out>)
> at
> /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_send.c:3203
> #13 0xffffffff8244b706 in zfs_ioc_recv (zc=<optimized out>)
> at
> /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c:4745
> #14 0xffffffff8244dd3d in zfsdev_ioctl (dev=<optimized out>,
> zcmd=<optimized out>, arg=<optimized out>, flag=<optimized out>,
> td=<optimized out>)
> at
> /usr/src/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c:6956
> #15 0xffffffff808a35e0 in devfs_ioctl (ap=0xfffffe0343231778)
> at /usr/src/sys/fs/devfs/devfs_vnops.c:797
> #16 0xffffffff80f3becb in VOP_IOCTL_APV (
> vop=0xffffffff816a2fe0 <devfs_specops>, a=0xfffffe0343231778)
> at vnode_if.c:1067
> #17 0xffffffff80ad1b0a in vn_ioctl (fp=0xfffff8001802b5a0,
> com=<optimized out>, data=0xfffffe0343231910,
> active_cred=0xfffff80032214300, td=0x20700000)
> at /usr/src/sys/kern/vfs_vnops.c:1508
> #18 0xffffffff808a3bce in devfs_ioctl_f (fp=0xfffff80777890070,
> com=18446744071600721588, data=0xffffffff824e34ed <.L.str+1>, cred=0x0,
> td=0xfffff80298850000) at /usr/src/sys/fs/devfs/devfs_vnops.c:755
> #19 0xffffffff80a5d807 in fo_ioctl (fp=0xfffff8001802b5a0, com=3222821403,
> data=0xffffffff824e34ed <.L.str+1>, active_cred=0x0,
> td=0xfffff80298850000) at /usr/src/sys/sys/file.h:337
> #20 kern_ioctl (td=0x20700000, fd=<optimized out>, com=3222821403,
> data=0xffffffff824e34ed <.L.str+1> "zrl->zr_mtx")
> at /usr/src/sys/kern/sys_generic.c:805
> #21 0xffffffff80a5d4aa in sys_ioctl (td=0xfffff80298850000,
> uap=0xfffff802988503c0) at /usr/src/sys/kern/sys_generic.c:713
> #22 0xffffffff80e40c87 in syscallenter (td=0xfffff80298850000)
> at /usr/src/sys/amd64/amd64/../../kern/subr_syscall.c:144
> #23 amd64_syscall (td=0xfffff80298850000, traced=0)
> at /usr/src/sys/amd64/amd64/trap.c:1163
> #24 <signal handler called>
> #25 0x00000008004a8cba in ?? ()
> Backtrace stopped: Cannot access memory at address 0x7fffffffa988
> (kgdb)
>
>
> _______________________________________________
> freebsd-stable@freebsd.org mailing list
> https://lists.freebsd.org/mailman/listinfo/freebsd-stable
> To unsubscribe, send any mail to "freebsd-stable-unsubscribe@freebsd.org"
>
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?5b3415cb-2176-895e-9d22-4f4f0f359d85>