From nobody Fri Jun 9 02:56:07 2023 X-Original-To: current@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id 4Qcm1X0slkz4bHKM for ; Fri, 9 Jun 2023 02:56:16 +0000 (UTC) (envelope-from glebius@freebsd.org) Received: from glebi.us (glebi.us [162.251.186.162]) by mx1.freebsd.org (Postfix) with ESMTP id 4Qcm1V58Jmz46RH; Fri, 9 Jun 2023 02:56:14 +0000 (UTC) (envelope-from glebius@freebsd.org) Authentication-Results: mx1.freebsd.org; dkim=none; spf=softfail (mx1.freebsd.org: 162.251.186.162 is neither permitted nor denied by domain of glebius@freebsd.org) smtp.mailfrom=glebius@freebsd.org; dmarc=none Received: by glebi.us (Postfix, from userid 1000) id A64C6226B6; Thu, 8 Jun 2023 19:56:07 -0700 (PDT) Date: Thu, 8 Jun 2023 19:56:07 -0700 From: Gleb Smirnoff To: current@freebsd.org Cc: mm@freebsd.org Subject: panic(s) in ZFS on CURRENT Message-ID: List-Id: Discussions about the use of FreeBSD-current List-Archive: https://lists.freebsd.org/archives/freebsd-current List-Help: List-Post: List-Subscribe: List-Unsubscribe: Sender: owner-freebsd-current@freebsd.org MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline X-Spamd-Result: default: False [0.50 / 15.00]; VIOLATED_DIRECT_SPF(3.50)[]; NEURAL_HAM_MEDIUM(-1.00)[-1.000]; NEURAL_HAM_LONG(-1.00)[-1.000]; NEURAL_HAM_SHORT(-1.00)[-1.000]; MIME_GOOD(-0.10)[text/plain]; RCVD_NO_TLS_LAST(0.10)[]; MLMMJ_DEST(0.00)[current@freebsd.org]; ASN(0.00)[asn:27348, ipnet:162.251.186.0/24, country:US]; FROM_EQ_ENVFROM(0.00)[]; R_DKIM_NA(0.00)[]; RCPT_COUNT_TWO(0.00)[2]; MIME_TRACE(0.00)[0:+]; ARC_NA(0.00)[]; R_SPF_SOFTFAIL(0.00)[~all]; FROM_HAS_DN(0.00)[]; FREEFALL_USER(0.00)[glebius]; RCVD_COUNT_TWO(0.00)[2]; TO_MATCH_ENVRCPT_ALL(0.00)[]; DMARC_NA(0.00)[freebsd.org]; TO_DN_NONE(0.00)[]; MID_RHS_MATCH_FROM(0.00)[] X-Rspamd-Queue-Id: 4Qcm1V58Jmz46RH X-Spamd-Bar: / X-ThisMailContainsUnwantedMimeParts: N Hi, I got several panics on my desktop running eb2b00da564 which is after the latest OpenZFS merge. #1 (couple cores with this backtrace) --- trap 0x9, rip = 0xffffffff803ab94b, rsp = 0xfffffe022e45ed30, rbp = 0xfffffe022e45ed50 --- buf_hash_insert() at buf_hash_insert+0xab/frame 0xfffffe022e45ed50 arc_write_done() at arc_write_done+0xfa/frame 0xfffffe022e45ed90 zio_done() at zio_done+0xf0b/frame 0xfffffe022e45ee00 zio_execute() at zio_execute+0x9f/frame 0xfffffe022e45ee40 taskqueue_run_locked() at taskqueue_run_locked+0x181/frame 0xfffffe022e45eec0 taskqueue_thread_loop() at taskqueue_thread_loop+0xc3/frame 0xfffffe022e45eef0 fork_exit() at fork_exit+0x7d/frame 0xfffffe022e45ef30 fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe022e45ef30 --- trap 0, rip = 0, rsp = 0, rbp = 0 --- (kgdb) frame 7 #7 buf_hash_insert (hdr=hdr@entry=0xfffff8001b21fa28, lockp=lockp@entry=0xfffffe022e45ed60) at /usr/src/FreeBSD/sys/contrib/openzfs/module/zfs/arc.c:1062 1062 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr)) (kgdb) p hdr $1 = (arc_buf_hdr_t *) 0xfffff8001b21fa28 (kgdb) p *hdr $2 = {b_dva = {dva_word = {16, 20406677952}}, b_birth = 38447120, b_type = ARC_BUFC_METADATA, b_complevel = 255 '\377', b_reserved1 = 0 '\000', b_reserved2 = 0, b_hash_next = 0x0, b_flags = (ARC_FLAG_L2CACHE | ARC_FLAG_IO_IN_PROGRESS | ARC_FLAG_BUFC_METADATA | ARC_FLAG_HAS_L1HDR | ARC_FLAG_COMPRESSED_ARC | ARC_FLAG_COMPRESS_0 | ARC_FLAG_COMPRESS_1 | ARC_FLAG_COMPRESS_2 | ARC_FLAG_COMPRESS_3), b_psize = 8, b_lsize = 32, b_spa = 1230587331341359116, b_l2hdr = { b_dev = 0x0, b_daddr = 0, b_hits = 0, b_arcs_state = ARC_STATE_ANON, b_l2node = {list_next = 0x0, list_prev = 0x0}}, b_l1hdr = {b_cv = { cv_description = 0xffffffff80bb5b02 "hdr->b_l1hdr.b_cv", cv_waiters = 0}, b_byteswap = 10 '\n', b_state = 0xffffffff80ef23c0 , b_arc_node = {list_next = 0x0, list_prev = 0x0}, b_arc_access = 0, b_mru_hits = 0, b_mru_ghost_hits = 0, b_mfu_hits = 0, b_mfu_ghost_hits = 0, b_bufcnt = 1, b_buf = 0xfffff80003139d80, b_refcnt = {rc_count = 2}, b_acb = 0x0, b_pabd = 0xfffff80a35dc6480}, b_crypt_hdr = {b_rabd = 0x10, b_ot = 2744191968, b_ebufcnt = 4, b_dsobj = 38340866, b_salt = "\001\000\000\000\000\000\000", b_iv = "\000\000\000\000\000\000\000\000\220\000\026\017", b_mac = "\b\000 \000\f\230\262m\250\354\023\021\000\000\000"}} #2 (single core) --- trap 0x9, rip = 0xffffffff803ab94b, rsp = 0xfffffe0256158780, rbp = 0xfffffe02561587a0 --- buf_hash_insert() at buf_hash_insert+0xab/frame 0xfffffe02561587a0 arc_hdr_realloc() at arc_hdr_realloc+0x138/frame 0xfffffe0256158800 arc_read() at arc_read+0x2dc/frame 0xfffffe02561588b0 dbuf_read() at dbuf_read+0xb3e/frame 0xfffffe02561589f0 dmu_buf_hold() at dmu_buf_hold+0x46/frame 0xfffffe0256158a30 zap_cursor_retrieve() at zap_cursor_retrieve+0x167/frame 0xfffffe0256158a90 zfs_freebsd_readdir() at zfs_freebsd_readdir+0x383/frame 0xfffffe0256158cc0 VOP_READDIR_APV() at VOP_READDIR_APV+0x1f/frame 0xfffffe0256158ce0 kern_getdirentries() at kern_getdirentries+0x186/frame 0xfffffe0256158dd0 sys_getdirentries() at sys_getdirentries+0x29/frame 0xfffffe0256158e00 amd64_syscall() at amd64_syscall+0x100/frame 0xfffffe0256158f30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0256158f30 (kgdb) frame 7 #7 buf_hash_insert (hdr=0xfffff80c906b03e8, lockp=lockp@entry=0x0) at /usr/src/FreeBSD/sys/contrib/openzfs/module/zfs/arc.c:1062 1062 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr)) (kgdb) p *hdr $1 = {b_dva = {dva_word = {16, 19965896928}}, b_birth = 36629088, b_type = ARC_BUFC_METADATA, b_complevel = 0 '\000', b_reserved1 = 0 '\000', b_reserved2 = 0, b_hash_next = 0x0, b_flags = (ARC_FLAG_BUFC_METADATA | ARC_FLAG_HAS_L1HDR | ARC_FLAG_HAS_L2HDR | ARC_FLAG_COMPRESSED_ARC | ARC_FLAG_COMPRESS_1), b_psize = 5, b_lsize = 5, b_spa = 3583499065027950438, b_l2hdr = {b_dev = 0xfffffe02306c8000, b_daddr = 4917395456, b_hits = 0, b_arcs_state = ARC_STATE_MRU, b_l2node = {list_next = 0xfffff801313fc9b0, list_prev = 0xfffff801313fca70}}, b_l1hdr = {b_cv = { cv_description = 0xffffffff80bb5b02 "hdr->b_l1hdr.b_cv", cv_waiters = 0}, b_byteswap = 10 '\n', b_state = 0xffffffff80f02900 , b_arc_node = {list_next = 0x0, list_prev = 0x0}, b_arc_access = 0, b_mru_hits = 0, b_mru_ghost_hits = 0, b_mfu_hits = 0, b_mfu_ghost_hits = 0, b_bufcnt = 0, b_buf = 0x0, b_refcnt = {rc_count = 0}, b_acb = 0x0, b_pabd = 0x0}, b_crypt_hdr = {b_rabd = 0x10, b_ot = 2786027712, b_ebufcnt = 4, b_dsobj = 36629088, b_salt = "\001\000\000\000\000\000\000", b_iv = "\240\2769$\001\370\377\377\220\000\036\017", b_mac = "\b\000 \000fw\357\327i%\2731\000\200l0"}} #3 (not ZFS, but VFS, could be related?) --- trap 0x9, rip = 0xffffffff80801408, rsp = 0xfffffe02348cbcc0, rbp = 0xfffffe02348cbcf0 --- pwd_chdir() at pwd_chdir+0x28/frame 0xfffffe02348cbcf0 kern_chdir() at kern_chdir+0x169/frame 0xfffffe02348cbe00 amd64_syscall() at amd64_syscall+0x100/frame 0xfffffe02348cbf30 fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe02348cbf30 (kgdb) frame 7 #7 pwd_alloc () at /usr/src/FreeBSD/sys/kern/kern_descrip.c:3920 warning: Source file is more recent than executable. 3920 bzero(pwd, sizeof(*pwd)); (kgdb) p pwd $1 = (struct pwd *) 0xfff2fff0fff1ffed (kgdb) p *pwd Cannot access memory at address 0xfff2fff0fff1ffed I'm switching to INVARIANTS kernel right now and will see if that panics earlier. -- Gleb Smirnoff