Date: Wed, 6 May 2020 16:57:55 -0700 From: Mark Millard <marklmi@yahoo.com> To: "vangyzen@freebsd.org" <vangyzen@FreeBSD.org>, svn-src-head@freebsd.org, FreeBSD Current <freebsd-current@freebsd.org>, FreeBSD Hackers <freebsd-hackers@freebsd.org>, FreeBSD PowerPC ML <freebsd-ppc@freebsd.org> Subject: Re: svn commit: r360233 - in head: contrib/jemalloc . . . : This partially breaks a 2-socket 32-bit powerpc (old PowerMac G4) based on head -r360311 Message-ID: <8AAB0462-3FA8-490C-8D8D-7C15B1C9E2DE@yahoo.com> In-Reply-To: <121B9B09-141B-4DC3-918B-1E7CFB99E779@yahoo.com> References: <C24EE1A1-FAED-42C2-8204-CA7B1D20A369@yahoo.com> <8479DD58-44F6-446A-9CA5-D01F0F7C1B38@yahoo.com> <17ACDA02-D7EF-4F26-874A-BB3E935CD072@yahoo.com> <695E6836-F860-4557-B7DE-CC1EDB347F18@yahoo.com> <DCABCD83-27B0-4F2D-9410-69102294A98E@yahoo.com> <121B9B09-141B-4DC3-918B-1E7CFB99E779@yahoo.com>
next in thread | previous in thread | raw e-mail | index | archive | help
[This explores process crashes that happen during system
shutdown, in a context not having MALLOC_PRODUCTION=3D .
So assert failures are reported as the stopping points.]
It looks like shutdown -p now, shutdown -r now, and the
like can lead some processes to assert during their exit
attempt, including a sshd failure (that I've not seen
before), rpcbind, and nfsd. I show information about the
observed asserts for those below.
sshd hit an assert, failing slab =3D=3D extent_slab_get(extent) :
(gdb) bt=20
#0 thr_kill () at thr_kill.S:4
#1 0x50927170 in __raise (s=3D6) at /usr/src/lib/libc/gen/raise.c:52
#2 0x50886cc0 in abort () at /usr/src/lib/libc/stdlib/abort.c:67
#3 0x508834b0 in arena_dalloc (tsdn=3D<optimized out>, ptr=3D<optimized =
out>, tcache=3D<optimized out>, alloc_ctx=3D<optimized out>, =
slow_path=3D<optimized out>)
at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h:315
#4 idalloctm (tsdn=3D0x500dd040, ptr=3D0x5008a180, tcache=3D0x500dd160, =
alloc_ctx=3D<optimized out>, is_internal=3D<optimized out>, =
slow_path=3D<optimized out>)
at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_inli=
nes_c.h:118
#5 0x5087b0a4 in ifree (tsd=3D0x500dd040, ptr=3D0x5008a180, =
tcache=3D0x500dd160, slow_path=3D<optimized out>) at =
jemalloc_jemalloc.c:2590
#6 0x5087acac in __je_free_default (ptr=3D0x5008a180) at =
jemalloc_jemalloc.c:2784
#7 0x5087b294 in __free (ptr=3D0x5008a180) at jemalloc_jemalloc.c:2852
#8 0x10029464 in server_accept_loop (config_s=3D<optimized out>, =
sock_in=3D<optimized out>, sock_out=3D<optimized out>, =
newsock=3D<optimized out>) at /usr/src/crypto/openssh/sshd.c:1185
#9 main (ac=3D<optimized out>, av=3D0xffffde3c) at =
/usr/src/crypto/openssh/sshd.c:2009
. . .
(gdb) up
#2 0x50886cc0 in abort () at /usr/src/lib/libc/stdlib/abort.c:67
67 (void)raise(SIGABRT);
(gdb) up
#3 0x508834b0 in arena_dalloc (tsdn=3D<optimized out>, ptr=3D<optimized =
out>, tcache=3D<optimized out>, alloc_ctx=3D<optimized out>, =
slow_path=3D<optimized out>)
at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h:315
315 assert(slab =3D=3D extent_slab_get(extent));
(gdb) list
310 rtree_ctx =3D tsd_rtree_ctx(tsdn_tsd(tsdn));
311 extent_t *extent =3D rtree_extent_read(tsdn, =
&extents_rtree,
312 rtree_ctx, (uintptr_t)ptr, true);
313 assert(szind =3D=3D extent_szind_get(extent));
314 assert(szind < SC_NSIZES);
315 assert(slab =3D=3D extent_slab_get(extent));
316 }
317=09
318 if (likely(slab)) {
319 /* Small allocation. */
More fully:
285 JEMALLOC_ALWAYS_INLINE void
286 arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
287 alloc_ctx_t *alloc_ctx, bool slow_path) {
288 assert(!tsdn_null(tsdn) || tcache =3D=3D NULL);
289 assert(ptr !=3D NULL);
290=09
291 if (unlikely(tcache =3D=3D NULL)) {
292 arena_dalloc_no_tcache(tsdn, ptr);
293 return;
294 }
295=09
296 szind_t szind;
297 bool slab;
298 rtree_ctx_t *rtree_ctx;
299 if (alloc_ctx !=3D NULL) {
300 szind =3D alloc_ctx->szind;
301 slab =3D alloc_ctx->slab;
302 assert(szind !=3D SC_NSIZES);
303 } else {
304 rtree_ctx =3D tsd_rtree_ctx(tsdn_tsd(tsdn));
305 rtree_szind_slab_read(tsdn, &extents_rtree, =
rtree_ctx,
306 (uintptr_t)ptr, true, &szind, &slab);
307 }
308=09
309 if (config_debug) {
310 rtree_ctx =3D tsd_rtree_ctx(tsdn_tsd(tsdn));
311 extent_t *extent =3D rtree_extent_read(tsdn, =
&extents_rtree,
312 rtree_ctx, (uintptr_t)ptr, true);
313 assert(szind =3D=3D extent_szind_get(extent));
314 assert(szind < SC_NSIZES);
315 assert(slab =3D=3D extent_slab_get(extent));
316 }
317=09
318 if (likely(slab)) {
319 /* Small allocation. */
320 tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, =
szind,
321 slow_path);
322 } else {
323 arena_dalloc_large(tsdn, ptr, tcache, szind, =
slow_path);
324 }
325 }
rpcbind hit an assert, failing ret =3D=3D sz_size2index_compute(size) :
(gdb) bt
#0 thr_kill () at thr_kill.S:4
#1 0x502f2170 in __raise (s=3D6) at /usr/src/lib/libc/gen/raise.c:52
#2 0x50251d04 in abort () at /usr/src/lib/libc/stdlib/abort.c:79
#3 0x5024f260 in sz_size2index_lookup (size=3D<optimized out>) at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:159
#4 sz_size2index (size=3D<optimized out>) at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:166
#5 imalloc_body (sopts=3D0xffffb360, dopts=3D0xffffb340, =
tsd=3D0x5009a018) at jemalloc_jemalloc.c:2066
#6 0x50244874 in imalloc (sopts=3D0xffffb360, dopts=3D0xffffb340) at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/tsd.h:331
#7 0x50244fe8 in __calloc (num=3D1, size=3D96) at =
jemalloc_jemalloc.c:2498
#8 0x50265690 in svc_xprt_alloc () at /usr/src/lib/libc/rpc/svc.c:541
#9 0x502635f4 in makefd_xprt (fd=3D14, sendsize=3D9000, recvsize=3D9000) =
at /usr/src/lib/libc/rpc/svc_vc.c:250
#10 0x502644b4 in rendezvous_request (xprt=3D0x5004c000, msg=3D<optimized =
out>) at /usr/src/lib/libc/rpc/svc_vc.c:315
#11 0x50265a98 in svc_getreq_common (fd=3D<optimized out>) at =
/usr/src/lib/libc/rpc/svc.c:640
#12 0x50265d1c in svc_getreq_poll (pfdp=3D<optimized out>, pollretval=3D1)=
at /usr/src/lib/libc/rpc/svc.c:739
#13 0x10018568 in my_svc_run () at =
/usr/src/usr.sbin/rpcbind/rpcb_svc_com.c:1167
#14 0x10014ad8 in main (argc=3D<optimized out>, argv=3D<optimized out>) =
at /usr/src/usr.sbin/rpcbind/rpcbind.c:250
(gdb) up 3
#3 0x5024f260 in sz_size2index_lookup (size=3D<optimized out>) at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:159
159 assert(ret =3D=3D sz_size2index_compute(size));
(gdb) print ret
$1 =3D 0
154 JEMALLOC_ALWAYS_INLINE szind_t
155 sz_size2index_lookup(size_t size) {
156 assert(size <=3D SC_LOOKUP_MAXCLASS);
157 szind_t ret =3D (sz_size2index_tab[(size + (ZU(1) << =
SC_LG_TINY_MIN) - 1)
158 >> SC_LG_TINY_MIN]);
159 assert(ret =3D=3D sz_size2index_compute(size));
160 return ret;
161 }
nfsd hit an assert, failing ret =3D=3D sz_size2index_compute(size)
(also, but a different caller of sz_size2index):
(gdb) bt
#0 thr_kill () at thr_kill.S:4
#1 0x502b2170 in __raise (s=3D6) at /usr/src/lib/libc/gen/raise.c:52
#2 0x50211cc0 in abort () at /usr/src/lib/libc/stdlib/abort.c:67
#3 0x50206104 in sz_index2size_lookup (index=3D<optimized out>) at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:200
#4 sz_index2size (index=3D<optimized out>) at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:207
#5 ifree (tsd=3D0x50094018, ptr=3D0x50041028, tcache=3D0x50094138, =
slow_path=3D<optimized out>) at jemalloc_jemalloc.c:2583
#6 0x50205cac in __je_free_default (ptr=3D0x50041028) at =
jemalloc_jemalloc.c:2784
#7 0x50206294 in __free (ptr=3D0x50041028) at jemalloc_jemalloc.c:2852
#8 0x50287ec8 in ns_src_free (src=3D0x50329004, srclistsize=3D<optimized =
out>) at /usr/src/lib/libc/net/nsdispatch.c:452
#9 ns_dbt_free (dbt=3D0x50329000) at =
/usr/src/lib/libc/net/nsdispatch.c:436
#10 vector_free (vec=3D0x50329000, count=3D<optimized out>, esize=3D12, =
free_elem=3D<optimized out>) at /usr/src/lib/libc/net/nsdispatch.c:253
#11 nss_atexit () at /usr/src/lib/libc/net/nsdispatch.c:578
#12 0x5028d958 in __cxa_finalize (dso=3D0x0) at =
/usr/src/lib/libc/stdlib/atexit.c:240
#13 0x502117f8 in exit (status=3D0) at =
/usr/src/lib/libc/stdlib/exit.c:74
#14 0x10013f9c in child_cleanup (signo=3D<optimized out>) at =
/usr/src/usr.sbin/nfsd/nfsd.c:969
#15 <signal handler called>
#16 0x00000000 in ?? ()
(gdb) up 3
#3 0x50206104 in sz_index2size_lookup (index=3D<optimized out>) at =
/usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:200
200 assert(ret =3D=3D sz_index2size_compute(index));
(ret is optimized out.)
197 JEMALLOC_ALWAYS_INLINE size_t
198 sz_index2size_lookup(szind_t index) {
199 size_t ret =3D (size_t)sz_index2size_tab[index];
200 assert(ret =3D=3D sz_index2size_compute(index));
201 return ret;
202 }
Booting and immediately trying something like:
service nfsd stop
did not lead to a failure. But may be after
a while it would and be less drastic than a
reboot or power down.
=3D=3D=3D
Mark Millard
marklmi at yahoo.com
( dsl-only.net went
away in early 2018-Mar)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?8AAB0462-3FA8-490C-8D8D-7C15B1C9E2DE>