Date: Wed, 9 Apr 2014 14:19:17 +0300 From: Konstantin Belousov <kostikbel@gmail.com> To: Karl Pielorz <kpielorz_lst@tdx.co.uk> Cc: freebsd-hackers@freebsd.org Subject: Re: Stuck CLOSED sockets / sshd / zombies... Message-ID: <20140409111917.GH21331@kib.kiev.ua> In-Reply-To: <2A722BB3B12E0D80CA9FF075@Mail-PC.tdx.co.uk> References: <3FE645E9723756F22EF901AE@Mail-PC.tdx.co.uk> <20140408164353.GB21331@kib.kiev.ua> <277FA3F7B4E7A98921F4D631@study64.tdx.co.uk> <201404081533.53990.jhb@freebsd.org> <92366925229B4C5B21B04D81@study64.tdx.co.uk> <20140408212319.GC21331@kib.kiev.ua> <D0B81EA30BF8126B37F98D18@study64.tdx.co.uk> <20140409084951.GE21331@kib.kiev.ua> <2A722BB3B12E0D80CA9FF075@Mail-PC.tdx.co.uk>
index | next in thread | previous in thread | raw e-mail
[-- Attachment #1 --] On Wed, Apr 09, 2014 at 11:15:20AM +0100, Karl Pielorz wrote: > > > --On 09 April 2014 11:49 +0300 Konstantin Belousov <kostikbel@gmail.com> > wrote: > > > Hm, I think my instructions were flawed, you have to install with > > DEBUG_FLAGS as well: > > make install DEBUG_FLAGS=-g > > > > You do not need to re-run the tests if rtld did not changed after > > the installation. Reinstall and get the backtrace again, please. > > Ok, did that - output below, > > -Karl > > --- > > " > [Switching to LWP 100218] > 0x00000008038ea89c in __error () from /lib/libthr.so.3 > (gdb) bt > #0 0x00000008038ea89c in __error () from /lib/libthr.so.3 > #1 0x00000008038e104f in __thr_rwlock_rdlock (rwlock=0x803af9480, > flags=<value optimized out>, tsp=<value optimized out>) > at /usr/src/lib/libthr/thread/thr_umtx.c:277 > #2 0x00000008038e821c in _thr_rtld_rlock_acquire (lock=0x803af9480) at > atomic.h:143 > #3 0x000000080064f9a2 in digest_dynamic1 (obj=0x80085fe00, early=32767, > dyn_rpath=0x80582a93c, dyn_soname=0x80582a93c, dyn_runpath=0x7fffffffba30) > at /usr/src/libexec/rtld-elf/rtld.c:1103 > #4 0x00000008006498c9 in objlist_call_init (list=<value optimized out>, > lockstate=0x0) at /usr/src/libexec/rtld-elf/rtld.c:287 > #5 0x00000008006470cd in _rtld_bind_start () at > /usr/src/libexec/rtld-elf/amd64/rtld_start.S:121 > #6 0x000000000041072c in grace_alarm_handler (sig=-17504) at > /usr/src/secure/usr.sbin/sshd/../../../crypto/openssh/sshd.c:378 > #7 <signal handler called> > #8 0x00000008038ea89c in __error () from /lib/libthr.so.3 > #9 0x00000008038e104f in __thr_rwlock_rdlock (rwlock=0x803af9480, > flags=<value optimized out>, tsp=<value optimized out>) > at /usr/src/lib/libthr/thread/thr_umtx.c:277 > #10 0x00000008038e821c in _thr_rtld_rlock_acquire (lock=0x803af9480) at > atomic.h:143 > #11 0x000000080064f9a2 in digest_dynamic1 (obj=0x80085fe00, early=32767, > dyn_rpath=0x8038d8e30, dyn_soname=0x100000001, dyn_runpath=0x7fffffffc040) > at /usr/src/libexec/rtld-elf/rtld.c:1103 > #12 0x00000008006498c9 in objlist_call_init (list=<value optimized out>, > lockstate=0xffff00001f80) at /usr/src/libexec/rtld-elf/rtld.c:287 > #13 0x00000008006470cd in _rtld_bind_start () at > /usr/src/libexec/rtld-elf/amd64/rtld_start.S:121 > #14 0x000000000042f9dd in sshpam_sigchld_handler (sig=<value optimized > out>) at > /usr/src/secure/usr.sbin/sshd/../../../crypto/openssh/auth-pam.c:152 > #15 <signal handler called> > #16 0x0000000800653aea in lmc_parse () at > /usr/src/libexec/rtld-elf/libmap.c:306 > #17 0x000000080064a835 in objlist_call_fini () at > /usr/src/libexec/rtld-elf/rtld.c:2267 > #18 0x000000080064a1eb in symlook_default (req=0x7fffffffd050, > refobj=<value optimized out>) at /usr/src/libexec/rtld-elf/rtld.c:3620 > #19 0x0000000800edd121 in openpam_clear_chains () from /usr/lib/libpam.so.5 > #20 0x0000000800edd0bc in openpam_clear_chains () from /usr/lib/libpam.so.5 > #21 0x0000000800edd0bc in openpam_clear_chains () from /usr/lib/libpam.so.5 > #22 0x0000000800edd061 in openpam_clear_chains () from /usr/lib/libpam.so.5 > #23 0x0000000800ed99e7 in pam_end () from /usr/lib/libpam.so.5 > #24 0x000000000042e15d in sshpam_cleanup () at > /usr/src/secure/usr.sbin/sshd/../../../crypto/openssh/auth-pam.c:614 > #25 0x000000000041d58f in do_cleanup (authctxt=0x80401a600) at > /usr/src/secure/usr.sbin/sshd/../../../crypto/openssh/session.c:2732 > #26 0x000000000041064f in ssh_cleanup_exit (i=255) at > /usr/src/secure/usr.sbin/sshd/../../../crypto/openssh/sshd.c:2545 > #27 0x0000000000428f83 in mm_request_receive (sock=<value optimized out>, > m=<value optimized out>) > at > /usr/src/secure/usr.sbin/sshd/../../../crypto/openssh/monitor_wrap.c:153 > #28 0x0000000000427e26 in monitor_read (pmonitor=0x804022220, ent=0x6465a0, > pent=0x7fffffffd0c0) > at /usr/src/secure/usr.sbin/sshd/../../../crypto/openssh/monitor.c:593 > #29 0x0000000000427b49 in monitor_child_preauth (_authctxt=<value optimized > out>, pmonitor=0x804022220) > at /usr/src/secure/usr.sbin/sshd/../../../crypto/openssh/monitor.c:387 > #30 0x000000000040fd15 in main (ac=<value optimized out>, av=<value > optimized out>) at > /usr/src/secure/usr.sbin/sshd/../../../crypto/openssh/sshd.c:679 > " It is still mostly nonsensical, due to bad and missing debugging information. First, my patch seems to be buggy, I miscalculated the offsets of the saved registers. Hopefully, improved version is at the end of the message. Also, I suspect that there is a mismatch between installed and built rtld. Please do the clean build with DEBUG_FLAGS=-g and patch applied and install (again with DEBUG_FLAGS=-g). Second, the debugging information in your libthr.so.3 is partial. Could you, please rebuild it and install with DEBUG_FLAGS=-g from the clean state ? Also, please rebuild you pam installation with '-g'. After this is done, reproduce the issue and take the backtrace once more. Sorry, but the current backtrace is not useful. diff --git a/libexec/rtld-elf/amd64/rtld_start.S b/libexec/rtld-elf/amd64/rtld_start.S index da3d156..2481f09 100644 --- a/libexec/rtld-elf/amd64/rtld_start.S +++ b/libexec/rtld-elf/amd64/rtld_start.S @@ -79,17 +79,39 @@ .globl _rtld_bind_start .type _rtld_bind_start,@function _rtld_bind_start: + .cfi_startproc + .cfi_adjust_cfa_offset 16 subq $8,%rsp + .cfi_adjust_cfa_offset 8 pushfq # Save rflags + .cfi_adjust_cfa_offset 8 pushq %rax # Save %rax + .cfi_adjust_cfa_offset 8 + .cfi_offset %rax,-32 pushq %rdx # Save %rdx + .cfi_adjust_cfa_offset 8 + .cfi_offset %rdx,-40 pushq %rcx # Save %rcx + .cfi_adjust_cfa_offset 8 + .cfi_offset %rcx,-48 pushq %rsi # Save %rsi + .cfi_adjust_cfa_offset 8 + .cfi_offset %rsi,-56 pushq %rdi # Save %rdi + .cfi_adjust_cfa_offset 8 + .cfi_offset %rdi,-64 pushq %r8 # Save %r8 + .cfi_adjust_cfa_offset 8 + .cfi_offset %r8,-72 pushq %r9 # Save %r9 + .cfi_adjust_cfa_offset 8 + .cfi_offset %r9,-80 pushq %r10 # Save %r10 + .cfi_adjust_cfa_offset 8 + .cfi_offset %r10,-88 pushq %r11 # Save %r11 + .cfi_adjust_cfa_offset 8 + .cfi_offset %r11,-96 movq 0x58(%rsp),%rdi # Fetch obj argument movq 0x60(%rsp),%rsi # Fetch reloff argument @@ -101,16 +123,37 @@ _rtld_bind_start: movq %rax,0x60(%rsp) # Store target over reloff argument popq %r11 # Restore %r11 + .cfi_adjust_cfa_offset -8 + .cfi_restore %r11 popq %r10 # Restore %r10 + .cfi_adjust_cfa_offset -8 + .cfi_restore %r10 popq %r9 # Restore %r9 + .cfi_adjust_cfa_offset -8 + .cfi_restore %r9 popq %r8 # Restore %r8 + .cfi_adjust_cfa_offset -8 + .cfi_restore %r8 popq %rdi # Restore %rdi + .cfi_adjust_cfa_offset -8 + .cfi_restore %rdi popq %rsi # Restore %rsi + .cfi_adjust_cfa_offset -8 + .cfi_restore %rsi popq %rcx # Restore %rcx + .cfi_adjust_cfa_offset -8 + .cfi_restore %rcx popq %rdx # Restore %rdx + .cfi_adjust_cfa_offset -8 + .cfi_restore %rdx popq %rax # Restore %rax + .cfi_adjust_cfa_offset -8 + .cfi_restore %rax popfq # Restore rflags + .cfi_adjust_cfa_offset -8 leaq 16(%rsp),%rsp # Discard spare, obj, do not change rflags ret # "Return" to target address + .cfi_endproc + .size _rtld_bind_start, . - _rtld_bind_start .section .note.GNU-stack,"",%progbits [-- Attachment #2 --] -----BEGIN PGP SIGNATURE----- Version: GnuPG v2.0.22 (FreeBSD) iQIcBAEBAgAGBQJTRSy1AAoJEJDCuSvBvK1BTwsQAJKgoOb0UHnHcDFbqRHYMpep B9x8gSO8olF3vmwRTG/XqEWA6GOvMoEjC4rsJ/hlWpz9RVhoiiY7g0VZuYzpUifx p2d+Swc5JbsiKBDnvfx5uYgAGKVVPbvvE81Pmf4wFLBVN4CXSFFPd978zdJwwh59 UEB5gLuqj2nd17EoWdS93QqiIoVvmAs2NtWKxzXCR/GYhUrADLAGf+uNXnaOG7SJ O45yS4EJJkP+oxzgozGP+3lzRWr40VH+nkCrE2ZBcwyPZrHtHbAqD5ekI/0hd5Gq DbnCShnorl1N0vkY/ACR/4BmNfClN3WJRDD6ZsauaFHNTo8IoRjEUasOoZ7eGccE KWlrT8nFoiUbAXzOEmqyF0Cag9CHo+VhO+I4OJ+rWa6FKYybDNuCi/9LvKgpRnVH OpVEonbCTer4iMJ8LfTc2jgrZJywefkgVd1Ig5QiPXrRxhPQqxQ6GmTKIeMOu1i6 1xLms4Yec/HqRf5s2McsncY1IWNFcbDU8nxl4NHIKGDrVA606CSKcCDkeTXRUOJ+ wpMtNsgjRsk+AJ3ArTA1tl72Kqqdx+DZmIAqjptcnIZKzkg3m+mj5jFvHD4dxhfr QD7ZYZabYxTT+RkKj8qkMqvMEnZG0G0X9wUA7LGxnem3DihBArak1Q6Hs+6c00wh muoLMqAOdd0K4X8tMn+5 =Xnmi -----END PGP SIGNATURE-----help
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20140409111917.GH21331>