From owner-freebsd-current@FreeBSD.ORG Wed Aug 25 16:34:52 2004 Return-Path: Delivered-To: freebsd-current@freebsd.org Received: from mx1.FreeBSD.org (mx1.freebsd.org [216.136.204.125]) by hub.freebsd.org (Postfix) with ESMTP id 3F56916A4CE; Wed, 25 Aug 2004 16:34:52 +0000 (GMT) Received: from TRANG.nuxi.com (trang.nuxi.com [66.93.134.19]) by mx1.FreeBSD.org (Postfix) with ESMTP id BA59043D64; Wed, 25 Aug 2004 16:34:51 +0000 (GMT) (envelope-from obrien@NUXI.com) Received: from dragon.nuxi.com (obrien@localhost [127.0.0.1]) by TRANG.nuxi.com (8.13.1/8.12.11) with ESMTP id i7PGYpQA085053; Wed, 25 Aug 2004 09:34:51 -0700 (PDT) (envelope-from obrien@dragon.nuxi.com) Received: (from obrien@localhost) by dragon.nuxi.com (8.13.1/8.13.1/Submit) id i7PGYpsA085052; Wed, 25 Aug 2004 09:34:51 -0700 (PDT) (envelope-from obrien) Date: Wed, 25 Aug 2004 09:34:51 -0700 From: "David O'Brien" To: John Baldwin Message-ID: <20040825163451.GD53710@dragon.nuxi.com> References: <20040822115345.Y94593@carver.gumbysoft.com> <200408231416.25464.jhb@FreeBSD.org> Mime-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <200408231416.25464.jhb@FreeBSD.org> User-Agent: Mutt/1.4.1i X-Operating-System: FreeBSD 6.0-CURRENT Organization: The NUXI BSD Group X-Pgp-Rsa-Fingerprint: B7 4D 3E E9 11 39 5F A3 90 76 5D 69 58 D9 98 7A X-Pgp-Rsa-Keyid: 1024/34F9F9D5 cc: freebsd-current@FreeBSD.org Subject: Re: PLEASE TEST: IPI deadlock avoidance patch X-BeenThere: freebsd-current@freebsd.org X-Mailman-Version: 2.1.1 Precedence: list Reply-To: obrien@FreeBSD.org List-Id: Discussions about the use of FreeBSD-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 25 Aug 2004 16:34:52 -0000 On Mon, Aug 23, 2004 at 02:16:25PM -0400, John Baldwin wrote: > On Sunday 22 August 2004 03:05 pm, Doug White wrote: > > If you have a reasonably fast i386 or amd64 multiprocessor and/or > > hyperthreading machine and are experiencing reproducible hangs during -j > > buildwords and other highly parallel operations, please try this patch: > > http://people.freebsd.org/~dwhite/smp_rv_mtx.patch .. > Looks good to me. I also think this might help the KSE + SMP hangs. Only > suggestion I might have is to rename smp_rv_mtx to smp_ipi_mtx and "SMP IPI > lock" or some such. Like this? Index: amd64/amd64/mp_machdep.c =================================================================== RCS file: /home/ncvs/src/sys/amd64/amd64/mp_machdep.c,v retrieving revision 1.243 diff -u -r1.243 mp_machdep.c --- amd64/amd64/mp_machdep.c 23 Aug 2004 21:39:28 -0000 1.243 +++ amd64/amd64/mp_machdep.c 24 Aug 2004 18:31:48 -0000 @@ -708,7 +708,7 @@ ncpu = mp_ncpus - 1; /* does not shootdown self */ if (ncpu < 1) return; /* no other cpus */ - mtx_assert(&smp_rv_mtx, MA_OWNED); + mtx_assert(&smp_ipi_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); @@ -794,7 +794,7 @@ if (ncpu < 1) return; } - mtx_assert(&smp_rv_mtx, MA_OWNED); + mtx_assert(&smp_ipi_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); Index: amd64/amd64/pmap.c =================================================================== RCS file: /home/ncvs/src/sys/amd64/amd64/pmap.c,v retrieving revision 1.493 diff -u -r1.493 pmap.c --- amd64/amd64/pmap.c 24 Aug 2004 00:17:52 -0000 1.493 +++ amd64/amd64/pmap.c 24 Aug 2004 18:31:48 -0000 @@ -631,7 +631,7 @@ if (smp_started) { if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_rv_mtx); + mtx_lock_spin(&smp_ipi_mtx); } else critical_enter(); /* @@ -652,7 +652,7 @@ smp_masked_invlpg(pmap->pm_active & other_cpus, va); } if (smp_started) - mtx_unlock_spin(&smp_rv_mtx); + mtx_unlock_spin(&smp_ipi_mtx); else critical_exit(); } @@ -667,7 +667,7 @@ if (smp_started) { if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_rv_mtx); + mtx_lock_spin(&smp_ipi_mtx); } else critical_enter(); /* @@ -691,7 +691,7 @@ sva, eva); } if (smp_started) - mtx_unlock_spin(&smp_rv_mtx); + mtx_unlock_spin(&smp_ipi_mtx); else critical_exit(); } @@ -705,7 +705,7 @@ if (smp_started) { if (!(read_rflags() & PSL_I)) panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_rv_mtx); + mtx_lock_spin(&smp_ipi_mtx); } else critical_enter(); /* @@ -726,7 +726,7 @@ smp_masked_invltlb(pmap->pm_active & other_cpus); } if (smp_started) - mtx_unlock_spin(&smp_rv_mtx); + mtx_unlock_spin(&smp_ipi_mtx); else critical_exit(); } Index: i386/i386/mp_machdep.c =================================================================== RCS file: /home/ncvs/src/sys/i386/i386/mp_machdep.c,v retrieving revision 1.236 diff -u -r1.236 mp_machdep.c --- i386/i386/mp_machdep.c 23 Aug 2004 21:39:28 -0000 1.236 +++ i386/i386/mp_machdep.c 24 Aug 2004 18:31:48 -0000 @@ -910,7 +910,7 @@ ncpu = mp_ncpus - 1; /* does not shootdown self */ if (ncpu < 1) return; /* no other cpus */ - mtx_assert(&smp_rv_mtx, MA_OWNED); + mtx_assert(&smp_ipi_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); @@ -996,7 +996,7 @@ if (ncpu < 1) return; } - mtx_assert(&smp_rv_mtx, MA_OWNED); + mtx_assert(&smp_ipi_mtx, MA_OWNED); smp_tlb_addr1 = addr1; smp_tlb_addr2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); Index: i386/i386/pmap.c =================================================================== RCS file: /home/ncvs/src/sys/i386/i386/pmap.c,v retrieving revision 1.495 diff -u -r1.495 pmap.c --- i386/i386/pmap.c 23 Aug 2004 21:39:28 -0000 1.495 +++ i386/i386/pmap.c 24 Aug 2004 18:31:48 -0000 @@ -582,7 +582,7 @@ if (smp_started) { if (!(read_eflags() & PSL_I)) panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_rv_mtx); + mtx_lock_spin(&smp_ipi_mtx); } else critical_enter(); /* @@ -603,7 +603,7 @@ smp_masked_invlpg(pmap->pm_active & other_cpus, va); } if (smp_started) - mtx_unlock_spin(&smp_rv_mtx); + mtx_unlock_spin(&smp_ipi_mtx); else critical_exit(); } @@ -618,7 +618,7 @@ if (smp_started) { if (!(read_eflags() & PSL_I)) panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_rv_mtx); + mtx_lock_spin(&smp_ipi_mtx); } else critical_enter(); /* @@ -642,7 +642,7 @@ sva, eva); } if (smp_started) - mtx_unlock_spin(&smp_rv_mtx); + mtx_unlock_spin(&smp_ipi_mtx); else critical_exit(); } @@ -656,7 +656,7 @@ if (smp_started) { if (!(read_eflags() & PSL_I)) panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_rv_mtx); + mtx_lock_spin(&smp_ipi_mtx); } else critical_enter(); /* @@ -677,7 +677,7 @@ smp_masked_invltlb(pmap->pm_active & other_cpus); } if (smp_started) - mtx_unlock_spin(&smp_rv_mtx); + mtx_unlock_spin(&smp_ipi_mtx); else critical_exit(); } @@ -1301,7 +1301,7 @@ while ((mask = pmap->pm_active) != 0) { spins = 50000000; mask = mask & -mask; /* Find least significant set bit */ - mtx_lock_spin(&smp_rv_mtx); + mtx_lock_spin(&smp_ipi_mtx); #ifdef PAE lazyptd = vtophys(pmap->pm_pdpt); #else @@ -1321,7 +1321,7 @@ break; } } - mtx_unlock_spin(&smp_rv_mtx); + mtx_unlock_spin(&smp_ipi_mtx); if (spins == 0) printf("pmap_lazyfix: spun for 50000000\n"); } Index: kern/subr_smp.c =================================================================== RCS file: /home/ncvs/src/sys/kern/subr_smp.c,v retrieving revision 1.189 diff -u -r1.189 subr_smp.c --- kern/subr_smp.c 23 Aug 2004 21:39:29 -0000 1.189 +++ kern/subr_smp.c 24 Aug 2004 18:31:48 -0000 @@ -109,7 +109,7 @@ * functions trigger at once and cause multiple CPUs to busywait with * interrupts disabled. */ -struct mtx smp_rv_mtx; +struct mtx smp_ipi_mtx; /* * Let the MD SMP code initialize mp_maxid very early if it can. @@ -135,7 +135,7 @@ return; } - mtx_init(&smp_rv_mtx, "smp rendezvous", NULL, MTX_SPIN); + mtx_init(&smp_ipi_mtx, "smp rendezvous", NULL, MTX_SPIN); cpu_mp_start(); printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n", mp_ncpus); @@ -331,7 +331,7 @@ } /* obtain rendezvous lock */ - mtx_lock_spin(&smp_rv_mtx); + mtx_lock_spin(&smp_ipi_mtx); /* set static function pointers */ smp_rv_setup_func = setup_func; @@ -348,7 +348,7 @@ smp_rendezvous_action(); /* release lock */ - mtx_unlock_spin(&smp_rv_mtx); + mtx_unlock_spin(&smp_ipi_mtx); } #else /* !SMP */ Index: sys/smp.h =================================================================== RCS file: /home/ncvs/src/sys/sys/smp.h,v retrieving revision 1.78 diff -u -r1.78 smp.h --- sys/smp.h 23 Aug 2004 21:39:29 -0000 1.78 +++ sys/smp.h 24 Aug 2004 18:31:48 -0000 @@ -95,7 +95,7 @@ int restart_cpus(cpumask_t); int stop_cpus(cpumask_t); void smp_rendezvous_action(void); -extern struct mtx smp_rv_mtx; +extern struct mtx smp_ipi_mtx; #endif /* SMP */ void smp_rendezvous(void (*)(void *), void (*)(void *),