From owner-svn-src-stable-7@FreeBSD.ORG Wed Sep 15 19:28:06 2010 Return-Path: Delivered-To: svn-src-stable-7@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id A76CA10656A3; Wed, 15 Sep 2010 19:28:06 +0000 (UTC) (envelope-from marius@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 954E58FC25; Wed, 15 Sep 2010 19:28:06 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id o8FJS6HC047915; Wed, 15 Sep 2010 19:28:06 GMT (envelope-from marius@svn.freebsd.org) Received: (from marius@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id o8FJS6pi047912; Wed, 15 Sep 2010 19:28:06 GMT (envelope-from marius@svn.freebsd.org) Message-Id: <201009151928.o8FJS6pi047912@svn.freebsd.org> From: Marius Strobl Date: Wed, 15 Sep 2010 19:28:06 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-7@freebsd.org X-SVN-Group: stable-7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r212690 - in stable/7/sys/sparc64: include sparc64 X-BeenThere: svn-src-stable-7@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for only the 7-stable src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 15 Sep 2010 19:28:06 -0000 Author: marius Date: Wed Sep 15 19:28:06 2010 New Revision: 212690 URL: http://svn.freebsd.org/changeset/base/212690 Log: MFC: r211050 (partial) - Introduce a cpu_ipi_single() function pointer in order to send IPIs to single CPUs more efficiently with Cheetah(-class) and Jalapeno CPUs. - Factor out the Jalapeno support from the Cheetah IPI send functions in order to be able to more easily and efficiently implement support for more than 32 target CPUs as well as a workaround for Cheetah+ erratum 25 for the latter. Modified: stable/7/sys/sparc64/include/smp.h stable/7/sys/sparc64/sparc64/mp_machdep.c Directory Properties: stable/7/sys/ (props changed) stable/7/sys/cddl/contrib/opensolaris/ (props changed) stable/7/sys/contrib/dev/acpica/ (props changed) stable/7/sys/contrib/pf/ (props changed) Modified: stable/7/sys/sparc64/include/smp.h ============================================================================== --- stable/7/sys/sparc64/include/smp.h Wed Sep 15 19:27:30 2010 (r212689) +++ stable/7/sys/sparc64/include/smp.h Wed Sep 15 19:28:06 2010 (r212690) @@ -95,6 +95,8 @@ void cpu_mp_shutdown(void); typedef void cpu_ipi_selected_t(u_int, u_long, u_long, u_long); extern cpu_ipi_selected_t *cpu_ipi_selected; +typedef void cpu_ipi_single_t(u_int, u_long, u_long, u_long); +extern cpu_ipi_single_t *cpu_ipi_single; void mp_init(u_int cpu_impl); @@ -229,7 +231,8 @@ ipi_tlb_range_demap(struct pmap *pm, vm_ ita->ita_pmap = pm; ita->ita_start = start; ita->ita_end = end; - cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_range_demap, (u_long)ita); + cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_range_demap, + (u_long)ita); return (&ita->ita_mask); } Modified: stable/7/sys/sparc64/sparc64/mp_machdep.c ============================================================================== --- stable/7/sys/sparc64/sparc64/mp_machdep.c Wed Sep 15 19:27:30 2010 (r212689) +++ stable/7/sys/sparc64/sparc64/mp_machdep.c Wed Sep 15 19:28:06 2010 (r212690) @@ -29,7 +29,7 @@ */ /*- * Copyright (c) 2002 Jake Burkholder. - * Copyright (c) 2007 Marius Strobl + * Copyright (c) 2007 - 2010 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -113,6 +113,7 @@ struct pcb stoppcbs[MAXCPU]; struct mtx ipi_mtx; cpu_ipi_selected_t *cpu_ipi_selected; +cpu_ipi_single_t *cpu_ipi_single; static vm_offset_t mp_tramp; static u_int cpuid_to_mid[MAXCPU]; @@ -124,11 +125,14 @@ static void ap_start(phandle_t node, u_i static void cpu_mp_unleash(void *v); static void foreach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid, u_int cpu_impl)); -static void spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2); static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg); static cpu_ipi_selected_t cheetah_ipi_selected; +static cpu_ipi_single_t cheetah_ipi_single; +static cpu_ipi_selected_t jalapeno_ipi_selected; +static cpu_ipi_single_t jalapeno_ipi_single; static cpu_ipi_selected_t spitfire_ipi_selected; +static cpu_ipi_single_t spitfire_ipi_single; SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); @@ -162,13 +166,18 @@ mp_init(u_int cpu_impl) * cpu_mp_start() wasn't so initialize these here. */ if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || - cpu_impl == CPU_IMPL_ULTRASPARCIIIip) + cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { isjbus = 1; - if (cpu_impl == CPU_IMPL_SPARC64V || - cpu_impl >= CPU_IMPL_ULTRASPARCIII) + cpu_ipi_selected = jalapeno_ipi_selected; + cpu_ipi_single = jalapeno_ipi_single; + } else if (cpu_impl == CPU_IMPL_SPARC64V || + cpu_impl >= CPU_IMPL_ULTRASPARCIII) { cpu_ipi_selected = cheetah_ipi_selected; - else + cpu_ipi_single = cheetah_ipi_single; + } else { cpu_ipi_selected = spitfire_ipi_selected; + cpu_ipi_single = spitfire_ipi_single; + } } static void @@ -179,7 +188,7 @@ foreach_ap(phandle_t node, void (*func)( phandle_t child; u_int cpuid; uint32_t cpu_impl; - + /* There's no need to traverse the whole OFW tree twice. */ if (mp_maxid > 0 && mp_ncpus >= mp_maxid + 1) return; @@ -199,7 +208,7 @@ foreach_ap(phandle_t node, void (*func)( panic("%s: couldn't determine CPU " "implementation", __func__); if (OF_getprop(node, cpu_cpuid_prop(cpu_impl), &cpuid, - sizeof(cpuid)) <= 0) + sizeof(cpuid)) <= 0) panic("%s: couldn't determine CPU module ID", __func__); if (cpuid == PCPU_GET(mid)) @@ -511,24 +520,25 @@ spitfire_ipi_selected(u_int cpus, u_long { u_int cpu; - KASSERT((cpus & (1 << curcpu)) == 0, - ("%s: CPU can't IPI itself", __func__)); while (cpus) { cpu = ffs(cpus) - 1; cpus &= ~(1 << cpu); - spitfire_ipi_send(cpuid_to_mid[cpu], d0, d1, d2); + spitfire_ipi_single(cpu, d0, d1, d2); } } static void -spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2) +spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) { register_t s; u_long ids; + u_int mid; int i; + KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, ("%s: outstanding dispatch", __func__)); + mid = cpuid_to_mid[cpu]; for (i = 0; i < IPI_RETRIES; i++) { s = intr_disable(); stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); @@ -569,6 +579,49 @@ spitfire_ipi_send(u_int mid, u_long d0, } static void +cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) +{ + register_t s; + u_long ids; + u_int mid; + int i; + + KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); + KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & + IDR_CHEETAH_ALL_BUSY) == 0, + ("%s: outstanding dispatch", __func__)); + mid = cpuid_to_mid[cpu]; + for (i = 0; i < IPI_RETRIES; i++) { + s = intr_disable(); + stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); + stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); + stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); + membar(Sync); + stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), + ASI_SDB_INTR_W, 0); + membar(Sync); + while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & + IDR_BUSY) != 0) + ; + intr_restore(s); + if ((ids & (IDR_BUSY | IDR_NACK)) == 0) + return; + /* + * Leave interrupts enabled for a bit before retrying + * in order to avoid deadlocks if the other CPU is also + * trying to send an IPI. + */ + DELAY(2); + } + if (kdb_active != 0 || panicstr != NULL) + printf("%s: couldn't send IPI to module 0x%u\n", + __func__, mid); + else + panic("%s: couldn't send IPI to module 0x%u", + __func__, mid); +} + +static void cheetah_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2) { register_t s; @@ -594,9 +647,8 @@ cheetah_ipi_selected(u_int cpus, u_long bnp = 0; for (cpu = 0; cpu < mp_ncpus; cpu++) { if ((cpus & (1 << cpu)) != 0) { - stxa(AA_INTR_SEND | - (cpuid_to_mid[cpu] << IDC_ITID_SHIFT) | - (isjbus ? 0 : bnp << IDC_BN_SHIFT), + stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << + IDC_ITID_SHIFT) | bnp << IDC_BN_SHIFT, ASI_SDB_INTR_W, 0); membar(Sync); bnp++; @@ -606,14 +658,13 @@ cheetah_ipi_selected(u_int cpus, u_long IDR_CHEETAH_ALL_BUSY) != 0) ; intr_restore(s); - if ((ids & (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) + if ((ids & + (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) return; bnp = 0; for (cpu = 0; cpu < mp_ncpus; cpu++) { if ((cpus & (1 << cpu)) != 0) { - if ((ids & (IDR_NACK << (isjbus ? - (2 * cpuid_to_mid[cpu]) : - (2 * bnp)))) == 0) + if ((ids & (IDR_NACK << (2 * bnp))) == 0) cpus &= ~(1 << cpu); bnp++; } @@ -639,3 +690,104 @@ cheetah_ipi_selected(u_int cpus, u_long panic("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)", __func__, cpus, ids); } + +static void +jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) +{ + register_t s; + u_long ids; + u_int busy, busynack, mid; + int i; + + KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); + KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & + IDR_CHEETAH_ALL_BUSY) == 0, + ("%s: outstanding dispatch", __func__)); + mid = cpuid_to_mid[cpu]; + busy = IDR_BUSY << (2 * mid); + busynack = (IDR_BUSY | IDR_NACK) << (2 * mid); + for (i = 0; i < IPI_RETRIES; i++) { + s = intr_disable(); + stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); + stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); + stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); + membar(Sync); + stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), + ASI_SDB_INTR_W, 0); + membar(Sync); + while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & + busy) != 0) + ; + intr_restore(s); + if ((ids & busynack) == 0) + return; + /* + * Leave interrupts enabled for a bit before retrying + * in order to avoid deadlocks if the other CPU is also + * trying to send an IPI. + */ + DELAY(2); + } + if (kdb_active != 0 || panicstr != NULL) + printf("%s: couldn't send IPI to module 0x%u\n", + __func__, mid); + else + panic("%s: couldn't send IPI to module 0x%u", + __func__, mid); +} + +static void +jalapeno_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2) +{ + register_t s; + u_long ids; + u_int cpu; + int i; + + KASSERT((cpus & (1 << curcpu)) == 0, + ("%s: CPU can't IPI itself", __func__)); + KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & + IDR_CHEETAH_ALL_BUSY) == 0, + ("%s: outstanding dispatch", __func__)); + if (cpus == 0) + return; + ids = 0; + for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { + s = intr_disable(); + stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); + stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); + stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); + membar(Sync); + for (cpu = 0; cpu < mp_ncpus; cpu++) { + if ((cpus & (1 << cpu)) != 0) { + stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << + IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0); + membar(Sync); + } + } + while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & + IDR_CHEETAH_ALL_BUSY) != 0) + ; + intr_restore(s); + if ((ids & + (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) + return; + for (cpu = 0; cpu < mp_ncpus; cpu++) + if ((cpus & (1 << cpu)) != 0) + if ((ids & (IDR_NACK << + (2 * cpuid_to_mid[cpu]))) == 0) + cpus &= ~(1 << cpu); + /* + * Leave interrupts enabled for a bit before retrying + * in order to avoid deadlocks if the other CPUs are + * also trying to send IPIs. + */ + DELAY(2 * mp_ncpus); + } + if (kdb_active != 0 || panicstr != NULL) + printf("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)\n", + __func__, cpus, ids); + else + panic("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)", + __func__, cpus, ids); +}