From owner-svn-src-all@FreeBSD.ORG Fri Oct 24 07:58:38 2008 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id D4687106569E; Fri, 24 Oct 2008 07:58:38 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id C28898FC1F; Fri, 24 Oct 2008 07:58:38 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id m9O7wcGJ084171; Fri, 24 Oct 2008 07:58:38 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id m9O7wcvt084168; Fri, 24 Oct 2008 07:58:38 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200810240758.m9O7wcvt084168@svn.freebsd.org> From: Kip Macy Date: Fri, 24 Oct 2008 07:58:38 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r184224 - in head/sys/i386: include xen X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 24 Oct 2008 07:58:39 -0000 Author: kmacy Date: Fri Oct 24 07:58:38 2008 New Revision: 184224 URL: http://svn.freebsd.org/changeset/base/184224 Log: Fix general issues with IPI support Modified: head/sys/i386/include/apicvar.h head/sys/i386/include/smp.h head/sys/i386/xen/mp_machdep.c Modified: head/sys/i386/include/apicvar.h ============================================================================== --- head/sys/i386/include/apicvar.h Fri Oct 24 07:57:48 2008 (r184223) +++ head/sys/i386/include/apicvar.h Fri Oct 24 07:58:38 2008 (r184224) @@ -114,14 +114,14 @@ #define APIC_IPI_INTS (APIC_LOCAL_INTS + 2) #ifdef XEN -#define IPI_RENDEZVOUS (0) /* Inter-CPU rendezvous. */ -#define IPI_INVLTLB (1) /* TLB Shootdown IPIs */ -#define IPI_INVLPG (2) -#define IPI_INVLRNG (3) -#define IPI_INVLCACHE (4) -#define IPI_LAZYPMAP (5) /* Lazy pmap release. */ +#define IPI_RENDEZVOUS (2) /* Inter-CPU rendezvous. */ +#define IPI_INVLTLB (3) /* TLB Shootdown IPIs */ +#define IPI_INVLPG (4) +#define IPI_INVLRNG (5) +#define IPI_INVLCACHE (6) +#define IPI_LAZYPMAP (7) /* Lazy pmap release. */ /* Vector to handle bitmap based IPIs */ -#define IPI_BITMAP_VECTOR (6) +#define IPI_BITMAP_VECTOR (8) #else #define IPI_RENDEZVOUS (APIC_IPI_INTS) /* Inter-CPU rendezvous. */ Modified: head/sys/i386/include/smp.h ============================================================================== --- head/sys/i386/include/smp.h Fri Oct 24 07:57:48 2008 (r184223) +++ head/sys/i386/include/smp.h Fri Oct 24 07:58:38 2008 (r184224) @@ -86,8 +86,8 @@ int ipi_nmi_handler(void); #ifdef XEN void ipi_to_irq_init(void); -#define CALL_FUNCTION_VECTOR 0 -#define RESCHEDULE_VECTOR 1 +#define RESCHEDULE_VECTOR 0 +#define CALL_FUNCTION_VECTOR 1 #define NR_IPIS 2 #endif Modified: head/sys/i386/xen/mp_machdep.c ============================================================================== --- head/sys/i386/xen/mp_machdep.c Fri Oct 24 07:57:48 2008 (r184223) +++ head/sys/i386/xen/mp_machdep.c Fri Oct 24 07:58:38 2008 (r184224) @@ -309,14 +309,12 @@ static void iv_invltlb(uintptr_t a, uintptr_t b) { xen_tlb_flush(); - atomic_add_int(&smp_tlb_wait, 1); } static void iv_invlpg(uintptr_t a, uintptr_t b) { xen_invlpg(a); - atomic_add_int(&smp_tlb_wait, 1); } static void @@ -329,7 +327,6 @@ iv_invlrng(uintptr_t a, uintptr_t b) xen_invlpg(start); start += PAGE_SIZE; } - atomic_add_int(&smp_tlb_wait, 1); } @@ -345,12 +342,36 @@ static void iv_lazypmap(uintptr_t a, uintptr_t b) { pmap_lazyfix_action(); + atomic_add_int(&smp_tlb_wait, 1); } + static void -iv_bitmap_vector(uintptr_t a, uintptr_t b) +iv_noop(uintptr_t a, uintptr_t b) { + atomic_add_int(&smp_tlb_wait, 1); +} +static call_data_func_t *ipi_vectors[IPI_BITMAP_VECTOR] = +{ + iv_noop, + iv_noop, + iv_rendezvous, + iv_invltlb, + iv_invlpg, + iv_invlrng, + iv_invlcache, + iv_lazypmap, +}; + +/* + * Reschedule call back. Nothing to do, + * all the work is done automatically when + * we return from the interrupt. + */ +static int +smp_reschedule_interrupt(void *unused) +{ int cpu = PCPU_GET(cpuid); u_int ipi_bitmap; @@ -369,37 +390,16 @@ iv_bitmap_vector(uintptr_t a, uintptr_t #endif /* Nothing to do for AST */ } -} - - -static call_data_func_t *ipi_vectors[IPI_BITMAP_VECTOR + 1] = -{ iv_rendezvous, - iv_invltlb, - iv_invlpg, - iv_invlrng, - iv_invlcache, - iv_lazypmap, - iv_bitmap_vector -}; - -/* - * Reschedule call back. Nothing to do, - * all the work is done automatically when - * we return from the interrupt. - */ -static int -smp_reschedule_interrupt(void *unused) -{ return (FILTER_HANDLED); } struct _call_data { - call_data_func_t *func; + uint16_t func_id; + uint16_t wait; uintptr_t arg1; uintptr_t arg2; atomic_t started; atomic_t finished; - int wait; }; static struct _call_data *call_data; @@ -407,17 +407,23 @@ static struct _call_data *call_data; static int smp_call_function_interrupt(void *unused) { - call_data_func_t *func = call_data->func; + call_data_func_t *func; uintptr_t arg1 = call_data->arg1; uintptr_t arg2 = call_data->arg2; int wait = call_data->wait; + atomic_t *started = &call_data->started; + atomic_t *finished = &call_data->finished; + if (call_data->func_id > IPI_BITMAP_VECTOR) + panic("invalid function id %u", call_data->func_id); + + func = ipi_vectors[call_data->func_id]; /* * Notify initiating CPU that I've grabbed the data and am * about to execute the function */ mb(); - atomic_inc(&call_data->started); + atomic_inc(started); /* * At this point the info structure may be out of scope unless wait==1 */ @@ -425,8 +431,9 @@ smp_call_function_interrupt(void *unused if (wait) { mb(); - atomic_inc(&call_data->finished); + atomic_inc(finished); } + atomic_add_int(&smp_tlb_wait, 1); return (FILTER_HANDLED); } @@ -967,12 +974,14 @@ smp_tlb_shootdown(u_int vector, vm_offse if (!(read_eflags() & PSL_I)) panic("%s: interrupts disabled", __func__); mtx_lock_spin(&smp_ipi_mtx); + call_data->func_id = vector; call_data->arg1 = addr1; call_data->arg2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); ipi_all_but_self(vector); while (smp_tlb_wait < ncpu) ia32_pause(); + call_data = NULL; mtx_unlock_spin(&smp_ipi_mtx); } @@ -980,6 +989,7 @@ static void smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) { int ncpu, othercpus; + struct _call_data data; othercpus = mp_ncpus - 1; if (mask == (u_int)-1) { @@ -1004,8 +1014,10 @@ smp_targeted_tlb_shootdown(u_int mask, u if (!(read_eflags() & PSL_I)) panic("%s: interrupts disabled", __func__); mtx_lock_spin(&smp_ipi_mtx); - smp_tlb_addr1 = addr1; - smp_tlb_addr2 = addr2; + call_data = &data; + call_data->func_id = vector; + call_data->arg1 = addr1; + call_data->arg2 = addr2; atomic_store_rel_int(&smp_tlb_wait, 0); if (mask == (u_int)-1) ipi_all_but_self(vector); @@ -1013,6 +1025,7 @@ smp_targeted_tlb_shootdown(u_int mask, u ipi_selected(mask, vector); while (smp_tlb_wait < ncpu) ia32_pause(); + call_data = NULL; mtx_unlock_spin(&smp_ipi_mtx); } @@ -1082,20 +1095,17 @@ smp_masked_invlpg_range(u_int mask, vm_o * send an IPI to a set of cpus. */ void -ipi_selected(u_int32_t cpus, u_int ipi) +ipi_selected(uint32_t cpus, u_int ipi) { int cpu; u_int bitmap = 0; u_int old_pending; u_int new_pending; - struct _call_data data; - - call_data = &data; if (IPI_IS_BITMAPED(ipi)) { bitmap = 1 << ipi; ipi = IPI_BITMAP_VECTOR; - } + } #ifdef STOP_NMI if (ipi == IPI_STOP && stop_cpus_with_nmi) { @@ -1117,10 +1127,13 @@ ipi_selected(u_int32_t cpus, u_int ipi) new_pending = old_pending | bitmap; } while (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending)); - if (old_pending) - continue; + if (!old_pending) + ipi_pcpu(cpu, RESCHEDULE_VECTOR); + continue; + } - call_data->func = ipi_vectors[ipi]; + + KASSERT(call_data != NULL, ("call_data not set")); ipi_pcpu(cpu, CALL_FUNCTION_VECTOR); } } @@ -1137,7 +1150,7 @@ ipi_all_but_self(u_int ipi) return; } CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); - ipi_selected((all_cpus & ~(1 << curcpu)), ipi); + ipi_selected(PCPU_GET(other_cpus), ipi); } #ifdef STOP_NMI