Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 24 Oct 2008 07:58:38 +0000 (UTC)
From:      Kip Macy <kmacy@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r184224 - in head/sys/i386: include xen
Message-ID:  <200810240758.m9O7wcvt084168@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kmacy
Date: Fri Oct 24 07:58:38 2008
New Revision: 184224
URL: http://svn.freebsd.org/changeset/base/184224

Log:
  Fix general issues with IPI support

Modified:
  head/sys/i386/include/apicvar.h
  head/sys/i386/include/smp.h
  head/sys/i386/xen/mp_machdep.c

Modified: head/sys/i386/include/apicvar.h
==============================================================================
--- head/sys/i386/include/apicvar.h	Fri Oct 24 07:57:48 2008	(r184223)
+++ head/sys/i386/include/apicvar.h	Fri Oct 24 07:58:38 2008	(r184224)
@@ -114,14 +114,14 @@
 
 #define	APIC_IPI_INTS	(APIC_LOCAL_INTS + 2)
 #ifdef XEN
-#define	IPI_RENDEZVOUS		(0)	/* Inter-CPU rendezvous. */
-#define	IPI_INVLTLB		(1)	/* TLB Shootdown IPIs */
-#define	IPI_INVLPG		(2)
-#define	IPI_INVLRNG		(3)
-#define	IPI_INVLCACHE		(4)
-#define	IPI_LAZYPMAP		(5)	/* Lazy pmap release. */
+#define	IPI_RENDEZVOUS		(2)	/* Inter-CPU rendezvous. */
+#define	IPI_INVLTLB		(3)	/* TLB Shootdown IPIs */
+#define	IPI_INVLPG		(4)
+#define	IPI_INVLRNG		(5)
+#define	IPI_INVLCACHE		(6)
+#define	IPI_LAZYPMAP		(7)	/* Lazy pmap release. */
 /* Vector to handle bitmap based IPIs */
-#define	IPI_BITMAP_VECTOR	(6)
+#define	IPI_BITMAP_VECTOR	(8)
 
 #else
 #define	IPI_RENDEZVOUS	(APIC_IPI_INTS)		/* Inter-CPU rendezvous. */

Modified: head/sys/i386/include/smp.h
==============================================================================
--- head/sys/i386/include/smp.h	Fri Oct 24 07:57:48 2008	(r184223)
+++ head/sys/i386/include/smp.h	Fri Oct 24 07:58:38 2008	(r184224)
@@ -86,8 +86,8 @@ int	ipi_nmi_handler(void);
 #ifdef XEN
 void ipi_to_irq_init(void);
 
-#define CALL_FUNCTION_VECTOR	0
-#define RESCHEDULE_VECTOR	1
+#define RESCHEDULE_VECTOR	0
+#define CALL_FUNCTION_VECTOR	1
 #define NR_IPIS			2
 
 #endif

Modified: head/sys/i386/xen/mp_machdep.c
==============================================================================
--- head/sys/i386/xen/mp_machdep.c	Fri Oct 24 07:57:48 2008	(r184223)
+++ head/sys/i386/xen/mp_machdep.c	Fri Oct 24 07:58:38 2008	(r184224)
@@ -309,14 +309,12 @@ static void
 iv_invltlb(uintptr_t a, uintptr_t b)
 {
 	xen_tlb_flush();
-	atomic_add_int(&smp_tlb_wait, 1);
 }
 
 static void
 iv_invlpg(uintptr_t a, uintptr_t b)
 {
 	xen_invlpg(a);
-	atomic_add_int(&smp_tlb_wait, 1);
 }
 
 static void
@@ -329,7 +327,6 @@ iv_invlrng(uintptr_t a, uintptr_t b)
 		xen_invlpg(start);
 		start += PAGE_SIZE;
 	}
-	atomic_add_int(&smp_tlb_wait, 1);
 }
 
 
@@ -345,12 +342,36 @@ static void
 iv_lazypmap(uintptr_t a, uintptr_t b)
 {
 	pmap_lazyfix_action();
+	atomic_add_int(&smp_tlb_wait, 1);
 }
 
+
 static void
-iv_bitmap_vector(uintptr_t a, uintptr_t b)
+iv_noop(uintptr_t a, uintptr_t b)
 {
+	atomic_add_int(&smp_tlb_wait, 1);
+}
 
+static call_data_func_t *ipi_vectors[IPI_BITMAP_VECTOR] = 
+{
+  iv_noop,
+  iv_noop,
+  iv_rendezvous,
+  iv_invltlb,
+  iv_invlpg,
+  iv_invlrng,
+  iv_invlcache,
+  iv_lazypmap,
+};
+
+/*
+ * Reschedule call back. Nothing to do,
+ * all the work is done automatically when
+ * we return from the interrupt.
+ */
+static int
+smp_reschedule_interrupt(void *unused)
+{
 	int cpu = PCPU_GET(cpuid);
 	u_int ipi_bitmap;
 
@@ -369,37 +390,16 @@ iv_bitmap_vector(uintptr_t a, uintptr_t 
 #endif
 		/* Nothing to do for AST */
 	}	
-}
-
-
-static call_data_func_t *ipi_vectors[IPI_BITMAP_VECTOR + 1] = 
-{ iv_rendezvous,
-  iv_invltlb,
-  iv_invlpg,
-  iv_invlrng,
-  iv_invlcache,
-  iv_lazypmap,
-  iv_bitmap_vector
-};
-
-/*
- * Reschedule call back. Nothing to do,
- * all the work is done automatically when
- * we return from the interrupt.
- */
-static int
-smp_reschedule_interrupt(void *unused)
-{
 	return (FILTER_HANDLED);
 }
 
 struct _call_data {
-	call_data_func_t *func;
+	uint16_t func_id;
+	uint16_t wait;
 	uintptr_t arg1;
 	uintptr_t arg2;
 	atomic_t started;
 	atomic_t finished;
-	int wait;
 };
 
 static struct _call_data *call_data;
@@ -407,17 +407,23 @@ static struct _call_data *call_data;
 static int
 smp_call_function_interrupt(void *unused)
 {	
-	call_data_func_t *func = call_data->func;
+	call_data_func_t *func;
 	uintptr_t arg1 = call_data->arg1;
 	uintptr_t arg2 = call_data->arg2;
 	int wait = call_data->wait;
+	atomic_t *started = &call_data->started;
+	atomic_t *finished = &call_data->finished;
 
+	if (call_data->func_id > IPI_BITMAP_VECTOR)
+		panic("invalid function id %u", call_data->func_id);
+	
+	func = ipi_vectors[call_data->func_id];
 	/*
 	 * Notify initiating CPU that I've grabbed the data and am
 	 * about to execute the function
 	 */
 	mb();
-	atomic_inc(&call_data->started);
+	atomic_inc(started);
 	/*
 	 * At this point the info structure may be out of scope unless wait==1
 	 */
@@ -425,8 +431,9 @@ smp_call_function_interrupt(void *unused
 
 	if (wait) {
 		mb();
-		atomic_inc(&call_data->finished);
+		atomic_inc(finished);
 	}
+	atomic_add_int(&smp_tlb_wait, 1);
 	return (FILTER_HANDLED);
 }
 
@@ -967,12 +974,14 @@ smp_tlb_shootdown(u_int vector, vm_offse
 	if (!(read_eflags() & PSL_I))
 		panic("%s: interrupts disabled", __func__);
 	mtx_lock_spin(&smp_ipi_mtx);
+	call_data->func_id = vector;
 	call_data->arg1 = addr1;
 	call_data->arg2 = addr2;
 	atomic_store_rel_int(&smp_tlb_wait, 0);
 	ipi_all_but_self(vector);
 	while (smp_tlb_wait < ncpu)
 		ia32_pause();
+	call_data = NULL;
 	mtx_unlock_spin(&smp_ipi_mtx);
 }
 
@@ -980,6 +989,7 @@ static void
 smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2)
 {
 	int ncpu, othercpus;
+	struct _call_data data;
 
 	othercpus = mp_ncpus - 1;
 	if (mask == (u_int)-1) {
@@ -1004,8 +1014,10 @@ smp_targeted_tlb_shootdown(u_int mask, u
 	if (!(read_eflags() & PSL_I))
 		panic("%s: interrupts disabled", __func__);
 	mtx_lock_spin(&smp_ipi_mtx);
-	smp_tlb_addr1 = addr1;
-	smp_tlb_addr2 = addr2;
+	call_data = &data;		
+	call_data->func_id = vector;
+	call_data->arg1 = addr1;
+	call_data->arg2 = addr2;
 	atomic_store_rel_int(&smp_tlb_wait, 0);
 	if (mask == (u_int)-1)
 		ipi_all_but_self(vector);
@@ -1013,6 +1025,7 @@ smp_targeted_tlb_shootdown(u_int mask, u
 		ipi_selected(mask, vector);
 	while (smp_tlb_wait < ncpu)
 		ia32_pause();
+	call_data = NULL;
 	mtx_unlock_spin(&smp_ipi_mtx);
 }
 
@@ -1082,20 +1095,17 @@ smp_masked_invlpg_range(u_int mask, vm_o
  * send an IPI to a set of cpus.
  */
 void
-ipi_selected(u_int32_t cpus, u_int ipi)
+ipi_selected(uint32_t cpus, u_int ipi)
 {
 	int cpu;
 	u_int bitmap = 0;
 	u_int old_pending;
 	u_int new_pending;
-	struct _call_data data;
-
-	call_data = &data;
 	
 	if (IPI_IS_BITMAPED(ipi)) { 
 		bitmap = 1 << ipi;
 		ipi = IPI_BITMAP_VECTOR;
-	}
+	} 
 
 #ifdef STOP_NMI
 	if (ipi == IPI_STOP && stop_cpus_with_nmi) {
@@ -1117,10 +1127,13 @@ ipi_selected(u_int32_t cpus, u_int ipi)
 				new_pending = old_pending | bitmap;
 			} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],old_pending, new_pending));	
 
-			if (old_pending)
-				continue;
+			if (!old_pending)
+				ipi_pcpu(cpu, RESCHEDULE_VECTOR);
+			continue;
+			
 		}
-		call_data->func = ipi_vectors[ipi];
+		
+		KASSERT(call_data != NULL, ("call_data not set"));
 		ipi_pcpu(cpu, CALL_FUNCTION_VECTOR);
 	}
 }
@@ -1137,7 +1150,7 @@ ipi_all_but_self(u_int ipi)
 		return;
 	}
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
-	ipi_selected((all_cpus & ~(1 << curcpu)), ipi);
+	ipi_selected(PCPU_GET(other_cpus), ipi);
 }
 
 #ifdef STOP_NMI



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200810240758.m9O7wcvt084168>