Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 02 Nov 2004 21:11:27 -0500
From:      Stephan Uphoff <ups@tree.com>
To:        Nik Azim Azam <nskyline_r35@yahoo.com>
Cc:        freebsd-current@freebsd.org
Subject:   Re: number of CPUs and IPI panic
Message-ID:  <1099447887.90396.91.camel@palm.tree.com>
In-Reply-To: <20041021081115.88930.qmail@web21125.mail.yahoo.com>
References:  <20041021081115.88930.qmail@web21125.mail.yahoo.com>

next in thread | previous in thread | raw e-mail | index | archive | help

--=-DgZP8J8610bg4bOMq9Wm
Content-Type: text/plain
Content-Transfer-Encoding: 7bit

On Thu, 2004-10-21 at 04:11, Nik Azim Azam wrote:
> the system still panics with the same message with the
> provided patch. please advise me what should i do to
> get more information out of this panic.
> 
> thanks,
> nik.

Could you try the attached patch?

Thanks
	Stephan

--=-DgZP8J8610bg4bOMq9Wm
Content-Disposition: attachment; filename=ipi2_patch
Content-Type: text/x-patch; name=ipi2_patch; charset=ASCII
Content-Transfer-Encoding: 7bit

Index: i386/apic_vector.s
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/apic_vector.s,v
retrieving revision 1.101
diff -u -r1.101 apic_vector.s
--- i386/apic_vector.s	26 May 2004 07:43:41 -0000	1.101
+++ i386/apic_vector.s	3 Nov 2004 00:04:58 -0000
@@ -286,7 +286,11 @@
 	movl	%eax, %es
 	movl	$KPSEL, %eax
 	movl	%eax, %fs
-
+		
+	movl	PCPU(CPUMASK), %eax
+	lock
+	or	%eax,ipi_ast_pending  
+	
 	movl	lapic, %edx
 	movl	$0, LA_EOI(%edx)	/* End Of Interrupt to APIC */
 
Index: i386/genassym.c
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/genassym.c,v
retrieving revision 1.148
diff -u -r1.148 genassym.c
--- i386/genassym.c	23 May 2004 16:50:55 -0000	1.148
+++ i386/genassym.c	3 Nov 2004 00:10:52 -0000
@@ -197,6 +197,7 @@
 ASSYM(PC_TSS_GDT, offsetof(struct pcpu, pc_tss_gdt));
 ASSYM(PC_CURRENTLDT, offsetof(struct pcpu, pc_currentldt));
 ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
+ASSYM(PC_CPUMASK, offsetof(struct pcpu, pc_cpumask));
 ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap));
 
 #ifdef DEV_APIC
Index: i386/local_apic.c
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/local_apic.c,v
retrieving revision 1.9
diff -u -r1.9 local_apic.c
--- i386/local_apic.c	14 Jul 2004 18:12:15 -0000	1.9
+++ i386/local_apic.c	4 Oct 2004 02:24:17 -0000
@@ -693,6 +693,62 @@
 	intr_restore(eflags);
 }
 
+
+static void
+lapic_ipi_wait_and_raw(int delay,register_t icrlo, u_int dest)
+{
+    int x, incr,idle;
+    register_t value, eflags;
+
+    
+/* XXX: Need more sanity checking of icrlo? */
+    KASSERT(lapic != NULL, ("%s called too early", __func__));
+	KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
+	    ("%s: invalid dest field", __func__));
+	KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
+	    ("%s: reserved bits set in ICR LO register", __func__));
+
+	if (delay == -1) {
+		incr = 0;
+		delay = 1;
+	} else
+		incr = 1;
+
+	/* Set destination in ICR HI register if it is being used. */
+	eflags = intr_disable();
+
+	idle = 0;
+
+	for (x = 0; x < delay; x += incr) {
+		if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE)
+		{
+		    idle = 1;
+		    break;
+		}
+		intr_restore(eflags);
+		ia32_pause();
+		eflags = intr_disable();
+
+	}
+	
+	if (!idle)  panic("APIC: Previous IPI is stuck");
+
+	if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
+		value = lapic->icr_hi;
+		value &= ~APIC_ID_MASK;
+		value |= dest << APIC_ID_SHIFT;
+		lapic->icr_hi = value;
+	}
+
+	/* Program the contents of the IPI and dispatch it. */
+	value = lapic->icr_lo;
+	value &= APIC_ICRLO_RESV_MASK;
+	value |= icrlo;
+	lapic->icr_lo = value;
+	intr_restore(eflags);
+}
+
+
 #define	BEFORE_SPIN	1000000
 #ifdef DETECT_DEADLOCK
 #define	AFTER_SPIN	1000
@@ -725,11 +781,8 @@
 		destfield = dest;
 	}
 
-	/* Wait for an earlier IPI to finish. */
-	if (!lapic_ipi_wait(BEFORE_SPIN))
-		panic("APIC: Previous IPI is stuck");
-
-	lapic_ipi_raw(icrlo, destfield);
+	
+	lapic_ipi_wait_and_raw(BEFORE_SPIN,icrlo, destfield);
 
 #ifdef DETECT_DEADLOCK
 	/* Wait for IPI to be delivered. */
Index: i386/mp_machdep.c
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/mp_machdep.c,v
retrieving revision 1.240
diff -u -r1.240 mp_machdep.c
--- i386/mp_machdep.c	1 Nov 2004 22:11:27 -0000	1.240
+++ i386/mp_machdep.c	2 Nov 2004 23:13:25 -0000
@@ -212,6 +212,14 @@
 static int	hlt_logical_cpus;
 static struct	sysctl_ctx_list logical_cpu_clist;
 
+
+static int ipi_statclock_pending;
+static int ipi_hardclock_pending;
+int ipi_ast_pending;
+
+
+
+
 static void
 mem_range_AP_init(void)
 {
@@ -1017,12 +1025,24 @@
 	smp_tlb_addr1 = addr1;
 	smp_tlb_addr2 = addr2;
 	atomic_store_rel_int(&smp_tlb_wait, 0);
+
+
+	/* Enable interrupts */
+	/* Thread switching still disabled */
+
+	enable_intr();
+
 	if (mask == (u_int)-1)
 		ipi_all_but_self(vector);
 	else
 		ipi_selected(mask, vector);
+
 	while (smp_tlb_wait < ncpu)
 		ia32_pause();
+
+	/* disable interrupts */
+	disable_intr();
+
 }
 
 void
@@ -1104,6 +1124,9 @@
 	struct thread *td;
 
 	CTR0(KTR_SMP, "forwarded_statclock");
+
+	atomic_clear_int(&ipi_statclock_pending,PCPU_GET(cpumask));	
+
 	td = curthread;
 	td->td_intr_nesting_level++;
 	if (profprocs != 0)
@@ -1123,7 +1146,7 @@
 	if (!smp_started || cold || panicstr)
 		return;
 
-	map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask);
+	map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask|ipi_statclock_pending);
 	if (map != 0)
 		ipi_selected(map, IPI_STATCLOCK);
 }
@@ -1141,6 +1164,9 @@
 	struct thread *td;
 
 	CTR0(KTR_SMP, "forwarded_hardclock");
+
+	atomic_clear_int(&ipi_hardclock_pending,PCPU_GET(cpumask));	
+
 	td = curthread;
 	td->td_intr_nesting_level++;
 	hardclock_process(&frame);
@@ -1157,7 +1183,7 @@
 	if (!smp_started || cold || panicstr)
 		return;
 
-	map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask);
+	map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask|ipi_hardclock_pending);
 	if (map != 0)
 		ipi_selected(map, IPI_HARDCLOCK);
 }
@@ -1170,6 +1196,10 @@
 {
 	int cpu;
 
+	if (ipi == IPI_AST)
+		cpus &= ~(ipi_ast_pending); /* XXX  Needs memory barrier */
+
+  
 	CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
 	while ((cpu = ffs(cpus)) != 0) {
 		cpu--;
Index: i386/pmap.c
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/pmap.c,v
retrieving revision 1.514
diff -u -r1.514 pmap.c
--- i386/pmap.c	29 Oct 2004 19:10:46 -0000	1.514
+++ i386/pmap.c	1 Nov 2004 19:29:08 -0000
@@ -1328,12 +1328,20 @@
 			atomic_store_rel_int((u_int *)&lazymask,
 			    (u_int)&pmap->pm_active);
 			atomic_store_rel_int(&lazywait, 0);
+			/* Enable interrupts */
+			/* Thread switching still disabled */
+
+			enable_intr();
 			ipi_selected(mask, IPI_LAZYPMAP);
 			while (lazywait == 0) {
 				ia32_pause();
 				if (--spins == 0)
 					break;
 			}
+
+			/* disable interrupts */
+			disable_intr();
+
 		}
 		mtx_unlock_spin(&smp_ipi_mtx);
 		if (spins == 0)

--=-DgZP8J8610bg4bOMq9Wm--



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?1099447887.90396.91.camel>