Date: Tue, 02 Nov 2004 21:11:27 -0500 From: Stephan Uphoff <ups@tree.com> To: Nik Azim Azam <nskyline_r35@yahoo.com> Cc: freebsd-current@freebsd.org Subject: Re: number of CPUs and IPI panic Message-ID: <1099447887.90396.91.camel@palm.tree.com> In-Reply-To: <20041021081115.88930.qmail@web21125.mail.yahoo.com> References: <20041021081115.88930.qmail@web21125.mail.yahoo.com>
next in thread | previous in thread | raw e-mail | index | archive | help
--=-DgZP8J8610bg4bOMq9Wm
Content-Type: text/plain
Content-Transfer-Encoding: 7bit
On Thu, 2004-10-21 at 04:11, Nik Azim Azam wrote:
> the system still panics with the same message with the
> provided patch. please advise me what should i do to
> get more information out of this panic.
>
> thanks,
> nik.
Could you try the attached patch?
Thanks
Stephan
--=-DgZP8J8610bg4bOMq9Wm
Content-Disposition: attachment; filename=ipi2_patch
Content-Type: text/x-patch; name=ipi2_patch; charset=ASCII
Content-Transfer-Encoding: 7bit
Index: i386/apic_vector.s
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/apic_vector.s,v
retrieving revision 1.101
diff -u -r1.101 apic_vector.s
--- i386/apic_vector.s 26 May 2004 07:43:41 -0000 1.101
+++ i386/apic_vector.s 3 Nov 2004 00:04:58 -0000
@@ -286,7 +286,11 @@
movl %eax, %es
movl $KPSEL, %eax
movl %eax, %fs
-
+
+ movl PCPU(CPUMASK), %eax
+ lock
+ or %eax,ipi_ast_pending
+
movl lapic, %edx
movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */
Index: i386/genassym.c
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/genassym.c,v
retrieving revision 1.148
diff -u -r1.148 genassym.c
--- i386/genassym.c 23 May 2004 16:50:55 -0000 1.148
+++ i386/genassym.c 3 Nov 2004 00:10:52 -0000
@@ -197,6 +197,7 @@
ASSYM(PC_TSS_GDT, offsetof(struct pcpu, pc_tss_gdt));
ASSYM(PC_CURRENTLDT, offsetof(struct pcpu, pc_currentldt));
ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
+ASSYM(PC_CPUMASK, offsetof(struct pcpu, pc_cpumask));
ASSYM(PC_CURPMAP, offsetof(struct pcpu, pc_curpmap));
#ifdef DEV_APIC
Index: i386/local_apic.c
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/local_apic.c,v
retrieving revision 1.9
diff -u -r1.9 local_apic.c
--- i386/local_apic.c 14 Jul 2004 18:12:15 -0000 1.9
+++ i386/local_apic.c 4 Oct 2004 02:24:17 -0000
@@ -693,6 +693,62 @@
intr_restore(eflags);
}
+
+static void
+lapic_ipi_wait_and_raw(int delay,register_t icrlo, u_int dest)
+{
+ int x, incr,idle;
+ register_t value, eflags;
+
+
+/* XXX: Need more sanity checking of icrlo? */
+ KASSERT(lapic != NULL, ("%s called too early", __func__));
+ KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0,
+ ("%s: invalid dest field", __func__));
+ KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0,
+ ("%s: reserved bits set in ICR LO register", __func__));
+
+ if (delay == -1) {
+ incr = 0;
+ delay = 1;
+ } else
+ incr = 1;
+
+ /* Set destination in ICR HI register if it is being used. */
+ eflags = intr_disable();
+
+ idle = 0;
+
+ for (x = 0; x < delay; x += incr) {
+ if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE)
+ {
+ idle = 1;
+ break;
+ }
+ intr_restore(eflags);
+ ia32_pause();
+ eflags = intr_disable();
+
+ }
+
+ if (!idle) panic("APIC: Previous IPI is stuck");
+
+ if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) {
+ value = lapic->icr_hi;
+ value &= ~APIC_ID_MASK;
+ value |= dest << APIC_ID_SHIFT;
+ lapic->icr_hi = value;
+ }
+
+ /* Program the contents of the IPI and dispatch it. */
+ value = lapic->icr_lo;
+ value &= APIC_ICRLO_RESV_MASK;
+ value |= icrlo;
+ lapic->icr_lo = value;
+ intr_restore(eflags);
+}
+
+
#define BEFORE_SPIN 1000000
#ifdef DETECT_DEADLOCK
#define AFTER_SPIN 1000
@@ -725,11 +781,8 @@
destfield = dest;
}
- /* Wait for an earlier IPI to finish. */
- if (!lapic_ipi_wait(BEFORE_SPIN))
- panic("APIC: Previous IPI is stuck");
-
- lapic_ipi_raw(icrlo, destfield);
+
+ lapic_ipi_wait_and_raw(BEFORE_SPIN,icrlo, destfield);
#ifdef DETECT_DEADLOCK
/* Wait for IPI to be delivered. */
Index: i386/mp_machdep.c
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/mp_machdep.c,v
retrieving revision 1.240
diff -u -r1.240 mp_machdep.c
--- i386/mp_machdep.c 1 Nov 2004 22:11:27 -0000 1.240
+++ i386/mp_machdep.c 2 Nov 2004 23:13:25 -0000
@@ -212,6 +212,14 @@
static int hlt_logical_cpus;
static struct sysctl_ctx_list logical_cpu_clist;
+
+static int ipi_statclock_pending;
+static int ipi_hardclock_pending;
+int ipi_ast_pending;
+
+
+
+
static void
mem_range_AP_init(void)
{
@@ -1017,12 +1025,24 @@
smp_tlb_addr1 = addr1;
smp_tlb_addr2 = addr2;
atomic_store_rel_int(&smp_tlb_wait, 0);
+
+
+ /* Enable interrupts */
+ /* Thread switching still disabled */
+
+ enable_intr();
+
if (mask == (u_int)-1)
ipi_all_but_self(vector);
else
ipi_selected(mask, vector);
+
while (smp_tlb_wait < ncpu)
ia32_pause();
+
+ /* disable interrupts */
+ disable_intr();
+
}
void
@@ -1104,6 +1124,9 @@
struct thread *td;
CTR0(KTR_SMP, "forwarded_statclock");
+
+ atomic_clear_int(&ipi_statclock_pending,PCPU_GET(cpumask));
+
td = curthread;
td->td_intr_nesting_level++;
if (profprocs != 0)
@@ -1123,7 +1146,7 @@
if (!smp_started || cold || panicstr)
return;
- map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask);
+ map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask|ipi_statclock_pending);
if (map != 0)
ipi_selected(map, IPI_STATCLOCK);
}
@@ -1141,6 +1164,9 @@
struct thread *td;
CTR0(KTR_SMP, "forwarded_hardclock");
+
+ atomic_clear_int(&ipi_hardclock_pending,PCPU_GET(cpumask));
+
td = curthread;
td->td_intr_nesting_level++;
hardclock_process(&frame);
@@ -1157,7 +1183,7 @@
if (!smp_started || cold || panicstr)
return;
- map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask);
+ map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask|ipi_hardclock_pending);
if (map != 0)
ipi_selected(map, IPI_HARDCLOCK);
}
@@ -1170,6 +1196,10 @@
{
int cpu;
+ if (ipi == IPI_AST)
+ cpus &= ~(ipi_ast_pending); /* XXX Needs memory barrier */
+
+
CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
while ((cpu = ffs(cpus)) != 0) {
cpu--;
Index: i386/pmap.c
===================================================================
RCS file: /cvsroot/src/sys/i386/i386/pmap.c,v
retrieving revision 1.514
diff -u -r1.514 pmap.c
--- i386/pmap.c 29 Oct 2004 19:10:46 -0000 1.514
+++ i386/pmap.c 1 Nov 2004 19:29:08 -0000
@@ -1328,12 +1328,20 @@
atomic_store_rel_int((u_int *)&lazymask,
(u_int)&pmap->pm_active);
atomic_store_rel_int(&lazywait, 0);
+ /* Enable interrupts */
+ /* Thread switching still disabled */
+
+ enable_intr();
ipi_selected(mask, IPI_LAZYPMAP);
while (lazywait == 0) {
ia32_pause();
if (--spins == 0)
break;
}
+
+ /* disable interrupts */
+ disable_intr();
+
}
mtx_unlock_spin(&smp_ipi_mtx);
if (spins == 0)
--=-DgZP8J8610bg4bOMq9Wm--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?1099447887.90396.91.camel>