Date: Mon, 29 Oct 2001 22:10:01 -0800 (PST) From: Tor.Egge@fast.no To: freebsd-bugs@FreeBSD.org Subject: Re: Re[4]: i386/31535: Can't reboot system: Tyan Thunder K7+ Dual AMD Athlon 1.2 MP / FreeBSD 4.4 STABLE or 4.3 RELEASE Message-ID: <200110300610.f9U6A1e68321@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
The following reply was made to PR i386/31535; it has been noted by GNATS.
From: Tor.Egge@fast.no
To: ilin@rinet.ru
Cc: freebsd-gnats-submit@FreeBSD.org
Subject: Re: Re[4]: i386/31535: Can't reboot system: Tyan Thunder K7+ Dual AMD Athlon 1.2 MP / FreeBSD 4.4 STABLE or 4.3 RELEASE
Date: Tue, 30 Oct 2001 07:02:00 +0100
Here is an updated patch which tries to shutdown the AP in a way
closer to the MP spec. Interrupts are masked in the IOAPICs and an
attempt is made to set the APs in the default HALT state before
shutting down the BSP.
- Tor Egge
Index: sys/i386/include/smp.h
===================================================================
RCS file: /home/ncvs/src/sys/i386/include/smp.h,v
retrieving revision 1.50.2.5
diff -u -r1.50.2.5 smp.h
--- sys/i386/include/smp.h 13 Feb 2001 22:32:45 -0000 1.50.2.5
+++ sys/i386/include/smp.h 30 Oct 2001 05:40:34 -0000
@@ -42,9 +42,11 @@
/* global data in mpboot.s */
extern int bootMP_size;
+extern int shutdownMP_size;
/* functions in mpboot.s */
void bootMP __P((void));
+void shutdownMP __P((void));
/* global data in mplock.s */
extern u_int mp_lock;
@@ -128,6 +130,7 @@
void smp_invltlb __P((void));
int stop_cpus __P((u_int));
int restart_cpus __P((u_int));
+void shutdown_other_cpus __P((int));
#ifdef BETTER_CLOCK
void forward_statclock __P((int pscnt));
void forward_hardclock __P((int pscnt));
@@ -158,6 +161,11 @@
void io_apic_set_id __P((int, int));
int io_apic_get_id __P((int));
int ext_int_setup __P((int, int));
+void shutdown_ioapics __P((void));
+int focus_apic_interrupt __P((int, int));
+int unfocus_apic_interrupt __P((int));
+int focus_apic_interrupts __P((int));
+int unfocus_apic_interrupts __P((void));
#if defined(READY)
void clr_io_apic_mask24 __P((int, u_int32_t));
Index: sys/i386/i386/mpboot.s
===================================================================
RCS file: /home/ncvs/src/sys/i386/i386/mpboot.s,v
retrieving revision 1.13.2.3
diff -u -r1.13.2.3 mpboot.s
--- sys/i386/i386/mpboot.s 7 Sep 2000 01:18:26 -0000 1.13.2.3
+++ sys/i386/i386/mpboot.s 30 Oct 2001 05:40:34 -0000
@@ -302,3 +302,19 @@
.globl _bootMP_size
_bootMP_size:
.long BOOTMP2 - BOOTMP1
+
+SHUTDOWNMP1:
+
+NON_GPROF_ENTRY(shutdownMP)
+ .code16
+ cli
+1:
+ hlt
+ jmp 1b
+
+SHUTDOWNMP2:
+ .global CNAME(shutdownMP_size)
+CNAME(shutdownMP_size):
+ .long SHUTDOWNMP2 - SHUTDOWNMP1
+
+
Index: sys/i386/i386/mp_machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/i386/i386/mp_machdep.c,v
retrieving revision 1.115.2.10
diff -u -r1.115.2.10 mp_machdep.c
--- sys/i386/i386/mp_machdep.c 5 Oct 2001 06:20:17 -0000 1.115.2.10
+++ sys/i386/i386/mp_machdep.c 30 Oct 2001 05:40:34 -0000
@@ -2107,6 +2130,78 @@
}
+void
+shutdown_other_cpus(int callercpu)
+{
+ u_char mpbiosreason;
+ u_long mpbioswarmvec;
+ int cpu;
+ u_int boot_addr;
+ u_long icr_lo, icr_hi;
+ int physical_cpu;
+
+ if (bsp_apic_ready == 0)
+ return;
+
+ boot_addr = boot_address; /* XXX */
+
+ /* save the current value of the warm-start vector */
+ mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
+#ifndef PC98
+ outb(CMOS_REG, BIOS_RESET);
+ mpbiosreason = inb(CMOS_DATA);
+#endif
+
+ /* install the AP shutdown code */
+ install_ap_shutdown_tramp(boot_addr);
+
+ for (cpu = 0; cpu < mp_ncpus; cpu++) {
+
+ if (cpu == callercpu)
+ continue;
+
+ /* setup a vector to our boot code */
+ *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
+ *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
+#ifndef PC98
+ outb(CMOS_REG, BIOS_RESET);
+ outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
+#endif
+ /* get the PHYSICAL APIC ID# */
+ physical_cpu = CPU_TO_ID(cpu);
+
+ /* setup the address for the target AP */
+ icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
+ icr_hi |= (physical_cpu << 24);
+ lapic.icr_hi = icr_hi;
+
+ /* do an INIT IPI: assert RESET */
+ icr_lo = lapic.icr_lo & 0xfff00000;
+ lapic.icr_lo = icr_lo | 0x0000c500;
+
+ /* wait for pending status end */
+ while (lapic.icr_lo & APIC_DELSTAT_MASK)
+ /* spin */ ;
+
+ /* do an INIT IPI: deassert RESET */
+ lapic.icr_lo = icr_lo | 0x00008500;
+
+ /* wait for pending status end */
+ u_sleep(10000); /* wait ~10mS */
+ while (lapic.icr_lo & APIC_DELSTAT_MASK)
+ /* spin */ ;
+
+ u_sleep(100000); /* wait ~100mS */
+ }
+ /* restore the warmstart vector */
+ *(u_long *) WARMBOOT_OFF = mpbioswarmvec;
+#ifndef PC98
+ outb(CMOS_REG, BIOS_RESET);
+ outb(CMOS_DATA, mpbiosreason);
+#endif
+}
+
+
/*
* load the 1st level AP boot code into base memory.
*/
@@ -2168,6 +2263,22 @@
/*
+ * load the level AP shutdown code into base memory.
+ */
+
+static void
+install_ap_shutdown_tramp(u_int boot_addr)
+{
+ int x;
+ int size = *(int *) ((u_long) & shutdownMP_size);
+ u_char *src = (u_char *) ((u_long) shutdownMP);
+ u_char *dst = (u_char *) boot_addr + KERNBASE;
+
+ for (x = 0; x < size; ++x)
+ *dst++ = *src++;
+}
+
+/*
* this function starts the AP (application processor) identified
* by the APIC ID 'physicalCpu'. It does quite a "song and dance"
* to accomplish this. This is necessary because of the nuances
Index: sys/i386/i386/vm_machdep.c
===================================================================
RCS file: /home/ncvs/src/sys/i386/i386/vm_machdep.c,v
retrieving revision 1.132.2.6
diff -u -r1.132.2.6 vm_machdep.c
--- sys/i386/i386/vm_machdep.c 20 Oct 2001 17:37:29 -0000 1.132.2.6
+++ sys/i386/i386/vm_machdep.c 30 Oct 2001 05:40:34 -0000
@@ -406,17 +415,28 @@
u_int saved_mp_lock;
cpu_reset_proxy_active = 1;
+ wbinvd();
while (cpu_reset_proxy_active == 1)
; /* Wait for other cpu to disable interupts */
saved_mp_lock = mp_lock;
mp_lock = 1;
- printf("cpu_reset_proxy: Grabbed mp lock for BSP\n");
cpu_reset_proxy_active = 3;
+ wbinvd();
+ printf("cpu_reset_proxy: Grabbed mp lock for BSP\n");
while (cpu_reset_proxy_active == 3)
; /* Wait for other cpu to enable interrupts */
stop_cpus((1<<cpu_reset_proxyid));
printf("cpu_reset_proxy: Stopped CPU %d\n", cpu_reset_proxyid);
- DELAY(1000000);
+ DELAY(100000);
+ shutdown_ioapics();
+ printf("cpu_reset_proxy: Disabled IOAPICs\n");
+ DELAY(100000);
+ shutdown_other_cpus(cpuid);
+ DELAY(100000);
+ printf("cpu_reset_proxy: Shutdown other CPUs\n");
+ DELAY(100000);
+ printf("cpu_reset_proxy: Resetting BSP\n");
+ DELAY(100000);
cpu_reset_real();
}
#endif
@@ -442,7 +462,16 @@
}
if (cpuid == 0) {
- DELAY(1000000);
+ DELAY(100000);
+ shutdown_ioapics();
+ printf("cpu_reset: Disabled IOAPICs\n");
+ DELAY(100000);
+ shutdown_other_cpus(cpuid);
+ DELAY(100000);
+ printf("cpu_reset: Shutdown other CPUs\n");
+ DELAY(100000);
+ printf("cpu_reset: Resetting BSP\n");
+ DELAY(100000);
cpu_reset_real();
/* NOTREACHED */
} else {
@@ -454,20 +483,43 @@
started_cpus = (1<<0); /* Restart CPU #0 */
cnt = 0;
- while (cpu_reset_proxy_active == 0 && cnt < 10000000)
- cnt++; /* Wait for BSP to announce restart */
+ wbinvd();
+ /* Wait for BSP to announce restart */
+ while (cpu_reset_proxy_active == 0 && cnt < 2000) {
+ cnt++;
+ DELAY(1000);
+ }
if (cpu_reset_proxy_active == 0)
printf("cpu_reset: Failed to restart BSP\n");
__asm __volatile("cli" : : : "memory");
cpu_reset_proxy_active = 2;
+ wbinvd();
cnt = 0;
- while (cpu_reset_proxy_active == 2 && cnt < 10000000)
+ /* Wait for BSP to grab mp lock */
+ while (cpu_reset_proxy_active == 2 && cnt < 2000) {
cnt++; /* Do nothing */
+ DELAY(1000);
+ }
if (cpu_reset_proxy_active == 2) {
printf("cpu_reset: BSP did not grab mp lock\n");
+ DELAY(100000);
+ shutdown_ioapics();
+ printf("cpu#%d: Disabled IOAPICs\n",
+ cpuid);
+ DELAY(100000);
+ shutdown_other_cpus(cpuid);
+ DELAY(100000);
+ printf("cpu#%d: Shutdown other CPUs\n",
+ cpuid);
+ DELAY(100000);
+ printf("cpu#%d: Resetting self (AP)\n",
+ cpuid);
+ DELAY(100000);
cpu_reset_real(); /* XXX: Bogus ? */
}
cpu_reset_proxy_active = 4;
+ wbinvd();
+ /* Enable interrupts to allow BSP stopping this AP */
__asm __volatile("sti" : : : "memory");
while (1);
/* NOTREACHED */
Index: sys/i386/i386/mpapic.c
===================================================================
RCS file: /home/ncvs/src/sys/i386/i386/mpapic.c,v
retrieving revision 1.37.2.5
diff -u -r1.37.2.5 mpapic.c
--- sys/i386/i386/mpapic.c 13 Feb 2001 22:32:44 -0000 1.37.2.5
+++ sys/i386/i386/mpapic.c 30 Oct 2001 05:40:34 -0000
@@ -283,6 +294,149 @@
}
/* return GOOD status */
+ return 0;
+}
+
+
+/*
+ * Block all interrupts on all IOAPICs in preparation for shutdown.
+ */
+
+void
+shutdown_ioapics(void)
+{
+ int apic;
+ int maxpin;
+ int pin;
+ u_char select; /* the select register is 8 bits */
+ u_int eflags;
+
+ for (apic = 0; apic < mp_napics; ++apic) {
+ maxpin = REDIRCNT_IOAPIC(apic); /* pins in APIC */
+ printf("Disabling interrupt on %d pins in IOAPIC #%d ...",
+ maxpin, apic);
+ for (pin = 0; pin < maxpin; ++pin) {
+ select = pin * 2 + IOAPIC_REDTBL0; /* register */
+ eflags = read_eflags();
+ __asm __volatile("cli" : : : "memory");
+ s_lock(&imen_lock);
+ io_apic_write(apic, select,
+ (io_apic_read(apic, select) &
+ ~IOART_INTMASK)
+ |IOART_INTMSET);
+ s_unlock(&imen_lock);
+ write_eflags(eflags);
+ }
+ printf(" Done\n");
+ }
+}
+
+
+/* Bind an interrupt to a CPU */
+
+int
+focus_apic_interrupt(int irq, int cpu)
+{
+ int apic;
+ int pin;
+ int redtbl0;
+ int redtbl1;
+ u_char select;
+ u_int eflags;
+
+ if (irq < 0 || irq >= APIC_INTMAPSIZE)
+ return -1;
+
+ apic = int_to_apicintpin[irq].ioapic;
+ pin = int_to_apicintpin[irq].int_pin;
+
+ if (apic == -1)
+ return -1;
+
+ select = pin * 2 + IOAPIC_REDTBL0;
+
+ eflags = read_eflags();
+ __asm __volatile("cli" : : : "memory");
+ s_lock(&imen_lock);
+ redtbl0 = io_apic_read(apic, select);
+ redtbl1 = io_apic_read(apic, select + 1);
+
+ redtbl0 &= ~IOART_DELMOD;
+ redtbl0 |= IOART_DELFIXED;
+
+ redtbl1 &= ~IOART_DEST;
+ redtbl1 |= (CPU_TO_ID(cpu) << 24);
+
+ io_apic_write(apic, select, redtbl0);
+ io_apic_write(apic, select + 1, redtbl1);
+ s_unlock(&imen_lock);
+ write_eflags(eflags);
+
+ return 0;
+}
+
+
+/* Allow an interrupt to be delivered to any CPU */
+
+int
+unfocus_apic_interrupt(int irq)
+{
+ int apic;
+ int pin;
+ int redtbl0;
+ int redtbl1;
+ u_char select;
+ u_int eflags;
+
+ if (irq < 0 || irq >= APIC_INTMAPSIZE)
+ return -1;
+
+ apic = int_to_apicintpin[irq].ioapic;
+ pin = int_to_apicintpin[irq].int_pin;
+
+ if (apic == -1)
+ return -1;
+
+ select = pin * 2 + IOAPIC_REDTBL0;
+
+ eflags = read_eflags();
+ __asm __volatile("cli" : : : "memory");
+ s_lock(&imen_lock);
+ redtbl0 = io_apic_read(apic, select);
+ redtbl1 = io_apic_read(apic, select + 1);
+
+ redtbl0 &= ~IOART_DELMOD;
+ redtbl0 |= IOART_DELLOPRI;
+
+ redtbl1 |= IOART_DEST;
+
+ io_apic_write(apic, select, redtbl0);
+ io_apic_write(apic, select + 1, redtbl1);
+ s_unlock(&imen_lock);
+ write_eflags(eflags);
+
+ return 0;
+}
+
+
+int
+focus_apic_interrupts(int cpu)
+{
+ int irq;
+
+ for (irq = 0; irq < APIC_INTMAPSIZE; irq++)
+ focus_apic_interrupt(irq, cpu);
+ return 0;
+}
+
+
+int
+unfocus_apic_interrupts(void)
+{
+ int irq;
+
+ for (irq = 0; irq < APIC_INTMAPSIZE; irq++)
+ unfocus_apic_interrupt(irq);
return 0;
}
#undef DEFAULT_ISA_FLAGS
To Unsubscribe: send mail to majordomo@FreeBSD.org
with "unsubscribe freebsd-bugs" in the body of the message
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200110300610.f9U6A1e68321>