From owner-freebsd-bugs Fri Jan 21 11:10:17 2000 Delivered-To: freebsd-bugs@freebsd.org Received: from freefall.freebsd.org (freefall.FreeBSD.ORG [204.216.27.21]) by hub.freebsd.org (Postfix) with ESMTP id 03A5815530 for ; Fri, 21 Jan 2000 11:10:03 -0800 (PST) (envelope-from gnats@FreeBSD.org) Received: (from gnats@localhost) by freefall.freebsd.org (8.9.3/8.9.2) id LAA68379; Fri, 21 Jan 2000 11:10:02 -0800 (PST) (envelope-from gnats@FreeBSD.org) Received: from fast.cs.utah.edu (fast.cs.utah.edu [155.99.212.1]) by hub.freebsd.org (Postfix) with ESMTP id 8CF6F154B9 for ; Fri, 21 Jan 2000 11:05:20 -0800 (PST) (envelope-from sclawson@fast.cs.utah.edu) Received: (from sclawson@localhost) by fast.cs.utah.edu (8.9.1/8.9.1) id MAA13074 for FreeBSD-gnats-submit@freebsd.org; Fri, 21 Jan 2000 12:05:13 -0700 (MST) Message-Id: <200001211905.MAA13074@fast.cs.utah.edu> Date: Fri, 21 Jan 2000 12:05:13 -0700 (MST) From: stephen clawson To: FreeBSD-gnats-submit@freebsd.org Subject: i386/16269: smp dosen't work with >2 cpus on AMI Goliath II Sender: owner-freebsd-bugs@FreeBSD.ORG Precedence: bulk X-Loop: FreeBSD.org >Number: 16269 >Category: i386 >Synopsis: smp dosen't work with >2 cpus on AMI Goliath II >Confidential: no >Severity: serious >Priority: medium >Responsible: freebsd-bugs >State: open >Quarter: >Keywords: >Date-Required: >Class: sw-bug >Submitter-Id: current-users >Arrival-Date: Fri Jan 21 11:10:02 PST 2000 >Closed-Date: >Last-Modified: >Originator: Stephen Clawson >Release: FreeBSD 4.0-CURRENT i386 >Organization: University of Utah CSL >Environment: A AMI Goliath II SMP system w/4 200MHz Pentium Pro's. >Description: For some reason the BIOS on the Goliath II sets the IO APIC's id to 2 in the mptable, while also setting cpu 2's id to 2. This causes SMP initialization to blow up. Because ID_TO_CPU() and ID_TO_IO() both refrence the same array, when we reach the IO APIC's mptable entry, it overwrites the mapping for cpu 2, causing panics like this: [other kernel messages deleted] SMP: AP CPU #3 Launched! SMP: AP CPU #1 Launched! SMP: AP CPU #2 Launched! SMP: cpuid = 2 SMP: apic_id = 0 PTD[MPPTDI] = 0x31d023 panic: cpuid mismatch! boom!! mp_lock = 02000001; cpuid = 2; lapic.id = 02000000 >How-To-Repeat: Boot an SMP kernel with NCPU > 2 on a Goliath II system. =) >Fix: Ideally there'd be a new BIOS for this board that wasn't so braindead. However, I spent some time trying to find an updated BIOS for it and had no luck. It seems that AMI dosen't seem to care much about it these days. =( At the very least there should be a check to see if we've already got a cpu at that apic id. ie. add this to mp_machdep.c:io_apic(): if (ID_TO_CPU(io_apic_id) != -1) panic("IO APIC #%d id conflicts with cpu #%d's local APIC id!", apic, ID_TO_CPU(io_apic_id)); However, the last paragraph of section 3.6.6 of v1.4 of the MP spec does clearly state that it's up to the OS to assign a unique id to io apic's in case there's a conflict, so this fix is potentially useful anyway, even if in this case AMI was just being lazy/stupid. =) So, the patch: It checks to see if there's a conflict between an io apic's id and a cpu's id. If there's a conflict, it assigns a new id starting with the lowest possible id, as per section 3.6.6. In the process it keeps track of the mapping from the original mptable id and the new id, since we'll also have to remap the destination apic id of any io interrupt mptable entries. Index: mp_machdep.c =================================================================== RCS file: /n/marker/usr/lsrc/FreeBSD/CVS/src/sys/i386/i386/mp_machdep.c,v retrieving revision 1.115 diff -c -r1.115 mp_machdep.c *** mp_machdep.c 2000/01/13 09:08:59 1.115 --- mp_machdep.c 2000/01/21 03:10:25 *************** *** 729,736 **** static int processor_entry __P((proc_entry_ptr entry, int cpu)); static int bus_entry __P((bus_entry_ptr entry, int bus)); ! static int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); ! static int int_entry __P((int_entry_ptr entry, int intr)); static int lookup_bus_type __P((char *name)); --- 729,738 ---- static int processor_entry __P((proc_entry_ptr entry, int cpu)); static int bus_entry __P((bus_entry_ptr entry, int bus)); ! static int io_apic_entry __P((io_apic_entry_ptr entry, int apic, ! int io_apic_id_map[])); ! static int int_entry __P((int_entry_ptr entry, int intr, ! int io_apic_id_map[])); static int lookup_bus_type __P((char *name)); *************** *** 883,888 **** --- 885,891 ---- int count; int type; int apic, bus, cpu, intr; + int io_apic_id_map[NAPICID]; POSTCODE(MPTABLE_PASS2_POST); *************** *** 891,896 **** --- 894,900 ---- ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ + io_apic_id_map[x] = -1; /* mptable apic id to apic id table */ } /* clear bus data table */ *************** *** 934,944 **** ++bus; break; case 2: ! if (io_apic_entry(position, apic)) ++apic; break; case 3: ! if (int_entry(position, intr)) ++intr; break; case 4: --- 938,948 ---- ++bus; break; case 2: ! if (io_apic_entry(position, apic, io_apic_id_map)) ++apic; break; case 3: ! if (int_entry(position, intr, io_apic_id_map)) ++intr; break; case 4: *************** *** 1012,1017 **** --- 1016,1038 ---- } } + #ifdef APIC_IO + static void + set_io_apic_id(int apic, int io_apic_id) + { + u_int32_t ux; + + /* reprogram the io-apic to it's new id. */ + ux = io_apic_read(apic, IOAPIC_ID); /* get current contents */ + ux &= ~APIC_ID_MASK; /* clear the ID field */ + ux |= (io_apic_id << 24); /* set it to io_apic_id */ + io_apic_write(apic, IOAPIC_ID, ux); /* write new value */ + ux = io_apic_read(apic, IOAPIC_ID); /* re-read && test */ + if ((ux & APIC_ID_MASK) != (io_apic_id << 24)) + panic("can't control IO APIC ID, reg: 0x%08x", ux); + } + #endif + /* * parse an Intel MP specification table */ *************** *** 1202,1214 **** static int ! io_apic_entry(io_apic_entry_ptr entry, int apic) { if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) return 0; ! IO_TO_ID(apic) = entry->apic_id; ! ID_TO_IO(entry->apic_id) = apic; return 1; } --- 1223,1262 ---- static int ! io_apic_entry(io_apic_entry_ptr entry, int apic, int io_apic_id_map[]) { + int io_apic_id = entry->apic_id; + if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) return 0; ! /* ! * section 3.6.6 of the mp spec v1.4 specifies that the OS ! * must check the uniqueness of the IO APIC id and assign a ! * unique id if there's a conflict. ! */ ! if (ID_TO_CPU(io_apic_id) != -1) { ! /* ! * section 3.6.6 also specifies that we must start ! * with the lowest id possible. ! */ ! io_apic_id = 0; ! while ((io_apic_id < NAPICID) && ! (ID_TO_CPU(io_apic_id) != -1)) ! io_apic_id++; ! ! if (io_apic_id == NAPICID) ! panic("can't remap IOAPIC id: all apic id's used"); ! ! printf("Remapping IO APIC #%d from id %d to id %d\n", ! apic, entry->apic_id, io_apic_id); ! ! set_io_apic_id(apic, io_apic_id); ! } ! ! io_apic_id_map[entry->apic_id] = io_apic_id; ! IO_TO_ID(apic) = io_apic_id; ! ID_TO_IO(io_apic_id) = apic; return 1; } *************** *** 1228,1234 **** static int ! int_entry(int_entry_ptr entry, int intr) { int apic; --- 1276,1282 ---- static int ! int_entry(int_entry_ptr entry, int intr, int io_apic_id_map[]) { int apic; *************** *** 1249,1255 **** else io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; } else ! io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; return 1; --- 1297,1305 ---- else io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; } else ! io_apic_ints[intr].dst_apic_id = ! io_apic_id_map[entry->dst_apic_id]; ! io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; return 1; *************** *** 1271,1278 **** /* ! * Given a traditional ISA INT mask, return an APIC mask. ! */ u_int isa_apic_mask(u_int isa_mask) { --- 1321,1327 ---- /* ! * Given a traditional ISA INT mask, return an APIC mask. */ u_int isa_apic_mask(u_int isa_mask) { *************** *** 1601,1607 **** { int ap_cpu_id; #if defined(APIC_IO) - u_int32_t ux; int io_apic_id; int pin; #endif /* APIC_IO */ --- 1650,1655 ---- *************** *** 1661,1673 **** #else if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { #endif /* REALLY_ANAL_IOAPICID_VALUE */ ! ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ ! ux &= ~APIC_ID_MASK; /* clear the ID field */ ! ux |= 0x02000000; /* set it to '2' */ ! io_apic_write(0, IOAPIC_ID, ux); /* write new value */ ! ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ ! if ((ux & APIC_ID_MASK) != 0x02000000) ! panic("can't control IO APIC ID, reg: 0x%08x", ux); io_apic_id = 2; } IO_TO_ID(0) = io_apic_id; --- 1709,1715 ---- #else if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { #endif /* REALLY_ANAL_IOAPICID_VALUE */ ! set_io_apic_id(0, 2); io_apic_id = 2; } IO_TO_ID(0) = io_apic_id; >Release-Note: >Audit-Trail: >Unformatted: Stephen Clawson To Unsubscribe: send mail to majordomo@FreeBSD.org with "unsubscribe freebsd-bugs" in the body of the message