From owner-freebsd-alpha@FreeBSD.ORG Wed Nov 3 18:09:50 2004 Return-Path: Delivered-To: freebsd-alpha@freebsd.org Received: from mx1.FreeBSD.org (mx1.freebsd.org [216.136.204.125]) by hub.freebsd.org (Postfix) with ESMTP id 13DC816A4CE for ; Wed, 3 Nov 2004 18:09:50 +0000 (GMT) Received: from mail2.speakeasy.net (mail2.speakeasy.net [216.254.0.202]) by mx1.FreeBSD.org (Postfix) with ESMTP id CB19E43D2F for ; Wed, 3 Nov 2004 18:09:49 +0000 (GMT) (envelope-from jhb@FreeBSD.org) Received: (qmail 10655 invoked from network); 3 Nov 2004 18:09:47 -0000 Received: from dsl027-160-063.atl1.dsl.speakeasy.net (HELO server.baldwin.cx) ([216.27.160.63]) (envelope-sender ) encrypted SMTP for ; 3 Nov 2004 18:09:46 -0000 Received: from [10.50.41.235] (gw1.twc.weather.com [216.133.140.1]) (authenticated bits=0) by server.baldwin.cx (8.12.11/8.12.11) with ESMTP id iA3I9YLK078281 for ; Wed, 3 Nov 2004 13:09:35 -0500 (EST) (envelope-from jhb@FreeBSD.org) From: John Baldwin To: alpha@FreeBSD.org Date: Wed, 3 Nov 2004 13:49:01 -0500 User-Agent: KMail/1.6.2 MIME-Version: 1.0 Content-Disposition: inline Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Message-Id: <200411031349.01668.jhb@FreeBSD.org> X-Spam-Checker-Version: SpamAssassin 2.63 (2004-01-11) on server.baldwin.cx Subject: Patch to tweak FreeBSD CPU IDs in the SMP support X-BeenThere: freebsd-alpha@freebsd.org X-Mailman-Version: 2.1.1 Precedence: list List-Id: Porting FreeBSD to the Alpha List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 03 Nov 2004 18:09:50 -0000 I would appreciate it if folks with SMP machines could please test the patch below. It should ensure among other things that the boot processor will now always have a FreeBSD cpu ID of 0 and that the FreeBSD cpu IDs will now be allocate independent of the PAL IDs from the HWPRB. I'm especially interested in reports from machines where the PAL IDs don't match the CPU IDs. (My one SMP alpha box always seems to boot from CPU 0 and so the IDs match up so I can't tell if I've accidentally gotten an id variable backwards somewhere.) --- //depot/vendor/freebsd/src/sys/alpha/alpha/machdep.c 2004/09/05 02:10:52 +++ //depot/projects/smpng/sys/alpha/alpha/machdep.c 2004/10/27 17:45:12 @@ -864,7 +864,8 @@ /* This is not a 'struct user' */ size_t sz = round_page(KSTACK_PAGES * PAGE_SIZE); pcpup = (struct pcpu *) pmap_steal_memory(sz); - pcpu_init(pcpup, alpha_pal_whami(), sz); + pcpu_init(pcpup, 0, sz); + pcpup->pc_pal_id = alpha_pal_whami(); alpha_pal_wrval((u_int64_t) pcpup); PCPU_GET(next_asn) = 1; /* 0 used for proc0 pmap */ PCPU_SET(curthread, &thread0); --- //depot/vendor/freebsd/src/sys/alpha/alpha/mp_machdep.c 2004/01/07 23:00:44 +++ //depot/projects/smpng/sys/alpha/alpha/mp_machdep.c 2004/11/02 22:27:44 @@ -61,23 +61,23 @@ static struct mtx ap_boot_mtx; -u_int boot_cpu_id; +u_int64_t boot_cpu_id; static void release_aps(void *dummy); static int smp_cpu_enabled(struct pcs *pcsp); extern void smp_init_secondary_glue(void); -static int smp_send_secondary_command(const char *command, int cpuid); -static int smp_start_secondary(int cpuid); +static int smp_send_secondary_command(const char *command, int pal_id); +static int smp_start_secondary(int pal_id, int cpuid); /* * Communicate with a console running on a secondary processor. * Return 1 on failure. */ static int -smp_send_secondary_command(const char *command, int cpuid) +smp_send_secondary_command(const char *command, int pal_id) { - u_int64_t mask = 1L << cpuid; - struct pcs *cpu = LOCATE_PCS(hwrpb, cpuid); + u_int64_t mask = 1L << pal_id; + struct pcs *cpu = LOCATE_PCS(hwrpb, pal_id); int i, len; /* @@ -165,7 +165,7 @@ /* * Set flags in our per-CPU slot in the HWRPB. */ - cpu = LOCATE_PCS(hwrpb, PCPU_GET(cpuid)); + cpu = LOCATE_PCS(hwrpb, PCPU_GET(pal_id)); cpu->pcs_flags &= ~PCS_BIP; cpu->pcs_flags |= PCS_RC; alpha_mb(); @@ -216,9 +216,9 @@ } static int -smp_start_secondary(int cpuid) +smp_start_secondary(int pal_id, int cpuid) { - struct pcs *cpu = LOCATE_PCS(hwrpb, cpuid); + struct pcs *cpu = LOCATE_PCS(hwrpb, pal_id); struct pcs *bootcpu = LOCATE_PCS(hwrpb, boot_cpu_id); struct alpha_pcb *pcb = (struct alpha_pcb *) cpu->pcs_hwpcb; struct pcpu *pcpu; @@ -226,12 +226,12 @@ size_t sz; if ((cpu->pcs_flags & PCS_PV) == 0) { - printf("smp_start_secondary: cpu %d PALcode invalid\n", cpuid); + printf("smp_start_secondary: cpu %d PALcode invalid\n", pal_id); return 0; } if (bootverbose) - printf("smp_start_secondary: starting cpu %d\n", cpuid); + printf("smp_start_secondary: starting cpu %d\n", pal_id); sz = round_page((UAREA_PAGES + KSTACK_PAGES) * PAGE_SIZE); pcpu = malloc(sz, M_TEMP, M_NOWAIT); @@ -241,6 +241,7 @@ } pcpu_init(pcpu, cpuid, sz); + pcpu->pc_pal_id = pal_id; /* * Copy the idle pcb and setup the address to start executing. @@ -270,7 +271,7 @@ /* * Fire it up and hope for the best. */ - if (!smp_send_secondary_command("START\r\n", cpuid)) { + if (!smp_send_secondary_command("START\r\n", pal_id)) { printf("smp_start_secondary: can't send START command\n"); pcpu_destroy(pcpu); free(pcpu, M_TEMP); @@ -296,7 +297,7 @@ * It worked (I think). */ if (bootverbose) - printf("smp_start_secondary: cpu %d started\n", cpuid); + printf("smp_start_secondary: cpu %d started\n", pal_id); return 1; } @@ -329,16 +330,18 @@ void cpu_mp_setmaxid(void) { - int i; + u_int64_t i; mp_maxid = 0; - for (i = 0; i < hwrpb->rpb_pcs_cnt && i < MAXCPU; i++) { - if (i == PCPU_GET(cpuid)) + for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) { + if (i == PCPU_GET(pal_id)) continue; if (!smp_cpu_enabled(LOCATE_PCS(hwrpb, i))) continue; - mp_maxid = i; + mp_maxid++; } + if (mp_maxid > MAXCPU) + mp_maxid = MAXCPU; } int @@ -348,7 +351,7 @@ /* XXX: Need to check for valid platforms here. */ - boot_cpu_id = PCPU_GET(cpuid); + boot_cpu_id = PCPU_GET(pal_id); KASSERT(boot_cpu_id == hwrpb->rpb_primary_cpu_id, ("cpu_mp_probe() called on non-primary CPU")); all_cpus = PCPU_GET(cpumask); @@ -358,12 +361,10 @@ /* Make sure we have at least one secondary CPU. */ cpus = 0; for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) { - if (i == PCPU_GET(cpuid)) + if (i == PCPU_GET(pal_id)) continue; if (!smp_cpu_enabled(LOCATE_PCS(hwrpb, i))) continue; - if (i > MAXCPU) - continue; cpus++; } return (cpus); @@ -372,10 +373,11 @@ void cpu_mp_start(void) { - int i; + int i, cpuid; mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); + cpuid = 1; for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) { struct pcs *pcsp; @@ -410,22 +412,30 @@ printf("CPU %d disabled by loader.\n", i); continue; } - all_cpus |= (1 << i); - mp_ncpus++; + if (smp_start_secondary(i, cpuid)) { + all_cpus |= (1 << cpuid); + mp_ncpus++; + cpuid++; + } } PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); - - for (i = 0; i < hwrpb->rpb_pcs_cnt; i++) { - if (i == boot_cpu_id) - continue; - if (!CPU_ABSENT(i)) - smp_start_secondary(i); - } } void cpu_mp_announce(void) { + struct pcpu *pc; + int i; + + /* List CPUs */ + printf(" cpu0 (BSP): PAL ID: %2lu\n", boot_cpu_id); + for (i = 1; i < MAXCPU; i++) { + if (CPU_ABSENT(i)) + continue; + pc = pcpu_find(i); + MPASS(pc != NULL); + printf(" cpu%d (AP): PAL ID: %2lu\n", i, pc->pc_pal_id); + } } /* @@ -446,8 +456,9 @@ if (pcpu) { atomic_set_64(&pcpu->pc_pending_ipis, ipi); alpha_mb(); - CTR1(KTR_SMP, "calling alpha_pal_wripir(%d)", cpuid); - alpha_pal_wripir(cpuid); + CTR1(KTR_SMP, "calling alpha_pal_wripir(%d)", + pcpu->pc_pal_id); + alpha_pal_wripir(pcpu->pc_pal_id); } } } @@ -529,8 +540,8 @@ * requests to provide PALcode to secondaries and to start up new * secondaries that are added to the system on the fly. */ - if (PCPU_GET(cpuid) == boot_cpu_id) { - u_int cpuid; + if (PCPU_GET(pal_id) == boot_cpu_id) { + u_int pal_id; u_int64_t txrdy; #ifdef DIAGNOSTIC struct pcs *cpu; @@ -539,18 +550,18 @@ alpha_mb(); while (hwrpb->rpb_txrdy != 0) { - cpuid = ffs(hwrpb->rpb_txrdy) - 1; + pal_id = ffs(hwrpb->rpb_txrdy) - 1; #ifdef DIAGNOSTIC - cpu = LOCATE_PCS(hwrpb, cpuid); + cpu = LOCATE_PCS(hwrpb, pal_id); bcopy(&cpu->pcs_buffer.txbuf, buf, cpu->pcs_buffer.txlen); buf[cpu->pcs_buffer.txlen] = '\0'; - printf("SMP From CPU%d: %s\n", cpuid, buf); + printf("SMP From CPU%d: %s\n", pal_id, buf); #endif do { txrdy = hwrpb->rpb_txrdy; } while (atomic_cmpset_64(&hwrpb->rpb_txrdy, txrdy, - txrdy & ~(1 << cpuid)) == 0); + txrdy & ~(1 << pal_id)) == 0); } } } --- //depot/vendor/freebsd/src/sys/alpha/include/pcpu.h 2001/12/11 23:42:08 +++ //depot/projects/smpng/sys/alpha/include/pcpu.h 2004/10/27 17:45:12 @@ -34,6 +34,7 @@ #define PCPU_MD_FIELDS \ struct alpha_pcb pc_idlepcb; /* pcb for idling */ \ + u_int64_t pc_pal_id; /* physical CPU ID */ \ u_int64_t pc_idlepcbphys; /* pa of pc_idlepcb */ \ u_int64_t pc_pending_ipis; /* pending IPI's */ \ u_int32_t pc_next_asn; /* next ASN to alloc */ \ --- //depot/vendor/freebsd/src/sys/alpha/include/smp.h 2001/08/13 23:42:17 +++ //depot/projects/smpng/sys/alpha/include/smp.h 2004/10/29 21:11:53 @@ -26,7 +26,7 @@ #ifndef LOCORE -extern u_int boot_cpu_id; +extern u_int64_t boot_cpu_id; void ipi_selected(u_int cpus, u_int64_t ipi); void ipi_all(u_int64_t ipi); -- John Baldwin <>< http://www.FreeBSD.org/~jhb/ "Power Users Use the Power to Serve" = http://www.FreeBSD.org