Date: Fri, 23 Oct 2009 03:17:02 +0000 (UTC) From: Nathan Whitehorn <nwhitehorn@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r198378 - in head/sys/powerpc: aim booke include powerpc Message-ID: <200910230317.n9N3H2YP047516@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: nwhitehorn Date: Fri Oct 23 03:17:02 2009 New Revision: 198378 URL: http://svn.freebsd.org/changeset/base/198378 Log: Add SMP support on U3-based G5 systems. This does not yet work perfectly: at least on my Xserve, getting the decrementer and timebase on APs to tick requires setting up a clock chip over I2C, which is not yet done. While here, correct the 64-bit tlbie function to set the CPU to 64-bit mode correctly. Hardware donated by: grehan Modified: head/sys/powerpc/aim/mmu_oea64.c head/sys/powerpc/aim/mp_cpudep.c head/sys/powerpc/aim/platform_chrp.c head/sys/powerpc/booke/mp_cpudep.c head/sys/powerpc/include/pcpu.h head/sys/powerpc/include/smp.h head/sys/powerpc/include/spr.h head/sys/powerpc/powerpc/cpu.c head/sys/powerpc/powerpc/mp_machdep.c Modified: head/sys/powerpc/aim/mmu_oea64.c ============================================================================== --- head/sys/powerpc/aim/mmu_oea64.c Thu Oct 22 21:07:32 2009 (r198377) +++ head/sys/powerpc/aim/mmu_oea64.c Fri Oct 23 03:17:02 2009 (r198378) @@ -182,35 +182,28 @@ va_to_vsid(pmap_t pm, vm_offset_t va) * Just to add to the fun, exceptions must be off as well * so that we can't trap in 64-bit mode. What a pain. */ +struct mtx tlbie_mutex; static __inline void TLBIE(pmap_t pmap, vm_offset_t va) { - register_t msr; - register_t scratch; - uint64_t vpn; register_t vpn_hi, vpn_lo; - -#if 1 - /* - * CPU documentation says that tlbie takes the VPN, not the - * VA. I think the code below does this correctly. We will see. - */ + register_t msr; + register_t scratch; vpn = (uint64_t)(va & ADDR_PIDX); if (pmap != NULL) vpn |= (va_to_vsid(pmap,va) << 28); -#else - vpn = va; -#endif vpn_hi = (uint32_t)(vpn >> 32); vpn_lo = (uint32_t)vpn; + mtx_lock_spin(&tlbie_mutex); __asm __volatile("\ mfmsr %0; \ clrldi %1,%0,49; \ - insrdi %1,1,1,0; \ + mtmsr %1; \ + insrdi %1,%5,1,0; \ mtmsrd %1; \ ptesync; \ \ @@ -222,7 +215,8 @@ TLBIE(pmap_t pmap, vm_offset_t va) { eieio; \ tlbsync; \ ptesync;" - : "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32)); + : "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)); + mtx_unlock_spin(&tlbie_mutex); } #define DISABLE_TRANS(msr) msr = mfmsr(); mtmsr(msr & ~PSL_DR); isync() @@ -352,7 +346,7 @@ static int moea64_pte_insert(u_int, str * PVO calls. */ static int moea64_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *, - vm_offset_t, vm_offset_t, uint64_t, int, int); + vm_offset_t, vm_offset_t, uint64_t, int); static void moea64_pvo_remove(struct pvo_entry *, int); static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t, int *); static struct lpte *moea64_pvo_to_pte(const struct pvo_entry *, int); @@ -825,6 +819,11 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_o MTX_RECURSE); /* + * Initialize the TLBIE lock. TLBIE can only be executed by one CPU. + */ + mtx_init(&tlbie_mutex, "tlbie mutex", NULL, MTX_SPIN); + + /* * Initialise the unmanaged pvo pool. */ moea64_bpvo_pool = (struct pvo_entry *)moea64_bootstrap_alloc( @@ -1254,7 +1253,7 @@ moea64_enter_locked(pmap_t pmap, vm_offs pvo_flags |= PVO_FAKE; error = moea64_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m), - pte_lo, pvo_flags, 0); + pte_lo, pvo_flags); if (pmap == kernel_pmap) TLBIE(pmap, va); @@ -1427,16 +1426,15 @@ moea64_uma_page_alloc(uma_zone_t zone, i if (pvo_allocator_start >= pvo_allocator_end) panic("Ran out of PVO allocator buffer space!"); - /* Now call pvo_enter in recursive mode */ moea64_pvo_enter(kernel_pmap, moea64_upvo_zone, &moea64_pvo_kunmanaged, va, VM_PAGE_TO_PHYS(m), LPTE_M, - PVO_WIRED | PVO_BOOTSTRAP, 1); + PVO_WIRED | PVO_BOOTSTRAP); TLBIE(kernel_pmap, va); - + if (needed_lock) PMAP_UNLOCK(kernel_pmap); - + if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) bzero((void *)va, PAGE_SIZE); @@ -1579,7 +1577,7 @@ moea64_kenter(mmu_t mmu, vm_offset_t va, PMAP_LOCK(kernel_pmap); error = moea64_pvo_enter(kernel_pmap, moea64_upvo_zone, &moea64_pvo_kunmanaged, va, pa, pte_lo, - PVO_WIRED | VM_PROT_EXECUTE, 0); + PVO_WIRED | VM_PROT_EXECUTE); TLBIE(kernel_pmap, va); @@ -1968,14 +1966,29 @@ static void tlbia(void) { vm_offset_t i; + register_t msr, scratch; - for (i = 0; i < 0xFF000; i += 0x00001000) - TLBIE(NULL,i); + for (i = 0; i < 0xFF000; i += 0x00001000) { + __asm __volatile("\ + mfmsr %0; \ + mr %1, %0; \ + insrdi %1,%3,1,0; \ + mtmsrd %1; \ + ptesync; \ + \ + tlbiel %2; \ + \ + mtmsrd %0; \ + eieio; \ + tlbsync; \ + ptesync;" + : "=r"(msr), "=r"(scratch) : "r"(i), "r"(1)); + } } static int moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head, - vm_offset_t va, vm_offset_t pa, uint64_t pte_lo, int flags, int recurse) + vm_offset_t va, vm_offset_t pa, uint64_t pte_lo, int flags) { struct pvo_entry *pvo; uint64_t vsid; @@ -2011,16 +2024,14 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t z * Remove any existing mapping for this page. Reuse the pvo entry if * there is a mapping. */ - if (!recurse) - LOCK_TABLE(); + LOCK_TABLE(); LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { if ((pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) == pa && (pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == (pte_lo & LPTE_PP)) { - if (!recurse) - UNLOCK_TABLE(); + UNLOCK_TABLE(); return (0); } moea64_pvo_remove(pvo, -1); @@ -2041,12 +2052,19 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t z moea64_bpvo_pool_index++; bootstrap = 1; } else { + /* + * Note: drop the table around the UMA allocation in + * case the UMA allocator needs to manipulate the page + * table. The mapping we are working with is already + * protected by the PMAP lock. + */ + UNLOCK_TABLE(); pvo = uma_zalloc(zone, M_NOWAIT); + LOCK_TABLE(); } if (pvo == NULL) { - if (!recurse) - UNLOCK_TABLE(); + UNLOCK_TABLE(); return (ENOMEM); } @@ -2093,8 +2111,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t z moea64_pte_overflow++; } - if (!recurse) - UNLOCK_TABLE(); + UNLOCK_TABLE(); return (first ? ENOENT : 0); } Modified: head/sys/powerpc/aim/mp_cpudep.c ============================================================================== --- head/sys/powerpc/aim/mp_cpudep.c Thu Oct 22 21:07:32 2009 (r198377) +++ head/sys/powerpc/aim/mp_cpudep.c Fri Oct 23 03:17:02 2009 (r198378) @@ -48,14 +48,34 @@ __FBSDID("$FreeBSD$"); #include <dev/ofw/openfirm.h> #include <machine/ofw_machdep.h> -extern void *rstcode; -extern register_t l2cr_config; -extern register_t l3cr_config; - void *ap_pcpu; +static register_t bsp_state[8]; + +static void cpudep_save_config(void *dummy); +SYSINIT(cpu_save_config, SI_SUB_CPU, SI_ORDER_ANY, cpudep_save_config, NULL); + +uintptr_t +cpudep_ap_bootstrap(void) +{ + register_t msr, sp; + + msr = PSL_KERNSET & ~PSL_EE; + mtmsr(msr); + isync(); + + __asm __volatile("mtsprg 0, %0" :: "r"(ap_pcpu)); + powerpc_sync(); + + pcpup->pc_curthread = pcpup->pc_idlethread; + pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb; + sp = pcpup->pc_curpcb->pcb_sp; + + return (sp); +} + static register_t -l2_enable(void) +mpc745x_l2_enable(register_t l2cr_config) { register_t ccr; @@ -77,7 +97,7 @@ l2_enable(void) } static register_t -l3_enable(void) +mpc745x_l3_enable(register_t l3cr_config) { register_t ccr; @@ -109,7 +129,7 @@ l3_enable(void) } static register_t -l1d_enable(void) +mpc745x_l1d_enable(void) { register_t hid; @@ -127,7 +147,7 @@ l1d_enable(void) } static register_t -l1i_enable(void) +mpc745x_l1i_enable(void) { register_t hid; @@ -144,43 +164,110 @@ l1i_enable(void) return (hid); } -uint32_t -cpudep_ap_bootstrap(void) +static void +cpudep_save_config(void *dummy) { - uint32_t hid, msr, reg, sp; - - // reg = mfspr(SPR_MSSCR0); - // mtspr(SPR_MSSCR0, reg | 0x3); - - __asm __volatile("mtsprg 0, %0" :: "r"(ap_pcpu)); - powerpc_sync(); + uint16_t vers; - __asm __volatile("mtspr 1023,%0" :: "r"(PCPU_GET(cpuid))); - __asm __volatile("mfspr %0,1023" : "=r"(pcpup->pc_pir)); - - msr = PSL_FP | PSL_IR | PSL_DR | PSL_ME | PSL_RI; - powerpc_sync(); - isync(); - mtmsr(msr); - isync(); + vers = mfpvr() >> 16; - if (l3cr_config != 0) - reg = l3_enable(); - if (l2cr_config != 0) - reg = l2_enable(); - reg = l1d_enable(); - reg = l1i_enable(); - - hid = mfspr(SPR_HID0); - hid &= ~(HID0_DOZE | HID0_SLEEP); - hid |= HID0_NAP | HID0_DPM; - mtspr(SPR_HID0, hid); - isync(); - - pcpup->pc_curthread = pcpup->pc_idlethread; - pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb; - sp = pcpup->pc_curpcb->pcb_sp; + switch(vers) { + case IBM970: + case IBM970FX: + case IBM970MP: + __asm __volatile ("mfspr %0,%2; mr %1,%0; srdi %0,%0,32" + : "=r" (bsp_state[0]),"=r" (bsp_state[1]) : "K" (SPR_HID0)); + __asm __volatile ("mfspr %0,%2; mr %1,%0; srdi %0,%0,32" + : "=r" (bsp_state[2]),"=r" (bsp_state[3]) : "K" (SPR_HID1)); + __asm __volatile ("mfspr %0,%2; mr %1,%0; srdi %0,%0,32" + : "=r" (bsp_state[4]),"=r" (bsp_state[5]) : "K" (SPR_HID4)); + __asm __volatile ("mfspr %0,%2; mr %1,%0; srdi %0,%0,32" + : "=r" (bsp_state[6]),"=r" (bsp_state[7]) : "K" (SPR_HID5)); + + break; + case MPC7450: + case MPC7455: + case MPC7457: + /* Only MPC745x CPUs have an L3 cache. */ + bsp_state[3] = mfspr(SPR_L3CR); + + /* Fallthrough */ + case MPC7400: + case MPC7410: + case MPC7447A: + case MPC7448: + bsp_state[2] = mfspr(SPR_L2CR); + bsp_state[1] = mfspr(SPR_HID1); + bsp_state[0] = mfspr(SPR_HID0); + break; + } +} - return (sp); +void +cpudep_ap_setup() +{ + register_t reg; + uint16_t vers; + + vers = mfpvr() >> 16; + + switch(vers) { + case IBM970: + case IBM970FX: + case IBM970MP: + /* Set HIOR to 0 */ + __asm __volatile("mtspr 311,%0" :: "r"(0)); + powerpc_sync(); + + /* + * The 970 has strange rules about how to update HID registers. + * See Table 2-3, 970MP manual + */ + + __asm __volatile(" \ + ld %0,0(%2); \ + mtspr %1, %0; \ + mfspr %0, %1; mfspr %0, %1; mfspr %0, %1; \ + mfspr %0, %1; mfspr %0, %1; mfspr %0, %1;" + : "=r"(reg) : "K"(SPR_HID0), "r"(bsp_state)); + __asm __volatile("ld %0, 8(%2); mtspr %1, %0; mtspr %1, %0; \ + isync" : "=r"(reg) : "K"(SPR_HID1), "r"(bsp_state)); + __asm __volatile("ld %0, 16(%2); sync; mtspr %1, %0; isync;" + : "=r"(reg) : "K"(SPR_HID4), "r"(bsp_state)); + __asm __volatile("ld %0, 24(%2); sync; mtspr %1, %0; isync;" + : "=r"(reg) : "K"(SPR_HID5), "r"(bsp_state)); + + powerpc_sync(); + break; + case MPC7450: + case MPC7455: + case MPC7457: + /* Only MPC745x CPUs have an L3 cache. */ + reg = mpc745x_l3_enable(bsp_state[3]); + + /* Fallthrough */ + case MPC7400: + case MPC7410: + case MPC7447A: + case MPC7448: + /* XXX: Program the CPU ID into PIR */ + __asm __volatile("mtspr 1023,%0" :: "r"(PCPU_GET(cpuid))); + + powerpc_sync(); + isync(); + + mtspr(SPR_HID0, bsp_state[0]); isync(); + mtspr(SPR_HID1, bsp_state[1]); isync(); + + reg = mpc745x_l2_enable(bsp_state[2]); + reg = mpc745x_l1d_enable(); + reg = mpc745x_l1i_enable(); + + break; + default: + printf("WARNING: Unknown CPU type. Cache performace may be " + "suboptimal.\n"); + break; + } } Modified: head/sys/powerpc/aim/platform_chrp.c ============================================================================== --- head/sys/powerpc/aim/platform_chrp.c Thu Oct 22 21:07:32 2009 (r198377) +++ head/sys/powerpc/aim/platform_chrp.c Fri Oct 23 03:17:02 2009 (r198378) @@ -239,12 +239,14 @@ chrp_smp_start_cpu(platform_t plat, stru rstvec = rstvec_virtbase + reset; *rstvec = 4; + (void)(*rstvec); powerpc_sync(); DELAY(1); *rstvec = 0; + (void)(*rstvec); powerpc_sync(); - timeout = 1000; + timeout = 10000; while (!pc->pc_awake && timeout--) DELAY(100); Modified: head/sys/powerpc/booke/mp_cpudep.c ============================================================================== --- head/sys/powerpc/booke/mp_cpudep.c Thu Oct 22 21:07:32 2009 (r198377) +++ head/sys/powerpc/booke/mp_cpudep.c Fri Oct 23 03:17:02 2009 (r198378) @@ -47,7 +47,7 @@ extern void icache_inval(void); volatile void *ap_pcpu; -uint32_t +uintptr_t cpudep_ap_bootstrap() { uint32_t msr, sp, csr; @@ -78,3 +78,8 @@ cpudep_ap_bootstrap() return (sp); } + +void +cpudep_ap_setup() +{ +} Modified: head/sys/powerpc/include/pcpu.h ============================================================================== --- head/sys/powerpc/include/pcpu.h Thu Oct 22 21:07:32 2009 (r198377) +++ head/sys/powerpc/include/pcpu.h Fri Oct 23 03:17:02 2009 (r198378) @@ -43,8 +43,8 @@ struct pmap; struct thread *pc_vecthread; /* current vec user */ \ uintptr_t pc_hwref; \ uint32_t pc_pir; \ - int pc_bsp:1; \ - int pc_awake:1; \ + int pc_bsp; \ + volatile int pc_awake; \ uint32_t pc_ipimask; \ register_t pc_tempsave[CPUSAVE_LEN]; \ register_t pc_disisave[CPUSAVE_LEN]; \ Modified: head/sys/powerpc/include/smp.h ============================================================================== --- head/sys/powerpc/include/smp.h Thu Oct 22 21:07:32 2009 (r198377) +++ head/sys/powerpc/include/smp.h Fri Oct 23 03:17:02 2009 (r198378) @@ -48,7 +48,8 @@ struct cpuref { }; void pmap_cpu_bootstrap(int); -uint32_t cpudep_ap_bootstrap(void); +uintptr_t cpudep_ap_bootstrap(void); +void cpudep_ap_setup(void); void machdep_ap_bootstrap(void); #endif /* !LOCORE */ Modified: head/sys/powerpc/include/spr.h ============================================================================== --- head/sys/powerpc/include/spr.h Thu Oct 22 21:07:32 2009 (r198377) +++ head/sys/powerpc/include/spr.h Fri Oct 23 03:17:02 2009 (r198378) @@ -50,7 +50,7 @@ #define mtspr64(reg,valhi,vallo,scratch) \ __asm __volatile(" \ mfmsr %0; \ - insrdi %0,1,1,0; \ + insrdi %0,%5,1,0; \ mtmsrd %0; \ isync; \ \ @@ -62,13 +62,13 @@ clrldi %0,%0,1; \ mtmsrd %0; \ isync;" \ - : "=r"(scratch), "=r"(valhi) : "r"(vallo), "K"(reg), "r"(32)) + : "=r"(scratch), "=r"(valhi) : "r"(vallo), "K"(reg), "r"(32), "r"(1)) #define mfspr64upper(reg,scratch) \ ( { register_t val; \ __asm __volatile(" \ mfmsr %0; \ - insrdi %0,1,1,0; \ + insrdi %0,%4,1,0; \ mtmsrd %0; \ isync; \ \ @@ -78,7 +78,7 @@ clrldi %0,%0,1; \ mtmsrd %0; \ isync;" \ - : "=r"(scratch), "=r"(val) : "K"(reg), "r"(32)); \ + : "=r"(scratch), "=r"(val) : "K"(reg), "r"(32), "r"(1)); \ val; } ) #endif /* _LOCORE */ Modified: head/sys/powerpc/powerpc/cpu.c ============================================================================== --- head/sys/powerpc/powerpc/cpu.c Thu Oct 22 21:07:32 2009 (r198377) +++ head/sys/powerpc/powerpc/cpu.c Fri Oct 23 03:17:02 2009 (r198378) @@ -69,6 +69,7 @@ #include <machine/bus.h> #include <machine/hid.h> #include <machine/md_var.h> +#include <machine/smp.h> #include <machine/spr.h> int powerpc_pow_enabled; @@ -112,9 +113,6 @@ static const struct cputab models[] = { static char model[64]; SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD, model, 0, ""); -register_t l2cr_config = 0; -register_t l3cr_config = 0; - static void cpu_print_speed(void); static void cpu_print_cacheinfo(u_int, uint16_t); @@ -258,11 +256,6 @@ cpu_setup(u_int cpuid) case MPC7450: case MPC7455: case MPC7457: - /* Only MPC745x CPUs have an L3 cache. */ - - l3cr_config = mfspr(SPR_L3CR); - - /* Fallthrough */ case MPC750: case IBM750FX: case MPC7400: @@ -272,8 +265,6 @@ cpu_setup(u_int cpuid) cpu_print_speed(); printf("\n"); - l2cr_config = mfspr(SPR_L2CR); - if (bootverbose) cpu_print_cacheinfo(cpuid, vers); break; @@ -366,15 +357,15 @@ cpu_print_cacheinfo(u_int cpuid, uint16_ printf("L1 D-cache %sabled\n", (hid & HID0_DCE) ? "en" : "dis"); printf("cpu%u: ", cpuid); - if (l2cr_config & L2CR_L2E) { + if (mfspr(SPR_L2CR) & L2CR_L2E) { switch (vers) { case MPC7450: case MPC7455: case MPC7457: printf("256KB L2 cache, "); - if (l3cr_config & L3CR_L3E) + if (mfspr(SPR_L3CR) & L3CR_L3E) printf("%cMB L3 backside cache", - l3cr_config & L3CR_L3SIZ ? '2' : '1'); + mfspr(SPR_L3CR) & L3CR_L3SIZ ? '2' : '1'); else printf("L3 cache disabled"); printf("\n"); @@ -383,7 +374,7 @@ cpu_print_cacheinfo(u_int cpuid, uint16_ printf("512KB L2 cache\n"); break; default: - switch (l2cr_config & L2CR_L2SIZ) { + switch (mfspr(SPR_L2CR) & L2CR_L2SIZ) { case L2SIZ_256K: printf("256KB "); break; @@ -394,9 +385,9 @@ cpu_print_cacheinfo(u_int cpuid, uint16_ printf("1MB "); break; } - printf("write-%s", (l2cr_config & L2CR_L2WT) + printf("write-%s", (mfspr(SPR_L2CR) & L2CR_L2WT) ? "through" : "back"); - if (l2cr_config & L2CR_L2PE) + if (mfspr(SPR_L2CR) & L2CR_L2PE) printf(", with parity"); printf(" backside cache\n"); break; Modified: head/sys/powerpc/powerpc/mp_machdep.c ============================================================================== --- head/sys/powerpc/powerpc/mp_machdep.c Thu Oct 22 21:07:32 2009 (r198377) +++ head/sys/powerpc/powerpc/mp_machdep.c Fri Oct 23 03:17:02 2009 (r198378) @@ -64,7 +64,10 @@ static u_int ipi_msg_cnt[32]; void machdep_ap_bootstrap(void) { + /* Set up important bits on the CPU (HID registers, etc.) */ + cpudep_ap_setup(); + /* Set PIR */ PCPU_SET(pir, mfspr(SPR_PIR)); PCPU_SET(awake, 1); __asm __volatile("msync; isync"); @@ -78,7 +81,7 @@ machdep_ap_bootstrap(void) __asm __volatile("mtdec %0" :: "r"(ap_decr)); atomic_add_int(&ap_awake, 1); - CTR1(KTR_SMP, "SMP: AP CPU%d launched", PCPU_GET(cpuid)); + printf("SMP: AP CPU #%d launched\n", PCPU_GET(cpuid)); /* Initialize curthread */ PCPU_SET(curthread, PCPU_GET(idlethread)); @@ -86,6 +89,8 @@ machdep_ap_bootstrap(void) /* Let the DEC and external interrupts go */ mtmsr(mfmsr() | PSL_EE); + + /* Announce ourselves awake, and enter the scheduler */ sched_throw(NULL); } @@ -247,6 +252,9 @@ cpu_mp_unleash(void *dummy) mp_ncpus, cpus, smp_cpus); } + /* Let the APs get into the scheduler */ + DELAY(10000); + smp_active = 1; smp_started = 1; }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200910230317.n9N3H2YP047516>