Date: Thu, 11 Sep 2003 16:16:54 -0700 (PDT) From: Peter Wemm <peter@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 37941 for review Message-ID: <200309112316.h8BNGs7A053776@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=37941 Change 37941 by peter@peter_work on 2003/09/11 16:16:06 another smp pass (not finished still) Affected files ... .. //depot/projects/hammer/sys/amd64/amd64/amd64_mem.c#2 edit .. //depot/projects/hammer/sys/amd64/amd64/mp_machdep.c#2 edit .. //depot/projects/hammer/sys/amd64/amd64/mptable.c#2 edit .. //depot/projects/hammer/sys/amd64/amd64/pmap.c#32 edit .. //depot/projects/hammer/sys/amd64/include/smp.h#7 edit .. //depot/projects/hammer/sys/amd64/include/smptests.h#2 edit Differences ... ==== //depot/projects/hammer/sys/amd64/amd64/amd64_mem.c#2 (text+ko) ==== ==== //depot/projects/hammer/sys/amd64/amd64/mp_machdep.c#2 (text+ko) ==== @@ -35,15 +35,9 @@ #error How did you get here? #endif -#if defined(I386_CPU) && !defined(COMPILING_LINT) -#error SMP not supported with I386_CPU -#endif #ifndef DEV_APIC #error The apic device is required for SMP, add "device apic" to your config file. #endif -#if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT) -#error SMP not supported with CPU_DISABLE_CMPXCHG -#endif #endif /* not lint */ #include <sys/param.h> @@ -63,41 +57,19 @@ #include <sys/proc.h> #include <sys/smp.h> #include <sys/sysctl.h> -#if 0 -#include <sys/user.h> -#endif #include <vm/vm.h> #include <vm/vm_param.h> #include <vm/pmap.h> #include <vm/vm_kern.h> #include <vm/vm_extern.h> -#if 0 -#include <vm/vm_map.h> -#endif #include <machine/apicreg.h> -#if 0 -#include <machine/atomic.h> -#endif #include <machine/clock.h> -#if 0 -#include <machine/cpu.h> -#include <machine/cpufunc.h> -#endif #include <machine/md_var.h> #include <machine/pcb.h> -#if 0 -#include <machine/psl.h> -#include <machine/segments.h> -#endif #include <machine/smp.h> -#include <machine/smptests.h> /** COUNT_XINVLTLB_HITS, USE_COMLOCK */ -#if 0 -#include <machine/tss.h> -#endif #include <machine/specialreg.h> -#include <machine/privatespace.h> #define WARMBOOT_TARGET 0 #define WARMBOOT_OFF (KERNBASE + 0x0467) @@ -109,43 +81,6 @@ #define BIOS_WARM (0x0a) /* - * this code MUST be enabled here and in mpboot.s. - * it follows the very early stages of AP boot by placing values in CMOS ram. - * it NORMALLY will never be needed and thus the primitive method for enabling. - * -#define CHECK_POINTS - */ - -#if defined(CHECK_POINTS) && !defined(PC98) -#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) -#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) - -#define CHECK_INIT(D); \ - CHECK_WRITE(0x34, (D)); \ - CHECK_WRITE(0x35, (D)); \ - CHECK_WRITE(0x36, (D)); \ - CHECK_WRITE(0x37, (D)); \ - CHECK_WRITE(0x38, (D)); \ - CHECK_WRITE(0x39, (D)); - -#define CHECK_PRINT(S); \ - printf("%s: %d, %d, %d, %d, %d, %d\n", \ - (S), \ - CHECK_READ(0x34), \ - CHECK_READ(0x35), \ - CHECK_READ(0x36), \ - CHECK_READ(0x37), \ - CHECK_READ(0x38), \ - CHECK_READ(0x39)); - -#else /* CHECK_POINTS */ - -#define CHECK_INIT(D) -#define CHECK_PRINT(S) - -#endif /* CHECK_POINTS */ - -/* * Values to send to the POST hardware. */ #define MP_BOOTADDRESS_POST 0x10 @@ -165,11 +100,6 @@ /* lock region used by kernel profiling */ int mcount_lock; -#ifdef USE_COMLOCK -/* locks com (tty) data/hardware accesses: a FASTINTR() */ -struct mtx com_mtx; -#endif - /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; @@ -247,7 +177,6 @@ u_int mp_bootaddress(u_int basemem) { - POSTCODE(MP_BOOTADDRESS_POST); boot_address = basemem & ~0xfff; /* round down to 4k boundary */ if ((basemem - boot_address) < bootMP_size) @@ -321,8 +250,6 @@ cpu_mp_start(void) { - POSTCODE(MP_START_POST); - /* Install an inter-CPU IPI for TLB invalidation */ setidt(IPI_INVLTLB, IDTVEC(invltlb), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); @@ -355,9 +282,6 @@ setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); -#ifdef USE_COMLOCK - mtx_init(&com_mtx, "com", NULL, MTX_SPIN); -#endif mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN); /* Setup BSP apic ID */ @@ -389,8 +313,6 @@ { int i, x; - POSTCODE(MP_ANNOUNCE_POST); - /* List CPUs */ printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id); for (i = 1, x = 0; x < MAXCPU; x++) { @@ -502,8 +424,6 @@ uintptr_t kptbase; int i, pg, apic_id, cpu; - POSTCODE(START_ALL_APS_POST); - mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); /* install the AP 1st level boot code */ @@ -563,16 +483,13 @@ bootAP = cpu; /* attempt to start the Application Processor */ - CHECK_INIT(99); /* setup checkpoints */ if (!start_ap(apic_id, boot_addr)) { printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id); - CHECK_PRINT("trace"); /* show checkpoints */ /* better panic as the AP may be running loose */ printf("panic y/n? [y] "); if (cngetc() != 'n') panic("bye-bye"); } - CHECK_PRINT("trace"); /* show checkpoints */ all_cpus |= (1 << cpu); /* record AP in CPU map */ } @@ -632,8 +549,6 @@ u_int16_t *dst16; u_int32_t *dst32; - POSTCODE(INSTALL_AP_TRAMP_POST); - for (x = 0; x < size; ++x) *dst++ = *src++; @@ -680,8 +595,6 @@ int vector, ms; int cpus; - POSTCODE(START_AP_POST); - /* calculate the vector */ vector = (boot_addr >> 12) & 0xff; ==== //depot/projects/hammer/sys/amd64/amd64/mptable.c#2 (text+ko) ==== @@ -225,10 +225,6 @@ u_long segment; u_int32_t target; -#if 0 - POSTCODE(MP_PROBE_POST); -#endif - /* see if EBDA exists */ if ((segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) != 0) { /* search first 1K of EBDA */ ==== //depot/projects/hammer/sys/amd64/amd64/pmap.c#32 (text+ko) ==== @@ -116,6 +116,9 @@ #include <sys/user.h> #include <sys/vmmeter.h> #include <sys/sysctl.h> +#ifdef SMP +#include <sys/smp.h> +#endif #include <vm/vm.h> #include <vm/vm_param.h> @@ -165,6 +168,9 @@ LIST_HEAD(pmaplist, pmap); static struct pmaplist allpmaps; static struct mtx allpmaps_lock; +#ifdef SMP +static struct mtx lazypmap_lock; +#endif vm_paddr_t avail_start; /* PA of first available physical page */ vm_paddr_t avail_end; /* PA of last available physical page */ @@ -479,6 +485,9 @@ kernel_pmap->pm_active = -1; /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvlist); LIST_INIT(&allpmaps); +#ifdef SMP + mtx_init(&lazypmap_lock, "lazypmap", NULL, MTX_SPIN); +#endif mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN); mtx_lock_spin(&allpmaps_lock); LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list); @@ -678,10 +687,121 @@ return 0; } -XXXX SMP VERSIONS +#ifdef SMP +/* + * For SMP, these functions have to use the IPI mechanism for coherence. + */ +void +pmap_invalidate_page(pmap_t pmap, vm_offset_t va) +{ + u_int cpumask; + u_int other_cpus; + + if (smp_started) { + if (!(read_eflags() & PSL_I)) + panic("%s: interrupts disabled", __func__); + mtx_lock_spin(&smp_tlb_mtx); + } else + critical_enter(); + /* + * We need to disable interrupt preemption but MUST NOT have + * interrupts disabled here. + * XXX we may need to hold schedlock to get a coherent pm_active + * XXX critical sections disable interrupts again + */ + if (pmap->pm_active == -1 || pmap->pm_active == all_cpus) { + invlpg(va); + smp_invlpg(va); + } else { + cpumask = PCPU_GET(cpumask); + other_cpus = PCPU_GET(other_cpus); + if (pmap->pm_active & cpumask) + invlpg(va); + if (pmap->pm_active & other_cpus) + smp_masked_invlpg(pmap->pm_active & other_cpus, va); + } + if (smp_started) + mtx_unlock_spin(&smp_tlb_mtx); + else + critical_exit(); +} + +void +pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) +{ + u_int cpumask; + u_int other_cpus; + vm_offset_t addr; + + if (smp_started) { + if (!(read_eflags() & PSL_I)) + panic("%s: interrupts disabled", __func__); + mtx_lock_spin(&smp_tlb_mtx); + } else + critical_enter(); + /* + * We need to disable interrupt preemption but MUST NOT have + * interrupts disabled here. + * XXX we may need to hold schedlock to get a coherent pm_active + * XXX critical sections disable interrupts again + */ + if (pmap->pm_active == -1 || pmap->pm_active == all_cpus) { + for (addr = sva; addr < eva; addr += PAGE_SIZE) + invlpg(addr); + smp_invlpg_range(sva, eva); + } else { + cpumask = PCPU_GET(cpumask); + other_cpus = PCPU_GET(other_cpus); + if (pmap->pm_active & cpumask) + for (addr = sva; addr < eva; addr += PAGE_SIZE) + invlpg(addr); + if (pmap->pm_active & other_cpus) + smp_masked_invlpg_range(pmap->pm_active & other_cpus, + sva, eva); + } + if (smp_started) + mtx_unlock_spin(&smp_tlb_mtx); + else + critical_exit(); +} + +void +pmap_invalidate_all(pmap_t pmap) +{ + u_int cpumask; + u_int other_cpus; + if (smp_started) { + if (!(read_eflags() & PSL_I)) + panic("%s: interrupts disabled", __func__); + mtx_lock_spin(&smp_tlb_mtx); + } else + critical_enter(); + /* + * We need to disable interrupt preemption but MUST NOT have + * interrupts disabled here. + * XXX we may need to hold schedlock to get a coherent pm_active + * XXX critical sections disable interrupts again + */ + if (pmap->pm_active == -1 || pmap->pm_active == all_cpus) { + invltlb(); + smp_invltlb(); + } else { + cpumask = PCPU_GET(cpumask); + other_cpus = PCPU_GET(other_cpus); + if (pmap->pm_active & cpumask) + invltlb(); + if (pmap->pm_active & other_cpus) + smp_masked_invltlb(pmap->pm_active & other_cpus); + } + if (smp_started) + mtx_unlock_spin(&smp_tlb_mtx); + else + critical_exit(); +} +#else /* !SMP */ /* - * Normal invalidation functions. + * Normal, non-SMP, invalidation functions. * We inline these within pmap.c for speed. */ PMAP_INLINE void @@ -709,6 +829,7 @@ if (pmap == kernel_pmap || pmap->pm_active) invltlb(); } +#endif /* !SMP */ /* * Are we current address space or kernel? @@ -1241,6 +1362,95 @@ * Pmap allocation/deallocation routines. ***************************************************/ +#ifdef SMP +/* + * Deal with a SMP shootdown of other users of the pmap that we are + * trying to dispose of. This can be a bit hairy. + */ +static u_int *lazymask; +static u_int lazyptd; +static volatile u_int lazywait; + +void pmap_lazyfix_action(void); + +void +pmap_lazyfix_action(void) +{ + u_int mymask = PCPU_GET(cpumask); + + if (rcr3() == lazyptd) + load_cr3(PCPU_GET(curpcb)->pcb_cr3); + atomic_clear_int(lazymask, mymask); + atomic_store_rel_int(&lazywait, 1); +} + +static void +pmap_lazyfix_self(u_int mymask) +{ + + if (rcr3() == lazyptd) + load_cr3(PCPU_GET(curpcb)->pcb_cr3); + atomic_clear_int(lazymask, mymask); +} + + +static void +pmap_lazyfix(pmap_t pmap) +{ + u_int mymask = PCPU_GET(cpumask); + u_int mask; + register u_int spins; + + while ((mask = pmap->pm_active) != 0) { + spins = 50000000; + mask = mask & -mask; /* Find least significant set bit */ + mtx_lock_spin(&lazypmap_lock); +#ifdef PAE + lazyptd = vtophys(pmap->pm_pdpt); +#else + lazyptd = vtophys(pmap->pm_pdir); +#endif + if (mask == mymask) { + lazymask = &pmap->pm_active; + pmap_lazyfix_self(mymask); + } else { + atomic_store_rel_int((u_int *)&lazymask, + (u_int)&pmap->pm_active); + atomic_store_rel_int(&lazywait, 0); + ipi_selected(mask, IPI_LAZYPMAP); + while (lazywait == 0) { + ia32_pause(); + if (--spins == 0) + break; + } + } + mtx_unlock_spin(&lazypmap_lock); + if (spins == 0) + printf("pmap_lazyfix: spun for 50000000\n"); + } +} + +#else /* SMP */ + +/* + * Cleaning up on uniprocessor is easy. For various reasons, we're + * unlikely to have to even execute this code, including the fact + * that the cleanup is deferred until the parent does a wait(2), which + * means that another userland process has run. + */ +static void +pmap_lazyfix(pmap_t pmap) +{ + u_int cr3; + + cr3 = vtophys(pmap->pm_pdir); + if (cr3 == rcr3()) { + load_cr3(PCPU_GET(curpcb)->pcb_cr3); + pmap->pm_active &= ~(PCPU_GET(cpumask)); + } +} +#endif /* SMP */ + /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. @@ -1261,6 +1471,7 @@ ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); + pmap_lazyfix(pmap); mtx_lock_spin(&allpmaps_lock); LIST_REMOVE(pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); @@ -2899,12 +3110,19 @@ pmap_activate(struct thread *td) { struct proc *p = td->td_proc; - pmap_t pmap; + pmap_t pmap, oldpmap; u_int64_t cr3; critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); + oldpmap = PCPU_GET(curpmap); +#ifdef SMP + atomic_clear_long(&oldpmap->pm_active, PCPU_GET(cpumask)); + atomic_set_long(&pmap->pm_active, PCPU_GET(cpumask)); +#else + oldpmap->pm_active &= ~PCPU_GET(cpumask); pmap->pm_active |= PCPU_GET(cpumask); +#endif cr3 = vtophys(pmap->pm_pml4); /* XXXKSE this is wrong. * pmap_activate is for the current thread on the current cpu ==== //depot/projects/hammer/sys/amd64/include/smp.h#7 (text+ko) ==== @@ -19,20 +19,6 @@ #ifndef LOCORE -/* - * For sending values to POST displays. - * XXX FIXME: where does this really belong, isa.h/isa.c perhaps? - */ -extern int current_postcode; /** XXX currently in mp_machdep.c */ -#define POSTCODE(X) current_postcode = (X), \ - outb(0x80, current_postcode) -#define POSTCODE_LO(X) current_postcode &= 0xf0, \ - current_postcode |= ((X) & 0x0f), \ - outb(0x80, current_postcode) -#define POSTCODE_HI(X) current_postcode &= 0x0f, \ - current_postcode |= (((X) << 4) & 0xf0), \ - outb(0x80, current_postcode) - #include <sys/bus.h> #include <machine/frame.h> #include <machine/intr_machdep.h> ==== //depot/projects/hammer/sys/amd64/include/smptests.h#2 (text+ko) ==== @@ -24,99 +24,3 @@ * * $FreeBSD: src/sys/i386/include/smptests.h,v 1.44 2003/01/23 01:04:27 peter Exp $ */ - -#ifndef _MACHINE_SMPTESTS_H_ -#define _MACHINE_SMPTESTS_H_ - - -/* - * Various 'tests in progress' and configuration parameters. - */ - -/* - * These defines enable critical region locking of areas that were - * protected via cli/sti in the UP kernel. - * - * COMLOCK protects the sio/cy drivers. - * known to be incomplete: - * joystick lkm - * ? - */ -#define USE_COMLOCK - - -/* - * Send CPUSTOP IPI for stop/restart of other CPUs on DDB break. -#define VERBOSE_CPUSTOP_ON_DDBBREAK - */ -#define CPUSTOP_ON_DDBBREAK - -/* - * Misc. counters. - * -#define COUNT_XINVLTLB_HITS - */ - -/* - * Address of POST hardware port. - * Defining this enables POSTCODE macros. - * -#define POST_ADDR 0x80 - */ - - -/* - * POST hardware macros. - */ -#ifdef POST_ADDR -#define ASMPOSTCODE_INC \ - pushl %eax ; \ - movl _current_postcode, %eax ; \ - incl %eax ; \ - andl $0xff, %eax ; \ - movl %eax, _current_postcode ; \ - outb %al, $POST_ADDR ; \ - popl %eax - -/* - * Overwrite the current_postcode value. - */ -#define ASMPOSTCODE(X) \ - pushl %eax ; \ - movl $X, %eax ; \ - movl %eax, _current_postcode ; \ - outb %al, $POST_ADDR ; \ - popl %eax - -/* - * Overwrite the current_postcode low nibble. - */ -#define ASMPOSTCODE_LO(X) \ - pushl %eax ; \ - movl _current_postcode, %eax ; \ - andl $0xf0, %eax ; \ - orl $X, %eax ; \ - movl %eax, _current_postcode ; \ - outb %al, $POST_ADDR ; \ - popl %eax - -/* - * Overwrite the current_postcode high nibble. - */ -#define ASMPOSTCODE_HI(X) \ - pushl %eax ; \ - movl _current_postcode, %eax ; \ - andl $0x0f, %eax ; \ - orl $(X<<4), %eax ; \ - movl %eax, _current_postcode ; \ - outb %al, $POST_ADDR ; \ - popl %eax -#else -#define ASMPOSTCODE_INC -#define ASMPOSTCODE(X) -#define ASMPOSTCODE_LO(X) -#define ASMPOSTCODE_HI(X) -#endif /* POST_ADDR */ - - -#endif /* _MACHINE_SMPTESTS_H_ */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200309112316.h8BNGs7A053776>