Date: Tue, 13 May 2003 23:59:33 -0700 (PDT) From: Peter Wemm <peter@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 31143 for review Message-ID: <200305140659.h4E6xXsR090608@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=31143 Change 31143 by peter@peter_hammer on 2003/05/13 23:59:13 reimplement the lazy critical section masking stuff. Affected files ... .. //depot/projects/hammer/sys/amd64/amd64/critical.c#7 edit .. //depot/projects/hammer/sys/amd64/amd64/exception.S#14 edit .. //depot/projects/hammer/sys/amd64/amd64/genassym.c#14 edit .. //depot/projects/hammer/sys/amd64/include/critical.h#6 edit .. //depot/projects/hammer/sys/amd64/include/pcpu.h#6 edit .. //depot/projects/hammer/sys/amd64/include/proc.h#6 edit .. //depot/projects/hammer/sys/amd64/isa/icu_vector.S#4 edit .. //depot/projects/hammer/sys/amd64/isa/intr_machdep.c#6 edit Differences ... ==== //depot/projects/hammer/sys/amd64/amd64/critical.c#7 (text+ko) ==== @@ -19,24 +19,148 @@ #include <machine/critical.h> /* - * cpu_critical_fork_exit() - cleanup after fork + * XXX this mess to get sched_ithd() and call_fast_unpend() + */ +#include <sys/bus.h> +#include <machine/frame.h> +#include <machine/segments.h> +#include <amd64/isa/icu.h> +#include <amd64/isa/intr_machdep.h> + +void amd64_unpend(void); /* NOTE: not static, called from assembly */ + +/* + * cpu_unpend() - called from critical_exit() inline after quick + * interrupt-pending check. */ void -cpu_critical_fork_exit(void) +cpu_unpend(void) { + register_t rflags; struct thread *td; td = curthread; - td->td_critnest = 1; - td->td_md.md_savecrit = read_rflags() | PSL_I; + rflags = intr_disable(); + if (PCPU_GET(int_pending)) { + ++td->td_intr_nesting_level; + amd64_unpend(); + --td->td_intr_nesting_level; + } + intr_restore(rflags); +} + +/* + * cpu_critical_fork_exit() - cleanup after fork + * + * For amd64 we do not have to do anything, td_critnest is + * handled by the fork trampoline code. + */ +void +cpu_critical_fork_exit(void) +{ } /* * cpu_thread_link() - thread linkup, initialize machine-dependant fields + * + * There are currently no machine-dependant fields that require + * initialization. */ void cpu_thread_link(struct thread *td) { +} + +/* + * Called from cpu_unpend or called from the assembly vector code + * to process any interrupts which may have occured while we were in + * a critical section. + * + * - interrupts must be disabled + * - td_critnest must be 0 + * - td_intr_nesting_level must be incremented by the caller + * + * NOT STATIC (called from assembly) + */ +static __inline u_int +bsfq(u_long mask) +{ + u_long result; + + __asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask)); + return (result); +} + +void +amd64_unpend(void) +{ + struct clockframe frame; + + frame.cf_cs = SEL_KPL; + frame.cf_rip = (register_t)amd64_unpend; + frame.cf_rflags = PSL_KERNEL; + KASSERT(curthread->td_critnest == 0, ("unpend critnest != 0")); + KASSERT((read_rflags() & PSL_I) == 0, ("unpend interrupts enabled1")); + curthread->td_critnest = 1; + for (;;) { + u_int64_t mask; + int irq; + + /* + * Fast interrupts have priority + */ + if ((mask = PCPU_GET(fpending)) != 0) { + irq = bsfq(mask); + PCPU_SET(fpending, mask & ~(1ul << irq)); + call_fast_unpend(irq); + KASSERT((read_rflags() & PSL_I) == 0, + ("unpend interrupts enabled2 %d", irq)); + continue; + } - td->td_md.md_savecrit = 0; + /* + * Threaded interrupts come next + */ + if ((mask = PCPU_GET(ipending)) != 0) { + irq = bsfq(mask); + PCPU_SET(ipending, mask & ~(1ul << irq)); + sched_ithd((void *)(uintptr_t)irq); + KASSERT((read_rflags() & PSL_I) == 0, + ("unpend interrupts enabled3 %d", irq)); + continue; + } + + /* + * Software interrupts and delayed IPIs are last + * + * XXX give the bits #defined names. see also + * isa/xxx_vector.s + */ + if ((mask = PCPU_GET(spending)) != 0) { + irq = bsfq(mask); + PCPU_SET(spending, mask & ~(1ul << irq)); + switch(irq) { + case 0: /* bit 0 - hardclock */ + hardclock_process(&frame); + break; + case 1: /* bit 1 - statclock */ + if (profprocs != 0) + profclock(&frame); + if (pscnt == psdiv) + statclock(&frame); + break; + } + KASSERT((read_rflags() & PSL_I) == 0, + ("unpend interrupts enabled4 %d", irq)); + continue; + } + break; + } + /* + * Interrupts are still disabled, we can safely clear int_pending + * and td_critnest. + */ + KASSERT((read_rflags() & PSL_I) == 0, ("unpend interrupts enabled5")); + PCPU_SET(int_pending, 0); + curthread->td_critnest = 0; } ==== //depot/projects/hammer/sys/amd64/amd64/exception.S#14 (text+ko) ==== @@ -297,6 +297,9 @@ movq %r12, %rdi /* function */ movq %rbx, %rsi /* arg1 */ movq %rsp, %rdx /* trapframe pointer */ + movq PCPU(CURTHREAD),%rbx /* setup critnest */ + movl $1,TD_CRITNEST(%rbx) + sti call fork_exit MEXITCOUNT jmp doreti /* Handle any ASTs */ ==== //depot/projects/hammer/sys/amd64/amd64/genassym.c#14 (text+ko) ==== @@ -181,6 +181,10 @@ ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb)); ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); ASSYM(PC_SCRATCH_RSP, offsetof(struct pcpu, pc_scratch_rsp)); +ASSYM(PC_INT_PENDING, offsetof(struct pcpu, pc_int_pending)); +ASSYM(PC_IPENDING, offsetof(struct pcpu, pc_ipending)); +ASSYM(PC_FPENDING, offsetof(struct pcpu, pc_fpending)); +ASSYM(PC_SPENDING, offsetof(struct pcpu, pc_spending)); ASSYM(KCSEL, GSEL(GCODE_SEL, SEL_KPL)); ASSYM(KDSEL, GSEL(GDATA_SEL, SEL_KPL)); ==== //depot/projects/hammer/sys/amd64/include/critical.h#6 (text+ko) ==== @@ -23,6 +23,7 @@ /* * Prototypes - see <arch>/<arch>/critical.c */ +void cpu_unpend(void); void cpu_critical_fork_exit(void); void cpu_thread_link(struct thread *td); @@ -33,15 +34,12 @@ * * This routine is called from critical_enter() on the 0->1 transition * of td_critnest, prior to it being incremented to 1. + * + * If new-style critical section handling we do not have to do anything. + * However, as a side effect any interrupts occuring while td_critnest + * is non-zero will be deferred. */ -static __inline void -cpu_critical_enter(void) -{ - struct thread *td; - - td = curthread; - td->td_md.md_savecrit = intr_disable(); -} +#define cpu_critical_enter() /* * cpu_critical_exit: @@ -49,14 +47,27 @@ * This routine is called from critical_exit() on a 1->0 transition * of td_critnest, after it has been decremented to 0. We are * exiting the last critical section. + * + * Note that the td->critnest (1->0) transition interrupt race against + * our int_pending/unpend() check below is handled by the interrupt + * code for us, so we do not have to do anything fancy. */ static __inline void cpu_critical_exit(void) { - struct thread *td; - - td = curthread; - intr_restore(td->td_md.md_savecrit); + /* + * We may have to schedule pending interrupts. Create + * conditions similar to an interrupt context and call + * unpend(). + * + * note: we do this even if we are in an interrupt + * nesting level. Deep nesting is protected by + * critical_*() and if we conditionalized it then we + * would have to check int_pending again whenever + * we decrement td_intr_nesting_level to 0. + */ + if (PCPU_GET(int_pending)) + cpu_unpend(); } #else /* !__GNUC__ */ ==== //depot/projects/hammer/sys/amd64/include/pcpu.h#6 (text+ko) ==== @@ -40,7 +40,12 @@ */ #define PCPU_MD_FIELDS \ struct pcpu *pc_prvspace; /* Self-reference */ \ - register_t pc_scratch_rsp; /* User %rsp in syscall */ + register_t pc_scratch_rsp; /* User %rsp in syscall */ \ + u_int64_t pc_int_pending; /* master int pending flag */ \ + u_int64_t pc_ipending; /* pending slow interrupts */ \ + u_int64_t pc_fpending; /* pending fast interrupts */ \ + u_int64_t pc_spending /* pending soft interrupts */ + #if defined(lint) ==== //depot/projects/hammer/sys/amd64/include/proc.h#6 (text+ko) ==== @@ -41,7 +41,7 @@ * Machine-dependent part of the proc structure for AMD64. */ struct mdthread { - register_t md_savecrit; + int __dummy__; }; struct mdproc { ==== //depot/projects/hammer/sys/amd64/isa/icu_vector.S#4 (text+ko) ==== @@ -4,6 +4,7 @@ */ #define IRQ_BIT(irq_num) (1 << ((irq_num) % 8)) +#define IRQ_LBIT(irq_num) (1 << (irq_num)) #define IRQ_BYTE(irq_num) ((irq_num) >> 3) #define ENABLE_ICU1 \ @@ -15,12 +16,11 @@ outb %al,$IO_ICU2 ; /* but do second icu first ... */ \ outb %al,$IO_ICU1 /* ... then first icu */ - /* * Macros for interrupt interrupt entry, call to handler, and exit. */ -#define FAST_INTR(irq_num, vec_name, enable_icus) \ +#define FAST_INTR(irq_num, vec_name, icu, enable_icus) \ .text ; \ SUPERALIGN_TEXT ; \ IDTVEC(vec_name) ; \ @@ -44,8 +44,19 @@ movq %r14,TF_R14(%rsp) ; \ movq %r15,TF_R15(%rsp) ; \ FAKE_MCOUNT((12)*4(%rsp)) ; \ - call critical_enter ; \ movq PCPU(CURTHREAD),%rbx ; \ + cmpl $0,TD_CRITNEST(%rbx) ; \ + je 1f ; \ + movq $1,PCPU(INT_PENDING) ; \ + orq $IRQ_LBIT(irq_num),PCPU(FPENDING) ; \ + movb imen + IRQ_BYTE(irq_num),%al ; \ + orb $IRQ_BIT(irq_num),%al ; \ + movb %al,imen + IRQ_BYTE(irq_num) ; \ + outb %al,$icu+ICU_IMR_OFFSET ; \ + enable_icus ; \ + jmp 10f ; \ +1: ; \ + incl TD_CRITNEST(%rbx) ; \ incl TD_INTR_NESTING_LEVEL(%rbx) ; \ movq intr_unit + (irq_num) * 8, %rdi ; \ call *intr_handler + (irq_num) * 8 ; /* do the work ASAP */ \ @@ -53,11 +64,47 @@ incl cnt+V_INTR ; /* book-keeping can wait */ \ movq intr_countp + (irq_num) * 8,%rax ; \ incq (%rax) ; \ + decl TD_CRITNEST(%rbx) ; \ + cmpq $0,PCPU(INT_PENDING) ; \ + je 2f ; \ + call amd64_unpend ; \ +2: ; \ decl TD_INTR_NESTING_LEVEL(%rbx) ; \ - call critical_exit ; \ +10: ; \ MEXITCOUNT ; \ jmp doreti +/* + * Restart a fast interrupt that was held up by a critical section. + * This routine is called from unpend(). unpend() ensures we are + * in a critical section and deals with the interrupt nesting level + * for us. If we previously masked the irq, we have to unmask it. + * + * We have a choice. We can regenerate the irq using the 'int' + * instruction or we can create a dummy frame and call the interrupt + * handler directly. I've chosen to use the dummy-frame method. + */ +#define FAST_UNPEND(irq_num, vec_name, icu) \ + .text ; \ + SUPERALIGN_TEXT ; \ +IDTVEC(vec_name) ; \ + pushfq ; /* rflags */ \ + mov %cs,%ax ; \ + pushq %rax ; /* cs */ \ + pushq 24(%rsp) ; /* original caller rip */ \ + subq $TF_RIP,%rsp ; /* skip rest including tf_err etc */ \ + movq intr_unit + (irq_num) * 8, %rdi ; \ + call *intr_handler + (irq_num) * 8 ; /* do the work ASAP */ \ + incl cnt+V_INTR ; /* book-keeping can wait */ \ + movq intr_countp + (irq_num) * 8,%rax ; \ + incq (%rax) ; \ + movb imen + IRQ_BYTE(irq_num),%al ; \ + andb $~IRQ_BIT(irq_num),%al ; \ + movb %al,imen + IRQ_BYTE(irq_num) ; \ + outb %al,$icu+ICU_IMR_OFFSET ; \ + addq $TF_RSP,%rsp ; /* dump frame */ \ + ret + /* * Slow, threaded interrupts. * @@ -75,7 +122,8 @@ testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \ jz 1f ; /* Yes, dont swapgs again */ \ swapgs ; \ -1: movq %rdi,TF_RDI(%rsp) ; \ +1: ; \ + movq %rdi,TF_RDI(%rsp) ; \ movq %rsi,TF_RSI(%rsp) ; \ movq %rdx,TF_RDX(%rsp) ; \ movq %rcx,TF_RCX(%rsp) ; \ @@ -97,33 +145,44 @@ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ movq PCPU(CURTHREAD),%rbx ; \ + cmpl $0,TD_CRITNEST(%rbx) ; \ + je 1f ; \ + movq $1,PCPU(INT_PENDING) ; \ + orq $IRQ_LBIT(irq_num),PCPU(IPENDING) ; \ + jmp 10f ; \ +1: ; \ incl TD_INTR_NESTING_LEVEL(%rbx) ; \ FAKE_MCOUNT(13*4(%rsp)) ; /* XXX late to avoid double count */ \ + cmpq $0,PCPU(INT_PENDING) ; \ + je 9f ; \ + call amd64_unpend ; \ +9: ; \ movq $irq_num, %rdi; /* pass the IRQ */ \ call sched_ithd ; \ decl TD_INTR_NESTING_LEVEL(%rbx) ; \ +10: ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* doreti, but it's probably better to use less cache. */ \ jmp doreti MCOUNT_LABEL(bintr) - FAST_INTR(0,fastintr0, ENABLE_ICU1) - FAST_INTR(1,fastintr1, ENABLE_ICU1) - FAST_INTR(2,fastintr2, ENABLE_ICU1) - FAST_INTR(3,fastintr3, ENABLE_ICU1) - FAST_INTR(4,fastintr4, ENABLE_ICU1) - FAST_INTR(5,fastintr5, ENABLE_ICU1) - FAST_INTR(6,fastintr6, ENABLE_ICU1) - FAST_INTR(7,fastintr7, ENABLE_ICU1) - FAST_INTR(8,fastintr8, ENABLE_ICU1_AND_2) - FAST_INTR(9,fastintr9, ENABLE_ICU1_AND_2) - FAST_INTR(10,fastintr10, ENABLE_ICU1_AND_2) - FAST_INTR(11,fastintr11, ENABLE_ICU1_AND_2) - FAST_INTR(12,fastintr12, ENABLE_ICU1_AND_2) - FAST_INTR(13,fastintr13, ENABLE_ICU1_AND_2) - FAST_INTR(14,fastintr14, ENABLE_ICU1_AND_2) - FAST_INTR(15,fastintr15, ENABLE_ICU1_AND_2) + FAST_INTR(0,fastintr0, IO_ICU1, ENABLE_ICU1) + FAST_INTR(1,fastintr1, IO_ICU1, ENABLE_ICU1) + FAST_INTR(2,fastintr2, IO_ICU1, ENABLE_ICU1) + FAST_INTR(3,fastintr3, IO_ICU1, ENABLE_ICU1) + FAST_INTR(4,fastintr4, IO_ICU1, ENABLE_ICU1) + FAST_INTR(5,fastintr5, IO_ICU1, ENABLE_ICU1) + FAST_INTR(6,fastintr6, IO_ICU1, ENABLE_ICU1) + FAST_INTR(7,fastintr7, IO_ICU1, ENABLE_ICU1) + FAST_INTR(8,fastintr8, IO_ICU2, ENABLE_ICU1_AND_2) + FAST_INTR(9,fastintr9, IO_ICU2, ENABLE_ICU1_AND_2) + FAST_INTR(10,fastintr10, IO_ICU2, ENABLE_ICU1_AND_2) + FAST_INTR(11,fastintr11, IO_ICU2, ENABLE_ICU1_AND_2) + FAST_INTR(12,fastintr12, IO_ICU2, ENABLE_ICU1_AND_2) + FAST_INTR(13,fastintr13, IO_ICU2, ENABLE_ICU1_AND_2) + FAST_INTR(14,fastintr14, IO_ICU2, ENABLE_ICU1_AND_2) + FAST_INTR(15,fastintr15, IO_ICU2, ENABLE_ICU1_AND_2) #define CLKINTR_PENDING movl $1,CNAME(clkintr_pending) /* Threaded interrupts */ @@ -144,5 +203,21 @@ INTR(14,intr14, IO_ICU2, ENABLE_ICU1_AND_2,) INTR(15,intr15, IO_ICU2, ENABLE_ICU1_AND_2,) + FAST_UNPEND(0,fastunpend0, IO_ICU1) + FAST_UNPEND(1,fastunpend1, IO_ICU1) + FAST_UNPEND(2,fastunpend2, IO_ICU1) + FAST_UNPEND(3,fastunpend3, IO_ICU1) + FAST_UNPEND(4,fastunpend4, IO_ICU1) + FAST_UNPEND(5,fastunpend5, IO_ICU1) + FAST_UNPEND(6,fastunpend6, IO_ICU1) + FAST_UNPEND(7,fastunpend7, IO_ICU1) + FAST_UNPEND(8,fastunpend8, IO_ICU2) + FAST_UNPEND(9,fastunpend9, IO_ICU2) + FAST_UNPEND(10,fastunpend10, IO_ICU2) + FAST_UNPEND(11,fastunpend11, IO_ICU2) + FAST_UNPEND(12,fastunpend12, IO_ICU2) + FAST_UNPEND(13,fastunpend13, IO_ICU2) + FAST_UNPEND(14,fastunpend14, IO_ICU2) + FAST_UNPEND(15,fastunpend15, IO_ICU2) + MCOUNT_LABEL(eintr) - ==== //depot/projects/hammer/sys/amd64/isa/intr_machdep.c#6 (text+ko) ==== @@ -87,6 +87,17 @@ IDTVEC(fastintr14), IDTVEC(fastintr15), }; +static unpendhand_t *fastunpend[ICU_LEN] = { + IDTVEC(fastunpend0), IDTVEC(fastunpend1), + IDTVEC(fastunpend2), IDTVEC(fastunpend3), + IDTVEC(fastunpend4), IDTVEC(fastunpend5), + IDTVEC(fastunpend6), IDTVEC(fastunpend7), + IDTVEC(fastunpend8), IDTVEC(fastunpend9), + IDTVEC(fastunpend10), IDTVEC(fastunpend11), + IDTVEC(fastunpend12), IDTVEC(fastunpend13), + IDTVEC(fastunpend14), IDTVEC(fastunpend15), +}; + static inthand_t *slowintr[ICU_LEN] = { IDTVEC(intr0), IDTVEC(intr1), IDTVEC(intr2), IDTVEC(intr3), IDTVEC(intr4), IDTVEC(intr5), IDTVEC(intr6), IDTVEC(intr7), @@ -521,3 +532,10 @@ return (ithread_remove_handler(cookie)); } + +void +call_fast_unpend(int irq) +{ + + fastunpend[irq](); +}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200305140659.h4E6xXsR090608>