Date: Fri, 14 Mar 2008 19:53:07 GMT From: John Baldwin <jhb@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 137740 for review Message-ID: <200803141953.m2EJr7PZ020996@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=137740 Change 137740 by jhb@jhb_mutex on 2008/03/14 19:53:02 IFC @137739 Affected files ... .. //depot/projects/smpng/sys/amd64/amd64/intr_machdep.c#29 integrate .. //depot/projects/smpng/sys/amd64/include/intr_machdep.h#17 integrate .. //depot/projects/smpng/sys/arm/arm/intr.c#16 integrate .. //depot/projects/smpng/sys/i386/i386/intr_machdep.c#29 integrate .. //depot/projects/smpng/sys/i386/include/intr_machdep.h#19 integrate .. //depot/projects/smpng/sys/ia64/ia64/interrupt.c#39 integrate .. //depot/projects/smpng/sys/kern/kern_intr.c#90 integrate .. //depot/projects/smpng/sys/powerpc/powerpc/intr_machdep.c#13 integrate .. //depot/projects/smpng/sys/sparc64/sparc64/intr_machdep.c#27 integrate .. //depot/projects/smpng/sys/sun4v/sun4v/intr_machdep.c#7 integrate .. //depot/projects/smpng/sys/sys/interrupt.h#23 integrate Differences ... ==== //depot/projects/smpng/sys/amd64/amd64/intr_machdep.c#29 (text+ko) ==== @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/amd64/amd64/intr_machdep.c,v 1.36 2008/03/14 03:44:41 jhb Exp $ + * $FreeBSD: src/sys/amd64/amd64/intr_machdep.c,v 1.37 2008/03/14 19:41:46 jhb Exp $ */ /* @@ -89,6 +89,7 @@ static void intr_assign_next_cpu(struct intsrc *isrc); #endif +static int intr_assign_cpu(void *arg, u_char cpu); static void intr_init(void *__dummy); static int intr_pic_registered(struct pic *pic); static void intrcnt_setname(const char *name, int index); @@ -146,10 +147,12 @@ #ifdef INTR_FILTER error = intr_event_create(&isrc->is_event, isrc, 0, (mask_fn)isrc->is_pic->pic_enable_source, - intr_eoi_src, intr_disab_eoi_src, "irq%d:", vector); + intr_eoi_src, intr_disab_eoi_src, intr_assign_cpu, "irq%d:", + vector); #else error = intr_event_create(&isrc->is_event, isrc, 0, - (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector); + (mask_fn)isrc->is_pic->pic_enable_source, intr_assign_cpu, "irq%d:", + vector); #endif if (error) return (error); @@ -431,6 +434,28 @@ sx_xunlock(&intr_table_lock); } +static int +intr_assign_cpu(void *arg, u_char cpu) +{ +#ifdef SMP + struct intsrc *isrc; + + /* + * Don't do anything during early boot. We will pick up the + * assignment once the APs are started. + */ + if (assign_cpu && cpu != NOCPU) { + isrc = arg; + sx_xlock(&intr_table_lock); + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); + sx_xunlock(&intr_table_lock); + } + return (0); +#else + return (EOPNOTSUPP); +#endif +} + static void intrcnt_setname(const char *name, int index) { @@ -542,15 +567,11 @@ static void intr_assign_next_cpu(struct intsrc *isrc) { - struct pic *pic; - u_int apic_id; /* * Assign this source to a local APIC in a round-robin fashion. */ - pic = isrc->is_pic; - apic_id = cpu_apic_ids[current_cpu]; - pic->pic_assign_cpu(isrc, apic_id); + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]); do { current_cpu++; if (current_cpu > mp_maxid) @@ -558,6 +579,18 @@ } while (!(intr_cpus & (1 << current_cpu))); } +/* Attempt to bind the specified IRQ to the specified CPU. */ +int +intr_bind(u_int vector, u_char cpu) +{ + struct intsrc *isrc; + + isrc = intr_lookup_source(vector); + if (isrc == NULL) + return (EINVAL); + return (intr_event_bind(isrc->is_event, cpu)); +} + /* * Add a CPU to our mask of valid CPUs that can be destinations of * interrupts. @@ -594,8 +627,18 @@ assign_cpu = 1; for (i = 0; i < NUM_IO_INTS; i++) { isrc = interrupt_sources[i]; - if (isrc != NULL && isrc->is_handlers > 0) - intr_assign_next_cpu(isrc); + if (isrc != NULL && isrc->is_handlers > 0) { + /* + * If this event is already bound to a CPU, + * then assign the source to that CPU instead + * of picking one via round-robin. + */ + if (isrc->is_event->ie_cpu != NOCPU) + isrc->is_pic->pic_assign_cpu(isrc, + cpu_apic_ids[isrc->is_event->ie_cpu]); + else + intr_assign_next_cpu(isrc); + } } sx_xunlock(&intr_table_lock); } ==== //depot/projects/smpng/sys/amd64/include/intr_machdep.h#17 (text+ko) ==== @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/amd64/include/intr_machdep.h,v 1.18 2007/05/08 21:29:13 jhb Exp $ + * $FreeBSD: src/sys/amd64/include/intr_machdep.h,v 1.19 2008/03/14 19:41:46 jhb Exp $ */ #ifndef __MACHINE_INTR_MACHDEP_H__ @@ -137,6 +137,9 @@ int intr_add_handler(const char *name, int vector, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep); +#ifdef SMP +int intr_bind(u_int vector, u_char cpu); +#endif int intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol); void intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame); ==== //depot/projects/smpng/sys/arm/arm/intr.c#16 (text+ko) ==== @@ -37,7 +37,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/arm/arm/intr.c,v 1.17 2007/07/27 14:26:42 cognet Exp $"); +__FBSDID("$FreeBSD: src/sys/arm/arm/intr.c,v 1.18 2008/03/14 19:41:46 jhb Exp $"); #include <sys/param.h> #include <sys/systm.h> #include <sys/syslog.h> @@ -92,10 +92,10 @@ #ifdef INTR_FILTER error = intr_event_create(&event, (void *)irq, 0, (void (*)(void *))arm_unmask_irq, intr_eoi_src, - intr_disab_eoi_src, "intr%d:", irq); + intr_disab_eoi_src, NULL, "intr%d:", irq); #else error = intr_event_create(&event, (void *)irq, 0, - (void (*)(void *))arm_unmask_irq, "intr%d:", irq); + (void (*)(void *))arm_unmask_irq, NULL, "intr%d:", irq); #endif if (error) return; ==== //depot/projects/smpng/sys/i386/i386/intr_machdep.c#29 (text+ko) ==== @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/i386/i386/intr_machdep.c,v 1.31 2008/03/14 03:44:42 jhb Exp $ + * $FreeBSD: src/sys/i386/i386/intr_machdep.c,v 1.32 2008/03/14 19:41:46 jhb Exp $ */ /* @@ -80,6 +80,7 @@ static void intr_assign_next_cpu(struct intsrc *isrc); #endif +static int intr_assign_cpu(void *arg, u_char cpu); static void intr_init(void *__dummy); static int intr_pic_registered(struct pic *pic); static void intrcnt_setname(const char *name, int index); @@ -137,10 +138,12 @@ #ifdef INTR_FILTER error = intr_event_create(&isrc->is_event, isrc, 0, (mask_fn)isrc->is_pic->pic_enable_source, - intr_eoi_src, intr_disab_eoi_src, "irq%d:", vector); + intr_eoi_src, intr_disab_eoi_src, intr_assign_cpu, "irq%d:", + vector); #else error = intr_event_create(&isrc->is_event, isrc, 0, - (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector); + (mask_fn)isrc->is_pic->pic_enable_source, intr_assign_cpu, "irq%d:", + vector); #endif if (error) return (error); @@ -429,6 +432,28 @@ sx_xunlock(&intr_table_lock); } +static int +intr_assign_cpu(void *arg, u_char cpu) +{ +#ifdef SMP + struct intsrc *isrc; + + /* + * Don't do anything during early boot. We will pick up the + * assignment once the APs are started. + */ + if (assign_cpu && cpu != NOCPU) { + isrc = arg; + sx_xlock(&intr_table_lock); + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); + sx_xunlock(&intr_table_lock); + } + return (0); +#else + return (EOPNOTSUPP); +#endif +} + static void intrcnt_setname(const char *name, int index) { @@ -518,15 +543,11 @@ static void intr_assign_next_cpu(struct intsrc *isrc) { - struct pic *pic; - u_int apic_id; /* * Assign this source to a local APIC in a round-robin fashion. */ - pic = isrc->is_pic; - apic_id = cpu_apic_ids[current_cpu]; - pic->pic_assign_cpu(isrc, apic_id); + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]); do { current_cpu++; if (current_cpu > mp_maxid) @@ -534,6 +555,18 @@ } while (!(intr_cpus & (1 << current_cpu))); } +/* Attempt to bind the specified IRQ to the specified CPU. */ +int +intr_bind(u_int vector, u_char cpu) +{ + struct intsrc *isrc; + + isrc = intr_lookup_source(vector); + if (isrc == NULL) + return (EINVAL); + return (intr_event_bind(isrc->is_event, cpu)); +} + /* * Add a CPU to our mask of valid CPUs that can be destinations of * interrupts. @@ -570,8 +603,18 @@ assign_cpu = 1; for (i = 0; i < NUM_IO_INTS; i++) { isrc = interrupt_sources[i]; - if (isrc != NULL && isrc->is_handlers > 0) - intr_assign_next_cpu(isrc); + if (isrc != NULL && isrc->is_handlers > 0) { + /* + * If this event is already bound to a CPU, + * then assign the source to that CPU instead + * of picking one via round-robin. + */ + if (isrc->is_event->ie_cpu != NOCPU) + isrc->is_pic->pic_assign_cpu(isrc, + cpu_apic_ids[isrc->is_event->ie_cpu]); + else + intr_assign_next_cpu(isrc); + } } sx_xunlock(&intr_table_lock); } ==== //depot/projects/smpng/sys/i386/include/intr_machdep.h#19 (text+ko) ==== @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/sys/i386/include/intr_machdep.h,v 1.20 2007/05/08 21:29:14 jhb Exp $ + * $FreeBSD: src/sys/i386/include/intr_machdep.h,v 1.21 2008/03/14 19:41:46 jhb Exp $ */ #ifndef __MACHINE_INTR_MACHDEP_H__ @@ -133,6 +133,9 @@ #endif int intr_add_handler(const char *name, int vector, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep); +#ifdef SMP +int intr_bind(u_int vector, u_char cpu); +#endif int intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol); void intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame); ==== //depot/projects/smpng/sys/ia64/ia64/interrupt.c#39 (text+ko) ==== @@ -1,4 +1,4 @@ -/* $FreeBSD: src/sys/ia64/ia64/interrupt.c,v 1.62 2007/11/21 04:03:50 scottl Exp $ */ +/* $FreeBSD: src/sys/ia64/ia64/interrupt.c,v 1.63 2008/03/14 19:41:47 jhb Exp $ */ /* $NetBSD: interrupt.c,v 1.23 1998/02/24 07:38:01 thorpej Exp $ */ /*- @@ -344,7 +344,7 @@ #ifdef INTR_FILTER ia64_intr_eoi, ia64_intr_mask, #endif - "irq%u:", irq); + NULL, "irq%u:", irq); if (error) { free(i, M_DEVBUF); return (error); ==== //depot/projects/smpng/sys/kern/kern_intr.c#90 (text+ko) ==== @@ -25,7 +25,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/kern/kern_intr.c,v 1.154 2008/03/12 10:11:59 jeff Exp $"); +__FBSDID("$FreeBSD: src/sys/kern/kern_intr.c,v 1.155 2008/03/14 19:41:47 jhb Exp $"); #include "opt_ddb.h" @@ -46,6 +46,7 @@ #include <sys/random.h> #include <sys/resourcevar.h> #include <sys/sched.h> +#include <sys/smp.h> #include <sys/sysctl.h> #include <sys/unistd.h> #include <sys/vmmeter.h> @@ -240,7 +241,8 @@ #ifndef INTR_FILTER int intr_event_create(struct intr_event **event, void *source, int flags, - void (*enable)(void *), const char *fmt, ...) + void (*enable)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, + ...) { struct intr_event *ie; va_list ap; @@ -251,7 +253,9 @@ ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); ie->ie_source = source; ie->ie_enable = enable; + ie->ie_assign_cpu = assign_cpu; ie->ie_flags = flags; + ie->ie_cpu = NOCPU; TAILQ_INIT(&ie->ie_handlers); mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); @@ -271,7 +275,7 @@ int intr_event_create(struct intr_event **event, void *source, int flags, void (*enable)(void *), void (*eoi)(void *), void (*disab)(void *), - const char *fmt, ...) + int (*assign_cpu)(void *, u_char), const char *fmt, ...) { struct intr_event *ie; va_list ap; @@ -282,9 +286,11 @@ ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); ie->ie_source = source; ie->ie_enable = enable; + ie->ie_assign_cpu = assign_cpu; ie->ie_eoi = eoi; ie->ie_disab = disab; ie->ie_flags = flags; + ie->ie_cpu = NOCPU; TAILQ_INIT(&ie->ie_handlers); mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); @@ -302,6 +308,52 @@ } #endif +/* + * Bind an interrupt event to the specified CPU. Note that not all + * platforms support binding an interrupt to a CPU. For those + * platforms this request will fail. For supported platforms, any + * associated ithreads as well as the primary interrupt context will + * be bound to the specificed CPU. Using a cpu id of NOCPU unbinds + * the interrupt event. + */ +int +intr_event_bind(struct intr_event *ie, u_char cpu) +{ + struct thread *td; + int error; + + /* Need a CPU to bind to. */ + if (cpu != NOCPU && CPU_ABSENT(cpu)) + return (EINVAL); + + if (ie->ie_assign_cpu == NULL) + return (EOPNOTSUPP); + + /* Don't allow a bind request if the interrupt is already bound. */ + mtx_lock(&ie->ie_lock); + if (ie->ie_cpu != NOCPU && cpu != NOCPU) { + mtx_unlock(&ie->ie_lock); + return (EBUSY); + } + mtx_unlock(&ie->ie_lock); + + error = ie->ie_assign_cpu(ie->ie_source, cpu); + if (error) + return (error); + mtx_lock(&ie->ie_lock); + if (ie->ie_thread != NULL) + td = ie->ie_thread->it_thread; + else + td = NULL; + if (td != NULL) + thread_lock(td); + ie->ie_cpu = cpu; + if (td != NULL) + thread_unlock(td); + mtx_unlock(&ie->ie_lock); + return (0); +} + int intr_event_destroy(struct intr_event *ie) { @@ -893,10 +945,10 @@ } else { #ifdef INTR_FILTER error = intr_event_create(&ie, NULL, IE_SOFT, - NULL, NULL, NULL, "swi%d:", pri); + NULL, NULL, NULL, NULL, "swi%d:", pri); #else error = intr_event_create(&ie, NULL, IE_SOFT, - NULL, "swi%d:", pri); + NULL, NULL, "swi%d:", pri); #endif if (error) return (error); @@ -1078,6 +1130,7 @@ struct intr_event *ie; struct thread *td; struct proc *p; + u_char cpu; td = curthread; p = td->td_proc; @@ -1086,6 +1139,7 @@ ("%s: ithread and proc linkage out of sync", __func__)); ie = ithd->it_event; ie->ie_count = 0; + cpu = NOCPU; /* * As long as we have interrupts outstanding, go through the @@ -1131,6 +1185,21 @@ ie->ie_count = 0; mi_switch(SW_VOL, NULL); } + +#ifdef SMP + /* + * Ensure we are bound to the correct CPU. We can't + * move ithreads until SMP is running however, so just + * leave interrupts on the boor CPU during boot. + */ + if (ie->ie_cpu != cpu && smp_started) { + cpu = ie->ie_cpu; + if (cpu == NOCPU) + sched_unbind(td); + else + sched_bind(td, cpu); + } +#endif thread_unlock(td); } } @@ -1147,6 +1216,7 @@ struct thread *td; struct proc *p; int priv; + u_char cpu; td = curthread; p = td->td_proc; @@ -1157,6 +1227,7 @@ ("%s: ithread and proc linkage out of sync", __func__)); ie = ithd->it_event; ie->ie_count = 0; + cpu = NOCPU; /* * As long as we have interrupts outstanding, go through the @@ -1205,6 +1276,21 @@ ie->ie_count = 0; mi_switch(SW_VOL, NULL); } + +#ifdef SMP + /* + * Ensure we are bound to the correct CPU. We can't + * move ithreads until SMP is running however, so just + * leave interrupts on the boor CPU during boot. + */ + if (!priv && ie->ie_cpu != cpu && smp_started) { + cpu = ie->ie_cpu; + if (cpu == NOCPU) + sched_unbind(td); + else + sched_bind(td, cpu); + } +#endif thread_unlock(td); } } @@ -1440,6 +1526,8 @@ db_printf("(pid %d)", it->it_thread->td_proc->p_pid); else db_printf("(no thread)"); + if (ie->ie_cpu != NOCPU) + db_printf(" (CPU %d)", ie->ie_cpu); if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 || (it != NULL && it->it_need)) { db_printf(" {"); ==== //depot/projects/smpng/sys/powerpc/powerpc/intr_machdep.c#13 (text+ko) ==== @@ -57,7 +57,7 @@ * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 * form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20 * - * $FreeBSD: src/sys/powerpc/powerpc/intr_machdep.c,v 1.17 2008/03/11 19:58:52 marcel Exp $ + * $FreeBSD: src/sys/powerpc/powerpc/intr_machdep.c,v 1.18 2008/03/14 19:41:47 jhb Exp $ */ #include <sys/param.h> @@ -233,7 +233,7 @@ #ifdef INTR_FILTER powerpc_intr_eoi, powerpc_intr_mask, #endif - "irq%u:", irq); + NULL, "irq%u:", irq); if (error) return (error); ==== //depot/projects/smpng/sys/sparc64/sparc64/intr_machdep.c#27 (text+ko) ==== @@ -59,7 +59,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/sparc64/sparc64/intr_machdep.c,v 1.28 2007/11/21 04:03:51 scottl Exp $"); +__FBSDID("$FreeBSD: src/sys/sparc64/sparc64/intr_machdep.c,v 1.29 2008/03/14 19:41:47 jhb Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -331,9 +331,9 @@ */ error = intr_event_create(&ie, iv, 0, intr_enable_eoi, #ifdef INTR_FILTER - ic->ic_eoi, ic->ic_disable, "vec%d:", vec); + ic->ic_eoi, ic->ic_disable, NULL, "vec%d:", vec); #else - "vec%d:", vec); + NULL, "vec%d:", vec); #endif if (error != 0) return (error); ==== //depot/projects/smpng/sys/sun4v/sun4v/intr_machdep.c#7 (text+ko) ==== @@ -59,7 +59,7 @@ */ #include <sys/cdefs.h> -__FBSDID("$FreeBSD: src/sys/sun4v/sun4v/intr_machdep.c,v 1.7 2007/05/31 19:25:35 piso Exp $"); +__FBSDID("$FreeBSD: src/sys/sun4v/sun4v/intr_machdep.c,v 1.8 2008/03/14 19:41:47 jhb Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -359,7 +359,7 @@ mtx_unlock_spin(&intr_table_lock); if (ie == NULL) { errcode = intr_event_create(&ie, (void *)(intptr_t)vec, 0, NULL, - "vec%d:", vec); + NULL, "vec%d:", vec); if (errcode) return (errcode); mtx_lock_spin(&intr_table_lock); ==== //depot/projects/smpng/sys/sys/interrupt.h#23 (text+ko) ==== @@ -23,7 +23,7 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * - * $FreeBSD: src/sys/sys/interrupt.h,v 1.37 2007/05/06 17:02:50 piso Exp $ + * $FreeBSD: src/sys/sys/interrupt.h,v 1.38 2008/03/14 19:41:48 jhb Exp $ */ #ifndef _SYS_INTERRUPT_H_ @@ -73,6 +73,7 @@ void *ie_source; /* Cookie used by MD code. */ struct intr_thread *ie_thread; /* Thread we are connected to. */ void (*ie_enable)(void *); + int (*ie_assign_cpu)(void *, u_char); #ifdef INTR_FILTER void (*ie_eoi)(void *); void (*ie_disab)(void *); @@ -81,6 +82,7 @@ int ie_count; /* Loop counter. */ int ie_warncnt; /* Rate-check interrupt storm warns. */ struct timeval ie_warntm; + u_char ie_cpu; /* CPU this event is bound to. */ }; /* Interrupt event flags kept in ie_flags. */ @@ -127,15 +129,18 @@ int intr_event_add_handler(struct intr_event *ie, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, void **cookiep); +int intr_event_bind(struct intr_event *ie, u_char cpu); #ifndef INTR_FILTER int intr_event_create(struct intr_event **event, void *source, - int flags, void (*enable)(void *), const char *fmt, ...) - __printflike(5, 6); + int flags, void (*enable)(void *), + int (*assign_cpu)(void *, u_char), const char *fmt, ...) + __printflike(6, 7); #else int intr_event_create(struct intr_event **event, void *source, int flags, void (*enable)(void *), void (*eoi)(void *), - void (*disab)(void *), const char *fmt, ...) - __printflike(7, 8); + void (*disab)(void *), int (*assign_cpu)(void *, u_char), + const char *fmt, ...) + __printflike(8, 9); #endif int intr_event_destroy(struct intr_event *ie); int intr_event_remove_handler(void *cookie);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200803141953.m2EJr7PZ020996>