From owner-p4-projects@FreeBSD.ORG Sun Oct 30 23:46:43 2005 Return-Path: X-Original-To: p4-projects@freebsd.org Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id D5D5A16A421; Sun, 30 Oct 2005 23:46:42 +0000 (GMT) X-Original-To: perforce@freebsd.org Delivered-To: perforce@freebsd.org Received: from mx1.FreeBSD.org (mx1.freebsd.org [216.136.204.125]) by hub.freebsd.org (Postfix) with ESMTP id AC8CA16A41F for ; Sun, 30 Oct 2005 23:46:42 +0000 (GMT) (envelope-from bb+lists.freebsd.perforce@cyrus.watson.org) Received: from repoman.freebsd.org (repoman.freebsd.org [216.136.204.115]) by mx1.FreeBSD.org (Postfix) with ESMTP id 543F743D4C for ; Sun, 30 Oct 2005 23:46:42 +0000 (GMT) (envelope-from bb+lists.freebsd.perforce@cyrus.watson.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.13.1/8.13.1) with ESMTP id j9UNkgCN082641 for ; Sun, 30 Oct 2005 23:46:42 GMT (envelope-from bb+lists.freebsd.perforce@cyrus.watson.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.13.1/8.13.1/Submit) id j9UNkfrK082638 for perforce@freebsd.org; Sun, 30 Oct 2005 23:46:41 GMT (envelope-from bb+lists.freebsd.perforce@cyrus.watson.org) Date: Sun, 30 Oct 2005 23:46:41 GMT Message-Id: <200510302346.j9UNkfrK082638@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to bb+lists.freebsd.perforce@cyrus.watson.org using -f From: Robert Watson To: Perforce Change Reviews Cc: Subject: PERFORCE change 86091 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 30 Oct 2005 23:46:43 -0000 http://perforce.freebsd.org/chv.cgi?CH=86091 Change 86091 by rwatson@rwatson_peppercorn on 2005/10/30 23:45:49 Not entirely pretty hack: allow preemption by the netisr thread to be configurable separate from global preemption. Specifically, if net.isr.preempt=0, pass SWI_NOPREEMPT to swi_sched(). Implement SWI_NOPREEMPT by adding IE_FLAG_NOPREEMPT (don't preempt due to an interrupt event, only wakeup), which in turn is implemented by adding SRQ_NOPREEMPT, a flag to the scheduler run queue calls. The reason to add this is that loopback traffic is seeing a lot of context switches, and avoiding the context switches through avoiding unnecessary preemption may help illustrate if this is a performance problem, and possibly be part of a fix. This may not work, either functionally, or in its overall goal, and is an experiment. Affected files ... .. //depot/projects/netsmp/src/sys/kern/kern_intr.c#4 edit .. //depot/projects/netsmp/src/sys/kern/sched_4bsd.c#2 edit .. //depot/projects/netsmp/src/sys/kern/sched_ule.c#6 edit .. //depot/projects/netsmp/src/sys/net/netisr.c#3 edit .. //depot/projects/netsmp/src/sys/sys/interrupt.h#4 edit .. //depot/projects/netsmp/src/sys/sys/proc.h#5 edit Differences ... ==== //depot/projects/netsmp/src/sys/kern/kern_intr.c#4 (text+ko) ==== @@ -485,7 +485,7 @@ } int -intr_event_schedule_thread(struct intr_event *ie) +intr_event_schedule_thread_flags(struct intr_event *ie, int flags) { struct intr_entropy entropy; struct intr_thread *it; @@ -531,7 +531,8 @@ CTR3(KTR_INTR, "%s: schedule pid %d (%s)", __func__, p->p_pid, p->p_comm); TD_CLR_IWAIT(td); - setrunqueue(td, SRQ_INTR); + setrunqueue(td, SRQ_INTR | + (flags & IE_FLAG_NOPREEMPT ? SRQ_NOPREEMPT : 0)); } else { CTR5(KTR_INTR, "%s: pid %d (%s): it_need %d, state %d", __func__, p->p_pid, p->p_comm, it->it_need, td->td_state); @@ -541,6 +542,13 @@ return (0); } +int +intr_event_schedule_thread(struct intr_event *ie) +{ + + return (intr_event_schedule_thread_flags(ie, 0)); +} + /* * Add a software interrupt handler to a specified event. If a given event * is not specified, then a new event is created. @@ -595,7 +603,8 @@ */ atomic_store_rel_int(&ih->ih_need, 1); if (!(flags & SWI_DELAY)) { - error = intr_event_schedule_thread(ie); + error = intr_event_schedule_thread_flags(ie, + (flags & SWI_NOPREEMPT) ? IE_FLAG_NOPREEMPT : 0); KASSERT(error == 0, ("stray software interrupt")); } } ==== //depot/projects/netsmp/src/sys/kern/sched_4bsd.c#2 (text+ko) ==== @@ -1174,7 +1174,9 @@ } if (!forwarded) { - if ((flags & SRQ_YIELDING) == 0 && maybe_preempt(td)) + if ((flags & SRQ_YIELDING) == 0 && + (flags & SRQ_NOPREEMPT) == 0 && + maybe_preempt(td)) return; else maybe_resched(td); @@ -1214,7 +1216,7 @@ * OURSELF case, we are puting ourself on the run queue * which also only happens when we are about to yield. */ - if((flags & SRQ_YIELDING) == 0) { + if ((flags & SRQ_YIELDING) == 0 && (flags & SRQ_NOPREEMPT) == 0) { if (maybe_preempt(td)) return; } ==== //depot/projects/netsmp/src/sys/kern/sched_ule.c#6 (text+ko) ==== @@ -1767,7 +1767,7 @@ ke = td->td_kse; kg = td->td_ksegrp; canmigrate = 1; - preemptive = !(flags & SRQ_YIELDING); + preemptive = !(flags & SRQ_YIELDING || flags & SRQ_NOPREEMPT); class = PRI_BASE(kg->kg_pri_class); kseq = KSEQ_SELF(); if ((ke->ke_flags & KEF_INTERNAL) == 0) ==== //depot/projects/netsmp/src/sys/net/netisr.c#3 (text+ko) ==== @@ -74,6 +74,8 @@ SYSCTL_INT(_debug, OID_AUTO, mpsafenet, CTLFLAG_RD, &debug_mpsafenet, 0, "Enable/disable MPSAFE network support"); +static int netisr_preempt = 1; /* netisr will preempt when woken. */ + volatile unsigned int netisr; /* scheduling bits for network */ struct netisr { @@ -159,7 +161,8 @@ void legacy_setsoftnet(void) { - swi_sched(net_ih, 0); + + swi_sched(net_ih, netisr_preempt ? 0 : SWI_NOPREEMPT); } void @@ -219,6 +222,9 @@ SYSCTL_INT(_net_isr, OID_AUTO, swi_count, CTLFLAG_RD, &isrstat.isrs_swi_count, 0, ""); +SYSCTL_INT(_net_isr, OID_AUTO, preempt, CTLFLAG_RW, + &netisr_preempt, 0, ""); + /* * Process all packets currently present in a netisr queue. Used to * drain an existing set of packets waiting for processing when we ==== //depot/projects/netsmp/src/sys/sys/interrupt.h#4 (text+ko) ==== @@ -81,8 +81,14 @@ #define IE_ENTROPY 0x000002 /* Interrupt is an entropy source. */ #define IE_ADDING_THREAD 0x000004 /* Currently building an ithread. */ +/* + * Flags passed to intr_event_schedule_thread_flags(). + */ +#define IE_FLAG_NOPREEMPT 0x00000001 /* Schedule; don't preempt. */ + /* Flags to pass to sched_swi. */ #define SWI_DELAY 0x2 +#define SWI_NOPREEMPT 0x4 /* * Software interrupt numbers in priority order. The priority determines @@ -121,6 +127,7 @@ int intr_event_destroy(struct intr_event *ie); int intr_event_remove_handler(void *cookie); int intr_event_schedule_thread(struct intr_event *ie); +int intr_event_schedule_thread_flags(struct intr_event *ie, int flags); int swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler, void *arg, int pri, enum intr_type flags, void **cookiep); ==== //depot/projects/netsmp/src/sys/sys/proc.h#5 (text+ko) ==== @@ -685,6 +685,7 @@ #define SRQ_OURSELF 0x0002 /* It is ourself (from mi_switch). */ #define SRQ_INTR 0x0004 /* It is probably urgent. */ #define SRQ_PREEMPTED 0x0008 /* has been preempted.. be kind */ +#define SRQ_NOPREEMPT 0x0010 /* Caller hints not to preempt. */ /* How values for thread_single(). */ #define SINGLE_NO_EXIT 0