From owner-svn-src-projects@FreeBSD.ORG Tue May 19 20:23:32 2009 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id C1BBA1065670; Tue, 19 May 2009 20:23:32 +0000 (UTC) (envelope-from rwatson@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id AFFBD8FC0C; Tue, 19 May 2009 20:23:32 +0000 (UTC) (envelope-from rwatson@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n4JKNWYw009023; Tue, 19 May 2009 20:23:32 GMT (envelope-from rwatson@svn.freebsd.org) Received: (from rwatson@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n4JKNWRQ009022; Tue, 19 May 2009 20:23:32 GMT (envelope-from rwatson@svn.freebsd.org) Message-Id: <200905192023.n4JKNWRQ009022@svn.freebsd.org> From: Robert Watson Date: Tue, 19 May 2009 20:23:32 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r192408 - projects/pnet/sys/net X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 19 May 2009 20:23:33 -0000 Author: rwatson Date: Tue May 19 20:23:32 2009 New Revision: 192408 URL: http://svn.freebsd.org/changeset/base/192408 Log: Implement a hybrid cross-CPU dispatch model for netisr2, in which a directly dispatching thread is willing to directly dispatch packets normally processed on another CPU if that CPU's netisr thread is idle. Modified: projects/pnet/sys/net/netisr2.c Modified: projects/pnet/sys/net/netisr2.c ============================================================================== --- projects/pnet/sys/net/netisr2.c Tue May 19 20:16:18 2009 (r192407) +++ projects/pnet/sys/net/netisr2.c Tue May 19 20:23:32 2009 (r192408) @@ -107,19 +107,25 @@ static struct rmlock netisr_rmlock; SYSCTL_NODE(_net, OID_AUTO, isr2, CTLFLAG_RW, 0, "netisr2"); /*- - * Three direct dispatch policies are supported: + * Four direct dispatch policies are supported: * * - Always defer: all work is scheduled for a netisr, regardless of context. + * (direct_enable == 0) * * - Always direct: if the executing context allows direct dispatch, always * direct dispatch. + * (direct_enable != 0 && direct_force != 0) * * - Hybrid: if the executing context allows direct dispatch, and we're * running on the CPU the work would be done on, then direct dispatch if it * wouldn't violate ordering constraints on the workstream. + * (direct_enable != 0 && direct_force == 0 && hybridxcpu_enable == 0) + * + * - Hybrid with cross-CPU dispatch: if the executing context allows direct + * dispatch, then direct dispatch if it wouldn't violate ordering + * constraints on the workstream. + * (direct_enable != 0 && direct_force == 0 && hybridxcpu_enable != 0) * - * These policies are captured using two sysctls -- direct_enable allows - * direct dispatch, and direct_force forces direct dispatch if enabled. * Notice that changing the global policy could lead to short periods of * disordered processing, but this is considered acceptable as compared to * the complexity of enforcing ordering during policy changes. @@ -132,6 +138,10 @@ static int netisr_direct_enable = 1; /* SYSCTL_INT(_net_isr2, OID_AUTO, direct_enable, CTLFLAG_RW, &netisr_direct_enable, 0, "Enable direct dispatch"); +static int netisr_hybridxcpu_enable = 1; /* Enable cross-CPU dispatch. */ +SYSCTL_INT(_net_isr2, OID_AUTO, hybridxcpu_enable, CTLFLAG_RW, + &netisr_hybridxcpu_enable, 0, "Enable cross-CPU hybrid direct dispatch."); + /* * Allow the administrator to limit the number of threads (CPUs) to use for * netisr2. Notice that we don't check netisr_maxthreads before creating the @@ -206,6 +216,7 @@ struct netisr_work { */ u_int64_t nw_dispatched; /* Number of direct dispatches. */ u_int64_t nw_hybrid_dispatched; /* "" hybrid dispatches. */ + u_int64_t nw_hybrid_xcpudispatched; /* "" cross-CPU hybrid. */ u_int64_t nw_qdrops; /* "" drops. */ u_int64_t nw_queued; /* "" enqueues. */ u_int64_t nw_handled; /* "" handled in worker. */ @@ -880,7 +891,7 @@ netisr2_dispatch_src(u_int proto, uintpt return (ENOBUFS); } sched_pin(); - if (cpuid != curcpu) + if (!netisr_hybridxcpu_enable && (cpuid != curcpu)) goto queue_fallback; nwsp = &nws[cpuid]; npwp = &nwsp->nws_work[proto]; @@ -915,7 +926,10 @@ netisr2_dispatch_src(u_int proto, uintpt NWS_LOCK(nwsp); nwsp->nws_flags &= ~NWS_DISPATCHING; npwp->nw_handled++; - npwp->nw_hybrid_dispatched++; + if (curcpu == cpuid) + npwp->nw_hybrid_dispatched++; + else + npwp->nw_hybrid_xcpudispatched++; /* * If other work was enqueued by another thread while we were direct @@ -1042,8 +1056,8 @@ DB_SHOW_COMMAND(netisr2, db_show_netisr2 int cpu, first, proto; db_printf("%3s %5s %6s %5s %5s %5s %8s %8s %8s %8s %8s\n", "CPU", - "Pend", "Proto", "Len", "WMark", "Max", "Disp", "HDisp", "Drop", - "Queue", "Handle"); + "Pend", "Proto", "Len", "WMark", "Max", "Disp", "HDisp", + "XHDisp", "Drop", "Queue"); for (cpu = 0; cpu < MAXCPU; cpu++) { nwsp = &nws[cpu]; if (nwsp->nws_intr_event == NULL) @@ -1063,7 +1077,8 @@ DB_SHOW_COMMAND(netisr2, db_show_netisr2 np[proto].np_name, nwp->nw_len, nwp->nw_watermark, nwp->nw_qlimit, nwp->nw_dispatched, nwp->nw_hybrid_dispatched, - nwp->nw_qdrops, nwp->nw_queued, nwp->nw_handled); + nwp->nw_hybrid_xcpudispatched, nwp->nw_qdrops, + nwp->nw_queued); } } }