From owner-svn-src-projects@FreeBSD.ORG Tue Feb 26 15:25:44 2013 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.FreeBSD.org [8.8.178.115]) by hub.freebsd.org (Postfix) with ESMTP id D182B670; Tue, 26 Feb 2013 15:25:44 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) by mx1.freebsd.org (Postfix) with ESMTP id B5518A43; Tue, 26 Feb 2013 15:25:44 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.5/8.14.5) with ESMTP id r1QFPi7x058087; Tue, 26 Feb 2013 15:25:44 GMT (envelope-from mav@svn.freebsd.org) Received: (from mav@localhost) by svn.freebsd.org (8.14.5/8.14.5/Submit) id r1QFPhLt058080; Tue, 26 Feb 2013 15:25:44 GMT (envelope-from mav@svn.freebsd.org) Message-Id: <201302261525.r1QFPhLt058080@svn.freebsd.org> From: Alexander Motin Date: Tue, 26 Feb 2013 15:25:43 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r247319 - in projects/calloutng/sys: kern sys X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 26 Feb 2013 15:25:44 -0000 Author: mav Date: Tue Feb 26 15:25:43 2013 New Revision: 247319 URL: http://svnweb.freebsd.org/changeset/base/247319 Log: Optimize callout_process() to use less variables and less conditions to implement the same logic. Now it fits better into CPU registers, and according to PMC significntly reduces number of resource stalls, reducing consumed by it CPU time during usleep(1) benchmark by 30%. Remove callout_new_inserted variable and call cpu_new_callout() function directly. Since most architectures are using it and there is #define to identify others, there is no point to keep this extra indirection. Reviewed by: davide Modified: projects/calloutng/sys/kern/kern_clocksource.c projects/calloutng/sys/kern/kern_timeout.c projects/calloutng/sys/sys/callout.h projects/calloutng/sys/sys/systm.h Modified: projects/calloutng/sys/kern/kern_clocksource.c ============================================================================== --- projects/calloutng/sys/kern/kern_clocksource.c Tue Feb 26 13:59:39 2013 (r247318) +++ projects/calloutng/sys/kern/kern_clocksource.c Tue Feb 26 15:25:43 2013 (r247319) @@ -73,8 +73,6 @@ static int round_freq(struct eventtimer static sbintime_t getnextcpuevent(int idle); static sbintime_t getnextevent(void); static int handleevents(sbintime_t now, int fake); -static void cpu_new_callout(int cpu, sbintime_t bt, - sbintime_t bt_opt); static struct mtx et_hw_mtx; @@ -99,7 +97,7 @@ static sbintime_t timerperiod; /* Timer static sbintime_t statperiod; /* statclock() events period. */ static sbintime_t profperiod; /* profclock() events period. */ static sbintime_t nexttick; /* Next global timer tick time. */ -static u_int busy = 0; /* Reconfiguration is in progress. */ +static u_int busy = 1; /* Reconfiguration is in progress. */ static int profiling = 0; /* Profiling events enabled. */ static char timername[32]; /* Wanted timer. */ @@ -527,6 +525,8 @@ configtimer(int start) state->nexthard = next; state->nextstat = next; state->nextprof = next; + state->nextcall = next; + state->nextcallopt = next; hardclock_sync(cpu); } busy = 0; @@ -609,7 +609,6 @@ cpu_initclocks_bsp(void) state->nextcall = -1; state->nextcallopt = -1; } - callout_new_inserted = cpu_new_callout; periodic = want_periodic; /* Grab requested timer or the best of present. */ if (timername[0]) @@ -803,6 +802,9 @@ clocksource_cyc_set(const struct bintime sbintime_t now, t; struct pcpu_state *state; + /* Do not touch anything if somebody reconfiguring timers. */ + if (busy) + return; t = bintime2sbintime(*bt); state = DPCPU_PTR(timerstate); if (periodic) @@ -831,12 +833,15 @@ clocksource_cyc_set(const struct bintime } #endif -static void +void cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt) { sbintime_t now; struct pcpu_state *state; + /* Do not touch anything if somebody reconfiguring timers. */ + if (busy) + return; CTR6(KTR_SPARE2, "new co at %d: on %d at %d.%08x - %d.%08x", curcpu, cpu, (int)(bt_opt >> 32), (u_int)(bt_opt & 0xffffffff), (int)(bt >> 32), (u_int)(bt & 0xffffffff)); Modified: projects/calloutng/sys/kern/kern_timeout.c ============================================================================== --- projects/calloutng/sys/kern/kern_timeout.c Tue Feb 26 13:59:39 2013 (r247318) +++ projects/calloutng/sys/kern/kern_timeout.c Tue Feb 26 15:25:43 2013 (r247319) @@ -179,7 +179,6 @@ struct callout_cpu cc_cpu; (sizeof(time_t) == (sizeof(int64_t)) ? INT64_MAX : INT32_MAX) static int timeout_cpu; -void (*callout_new_inserted)(int cpu, sbintime_t bt, sbintime_t bt_opt) = NULL; static void softclock_call_cc(struct callout *c, struct callout_cpu *cc, #ifdef CALLOUT_PROFILING @@ -391,12 +390,11 @@ callout_process(sbintime_t now) struct callout_tailq *sc; sbintime_t first, last, max, tmp_max; uint32_t lookahead; - int firstb, lastb, nowb, need_softclock, exit_allowed, exit_wanted; + u_int firstb, lastb, nowb; #ifdef CALLOUT_PROFILING int depth_dir = 0, mpcalls_dir = 0, lockcalls_dir = 0; #endif - need_softclock = 0; cc = CC_SELF(); mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET); @@ -427,14 +425,10 @@ callout_process(sbintime_t now) lastb = firstb - 1; if (nowb - firstb >= callwheelsize) nowb = firstb - 1; - nowb &= callwheelmask; - lastb &= callwheelmask; - firstb &= callwheelmask; /* Iterate callwheel from firstb to nowb and then up to lastb. */ - exit_allowed = exit_wanted = 0; - for (;;) { - sc = &cc->cc_callwheel[firstb]; + do { + sc = &cc->cc_callwheel[firstb & callwheelmask]; tmp = TAILQ_FIRST(sc); while (tmp != NULL) { /* Run the callout if present time within allowed. */ @@ -462,7 +456,6 @@ callout_process(sbintime_t now) TAILQ_INSERT_TAIL(&cc->cc_expireq, tmp, c_links.tqe); tmp->c_flags |= CALLOUT_PROCESSED; - need_softclock = 1; tmp = tmpn; } continue; @@ -475,37 +468,31 @@ callout_process(sbintime_t now) * time, so it cannot be aggregated. */ if (tmp->c_time > last) { - exit_wanted = 1; + lastb = nowb; goto next; } /* Update first and last time, respecting this event. */ if (tmp->c_time < first) first = tmp->c_time; - tmp_max = tmp->c_time; - tmp_max += tmp->c_precision; + tmp_max = tmp->c_time + tmp->c_precision; if (tmp_max < last) last = tmp_max; next: tmp = TAILQ_NEXT(tmp, c_links.tqe); } - /* Stop if we looked far enough into the future. */ - if (firstb == lastb) - break; + /* Proceed with the next bucket. */ + firstb++; /* * Stop if we looked after present time and found * some event we can't execute at now. + * Stop if we looked far enough into the future. */ - if (firstb == nowb) - exit_allowed = 1; - if (exit_allowed && exit_wanted) - break; - /* Proceed with the next bucket. */ - firstb = (firstb + 1) & callwheelmask; - } + } while (firstb <= lastb); cc->cc_exec_next_dir = NULL; - if (callout_new_inserted != NULL) - (*callout_new_inserted)(curcpu, last, first); cc->cc_firstevent = last; +#ifndef NO_EVENTTIMERS + cpu_new_callout(curcpu, last, first); +#endif #ifdef CALLOUT_PROFILING avg_depth_dir += (depth_dir * 1000 - avg_depth_dir) >> 8; avg_mpcalls_dir += (mpcalls_dir * 1000 - avg_mpcalls_dir) >> 8; @@ -516,7 +503,7 @@ next: * swi_sched acquires the thread lock, so we don't want to call it * with cc_lock held; incorrect locking order. */ - if (need_softclock) + if (!TAILQ_EMPTY(&cc->cc_expireq)) swi_sched(cc->cc_cookie, 0); } @@ -549,7 +536,6 @@ callout_cc_add(struct callout *c, struct sbintime_t sbt, sbintime_t precision, void (*func)(void *), void *arg, int cpu, int flags) { - sbintime_t last; int bucket; CC_LOCK_ASSERT(cc); @@ -568,16 +554,17 @@ callout_cc_add(struct callout *c, struct c, (int)(c->c_precision >> 32), (u_int)(c->c_precision & 0xffffffff)); TAILQ_INSERT_TAIL(&cc->cc_callwheel[bucket], c, c_links.tqe); +#ifndef NO_EVENTTIMERS /* * Inform the eventtimers(4) subsystem there's a new callout * that has been inserted, but only if really required. */ - last = c->c_time + c->c_precision; - if (callout_new_inserted != NULL && ((last < cc->cc_firstevent) || - (cc->cc_firstevent == 0))) { - cc->cc_firstevent = last; - (*callout_new_inserted)(cpu, last, c->c_time); + sbt = c->c_time + c->c_precision; + if (sbt < cc->cc_firstevent || cc->cc_firstevent == 0) { + cc->cc_firstevent = sbt; + cpu_new_callout(cpu, sbt, c->c_time); } +#endif } static void Modified: projects/calloutng/sys/sys/callout.h ============================================================================== --- projects/calloutng/sys/sys/callout.h Tue Feb 26 13:59:39 2013 (r247318) +++ projects/calloutng/sys/sys/callout.h Tue Feb 26 15:25:43 2013 (r247319) @@ -97,7 +97,6 @@ int callout_schedule_on(struct callout * #define callout_stop(c) _callout_stop_safe(c, 0) int _callout_stop_safe(struct callout *, int); void callout_process(sbintime_t now); -extern void (*callout_new_inserted)(int cpu, sbintime_t bt, sbintime_t bt_opt); #endif Modified: projects/calloutng/sys/sys/systm.h ============================================================================== --- projects/calloutng/sys/sys/systm.h Tue Feb 26 13:59:39 2013 (r247318) +++ projects/calloutng/sys/sys/systm.h Tue Feb 26 15:25:43 2013 (r247319) @@ -269,6 +269,7 @@ void cpu_startprofclock(void); void cpu_stopprofclock(void); sbintime_t cpu_idleclock(void); void cpu_activeclock(void); +void cpu_new_callout(int cpu, sbintime_t bt, sbintime_t bt_opt); extern int cpu_can_deep_sleep; extern int cpu_disable_deep_sleep;