Date: Thu, 4 Dec 2008 16:08:07 +0200 From: "Vladimir V. Kobal" <vlad@prokk.net> To: "'Alexander Motin'" <mav@freebsd.org> Cc: freebsd-net@freebsd.org Subject: RE: Multiple netgraph threads Message-ID: <003001c95619$be6e98a0$3b4bc9e0$@net> In-Reply-To: <493640A9.8080701@FreeBSD.org> References: <1228234984.00043656.1228222202@10.7.7.3> <493640A9.8080701@FreeBSD.org>
next in thread | previous in thread | raw e-mail | index | archive | help
]rn qnqr`bmne qnnayemhe b tnpl`re MIME.
------=_NextPart_000_0031_01C9562A.81F768A0
Content-Type: text/plain;
charset="koi8-r"
Content-Transfer-Encoding: 7bit
Alexander Motin wrote:
>I have uploaded that patch back. Not sure is it correct at this moment,
>there was some changes, but that time it worked fine.
Thanks a lot. I've installed the patch with some changes. I'm using the
7-STABLE (ng_base.c,v 1.135.2.10) at a production NAS. Corresponding patch
is in the attachment.
I can't say anything of performance yet. Not enough statistics data is
collected at the moment. But there is a strangeness with ng_queue threads:
under high network load top display too low WCPU (while TIME has a normal
value) for these threads. The maximum WCPU I saw under lower network load
was 35-40% per ng_queue thread. Are there any ideas? Am I using correctly
the kthread_create() in the patch?
last pid: 21000; load averages: 1.52, 1.40, 1.02 up 0+00:25:39
14:24:28
79 processes: 5 running, 63 sleeping, 11 waiting
CPU: 0.3% user, 0.0% nice, 41.4% system, 0.6% interrupt, 57.7% idle
Mem: 40M Active, 11M Inact, 166M Wired, 112K Cache, 77M Buf, 1731M Free
Swap: 2048M Total, 2048M Free
PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND
12 root 1 171 ki31 0K 16K RUN 2 18:36 58.50% idle: cpu2
11 root 1 171 ki31 0K 16K RUN 3 18:02 63.87% idle: cpu3
13 root 1 171 ki31 0K 16K CPU1 1 17:56 41.26% idle: cpu1
14 root 1 171 ki31 0K 16K CPU0 0 17:20 59.96% idle: cpu0
2 root 1 96 - 0K 16K sleep 0 7:16 0.20% ng_queue0
3 root 1 96 - 0K 16K sleep 1 7:13 0.20% ng_queue1
25 root 1 -68 - 0K 16K - 1 5:44 36.67% em0 taskq
26 root 1 -68 - 0K 16K - 2 5:26 35.60% em1 taskq
16 root 1 -32 - 0K 16K WAIT 0 1:30 3.86% swi4:
clock
2538 root 1 48 0 36720K 15228K select 3 0:25 0.59% mpd5
--
Vladimir Kobal
------=_NextPart_000_0031_01C9562A.81F768A0
Content-Type: application/octet-stream;
name="netgraph.threads.patch"
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;
filename="netgraph.threads.patch"
--- ng_base.c.orig 2008-12-03 11:29:36.000000000 +0200=0A=
+++ ng_base.c 2008-12-03 16:12:20.000000000 +0200=0A=
@@ -61,6 +61,8 @@=0A=
#include <sys/syslog.h>=0A=
#include <sys/refcount.h>=0A=
#include <sys/proc.h>=0A=
+#include <sys/kthread.h>=0A=
+#include <sys/smp.h>=0A=
#include <machine/cpu.h>=0A=
=0A=
#include <net/netisr.h>=0A=
@@ -203,7 +205,7 @@ static int ng_generic_msg(node_p here, i=0A=
static ng_ID_t ng_decodeidname(const char *name);=0A=
static int ngb_mod_event(module_t mod, int event, void *data);=0A=
static void ng_worklist_add(node_p node);=0A=
-static void ngintr(void);=0A=
+static void ngthread(void *);=0A=
static int ng_apply_item(node_p node, item_p item, int rw);=0A=
static void ng_flush_input_queue(struct ng_queue * ngq);=0A=
static node_p ng_ID2noderef(ng_ID_t ID);=0A=
@@ -251,6 +253,10 @@ MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_=0A=
mtx_lock(&ng_worklist_mtx)=0A=
#define NG_WORKLIST_UNLOCK() \=0A=
mtx_unlock(&ng_worklist_mtx)=0A=
+#define NG_WORKLIST_SLEEP() \=0A=
+ mtx_sleep(&ng_worklist, &ng_worklist_mtx, 0, "sleep", 0)=0A=
+#define NG_WORKLIST_WAKEUP() \=0A=
+ wakeup_one(&ng_worklist)=0A=
=0A=
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/=0A=
/*=0A=
@@ -2858,9 +2864,13 @@ out:=0A=
=0A=
uma_zone_t ng_qzone;=0A=
uma_zone_t ng_qdzone;=0A=
+static int numthreads =3D 0; /* number of queue threads */=0A=
static int maxalloc =3D 4096;/* limit the damage of a leak */=0A=
static int maxdata =3D 512; /* limit the damage of a DoS */=0A=
=0A=
+TUNABLE_INT("net.graph.threads", &numthreads);=0A=
+SYSCTL_INT(_net_graph, OID_AUTO, threads, CTLFLAG_RDTUN, &numthreads,=0A=
+ 0, "Number of queue processing threads to create");=0A=
TUNABLE_INT("net.graph.maxalloc", &maxalloc);=0A=
SYSCTL_INT(_net_graph, OID_AUTO, maxalloc, CTLFLAG_RDTUN, &maxalloc,=0A=
0, "Maximum number of non-data queue items to allocate");=0A=
@@ -3055,6 +3065,7 @@ static int=0A=
ngb_mod_event(module_t mod, int event, void *data)=0A=
{=0A=
int error =3D 0;=0A=
+ int i;=0A=
=0A=
switch (event) {=0A=
case MOD_LOAD:=0A=
@@ -3080,8 +3091,17 @@ ngb_mod_event(module_t mod, int event, v=0A=
ng_qdzone =3D uma_zcreate("NetGraph data items", sizeof(struct =
ng_item),=0A=
NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0);=0A=
uma_zone_set_max(ng_qdzone, maxdata);=0A=
- netisr_register(NETISR_NETGRAPH, (netisr_t *)ngintr, NULL,=0A=
- NETISR_MPSAFE);=0A=
+ /* Autoconfigure number of threads. */=0A=
+ if (numthreads <=3D 0)=0A=
+ numthreads =3D mp_ncpus;=0A=
+ /* Create threads. */=0A=
+ for (i =3D 0; i < numthreads; i++) {=0A=
+ if (kthread_create(ngthread, NULL, NULL,=0A=
+ 0, 0, "ng_queue%d", i)) {=0A=
+ numthreads =3D i - 1;=0A=
+ break;=0A=
+ }=0A=
+ }=0A=
break;=0A=
case MOD_UNLOAD:=0A=
/* You can't unload it because an interface may be using it. */=0A=
@@ -3236,29 +3256,25 @@ SYSCTL_PROC(_debug, OID_AUTO, ng_dump_it=0A=
/***********************************************************************=0A=
* Worklist routines=0A=
**********************************************************************/=0A=
-/* NETISR thread enters here */=0A=
/*=0A=
* Pick a node off the list of nodes with work,=0A=
- * try get an item to process off it.=0A=
- * If there are no more, remove the node from the list.=0A=
+ * try get an item to process off it. Remove the node from the list.=0A=
*/=0A=
static void=0A=
-ngintr(void)=0A=
+ngthread(void *arg)=0A=
{=0A=
for (;;) {=0A=
node_p node;=0A=
=0A=
/* Get node from the worklist. */=0A=
NG_WORKLIST_LOCK();=0A=
- node =3D TAILQ_FIRST(&ng_worklist);=0A=
- if (!node) {=0A=
- NG_WORKLIST_UNLOCK();=0A=
- break;=0A=
- }=0A=
+ while ((node =3D TAILQ_FIRST(&ng_worklist)) =3D=3D NULL)=0A=
+ NG_WORKLIST_SLEEP();=0A=
TAILQ_REMOVE(&ng_worklist, node, nd_work);=0A=
NG_WORKLIST_UNLOCK();=0A=
CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist",=0A=
__func__, node->nd_ID, node);=0A=
+=0A=
/*=0A=
* We have the node. We also take over the reference=0A=
* that the list had on it.=0A=
@@ -3310,9 +3326,9 @@ ng_worklist_add(node_p node)=0A=
NG_WORKLIST_LOCK();=0A=
TAILQ_INSERT_TAIL(&ng_worklist, node, nd_work);=0A=
NG_WORKLIST_UNLOCK();=0A=
- schednetisr(NETISR_NETGRAPH);=0A=
CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__,=0A=
node->nd_ID, node);=0A=
+ NG_WORKLIST_WAKEUP();=0A=
} else {=0A=
CTR3(KTR_NET, "%20s: node [%x] (%p) already on worklist",=0A=
__func__, node->nd_ID, node);=0A=
------=_NextPart_000_0031_01C9562A.81F768A0--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?003001c95619$be6e98a0$3b4bc9e0$>