Date: Thu, 4 Dec 2008 16:08:07 +0200 From: "Vladimir V. Kobal" <vlad@prokk.net> To: "'Alexander Motin'" <mav@freebsd.org> Cc: freebsd-net@freebsd.org Subject: RE: Multiple netgraph threads Message-ID: <003001c95619$be6e98a0$3b4bc9e0$@net> In-Reply-To: <493640A9.8080701@FreeBSD.org> References: <1228234984.00043656.1228222202@10.7.7.3> <493640A9.8080701@FreeBSD.org>
next in thread | previous in thread | raw e-mail | index | archive | help
[-- Attachment #1 --]
Alexander Motin wrote:
>I have uploaded that patch back. Not sure is it correct at this moment,
>there was some changes, but that time it worked fine.
Thanks a lot. I've installed the patch with some changes. I'm using the
7-STABLE (ng_base.c,v 1.135.2.10) at a production NAS. Corresponding patch
is in the attachment.
I can't say anything of performance yet. Not enough statistics data is
collected at the moment. But there is a strangeness with ng_queue threads:
under high network load top display too low WCPU (while TIME has a normal
value) for these threads. The maximum WCPU I saw under lower network load
was 35-40% per ng_queue thread. Are there any ideas? Am I using correctly
the kthread_create() in the patch?
last pid: 21000; load averages: 1.52, 1.40, 1.02 up 0+00:25:39
14:24:28
79 processes: 5 running, 63 sleeping, 11 waiting
CPU: 0.3% user, 0.0% nice, 41.4% system, 0.6% interrupt, 57.7% idle
Mem: 40M Active, 11M Inact, 166M Wired, 112K Cache, 77M Buf, 1731M Free
Swap: 2048M Total, 2048M Free
PID USERNAME THR PRI NICE SIZE RES STATE C TIME WCPU COMMAND
12 root 1 171 ki31 0K 16K RUN 2 18:36 58.50% idle: cpu2
11 root 1 171 ki31 0K 16K RUN 3 18:02 63.87% idle: cpu3
13 root 1 171 ki31 0K 16K CPU1 1 17:56 41.26% idle: cpu1
14 root 1 171 ki31 0K 16K CPU0 0 17:20 59.96% idle: cpu0
2 root 1 96 - 0K 16K sleep 0 7:16 0.20% ng_queue0
3 root 1 96 - 0K 16K sleep 1 7:13 0.20% ng_queue1
25 root 1 -68 - 0K 16K - 1 5:44 36.67% em0 taskq
26 root 1 -68 - 0K 16K - 2 5:26 35.60% em1 taskq
16 root 1 -32 - 0K 16K WAIT 0 1:30 3.86% swi4:
clock
2538 root 1 48 0 36720K 15228K select 3 0:25 0.59% mpd5
--
Vladimir Kobal
[-- Attachment #2 --]
--- ng_base.c.orig 2008-12-03 11:29:36.000000000 +0200
+++ ng_base.c 2008-12-03 16:12:20.000000000 +0200
@@ -61,6 +61,8 @@
#include <sys/syslog.h>
#include <sys/refcount.h>
#include <sys/proc.h>
+#include <sys/kthread.h>
+#include <sys/smp.h>
#include <machine/cpu.h>
#include <net/netisr.h>
@@ -203,7 +205,7 @@ static int ng_generic_msg(node_p here, i
static ng_ID_t ng_decodeidname(const char *name);
static int ngb_mod_event(module_t mod, int event, void *data);
static void ng_worklist_add(node_p node);
-static void ngintr(void);
+static void ngthread(void *);
static int ng_apply_item(node_p node, item_p item, int rw);
static void ng_flush_input_queue(struct ng_queue * ngq);
static node_p ng_ID2noderef(ng_ID_t ID);
@@ -251,6 +253,10 @@ MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_
mtx_lock(&ng_worklist_mtx)
#define NG_WORKLIST_UNLOCK() \
mtx_unlock(&ng_worklist_mtx)
+#define NG_WORKLIST_SLEEP() \
+ mtx_sleep(&ng_worklist, &ng_worklist_mtx, 0, "sleep", 0)
+#define NG_WORKLIST_WAKEUP() \
+ wakeup_one(&ng_worklist)
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
/*
@@ -2858,9 +2864,13 @@ out:
uma_zone_t ng_qzone;
uma_zone_t ng_qdzone;
+static int numthreads = 0; /* number of queue threads */
static int maxalloc = 4096;/* limit the damage of a leak */
static int maxdata = 512; /* limit the damage of a DoS */
+TUNABLE_INT("net.graph.threads", &numthreads);
+SYSCTL_INT(_net_graph, OID_AUTO, threads, CTLFLAG_RDTUN, &numthreads,
+ 0, "Number of queue processing threads to create");
TUNABLE_INT("net.graph.maxalloc", &maxalloc);
SYSCTL_INT(_net_graph, OID_AUTO, maxalloc, CTLFLAG_RDTUN, &maxalloc,
0, "Maximum number of non-data queue items to allocate");
@@ -3055,6 +3065,7 @@ static int
ngb_mod_event(module_t mod, int event, void *data)
{
int error = 0;
+ int i;
switch (event) {
case MOD_LOAD:
@@ -3080,8 +3091,17 @@ ngb_mod_event(module_t mod, int event, v
ng_qdzone = uma_zcreate("NetGraph data items", sizeof(struct ng_item),
NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0);
uma_zone_set_max(ng_qdzone, maxdata);
- netisr_register(NETISR_NETGRAPH, (netisr_t *)ngintr, NULL,
- NETISR_MPSAFE);
+ /* Autoconfigure number of threads. */
+ if (numthreads <= 0)
+ numthreads = mp_ncpus;
+ /* Create threads. */
+ for (i = 0; i < numthreads; i++) {
+ if (kthread_create(ngthread, NULL, NULL,
+ 0, 0, "ng_queue%d", i)) {
+ numthreads = i - 1;
+ break;
+ }
+ }
break;
case MOD_UNLOAD:
/* You can't unload it because an interface may be using it. */
@@ -3236,29 +3256,25 @@ SYSCTL_PROC(_debug, OID_AUTO, ng_dump_it
/***********************************************************************
* Worklist routines
**********************************************************************/
-/* NETISR thread enters here */
/*
* Pick a node off the list of nodes with work,
- * try get an item to process off it.
- * If there are no more, remove the node from the list.
+ * try get an item to process off it. Remove the node from the list.
*/
static void
-ngintr(void)
+ngthread(void *arg)
{
for (;;) {
node_p node;
/* Get node from the worklist. */
NG_WORKLIST_LOCK();
- node = TAILQ_FIRST(&ng_worklist);
- if (!node) {
- NG_WORKLIST_UNLOCK();
- break;
- }
+ while ((node = TAILQ_FIRST(&ng_worklist)) == NULL)
+ NG_WORKLIST_SLEEP();
TAILQ_REMOVE(&ng_worklist, node, nd_work);
NG_WORKLIST_UNLOCK();
CTR3(KTR_NET, "%20s: node [%x] (%p) taken off worklist",
__func__, node->nd_ID, node);
+
/*
* We have the node. We also take over the reference
* that the list had on it.
@@ -3310,9 +3326,9 @@ ng_worklist_add(node_p node)
NG_WORKLIST_LOCK();
TAILQ_INSERT_TAIL(&ng_worklist, node, nd_work);
NG_WORKLIST_UNLOCK();
- schednetisr(NETISR_NETGRAPH);
CTR3(KTR_NET, "%20s: node [%x] (%p) put on worklist", __func__,
node->nd_ID, node);
+ NG_WORKLIST_WAKEUP();
} else {
CTR3(KTR_NET, "%20s: node [%x] (%p) already on worklist",
__func__, node->nd_ID, node);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?003001c95619$be6e98a0$3b4bc9e0$>