Date: Sat, 1 May 2021 15:39:30 +0300 From: =?UTF-8?B?w5Z6a2FuIEtJUklL?= <ozkan.kirik@gmail.com> To: Mark Johnston <markj@freebsd.org> Cc: FreeBSD Net <freebsd-net@freebsd.org> Subject: Re: IPsec performace - netisr hits %100 Message-ID: <CAAcX-AE_jRirL64tbL4ikRa4XDuvkeQgDObLqphJN7HtXyqwLg@mail.gmail.com> In-Reply-To: <CAAcX-AGHNzU%2BvWD0Dvr_BQYcb25V=RHqyLeT7n_XkQiVXSwN0g@mail.gmail.com> References: <CAAcX-AF=0s5tueCuanFKkoALNkRnWJ-8QrzfCqSu=ReoWvqMug@mail.gmail.com> <YIxpdL9b6v8%2BN%2BLg@nuc> <CAAcX-AHSk92gXQ3HXw4KYpXQ-jTVCjX0svStu5z49ykH-tk2QQ@mail.gmail.com> <CAAcX-AG2KyN-7yMm%2BMpKbCRDKivFQjq6BVR0r50t4P3HpDRx=Q@mail.gmail.com> <YIx6eHEH53B4g1iB@nuc> <CAAcX-AGHNzU%2BvWD0Dvr_BQYcb25V=RHqyLeT7n_XkQiVXSwN0g@mail.gmail.com>
next in thread | previous in thread | raw e-mail | index | archive | help
I've pulled the latest stable/13 branch and make buildworld.
Same configuration and same usage, ping works between two sides, iperf can
connect to server but no data transferred. throughput is around 10Kbps.
I tried both with and without netipsec/ipsec_input.c patch, but no change.
There is something wrong with ipsec (or ipsec in jail) in stable/13.
iperf between host and jail works without ipsec.
I saw some errors in truss output show as a brief:
write(3,"@\^A\0x\0\0\0\^A\0\0\^S\M^I\0\0"...,131072) ERR#35 'Resource
temporarily unavailable'
recvfrom(3,0x8018048c0,131072,0,0x0,0x0) ERR#35 'Resource temporarily
unavailable'
_umtx_op(0x80026e008,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfffdec8)
ERR#60 'Operation timed out'
Full output of truss output is below:
root@host # jexec client bash
[root@client /]# truss iperf -B 172.16.70.1 -c 172.16.68.1 2>&1 | grep -v
clock_nanosleep
mmap(0x0,135168,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) =3D
34362150912 (0x80024d000)
mprotect(0x80024a000,4096,PROT_READ) =3D 0 (0x0)
issetugid() =3D 0 (0x0)
sigfastblock(0x1,0x80024c510) =3D 0 (0x0)
open("/etc/libmap.conf",O_RDONLY|O_CLOEXEC,011136710) =3D 3 (0x3)
fstat(3,{ mode=3D-rw-r--r-- ,inode=3D482,size=3D47,blksize=3D4096 }) =3D 0 =
(0x0)
read(3,"# $FreeBSD$\nincludedir /usr/loc"...,47) =3D 47 (0x2f)
close(3) =3D 0 (0x0)
open("/usr/local/etc/libmap.d",O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC,01=
65)
ERR#2 'No such file or directory'
open("/var/run/ld-elf.so.hints",O_RDONLY|O_CLOEXEC,010015464) =3D 3 (0x3)
read(3,"Ehnt\^A\0\0\0\M^@\0\0\0\M-U\0\0"...,128) =3D 128 (0x80)
fstat(3,{ mode=3D-r--r--r-- ,inode=3D187,size=3D341,blksize=3D4096 }) =3D 0=
(0x0)
pread(3,"/lib:/usr/lib:/usr/local/lib:/us"...,213,0x80) =3D 213 (0xd5)
close(3) =3D 0 (0x0)
open("/lib/libthr.so.3",O_RDONLY|O_CLOEXEC|O_VERIFY,00) =3D 3 (0x3)
fstat(3,{ mode=3D-r--r--r-- ,inode=3D186701,size=3D125952,blksize=3D32768 }=
) =3D 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) =3D 3436228608=
0
(0x80026e000)
mmap(0x0,184320,PROT_NONE,MAP_GUARD,-1,0x0) =3D 34362290176 (0x80026f000)
mmap(0x80026f000,53248,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFA=
ULT_READ,3,0x0)
=3D 34362290176 (0x80026f000)
mmap(0x80027c000,73728,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE=
|MAP_PREFAULT_READ,3,0xc000)
=3D 34362343424 (0x80027c000)
mmap(0x80028e000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x1d000)
=3D 34362417152 (0x80028e000)
mmap(0x80028f000,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x1d000)
=3D 34362421248 (0x80028f000)
mmap(0x800291000,45056,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANON,=
-1,0x0)
=3D 34362429440 (0x800291000)
munmap(0x80026e000,4096) =3D 0 (0x0)
close(3) =3D 0 (0x0)
open("/lib/librt.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,066000) ERR#2 'No such
file or directory'
open("/usr/lib/librt.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,066000) =3D 3 (0x3)
fstat(3,{ mode=3D-r--r--r-- ,inode=3D411606,size=3D22824,blksize=3D32768 })=
=3D 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) =3D 3436228608=
0
(0x80026e000)
mmap(0x0,36864,PROT_NONE,MAP_GUARD,-1,0x0) =3D 34362474496 (0x80029c000)
mmap(0x80029c000,12288,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFA=
ULT_READ,3,0x0)
=3D 34362474496 (0x80029c000)
mmap(0x80029f000,12288,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE=
|MAP_PREFAULT_READ,3,0x2000)
=3D 34362486784 (0x80029f000)
mmap(0x8002a2000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x4000)
=3D 34362499072 (0x8002a2000)
mmap(0x8002a3000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x4000)
=3D 34362503168 (0x8002a3000)
mmap(0x8002a4000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANON,-=
1,0x0)
=3D 34362507264 (0x8002a4000)
munmap(0x80026e000,4096) =3D 0 (0x0)
close(3) =3D 0 (0x0)
open("/lib/libc++.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,00) ERR#2 'No such file
or directory'
open("/usr/lib/libc++.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,00) =3D 3 (0x3)
fstat(3,{ mode=3D-r--r--r-- ,inode=3D411469,size=3D824056,blksize=3D32768 }=
) =3D 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) =3D 3436228608=
0
(0x80026e000)
mmap(0x0,860160,PROT_NONE,MAP_GUARD,-1,0x0) =3D 34362511360 (0x8002a5000)
mmap(0x8002a5000,376832,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREF=
AULT_READ,3,0x0)
=3D 34362511360 (0x8002a5000)
mmap(0x800301000,425984,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCOR=
E|MAP_PREFAULT_READ,3,0x5b000)
=3D 34362888192 (0x800301000)
mmap(0x800369000,24576,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFA=
ULT_READ,3,0xc2000)
=3D 34363314176 (0x800369000)
mmap(0x80036f000,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0xc7000)
=3D 34363338752 (0x80036f000)
mmap(0x800371000,24576,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANON,=
-1,0x0)
=3D 34363346944 (0x800371000)
munmap(0x80026e000,4096) =3D 0 (0x0)
close(3) =3D 0 (0x0)
open("/lib/libcxxrt.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,041400) =3D 3 (0x3)
fstat(3,{ mode=3D-r--r--r-- ,inode=3D186668,size=3D113688,blksize=3D32768 }=
) =3D 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) =3D 3436228608=
0
(0x80026e000)
mmap(0x0,143360,PROT_NONE,MAP_GUARD,-1,0x0) =3D 34363371520 (0x800377000)
mmap(0x800377000,53248,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFA=
ULT_READ,3,0x0)
=3D 34363371520 (0x800377000)
mmap(0x800384000,61440,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE=
|MAP_PREFAULT_READ,3,0xc000)
=3D 34363424768 (0x800384000)
mmap(0x800393000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x1a000)
=3D 34363486208 (0x800393000)
mmap(0x800394000,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x1a000)
=3D 34363490304 (0x800394000)
mmap(0x800396000,16384,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANON,=
-1,0x0)
=3D 34363498496 (0x800396000)
munmap(0x80026e000,4096) =3D 0 (0x0)
close(3) =3D 0 (0x0)
open("/lib/libm.so.5",O_RDONLY|O_CLOEXEC|O_VERIFY,00) =3D 3 (0x3)
fstat(3,{ mode=3D-r--r--r-- ,inode=3D186671,size=3D214192,blksize=3D32768 }=
) =3D 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) =3D 3436228608=
0
(0x80026e000)
mmap(0x0,225280,PROT_NONE,MAP_GUARD,-1,0x0) =3D 34363514880 (0x80039a000)
mmap(0x80039a000,73728,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFA=
ULT_READ,3,0x0)
=3D 34363514880 (0x80039a000)
mmap(0x8003ac000,143360,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCOR=
E|MAP_PREFAULT_READ,3,0x11000)
=3D 34363588608 (0x8003ac000)
mmap(0x8003cf000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x33000)
=3D 34363731968 (0x8003cf000)
mmap(0x8003d0000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x33000)
=3D 34363736064 (0x8003d0000)
munmap(0x80026e000,4096) =3D 0 (0x0)
close(3) =3D 0 (0x0)
open("/lib/libgcc_s.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,020643400) =3D 3 (0x3=
)
fstat(3,{ mode=3D-r--r--r-- ,inode=3D186687,size=3D93448,blksize=3D32768 })=
=3D 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) =3D 3436228608=
0
(0x80026e000)
mmap(0x0,106496,PROT_NONE,MAP_GUARD,-1,0x0) =3D 34363740160 (0x8003d1000)
mmap(0x8003d1000,36864,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREFA=
ULT_READ,3,0x0)
=3D 34363740160 (0x8003d1000)
mmap(0x8003da000,57344,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE=
|MAP_PREFAULT_READ,3,0x8000)
=3D 34363777024 (0x8003da000)
mmap(0x8003e8000,8192,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x15000)
=3D 34363834368 (0x8003e8000)
mmap(0x8003ea000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFAU=
LT_READ,3,0x16000)
=3D 34363842560 (0x8003ea000)
munmap(0x80026e000,4096) =3D 0 (0x0)
close(3) =3D 0 (0x0)
open("/lib/libc.so.7",O_RDONLY|O_CLOEXEC|O_VERIFY,00) =3D 3 (0x3)
fstat(3,{ mode=3D-r--r--r-- ,inode=3D186667,size=3D1904424,blksize=3D32768 =
}) =3D 0
(0x0)
mmap(0x0,4096,PROT_READ,MAP_PRIVATE|MAP_PREFAULT_READ,3,0x0) =3D 3436228608=
0
(0x80026e000)
mmap(0x0,4182016,PROT_NONE,MAP_GUARD,-1,0x0) =3D 34363846656 (0x8003eb000)
mmap(0x8003eb000,520192,PROT_READ,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE|MAP_PREF=
AULT_READ,3,0x0)
=3D 34363846656 (0x8003eb000)
mmap(0x80046a000,1331200,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCO=
RE|MAP_PREFAULT_READ,3,0x7e000)
=3D 34364366848 (0x80046a000)
mmap(0x8005af000,36864,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFA=
ULT_READ,3,0x1c2000)
=3D 34365698048 (0x8005af000)
mmap(0x8005b8000,28672,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_PREFA=
ULT_READ,3,0x1ca000)
=3D 34365734912 (0x8005b8000)
mmap(0x8005bf000,2265088,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_ANO=
N,-1,0x0)
=3D 34365763584 (0x8005bf000)
munmap(0x80026e000,4096) =3D 0 (0x0)
close(3) =3D 0 (0x0)
mprotect(0x80028e000,4096,PROT_READ) =3D 0 (0x0)
mprotect(0x8002a2000,4096,PROT_READ) =3D 0 (0x0)
mprotect(0x800369000,20480,PROT_READ) =3D 0 (0x0)
mprotect(0x800393000,4096,PROT_READ) =3D 0 (0x0)
mprotect(0x8003cf000,4096,PROT_READ) =3D 0 (0x0)
mprotect(0x8003e8000,4096,PROT_READ) =3D 0 (0x0)
mprotect(0x8005af000,36864,PROT_READ) =3D 0 (0x0)
mprotect(0x8005af000,36864,PROT_READ|PROT_WRITE) =3D 0 (0x0)
mprotect(0x8005af000,36864,PROT_READ) =3D 0 (0x0)
readlink("/etc/malloc.conf",0x7fffffffd5c0,1024) ERR#2 'No such file or
directory'
issetugid() =3D 0 (0x0)
__sysctl("vm.overcommit",2,0x7fffffffbb54,0x7fffffffbb48,0x0,0) =3D 0 (0x0)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(21),=
-1,0x0)
=3D 34368126976 (0x800800000)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(12),=
-1,0x0)
=3D 34370224128 (0x800a00000)
mmap(0x0,4194304,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(21),=
-1,0x0)
=3D 34372321280 (0x800c00000)
__sysctl("kern.usrstack",2,0x8002903c8,0x7fffffffd878,0x0,0) =3D 0 (0x0)
getrlimit(RLIMIT_STACK,{ cur=3D536870912,max=3D536870912 }) =3D 0 (0x0)
thr_self(0x800a12000) =3D 0 (0x0)
mmap(0x7fffdfffe000,4096,PROT_NONE,MAP_ANON,-1,0x0) =3D 140736951476224
(0x7fffdfffe000)
rtprio_thread(RTP_LOOKUP,102215,0x7fffffffd838) =3D 0 (0x0)
sigaction(SIGTHR,{ 0x8002884d0 SA_SIGINFO ss_t },0x0) =3D 0 (0x0)
sigprocmask(SIG_UNBLOCK,{ },0x0) =3D 0 (0x0)
_umtx_op(0x7fffffffd830,UMTX_OP_WAKE,0x1,0x0,0x0) =3D 0 (0x0)
mprotect(0x0,0,PROT_NONE) =3D 0 (0x0)
getpid() =3D 7030 (0x1b76)
getpid() =3D 7030 (0x1b76)
sigprocmask(SIG_BLOCK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|S=
IGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHL=
D|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|=
SIGUSR1|SIGUSR2
},{ }) =3D 0 (0x0)
sigfastblock(0x3,0x0) =3D 0 (0x0)
sigprocmask(SIG_SETMASK,{ },0x0) =3D 0 (0x0)
sigfastblock(0x1,0x800a12038) =3D 0 (0x0)
getcontext(0x7fffffffcbd0) =3D 0 (0x0)
sysarch(AMD64_GET_XFPUSTATE,0x7fffffffcb98) =3D 0 (0x0)
mmap(0x0,135168,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) =3D
34376515584 (0x801000000)
mprotect(0x22c000,4096,PROT_READ) =3D 0 (0x0)
sigprocmask(SIG_SETMASK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|S=
IGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHL=
D|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|=
SIGUSR1|SIGUSR2
},{ }) =3D 0 (0x0)
sigaction(SIGTERM,{ 0x800288380 SA_RESTART|SA_SIGINFO ss_t },{ SIG_DFL 0x0
ss_t }) =3D 0 (0x0)
sigprocmask(SIG_SETMASK,{ },0x0) =3D 0 (0x0)
sigprocmask(SIG_SETMASK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|S=
IGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHL=
D|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|=
SIGUSR1|SIGUSR2
},{ }) =3D 0 (0x0)
sigaction(SIGINT,{ 0x800288380 SA_RESTART|SA_SIGINFO ss_t },{ SIG_DFL 0x0
ss_t }) =3D 0 (0x0)
sigprocmask(SIG_SETMASK,{ },0x0) =3D 0 (0x0)
sigprocmask(SIG_SETMASK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|S=
IGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHL=
D|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|=
SIGUSR1|SIGUSR2
},{ }) =3D 0 (0x0)
sigaction(SIGALRM,{ 0x800288380 SA_SIGINFO ss_t },{ SIG_DFL 0x0 ss_t }) =3D=
0
(0x0)
sigprocmask(SIG_SETMASK,{ },0x0) =3D 0 (0x0)
sigprocmask(SIG_SETMASK,{
SIGHUP|SIGINT|SIGQUIT|SIGILL|SIGTRAP|SIGABRT|SIGEMT|SIGFPE|SIGKILL|SIGBUS|S=
IGSEGV|SIGSYS|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHL=
D|SIGTTIN|SIGTTOU|SIGIO|SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|=
SIGUSR1|SIGUSR2
},{ }) =3D 0 (0x0)
sigaction(SIGPIPE,{ SIG_IGN SA_RESTART ss_t },{ SIG_DFL 0x0 ss_t }) =3D 0
(0x0)
sigprocmask(SIG_SETMASK,{ },0x0) =3D 0 (0x0)
mmap(0x0,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) =3D
34362286080 (0x80026e000)
mmap(0x0,147456,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) =3D
34376650752 (0x801021000)
mmap(0x7fffdfdfd000,2101248,PROT_READ|PROT_WRITE,MAP_STACK,-1,0x0) =3D
140736949374976 (0x7fffdfdfd000)
mprotect(0x7fffdfdfd000,4096,PROT_NONE) =3D 0 (0x0)
thr_new(0x7fffffffe880,0x68) =3D 0 (0x0)
<new thread 102901>
sigfastblock(0x1,0x800a12738) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) =3D 0 (0x0)
mmap(0x7fffdfbfc000,2101248,PROT_READ|PROT_WRITE,MAP_STACK,-1,0x0) =3D
140736947273728 (0x7fffdfbfc000)
mprotect(0x7fffdfbfc000,4096,PROT_NONE) =3D 0 (0x0)
thr_new(0x7fffffffe880,0x68) =3D 0 (0x0)
<new thread 102902>
sigfastblock(0x1,0x800a12e38) =3D 0 (0x0)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(21),=
-1,0x0)
=3D 34378612736 (0x801200000)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(21),=
-1,0x0)
=3D 34380709888 (0x801400000)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(12),=
-1,0x0)
=3D 34382807040 (0x801600000)
mmap(0x0,2097152,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(12),=
-1,0x0)
=3D 34384904192 (0x801800000)
socket(PF_INET,SOCK_STREAM,0) =3D 3 (0x3)
bind(3,{ AF_INET 172.16.70.1:0 },16) =3D 0 (0x0)
connect(3,{ AF_INET 172.16.68.1:5001 },16) =3D 0 (0x0)
setsockopt(3,SOL_SOCKET,SO_SNDTIMEO,0x7fffdfdfced0,16) =3D 0 (0x0)
getsockname(3,{ AF_INET 172.16.70.1:34901 },0x800a1b200) =3D 0 (0x0)
getpeername(3,{ AF_INET 172.16.68.1:5001 },0x800a1b178) =3D 0 (0x0)
getsockname(3,{ AF_INET 172.16.70.1:34901 },0x7fffdfdfce3c) =3D 0 (0x0)
__sysctl("net.routetable.0.0.5.0",6,0x0,0x7fffdfdfcdc8,0x0,0) =3D 0 (0x0)
__sysctl("net.routetable.0.0.5.0",6,0x801826000,0x7fffdfdfcdc8,0x0,0) =3D 0
(0x0)
_umtx_op(0x80026e008,UMTX_OP_WAKE_PRIVATE,0x7fffffff,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x80026e008,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfffdec8) =3D=
0
(0x0)
getpid() =3D 7030 (0x1b76)
getsockopt(3,IPPROTO_TCP,TCP_MAXSEG,0x7fffdfdfceb4,0x7fffdfdfceb0) =3D 0 (0=
x0)
getsockopt(3,SOL_SOCKET,SO_SNDBUF,0x7fffdfdfceb4,0x7fffdfdfceb0) =3D 0 (0x0=
)
fstat(1,{ mode=3Dp--------- ,inode=3D12766,size=3D0,blksize=3D4096 }) =3D 0=
(0x0)
mmap(0x0,2621440,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON|MAP_ALIGNED(12),=
-1,0x0)
=3D 34387001344 (0x801a00000)
sendto(3,"@\^A\0x\0\0\0\^A\0\0\^S\M^I\0\0"...,60,MSG_DONTWAIT,NULL,0) =3D 6=
0
(0x3c)
getsockopt(3,SOL_SOCKET,SO_SNDBUF,0x7fffdfffde64,0x7fffdfffde60) =3D 0 (0x0=
)
setsockopt(3,SOL_SOCKET,SO_SNDTIMEO,0x7fffdfdfced0,16) =3D 0 (0x0)
------------------------------------------------------------
Client connecting to 172.16.68.1, TCP port 5001
TCP window size: 33.2 KByte (default)
------------------------------------------------------------
write(1,"--------------------------------"...,208) =3D 208 (0xd0)
[ 1] local 172.16.70.1 port 34901 connected with 172.16.68.1 port 5001
write(1,"[ 1] local 172.16.70.1 port 349"...,72) =3D 72 (0x48)
write(3,"@\^A\0x\0\0\0\^A\0\0\^S\M^I\0\0"...,131072) =3D 34000 (0x84d0)
write(3,"@\^A\0x\0\0\0\^A\0\0\^S\M^I\0\0"...,131072) ERR#35 'Resource
temporarily unavailable'
setitimer(0,{ 0.000000, 0.000000 },0x0) =3D 0 (0x0)
shutdown(3,SHUT_WR) =3D 0 (0x0)
setsockopt(3,SOL_SOCKET,SO_RCVTIMEO,0x7fffdfdfce60,16) =3D 0 (0x0)
recvfrom(3,"\0\0\0\^B\0\0\0\^\\0\0\0\0\0\^B"...,131072,0,NULL,0x0) =3D 28
(0x1c)
recvfrom(3,0x8018048c0,131072,0,0x0,0x0) ERR#35 'Resource temporarily
unavailable'
_umtx_op(0x80026e020,UMTX_OP_WAKE_PRIVATE,0x7fffffff,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x80026e020,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfdfcd38) =3D=
0
(0x0)
[ ID] Interval Transfer Bandwidth
[ 1] 0.00-20.05 sec 33.3 KBytes 13.6 Kbits/sec
write(1,"[ ID] Interval Transfer "...,94) =3D 94 (0x5e)
_umtx_op(0x80026e020,UMTX_OP_WAKE_PRIVATE,0x7fffffff,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x80026e020,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfdfcd38) =3D=
0
(0x0)
close(3) =3D 0 (0x0)
_umtx_op(0x800a130c0,UMTX_OP_NWAKE_PRIVATE,0x1,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x8002908a8,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x0,0x0) =3D 0 (0x0)
__sysctl("kern.ostype",2,0x80024be6a,0x7fffdfdfb720,0x0,0) =3D 0 (0x0)
__sysctl("kern.hostname",2,0x80024bf6a,0x7fffdfdfb720,0x0,0) =3D 0 (0x0)
__sysctl("kern.osrelease",2,0x80024c06a,0x7fffdfdfb720,0x0,0) =3D 0 (0x0)
__sysctl("kern.version",2,0x80024c16a,0x7fffdfdfb720,0x0,0) =3D 0 (0x0)
__sysctl("hw.machine",2,0x80024c26a,0x7fffdfdfb720,0x0,0) =3D 0 (0x0)
open("/lib/libgcc_s.so.1",O_RDONLY|O_CLOEXEC|O_VERIFY,00) =3D 3 (0x3)
fstat(3,{ mode=3D-r--r--r-- ,inode=3D186687,size=3D93448,blksize=3D32768 })=
=3D 0
(0x0)
close(3) =3D 0 (0x0)
madvise(0x801a00000,1839104,MADV_FREE) =3D 0 (0x0)
madvise(0x80184f000,4096,MADV_FREE) =3D 0 (0x0)
madvise(0x801851000,36864,MADV_FREE) =3D 0 (0x0)
madvise(0x801801000,274432,MADV_FREE) =3D 0 (0x0)
<thread 102902 exited>
_umtx_op(0x80026e008,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x18,0x7fffdfffdec8)
ERR#60 'Operation timed out'
_umtx_op(0x800a129c0,UMTX_OP_NWAKE_PRIVATE,0x1,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x8002908a8,UMTX_OP_WAIT_UINT_PRIVATE,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_UNLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_WRLOCK,0x0,0x0,0x0) =3D 0 (0x0)
_umtx_op(0x800290d80,UMTX_OP_RW_RDLOCK,0x0,0x0,0x0) =3D 0 (0x0)
<thread 102901 exited>
exit(0x0)
process exit, rval =3D 0
On Sat, May 1, 2021 at 1:26 AM =C3=96zkan KIRIK <ozkan.kirik@gmail.com> wro=
te:
> Here is fresh results;
> a bit performance gain achieved. But strangely QAT is better now.
>
> - with CCR - 2.14 Gbps
> PID USERNAME PRI NICE SIZE RES STATE C TIME WCPU COMMAN=
D
> 15 root -16 - 0B 16K CPU7 7 0:11 100.00% [crypt=
o
> returns 9]
> 11 root -92 - 0B 1120K CPU4 4 0:10 98.74%
> [intr{irq295: t6nex0:0a0}]
> 14 root -16 - 0B 16K CPU11 11 0:09 95.32% [crypt=
o
> returns 8]
> 11 root -92 - 0B 1120K CPU8 8 0:07 63.49%
> [intr{irq297: t6nex0:0a2}]
> 11 root -92 - 0B 1120K WAIT 12 0:05 42.73%
> [intr{irq307: t6nex0:1a2}]
> 16 root -16 - 0B 16K CPU15 15 0:04 33.82% [crypt=
o
> returns 10]
> 13 root -16 - 0B 16K RUN 3 0:02 21.78% [crypt=
o
> returns 7]
> 4317 root 22 0 23M 4348K sbwait 14 0:01 7.24% iperf
> -B 172.16.70.6 -c 172.16.68.1 -P 2 -t 20{iperf}
> 4329 root 23 0 23M 4348K sbwait 14 0:01 7.18% iperf
> -B 172.16.70.10 -c 172.16.68.1 -P 2 -t 20{iperf}
> ...
> 11 root -92 - 0B 1120K WAIT 3 0:00 0.31%
> [intr{irq294: t6nex0:evt}]
> 0 root -92 - 0B 2144K - 14 0:00 0.30%
> [kernel{t6nex0 tq0}]
> ...
>
>
> - with QAT - 2.63 Gbps
> PID USERNAME PRI NICE SIZE RES STATE C TIME WCPU COMMAND
> 8 root -16 - 0B 16K CPU7 7 0:31 92.05% [crypt=
o
> returns 5]
> 7 root -16 - 0B 16K CPU0 0 0:29 86.03% [crypt=
o
> returns 4]
> 9 root -16 - 0B 16K RUN 15 0:15 43.32% [crypt=
o
> returns 6]
> 11 root -92 - 0B 1664K RUN 4 0:15 43.03%
> [intr{irq297: t6nex0:0a2}]
> 6 root -16 - 0B 16K crypto 4 0:09 26.01% [crypt=
o
> returns 3]
> 11 root -92 - 0B 1664K WAIT 12 0:07 19.88%
> [intr{irq307: t6nex0:1a2}]
> 11 root -92 - 0B 1664K WAIT 8 0:02 7.22%
> [intr{irq368: qat1}]
> 4313 root 22 0 23M 4348K sbwait 2 0:01 6.51% iperf
> -B 172.16.70.8 -c 172.16.68.1 -P 2 -t 20{iperf}
> 4316 root 22 0 23M 4348K sbwait 3 0:01 6.18% iperf
> -B 172.16.70.9 -c 172.16.68.1 -P 2 -t 20{iperf}
> ...
> 11 root -92 - 0B 1664K WAIT 12 0:02 4.52%
> [intr{irq355: qat0}]
> ...
> 11 root -92 - 0B 1664K WAIT 12 0:01 1.89%
> [intr{irq372: qat1}]
> 11 root -92 - 0B 1664K WAIT 0 0:00 1.12%
> [intr{irq360: qat1}]
> 11 root -92 - 0B 1664K WAIT 2 0:00 1.11%
> [intr{irq362: qat1}]
> 11 root -92 - 0B 1664K WAIT 14 0:00 1.10%
> [intr{irq374: qat1}]
> 11 root -92 - 0B 1664K WAIT 4 0:00 1.10%
> [intr{irq364: qat1}]
> 11 root -92 - 0B 1664K WAIT 10 0:00 1.10%
> [intr{irq370: qat1}]
> 11 root -92 - 0B 1664K WAIT 7 0:00 1.09%
> [intr{irq367: qat1}]
> 11 root -92 - 0B 1664K WAIT 11 0:00 1.07%
> [intr{irq371: qat1}]
> 11 root -92 - 0B 1664K WAIT 5 0:00 1.04%
> [intr{irq365: qat1}]
> 11 root -92 - 0B 1664K WAIT 15 0:00 1.04%
> [intr{irq375: qat1}]
> 11 root -92 - 0B 1664K WAIT 1 0:00 1.04%
> [intr{irq361: qat1}]
> 11 root -92 - 0B 1664K WAIT 6 0:00 1.03%
> [intr{irq366: qat1}]
>
> On Sat, May 1, 2021 at 12:45 AM Mark Johnston <markj@freebsd.org> wrote:
>
>> On Sat, May 01, 2021 at 12:31:57AM +0300, =C3=96zkan KIRIK wrote:
>> > Hello again,
>> >
>> > patch is applied, now netisr is not eating CPU. but performance drops
>> > around 0.2Gbps according to previous kernel.
>> >
>> > I tried also both net.isr.maxthreads=3D1 and net.isr.maxthreads=3D4 .
>> results
>> > are same
>> >
>> > Results are:
>> >
>> > - with CCR - 1.8Gbps
>> > top:
>> > PID USERNAME PRI NICE SIZE RES STATE C TIME WCPU
>> COMMAND
>> > 14 root -16 - 0B 16K CPU5 5 1:38 100.00%
>> [crypto
>> > returns 8]
>> > 3 root -16 - 0B 16K CPU1 1 0:58 77.83%
>> [crypto
>> > returns 0]
>>
>> Could you also try this patch? It won't help with aesni.
>>
>> diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c
>> index dfd22662e87f..bc23056e86ad 100644
>> --- a/sys/opencrypto/crypto.c
>> +++ b/sys/opencrypto/crypto.c
>> @@ -94,6 +94,7 @@ struct crypto_session {
>> void *softc;
>> uint32_t hid;
>> uint32_t capabilities;
>> + uint64_t id;
>> };
>>
>> SDT_PROVIDER_DEFINE(opencrypto);
>> @@ -572,6 +573,7 @@ crypto_select_driver(const struct cryptoini *cri, in=
t
>> flags)
>> int
>> crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int
>> crid)
>> {
>> + static uint64_t sessid =3D 0;
>> crypto_session_t res;
>> void *softc_mem;
>> struct cryptocap *cap;
>> @@ -616,6 +618,7 @@ crypto_newsession(crypto_session_t *cses, struct
>> cryptoini *cri, int crid)
>> softc_mem =3D malloc(softc_size, M_CRYPTO_DATA, M_WAITOK | M_ZER=
O);
>> res =3D uma_zalloc(cryptoses_zone, M_WAITOK | M_ZERO);
>> res->softc =3D softc_mem;
>> + res->id =3D atomic_fetchadd_64(&sessid, 1);
>>
>> CRYPTO_DRIVER_LOCK();
>> cap =3D crypto_checkdriver(hid);
>> @@ -1016,7 +1019,7 @@ crypto_dispatch(struct cryptop *crp)
>> binuptime(&crp->crp_tstamp);
>> #endif
>>
>> - crp->crp_retw_id =3D ((uintptr_t)crp->crp_session) %
>> crypto_workers_num;
>> + crp->crp_retw_id =3D crp->crp_session->id % crypto_workers_num;
>>
>> if (CRYPTOP_ASYNC(crp)) {
>> if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
>>
>
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?CAAcX-AE_jRirL64tbL4ikRa4XDuvkeQgDObLqphJN7HtXyqwLg>