Date: Sun, 10 May 2009 18:19:16 +0000 (UTC) From: Peter Holm <pho@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r191969 - projects/stress2/misc Message-ID: <200905101819.n4AIJGWT064893@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: pho Date: Sun May 10 18:19:16 2009 New Revision: 191969 URL: http://svn.freebsd.org/changeset/base/191969 Log: Added test scenarios for page faults while doing filesystem data move to or from userspace during read(2) and write(2) Added: projects/stress2/misc/datamove.sh (contents, props changed) projects/stress2/misc/datamove2.sh (contents, props changed) projects/stress2/misc/datamove3.sh (contents, props changed) Added: projects/stress2/misc/datamove.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/stress2/misc/datamove.sh Sun May 10 18:19:16 2009 (r191969) @@ -0,0 +1,213 @@ +#!/bin/sh + +# +# Copyright (c) 2009 Peter Holm <pho@FreeBSD.org> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ +# + +# There is a well-known problem in FreeBSD, caused by allowing page faults +# while doing filesystem data move to or from userspace during read(2) and +# write(2). The issue is that if the userspace address being read or write +# from/to is backed by the mapping of the same file we are doing i/o to, +# we deadlock. + +# Test scenario by ups + +[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 + +here=`pwd` +cd /tmp +sed '1,/^EOF/d' < $here/$0 > dl.c +cc -o dl -Wall dl.c +rm -f dl.c + +n=5 +old=`sysctl vm.old_msync | awk '{print $NF}'` +sysctl vm.old_msync=1 +for i in `jot $n`; do + mkdir -p /tmp/dl.dir.$i + cd /tmp/dl.dir.$i + /tmp/dl & +done +cd /tmp +for i in `jot $n`; do + wait +done +for i in `jot $n`; do + rm -rf /tmp/dl.dir.$i +done +sysctl vm.old_msync=$old + +rm -rf /tmp/dl +exit 0 +EOF +#include <sys/types.h> +#include <unistd.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/mman.h> + + +int prepareFile(char* filename,int* fdp); +int mapBuffer(char** bufferp,int fd1,int fd2); +int startIO(int fd,char *buffer); + +int pagesize; + +#define FILESIZE (32*1024) +char wbuffer[FILESIZE]; + +/* Create a FILESIZE sized file - then remove file data from the cache*/ +int prepareFile(char* filename,int* fdp) +{ + int fd; + int len; + int status; + void *addr; + + fd = open(filename,O_CREAT | O_TRUNC | O_RDWR,S_IRWXU); + if (fd == -1) + { + perror("Creating file"); + return fd; + } + + len = write(fd,wbuffer,FILESIZE); + if (len < 0) + { + perror("Write failed"); + return 1; + } + + status = fsync(fd); + if (status != 0) + { + perror("fsync failed"); + return 1; + } + + addr = mmap(NULL,FILESIZE, PROT_READ | PROT_WRITE , MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) + { + perror("Mmap failed"); + return 1; + } + + status = msync(addr,FILESIZE,MS_INVALIDATE | MS_SYNC); + if (status != 0) + { + perror("Msync failed"); + return 1; + } + + munmap(addr,FILESIZE); + + *fdp = fd; + return 0; +} + + +/* mmap a 2 page buffer - first page is from fd1, second page from fd2 */ +int mapBuffer(char** bufferp,int fd1,int fd2) +{ + void* addr; + char *buffer; + + addr = mmap(NULL,pagesize*2, PROT_READ | PROT_WRITE , MAP_SHARED, fd1, 0); + if (addr == MAP_FAILED) + { + perror("Mmap failed"); + return 1; + } + + buffer = addr; + addr = mmap(buffer + pagesize,pagesize, PROT_READ | PROT_WRITE , MAP_FIXED | +MAP_SHARED, fd2, 0); + + if (addr == MAP_FAILED) + { + perror("Mmap2 failed"); + return 1; + } + *bufferp = buffer; + return 0; +} + + +int startIO(int fd,char *buffer) +{ + ssize_t len; + len = write(fd,buffer,2*pagesize); + if (len == -1) + { + perror("write failed"); + return 1; + } + return 0; +} + + +int main(int argc,char *argv[],char *envp[]) +{ + + int fdA,fdB,fdDelayA,fdDelayB; + int status; + char *bufferA,*bufferB; + pid_t pid; + + pagesize = getpagesize(); + + if ((prepareFile("A",&fdA)) + || (prepareFile("B",&fdB)) + || (prepareFile("DelayA",&fdDelayA)) + || (prepareFile("DelayB",&fdDelayB)) + || (mapBuffer(&bufferA,fdDelayA,fdB)) + || (mapBuffer(&bufferB,fdDelayB,fdA))) + exit(1); + + pid = fork(); + + if (pid == 0) + { + status = startIO(fdA,bufferA); + exit(status); + } + + if (pid == -1) + { + exit(1); + } + status = startIO(fdB,bufferB); + exit(status); + +} + + + + + Added: projects/stress2/misc/datamove2.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/stress2/misc/datamove2.sh Sun May 10 18:19:16 2009 (r191969) @@ -0,0 +1,210 @@ +#!/bin/sh + +# +# Copyright (c) 2009 Peter Holm <pho@FreeBSD.org> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ +# + +# Variation of the datamove.sh scenario by not using "sysctl vm.old_msync=1" + +# Deadlock seen + +# Test scenario by ups + +here=`pwd` +cd /tmp +sed '1,/^EOF/d' < $here/$0 > dl.c +cc -o dl -Wall dl.c +rm -f dl.c + +for i in `jot 3`; do + $here/../testcases/swap/swap -t 10m -i 200 -h & + /tmp/dl + ps | grep swap | grep -v swap | awk '{print $1}' | xargs kill +done +rm -rf /tmp/dl +exit 0 +EOF +#include <err.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + + +int prepareFile(char *filename, int *fdp); +int mapBuffer (char **bufferp, int fd1, int fd2); +int startIO (int fd, char *buffer); + +int pagesize; + +#define FILESIZE (32*1024) +char wbuffer [FILESIZE]; + +/* Create a FILESIZE sized file - then remove file data from the cache */ +int +prepareFile(char *filename, int *fdp) +{ + int fd; + int len; + int status; + void *addr; + + fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU); + if (fd == -1) { + perror("Creating file"); + return fd; + } + len = write(fd, wbuffer, FILESIZE); + if (len < 0) { + perror("Write failed"); + return 1; + } + status = fsync(fd); + if (status != 0) { + perror("fsync failed"); + return 1; + } + addr = mmap(NULL, FILESIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + perror("Mmap failed"); + return 1; + } + status = msync(addr, FILESIZE, MS_INVALIDATE | MS_SYNC); + if (status != 0) { + perror("Msync failed"); + return 1; + } + if (munmap(addr, FILESIZE) == -1) { + perror("munmap failed"); + return 1; + } + + *fdp = fd; + return 0; +} + + +/* mmap a 2 page buffer - first page is from fd1, second page from fd2 */ +int +mapBuffer(char **bufferp, int fd1, int fd2) +{ + void *addr; + char *buffer; + + addr = mmap(NULL, pagesize * 2, PROT_READ | PROT_WRITE, MAP_SHARED, fd1, 0); + if (addr == MAP_FAILED) { + perror("Mmap failed"); + return 1; + } + buffer = addr; + addr = mmap(buffer + pagesize, pagesize, PROT_READ | PROT_WRITE, MAP_FIXED | + MAP_SHARED, fd2, 0); + + if (addr == MAP_FAILED) { + perror("Mmap2 failed"); + return 1; + } + *bufferp = buffer; + return 0; +} + +void +unmapBuffer(char *bufferp) +{ + if (munmap(bufferp, pagesize * 2) == -1) + err(1, "unmap 1. buffer"); + if (munmap(bufferp + pagesize * 2, pagesize * 2) == -1) + err(1, "unmap 2. buffer"); +} + +int +startIO(int fd, char *buffer) +{ + ssize_t len; + + len = write(fd, buffer, 2 * pagesize); + if (len == -1) { + perror("write failed"); + return 1; + } + return 0; +} + + +int +main(int argc, char *argv[], char *envp[]) +{ + + int fdA, fdB, fdDelayA, fdDelayB; + int status; + int i; + char *bufferA, *bufferB; + pid_t pid; + + pagesize = getpagesize(); + + for (i = 0; i < 1000; i++) { + if ((prepareFile("A", &fdA)) + || (prepareFile("B", &fdB)) + || (prepareFile("DelayA", &fdDelayA)) + || (prepareFile("DelayB", &fdDelayB)) + || (mapBuffer(&bufferA, fdDelayA, fdB)) + || (mapBuffer(&bufferB, fdDelayB, fdA))) + exit(1); + + pid = fork(); + + if (pid == 0) { + status = startIO(fdA, bufferA); + exit(status); + } + if (pid == -1) { + perror("fork"); + exit(1); + } + status = startIO(fdB, bufferB); + if (wait(&status) == -1) + err(1, "wait"); + + close(fdA); + close(fdB); + close(fdDelayA); + close(fdDelayB); + unmapBuffer(bufferA); + unmapBuffer(bufferB); + unlink("A"); + unlink("B"); + unlink("DelayA"); + unlink("DelayB"); + } + exit(status); + +} Added: projects/stress2/misc/datamove3.sh ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/stress2/misc/datamove3.sh Sun May 10 18:19:16 2009 (r191969) @@ -0,0 +1,219 @@ +#!/bin/sh + +# +# Copyright (c) 2009 Peter Holm <pho@FreeBSD.org> +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ +# + +# Threaded variation of datamove.sh + +# Based on a test scenario by ups and suggestions by kib + +[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 + +here=`pwd` +cd /tmp +sed '1,/^EOF/d' < $here/$0 > dl.c +cc -o dl -Wall dl.c -lpthread +rm -f dl.c + +n=5 +old=`sysctl vm.old_msync | awk '{print $NF}'` +sysctl vm.old_msync=1 +for i in `jot $n`; do + mkdir -p /tmp/dl.dir.$i + cd /tmp/dl.dir.$i + /tmp/dl & +done +cd /tmp +for i in `jot $n`; do + wait +done +for i in `jot $n`; do + rm -rf /tmp/dl.dir.$i +done +sysctl vm.old_msync=$old + +rm -rf /tmp/dl +exit 0 +EOF +#include <err.h> +#include <fcntl.h> +#include <pthread.h> +#include <stdio.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +struct args { + char *bp; + int fd1; + int fd2; +} a[2]; + +int prepareFile(char *, int *); +void * mapBuffer(void *); +int startIO(int, char *); + +int pagesize; + +#define FILESIZE (32*1024) +char wbuffer [FILESIZE]; + +/* Create a FILESIZE sized file - then remove file data from the cache */ +int +prepareFile(char *filename, int *fdp) +{ + int fd; + int len; + int status; + void *addr; + + fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, S_IRWXU); + if (fd == -1) { + perror("Creating file"); + return fd; + } + len = write(fd, wbuffer, FILESIZE); + if (len < 0) { + perror("Write failed"); + return 1; + } + status = fsync(fd); + if (status != 0) { + perror("fsync failed"); + return 1; + } + addr = mmap(NULL, FILESIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (addr == MAP_FAILED) { + perror("Mmap failed"); + return 1; + } + status = msync(addr, FILESIZE, MS_INVALIDATE | MS_SYNC); + if (status != 0) { + perror("Msync failed"); + return 1; + } + munmap(addr, FILESIZE); + + *fdp = fd; + return 0; +} + + +/* mmap a 2 page buffer - first page is from fd1, second page from fd2 */ +void * +mapBuffer(void *ar) +{ + void *addr; + char *buffer; + int i; + + i = (int )ar; + addr = mmap(NULL, pagesize * 2, PROT_READ | PROT_WRITE, MAP_SHARED, a[i].fd1, 0); + if (addr == MAP_FAILED) { + err(1, "Mmap failed"); + } + buffer = addr; + addr = mmap(buffer + pagesize, pagesize, PROT_READ | PROT_WRITE, MAP_FIXED | + MAP_SHARED, a[i].fd2, 0); + + if (addr == MAP_FAILED) { + err(1, "Mmap2 failed"); + } + a[i].bp = buffer; + sleep(1); + return (NULL); +} + + +int +startIO(int fd, char *buffer) +{ + ssize_t len; + + len = write(fd, buffer, 2 * pagesize); + if (len == -1) { + warn("startIO(%d, %p): write failed", fd, buffer); + return 1; + } + return 0; +} + + + +int +main(int argc, char *argv[], char *envp[]) +{ + + int fdA, fdB, fdDelayA, fdDelayB; + int r, status; + char *bufferA, *bufferB; + pid_t pid; + pthread_t threads[2]; + + pagesize = getpagesize(); + + if ((prepareFile("A", &fdA)) + || (prepareFile("B", &fdB)) + || (prepareFile("DelayA", &fdDelayA)) + || (prepareFile("DelayB", &fdDelayB))) + exit(1); + + a[0].fd1 = fdDelayA; + a[0].fd2 = fdB; + + a[1].fd1 = fdDelayB; + a[1].fd2 = fdA; + + if ((r = pthread_create(&threads[0], NULL, mapBuffer, (void *)0)) != 0) + err(1, "pthread_create(): %s\n", strerror(r)); + if ((r = pthread_create(&threads[1], NULL, mapBuffer, (void *)1)) != 0) + err(1, "pthread_create(): %s\n", strerror(r)); + + while (a[0].bp == NULL || a[1].bp == NULL) + pthread_yield(); + + bufferA = a[0].bp; + bufferB = a[1].bp; + + pid = fork(); + + if (pid == 0) { + status = startIO(fdA, bufferA); + exit(status); + } + if (pid == -1) { + exit(1); + } + status = startIO(fdB, bufferB); + exit(status); + +}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200905101819.n4AIJGWT064893>