Skip site navigation (1)Skip section navigation (2) 
Date:      Thu, 30 Oct 2008 14:05:58 +0000 (UTC)
From:      Ivan Voras <ivoras@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r184471 - in head/bin: cat cp
Message-ID:  <200810301405.m9UE5wwZ053250@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: ivoras
Date: Thu Oct 30 14:05:57 2008
New Revision: 184471
URL: http://svn.freebsd.org/changeset/base/184471

Log:
  Teach cat(1) and cp(1) to use a larger buffer if enough memory is present
  in the system. A simple heuristics is used to detect what is "enough"
  memory: if number of physmem pages is greater than 32k (equalling 128 MB
  on machines with 4 kB pages).
  
  Typical immediate result of these changes is reduction in context switches
  and the goal is to increase efficiency by using large buffers:
  before: /usr/bin/time -hlp cat file1 > file2
  ...
         163  voluntary context switches
       11194  involuntary context switches
  after: /usr/bin/time -hlp ./cat file1 > file2
  ...
         417  voluntary context switches
         272  involuntary context switches
  
  Reviewed by:    hackers@ (no objections to earlier version of cat patch)
  Approved by:    gnn (mentor)
  MFC after:      4 months

Modified:
  head/bin/cat/cat.c
  head/bin/cp/utils.c

Modified: head/bin/cat/cat.c
==============================================================================
--- head/bin/cat/cat.c	Thu Oct 30 13:18:28 2008	(r184470)
+++ head/bin/cat/cat.c	Thu Oct 30 14:05:57 2008	(r184471)
@@ -77,6 +77,17 @@ static void raw_cat(int);
 static int udom_open(const char *path, int flags);
 #endif
 
+/* Memory strategy threshold, in pages: if physmem is larger then this, use a 
+ * large buffer */
+#define PHYSPAGES_THRESHOLD (32*1024)
+
+/* Maximum buffer size in bytes - do not allow it to grow larger than this */
+#define BUFSIZE_MAX (2*1024*1024)
+
+/* Small (default) buffer size in bytes. It's inefficient for this to be
+ * smaller than MAXPHYS */
+#define BUFSIZE_SMALL (MAXPHYS)
+
 int
 main(int argc, char *argv[])
 {
@@ -247,9 +258,17 @@ raw_cat(int rfd)
 	if (buf == NULL) {
 		if (fstat(wfd, &sbuf))
 			err(1, "%s", filename);
-		bsize = MAX(sbuf.st_blksize, 1024);
+		if (S_ISREG(sbuf.st_mode)) {
+			/* If there's plenty of RAM, use a large copy buffer */
+			if (sysconf(_SC_PHYS_PAGES) > PHYSPAGES_THRESHOLD)
+				bsize = MIN(BUFSIZE_MAX, MAXPHYS*8);
+			else
+				bsize = BUFSIZE_SMALL;
+		} else
+			bsize = MAX(sbuf.st_blksize, 
+					(blksize_t)sysconf(_SC_PAGESIZE));
 		if ((buf = malloc(bsize)) == NULL)
-			err(1, "buffer");
+			err(1, "malloc() failure of IO buffer");
 	}
 	while ((nr = read(rfd, buf, bsize)) > 0)
 		for (off = 0; nr; nr -= nw, off += nw)

Modified: head/bin/cp/utils.c
==============================================================================
--- head/bin/cp/utils.c	Thu Oct 30 13:18:28 2008	(r184470)
+++ head/bin/cp/utils.c	Thu Oct 30 14:05:57 2008	(r184471)
@@ -57,10 +57,22 @@ __FBSDID("$FreeBSD$");
 
 #define	cp_pct(x, y)	((y == 0) ? 0 : (int)(100.0 * (x) / (y)))
 
+/* Memory strategy threshold, in pages: if physmem is larger then this, use a 
+ * large buffer */
+#define PHYSPAGES_THRESHOLD (32*1024)
+
+/* Maximum buffer size in bytes - do not allow it to grow larger than this */
+#define BUFSIZE_MAX (2*1024*1024)
+
+/* Small (default) buffer size in bytes. It's inefficient for this to be
+ * smaller than MAXPHYS */
+#define BUFSIZE_SMALL (MAXPHYS)
+
 int
 copy_file(const FTSENT *entp, int dne)
 {
-	static char buf[MAXBSIZE];
+	static char *buf = NULL;
+	static size_t bufsize;
 	struct stat *fs;
 	ssize_t wcount;
 	size_t wresid;
@@ -174,8 +186,23 @@ copy_file(const FTSENT *entp, int dne)
 		} else
 #endif
 		{
+			if (buf == NULL) {
+				/*
+				 * Note that buf and bufsize are static. If
+				 * malloc() fails, it will fail at the start
+				 * and not copy only some files. 
+				 */ 
+				if (sysconf(_SC_PHYS_PAGES) > 
+				    PHYSPAGES_THRESHOLD)
+					bufsize = MIN(BUFSIZE_MAX, MAXPHYS * 8);
+				else
+					bufsize = BUFSIZE_SMALL;
+				buf = malloc(bufsize);
+				if (buf == NULL)
+					err(1, "Not enough memory");
+			}
 			wtotal = 0;
-			while ((rcount = read(from_fd, buf, MAXBSIZE)) > 0) {
+			while ((rcount = read(from_fd, buf, bufsize)) > 0) {
 				for (bufp = buf, wresid = rcount; ;
 			    	bufp += wcount, wresid -= wcount) {
 					wcount = write(to_fd, bufp, wresid);

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200810301405.m9UE5wwZ053250>