From owner-svn-src-all@FreeBSD.ORG  Thu Jul 22 11:23:18 2010
Return-Path: <owner-svn-src-all@FreeBSD.ORG>
Delivered-To: svn-src-all@freebsd.org
Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34])
	by hub.freebsd.org (Postfix) with ESMTP id A60D0106564A;
	Thu, 22 Jul 2010 11:23:18 +0000 (UTC)
	(envelope-from simon@FreeBSD.org)
Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c])
	by mx1.freebsd.org (Postfix) with ESMTP id 8AF4B8FC17;
	Thu, 22 Jul 2010 11:23:18 +0000 (UTC)
Received: from svn.freebsd.org (localhost [127.0.0.1])
	by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id o6MBNIsJ030866;
	Thu, 22 Jul 2010 11:23:18 GMT (envelope-from simon@svn.freebsd.org)
Received: (from simon@localhost)
	by svn.freebsd.org (8.14.3/8.14.3/Submit) id o6MBNIl8030833;
	Thu, 22 Jul 2010 11:23:18 GMT (envelope-from simon@svn.freebsd.org)
Message-Id: <201007221123.o6MBNIl8030833@svn.freebsd.org>
From: "Simon L. Nielsen" <simon@FreeBSD.org>
Date: Thu, 22 Jul 2010 11:23:18 +0000 (UTC)
To: src-committers@freebsd.org, svn-src-all@freebsd.org,
	svn-src-head@freebsd.org
X-SVN-Group: head
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Cc: 
Subject: svn commit: r210372 - head/usr.sbin/newsyslog
X-BeenThere: svn-src-all@freebsd.org
X-Mailman-Version: 2.1.5
Precedence: list
List-Id: "SVN commit messages for the entire src tree \(except for &quot;
	user&quot; and &quot; projects&quot; \)" <svn-src-all.freebsd.org>
List-Unsubscribe: <http://lists.freebsd.org/mailman/listinfo/svn-src-all>,
	<mailto:svn-src-all-request@freebsd.org?subject=unsubscribe>
List-Archive: <http://lists.freebsd.org/pipermail/svn-src-all>
List-Post: <mailto:svn-src-all@freebsd.org>
List-Help: <mailto:svn-src-all-request@freebsd.org?subject=help>
List-Subscribe: <http://lists.freebsd.org/mailman/listinfo/svn-src-all>,
	<mailto:svn-src-all-request@freebsd.org?subject=subscribe>
X-List-Received-Date: Thu, 22 Jul 2010 11:23:18 -0000

Author: simon
Date: Thu Jul 22 11:23:18 2010
New Revision: 210372
URL: http://svn.freebsd.org/changeset/base/210372

Log:
  Add support for creating the archived log filenames using a time-stamp
  instead of the traditional simple counter.
  
  Using the time-stamp based file-names, once a log file is archived, it
  will not change name until it is deleted.  This means that many backup
  systems will only perform one backup of the archived log file, instead
  for performing a new backup of the logfile upon each logfile rotation.
  
  This implementation is separate from the patches in the mentioned PR,
  as I wasn't aware of the existence of the PR until after I had
  implemented the same functionality as the patches in the PR provide.
  Unlike the PR, this new code does honor the 'log count' in
  newsyslog.conf so old logfiles are deleted.  This new code does not
  currently support never deleting the archived logfiles.
  
  PR:		bin/29363
  MFC after:	3 weeks

Modified:
  head/usr.sbin/newsyslog/newsyslog.8
  head/usr.sbin/newsyslog/newsyslog.c

Modified: head/usr.sbin/newsyslog/newsyslog.8
==============================================================================
--- head/usr.sbin/newsyslog/newsyslog.8	Thu Jul 22 10:24:28 2010	(r210371)
+++ head/usr.sbin/newsyslog/newsyslog.8	Thu Jul 22 11:23:18 2010	(r210372)
@@ -30,6 +30,7 @@
 .Op Fl a Ar directory
 .Op Fl d Ar directory
 .Op Fl f Ar config_file
+.Op Fl t Ar timefmt
 .Op Ar
 .Sh DESCRIPTION
 The
@@ -50,6 +51,11 @@ the last period's logs in it,
 has the next to last
 period's logs in it, and so on, up to a user-specified number of
 archived logs.
+It is also possible to let archived log filenames be created using the
+time the log file was archived instead of the sequential number using
+the
+.Fl t
+option.
 Optionally the archived logs can be compressed to save
 space.
 .Pp
@@ -141,6 +147,31 @@ However, this option is most likely to b
 with the
 .Fl R
 option, and in that case the compression will be done.
+.It Fl t Ar timefmt
+If specified
+.Nm
+will create the
+.Dq rotated
+logfiles using the specified time format instead of the default
+sequential filenames.
+The time format is described in the
+.Xr strftime 3
+manual page.
+If the
+.Ar timefmt
+argument is set to an empty string or the string
+.Dq DEFAULT ,
+the default built in time format
+is used.
+If the
+.Ar timefmt
+string is changed the old files created using the previous time format
+will not be be automatically removed (unless the new format is very
+similar to the old format).
+This is also the case when changing from sequential filenames to time
+based file names, and the other way around.
+The time format should contain at least year, month, day, and hour to
+make sure rotating of old logfiles can select the correct logfiles.
 .It Fl C
 If specified once, then
 .Nm

Modified: head/usr.sbin/newsyslog/newsyslog.c
==============================================================================
--- head/usr.sbin/newsyslog/newsyslog.c	Thu Jul 22 10:24:28 2010	(r210371)
+++ head/usr.sbin/newsyslog/newsyslog.c	Thu Jul 22 11:23:18 2010	(r210372)
@@ -69,9 +69,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 #include <sys/wait.h>
 
+#include <assert.h>
 #include <ctype.h>
 #include <err.h>
 #include <errno.h>
+#include <dirent.h>
 #include <fcntl.h>
 #include <fnmatch.h>
 #include <glob.h>
@@ -80,6 +82,7 @@ __FBSDID("$FreeBSD$");
 #include <pwd.h>
 #include <signal.h>
 #include <stdio.h>
+#include <libgen.h>
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
@@ -112,6 +115,9 @@ __FBSDID("$FreeBSD$");
 #define	DEFAULT_MARKER	"<default>"
 #define	DEBUG_MARKER	"<debug>"
 #define	INCLUDE_MARKER	"<include>"
+#define	DEFAULT_TIMEFNAME_FMT	"%Y%m%dT%H%M%S"
+
+#define	MAX_OLDLOGS 65536	/* Default maximum number of old logfiles */
 
 struct conf_entry {
 	STAILQ_ENTRY(conf_entry) cf_nextp;
@@ -155,6 +161,11 @@ struct include_entry {
 	const char *file;	/* Name of file to process */
 };
 
+struct oldlog_entry {
+	char *fname;		/* Filename of the log file */
+	time_t t;		/* Parses timestamp of the logfile */
+};
+
 typedef enum {
 	FREE_ENT, KEEP_ENT
 }	fk_entry;
@@ -182,6 +193,7 @@ int rotatereq = 0;		/* -R = Always rotat
 				/*    that a list of files *are* given on  */
 				/*    the run command). */
 char *requestor;		/* The name given on a -R request */
+char *timefnamefmt = NULL;	/* Use time based filenames instead of .0 etc */
 char *archdirname;		/* Directory path to old logfiles archive */
 char *destdir = NULL;		/* Directory to treat at root for logs */
 const char *conf;		/* Configuration file to use */
@@ -585,7 +597,7 @@ parse_args(int argc, char **argv)
 		*p = '\0';
 
 	/* Parse command line options. */
-	while ((ch = getopt(argc, argv, "a:d:f:nrsvCD:FNPR:")) != -1)
+	while ((ch = getopt(argc, argv, "a:d:f:nrst:vCD:FNPR:")) != -1)
 		switch (ch) {
 		case 'a':
 			archtodir++;
@@ -606,6 +618,13 @@ parse_args(int argc, char **argv)
 		case 's':
 			nosignal = 1;
 			break;
+		case 't':
+			if (optarg[0] == '\0' ||
+			    strcmp(optarg, "DEFAULT") == 0)
+				timefnamefmt = strdup(DEFAULT_TIMEFNAME_FMT);
+			else
+				timefnamefmt = strdup(optarg);
+			break;
 		case 'v':
 			verbose++;
 			break;
@@ -728,7 +747,7 @@ usage(void)
 
 	fprintf(stderr,
 	    "usage: newsyslog [-CFNnrsv] [-a directory] [-d directory] [-f config-file]\n"
-	    "                 [ [-R requestor] filename ... ]\n");
+	    "                 [-t timefmt ] [ [-R requestor] filename ... ]\n");
 	exit(1);
 }
 
@@ -1365,6 +1384,177 @@ missing_field(char *p, char *errline)
 }
 
 /*
+ * In our sort we return it in the reverse of what qsort normally
+ * would do, as we want the newest files first.  If we have two
+ * entries with the same time we don't really care about order.
+ *
+ * Support function for qsort() in delete_oldest_timelog().
+ */
+static int
+oldlog_entry_compare(const void *a, const void *b)
+{
+	const struct oldlog_entry *ola = a, *olb = b;
+
+	if (ola->t > olb->t)
+		return (-1);
+	else if (ola->t < olb->t)
+		return (1);
+	else
+		return (0);
+}
+
+/*
+ * Delete the oldest logfiles, when using time based filenames.
+ */
+static void
+delete_oldest_timelog(const struct conf_entry *ent, const char *archive_dir)
+{
+	char *logfname, *s, *dir, errbuf[80];
+	int logcnt, max_logcnt, dirfd, i;
+	struct oldlog_entry *oldlogs;
+	size_t logfname_len;
+	struct dirent *dp;
+	const char *cdir;
+	struct tm tm;
+	DIR *dirp;
+
+	oldlogs = malloc(MAX_OLDLOGS * sizeof(struct oldlog_entry));
+	max_logcnt = MAX_OLDLOGS;
+	logcnt = 0;
+
+	if (archive_dir != NULL && archive_dir[0] != '\0')
+		cdir = archive_dir;
+	else
+		if ((cdir = dirname(ent->log)) == NULL)
+			err(1, "dirname()");
+	if ((dir = strdup(cdir)) == NULL)
+		err(1, "strdup()");
+
+	if ((s = basename(ent->log)) == NULL)
+		err(1, "basename()");
+	if ((logfname = strdup(s)) == NULL)
+		err(1, "strdup()");
+	logfname_len = strlen(logfname);
+	if (strcmp(logfname, "/") == 0)
+		errx(1, "Invalid log filename - became '/'");
+
+	if (verbose > 2)
+		printf("Searching for old logs in %s\n", dir);
+
+	/* First we create a 'list' of all archived logfiles */
+	if ((dirp = opendir(dir)) == NULL)
+		err(1, "Cannot open log directory '%s'", dir);
+	dirfd = dirfd(dirp);
+	while ((dp = readdir(dirp)) != NULL) {
+		if (dp->d_type != DT_REG)
+			continue;
+
+		/* Ignore everything but files with our logfile prefix */
+		if (strncmp(dp->d_name, logfname, logfname_len) != 0)
+			continue;
+		/* Ignore the actual non-rotated logfile */
+		if (dp->d_namlen == logfname_len)
+			continue;
+		/*
+		 * Make sure we created have found a logfile, so the
+		 * postfix is valid, IE format is: '.<time>(.[bg]z)?'.
+		 */
+		if (dp->d_name[logfname_len] != '.') {
+			if (verbose)
+				printf("Ignoring %s which has unexpected "
+				    "extension '%s'\n", dp->d_name,
+				    &dp->d_name[logfname_len]);
+			continue;
+		}
+		if ((s = strptime(&dp->d_name[logfname_len + 1],
+			    timefnamefmt, &tm)) == NULL) {
+			/*
+			 * We could special case "old" sequentially
+			 * named logfiles here, but we do not as that
+			 * would require special handling to decide
+			 * which one was the oldest compared to "new"
+			 * time based logfiles.
+			 */
+			if (verbose)
+				printf("Ignoring %s which does not "
+				    "match time format\n", dp->d_name);
+			continue;
+		}
+		if (*s != '\0' && !(strcmp(s, BZCOMPRESS_POSTFIX) == 0 ||
+			strcmp(s, COMPRESS_POSTFIX) == 0))  {
+			    if (verbose)
+				printf("Ignoring %s which has unexpected "
+				    "extension '%s'\n", dp->d_name, s);
+			continue;
+		}
+
+		/*
+		 * We should now have old an old rotated logfile, so
+		 * add it to the 'list'.
+		 */
+		if ((oldlogs[logcnt].t = timegm(&tm)) == -1)
+			err(1, "Could not convert time string to time value");
+		if ((oldlogs[logcnt].fname = strdup(dp->d_name)) == NULL)
+			err(1, "strdup()");
+		logcnt++;
+
+		/*
+		 * It is very unlikely we ever run out of space in the
+		 * logfile array from the default size, but lets
+		 * handle it anyway...
+		 */
+		if (logcnt >= max_logcnt) {
+			max_logcnt *= 4;
+			/* Detect integer overflow */
+			if (max_logcnt < logcnt)
+				errx(1, "Too many old logfiles found");
+			oldlogs = realloc(oldlogs,
+			    max_logcnt * sizeof(struct oldlog_entry));
+			if (oldlogs == NULL)
+				err(1, "realloc()");
+		}
+	}
+
+	/* Second, if needed we delete oldest archived logfiles */
+	if (logcnt > 0 && logcnt >= ent->numlogs && ent->numlogs > 1) {
+		oldlogs = realloc(oldlogs, logcnt *
+		    sizeof(struct oldlog_entry));
+		if (oldlogs == NULL)
+			err(1, "realloc()");
+
+		/*
+		 * We now sort the logs in the order of newest to
+		 * oldest.  That way we can simply skip over the
+		 * number of records we want to keep.
+		 */
+		qsort(oldlogs, logcnt, sizeof(struct oldlog_entry),
+		    oldlog_entry_compare);
+		for (i = ent->numlogs - 1; i < logcnt; i++) {
+			if (noaction)
+				printf("\trm -f %s/%s\n", dir,
+				    oldlogs[i].fname);
+			else if (unlinkat(dirfd, oldlogs[i].fname, 0) != 0) {
+				snprintf(errbuf, sizeof(errbuf),
+				    "Could not delet old logfile '%s'",
+				    oldlogs[i].fname);
+				perror(errbuf);
+			}
+		}
+	} else if (verbose > 1)
+		printf("No old logs to delete for logfile %s\n", ent->log);
+
+	/* Third, cleanup */
+	closedir(dirp);
+	for (i = 0; i < logcnt; i++) {
+		assert(oldlogs[i].fname != NULL);
+		free(oldlogs[i].fname);
+	}
+	free(oldlogs);
+	free(logfname);
+	free(dir);
+}
+
+/*
  * Only add to the queue if the file hasn't already been added. This is
  * done to prevent circular include loops.
  */
@@ -1398,10 +1588,13 @@ do_rotate(const struct conf_entry *ent)
 	char file1[MAXPATHLEN], file2[MAXPATHLEN];
 	char zfile1[MAXPATHLEN], zfile2[MAXPATHLEN];
 	char jfile1[MAXPATHLEN];
+	char datetimestr[30];
 	int flags, numlogs_c;
 	fk_entry free_or_keep;
 	struct sigwork_entry *swork;
 	struct stat st;
+	struct tm tm;
+	time_t now;
 
 	flags = ent->flags;
 	free_or_keep = FREE_ENT;
@@ -1435,32 +1628,59 @@ do_rotate(const struct conf_entry *ent)
 		/* name of oldest log */
 		(void) snprintf(file1, sizeof(file1), "%s/%s.%d", dirpart,
 		    namepart, ent->numlogs);
-		(void) snprintf(zfile1, sizeof(zfile1), "%s%s", file1,
-		    COMPRESS_POSTFIX);
-		snprintf(jfile1, sizeof(jfile1), "%s%s", file1,
-		    BZCOMPRESS_POSTFIX);
 	} else {
+		/*
+		 * Tell delete_oldest_timelog() we are not using an
+		 * archive dir.
+		 */
+		dirpart[0] = '\0';
+
 		/* name of oldest log */
 		(void) snprintf(file1, sizeof(file1), "%s.%d", ent->log,
 		    ent->numlogs);
+	}
+
+	/* Delete old logs */
+	if (timefnamefmt != NULL)
+		delete_oldest_timelog(ent, dirpart);
+	else {
+		/* name of oldest log */
 		(void) snprintf(zfile1, sizeof(zfile1), "%s%s", file1,
 		    COMPRESS_POSTFIX);
 		snprintf(jfile1, sizeof(jfile1), "%s%s", file1,
 		    BZCOMPRESS_POSTFIX);
-	}
 
-	if (noaction) {
-		printf("\trm -f %s\n", file1);
-		printf("\trm -f %s\n", zfile1);
-		printf("\trm -f %s\n", jfile1);
-	} else {
-		(void) unlink(file1);
-		(void) unlink(zfile1);
-		(void) unlink(jfile1);
+		if (noaction) {
+			printf("\trm -f %s\n", file1);
+			printf("\trm -f %s\n", zfile1);
+			printf("\trm -f %s\n", jfile1);
+		} else {
+			(void) unlink(file1);
+			(void) unlink(zfile1);
+			(void) unlink(jfile1);
+		}
 	}
 
+	if (timefnamefmt != NULL) {
+		/* If time functions fails we can't really do any sensible */
+		if (time(&now) == (time_t)-1 ||
+		    localtime_r(&now, &tm) == NULL)
+			bzero(&tm, sizeof(tm));
+
+		strftime(datetimestr, sizeof(datetimestr), timefnamefmt, &tm);
+		if (archtodir)
+			(void) snprintf(file1, sizeof(file1), "%s/%s.%s",
+			    dirpart, namepart, datetimestr);
+		else
+			(void) snprintf(file1, sizeof(file1), "%s.%s",
+			    ent->log, datetimestr);
+
+		/* Don't run the code to move down logs */
+		numlogs_c = 0;
+	} else
+		numlogs_c = ent->numlogs;		/* copy for countdown */
+
 	/* Move down log files */
-	numlogs_c = ent->numlogs;		/* copy for countdown */
 	while (numlogs_c--) {
 
 		(void) strlcpy(file2, file1, sizeof(file2));