Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 25 Oct 2017 00:51:00 +0000 (UTC)
From:      Mark Johnston <markj@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r324965 - in head: etc/defaults etc/rc.d sbin/dumpon sbin/savecore share/man/man5 sys/dev/null sys/geom sys/kern sys/sys
Message-ID:  <201710250051.v9P0p0Tl052928@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: markj
Date: Wed Oct 25 00:51:00 2017
New Revision: 324965
URL: https://svnweb.freebsd.org/changeset/base/324965

Log:
  Add support for compressed kernel dumps.
  
  When using a kernel built with the GZIO config option, dumpon -z can be
  used to configure gzip compression using the in-kernel copy of zlib.
  This is useful on systems with large amounts of RAM, which require a
  correspondingly large dump device. Recovery of compressed dumps is also
  faster since fewer bytes need to be copied from the dump device.
  
  Because we have no way of knowing the final size of a compressed dump
  until it is written, the kernel will always attempt to dump when
  compression is configured, regardless of the dump device size. If the
  dump is aborted because we run out of space, an error is reported on
  the console.
  
  savecore(8) is modified to handle compressed dumps and save them to
  vmcore.<index>.gz, as it does when given the -z option.
  
  A new rc.conf variable, dumpon_flags, is added. Its value is added to
  the boot-time dumpon(8) invocation that occurs when a dump device is
  configured in rc.conf.
  
  Reviewed by:	cem (earlier version)
  Discussed with:	def, rgrimes
  Relnotes:	yes
  Sponsored by:	Dell EMC Isilon
  Differential Revision:	https://reviews.freebsd.org/D11723

Modified:
  head/etc/defaults/rc.conf
  head/etc/rc.d/dumpon
  head/sbin/dumpon/dumpon.8
  head/sbin/dumpon/dumpon.c
  head/sbin/savecore/savecore.8
  head/sbin/savecore/savecore.c
  head/share/man/man5/rc.conf.5
  head/sys/dev/null/null.c
  head/sys/geom/geom_dev.c
  head/sys/kern/kern_gzio.c
  head/sys/kern/kern_shutdown.c
  head/sys/sys/conf.h
  head/sys/sys/disk.h
  head/sys/sys/gzio.h
  head/sys/sys/kerneldump.h

Modified: head/etc/defaults/rc.conf
==============================================================================
--- head/etc/defaults/rc.conf	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/etc/defaults/rc.conf	Wed Oct 25 00:51:00 2017	(r324965)
@@ -596,9 +596,8 @@ nscd_enable="NO"	# Run the nsswitch caching daemon.
 chkprintcap_enable="NO"	# Run chkprintcap(8) before running lpd.
 chkprintcap_flags="-d"	# Create missing directories by default.
 dumpdev="AUTO"		# Device to crashdump to (device name, AUTO, or NO).
+dumpon_flags=""		# Options to pass to dumpon(8), followed by dumpdev.
 dumpdir="/var/crash"	# Directory where crash dumps are to be stored
-dumppubkey=""		# Public key for encrypted kernel crash dumps.
-			# See dumpon(8) for more details.
 savecore_enable="YES"	# Extract core from dump devices if any
 savecore_flags="-m 10"	# Used if dumpdev is enabled above, and present.
 			# By default, only the 10 most recent kernel dumps

Modified: head/etc/rc.d/dumpon
==============================================================================
--- head/etc/rc.d/dumpon	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/etc/rc.d/dumpon	Wed Oct 25 00:51:00 2017	(r324965)
@@ -16,11 +16,14 @@ stop_cmd="dumpon_stop"
 
 dumpon_try()
 {
+	local flags
+
+	flags=${dumpon_flags}
 	if [ -n "${dumppubkey}" ]; then
-		/sbin/dumpon -k "${dumppubkey}" "${1}"
-	else
-		/sbin/dumpon "${1}"
+		warn "The dumppubkey variable is deprecated.  Use dumpon_flags."
+		flags="${flags} -k ${dumppubkey}"
 	fi
+	/sbin/dumpon ${flags} "${1}"
 	if [ $? -eq 0 ]; then
 		# Make a symlink in devfs for savecore
 		ln -fs "${1}" /dev/dumpdev

Modified: head/sbin/dumpon/dumpon.8
==============================================================================
--- head/sbin/dumpon/dumpon.8	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sbin/dumpon/dumpon.8	Wed Oct 25 00:51:00 2017	(r324965)
@@ -28,7 +28,7 @@
 .\"     From: @(#)swapon.8	8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd December 10, 2016
+.Dd October 24, 2017
 .Dt DUMPON 8
 .Os
 .Sh NAME
@@ -38,6 +38,7 @@
 .Nm
 .Op Fl v
 .Op Fl k Ar public_key_file
+.Op Fl z
 .Ar special_file
 .Nm
 .Op Fl v
@@ -114,6 +115,22 @@ This flag requires a kernel compiled with the
 kernel option.
 .Pp
 The
+.Fl z
+option configures the kernel to compress the dump in gzip format before writing
+it to the dump device.
+This reduces the amount of space required for the dump and accelerates
+recovery with
+.Xr savecore 8
+since less data needs to be copied from the dump device.
+When compression is enabled, the
+.Nm
+utility will not verify that the dump device is sufficiently large for a full
+dump.
+This flag requires a kernel compiled with the
+.Dv GZIO
+kernel option.
+.Pp
+The
 .Fl l
 flag causes
 .Nm
@@ -272,3 +289,7 @@ utility appeared in
 .Sh BUGS
 Because the file system layer is already dead by the time a crash dump
 is taken, it is not possible to send crash dumps directly to a file.
+.Pp
+It is currently not possible to configure both compression and encryption.
+The encrypted dump format assumes that the kernel dump size is a multiple
+of the cipher block size, which may not be true when the dump is compressed.

Modified: head/sbin/dumpon/dumpon.c
==============================================================================
--- head/sbin/dumpon/dumpon.c	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sbin/dumpon/dumpon.c	Wed Oct 25 00:51:00 2017	(r324965)
@@ -71,7 +71,7 @@ static void
 usage(void)
 {
 	fprintf(stderr, "%s\n%s\n%s\n",
-	    "usage: dumpon [-v] [-k public_key_file] special_file",
+	    "usage: dumpon [-v] [-k public_key_file] [-z] special_file",
 	    "       dumpon [-v] off",
 	    "       dumpon [-v] -l");
 	exit(EX_USAGE);
@@ -190,11 +190,12 @@ main(int argc, char *argv[])
 	int ch;
 	int i, fd;
 	int do_listdumpdev = 0;
-	bool enable;
+	bool enable, gzip;
 
+	gzip = false;
 	pubkeyfile = NULL;
 
-	while ((ch = getopt(argc, argv, "k:lv")) != -1)
+	while ((ch = getopt(argc, argv, "k:lvz")) != -1)
 		switch((char)ch) {
 		case 'k':
 			pubkeyfile = optarg;
@@ -205,6 +206,9 @@ main(int argc, char *argv[])
 		case 'v':
 			verbose = 1;
 			break;
+		case 'z':
+			gzip = true;
+			break;
 		default:
 			usage();
 		}
@@ -247,9 +251,11 @@ main(int argc, char *argv[])
 		fd = open(dumpdev, O_RDONLY);
 		if (fd < 0)
 			err(EX_OSFILE, "%s", dumpdev);
-		check_size(fd, dumpdev);
-		bzero(&kda, sizeof(kda));
 
+		if (!gzip)
+			check_size(fd, dumpdev);
+
+		bzero(&kda, sizeof(kda));
 		kda.kda_enable = 0;
 		i = ioctl(fd, DIOCSKERNELDUMP, &kda);
 		explicit_bzero(&kda, sizeof(kda));
@@ -260,6 +266,8 @@ main(int argc, char *argv[])
 #endif
 
 		kda.kda_enable = 1;
+		kda.kda_compression = gzip ? KERNELDUMP_COMP_GZIP :
+		    KERNELDUMP_COMP_NONE;
 		i = ioctl(fd, DIOCSKERNELDUMP, &kda);
 		explicit_bzero(kda.kda_encryptedkey, kda.kda_encryptedkeysize);
 		free(kda.kda_encryptedkey);

Modified: head/sbin/savecore/savecore.8
==============================================================================
--- head/sbin/savecore/savecore.8	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sbin/savecore/savecore.8	Wed Oct 25 00:51:00 2017	(r324965)
@@ -28,7 +28,7 @@
 .\"     From: @(#)savecore.8	8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd December 10, 2016
+.Dd October 24, 2017
 .Dt SAVECORE 8
 .Os
 .Sh NAME
@@ -96,8 +96,12 @@ the counter will restart from
 Print out some additional debugging information.
 Specify twice for more information.
 .It Fl z
-Compress the core dump and kernel (see
+Compress the dump (see
 .Xr gzip 1 ) .
+The dump may already be compressed if the kernel was configured to
+do so by
+.Xr dumpon 8 .
+In this case, the option has no effect.
 .El
 .Pp
 The

Modified: head/sbin/savecore/savecore.c
==============================================================================
--- head/sbin/savecore/savecore.c	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sbin/savecore/savecore.c	Wed Oct 25 00:51:00 2017	(r324965)
@@ -121,6 +121,9 @@ printheader(xo_handle_t *xo, const struct kerneldumphe
 	    (long long)dumplen);
 	xo_emit_h(xo, "{P:  }{Lwc:Blocksize}{:blocksize/%d}\n",
 	    dtoh32(h->blocksize));
+	xo_emit_h(xo, "{P:  }{Lwc:Compression}{:compression/%s}\n",
+	    h->compression == KERNELDUMP_COMP_GZIP ?
+	    "gzip" : "none");
 
 	t = dtoh64(h->dumptime);
 	xo_emit_h(xo, "{P:  }{Lwc:Dumptime}{:dumptime/%s}", ctime(&t));
@@ -357,7 +360,7 @@ compare_magic(const struct kerneldumpheader *kdh, cons
 #define BLOCKMASK (~(BLOCKSIZE-1))
 
 static int
-DoRegularFile(int fd, bool isencrypted, off_t dumpsize, char *buf,
+DoRegularFile(int fd, off_t dumpsize, u_int sectorsize, bool sparse, char *buf,
     const char *device, const char *filename, FILE *fp)
 {
 	int he, hs, nr, nw, wl;
@@ -370,8 +373,8 @@ DoRegularFile(int fd, bool isencrypted, off_t dumpsize
 		wl = BUFFERSIZE;
 		if (wl > dumpsize)
 			wl = dumpsize;
-		nr = read(fd, buf, wl);
-		if (nr != wl) {
+		nr = read(fd, buf, roundup(wl, sectorsize));
+		if (nr != (int)roundup(wl, sectorsize)) {
 			if (nr == 0)
 				syslog(LOG_WARNING,
 				    "WARNING: EOF on dump device");
@@ -380,7 +383,7 @@ DoRegularFile(int fd, bool isencrypted, off_t dumpsize
 			nerr++;
 			return (-1);
 		}
-		if (compress || isencrypted) {
+		if (!sparse) {
 			nw = fwrite(buf, 1, wl, fp);
 		} else {
 			for (nw = 0; nw < nr; nw = he) {
@@ -506,15 +509,14 @@ DoFile(const char *savedir, const char *device)
 	char *temp = NULL;
 	struct kerneldumpheader kdhf, kdhl;
 	uint8_t *dumpkey;
-	off_t mediasize, dumpsize, firsthd, lasthd;
+	off_t mediasize, dumpextent, dumplength, firsthd, lasthd;
 	FILE *info, *fp;
 	mode_t oumask;
 	int fd, fdinfo, error;
 	int bounds, status;
 	u_int sectorsize, xostyle;
-	int istextdump;
 	uint32_t dumpkeysize;
-	bool isencrypted, ret;
+	bool iscompressed, isencrypted, istextdump, ret;
 
 	bounds = getbounds();
 	dumpkey = NULL;
@@ -582,12 +584,12 @@ DoFile(const char *savedir, const char *device)
 		goto closefd;
 	}
 	memcpy(&kdhl, temp, sizeof(kdhl));
-	istextdump = 0;
+	iscompressed = istextdump = false;
 	if (compare_magic(&kdhl, TEXTDUMPMAGIC)) {
 		if (verbose)
 			printf("textdump magic on last dump header on %s\n",
 			    device);
-		istextdump = 1;
+		istextdump = true;
 		if (dtoh32(kdhl.version) != KERNELDUMP_TEXT_VERSION) {
 			syslog(LOG_ERR,
 			    "unknown version (%d) in last dump header on %s",
@@ -607,6 +609,20 @@ DoFile(const char *savedir, const char *device)
 			if (force == 0)
 				goto closefd;
 		}
+		switch (kdhl.compression) {
+		case KERNELDUMP_COMP_NONE:
+			break;
+		case KERNELDUMP_COMP_GZIP:
+			if (compress && verbose)
+				printf("dump is already compressed\n");
+			compress = false;
+			iscompressed = true;
+			break;
+		default:
+			syslog(LOG_ERR, "unknown compression type %d on %s",
+			    kdhl.compression, device);
+			break;
+		}
 	} else {
 		if (verbose)
 			printf("magic mismatch on last dump header on %s\n",
@@ -619,8 +635,7 @@ DoFile(const char *savedir, const char *device)
 		if (compare_magic(&kdhl, KERNELDUMPMAGIC_CLEARED)) {
 			if (verbose)
 				printf("forcing magic on %s\n", device);
-			memcpy(kdhl.magic, KERNELDUMPMAGIC,
-			    sizeof kdhl.magic);
+			memcpy(kdhl.magic, KERNELDUMPMAGIC, sizeof(kdhl.magic));
 		} else {
 			syslog(LOG_ERR, "unable to force dump - bad magic");
 			goto closefd;
@@ -648,9 +663,10 @@ DoFile(const char *savedir, const char *device)
 		if (force == 0)
 			goto closefd;
 	}
-	dumpsize = dtoh64(kdhl.dumplength);
+	dumpextent = dtoh64(kdhl.dumpextent);
+	dumplength = dtoh64(kdhl.dumplength);
 	dumpkeysize = dtoh32(kdhl.dumpkeysize);
-	firsthd = lasthd - dumpsize - sectorsize - dumpkeysize;
+	firsthd = lasthd - dumpextent - sectorsize - dumpkeysize;
 	if (lseek(fd, firsthd, SEEK_SET) != firsthd ||
 	    read(fd, temp, sectorsize) != (ssize_t)sectorsize) {
 		syslog(LOG_ERR,
@@ -696,7 +712,7 @@ DoFile(const char *savedir, const char *device)
 	if (verbose)
 		printf("Checking for available free space\n");
 
-	if (!check_space(savedir, dumpsize, bounds)) {
+	if (!check_space(savedir, dumplength, bounds)) {
 		nerr++;
 		goto closefd;
 	}
@@ -724,6 +740,9 @@ DoFile(const char *savedir, const char *device)
 		    istextdump ? "textdump.tar" :
 		    (isencrypted ? "vmcore_encrypted" : "vmcore"), bounds);
 		fp = zopen(corename, "w");
+	} else if (iscompressed && !isencrypted) {
+		snprintf(corename, sizeof(corename), "vmcore.%d.gz", bounds);
+		fp = fopen(corename, "w");
 	} else {
 		snprintf(corename, sizeof(corename), "%s.%d",
 		    istextdump ? "textdump.tar" :
@@ -792,11 +811,12 @@ DoFile(const char *savedir, const char *device)
 	    savedir, corename);
 
 	if (istextdump) {
-		if (DoTextdumpFile(fd, dumpsize, lasthd, buf, device,
+		if (DoTextdumpFile(fd, dumplength, lasthd, buf, device,
 		    corename, fp) < 0)
 			goto closeall;
 	} else {
-		if (DoRegularFile(fd, isencrypted, dumpsize, buf, device,
+		if (DoRegularFile(fd, dumplength, sectorsize,
+		    !(compress || iscompressed || isencrypted), buf, device,
 		    corename, fp) < 0) {
 			goto closeall;
 		}
@@ -822,7 +842,7 @@ DoFile(const char *savedir, const char *device)
 			    "key.last");
 		}
 	}
-	if (compress) {
+	if (compress || iscompressed) {
 		snprintf(linkname, sizeof(linkname), "%s.last.gz",
 		    istextdump ? "textdump.tar" :
 		    (isencrypted ? "vmcore_encrypted" : "vmcore"));

Modified: head/share/man/man5/rc.conf.5
==============================================================================
--- head/share/man/man5/rc.conf.5	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/share/man/man5/rc.conf.5	Wed Oct 25 00:51:00 2017	(r324965)
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd February 26, 2017
+.Dd October 24, 2017
 .Dt RC.CONF 5
 .Os
 .Sh NAME
@@ -3380,6 +3380,13 @@ Otherwise, the value of this variable is passed as the
 .Xr dumpon 8 .
 To disable crash dumps, set this variable to
 .Dq Li NO .
+.It Va dumpon_flags
+.Pq Vt str
+Flags to pass to
+.Xr dumpon 8
+when configuring
+.Va dumpdev
+as the system dump device.
 .It Va dumpdir
 .Pq Vt str
 When the system reboots after a crash and a crash dump is found on the
@@ -3400,18 +3407,6 @@ to not run
 at boot time when
 .Va dumpdir
 is set.
-.It Va dumppubkey
-.Pq Vt str
-Path to a public key.
-It is used by
-.Xr dumpon 8
-to encrypt a one-time key for a crash dump.
-The public key has to match a private key used by
-.Xr decryptcore 8
-to decrypt a crash dump after reboot.
-See
-.Xr dumpon 8
-for more details.
 .It Va savecore_enable
 .Pq Vt bool
 If set to

Modified: head/sys/dev/null/null.c
==============================================================================
--- head/sys/dev/null/null.c	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sys/dev/null/null.c	Wed Oct 25 00:51:00 2017	(r324965)
@@ -114,7 +114,7 @@ null_ioctl(struct cdev *dev __unused, u_long cmd, cadd
 	case DIOCSKERNELDUMP_FREEBSD11:
 #endif
 	case DIOCSKERNELDUMP:
-		error = set_dumper(NULL, NULL, td, 0, NULL, 0, NULL);
+		error = set_dumper(NULL, NULL, td, 0, 0, NULL, 0, NULL);
 		break;
 	case FIONBIO:
 		break;

Modified: head/sys/geom/geom_dev.c
==============================================================================
--- head/sys/geom/geom_dev.c	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sys/geom/geom_dev.c	Wed Oct 25 00:51:00 2017	(r324965)
@@ -138,7 +138,7 @@ g_dev_setdumpdev(struct cdev *dev, struct diocskerneld
 	int error, len;
 
 	if (dev == NULL || kda == NULL)
-		return (set_dumper(NULL, NULL, td, 0, NULL, 0, NULL));
+		return (set_dumper(NULL, NULL, td, 0, 0, NULL, 0, NULL));
 
 	cp = dev->si_drv2;
 	len = sizeof(kd);
@@ -148,8 +148,9 @@ g_dev_setdumpdev(struct cdev *dev, struct diocskerneld
 	if (error != 0)
 		return (error);
 
-	error = set_dumper(&kd.di, devtoname(dev), td, kda->kda_encryption,
-	    kda->kda_key, kda->kda_encryptedkeysize, kda->kda_encryptedkey);
+	error = set_dumper(&kd.di, devtoname(dev), td, kda->kda_compression,
+	    kda->kda_encryption, kda->kda_key, kda->kda_encryptedkeysize,
+	    kda->kda_encryptedkey);
 	if (error == 0)
 		dev->si_flags |= SI_DUMPDEV;
 
@@ -832,7 +833,7 @@ g_dev_orphan(struct g_consumer *cp)
 
 	/* Reset any dump-area set on this device */
 	if (dev->si_flags & SI_DUMPDEV)
-		(void)set_dumper(NULL, NULL, curthread, 0, NULL, 0, NULL);
+		(void)set_dumper(NULL, NULL, curthread, 0, 0, NULL, 0, NULL);
 
 	/* Destroy the struct cdev *so we get no more requests */
 	destroy_dev_sched_cb(dev, g_dev_callback, cp);

Modified: head/sys/kern/kern_gzio.c
==============================================================================
--- head/sys/kern/kern_gzio.c	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sys/kern/kern_gzio.c	Wed Oct 25 00:51:00 2017	(r324965)
@@ -60,7 +60,6 @@ struct gzio_stream *
 gzio_init(gzio_cb cb, enum gzio_mode mode, size_t bufsz, int level, void *arg)
 {
 	struct gzio_stream *s;
-	uint8_t *hdr;
 	int error;
 
 	if (bufsz < KERN_GZ_HDRLEN)
@@ -72,7 +71,6 @@ gzio_init(gzio_cb cb, enum gzio_mode mode, size_t bufs
 	s->gz_bufsz = bufsz;
 	s->gz_buffer = gz_alloc(NULL, 1, s->gz_bufsz);
 	s->gz_mode = mode;
-	s->gz_crc = ~0U;
 	s->gz_cb = cb;
 	s->gz_arg = arg;
 
@@ -87,6 +85,26 @@ gzio_init(gzio_cb cb, enum gzio_mode mode, size_t bufs
 	if (error != 0)
 		goto fail;
 
+	gzio_reset(s);
+
+	return (s);
+
+fail:
+	gz_free(NULL, s->gz_buffer);
+	gz_free(NULL, s);
+	return (NULL);
+}
+
+void
+gzio_reset(struct gzio_stream *s)
+{
+	uint8_t *hdr;
+
+	(void)deflateReset(&s->gz_stream);
+
+	s->gz_off = 0;
+	s->gz_crc = ~0U;
+
 	s->gz_stream.avail_out = s->gz_bufsz;
 	s->gz_stream.next_out = s->gz_buffer;
 
@@ -99,13 +117,6 @@ gzio_init(gzio_cb cb, enum gzio_mode mode, size_t bufs
 	hdr[9] = OS_CODE;
 	s->gz_stream.next_out += KERN_GZ_HDRLEN;
 	s->gz_stream.avail_out -= KERN_GZ_HDRLEN;
-
-	return (s);
-
-fail:
-	gz_free(NULL, s->gz_buffer);
-	gz_free(NULL, s);
-	return (NULL);
 }
 
 int

Modified: head/sys/kern/kern_shutdown.c
==============================================================================
--- head/sys/kern/kern_shutdown.c	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sys/kern/kern_shutdown.c	Wed Oct 25 00:51:00 2017	(r324965)
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_ddb.h"
 #include "opt_ekcd.h"
+#include "opt_gzio.h"
 #include "opt_kdb.h"
 #include "opt_panic.h"
 #include "opt_sched.h"
@@ -52,6 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/cons.h>
 #include <sys/eventhandler.h>
 #include <sys/filedesc.h>
+#include <sys/gzio.h>
 #include <sys/jail.h>
 #include <sys/kdb.h>
 #include <sys/kernel.h>
@@ -162,6 +164,24 @@ struct kerneldumpcrypto {
 };
 #endif
 
+#ifdef GZIO
+struct kerneldumpgz {
+	struct gzio_stream	*kdgz_stream;
+	uint8_t			*kdgz_buf;
+	size_t			kdgz_resid;
+};
+
+static struct kerneldumpgz *kerneldumpgz_create(struct dumperinfo *di,
+		    uint8_t compression);
+static void	kerneldumpgz_destroy(struct dumperinfo *di);
+static int	kerneldumpgz_write_cb(void *cb, size_t len, off_t off, void *arg);
+
+static int kerneldump_gzlevel = 6;
+SYSCTL_INT(_kern, OID_AUTO, kerneldump_gzlevel, CTLFLAG_RWTUN,
+    &kerneldump_gzlevel, 0,
+    "Kernel crash dump gzip compression level");
+#endif /* GZIO */
+
 /*
  * Variable panicstr contains argument to first call to panic; used as flag
  * to indicate that the kernel has already called panic.
@@ -857,6 +877,9 @@ static char dumpdevname[sizeof(((struct cdev*)NULL)->s
 SYSCTL_STRING(_kern_shutdown, OID_AUTO, dumpdevname, CTLFLAG_RD,
     dumpdevname, 0, "Device for kernel dumps");
 
+static int	_dump_append(struct dumperinfo *di, void *virtual,
+		    vm_offset_t physical, size_t length);
+
 #ifdef EKCD
 static struct kerneldumpcrypto *
 kerneldumpcrypto_create(size_t blocksize, uint8_t encryption,
@@ -947,11 +970,45 @@ kerneldumpcrypto_dumpkeysize(const struct kerneldumpcr
 }
 #endif /* EKCD */
 
+#ifdef GZIO
+static struct kerneldumpgz *
+kerneldumpgz_create(struct dumperinfo *di, uint8_t compression)
+{
+	struct kerneldumpgz *kdgz;
+
+	if (compression != KERNELDUMP_COMP_GZIP)
+		return (NULL);
+	kdgz = malloc(sizeof(*kdgz), M_DUMPER, M_WAITOK | M_ZERO);
+	kdgz->kdgz_stream = gzio_init(kerneldumpgz_write_cb, GZIO_DEFLATE,
+	    di->maxiosize, kerneldump_gzlevel, di);
+	if (kdgz->kdgz_stream == NULL) {
+		free(kdgz, M_DUMPER);
+		return (NULL);
+	}
+	kdgz->kdgz_buf = malloc(di->maxiosize, M_DUMPER, M_WAITOK | M_NODUMP);
+	return (kdgz);
+}
+
+static void
+kerneldumpgz_destroy(struct dumperinfo *di)
+{
+	struct kerneldumpgz *kdgz;
+
+	kdgz = di->kdgz;
+	if (kdgz == NULL)
+		return;
+	gzio_fini(kdgz->kdgz_stream);
+	explicit_bzero(kdgz->kdgz_buf, di->maxiosize);
+	free(kdgz->kdgz_buf, M_DUMPER);
+	free(kdgz, M_DUMPER);
+}
+#endif /* GZIO */
+
 /* Registration of dumpers */
 int
 set_dumper(struct dumperinfo *di, const char *devname, struct thread *td,
-    uint8_t encryption, const uint8_t *key, uint32_t encryptedkeysize,
-    const uint8_t *encryptedkey)
+    uint8_t compression, uint8_t encryption, const uint8_t *key,
+    uint32_t encryptedkeysize, const uint8_t *encryptedkey)
 {
 	size_t wantcopy;
 	int error;
@@ -969,6 +1026,7 @@ set_dumper(struct dumperinfo *di, const char *devname,
 	dumper = *di;
 	dumper.blockbuf = NULL;
 	dumper.kdc = NULL;
+	dumper.kdgz = NULL;
 
 	if (encryption != KERNELDUMP_ENC_NONE) {
 #ifdef EKCD
@@ -987,9 +1045,30 @@ set_dumper(struct dumperinfo *di, const char *devname,
 	wantcopy = strlcpy(dumpdevname, devname, sizeof(dumpdevname));
 	if (wantcopy >= sizeof(dumpdevname)) {
 		printf("set_dumper: device name truncated from '%s' -> '%s'\n",
-			devname, dumpdevname);
+		    devname, dumpdevname);
 	}
 
+	if (compression != KERNELDUMP_COMP_NONE) {
+#ifdef GZIO
+		/*
+		 * We currently can't support simultaneous encryption and
+		 * compression.
+		 */
+		if (encryption != KERNELDUMP_ENC_NONE) {
+			error = EOPNOTSUPP;
+			goto cleanup;
+		}
+		dumper.kdgz = kerneldumpgz_create(&dumper, compression);
+		if (dumper.kdgz == NULL) {
+			error = EINVAL;
+			goto cleanup;
+		}
+#else
+		error = EOPNOTSUPP;
+		goto cleanup;
+#endif
+	}
+
 	dumper.blockbuf = malloc(di->blocksize, M_DUMPER, M_WAITOK | M_ZERO);
 	return (0);
 cleanup:
@@ -1000,6 +1079,11 @@ cleanup:
 		free(dumper.kdc, M_EKCD);
 	}
 #endif
+
+#ifdef GZIO
+	kerneldumpgz_destroy(&dumper);
+#endif
+
 	if (dumper.blockbuf != NULL) {
 		explicit_bzero(dumper.blockbuf, dumper.blocksize);
 		free(dumper.blockbuf, M_DUMPER);
@@ -1090,22 +1174,57 @@ dump_encrypted_write(struct dumperinfo *di, void *virt
 }
 
 static int
-dump_write_key(struct dumperinfo *di, vm_offset_t physical, off_t offset)
+dump_write_key(struct dumperinfo *di, off_t offset)
 {
 	struct kerneldumpcrypto *kdc;
 
 	kdc = di->kdc;
 	if (kdc == NULL)
 		return (0);
-
-	return (dump_write(di, kdc->kdc_dumpkey, physical, offset,
+	return (dump_write(di, kdc->kdc_dumpkey, 0, offset,
 	    kdc->kdc_dumpkeysize));
 }
 #endif /* EKCD */
 
+#ifdef GZIO
 static int
+kerneldumpgz_write_cb(void *base, size_t length, off_t offset, void *arg)
+{
+	struct dumperinfo *di;
+	size_t resid, rlength;
+	int error;
+
+	di = arg;
+
+	if (length % di->blocksize != 0) {
+		/*
+		 * This must be the final write after flushing the compression
+		 * stream. Write as many full blocks as possible and stash the
+		 * residual data in the dumper's block buffer. It will be
+		 * padded and written in dump_finish().
+		 */
+		rlength = rounddown(length, di->blocksize);
+		if (rlength != 0) {
+			error = _dump_append(di, base, 0, rlength);
+			if (error != 0)
+				return (error);
+		}
+		resid = length - rlength;
+		memmove(di->blockbuf, (uint8_t *)base + rlength, resid);
+		di->kdgz->kdgz_resid = resid;
+		return (EAGAIN);
+	}
+	return (_dump_append(di, base, 0, length));
+}
+#endif /* GZIO */
+
+/*
+ * Write a kerneldumpheader at the specified offset. The header structure is 512
+ * bytes in size, but we must pad to the device sector size.
+ */
+static int
 dump_write_header(struct dumperinfo *di, struct kerneldumpheader *kdh,
-    vm_offset_t physical, off_t offset)
+    off_t offset)
 {
 	void *buf;
 	size_t hdrsz;
@@ -1122,7 +1241,7 @@ dump_write_header(struct dumperinfo *di, struct kernel
 		memcpy(buf, kdh, hdrsz);
 	}
 
-	return (dump_write(di, buf, physical, offset, di->blocksize));
+	return (dump_write(di, buf, 0, offset, di->blocksize));
 }
 
 /*
@@ -1132,19 +1251,30 @@ dump_write_header(struct dumperinfo *di, struct kernel
 #define	SIZEOF_METADATA		(64 * 1024)
 
 /*
- * Do some preliminary setup for a kernel dump: verify that we have enough space
- * on the dump device, write the leading header, and optionally write the crypto
- * key.
+ * Do some preliminary setup for a kernel dump: initialize state for encryption,
+ * if requested, and make sure that we have enough space on the dump device.
+ *
+ * We set things up so that the dump ends before the last sector of the dump
+ * device, at which the trailing header is written.
+ *
+ *     +-----------+------+-----+----------------------------+------+
+ *     |           | lhdr | key |    ... kernel dump ...     | thdr |
+ *     +-----------+------+-----+----------------------------+------+
+ *                   1 blk  opt <------- dump extent --------> 1 blk
+ *
+ * Dumps written using dump_append() start at the beginning of the extent.
+ * Uncompressed dumps will use the entire extent, but compressed dumps typically
+ * will not. The true length of the dump is recorded in the leading and trailing
+ * headers once the dump has been completed.
  */
 int
 dump_start(struct dumperinfo *di, struct kerneldumpheader *kdh)
 {
-	uint64_t dumpsize;
+	uint64_t dumpextent;
 	uint32_t keysize;
-	int error;
 
 #ifdef EKCD
-	error = kerneldumpcrypto_init(di->kdc);
+	int error = kerneldumpcrypto_init(di->kdc);
 	if (error != 0)
 		return (error);
 	keysize = kerneldumpcrypto_dumpkeysize(di->kdc);
@@ -1152,30 +1282,36 @@ dump_start(struct dumperinfo *di, struct kerneldumphea
 	keysize = 0;
 #endif
 
-	dumpsize = dtoh64(kdh->dumplength) + 2 * di->blocksize + keysize;
-	if (di->mediasize < SIZEOF_METADATA + dumpsize)
-		return (E2BIG);
-
-	di->dumpoff = di->mediaoffset + di->mediasize - dumpsize;
-
-	error = dump_write_header(di, kdh, 0, di->dumpoff);
-	if (error != 0)
-		return (error);
-	di->dumpoff += di->blocksize;
-
-#ifdef EKCD
-	error = dump_write_key(di, 0, di->dumpoff);
-	if (error != 0)
-		return (error);
-	di->dumpoff += keysize;
+	dumpextent = dtoh64(kdh->dumpextent);
+	if (di->mediasize < SIZEOF_METADATA + dumpextent + 2 * di->blocksize +
+	    keysize) {
+#ifdef GZIO
+		if (di->kdgz != NULL) {
+			/*
+			 * We don't yet know how much space the compressed dump
+			 * will occupy, so try to use the whole swap partition
+			 * (minus the first 64KB) in the hope that the
+			 * compressed dump will fit. If that doesn't turn out to
+			 * be enouch, the bounds checking in dump_write()
+			 * will catch us and cause the dump to fail.
+			 */
+			dumpextent = di->mediasize - SIZEOF_METADATA -
+			    2 * di->blocksize - keysize;
+			kdh->dumpextent = htod64(dumpextent);
+		} else
 #endif
+			return (E2BIG);
+	}
 
+	/* The offset at which to begin writing the dump. */
+	di->dumpoff = di->mediaoffset + di->mediasize - di->blocksize -
+	    dumpextent;
+
 	return (0);
 }
 
-/* Write to the dump device at the current dump offset. */
-int
-dump_append(struct dumperinfo *di, void *virtual, vm_offset_t physical,
+static int
+_dump_append(struct dumperinfo *di, void *virtual, vm_offset_t physical,
     size_t length)
 {
 	int error;
@@ -1192,8 +1328,34 @@ dump_append(struct dumperinfo *di, void *virtual, vm_o
 	return (error);
 }
 
-/* Perform a raw write to the dump device at the specified offset. */
+/*
+ * Write to the dump device starting at dumpoff. When compression is enabled,
+ * writes to the device will be performed using a callback that gets invoked
+ * when the compression stream's output buffer is full.
+ */
 int
+dump_append(struct dumperinfo *di, void *virtual, vm_offset_t physical,
+    size_t length)
+{
+#ifdef GZIO
+	void *buf;
+
+	if (di->kdgz != NULL) {
+		/* Bounce through a buffer to avoid gzip CRC errors. */
+		if (length > di->maxiosize)
+			return (EINVAL);
+		buf = di->kdgz->kdgz_buf;
+		memmove(buf, virtual, length);
+		return (gzio_write(di->kdgz->kdgz_stream, buf, length));
+	}
+#endif
+	return (_dump_append(di, virtual, physical, length));
+}
+
+/*
+ * Write to the dump device at the specified offset.
+ */
+int
 dump_write(struct dumperinfo *di, void *virtual, vm_offset_t physical,
     off_t offset, size_t length)
 {
@@ -1206,18 +1368,74 @@ dump_write(struct dumperinfo *di, void *virtual, vm_of
 }
 
 /*
- * Write the trailing kernel dump header and signal to the lower layers that the
- * dump has completed.
+ * Perform kernel dump finalization: flush the compression stream, if necessary,
+ * write the leading and trailing kernel dump headers now that we know the true
+ * length of the dump, and optionally write the encryption key following the
+ * leading header.
  */
 int
 dump_finish(struct dumperinfo *di, struct kerneldumpheader *kdh)
 {
+	uint64_t extent;
+	uint32_t keysize;
 	int error;
 
-	error = dump_write_header(di, kdh, 0, di->dumpoff);
+	extent = dtoh64(kdh->dumpextent);
+
+#ifdef EKCD
+	keysize = kerneldumpcrypto_dumpkeysize(di->kdc);
+#else
+	keysize = 0;
+#endif
+
+#ifdef GZIO
+	if (di->kdgz != NULL) {
+		error = gzio_flush(di->kdgz->kdgz_stream);
+		if (error == EAGAIN) {
+			/* We have residual data in di->blockbuf. */
+			error = dump_write(di, di->blockbuf, 0, di->dumpoff,
+			    di->blocksize);
+			di->dumpoff += di->kdgz->kdgz_resid;
+			di->kdgz->kdgz_resid = 0;
+		}
+		if (error != 0)
+			return (error);
+
+		/*
+		 * We now know the size of the compressed dump, so update the
+		 * header accordingly and recompute parity.
+		 */
+		kdh->dumplength = htod64(di->dumpoff -
+		    (di->mediaoffset + di->mediasize - di->blocksize - extent));
+		kdh->parity = 0;
+		kdh->parity = kerneldump_parity(kdh);
+
+		gzio_reset(di->kdgz->kdgz_stream);
+	}
+#endif
+
+	/*
+	 * Write kerneldump headers at the beginning and end of the dump extent.
+	 * Write the key after the leading header.
+	 */
+	error = dump_write_header(di, kdh,
+	    di->mediaoffset + di->mediasize - 2 * di->blocksize - extent -
+	    keysize);
 	if (error != 0)
 		return (error);
 
+#ifdef EKCD
+	error = dump_write_key(di,
+	    di->mediaoffset + di->mediasize - di->blocksize - extent - keysize);
+	if (error != 0)
+		return (error);
+#endif
+
+	error = dump_write_header(di, kdh,
+	    di->mediaoffset + di->mediasize - di->blocksize);
+	if (error != 0)
+		return (error);
+
 	(void)dump_write(di, NULL, 0, 0, 0);
 	return (0);
 }
@@ -1234,6 +1452,7 @@ dump_init_header(const struct dumperinfo *di, struct k
 	kdh->version = htod32(KERNELDUMPVERSION);
 	kdh->architectureversion = htod32(archver);
 	kdh->dumplength = htod64(dumplen);
+	kdh->dumpextent = kdh->dumplength;
 	kdh->dumptime = htod64(time_second);
 #ifdef EKCD
 	kdh->dumpkeysize = htod32(kerneldumpcrypto_dumpkeysize(di->kdc));
@@ -1247,6 +1466,10 @@ dump_init_header(const struct dumperinfo *di, struct k
 		kdh->versionstring[dstsize - 2] = '\n';
 	if (panicstr != NULL)
 		strlcpy(kdh->panicstring, panicstr, sizeof(kdh->panicstring));
+#ifdef GZIO
+	if (di->kdgz != NULL)
+		kdh->compression = KERNELDUMP_COMP_GZIP;
+#endif
 	kdh->parity = kerneldump_parity(kdh);
 }
 

Modified: head/sys/sys/conf.h
==============================================================================
--- head/sys/sys/conf.h	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sys/sys/conf.h	Wed Oct 25 00:51:00 2017	(r324965)
@@ -338,14 +338,15 @@ struct dumperinfo {
 	void	*blockbuf;	/* Buffer for padding shorter dump blocks */
 	off_t	dumpoff;	/* Offset of ongoing kernel dump. */
 	struct kerneldumpcrypto	*kdc; /* Kernel dump crypto. */
+	struct kerneldumpgz *kdgz; /* Kernel dump compression. */
 };
 
 extern int dumping;		/* system is dumping */
 
 int doadump(boolean_t);
 int set_dumper(struct dumperinfo *di, const char *devname, struct thread *td,
-    uint8_t encrypt, const uint8_t *key, uint32_t encryptedkeysize,
-    const uint8_t *encryptedkey);
+    uint8_t compression, uint8_t encryption, const uint8_t *key,
+    uint32_t encryptedkeysize, const uint8_t *encryptedkey);
 
 int dump_start(struct dumperinfo *di, struct kerneldumpheader *kdh);
 int dump_append(struct dumperinfo *, void *, vm_offset_t, size_t);

Modified: head/sys/sys/disk.h
==============================================================================
--- head/sys/sys/disk.h	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sys/sys/disk.h	Wed Oct 25 00:51:00 2017	(r324965)
@@ -143,6 +143,7 @@ struct diocgattr_arg {
 
 struct diocskerneldump_arg {
 	uint8_t		 kda_enable;
+	uint8_t		 kda_compression;
 	uint8_t		 kda_encryption;
 	uint8_t		 kda_key[KERNELDUMP_KEY_MAX_SIZE];
 	uint32_t	 kda_encryptedkeysize;

Modified: head/sys/sys/gzio.h
==============================================================================
--- head/sys/sys/gzio.h	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sys/sys/gzio.h	Wed Oct 25 00:51:00 2017	(r324965)
@@ -40,6 +40,7 @@ typedef int (*gzio_cb)(void *, size_t, off_t, void *);
 struct gzio_stream;
 
 struct gzio_stream *gzio_init(gzio_cb cb, enum gzio_mode, size_t, int, void *);
+void		gzio_reset(struct gzio_stream *);
 int		gzio_write(struct gzio_stream *, void *, u_int);
 int		gzio_flush(struct gzio_stream *);
 void		gzio_fini(struct gzio_stream *);

Modified: head/sys/sys/kerneldump.h
==============================================================================
--- head/sys/sys/kerneldump.h	Tue Oct 24 21:18:50 2017	(r324964)
+++ head/sys/sys/kerneldump.h	Wed Oct 25 00:51:00 2017	(r324965)
@@ -55,6 +55,9 @@
 #define	htod64(x)	(x)
 #endif
 
+#define	KERNELDUMP_COMP_NONE		0
+#define	KERNELDUMP_COMP_GZIP		1
+
 #define	KERNELDUMP_ENC_NONE		0
 #define	KERNELDUMP_ENC_AES_256_CBC	1
 
@@ -75,8 +78,8 @@ struct kerneldumpheader {
 #define	KERNELDUMPMAGIC_CLEARED	"Cleared Kernel Dump"
 	char		architecture[12];
 	uint32_t	version;
-#define	KERNELDUMPVERSION		2
-#define	KERNELDUMP_TEXT_VERSION		2
+#define	KERNELDUMPVERSION		3
+#define	KERNELDUMP_TEXT_VERSION		3
 	uint32_t	architectureversion;
 #define	KERNELDUMP_AARCH64_VERSION	1
 #define	KERNELDUMP_AMD64_VERSION	2
@@ -87,12 +90,14 @@ struct kerneldumpheader {
 #define	KERNELDUMP_RISCV_VERSION	1
 #define	KERNELDUMP_SPARC64_VERSION	1
 	uint64_t	dumplength;		/* excl headers */
+	uint64_t	dumpextent;
 	uint64_t	dumptime;
 	uint32_t	dumpkeysize;
 	uint32_t	blocksize;
+	uint8_t		compression;
 	char		hostname[64];
 	char		versionstring[192];
-	char		panicstring[188];
+	char		panicstring[179];

*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201710250051.v9P0p0Tl052928>