Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 13 Feb 2018 19:28:02 +0000 (UTC)
From:      Mark Johnston <markj@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r329240 - in head: sbin/dumpon sbin/savecore share/man/man5 sys/conf sys/kern sys/sys
Message-ID:  <201802131928.w1DJS203052532@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: markj
Date: Tue Feb 13 19:28:02 2018
New Revision: 329240
URL: https://svnweb.freebsd.org/changeset/base/329240

Log:
  Add support for zstd-compressed user and kernel core dumps.
  
  This works similarly to the existing gzip compression support, but
  zstd is typically faster and gives better compression ratios.
  
  Support for this functionality must be configured by adding ZSTDIO to
  one's kernel configuration file. dumpon(8)'s new -Z option is used to
  configure zstd compression for kernel dumps. savecore(8) now recognizes
  and saves zstd-compressed kernel dumps with a .zst extension.
  
  Submitted by:	cem (original version)
  Relnotes:	yes
  Sponsored by:	Dell EMC Isilon
  Differential Revision:	https://reviews.freebsd.org/D13101,
  			https://reviews.freebsd.org/D13633

Modified:
  head/sbin/dumpon/dumpon.8
  head/sbin/dumpon/dumpon.c
  head/sbin/savecore/savecore.c
  head/share/man/man5/core.5
  head/sys/conf/NOTES
  head/sys/conf/files
  head/sys/conf/options
  head/sys/kern/kern_shutdown.c
  head/sys/kern/kern_sig.c
  head/sys/kern/subr_compressor.c
  head/sys/sys/compressor.h
  head/sys/sys/kerneldump.h

Modified: head/sbin/dumpon/dumpon.8
==============================================================================
--- head/sbin/dumpon/dumpon.8	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sbin/dumpon/dumpon.8	Tue Feb 13 19:28:02 2018	(r329240)
@@ -28,7 +28,7 @@
 .\"     From: @(#)swapon.8	8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd October 24, 2017
+.Dd February 13, 2018
 .Dt DUMPON 8
 .Os
 .Sh NAME
@@ -39,6 +39,7 @@
 .Op Fl v
 .Op Fl k Ar public_key_file
 .Op Fl z
+.Op Fl Z
 .Ar special_file
 .Nm
 .Op Fl v
@@ -116,8 +117,10 @@ kernel option.
 .Pp
 The
 .Fl z
-option configures the kernel to compress the dump in gzip format before writing
-it to the dump device.
+and
+.Fl Z
+options configure the kernel to compress the dump before writing it to
+the dump device.
 This reduces the amount of space required for the dump and accelerates
 recovery with
 .Xr savecore 8
@@ -126,10 +129,22 @@ When compression is enabled, the
 .Nm
 utility will not verify that the dump device is sufficiently large for a full
 dump.
-This flag requires a kernel compiled with the
+The
+.Fl z
+and
+.Fl Z
+options cause the dump to be written in
+.Xr gzip 1
+and
+.Xr zstd 1
+format, respectively.
+These flags require a kernel compiled with the
 .Dv GZIO
-kernel option.
+or
+.Dv ZSTDIO
+kernel options.
 .Pp
+.Pp
 The
 .Fl l
 flag causes
@@ -269,15 +284,17 @@ The core was decrypted properly if
 .Xr kgdb 1
 does not print any errors.
 .Sh SEE ALSO
+.Xr gzip 1 ,
 .Xr kgdb 1 ,
+.Xr zstd 1 ,
 .Xr ddb 4 ,
 .Xr fstab 5 ,
 .Xr rc.conf 5 ,
 .Xr config 8 ,
+.Xr decryptcore 8 ,
 .Xr init 8 ,
 .Xr loader 8 ,
 .Xr rc 8 ,
-.Xr decryptcore 8 ,
 .Xr savecore 8 ,
 .Xr swapon 8 ,
 .Xr panic 9

Modified: head/sbin/dumpon/dumpon.c
==============================================================================
--- head/sbin/dumpon/dumpon.c	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sbin/dumpon/dumpon.c	Tue Feb 13 19:28:02 2018	(r329240)
@@ -73,7 +73,7 @@ static void
 usage(void)
 {
 	fprintf(stderr, "%s\n%s\n%s\n",
-	    "usage: dumpon [-v] [-k public_key_file] [-z] special_file",
+	    "usage: dumpon [-v] [-k public_key_file] [-Zz] special_file",
 	    "       dumpon [-v] off",
 	    "       dumpon [-v] -l");
 	exit(EX_USAGE);
@@ -192,12 +192,12 @@ main(int argc, char *argv[])
 	int ch;
 	int i, fd;
 	int do_listdumpdev = 0;
-	bool enable, gzip;
+	bool enable, gzip, zstd;
 
-	gzip = false;
+	gzip = zstd = false;
 	pubkeyfile = NULL;
 
-	while ((ch = getopt(argc, argv, "k:lvz")) != -1)
+	while ((ch = getopt(argc, argv, "k:lvZz")) != -1)
 		switch((char)ch) {
 		case 'k':
 			pubkeyfile = optarg;
@@ -208,6 +208,9 @@ main(int argc, char *argv[])
 		case 'v':
 			verbose = 1;
 			break;
+		case 'Z':
+			zstd = true;
+			break;
 		case 'z':
 			gzip = true;
 			break;
@@ -215,6 +218,9 @@ main(int argc, char *argv[])
 			usage();
 		}
 
+	if (gzip && zstd)
+		errx(EX_USAGE, "The -z and -Z options are mutually exclusive.");
+
 	argc -= optind;
 	argv += optind;
 
@@ -254,7 +260,7 @@ main(int argc, char *argv[])
 		if (fd < 0)
 			err(EX_OSFILE, "%s", dumpdev);
 
-		if (!gzip)
+		if (!gzip && !zstd)
 			check_size(fd, dumpdev);
 
 		bzero(&kda, sizeof(kda));
@@ -268,8 +274,11 @@ main(int argc, char *argv[])
 #endif
 
 		kda.kda_enable = 1;
-		kda.kda_compression = gzip ? KERNELDUMP_COMP_GZIP :
-		    KERNELDUMP_COMP_NONE;
+		kda.kda_compression = KERNELDUMP_COMP_NONE;
+		if (zstd)
+			kda.kda_compression = KERNELDUMP_COMP_ZSTD;
+		else if (gzip)
+			kda.kda_compression = KERNELDUMP_COMP_GZIP;
 		i = ioctl(fd, DIOCSKERNELDUMP, &kda);
 		explicit_bzero(kda.kda_encryptedkey, kda.kda_encryptedkeysize);
 		free(kda.kda_encryptedkey);

Modified: head/sbin/savecore/savecore.c
==============================================================================
--- head/sbin/savecore/savecore.c	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sbin/savecore/savecore.c	Tue Feb 13 19:28:02 2018	(r329240)
@@ -109,6 +109,7 @@ printheader(xo_handle_t *xo, const struct kerneldumphe
 	uint64_t dumplen;
 	time_t t;
 	const char *stat_str;
+	const char *comp_str;
 
 	xo_flush_h(xo);
 	xo_emit_h(xo, "{Lwc:Dump header from device}{:dump_device/%s}\n",
@@ -123,10 +124,21 @@ printheader(xo_handle_t *xo, const struct kerneldumphe
 	    (long long)dumplen);
 	xo_emit_h(xo, "{P:  }{Lwc:Blocksize}{:blocksize/%d}\n",
 	    dtoh32(h->blocksize));
-	xo_emit_h(xo, "{P:  }{Lwc:Compression}{:compression/%s}\n",
-	    h->compression == KERNELDUMP_COMP_GZIP ?
-	    "gzip" : "none");
-
+	switch (h->compression) {
+	case KERNELDUMP_COMP_NONE:
+		comp_str = "none";
+		break;
+	case KERNELDUMP_COMP_GZIP:
+		comp_str = "gzip";
+		break;
+	case KERNELDUMP_COMP_ZSTD:
+		comp_str = "zstd";
+		break;
+	default:
+		comp_str = "???";
+		break;
+	}
+	xo_emit_h(xo, "{P:  }{Lwc:Compression}{:compression/%s}\n", comp_str);
 	t = dtoh64(h->dumptime);
 	xo_emit_h(xo, "{P:  }{Lwc:Dumptime}{:dumptime/%s}", ctime(&t));
 	xo_emit_h(xo, "{P:  }{Lwc:Hostname}{:hostname/%s}\n", h->hostname);
@@ -249,6 +261,8 @@ saved_dump_size(int bounds)
 	dumpsize += file_size(path);
 	(void)snprintf(path, sizeof(path), "vmcore.%d.gz", bounds);
 	dumpsize += file_size(path);
+	(void)snprintf(path, sizeof(path), "vmcore.%d.zst", bounds);
+	dumpsize += file_size(path);
 	(void)snprintf(path, sizeof(path), "textdump.tar.%d", bounds);
 	dumpsize += file_size(path);
 	(void)snprintf(path, sizeof(path), "textdump.tar.%d.gz", bounds);
@@ -268,6 +282,8 @@ saved_dump_remove(int bounds)
 	(void)unlink(path);
 	(void)snprintf(path, sizeof(path), "vmcore.%d.gz", bounds);
 	(void)unlink(path);
+	(void)snprintf(path, sizeof(path), "vmcore.%d.zst", bounds);
+	(void)unlink(path);
 	(void)snprintf(path, sizeof(path), "textdump.tar.%d", bounds);
 	(void)unlink(path);
 	(void)snprintf(path, sizeof(path), "textdump.tar.%d.gz", bounds);
@@ -282,6 +298,7 @@ symlinks_remove(void)
 	(void)unlink("key.last");
 	(void)unlink("vmcore.last");
 	(void)unlink("vmcore.last.gz");
+	(void)unlink("vmcore.last.zstd");
 	(void)unlink("vmcore_encrypted.last");
 	(void)unlink("vmcore_encrypted.last.gz");
 	(void)unlink("textdump.tar.last");
@@ -615,6 +632,7 @@ DoFile(const char *savedir, const char *device)
 		case KERNELDUMP_COMP_NONE:
 			break;
 		case KERNELDUMP_COMP_GZIP:
+		case KERNELDUMP_COMP_ZSTD:
 			if (compress && verbose)
 				printf("dump is already compressed\n");
 			compress = false;
@@ -743,7 +761,8 @@ DoFile(const char *savedir, const char *device)
 		    (isencrypted ? "vmcore_encrypted" : "vmcore"), bounds);
 		fp = zopen(corename, "w");
 	} else if (iscompressed && !isencrypted) {
-		snprintf(corename, sizeof(corename), "vmcore.%d.gz", bounds);
+		snprintf(corename, sizeof(corename), "vmcore.%d.%s", bounds,
+		    (kdhl.compression == KERNELDUMP_COMP_GZIP) ? "gz" : "zst");
 		fp = fopen(corename, "w");
 	} else {
 		snprintf(corename, sizeof(corename), "%s.%d",
@@ -845,9 +864,10 @@ DoFile(const char *savedir, const char *device)
 		}
 	}
 	if (compress || iscompressed) {
-		snprintf(linkname, sizeof(linkname), "%s.last.gz",
+		snprintf(linkname, sizeof(linkname), "%s.last.%s",
 		    istextdump ? "textdump.tar" :
-		    (isencrypted ? "vmcore_encrypted" : "vmcore"));
+		    (isencrypted ? "vmcore_encrypted" : "vmcore"),
+		    (kdhl.compression == KERNELDUMP_COMP_ZSTD) ? "zst" : "gz");
 	} else {
 		snprintf(linkname, sizeof(linkname), "%s.last",
 		    istextdump ? "textdump.tar" :

Modified: head/share/man/man5/core.5
==============================================================================
--- head/share/man/man5/core.5	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/share/man/man5/core.5	Tue Feb 13 19:28:02 2018	(r329240)
@@ -28,7 +28,7 @@
 .\"     @(#)core.5	8.3 (Berkeley) 12/11/93
 .\" $FreeBSD$
 .\"
-.Dd January 8, 2018
+.Dd February 13, 2018
 .Dt CORE 5
 .Os
 .Sh NAME
@@ -112,10 +112,17 @@ The following sysctl control core file compression:
 .Bl -tag -width "kern.compress_user_cores_level" -compact -offset "12345"
 .It Em kern.compress_user_cores
 Enable compression of user cores.
-A value of 1 configures gzip compression.
-gzip-compressed core files will have a suffix of
+A value of 1 configures
+.Xr gzip 1
+compression,
+and a value of 2 configures
+.Xr zstd 1
+compression.
+Compressed core files will have a suffix of
 .Ql .gz
-appended to their filenames.
+or
+.Ql .zst
+appended to their filenames depending on the selected format.
 .It Em kern.compress_user_cores_level
 Compression level.
 Defaults to 6.

Modified: head/sys/conf/NOTES
==============================================================================
--- head/sys/conf/NOTES	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sys/conf/NOTES	Tue Feb 13 19:28:02 2018	(r329240)
@@ -3018,6 +3018,10 @@ options         IMAGACT_BINMISC
 # This enables support for compressed core dumps.
 options 	GZIO
 
+# zstd I/O stream support
+# This enables support for Zstd compressed core dumps.
+options 	ZSTDIO
+
 # BHND(4) drivers
 options		BHND_LOGLEVEL	# Logging threshold level
 

Modified: head/sys/conf/files
==============================================================================
--- head/sys/conf/files	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sys/conf/files	Tue Feb 13 19:28:02 2018	(r329240)
@@ -3836,7 +3836,8 @@ kern/subr_bus_dma.c		standard
 kern/subr_bufring.c		standard
 kern/subr_capability.c		standard
 kern/subr_clock.c		standard
-kern/subr_compressor.c		standard
+kern/subr_compressor.c		standard \
+	compile-with "${NORMAL_C} -I$S/contrib/zstd/lib/freebsd"
 kern/subr_counter.c		standard
 kern/subr_devstat.c		standard
 kern/subr_disk.c		standard

Modified: head/sys/conf/options
==============================================================================
--- head/sys/conf/options	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sys/conf/options	Tue Feb 13 19:28:02 2018	(r329240)
@@ -222,6 +222,7 @@ TURNSTILE_PROFILING
 UMTX_PROFILING
 UMTX_CHAINS	opt_global.h
 VERBOSE_SYSINIT
+ZSTDIO		opt_zstdio.h
 
 # POSIX kernel options
 P1003_1B_MQUEUE			opt_posix.h

Modified: head/sys/kern/kern_shutdown.c
==============================================================================
--- head/sys/kern/kern_shutdown.c	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sys/kern/kern_shutdown.c	Tue Feb 13 19:28:02 2018	(r329240)
@@ -174,6 +174,7 @@ struct kerneldumpcrypto {
 #endif
 
 struct kerneldumpcomp {
+	uint8_t			kdc_format;
 	struct compressor	*kdc_stream;
 	uint8_t			*kdc_buf;
 	size_t			kdc_resid;
@@ -987,12 +988,23 @@ static struct kerneldumpcomp *
 kerneldumpcomp_create(struct dumperinfo *di, uint8_t compression)
 {
 	struct kerneldumpcomp *kdcomp;
+	int format;
 
-	if (compression != KERNELDUMP_COMP_GZIP)
+	switch (compression) {
+	case KERNELDUMP_COMP_GZIP:
+		format = COMPRESS_GZIP;
+		break;
+	case KERNELDUMP_COMP_ZSTD:
+		format = COMPRESS_ZSTD;
+		break;
+	default:
 		return (NULL);
+	}
+
 	kdcomp = malloc(sizeof(*kdcomp), M_DUMPER, M_WAITOK | M_ZERO);
+	kdcomp->kdc_format = compression;
 	kdcomp->kdc_stream = compressor_init(kerneldumpcomp_write_cb,
-	    COMPRESS_GZIP, di->maxiosize, kerneldump_gzlevel, di);
+	    format, di->maxiosize, kerneldump_gzlevel, di);
 	if (kdcomp->kdc_stream == NULL) {
 		free(kdcomp, M_DUMPER);
 		return (NULL);
@@ -1293,7 +1305,7 @@ dump_start(struct dumperinfo *di, struct kerneldumphea
 			 * will occupy, so try to use the whole swap partition
 			 * (minus the first 64KB) in the hope that the
 			 * compressed dump will fit. If that doesn't turn out to
-			 * be enouch, the bounds checking in dump_write()
+			 * be enough, the bounds checking in dump_write()
 			 * will catch us and cause the dump to fail.
 			 */
 			dumpextent = di->mediasize - SIZEOF_METADATA -
@@ -1463,7 +1475,7 @@ dump_init_header(const struct dumperinfo *di, struct k
 	if (panicstr != NULL)
 		strlcpy(kdh->panicstring, panicstr, sizeof(kdh->panicstring));
 	if (di->kdcomp != NULL)
-		kdh->compression = KERNELDUMP_COMP_GZIP;
+		kdh->compression = di->kdcomp->kdc_format;
 	kdh->parity = kerneldump_parity(kdh);
 }
 

Modified: head/sys/kern/kern_sig.c
==============================================================================
--- head/sys/kern/kern_sig.c	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sys/kern/kern_sig.c	Tue Feb 13 19:28:02 2018	(r329240)
@@ -3253,7 +3253,8 @@ sysctl_debug_num_cores_check (SYSCTL_HANDLER_ARGS)
 SYSCTL_PROC(_debug, OID_AUTO, ncores, CTLTYPE_INT|CTLFLAG_RW,
 	    0, sizeof(int), sysctl_debug_num_cores_check, "I", "");
 
-#define	GZ_SUFFIX	".gz"
+#define	GZIP_SUFFIX	".gz"
+#define	ZSTD_SUFFIX	".zst"
 
 int compress_user_cores = 0;
 
@@ -3273,7 +3274,9 @@ sysctl_compress_user_cores(SYSCTL_HANDLER_ARGS)
 }
 SYSCTL_PROC(_kern, OID_AUTO, compress_user_cores, CTLTYPE_INT | CTLFLAG_RWTUN,
     0, sizeof(int), sysctl_compress_user_cores, "I",
-    "Enable compression of user corefiles (" __XSTRING(COMPRESS_GZIP) " = gzip)");
+    "Enable compression of user corefiles ("
+    __XSTRING(COMPRESS_GZIP) " = gzip, "
+    __XSTRING(COMPRESS_ZSTD) " = zstd)");
 
 int compress_user_cores_level = 6;
 SYSCTL_INT(_kern, OID_AUTO, compress_user_cores_level, CTLFLAG_RWTUN,
@@ -3377,7 +3380,9 @@ corefile_open(const char *comm, uid_t uid, pid_t pid, 
 	sx_sunlock(&corefilename_lock);
 	free(hostname, M_TEMP);
 	if (compress == COMPRESS_GZIP)
-		sbuf_printf(&sb, GZ_SUFFIX);
+		sbuf_printf(&sb, GZIP_SUFFIX);
+	else if (compress == COMPRESS_ZSTD)
+		sbuf_printf(&sb, ZSTD_SUFFIX);
 	if (sbuf_error(&sb) != 0) {
 		log(LOG_ERR, "pid %ld (%s), uid (%lu): corename is too "
 		    "long\n", (long)pid, comm, (u_long)uid);

Modified: head/sys/kern/subr_compressor.c
==============================================================================
--- head/sys/kern/subr_compressor.c	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sys/kern/subr_compressor.c	Tue Feb 13 19:28:02 2018	(r329240)
@@ -2,6 +2,7 @@
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2014, 2017 Mark Johnston <markj@FreeBSD.org>
+ * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are
@@ -34,8 +35,10 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_gzio.h"
+#include "opt_zstdio.h"
 
 #include <sys/param.h>
+#include <sys/systm.h>
 
 #include <sys/compressor.h>
 #include <sys/kernel.h>
@@ -244,6 +247,232 @@ struct compressor_methods gzip_methods = {
 DATA_SET(compressors, gzip_methods);
 
 #endif /* GZIO */
+
+#ifdef ZSTDIO
+
+#define	ZSTD_STATIC_LINKING_ONLY
+#include <contrib/zstd/lib/zstd.h>
+
+struct zstdio_stream {
+	ZSTD_CCtx	*zst_stream;
+	ZSTD_inBuffer	zst_inbuffer;
+	ZSTD_outBuffer	zst_outbuffer;
+	uint8_t *	zst_buffer;	/* output buffer */
+	size_t		zst_maxiosz;	/* Max output IO size */
+	off_t		zst_off;	/* offset into the output stream */
+	void *		zst_static_wkspc;
+};
+
+static void 	*zstdio_init(size_t maxiosize, int level);
+static void	zstdio_reset(void *stream);
+static int	zstdio_write(void *stream, void *data, size_t len,
+		    compressor_cb_t, void *);
+static void	zstdio_fini(void *stream);
+
+static void *
+zstdio_init(size_t maxiosize, int level)
+{
+	ZSTD_CCtx *dump_compressor;
+	struct zstdio_stream *s;
+	void *wkspc, *owkspc, *buffer;
+	size_t wkspc_size, buf_size;
+
+	wkspc_size = ZSTD_estimateCStreamSize(level);
+	owkspc = wkspc = malloc(wkspc_size + 8, M_COMPRESS,
+	    M_WAITOK | M_NODUMP);
+	/* Zstd API requires 8-byte alignment. */
+	if ((uintptr_t)wkspc % 8 != 0)
+		wkspc = (void *)roundup2((uintptr_t)wkspc, 8);
+
+	dump_compressor = ZSTD_initStaticCCtx(wkspc, wkspc_size);
+	if (dump_compressor == NULL) {
+		free(owkspc, M_COMPRESS);
+		printf("%s: workspace too small.\n", __func__);
+		return (NULL);
+	}
+
+	(void)ZSTD_CCtx_setParameter(dump_compressor, ZSTD_p_checksumFlag, 1);
+
+	buf_size = ZSTD_CStreamOutSize() * 2;
+	buffer = malloc(buf_size, M_COMPRESS, M_WAITOK | M_NODUMP);
+
+	s = malloc(sizeof(*s), M_COMPRESS, M_NODUMP | M_WAITOK);
+	s->zst_buffer = buffer;
+	s->zst_outbuffer.dst = buffer;
+	s->zst_outbuffer.size = buf_size;
+	s->zst_maxiosz = maxiosize;
+	s->zst_stream = dump_compressor;
+	s->zst_static_wkspc = owkspc;
+
+	zstdio_reset(s);
+
+	return (s);
+}
+
+static void
+zstdio_reset(void *stream)
+{
+	struct zstdio_stream *s;
+	size_t res;
+
+	s = stream;
+	res = ZSTD_resetCStream(s->zst_stream, 0);
+	if (ZSTD_isError(res))
+		panic("%s: could not reset stream %p: %s\n", __func__, s,
+		    ZSTD_getErrorName(res));
+
+	s->zst_off = 0;
+	s->zst_inbuffer.src = NULL;
+	s->zst_inbuffer.size = 0;
+	s->zst_inbuffer.pos = 0;
+	s->zst_outbuffer.pos = 0;
+}
+
+static int
+zst_flush_intermediate(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
+{
+	size_t bytes_to_dump;
+	int error;
+
+	/* Flush as many full output blocks as possible. */
+	/* XXX: 4096 is arbitrary safe HDD block size for kernel dumps */
+	while (s->zst_outbuffer.pos >= 4096) {
+		bytes_to_dump = rounddown(s->zst_outbuffer.pos, 4096);
+
+		if (bytes_to_dump > s->zst_maxiosz)
+			bytes_to_dump = s->zst_maxiosz;
+
+		error = cb(s->zst_buffer, bytes_to_dump, s->zst_off, arg);
+		if (error != 0)
+			return (error);
+
+		/*
+		 * Shift any non-full blocks up to the front of the output
+		 * buffer.
+		 */
+		s->zst_outbuffer.pos -= bytes_to_dump;
+		memmove(s->zst_outbuffer.dst,
+		    (char *)s->zst_outbuffer.dst + bytes_to_dump,
+		    s->zst_outbuffer.pos);
+		s->zst_off += bytes_to_dump;
+	}
+	return (0);
+}
+
+static int
+zstdio_flush(struct zstdio_stream *s, compressor_cb_t cb, void *arg)
+{
+	size_t rc, lastpos;
+	int error;
+
+	/*
+	 * Positive return indicates unflushed data remaining; need to call
+	 * endStream again after clearing out room in output buffer.
+	 */
+	rc = 1;
+	lastpos = s->zst_outbuffer.pos;
+	while (rc > 0) {
+		rc = ZSTD_endStream(s->zst_stream, &s->zst_outbuffer);
+		if (ZSTD_isError(rc)) {
+			printf("%s: ZSTD_endStream failed (%s)\n", __func__,
+			    ZSTD_getErrorName(rc));
+			return (EIO);
+		}
+		if (lastpos == s->zst_outbuffer.pos) {
+			printf("%s: did not make forward progress endStream %zu\n",
+			    __func__, lastpos);
+			return (EIO);
+		}
+
+		error = zst_flush_intermediate(s, cb, arg);
+		if (error != 0)
+			return (error);
+
+		lastpos = s->zst_outbuffer.pos;
+	}
+
+	/*
+	 * We've already done an intermediate flush, so all full blocks have
+	 * been written.  Only a partial block remains.  Padding happens in a
+	 * higher layer.
+	 */
+	if (s->zst_outbuffer.pos != 0) {
+		error = cb(s->zst_buffer, s->zst_outbuffer.pos, s->zst_off,
+		    arg);
+		if (error != 0)
+			return (error);
+	}
+
+	return (0);
+}
+
+static int
+zstdio_write(void *stream, void *data, size_t len, compressor_cb_t cb,
+    void *arg)
+{
+	struct zstdio_stream *s;
+	size_t lastpos, rc;
+	int error;
+
+	s = stream;
+	if (data == NULL)
+		return (zstdio_flush(s, cb, arg));
+
+	s->zst_inbuffer.src = data;
+	s->zst_inbuffer.size = len;
+	s->zst_inbuffer.pos = 0;
+	lastpos = 0;
+
+	while (s->zst_inbuffer.pos < s->zst_inbuffer.size) {
+		rc = ZSTD_compressStream(s->zst_stream, &s->zst_outbuffer,
+		    &s->zst_inbuffer);
+		if (ZSTD_isError(rc)) {
+			printf("%s: Compress failed on %p! (%s)\n",
+			    __func__, data, ZSTD_getErrorName(rc));
+			return (EIO);
+		}
+
+		if (lastpos == s->zst_inbuffer.pos) {
+			/*
+			 * XXX: May need flushStream to make forward progress
+			 */
+			printf("ZSTD: did not make forward progress @pos %zu\n",
+			    lastpos);
+			return (EIO);
+		}
+		lastpos = s->zst_inbuffer.pos;
+
+		error = zst_flush_intermediate(s, cb, arg);
+		if (error != 0)
+			return (error);
+	}
+	return (0);
+}
+
+static void
+zstdio_fini(void *stream)
+{
+	struct zstdio_stream *s;
+
+	s = stream;
+	if (s->zst_static_wkspc != NULL)
+		free(s->zst_static_wkspc, M_COMPRESS);
+	else
+		ZSTD_freeCCtx(s->zst_stream);
+	free(s->zst_buffer, M_COMPRESS);
+	free(s, M_COMPRESS);
+}
+
+static struct compressor_methods zstd_methods = {
+	.format = COMPRESS_ZSTD,
+	.init = zstdio_init,
+	.reset = zstdio_reset,
+	.write = zstdio_write,
+	.fini = zstdio_fini,
+};
+DATA_SET(compressors, zstd_methods);
+
+#endif /* ZSTDIO */
 
 bool
 compressor_avail(int format)

Modified: head/sys/sys/compressor.h
==============================================================================
--- head/sys/sys/compressor.h	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sys/sys/compressor.h	Tue Feb 13 19:28:02 2018	(r329240)
@@ -35,6 +35,7 @@
 
 /* Supported formats. */
 #define	COMPRESS_GZIP	1
+#define	COMPRESS_ZSTD	2
 
 typedef int (*compressor_cb_t)(void *, size_t, off_t, void *);
 

Modified: head/sys/sys/kerneldump.h
==============================================================================
--- head/sys/sys/kerneldump.h	Tue Feb 13 19:19:26 2018	(r329239)
+++ head/sys/sys/kerneldump.h	Tue Feb 13 19:28:02 2018	(r329240)
@@ -59,6 +59,7 @@
 
 #define	KERNELDUMP_COMP_NONE		0
 #define	KERNELDUMP_COMP_GZIP		1
+#define	KERNELDUMP_COMP_ZSTD		2
 
 #define	KERNELDUMP_ENC_NONE		0
 #define	KERNELDUMP_ENC_AES_256_CBC	1



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201802131928.w1DJS203052532>