Date: Thu, 1 Aug 2019 06:35:33 +0000 (UTC) From: Xin LI <delphij@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r350496 - in head/sys: cddl/contrib/opensolaris/uts/common/fs/zfs cddl/contrib/opensolaris/uts/common/zmod conf contrib/zlib dev/zlib libkern modules/crypto modules/zfs modules/zlib ope... Message-ID: <201908010635.x716ZXUh057079@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: delphij Date: Thu Aug 1 06:35:33 2019 New Revision: 350496 URL: https://svnweb.freebsd.org/changeset/base/350496 Log: Allow Kernel to link in both legacy libkern/zlib and new sys/contrib/zlib, with an eventual goal to convert all legacl zlib callers to the new zlib version: * Move generic zlib shims that are not specific to zlib 1.0.4 to sys/dev/zlib. * Connect new zlib (1.2.11) to the zlib kernel module, currently built with Z_SOLO. * Prefix the legacy zlib (1.0.4) with 'zlib104_' namespace. * Convert sys/opencrypto/cryptodeflate.c to use new zlib. * Remove bundled zlib 1.2.3 from ZFS and adapt it to new zlib and make it depend on the zlib module. * Fix Z_SOLO build of new zlib. PR: 229763 Submitted by: Yoshihiro Ota <ota j email ne jp> Reviewed by: markm (sys/dev/zlib/zlib_kmod.c) Relnotes: yes Differential Revision: https://reviews.freebsd.org/D19706 Added: head/sys/dev/zlib/ head/sys/dev/zlib/zcalloc.c (contents, props changed) head/sys/dev/zlib/zcalloc.h (contents, props changed) head/sys/dev/zlib/zlib_mod.c - copied, changed from r350463, head/sys/libkern/zlib.c Deleted: head/sys/cddl/contrib/opensolaris/uts/common/zmod/adler32.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/crc32.h head/sys/cddl/contrib/opensolaris/uts/common/zmod/deflate.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/deflate.h head/sys/cddl/contrib/opensolaris/uts/common/zmod/inffast.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/inffast.h head/sys/cddl/contrib/opensolaris/uts/common/zmod/inffixed.h head/sys/cddl/contrib/opensolaris/uts/common/zmod/inflate.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/inflate.h head/sys/cddl/contrib/opensolaris/uts/common/zmod/inftrees.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/inftrees.h head/sys/cddl/contrib/opensolaris/uts/common/zmod/opensolaris_crc32.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/trees.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/zconf.h head/sys/cddl/contrib/opensolaris/uts/common/zmod/zlib.h head/sys/cddl/contrib/opensolaris/uts/common/zmod/zmod_subr.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/zutil.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/zutil.h Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c head/sys/cddl/contrib/opensolaris/uts/common/zmod/zmod.c head/sys/conf/NOTES head/sys/conf/files head/sys/conf/kern.pre.mk head/sys/conf/kmod.mk head/sys/contrib/zlib/deflate.c head/sys/contrib/zlib/zconf.h head/sys/contrib/zlib/zutil.h head/sys/libkern/zlib.c head/sys/modules/crypto/Makefile head/sys/modules/zfs/Makefile head/sys/modules/zlib/Makefile head/sys/opencrypto/cryptodeflate.c head/sys/opencrypto/deflate.h head/sys/sys/zlib.h Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c ============================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c Thu Aug 1 05:30:31 2019 (r350495) +++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c Thu Aug 1 06:35:33 2019 (r350496) @@ -7172,3 +7172,4 @@ MODULE_VERSION(zfsctrl, 1); MODULE_DEPEND(zfsctrl, opensolaris, 1, 1, 1); MODULE_DEPEND(zfsctrl, krpc, 1, 1, 1); MODULE_DEPEND(zfsctrl, acl_nfs4, 1, 1, 1); +MODULE_DEPEND(zfsctrl, zlib, 1, 1, 1); Modified: head/sys/cddl/contrib/opensolaris/uts/common/zmod/zmod.c ============================================================================== --- head/sys/cddl/contrib/opensolaris/uts/common/zmod/zmod.c Thu Aug 1 05:30:31 2019 (r350495) +++ head/sys/cddl/contrib/opensolaris/uts/common/zmod/zmod.c Thu Aug 1 06:35:33 2019 (r350496) @@ -25,11 +25,49 @@ */ #include <sys/types.h> +#include <sys/cmn_err.h> +#include <sys/systm.h> +#include <sys/kobj.h> #include <sys/zmod.h> -#include "zlib.h" -#include "zutil.h" +#include <contrib/zlib/zlib.h> +#include <contrib/zlib/zutil.h> +struct zchdr { + uint_t zch_magic; + uint_t zch_size; +}; + +#define ZCH_MAGIC 0x3cc13cc1 + +/*ARGSUSED*/ +static void * +zfs_zcalloc(void *opaque, uint_t items, uint_t size) +{ + size_t nbytes = sizeof (struct zchdr) + items * size; + struct zchdr *z = kobj_zalloc(nbytes, KM_NOWAIT|KM_TMP); + + if (z == NULL) + return (NULL); + + z->zch_magic = ZCH_MAGIC; + z->zch_size = nbytes; + + return (z + 1); +} + +/*ARGSUSED*/ +static void +zfs_zcfree(void *opaque, void *ptr) +{ + struct zchdr *z = ((struct zchdr *)ptr) - 1; + + if (z->zch_magic != ZCH_MAGIC) + panic("zcfree region corrupt: hdr=%p ptr=%p", (void *)z, ptr); + + kobj_free(z, z->zch_size); +} + /* * Uncompress the buffer 'src' into the buffer 'dst'. The caller must store * the expected decompressed data size externally so it can be passed in. @@ -47,6 +85,8 @@ z_uncompress(void *dst, size_t *dstlen, const void *sr zs.avail_in = srclen; zs.next_out = dst; zs.avail_out = *dstlen; + zs.zalloc = zfs_zcalloc; + zs.zfree = zfs_zcfree; /* * Call inflateInit2() specifying a window size of DEF_WBITS @@ -78,6 +118,8 @@ z_compress_level(void *dst, size_t *dstlen, const void zs.avail_in = srclen; zs.next_out = dst; zs.avail_out = *dstlen; + zs.zalloc = zfs_zcalloc; + zs.zfree = zfs_zcfree; if ((err = deflateInit(&zs, level)) != Z_OK) return (err); Modified: head/sys/conf/NOTES ============================================================================== --- head/sys/conf/NOTES Thu Aug 1 05:30:31 2019 (r350495) +++ head/sys/conf/NOTES Thu Aug 1 06:35:33 2019 (r350496) @@ -2981,4 +2981,6 @@ device spigen # Generic access to SPI devices from u # Enable legacy /dev/spigenN name aliases for /dev/spigenX.Y devices. options SPIGEN_LEGACY_CDEVNAME # legacy device names for spigen +# Compression supports. +device zlib # gzip/zlib compression/decompression library device xz # xz_embedded LZMA de-compression library Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Thu Aug 1 05:30:31 2019 (r350495) +++ head/sys/conf/files Thu Aug 1 06:35:33 2019 (r350496) @@ -273,16 +273,7 @@ cddl/contrib/opensolaris/uts/common/os/callb.c opti cddl/contrib/opensolaris/uts/common/os/fm.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/list.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/os/nvpair_alloc_system.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/adler32.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/deflate.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/inffast.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/inflate.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/inftrees.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/opensolaris_crc32.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/trees.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/zmod.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/zmod_subr.c optional zfs compile-with "${ZFS_C}" -cddl/contrib/opensolaris/uts/common/zmod/zutil.c optional zfs compile-with "${ZFS_C}" +cddl/contrib/opensolaris/uts/common/zmod/zmod.c optional zfs compile-with "${ZFS_C} ${ZLIB_CFLAGS}" # zfs lua support cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lapi.c optional zfs compile-with "${ZFS_C}" cddl/contrib/opensolaris/uts/common/fs/zfs/lua/lauxlib.c optional zfs compile-with "${ZFS_C}" @@ -4005,6 +3996,36 @@ libkern/strvalid.c standard libkern/timingsafe_bcmp.c standard libkern/zlib.c optional crypto | geom_uzip | ipsec | \ ipsec_support | mxge | netgraph_deflate | ddb_ctf | gzio +contrib/zlib/adler32.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C}" +contrib/zlib/crc32.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C} -Wno-cast-qual" +contrib/zlib/deflate.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C} -Wno-cast-qual" +contrib/zlib/inffast.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C}" +contrib/zlib/inflate.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C}" +contrib/zlib/inftrees.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C}" +contrib/zlib/trees.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C}" +contrib/zlib/zutil.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C}" +dev/zlib/zlib_mod.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C}" +dev/zlib/zcalloc.c optional crypto | geom_uzip | ipsec | \ + ipsec_support | mxge | ddb_ctf | gzio | zfs | zlib \ + compile-with "${ZLIB_C}" net/altq/altq_cbq.c optional altq net/altq/altq_codel.c optional altq net/altq/altq_hfsc.c optional altq @@ -4756,7 +4777,8 @@ opencrypto/crypto.c optional crypto | ipsec | ipsec_s opencrypto/cryptodev.c optional cryptodev opencrypto/cryptodev_if.m optional crypto | ipsec | ipsec_support opencrypto/cryptosoft.c optional crypto | ipsec | ipsec_support -opencrypto/cryptodeflate.c optional crypto | ipsec | ipsec_support +opencrypto/cryptodeflate.c optional crypto | ipsec | ipsec_support \ + compile-with "${ZLIB_C}" opencrypto/gmac.c optional crypto | ipsec | ipsec_support opencrypto/gfmult.c optional crypto | ipsec | ipsec_support opencrypto/rmd160.c optional crypto | ipsec | ipsec_support Modified: head/sys/conf/kern.pre.mk ============================================================================== --- head/sys/conf/kern.pre.mk Thu Aug 1 05:30:31 2019 (r350495) +++ head/sys/conf/kern.pre.mk Thu Aug 1 06:35:33 2019 (r350496) @@ -173,6 +173,10 @@ NORMAL_FW= uudecode -o ${.TARGET} ${.ALLSRC} NORMAL_FWO= ${LD} -b binary --no-warn-mismatch -d -warn-common -r \ -m ${LD_EMULATION} -o ${.TARGET} ${.ALLSRC:M*.fw} +# for zlib in the kernel +ZLIB_CFLAGS+= -DZ_SOLO +ZLIB_C= ${CC} -c ${ZLIB_CFLAGS} ${CFLAGS} ${.IMPSRC} + # for ZSTD in the kernel (include zstd/lib/freebsd before other CFLAGS) ZSTD_C= ${CC} -c -DZSTD_HEAPMODE=1 -I$S/contrib/zstd/lib/freebsd ${CFLAGS} -I$S/contrib/zstd/lib -I$S/contrib/zstd/lib/common ${WERROR} -Wno-inline -Wno-missing-prototypes ${PROF} -U__BMI__ ${.IMPSRC} Modified: head/sys/conf/kmod.mk ============================================================================== --- head/sys/conf/kmod.mk Thu Aug 1 05:30:31 2019 (r350495) +++ head/sys/conf/kmod.mk Thu Aug 1 06:35:33 2019 (r350496) @@ -104,6 +104,8 @@ __KLD_SHARED=yes __KLD_SHARED=no .endif +ZLIB_CFLAGS+= -DZ_SOLO + .if !empty(CFLAGS:M-O[23s]) && empty(CFLAGS:M-fno-strict-aliasing) CFLAGS+= -fno-strict-aliasing .endif Modified: head/sys/contrib/zlib/deflate.c ============================================================================== --- head/sys/contrib/zlib/deflate.c Thu Aug 1 05:30:31 2019 (r350495) +++ head/sys/contrib/zlib/deflate.c Thu Aug 1 06:35:33 2019 (r350496) @@ -189,9 +189,10 @@ local const config configuration_table[10] = { * Initialize the hash table (avoiding 64K overflow for 16 bit systems). * prev[] will be initialized on the fly. */ -#define CLEAR_HASH(s) \ +#define CLEAR_HASH(s) do { \ s->head[s->hash_size-1] = NIL; \ - zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); + zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); \ + } while (0) /* =========================================================================== * Slide the hash table when sliding the window down (could be avoided with 32 @@ -1622,8 +1623,10 @@ local void fill_window(s) /* Maximum stored block length in deflate format (not including header). */ #define MAX_STORED 65535 +#if !defined(MIN) /* Minimum of a and b. */ #define MIN(a, b) ((a) > (b) ? (b) : (a)) +#endif /* =========================================================================== * Copy without compression as much as possible from the input stream, return Modified: head/sys/contrib/zlib/zconf.h ============================================================================== --- head/sys/contrib/zlib/zconf.h Thu Aug 1 05:30:31 2019 (r350495) +++ head/sys/contrib/zlib/zconf.h Thu Aug 1 06:35:33 2019 (r350496) @@ -503,6 +503,9 @@ typedef uLong FAR uLongf; /* * This is hard-configured for FreeBSD. */ +#ifdef Z_SOLO +# include <sys/types.h> /* for off_t */ +#endif #define z_off_t off_t #ifndef _FILE_OFFSET_BITS #define _FILE_OFFSET_BITS 64 Modified: head/sys/contrib/zlib/zutil.h ============================================================================== --- head/sys/contrib/zlib/zutil.h Thu Aug 1 05:30:31 2019 (r350495) +++ head/sys/contrib/zlib/zutil.h Thu Aug 1 06:35:33 2019 (r350496) @@ -30,7 +30,10 @@ #endif #ifdef Z_SOLO +#ifndef _PTRDIFF_T_DECLARED typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */ +#define _PTRDIFF_T_DECLARED +#endif #endif #ifndef local Added: head/sys/dev/zlib/zcalloc.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/dev/zlib/zcalloc.c Thu Aug 1 06:35:33 2019 (r350496) @@ -0,0 +1,32 @@ +/* + * This file is in the public domain. + * $FreeBSD$ + */ + +#include <sys/param.h> +#include <dev/zlib/zcalloc.h> +#include <sys/kernel.h> +#include <sys/malloc.h> + +MALLOC_DEFINE(M_ZLIB, "zlib", "ZLIB Compressor"); + +void * +zcalloc_waitok(void *nil, u_int items, u_int size) +{ + + return mallocarray(items, size, M_ZLIB, M_WAITOK); +} + +void * +zcalloc_nowait(void *nil, u_int items, u_int size) +{ + + return mallocarray(items, size, M_ZLIB, M_NOWAIT); +} + +void +zcfree(void *nil, void *ptr) +{ + + free(ptr, M_ZLIB); +} Added: head/sys/dev/zlib/zcalloc.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/dev/zlib/zcalloc.h Thu Aug 1 06:35:33 2019 (r350496) @@ -0,0 +1,13 @@ +/* + * This file is in the public domain. + * $FreeBSD$ + */ + +#ifndef _DEV_ZLIB_ZCALLOC_ +#define _DEV_ZLIB_ZCALLOC_ + +void * zcalloc_waitok(void *nil, u_int items, u_int size); +void * zcalloc_nowait(void *nil, u_int items, u_int size); +void zcfree(void *nil, void *ptr); + +#endif Copied and modified: head/sys/dev/zlib/zlib_mod.c (from r350463, head/sys/libkern/zlib.c) ============================================================================== --- head/sys/libkern/zlib.c Wed Jul 31 05:38:39 2019 (r350463, copy source) +++ head/sys/dev/zlib/zlib_mod.c Thu Aug 1 06:35:33 2019 (r350496) @@ -1,5397 +1,37 @@ -/* - * This file is derived from various .h and .c files from the zlib-1.0.4 - * distribution by Jean-loup Gailly and Mark Adler, with some additions - * by Paul Mackerras to aid in implementing Deflate compression and - * decompression for PPP packets. See zlib.h for conditions of - * distribution and use. +/*- + * SPDX-License-Identifier: BSD-2-Clause * - * Changes that have been made include: - * - added Z_PACKET_FLUSH (see zlib.h for details) - * - added inflateIncomp and deflateOutputPending - * - allow strm->next_out to be NULL, meaning discard the output + * Copyright (c) 2019 The FreeBSD Foundation * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * * $FreeBSD$ */ -/* - * ==FILEVERSION 971210== - * - * This marker is used by the Linux installation script to determine - * whether an up-to-date version of this file is already installed. - */ - -#define NO_DUMMY_DECL -#define NO_ZCFUNCS -#define MY_ZCALLOC - -#if defined(__FreeBSD__) && defined(_KERNEL) -#define _tr_init _zlib104_tr_init -#define _tr_align _zlib104_tr_align -#define _tr_tally _zlib104_tr_tally -#define _tr_flush_block _zlib104_tr_flush_block -#define _tr_stored_block _zlib104_tr_stored_block -#define inflate_fast _zlib104_inflate_fast -#define inflate _zlib104_inflate -#define zlibVersion _zlib104_Version -#endif - - -/* +++ zutil.h */ -/*- - * zutil.h -- internal interface and configuration of the compression library - * Copyright (C) 1995-1996 Jean-loup Gailly. - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* WARNING: this file should *not* be used by applications. It is - part of the implementation of the compression library and is - subject to change. Applications should only use zlib.h. - */ - -/* From: zutil.h,v 1.16 1996/07/24 13:41:13 me Exp $ */ - -#ifndef _Z_UTIL_H -#define _Z_UTIL_H - -#ifdef _KERNEL -#include <sys/zlib.h> -#else -#include "zlib.h" -#endif - -#ifdef _KERNEL -/* Assume this is a *BSD or SVR4 kernel */ -#include <sys/types.h> -#include <sys/time.h> -#include <sys/systm.h> #include <sys/param.h> +#include <sys/time.h> #include <sys/kernel.h> #include <sys/module.h> -# define HAVE_MEMCPY -#else -#if defined(__KERNEL__) -/* Assume this is a Linux kernel */ -#include <linux/string.h> -#define HAVE_MEMCPY - -#else /* not kernel */ - -#if defined(MSDOS)||defined(VMS)||defined(CRAY)||defined(WIN32)||defined(RISCOS) -# include <stddef.h> -# include <errno.h> -#else - extern int errno; -#endif -#ifdef STDC -# include <string.h> -# include <stdlib.h> -#endif -#endif /* __KERNEL__ */ -#endif /* _KERNEL */ - -#ifndef local -# define local static -#endif -/* compile with -Dlocal if your debugger can't find static symbols */ - -typedef unsigned char uch; -typedef uch FAR uchf; -typedef unsigned short ush; -typedef ush FAR ushf; -typedef unsigned long ulg; - -static const char *z_errmsg[10]; /* indexed by 2-zlib_error */ -/* (size given to avoid silly warnings with Visual C++) */ - -#define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] - -#define ERR_RETURN(strm,err) \ - return (strm->msg = (const char*)ERR_MSG(err), (err)) -/* To be used only when the state is known to be valid */ - - /* common constants */ - -#ifndef DEF_WBITS -# define DEF_WBITS MAX_WBITS -#endif -/* default windowBits for decompression. MAX_WBITS is for compression only */ - -#if MAX_MEM_LEVEL >= 8 -# define DEF_MEM_LEVEL 8 -#else -# define DEF_MEM_LEVEL MAX_MEM_LEVEL -#endif -/* default memLevel */ - -#define STORED_BLOCK 0 -#define STATIC_TREES 1 -#define DYN_TREES 2 -/* The three kinds of block type */ - -#define MIN_MATCH 3 -#define MAX_MATCH 258 -/* The minimum and maximum match lengths */ - -#define PRESET_DICT 0x20 /* preset dictionary flag in zlib header */ - - /* target dependencies */ - -#ifdef MSDOS -# define OS_CODE 0x00 -# ifdef __TURBOC__ -# include <alloc.h> -# else /* MSC or DJGPP */ -# include <malloc.h> -# endif -#endif - -#ifdef OS2 -# define OS_CODE 0x06 -#endif - -#ifdef WIN32 /* Window 95 & Windows NT */ -# define OS_CODE 0x0b -#endif - -#if defined(VAXC) || defined(VMS) -# define OS_CODE 0x02 -# define FOPEN(name, mode) \ - fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") -#endif - -#ifdef AMIGA -# define OS_CODE 0x01 -#endif - -#if defined(ATARI) || defined(atarist) -# define OS_CODE 0x05 -#endif - -#ifdef MACOS -# define OS_CODE 0x07 -#endif - -#ifdef __50SERIES /* Prime/PRIMOS */ -# define OS_CODE 0x0F -#endif - -#ifdef TOPS20 -# define OS_CODE 0x0a -#endif - -#if defined(_BEOS_) || defined(RISCOS) -# define fdopen(fd,mode) NULL /* No fdopen() */ -#endif - - /* Common defaults */ - -#ifndef OS_CODE -# define OS_CODE 0x03 /* assume Unix */ -#endif - -#ifndef FOPEN -# define FOPEN(name, mode) fopen((name), (mode)) -#endif - - /* functions */ - -#ifdef HAVE_STRERROR - extern char *strerror OF((int)); -# define zstrerror(errnum) strerror(errnum) -#else -# define zstrerror(errnum) "" -#endif - -#if defined(pyr) -# define NO_MEMCPY -#endif -#if (defined(M_I86SM) || defined(M_I86MM)) && !defined(_MSC_VER) - /* Use our own functions for small and medium model with MSC <= 5.0. - * You may have to use the same strategy for Borland C (untested). - */ -# define NO_MEMCPY -#endif -#if defined(STDC) && !defined(HAVE_MEMCPY) && !defined(NO_MEMCPY) -# define HAVE_MEMCPY -#endif -#ifdef HAVE_MEMCPY -# ifdef SMALL_MEDIUM /* MSDOS small or medium model */ -# define zmemcpy _fmemcpy -# define zmemcmp _fmemcmp -# define zmemzero(dest, len) _fmemset(dest, 0, len) -# else -# define zmemcpy memcpy -# define zmemcmp memcmp -# define zmemzero(dest, len) memset(dest, 0, len) -# endif -#else - extern void zmemcpy OF((Bytef* dest, Bytef* source, uInt len)); - extern int zmemcmp OF((Bytef* s1, Bytef* s2, uInt len)); - extern void zmemzero OF((Bytef* dest, uInt len)); -#endif - -/* Diagnostic functions */ -#ifdef DEBUG_ZLIB -# include <stdio.h> -# ifndef verbose -# define verbose 0 -# endif - extern void z_error OF((char *m)); -# define Assert(cond,msg) {if(!(cond)) z_error(msg);} -# define Trace(x) fprintf x -# define Tracev(x) {if (verbose) fprintf x ;} -# define Tracevv(x) {if (verbose>1) fprintf x ;} -# define Tracec(c,x) {if (verbose && (c)) fprintf x ;} -# define Tracecv(c,x) {if (verbose>1 && (c)) fprintf x ;} -#else -# define Assert(cond,msg) -# define Trace(x) -# define Tracev(x) -# define Tracevv(x) -# define Tracec(c,x) -# define Tracecv(c,x) -#endif - - -typedef uLong (*check_func) OF((uLong check, const Bytef *buf, uInt len)); - -voidpf zcalloc OF((voidpf opaque, unsigned items, unsigned size)); -void zcfree OF((voidpf opaque, voidpf ptr)); - -#define ZALLOC(strm, items, size) \ - (*((strm)->zalloc))((strm)->opaque, (items), (size)) -#define ZFREE(strm, addr) (*((strm)->zfree))((strm)->opaque, (voidpf)(addr)) -#define TRY_FREE(s, p) {if (p) ZFREE(s, p);} - -#endif /* _Z_UTIL_H */ -/* --- zutil.h */ - -/* +++ deflate.h */ -/* deflate.h -- internal compression state - * Copyright (C) 1995-1996 Jean-loup Gailly - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* WARNING: this file should *not* be used by applications. It is - part of the implementation of the compression library and is - subject to change. Applications should only use zlib.h. - */ - -/* From: deflate.h,v 1.10 1996/07/02 12:41:00 me Exp $ */ - -#ifndef _DEFLATE_H -#define _DEFLATE_H - -/* #include "zutil.h" */ - -/* =========================================================================== - * Internal compression state. - */ - -#define LENGTH_CODES 29 -/* number of length codes, not counting the special END_BLOCK code */ - -#define LITERALS 256 -/* number of literal bytes 0..255 */ - -#define L_CODES (LITERALS+1+LENGTH_CODES) -/* number of Literal or Length codes, including the END_BLOCK code */ - -#define D_CODES 30 -/* number of distance codes */ - -#define BL_CODES 19 -/* number of codes used to transfer the bit lengths */ - -#define HEAP_SIZE (2*L_CODES+1) -/* maximum heap size */ - -#define MAX_BITS 15 -/* All codes must not exceed MAX_BITS bits */ - -#define INIT_STATE 42 -#define BUSY_STATE 113 -#define FINISH_STATE 666 -/* Stream status */ - - -/* Data structure describing a single value and its code string. */ -typedef struct ct_data_s { - union { - ush freq; /* frequency count */ - ush code; /* bit string */ - } fc; - union { - ush dad; /* father node in Huffman tree */ - ush len; /* length of bit string */ - } dl; -} FAR ct_data; - -#define Freq fc.freq -#define Code fc.code -#define Dad dl.dad -#define Len dl.len - -typedef struct static_tree_desc_s static_tree_desc; - -typedef struct tree_desc_s { - ct_data *dyn_tree; /* the dynamic tree */ - int max_code; /* largest code with non zero frequency */ - static_tree_desc *stat_desc; /* the corresponding static tree */ -} FAR tree_desc; - -typedef ush Pos; -typedef Pos FAR Posf; -typedef unsigned IPos; - -/* A Pos is an index in the character window. We use short instead of int to - * save space in the various tables. IPos is used only for parameter passing. - */ - -typedef struct deflate_state { - z_streamp strm; /* pointer back to this zlib stream */ - int status; /* as the name implies */ - Bytef *pending_buf; /* output still pending */ - ulg pending_buf_size; /* size of pending_buf */ - Bytef *pending_out; /* next pending byte to output to the stream */ - int pending; /* nb of bytes in the pending buffer */ - int noheader; /* suppress zlib header and adler32 */ - Byte data_type; /* UNKNOWN, BINARY or ASCII */ - Byte method; /* STORED (for zip only) or DEFLATED */ - int last_flush; /* value of flush param for previous deflate call */ - - /* used by deflate.c: */ - - uInt w_size; /* LZ77 window size (32K by default) */ - uInt w_bits; /* log2(w_size) (8..16) */ - uInt w_mask; /* w_size - 1 */ - - Bytef *window; - /* Sliding window. Input bytes are read into the second half of the window, - * and move to the first half later to keep a dictionary of at least wSize - * bytes. With this organization, matches are limited to a distance of - * wSize-MAX_MATCH bytes, but this ensures that IO is always - * performed with a length multiple of the block size. Also, it limits - * the window size to 64K, which is quite useful on MSDOS. - * To do: use the user input buffer as sliding window. - */ - - ulg window_size; - /* Actual size of window: 2*wSize, except when the user input buffer - * is directly used as sliding window. - */ - - Posf *prev; - /* Link to older string with same hash index. To limit the size of this - * array to 64K, this link is maintained only for the last 32K strings. - * An index in this array is thus a window index modulo 32K. - */ - - Posf *head; /* Heads of the hash chains or NIL. */ - - uInt ins_h; /* hash index of string to be inserted */ - uInt hash_size; /* number of elements in hash table */ - uInt hash_bits; /* log2(hash_size) */ - uInt hash_mask; /* hash_size-1 */ - - uInt hash_shift; - /* Number of bits by which ins_h must be shifted at each input - * step. It must be such that after MIN_MATCH steps, the oldest - * byte no longer takes part in the hash key, that is: - * hash_shift * MIN_MATCH >= hash_bits - */ - - long block_start; - /* Window position at the beginning of the current output block. Gets - * negative when the window is moved backwards. - */ - - uInt match_length; /* length of best match */ - IPos prev_match; /* previous match */ - int match_available; /* set if previous match exists */ - uInt strstart; /* start of string to insert */ - uInt match_start; /* start of matching string */ - uInt lookahead; /* number of valid bytes ahead in window */ - - uInt prev_length; - /* Length of the best match at previous step. Matches not greater than this - * are discarded. This is used in the lazy match evaluation. - */ - - uInt max_chain_length; - /* To speed up deflation, hash chains are never searched beyond this - * length. A higher limit improves compression ratio but degrades the - * speed. - */ - - uInt max_lazy_match; - /* Attempt to find a better match only when the current match is strictly - * smaller than this value. This mechanism is used only for compression - * levels >= 4. - */ -# define max_insert_length max_lazy_match - /* Insert new strings in the hash table only if the match length is not - * greater than this length. This saves time but degrades compression. - * max_insert_length is used only for compression levels <= 3. - */ - - int level; /* compression level (1..9) */ - int strategy; /* favor or force Huffman coding*/ - - uInt good_match; - /* Use a faster search when the previous match is longer than this */ - - int nice_match; /* Stop searching when current match exceeds this */ - - /* used by trees.c: */ - /* Didn't use ct_data typedef below to supress compiler warning */ - struct ct_data_s dyn_ltree[HEAP_SIZE]; /* literal and length tree */ - struct ct_data_s dyn_dtree[2*D_CODES+1]; /* distance tree */ - struct ct_data_s bl_tree[2*BL_CODES+1]; /* Huffman tree for bit lengths */ - - struct tree_desc_s l_desc; /* desc. for literal tree */ - struct tree_desc_s d_desc; /* desc. for distance tree */ - struct tree_desc_s bl_desc; /* desc. for bit length tree */ - - ush bl_count[MAX_BITS+1]; - /* number of codes at each bit length for an optimal tree */ - - int heap[2*L_CODES+1]; /* heap used to build the Huffman trees */ - int heap_len; /* number of elements in the heap */ - int heap_max; /* element of largest frequency */ - /* The sons of heap[n] are heap[2*n] and heap[2*n+1]. heap[0] is not used. - * The same heap array is used to build all trees. - */ - - uch depth[2*L_CODES+1]; - /* Depth of each subtree used as tie breaker for trees of equal frequency - */ - - uchf *l_buf; /* buffer for literals or lengths */ - - uInt lit_bufsize; - /* Size of match buffer for literals/lengths. There are 4 reasons for - * limiting lit_bufsize to 64K: - * - frequencies can be kept in 16 bit counters - * - if compression is not successful for the first block, all input - * data is still in the window so we can still emit a stored block even - * when input comes from standard input. (This can also be done for - * all blocks if lit_bufsize is not greater than 32K.) - * - if compression is not successful for a file smaller than 64K, we can - * even emit a stored file instead of a stored block (saving 5 bytes). - * This is applicable only for zip (not gzip or zlib). - * - creating new Huffman trees less frequently may not provide fast - * adaptation to changes in the input data statistics. (Take for - * example a binary file with poorly compressible code followed by - * a highly compressible string table.) Smaller buffer sizes give - * fast adaptation but have of course the overhead of transmitting - * trees more frequently. - * - I can't count above 4 - */ - - uInt last_lit; /* running index in l_buf */ - - ushf *d_buf; - /* Buffer for distances. To simplify the code, d_buf and l_buf have - * the same number of elements. To use different lengths, an extra flag - * array would be necessary. - */ - - ulg opt_len; /* bit length of current block with optimal trees */ - ulg static_len; /* bit length of current block with static trees */ - ulg compressed_len; /* total bit length of compressed file */ - uInt matches; /* number of string matches in current block */ - int last_eob_len; /* bit length of EOB code for last block */ - -#ifdef DEBUG_ZLIB - ulg bits_sent; /* bit length of the compressed data */ -#endif - - ush bi_buf; - /* Output buffer. bits are inserted starting at the bottom (least - * significant bits). - */ - int bi_valid; - /* Number of valid bits in bi_buf. All bits above the last valid bit - * are always zero. - */ - -} FAR deflate_state; - -/* Output a byte on the stream. - * IN assertion: there is enough room in pending_buf. - */ -#define put_byte(s, c) {s->pending_buf[s->pending++] = (c);} - - -#define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) -/* Minimum amount of lookahead, except at the end of the input file. - * See deflate.c for comments about the MIN_MATCH+1. - */ - -#define MAX_DIST(s) ((s)->w_size-MIN_LOOKAHEAD) -/* In order to simplify the code, particularly on 16 bit machines, match - * distances are limited to MAX_DIST instead of WSIZE. - */ - - /* in trees.c */ -void _tr_init OF((deflate_state *s)); -int _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); -ulg _tr_flush_block OF((deflate_state *s, charf *buf, ulg stored_len, - int eof)); -void _tr_align OF((deflate_state *s)); -void _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len, - int eof)); -void _tr_stored_type_only OF((deflate_state *)); - -#endif -/* --- deflate.h */ - -/* +++ deflate.c */ -/* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-1996 Jean-loup Gailly. - * For conditions of distribution and use, see copyright notice in zlib.h - */ - -/* - * ALGORITHM - * - * The "deflation" process depends on being able to identify portions - * of the input text which are identical to earlier input (within a - * sliding window trailing behind the input currently being processed). - * - * The most straightforward technique turns out to be the fastest for - * most input files: try all possible matches and select the longest. - * The key feature of this algorithm is that insertions into the string - * dictionary are very simple and thus fast, and deletions are avoided - * completely. Insertions are performed at each input character, whereas - * string matches are performed only when the previous match ends. So it - * is preferable to spend more time in matches to allow very fast string - * insertions and avoid deletions. The matching algorithm for small - * strings is inspired from that of Rabin & Karp. A brute force approach - * is used to find longer strings when a small match has been found. - * A similar algorithm is used in comic (by Jan-Mark Wams) and freeze - * (by Leonid Broukhis). - * A previous version of this file used a more sophisticated algorithm - * (by Fiala and Greene) which is guaranteed to run in linear amortized - * time, but has a larger average cost, uses more memory and is patented. - * However the F&G algorithm may be faster for some highly redundant - * files if the parameter max_chain_length (described below) is too large. - * - * ACKNOWLEDGEMENTS - * - * The idea of lazy evaluation of matches is due to Jan-Mark Wams, and - * I found it in 'freeze' written by Leonid Broukhis. - * Thanks to many people for bug reports and testing. - * - * REFERENCES - * - * Deutsch, L.P.,"DEFLATE Compressed Data Format Specification". - * Available in ftp://ds.internic.net/rfc/rfc1951.txt - * - * A description of the Rabin and Karp algorithm is given in the book - * "Algorithms" by R. Sedgewick, Addison-Wesley, p252. - * - * Fiala,E.R., and Greene,D.H. - * Data Compression with Finite Windows, Comm.ACM, 32,4 (1989) 490-595 - * - */ - -/* From: deflate.c,v 1.15 1996/07/24 13:40:58 me Exp $ */ - -/* #include "deflate.h" */ - -char deflate_copyright[] = " deflate 1.0.4 Copyright 1995-1996 Jean-loup Gailly "; -/* - If you use the zlib library in a product, an acknowledgment is welcome - in the documentation of your product. If for some reason you cannot - include such an acknowledgment, I would appreciate that you keep this - copyright string in the executable of your product. - */ - -/* =========================================================================== - * Function prototypes. - */ -typedef enum { - need_more, /* block not completed, need more input or more output */ - block_done, /* block flush performed */ - finish_started, /* finish started, need only more output at next deflate */ - finish_done /* finish done, accept no more input or output */ -} block_state; - -typedef block_state (*compress_func) OF((deflate_state *s, int flush)); -/* Compression function. Returns the block state after the call. */ - -local void fill_window OF((deflate_state *s)); -local block_state deflate_stored OF((deflate_state *s, int flush)); -local block_state deflate_fast OF((deflate_state *s, int flush)); -local block_state deflate_slow OF((deflate_state *s, int flush)); -local void lm_init OF((deflate_state *s)); -local void putShortMSB OF((deflate_state *s, uInt b)); -local void flush_pending OF((z_streamp strm)); -local int read_buf OF((z_streamp strm, charf *buf, unsigned size)); -#ifdef ASMV - void match_init OF((void)); /* asm code initialization */ - uInt longest_match OF((deflate_state *s, IPos cur_match)); -#else -local uInt longest_match OF((deflate_state *s, IPos cur_match)); -#endif - -#ifdef DEBUG_ZLIB -local void check_match OF((deflate_state *s, IPos start, IPos match, - int length)); -#endif - -/* =========================================================================== - * Local data - */ - -#define NIL 0 -/* Tail of hash chains */ - -#ifndef TOO_FAR -# define TOO_FAR 4096 -#endif -/* Matches of length 3 are discarded if their distance exceeds TOO_FAR */ - -#define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) -/* Minimum amount of lookahead, except at the end of the input file. - * See deflate.c for comments about the MIN_MATCH+1. - */ - -/* Values for max_lazy_match, good_match and max_chain_length, depending on - * the desired pack level (0..9). The values given below have been tuned to - * exclude worst case performance for pathological files. Better values may be - * found for specific files. - */ -typedef struct config_s { - ush good_length; /* reduce lazy search above this match length */ - ush max_lazy; /* do not perform lazy search above this match length */ - ush nice_length; /* quit search above this match length */ - ush max_chain; - compress_func func; -} config; - -local config configuration_table[10] = { -/* good lazy nice chain */ -/* 0 */ {0, 0, 0, 0, deflate_stored}, /* store only */ -/* 1 */ {4, 4, 8, 4, deflate_fast}, /* maximum speed, no lazy matches */ -/* 2 */ {4, 5, 16, 8, deflate_fast}, -/* 3 */ {4, 6, 32, 32, deflate_fast}, - -/* 4 */ {4, 4, 16, 16, deflate_slow}, /* lazy matches */ -/* 5 */ {8, 16, 32, 32, deflate_slow}, -/* 6 */ {8, 16, 128, 128, deflate_slow}, -/* 7 */ {8, 32, 128, 256, deflate_slow}, -/* 8 */ {32, 128, 258, 1024, deflate_slow}, -/* 9 */ {32, 258, 258, 4096, deflate_slow}}; /* maximum compression */ - -/* Note: the deflate() code requires max_lazy >= MIN_MATCH and max_chain >= 4 - * For deflate_fast() (levels <= 3) good is ignored and lazy has a different *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201908010635.x716ZXUh057079>