From owner-svn-src-all@FreeBSD.ORG Sat Dec 31 22:35:47 2011 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 4502E106564A; Sat, 31 Dec 2011 22:35:47 +0000 (UTC) (envelope-from adrian@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 2FBED8FC1B; Sat, 31 Dec 2011 22:35:47 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id pBVMZlrg085979; Sat, 31 Dec 2011 22:35:47 GMT (envelope-from adrian@svn.freebsd.org) Received: (from adrian@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id pBVMZll5085971; Sat, 31 Dec 2011 22:35:47 GMT (envelope-from adrian@svn.freebsd.org) Message-Id: <201112312235.pBVMZll5085971@svn.freebsd.org> From: Adrian Chadd Date: Sat, 31 Dec 2011 22:35:47 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r229159 - in vendor/xz-embedded: . dist dist/linux dist/linux/Documentation dist/linux/include dist/linux/include/linux dist/linux/include/linux/decompress dist/linux/lib dist/linux/lib... X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 31 Dec 2011 22:35:47 -0000 Author: adrian Date: Sat Dec 31 22:35:46 2011 New Revision: 229159 URL: http://svn.freebsd.org/changeset/base/229159 Log: Import xz-embedded from git. This is from commit hash '48f4588342f4a4e0182a6740e25675fd8e6c6295'. Added: vendor/xz-embedded/ vendor/xz-embedded/dist/ vendor/xz-embedded/dist/COPYING vendor/xz-embedded/dist/README vendor/xz-embedded/dist/linux/ vendor/xz-embedded/dist/linux/Documentation/ vendor/xz-embedded/dist/linux/Documentation/xz.txt vendor/xz-embedded/dist/linux/include/ vendor/xz-embedded/dist/linux/include/linux/ vendor/xz-embedded/dist/linux/include/linux/decompress/ vendor/xz-embedded/dist/linux/include/linux/decompress/unxz.h vendor/xz-embedded/dist/linux/include/linux/xz.h vendor/xz-embedded/dist/linux/lib/ vendor/xz-embedded/dist/linux/lib/decompress_unxz.c vendor/xz-embedded/dist/linux/lib/xz/ vendor/xz-embedded/dist/linux/lib/xz/Kconfig vendor/xz-embedded/dist/linux/lib/xz/Makefile vendor/xz-embedded/dist/linux/lib/xz/xz_crc32.c vendor/xz-embedded/dist/linux/lib/xz/xz_dec_bcj.c vendor/xz-embedded/dist/linux/lib/xz/xz_dec_lzma2.c vendor/xz-embedded/dist/linux/lib/xz/xz_dec_stream.c vendor/xz-embedded/dist/linux/lib/xz/xz_dec_syms.c vendor/xz-embedded/dist/linux/lib/xz/xz_dec_test.c vendor/xz-embedded/dist/linux/lib/xz/xz_lzma2.h vendor/xz-embedded/dist/linux/lib/xz/xz_private.h vendor/xz-embedded/dist/linux/lib/xz/xz_stream.h vendor/xz-embedded/dist/linux/scripts/ vendor/xz-embedded/dist/linux/scripts/xz_wrap.sh (contents, props changed) vendor/xz-embedded/dist/userspace/ vendor/xz-embedded/dist/userspace/Makefile vendor/xz-embedded/dist/userspace/boottest.c vendor/xz-embedded/dist/userspace/buftest.c vendor/xz-embedded/dist/userspace/xz_config.h vendor/xz-embedded/dist/userspace/xzminidec.c Added: vendor/xz-embedded/dist/COPYING ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/xz-embedded/dist/COPYING Sat Dec 31 22:35:46 2011 (r229159) @@ -0,0 +1,10 @@ + +Licensing of XZ Embedded +======================== + + All the files in this package have been written by Lasse Collin + and/or Igor Pavlov. All these files have been put into the + public domain. You can do whatever you want with these files. + + As usual, this software is provided "as is", without any warranty. + Added: vendor/xz-embedded/dist/README ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/xz-embedded/dist/README Sat Dec 31 22:35:46 2011 (r229159) @@ -0,0 +1,127 @@ + +XZ Embedded +=========== + + XZ Embedded is a relatively small, limited implementation of the .xz + file format. Currently only decoding is implemented. + + XZ Embedded was written for use in the Linux kernel, but the code can + be easily used in other environments too, including regular userspace + applications. + + This README contains information that is useful only when the copy + of XZ Embedded isn't part of the Linux kernel tree. You should also + read linux/Documentation/xz.txt even if you aren't using XZ Embedded + as part of Linux; information in that file is not repeated in this + README. + +Compiling the Linux kernel module + + The xz_dec module depends on crc32 module, so make sure that you have + it enabled (CONFIG_CRC32). + + Building the xz_dec and xz_dec_test modules without support for BCJ + filters: + + cd linux/lib/xz + make -C /path/to/kernel/source \ + KCPPFLAGS=-I"$(pwd)/../../include" M="$(pwd)" \ + CONFIG_XZ_DEC=m CONFIG_XZ_DEC_TEST=m + + Building the xz_dec and xz_dec_test modules with support for BCJ + filters: + + cd linux/lib/xz + make -C /path/to/kernel/source \ + KCPPFLAGS=-I"$(pwd)/../../include" M="$(pwd)" \ + CONFIG_XZ_DEC=m CONFIG_XZ_DEC_TEST=m CONFIG_XZ_DEC_BCJ=y \ + CONFIG_XZ_DEC_X86=y CONFIG_XZ_DEC_POWERPC=y \ + CONFIG_XZ_DEC_IA64=y CONFIG_XZ_DEC_ARM=y \ + CONFIG_XZ_DEC_ARMTHUMB=y CONFIG_XZ_DEC_SPARC=y + + If you want only one or a few of the BCJ filters, omit the appropriate + variables. CONFIG_XZ_DEC_BCJ=y is always required to build the support + code shared between all BCJ filters. + + Most people don't need the xz_dec_test module. You can skip building + it by omitting CONFIG_XZ_DEC_TEST=m from the make command line. + +Compiler requirements + + XZ Embedded should compile as either GNU-C89 (used in the Linux + kernel) or with any C99 compiler. Getting the code to compile with + non-GNU C89 compiler or a C++ compiler should be quite easy as + long as there is a data type for unsigned 64-bit integer (or the + code is modified not to support large files, which needs some more + care than just using 32-bit integer instead of 64-bit). + + If you use GCC, try to use a recent version. For example, on x86-32, + xz_dec_lzma2.c compiled with GCC 3.3.6 is 15-25 % slower than when + compiled with GCC 4.3.3. + +Embedding into userspace applications + + To embed the XZ decoder, copy the following files into a single + directory in your source code tree: + + linux/include/linux/xz.h + linux/lib/xz/xz_crc32.c + linux/lib/xz/xz_dec_lzma2.c + linux/lib/xz/xz_dec_stream.c + linux/lib/xz/xz_lzma2.h + linux/lib/xz/xz_private.h + linux/lib/xz/xz_stream.h + userspace/xz_config.h + + Alternatively, xz.h may be placed into a different directory but then + that directory must be in the compiler include path when compiling + the .c files. + + Your code should use only the functions declared in xz.h. The rest of + the .h files are meant only for internal use in XZ Embedded. + + You may want to modify xz_config.h to be more suitable for your build + environment. Probably you should at least skim through it even if the + default file works as is. + +BCJ filter support + + If you want support for one or more BCJ filters, you need to copy also + linux/lib/xz/xz_dec_bcj.c into your application, and use appropriate + #defines in xz_config.h or in compiler flags. You don't need these + #defines in the code that just uses XZ Embedded via xz.h, but having + them always #defined doesn't hurt either. + + #define Instruction set BCJ filter endianness + XZ_DEC_X86 x86-32 or x86-64 Little endian only + XZ_DEC_POWERPC PowerPC Big endian only + XZ_DEC_IA64 Itanium (IA-64) Big or little endian + XZ_DEC_ARM ARM Little endian only + XZ_DEC_ARMTHUMB ARM-Thumb Little endian only + XZ_DEC_SPARC SPARC Big or little endian + + While some architectures are (partially) bi-endian, the endianness + setting doesn't change the endianness of the instructions on all + architectures. That's why Itanium and SPARC filters work for both big + and little endian executables (Itanium has little endian instructions + and SPARC has big endian instructions). + + There currently is no filter for little endian PowerPC or big endian + ARM or ARM-Thumb. Implementing filters for them can be considered if + there is a need for such filters in real-world applications. + +Notes about shared libraries + + If you are including XZ Embedded into a shared library, you very + probably should rename the xz_* functions to prevent symbol + conflicts in case your library is linked against some other library + or application that also has XZ Embedded in it (which may even be + a different version of XZ Embedded). TODO: Provide an easy way + to do this. + + Please don't create a shared library of XZ Embedded itself unless + it is fine to rebuild everything depending on that shared library + everytime you upgrade to a newer version of XZ Embedded. There are + no API or ABI stability guarantees between different versions of + XZ Embedded. + Added: vendor/xz-embedded/dist/linux/Documentation/xz.txt ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/xz-embedded/dist/linux/Documentation/xz.txt Sat Dec 31 22:35:46 2011 (r229159) @@ -0,0 +1,122 @@ + +XZ data compression in Linux +============================ + +Introduction + + XZ is a general purpose data compression format with high compression + ratio and relatively fast decompression. The primary compression + algorithm (filter) is LZMA2. Additional filters can be used to improve + compression ratio even further. E.g. Branch/Call/Jump (BCJ) filters + improve compression ratio of executable data. + + The XZ decompressor in Linux is called XZ Embedded. It supports + the LZMA2 filter and optionally also BCJ filters. CRC32 is supported + for integrity checking. The home page of XZ Embedded is at + , where you can find the + latest version and also information about using the code outside + the Linux kernel. + + For userspace, XZ Utils provide a zlib-like compression library + and a gzip-like command line tool. XZ Utils can be downloaded from + . + +XZ related components in the kernel + + The xz_dec module provides XZ decompressor with single-call (buffer + to buffer) and multi-call (stateful) APIs. The usage of the xz_dec + module is documented in include/linux/xz.h. + + The xz_dec_test module is for testing xz_dec. xz_dec_test is not + useful unless you are hacking the XZ decompressor. xz_dec_test + allocates a char device major dynamically to which one can write + .xz files from userspace. The decompressed output is thrown away. + Keep an eye on dmesg to see diagnostics printed by xz_dec_test. + See the xz_dec_test source code for the details. + + For decompressing the kernel image, initramfs, and initrd, there + is a wrapper function in lib/decompress_unxz.c. Its API is the + same as in other decompress_*.c files, which is defined in + include/linux/decompress/generic.h. + + scripts/xz_wrap.sh is a wrapper for the xz command line tool found + from XZ Utils. The wrapper sets compression options to values suitable + for compressing the kernel image. + + For kernel makefiles, two commands are provided for use with + $(call if_needed). The kernel image should be compressed with + $(call if_needed,xzkern) which will use a BCJ filter and a big LZMA2 + dictionary. It will also append a four-byte trailer containing the + uncompressed size of the file, which is needed by the boot code. + Other things should be compressed with $(call if_needed,xzmisc) + which will use no BCJ filter and 1 MiB LZMA2 dictionary. + +Notes on compression options + + Since the XZ Embedded supports only streams with no integrity check or + CRC32, make sure that you don't use some other integrity check type + when encoding files that are supposed to be decoded by the kernel. With + liblzma, you need to use either LZMA_CHECK_NONE or LZMA_CHECK_CRC32 + when encoding. With the xz command line tool, use --check=none or + --check=crc32. + + Using CRC32 is strongly recommended unless there is some other layer + which will verify the integrity of the uncompressed data anyway. + Double checking the integrity would probably be waste of CPU cycles. + Note that the headers will always have a CRC32 which will be validated + by the decoder; you can only change the integrity check type (or + disable it) for the actual uncompressed data. + + In userspace, LZMA2 is typically used with dictionary sizes of several + megabytes. The decoder needs to have the dictionary in RAM, thus big + dictionaries cannot be used for files that are intended to be decoded + by the kernel. 1 MiB is probably the maximum reasonable dictionary + size for in-kernel use (maybe more is OK for initramfs). The presets + in XZ Utils may not be optimal when creating files for the kernel, + so don't hesitate to use custom settings. Example: + + xz --check=crc32 --lzma2=dict=512KiB inputfile + + An exception to above dictionary size limitation is when the decoder + is used in single-call mode. Decompressing the kernel itself is an + example of this situation. In single-call mode, the memory usage + doesn't depend on the dictionary size, and it is perfectly fine to + use a big dictionary: for maximum compression, the dictionary should + be at least as big as the uncompressed data itself. + +Future plans + + Creating a limited XZ encoder may be considered if people think it is + useful. LZMA2 is slower to compress than e.g. Deflate or LZO even at + the fastest settings, so it isn't clear if LZMA2 encoder is wanted + into the kernel. + + Support for limited random-access reading is planned for the + decompression code. I don't know if it could have any use in the + kernel, but I know that it would be useful in some embedded projects + outside the Linux kernel. + +Conformance to the .xz file format specification + + There are a couple of corner cases where things have been simplified + at expense of detecting errors as early as possible. These should not + matter in practice all, since they don't cause security issues. But + it is good to know this if testing the code e.g. with the test files + from XZ Utils. + +Reporting bugs + + Before reporting a bug, please check that it's not fixed already + at upstream. See to get the + latest code. + + Report bugs to or visit #tukaani on + Freenode and talk to Larhzu. I don't actively read LKML or other + kernel-related mailing lists, so if there's something I should know, + you should email to me personally or use IRC. + + Don't bother Igor Pavlov with questions about the XZ implementation + in the kernel or about XZ Utils. While these two implementations + include essential code that is directly based on Igor Pavlov's code, + these implementations aren't maintained nor supported by him. + Added: vendor/xz-embedded/dist/linux/include/linux/decompress/unxz.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/xz-embedded/dist/linux/include/linux/decompress/unxz.h Sat Dec 31 22:35:46 2011 (r229159) @@ -0,0 +1,19 @@ +/* + * Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd + * + * Author: Lasse Collin + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +#ifndef DECOMPRESS_UNXZ_H +#define DECOMPRESS_UNXZ_H + +int unxz(unsigned char *in, int in_size, + int (*fill)(void *dest, unsigned int size), + int (*flush)(void *src, unsigned int size), + unsigned char *out, int *in_used, + void (*error)(char *x)); + +#endif Added: vendor/xz-embedded/dist/linux/include/linux/xz.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/xz-embedded/dist/linux/include/linux/xz.h Sat Dec 31 22:35:46 2011 (r229159) @@ -0,0 +1,273 @@ +/* + * XZ decompressor + * + * Authors: Lasse Collin + * Igor Pavlov + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +#ifndef XZ_H +#define XZ_H + +#ifdef __KERNEL__ +# include +# include +#else +# include +# include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* In Linux, this is used to make extern functions static when needed. */ +#ifndef XZ_EXTERN +# define XZ_EXTERN extern +#endif + +/** + * enum xz_mode - Operation mode + * + * @XZ_SINGLE: Single-call mode. This uses less RAM than + * than multi-call modes, because the LZMA2 + * dictionary doesn't need to be allocated as + * part of the decoder state. All required data + * structures are allocated at initialization, + * so xz_dec_run() cannot return XZ_MEM_ERROR. + * @XZ_PREALLOC: Multi-call mode with preallocated LZMA2 + * dictionary buffer. All data structures are + * allocated at initialization, so xz_dec_run() + * cannot return XZ_MEM_ERROR. + * @XZ_DYNALLOC: Multi-call mode. The LZMA2 dictionary is + * allocated once the required size has been + * parsed from the stream headers. If the + * allocation fails, xz_dec_run() will return + * XZ_MEM_ERROR. + * + * It is possible to enable support only for a subset of the above + * modes at compile time by defining XZ_DEC_SINGLE, XZ_DEC_PREALLOC, + * or XZ_DEC_DYNALLOC. The xz_dec kernel module is always compiled + * with support for all operation modes, but the preboot code may + * be built with fewer features to minimize code size. + */ +enum xz_mode { + XZ_SINGLE, + XZ_PREALLOC, + XZ_DYNALLOC +}; + +/** + * enum xz_ret - Return codes + * @XZ_OK: Everything is OK so far. More input or more + * output space is required to continue. This + * return code is possible only in multi-call mode + * (XZ_PREALLOC or XZ_DYNALLOC). + * @XZ_STREAM_END: Operation finished successfully. + * @XZ_UNSUPPORTED_CHECK: Integrity check type is not supported. Decoding + * is still possible in multi-call mode by simply + * calling xz_dec_run() again. + * Note that this return value is used only if + * XZ_DEC_ANY_CHECK was defined at build time, + * which is not used in the kernel. Unsupported + * check types return XZ_OPTIONS_ERROR if + * XZ_DEC_ANY_CHECK was not defined at build time. + * @XZ_MEM_ERROR: Allocating memory failed. This return code is + * possible only if the decoder was initialized + * with XZ_DYNALLOC. The amount of memory that was + * tried to be allocated was no more than the + * dict_max argument given to xz_dec_init(). + * @XZ_MEMLIMIT_ERROR: A bigger LZMA2 dictionary would be needed than + * allowed by the dict_max argument given to + * xz_dec_init(). This return value is possible + * only in multi-call mode (XZ_PREALLOC or + * XZ_DYNALLOC); the single-call mode (XZ_SINGLE) + * ignores the dict_max argument. + * @XZ_FORMAT_ERROR: File format was not recognized (wrong magic + * bytes). + * @XZ_OPTIONS_ERROR: This implementation doesn't support the requested + * compression options. In the decoder this means + * that the header CRC32 matches, but the header + * itself specifies something that we don't support. + * @XZ_DATA_ERROR: Compressed data is corrupt. + * @XZ_BUF_ERROR: Cannot make any progress. Details are slightly + * different between multi-call and single-call + * mode; more information below. + * + * In multi-call mode, XZ_BUF_ERROR is returned when two consecutive calls + * to XZ code cannot consume any input and cannot produce any new output. + * This happens when there is no new input available, or the output buffer + * is full while at least one output byte is still pending. Assuming your + * code is not buggy, you can get this error only when decoding a compressed + * stream that is truncated or otherwise corrupt. + * + * In single-call mode, XZ_BUF_ERROR is returned only when the output buffer + * is too small or the compressed input is corrupt in a way that makes the + * decoder produce more output than the caller expected. When it is + * (relatively) clear that the compressed input is truncated, XZ_DATA_ERROR + * is used instead of XZ_BUF_ERROR. + */ +enum xz_ret { + XZ_OK, + XZ_STREAM_END, + XZ_UNSUPPORTED_CHECK, + XZ_MEM_ERROR, + XZ_MEMLIMIT_ERROR, + XZ_FORMAT_ERROR, + XZ_OPTIONS_ERROR, + XZ_DATA_ERROR, + XZ_BUF_ERROR +}; + +/** + * struct xz_buf - Passing input and output buffers to XZ code + * @in: Beginning of the input buffer. This may be NULL if and only + * if in_pos is equal to in_size. + * @in_pos: Current position in the input buffer. This must not exceed + * in_size. + * @in_size: Size of the input buffer + * @out: Beginning of the output buffer. This may be NULL if and only + * if out_pos is equal to out_size. + * @out_pos: Current position in the output buffer. This must not exceed + * out_size. + * @out_size: Size of the output buffer + * + * Only the contents of the output buffer from out[out_pos] onward, and + * the variables in_pos and out_pos are modified by the XZ code. + */ +struct xz_buf { + const uint8_t *in; + size_t in_pos; + size_t in_size; + + uint8_t *out; + size_t out_pos; + size_t out_size; +}; + +/** + * struct xz_dec - Opaque type to hold the XZ decoder state + */ +struct xz_dec; + +/** + * xz_dec_init() - Allocate and initialize a XZ decoder state + * @mode: Operation mode + * @dict_max: Maximum size of the LZMA2 dictionary (history buffer) for + * multi-call decoding. This is ignored in single-call mode + * (mode == XZ_SINGLE). LZMA2 dictionary is always 2^n bytes + * or 2^n + 2^(n-1) bytes (the latter sizes are less common + * in practice), so other values for dict_max don't make sense. + * In the kernel, dictionary sizes of 64 KiB, 128 KiB, 256 KiB, + * 512 KiB, and 1 MiB are probably the only reasonable values, + * except for kernel and initramfs images where a bigger + * dictionary can be fine and useful. + * + * Single-call mode (XZ_SINGLE): xz_dec_run() decodes the whole stream at + * once. The caller must provide enough output space or the decoding will + * fail. The output space is used as the dictionary buffer, which is why + * there is no need to allocate the dictionary as part of the decoder's + * internal state. + * + * Because the output buffer is used as the workspace, streams encoded using + * a big dictionary are not a problem in single-call mode. It is enough that + * the output buffer is big enough to hold the actual uncompressed data; it + * can be smaller than the dictionary size stored in the stream headers. + * + * Multi-call mode with preallocated dictionary (XZ_PREALLOC): dict_max bytes + * of memory is preallocated for the LZMA2 dictionary. This way there is no + * risk that xz_dec_run() could run out of memory, since xz_dec_run() will + * never allocate any memory. Instead, if the preallocated dictionary is too + * small for decoding the given input stream, xz_dec_run() will return + * XZ_MEMLIMIT_ERROR. Thus, it is important to know what kind of data will be + * decoded to avoid allocating excessive amount of memory for the dictionary. + * + * Multi-call mode with dynamically allocated dictionary (XZ_DYNALLOC): + * dict_max specifies the maximum allowed dictionary size that xz_dec_run() + * may allocate once it has parsed the dictionary size from the stream + * headers. This way excessive allocations can be avoided while still + * limiting the maximum memory usage to a sane value to prevent running the + * system out of memory when decompressing streams from untrusted sources. + * + * On success, xz_dec_init() returns a pointer to struct xz_dec, which is + * ready to be used with xz_dec_run(). If memory allocation fails, + * xz_dec_init() returns NULL. + */ +XZ_EXTERN struct xz_dec *xz_dec_init(enum xz_mode mode, uint32_t dict_max); + +/** + * xz_dec_run() - Run the XZ decoder + * @s: Decoder state allocated using xz_dec_init() + * @b: Input and output buffers + * + * The possible return values depend on build options and operation mode. + * See enum xz_ret for details. + * + * Note that if an error occurs in single-call mode (return value is not + * XZ_STREAM_END), b->in_pos and b->out_pos are not modified and the + * contents of the output buffer from b->out[b->out_pos] onward are + * undefined. This is true even after XZ_BUF_ERROR, because with some filter + * chains, there may be a second pass over the output buffer, and this pass + * cannot be properly done if the output buffer is truncated. Thus, you + * cannot give the single-call decoder a too small buffer and then expect to + * get that amount valid data from the beginning of the stream. You must use + * the multi-call decoder if you don't want to uncompress the whole stream. + */ +XZ_EXTERN enum xz_ret xz_dec_run(struct xz_dec *s, struct xz_buf *b); + +/** + * xz_dec_reset() - Reset an already allocated decoder state + * @s: Decoder state allocated using xz_dec_init() + * + * This function can be used to reset the multi-call decoder state without + * freeing and reallocating memory with xz_dec_end() and xz_dec_init(). + * + * In single-call mode, xz_dec_reset() is always called in the beginning of + * xz_dec_run(). Thus, explicit call to xz_dec_reset() is useful only in + * multi-call mode. + */ +XZ_EXTERN void xz_dec_reset(struct xz_dec *s); + +/** + * xz_dec_end() - Free the memory allocated for the decoder state + * @s: Decoder state allocated using xz_dec_init(). If s is NULL, + * this function does nothing. + */ +XZ_EXTERN void xz_dec_end(struct xz_dec *s); + +/* + * Standalone build (userspace build or in-kernel build for boot time use) + * needs a CRC32 implementation. For normal in-kernel use, kernel's own + * CRC32 module is used instead, and users of this module don't need to + * care about the functions below. + */ +#ifndef XZ_INTERNAL_CRC32 +# ifdef __KERNEL__ +# define XZ_INTERNAL_CRC32 0 +# else +# define XZ_INTERNAL_CRC32 1 +# endif +#endif + +#if XZ_INTERNAL_CRC32 +/* + * This must be called before any other xz_* function to initialize + * the CRC32 lookup table. + */ +XZ_EXTERN void xz_crc32_init(void); + +/* + * Update CRC32 value using the polynomial from IEEE-802.3. To start a new + * calculation, the third argument must be zero. To continue the calculation, + * the previously returned value is passed as the third argument. + */ +XZ_EXTERN uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc); +#endif + +#ifdef __cplusplus +} +#endif + +#endif Added: vendor/xz-embedded/dist/linux/lib/decompress_unxz.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/xz-embedded/dist/linux/lib/decompress_unxz.c Sat Dec 31 22:35:46 2011 (r229159) @@ -0,0 +1,397 @@ +/* + * Wrapper for decompressing XZ-compressed kernel, initramfs, and initrd + * + * Author: Lasse Collin + * + * This file has been put into the public domain. + * You can do whatever you want with this file. + */ + +/* + * Important notes about in-place decompression + * + * At least on x86, the kernel is decompressed in place: the compressed data + * is placed to the end of the output buffer, and the decompressor overwrites + * most of the compressed data. There must be enough safety margin to + * guarantee that the write position is always behind the read position. + * + * The safety margin for XZ with LZMA2 or BCJ+LZMA2 is calculated below. + * Note that the margin with XZ is bigger than with Deflate (gzip)! + * + * The worst case for in-place decompression is that the beginning of + * the file is compressed extremely well, and the rest of the file is + * uncompressible. Thus, we must look for worst-case expansion when the + * compressor is encoding uncompressible data. + * + * The structure of the .xz file in case of a compresed kernel is as follows. + * Sizes (as bytes) of the fields are in parenthesis. + * + * Stream Header (12) + * Block Header: + * Block Header (8-12) + * Compressed Data (N) + * Block Padding (0-3) + * CRC32 (4) + * Index (8-20) + * Stream Footer (12) + * + * Normally there is exactly one Block, but let's assume that there are + * 2-4 Blocks just in case. Because Stream Header and also Block Header + * of the first Block don't make the decompressor produce any uncompressed + * data, we can ignore them from our calculations. Block Headers of possible + * additional Blocks have to be taken into account still. With these + * assumptions, it is safe to assume that the total header overhead is + * less than 128 bytes. + * + * Compressed Data contains LZMA2 or BCJ+LZMA2 encoded data. Since BCJ + * doesn't change the size of the data, it is enough to calculate the + * safety margin for LZMA2. + * + * LZMA2 stores the data in chunks. Each chunk has a header whose size is + * a maximum of 6 bytes, but to get round 2^n numbers, let's assume that + * the maximum chunk header size is 8 bytes. After the chunk header, there + * may be up to 64 KiB of actual payload in the chunk. Often the payload is + * quite a bit smaller though; to be safe, let's assume that an average + * chunk has only 32 KiB of payload. + * + * The maximum uncompressed size of the payload is 2 MiB. The minimum + * uncompressed size of the payload is in practice never less than the + * payload size itself. The LZMA2 format would allow uncompressed size + * to be less than the payload size, but no sane compressor creates such + * files. LZMA2 supports storing uncompressible data in uncompressed form, + * so there's never a need to create payloads whose uncompressed size is + * smaller than the compressed size. + * + * The assumption, that the uncompressed size of the payload is never + * smaller than the payload itself, is valid only when talking about + * the payload as a whole. It is possible that the payload has parts where + * the decompressor consumes more input than it produces output. Calculating + * the worst case for this would be tricky. Instead of trying to do that, + * let's simply make sure that the decompressor never overwrites any bytes + * of the payload which it is currently reading. + * + * Now we have enough information to calculate the safety margin. We need + * - 128 bytes for the .xz file format headers; + * - 8 bytes per every 32 KiB of uncompressed size (one LZMA2 chunk header + * per chunk, each chunk having average payload size of 32 KiB); and + * - 64 KiB (biggest possible LZMA2 chunk payload size) to make sure that + * the decompressor never overwrites anything from the LZMA2 chunk + * payload it is currently reading. + * + * We get the following formula: + * + * safety_margin = 128 + uncompressed_size * 8 / 32768 + 65536 + * = 128 + (uncompressed_size >> 12) + 65536 + * + * For comparision, according to arch/x86/boot/compressed/misc.c, the + * equivalent formula for Deflate is this: + * + * safety_margin = 18 + (uncompressed_size >> 12) + 32768 + * + * Thus, when updating Deflate-only in-place kernel decompressor to + * support XZ, the fixed overhead has to be increased from 18+32768 bytes + * to 128+65536 bytes. + */ + +/* + * STATIC is defined to "static" if we are being built for kernel + * decompression (pre-boot code). will define + * STATIC to empty if it wasn't already defined. Since we will need to + * know later if we are being used for kernel decompression, we define + * XZ_PREBOOT here. + */ +#ifdef STATIC +# define XZ_PREBOOT +#endif +#ifdef __KERNEL__ +# include +#endif +#define XZ_EXTERN STATIC + +#ifndef XZ_PREBOOT +# include +# include +#else +/* + * Use the internal CRC32 code instead of kernel's CRC32 module, which + * is not available in early phase of booting. + */ +#define XZ_INTERNAL_CRC32 1 + +/* + * For boot time use, we enable only the BCJ filter of the current + * architecture or none if no BCJ filter is available for the architecture. + */ +#ifdef CONFIG_X86 +# define XZ_DEC_X86 +#endif +#ifdef CONFIG_PPC +# define XZ_DEC_POWERPC +#endif +#ifdef CONFIG_ARM +# define XZ_DEC_ARM +#endif +#ifdef CONFIG_IA64 +# define XZ_DEC_IA64 +#endif +#ifdef CONFIG_SPARC +# define XZ_DEC_SPARC +#endif + +/* + * This will get the basic headers so that memeq() and others + * can be defined. + */ +#include "xz/xz_private.h" + +/* + * Replace the normal allocation functions with the versions from + * . vfree() needs to support vfree(NULL) + * when XZ_DYNALLOC is used, but the pre-boot free() doesn't support it. + * Workaround it here because the other decompressors don't need it. + */ +#undef kmalloc +#undef kfree +#undef vmalloc +#undef vfree +#define kmalloc(size, flags) malloc(size) +#define kfree(ptr) free(ptr) +#define vmalloc(size) malloc(size) +#define vfree(ptr) do { if (ptr != NULL) free(ptr); } while (0) + +/* + * FIXME: Not all basic memory functions are provided in architecture-specific + * files (yet). We define our own versions here for now, but this should be + * only a temporary solution. + * + * memeq and memzero are not used much and any remotely sane implementation + * is fast enough. memcpy/memmove speed matters in multi-call mode, but + * the kernel image is decompressed in single-call mode, in which only + * memcpy speed can matter and only if there is a lot of uncompressible data + * (LZMA2 stores uncompressible chunks in uncompressed form). Thus, the + * functions below should just be kept small; it's probably not worth + * optimizing for speed. + */ + +#ifndef memeq +static bool memeq(const void *a, const void *b, size_t size) +{ + const uint8_t *x = a; + const uint8_t *y = b; + size_t i; + + for (i = 0; i < size; ++i) + if (x[i] != y[i]) + return false; + + return true; +} +#endif + +#ifndef memzero +static void memzero(void *buf, size_t size) +{ + uint8_t *b = buf; + uint8_t *e = b + size; + + while (b != e) + *b++ = '\0'; +} +#endif + +#ifndef memmove +/* Not static to avoid a conflict with the prototype in the Linux headers. */ +void *memmove(void *dest, const void *src, size_t size) +{ + uint8_t *d = dest; + const uint8_t *s = src; + size_t i; + + if (d < s) { + for (i = 0; i < size; ++i) + d[i] = s[i]; + } else if (d > s) { + i = size; + while (i-- > 0) + d[i] = s[i]; + } + + return dest; +} +#endif + +/* + * Since we need memmove anyway, would use it as memcpy too. + * Commented out for now to avoid breaking things. + */ +/* +#ifndef memcpy +# define memcpy memmove +#endif +*/ + +#include "xz/xz_crc32.c" +#include "xz/xz_dec_stream.c" +#include "xz/xz_dec_lzma2.c" +#include "xz/xz_dec_bcj.c" + +#endif /* XZ_PREBOOT */ + +/* Size of the input and output buffers in multi-call mode */ +#define XZ_IOBUF_SIZE 4096 + +/* + * This function implements the API defined in . + * + * This wrapper will automatically choose single-call or multi-call mode + * of the native XZ decoder API. The single-call mode can be used only when + * both input and output buffers are available as a single chunk, i.e. when + * fill() and flush() won't be used. + */ +STATIC int INIT unxz(unsigned char *in, int in_size, + int (*fill)(void *dest, unsigned int size), + int (*flush)(void *src, unsigned int size), + unsigned char *out, int *in_used, + void (*error)(char *x)) +{ + struct xz_buf b; + struct xz_dec *s; + enum xz_ret ret; + bool must_free_in = false; + +#if XZ_INTERNAL_CRC32 + xz_crc32_init(); +#endif + + if (in_used != NULL) + *in_used = 0; + + if (fill == NULL && flush == NULL) + s = xz_dec_init(XZ_SINGLE, 0); + else + s = xz_dec_init(XZ_DYNALLOC, (uint32_t)-1); + + if (s == NULL) + goto error_alloc_state; + + if (flush == NULL) { + b.out = out; + b.out_size = (size_t)-1; + } else { + b.out_size = XZ_IOBUF_SIZE; + b.out = malloc(XZ_IOBUF_SIZE); + if (b.out == NULL) + goto error_alloc_out; + } + + if (in == NULL) { + must_free_in = true; + in = malloc(XZ_IOBUF_SIZE); + if (in == NULL) + goto error_alloc_in; + } + + b.in = in; + b.in_pos = 0; + b.in_size = in_size; + b.out_pos = 0; + + if (fill == NULL && flush == NULL) { + ret = xz_dec_run(s, &b); + } else { + do { + if (b.in_pos == b.in_size && fill != NULL) { + if (in_used != NULL) + *in_used += b.in_pos; + + b.in_pos = 0; + + in_size = fill(in, XZ_IOBUF_SIZE); + if (in_size < 0) { + /* + * This isn't an optimal error code + * but it probably isn't worth making + * a new one either. + */ + ret = XZ_BUF_ERROR; + break; + } + + b.in_size = in_size; + } + + ret = xz_dec_run(s, &b); + + if (flush != NULL && (b.out_pos == b.out_size + || (ret != XZ_OK && b.out_pos > 0))) { + /* + * Setting ret here may hide an error + * returned by xz_dec_run(), but probably + * it's not too bad. + */ + if (flush(b.out, b.out_pos) != (int)b.out_pos) + ret = XZ_BUF_ERROR; + + b.out_pos = 0; + } + } while (ret == XZ_OK); + + if (must_free_in) + free(in); + + if (flush != NULL) + free(b.out); + } + + if (in_used != NULL) + *in_used += b.in_pos; + + xz_dec_end(s); + + switch (ret) { + case XZ_STREAM_END: + return 0; + + case XZ_MEM_ERROR: + /* This can occur only in multi-call mode. */ + error("XZ decompressor ran out of memory"); + break; + + case XZ_FORMAT_ERROR: + error("Input is not in the XZ format (wrong magic bytes)"); + break; + + case XZ_OPTIONS_ERROR: + error("Input was encoded with settings that are not " + "supported by this XZ decoder"); + break; + + case XZ_DATA_ERROR: + case XZ_BUF_ERROR: + error("XZ-compressed data is corrupt"); + break; + + default: + error("Bug in the XZ decompressor"); + break; + } + + return -1; + +error_alloc_in: + if (flush != NULL) + free(b.out); + +error_alloc_out: + xz_dec_end(s); + +error_alloc_state: + error("XZ decompressor ran out of memory"); + return -1; +} + +/* + * This macro is used by architecture-specific files to decompress + * the kernel image. + */ +#define decompress unxz Added: vendor/xz-embedded/dist/linux/lib/xz/Kconfig ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ vendor/xz-embedded/dist/linux/lib/xz/Kconfig Sat Dec 31 22:35:46 2011 (r229159) @@ -0,0 +1,59 @@ +config XZ_DEC + tristate "XZ decompression support" + select CRC32 + help + LZMA2 compression algorithm and BCJ filters are supported using + the .xz file format as the container. For integrity checking, + CRC32 is supported. See Documentation/xz.txt for more information. + +config XZ_DEC_X86 + bool "x86 BCJ filter decoder" if EXPERT + default y + depends on XZ_DEC + select XZ_DEC_BCJ + +config XZ_DEC_POWERPC + bool "PowerPC BCJ filter decoder" if EXPERT + default y + depends on XZ_DEC + select XZ_DEC_BCJ + +config XZ_DEC_IA64 + bool "IA-64 BCJ filter decoder" if EXPERT + default y + depends on XZ_DEC + select XZ_DEC_BCJ + +config XZ_DEC_ARM + bool "ARM BCJ filter decoder" if EXPERT + default y + depends on XZ_DEC + select XZ_DEC_BCJ *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***