Date: Wed, 4 Jan 2012 23:45:11 +0000 (UTC) From: Aleksandr Rybalko <ray@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r229538 - head/usr.bin/mkulzma Message-ID: <201201042345.q04NjBBD004255@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: ray Date: Wed Jan 4 23:45:10 2012 New Revision: 229538 URL: http://svn.freebsd.org/changeset/base/229538 Log: mkulzma used to create lzma compressed images, just like mkuzip do. Approved by: adrian (mentor) Added: head/usr.bin/mkulzma/ head/usr.bin/mkulzma/Makefile (contents, props changed) head/usr.bin/mkulzma/mkulzma.8 (contents, props changed) head/usr.bin/mkulzma/mkulzma.c (contents, props changed) Added: head/usr.bin/mkulzma/Makefile ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/usr.bin/mkulzma/Makefile Wed Jan 4 23:45:10 2012 (r229538) @@ -0,0 +1,8 @@ +# $FreeBSD$ + +PROG= mkulzma +MAN= mkulzma.8 +DPADD= ${LIBLZMA} +LDADD= -llzma + +.include <bsd.prog.mk> Added: head/usr.bin/mkulzma/mkulzma.8 ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/usr.bin/mkulzma/mkulzma.8 Wed Jan 4 23:45:10 2012 (r229538) @@ -0,0 +1,107 @@ +.\" ---------------------------------------------------------------------------- +.\" Derived from mkuzip.8 by Aleksandr Rybalko <ray@ddteam.net> +.\" ---------------------------------------------------------------------------- +.\" "THE BEER-WARE LICENSE" (Revision 42): +.\" <sobomax@FreeBSD.ORG> wrote this file. As long as you retain this notice you +.\" can do whatever you want with this stuff. If we meet some day, and you think +.\" this stuff is worth it, you can buy me a beer in return. Maxim Sobolev +.\" ---------------------------------------------------------------------------- +.\" +.\" $FreeBSD$ +.\" +.Dd March 17, 2006 +.Dt mkulzma 8 +.Os +.Sh NAME +.Nm mkulzma +.Nd compress disk image for use with +.Xr geom_uncompress 4 +class +.Sh SYNOPSIS +.Nm +.Op Fl v +.Op Fl o Ar outfile +.Op Fl s Ar cluster_size +.Ar infile +.Sh DESCRIPTION +The +.Nm +utility compresses a disk image file so that the +.Xr geom_uncompress 4 +class will be able to decompress the resulting image at run-time. +This allows for a significant reduction of size of disk image at +the expense of some CPU time required to decompress the data each +time it is read. +The +.Nm +utility +works in two phases: +.Bl -enum +.It +An +.Ar infile +image is split into clusters; each cluster is compressed using liblzma. +.It +The resulting set of compressed clusters along with headers that allow +locating each individual cluster is written to the output file. +.El +.Pp +The options are: +.Bl -tag -width indent +.It Fl o Ar outfile +Name of the output file +.Ar outfile . +The default is to use the input name with the suffix +.Pa .ulzma . +.It Fl s Ar cluster_size +Split the image into clusters of +.Ar cluster_size +bytes, 16384 bytes by default. +The +.Ar cluster_size +should be a multiple of 512 bytes. +.It Fl v +Display verbose messages. +.El +.Sh NOTES +The compression ratio largely depends on the cluster size used. +.\" The following two sentences are unclear: how can xz(1) be +.\" used in a comparable fashion, and wouldn't a lzma-compressed +.\" image suffer from larger cluster sizes as well? +For large cluster sizes (16K and higher), typical compression ratios +are only 1-2% less than those achieved with +.Xr lzma 1 . +However, it should be kept in mind that larger cluster +sizes lead to higher overhead in the +.Xr geom_uncompress 4 +class, as the class has to decompress the whole cluster even if +only a few bytes from that cluster have to be read. +.Pp +The +.Nm +utility +inserts a short shell script at the beginning of the generated image, +which makes it possible to +.Dq run +the image just like any other shell script. +The script tries to load the +.Xr geom_uncompress 4 +class if it is not loaded, configure the image as an +.Xr md 4 +disk device using +.Xr mdconfig 8 , +and automatically mount it using +.Xr mount_cd9660 8 +on the mount point provided as the first argument to the script. +.Sh EXIT STATUS +.Ex -std +.Sh SEE ALSO +.Xr lzma 1 , +.Xr geom 4 , +.Xr geom_uncompress 4 , +.Xr md 4 , +.Xr mdconfig 8 , +.Xr mount_cd9660 8 +.Sh AUTHORS +.An Maxim Sobolev Aq sobomax@FreeBSD.org +.An Aleksandr Rybalko Aq ray@ddteam.net Added: head/usr.bin/mkulzma/mkulzma.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/usr.bin/mkulzma/mkulzma.c Wed Jan 4 23:45:10 2012 (r229538) @@ -0,0 +1,330 @@ +/* + * ---------------------------------------------------------------------------- + * Derived from mkuzip.c by Aleksandr Rybalko <ray@ddteam.net> + * ---------------------------------------------------------------------------- + * "THE BEER-WARE LICENSE" (Revision 42): + * <sobomax@FreeBSD.ORG> wrote this file. As long as you retain this notice you + * can do whatever you want with this stuff. If we meet some day, and you think + * this stuff is worth it, you can buy me a beer in return. Maxim Sobolev + * ---------------------------------------------------------------------------- + * + * $FreeBSD$ + * + */ + +#include <sys/disk.h> +#include <sys/endian.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/uio.h> +#include <netinet/in.h> +#include <err.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include <lzma.h> + +#define CLSTSIZE 16384 +#define DEFAULT_SUFX ".ulzma" + +#define USED_BLOCKSIZE DEV_BSIZE + +#define CLOOP_MAGIC_LEN 128 +/* Format L3.0, since we move to XZ API */ +static char CLOOP_MAGIC_START[] = + "#!/bin/sh\n" + "#L3.0\n" + "n=uncompress\n" + "m=geom_$n\n" + "(kldstat -m $m 2>&-||kldload $m)>&-&&" + "mount_cd9660 /dev/`mdconfig -af $0`.$n $1\n" + "exit $?\n"; + +static char *readblock(int, char *, u_int32_t); +static void usage(void); +static void *safe_malloc(size_t); +static void cleanup(void); + +static char *cleanfile = NULL; + +int main(int argc, char **argv) +{ + char *iname, *oname, *obuf, *ibuf; + int fdr, fdw, i, opt, verbose, tmp; + struct iovec iov[2]; + struct stat sb; + uint32_t destlen; + uint64_t offset; + uint64_t *toc; + lzma_filter filters[2]; + lzma_options_lzma opt_lzma; + lzma_ret ret; + lzma_stream strm = LZMA_STREAM_INIT; + struct cloop_header { + char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ + uint32_t blksz; /* block size */ + uint32_t nblocks; /* number of blocks */ + } hdr; + + memset(&hdr, 0, sizeof(hdr)); + hdr.blksz = CLSTSIZE; + strcpy(hdr.magic, CLOOP_MAGIC_START); + oname = NULL; + verbose = 0; + + while((opt = getopt(argc, argv, "o:s:v")) != -1) { + switch(opt) { + case 'o': + oname = optarg; + break; + + case 's': + tmp = atoi(optarg); + if (tmp <= 0) { + errx(1, + "invalid cluster size specified: %s", + optarg); + /* Not reached */ + } + if (tmp % USED_BLOCKSIZE != 0) { + errx(1, + "cluster size should be multiple of %d", + USED_BLOCKSIZE); + /* Not reached */ + } + if ( tmp > MAXPHYS) { + errx(1, "cluster size is too large"); + /* Not reached */ + } + hdr.blksz = tmp; + break; + + case 'v': + verbose = 1; + break; + + default: + usage(); + /* Not reached */ + } + } + argc -= optind; + argv += optind; + + if (argc != 1) { + usage(); + /* Not reached */ + } + + iname = argv[0]; + if (oname == NULL) { + asprintf(&oname, "%s%s", iname, DEFAULT_SUFX); + if (oname == NULL) { + err(1, "can't allocate memory"); + /* Not reached */ + } + } + + obuf = safe_malloc(hdr.blksz*2); + ibuf = safe_malloc(hdr.blksz); + + signal(SIGHUP, exit); + signal(SIGINT, exit); + signal(SIGTERM, exit); + signal(SIGXCPU, exit); + signal(SIGXFSZ, exit); + atexit(cleanup); + + fdr = open(iname, O_RDONLY); + if (fdr < 0) { + err(1, "open(%s)", iname); + /* Not reached */ + } + if (fstat(fdr, &sb) != 0) { + err(1, "fstat(%s)", iname); + /* Not reached */ + } + if (S_ISCHR(sb.st_mode)) { + off_t ms; + + if (ioctl(fdr, DIOCGMEDIASIZE, &ms) < 0) { + err(1, "ioctl(DIOCGMEDIASIZE)"); + /* Not reached */ + } + sb.st_size = ms; + } else if (!S_ISREG(sb.st_mode)) { + fprintf(stderr, + "%s: not a character device or regular file\n", + iname); + exit(1); + } + hdr.nblocks = sb.st_size / hdr.blksz; + if ((sb.st_size % hdr.blksz) != 0) { + if (verbose != 0) + fprintf(stderr, "file size is not multiple " + "of %d, padding data\n", hdr.blksz); + hdr.nblocks++; + } + toc = safe_malloc((hdr.nblocks + 1) * sizeof(*toc)); + + fdw = open(oname, O_WRONLY | O_TRUNC | O_CREAT, + S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); + if (fdw < 0) { + err(1, "open(%s)", oname); + /* Not reached */ + } + cleanfile = oname; + + /* + * Prepare header that we will write later when we have index ready. + */ + iov[0].iov_base = (char *)&hdr; + iov[0].iov_len = sizeof(hdr); + iov[1].iov_base = (char *)toc; + iov[1].iov_len = (hdr.nblocks + 1) * sizeof(*toc); + offset = iov[0].iov_len + iov[1].iov_len; + + /* Reserve space for header */ + lseek(fdw, offset, SEEK_SET); + + if (verbose != 0) + fprintf(stderr, "data size %ju bytes, number of clusters " + "%u, index length %zu bytes\n", sb.st_size, + hdr.nblocks, iov[1].iov_len); + + /* Init lzma encoder */ + if (lzma_lzma_preset(&opt_lzma, LZMA_PRESET_DEFAULT)) + errx(1, "Error loading LZMA preset"); + + filters[0].id = LZMA_FILTER_LZMA2; + filters[0].options = &opt_lzma; + filters[1].id = LZMA_VLI_UNKNOWN; + + for(i = 0; i == 0 || ibuf != NULL; i++) { + ibuf = readblock(fdr, ibuf, hdr.blksz); + if (ibuf != NULL) { + destlen = hdr.blksz*2; + + ret = lzma_stream_encoder(&strm, filters, + LZMA_CHECK_CRC32); + if (ret != LZMA_OK) { + if (ret == LZMA_MEMLIMIT_ERROR) + errx(1, "can't compress data: " + "LZMA_MEMLIMIT_ERROR"); + + errx(1, "can't compress data: " + "LZMA compressor ERROR"); + } + + strm.next_in = ibuf; + strm.avail_in = hdr.blksz; + strm.next_out = obuf; + strm.avail_out = hdr.blksz*2; + + ret = lzma_code(&strm, LZMA_FINISH); + + if (ret != LZMA_STREAM_END) { + /* Error */ + errx(1, "lzma_code FINISH failed, code=%d, " + "pos(in=%zd, out=%zd)", + ret, + (hdr.blksz - strm.avail_in), + (hdr.blksz*2 - strm.avail_out)); + } + + destlen -= strm.avail_out; + + lzma_end(&strm); + + if (verbose != 0) + fprintf(stderr, "cluster #%d, in %u bytes, " + "out %u bytes\n", i, hdr.blksz, destlen); + } else { + destlen = USED_BLOCKSIZE - (offset % USED_BLOCKSIZE); + memset(obuf, 0, destlen); + if (verbose != 0) + fprintf(stderr, "padding data with %u bytes" + " so that file size is multiple of %d\n", + destlen, + USED_BLOCKSIZE); + } + if (write(fdw, obuf, destlen) < 0) { + err(1, "write(%s)", oname); + /* Not reached */ + } + toc[i] = htobe64(offset); + offset += destlen; + } + close(fdr); + + if (verbose != 0) + fprintf(stderr, "compressed data to %ju bytes, saved %lld " + "bytes, %.2f%% decrease.\n", offset, + (long long)(sb.st_size - offset), + 100.0 * (long long)(sb.st_size - offset) / + (float)sb.st_size); + + /* Convert to big endian */ + hdr.blksz = htonl(hdr.blksz); + hdr.nblocks = htonl(hdr.nblocks); + /* Write headers into pre-allocated space */ + lseek(fdw, 0, SEEK_SET); + if (writev(fdw, iov, 2) < 0) { + err(1, "writev(%s)", oname); + /* Not reached */ + } + cleanfile = NULL; + close(fdw); + + exit(0); +} + +static char * +readblock(int fd, char *ibuf, u_int32_t clstsize) +{ + int numread; + + bzero(ibuf, clstsize); + numread = read(fd, ibuf, clstsize); + if (numread < 0) { + err(1, "read() failed"); + /* Not reached */ + } + if (numread == 0) { + return NULL; + } + return ibuf; +} + +static void +usage(void) +{ + + fprintf(stderr, "usage: mkulzma [-v] [-o outfile] [-s cluster_size] " + "infile\n"); + exit(1); +} + +static void * +safe_malloc(size_t size) +{ + void *retval; + + retval = malloc(size); + if (retval == NULL) { + err(1, "can't allocate memory"); + /* Not reached */ + } + return retval; +} + +static void +cleanup(void) +{ + + if (cleanfile != NULL) + unlink(cleanfile); +}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201201042345.q04NjBBD004255>