From owner-svn-src-all@FreeBSD.ORG Mon Jan 26 05:44:41 2009 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 3E6EB106566C; Mon, 26 Jan 2009 05:44:41 +0000 (UTC) (envelope-from kientzle@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 2BF9D8FC14; Mon, 26 Jan 2009 05:44:41 +0000 (UTC) (envelope-from kientzle@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n0Q5if2F037940; Mon, 26 Jan 2009 05:44:41 GMT (envelope-from kientzle@svn.freebsd.org) Received: (from kientzle@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n0Q5ifeI037937; Mon, 26 Jan 2009 05:44:41 GMT (envelope-from kientzle@svn.freebsd.org) Message-Id: <200901260544.n0Q5ifeI037937@svn.freebsd.org> From: Tim Kientzle Date: Mon, 26 Jan 2009 05:44:41 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r187704 - in head/lib/libarchive: . test X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 26 Jan 2009 05:44:41 -0000 Author: kientzle Date: Mon Jan 26 05:44:40 2009 New Revision: 187704 URL: http://svn.freebsd.org/changeset/base/187704 Log: Fix ARCHIVE_EXTRACT_SPARSE handling in libarchive. Add a test to exercise this feature. This should fix --sparse/-S support in tar. Thanks to: Daichi GOTO MFC after: 1 week Added: head/lib/libarchive/test/test_write_disk_sparse.c (contents, props changed) Modified: head/lib/libarchive/archive_write_disk.c head/lib/libarchive/test/Makefile Modified: head/lib/libarchive/archive_write_disk.c ============================================================================== --- head/lib/libarchive/archive_write_disk.c Mon Jan 26 02:15:22 2009 (r187703) +++ head/lib/libarchive/archive_write_disk.c Mon Jan 26 05:44:40 2009 (r187704) @@ -178,6 +178,8 @@ struct archive_write_disk { int fd; /* Current offset for writing data to the file. */ off_t offset; + /* Last offset actually written to disk. */ + off_t fd_offset; /* Maximum size of file, -1 if unknown. */ off_t filesize; /* Dir we were in before this restore; only for deep paths. */ @@ -187,8 +189,6 @@ struct archive_write_disk { /* UID/GID to use in restoring this entry. */ uid_t uid; gid_t gid; - /* Last offset written to disk. */ - off_t last_offset; }; /* @@ -235,7 +235,7 @@ static struct fixup_entry *sort_dir_list static gid_t trivial_lookup_gid(void *, const char *, gid_t); static uid_t trivial_lookup_uid(void *, const char *, uid_t); static ssize_t write_data_block(struct archive_write_disk *, - const char *, size_t, off_t); + const char *, size_t); static struct archive_vtable *archive_write_disk_vtable(void); @@ -337,7 +337,7 @@ _archive_write_header(struct archive *_a } a->entry = archive_entry_clone(entry); a->fd = -1; - a->last_offset = 0; + a->fd_offset = 0; a->offset = 0; a->uid = a->user_uid; a->mode = archive_entry_mode(a->entry); @@ -513,9 +513,9 @@ archive_write_disk_set_skip_file(struct } static ssize_t -write_data_block(struct archive_write_disk *a, - const char *buff, size_t size, off_t offset) +write_data_block(struct archive_write_disk *a, const char *buff, size_t size) { + uint64_t start_size = size; ssize_t bytes_written = 0; ssize_t block_size = 0, bytes_to_write; @@ -538,8 +538,9 @@ write_data_block(struct archive_write_di #endif } - if (a->filesize >= 0 && (off_t)(offset + size) > a->filesize) - size = (size_t)(a->filesize - offset); + /* If this write would run beyond the file size, truncate it. */ + if (a->filesize >= 0 && (off_t)(a->offset + size) > a->filesize) + start_size = size = (size_t)(a->filesize - a->offset); /* Write the data. */ while (size > 0) { @@ -555,7 +556,7 @@ write_data_block(struct archive_write_di if (*p != '\0') break; } - offset += p - buff; + a->offset += p - buff; size -= p - buff; buff = p; if (size == 0) @@ -563,22 +564,25 @@ write_data_block(struct archive_write_di /* Calculate next block boundary after offset. */ block_end - = (offset / block_size) * block_size + block_size; + = (a->offset / block_size + 1) * block_size; /* If the adjusted write would cross block boundary, * truncate it to the block boundary. */ bytes_to_write = size; - if (offset + bytes_to_write > block_end) - bytes_to_write = block_end - offset; + if (a->offset + bytes_to_write > block_end) + bytes_to_write = block_end - a->offset; } /* Seek if necessary to the specified offset. */ - if (offset != a->last_offset) { - if (lseek(a->fd, offset, SEEK_SET) < 0) { + if (a->offset != a->fd_offset) { + if (lseek(a->fd, a->offset, SEEK_SET) < 0) { archive_set_error(&a->archive, errno, "Seek failed"); return (ARCHIVE_FATAL); } + a->fd_offset = a->offset; + a->archive.file_position = a->offset; + a->archive.raw_position = a->offset; } bytes_written = write(a->fd, buff, bytes_to_write); if (bytes_written < 0) { @@ -587,12 +591,12 @@ write_data_block(struct archive_write_di } buff += bytes_written; size -= bytes_written; - offset += bytes_written; + a->offset += bytes_written; a->archive.file_position += bytes_written; a->archive.raw_position += bytes_written; - a->last_offset = a->offset = offset; + a->fd_offset = a->offset; } - return (bytes_written); + return (start_size - size); } static ssize_t @@ -605,9 +609,9 @@ _archive_write_data_block(struct archive __archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, ARCHIVE_STATE_DATA, "archive_write_disk_block"); - r = write_data_block(a, buff, size, offset); - - if (r < 0) + a->offset = offset; + r = write_data_block(a, buff, size); + if (r < ARCHIVE_OK) return (r); if ((size_t)r < size) { archive_set_error(&a->archive, 0, @@ -625,7 +629,7 @@ _archive_write_data(struct archive *_a, __archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC, ARCHIVE_STATE_DATA, "archive_write_data"); - return (write_data_block(a, buff, size, a->offset)); + return (write_data_block(a, buff, size)); } static int @@ -646,7 +650,7 @@ _archive_write_finish_entry(struct archi /* There's no file. */ } else if (a->filesize < 0) { /* File size is unknown, so we can't set the size. */ - } else if (a->last_offset == a->filesize) { + } else if (a->fd_offset == a->filesize) { /* Last write ended at exactly the filesize; we're done. */ /* Hopefully, this is the common case. */ } else { Modified: head/lib/libarchive/test/Makefile ============================================================================== --- head/lib/libarchive/test/Makefile Mon Jan 26 02:15:22 2009 (r187703) +++ head/lib/libarchive/test/Makefile Mon Jan 26 05:44:40 2009 (r187704) @@ -62,6 +62,7 @@ TESTS= \ test_write_disk_hardlink.c \ test_write_disk_perms.c \ test_write_disk_secure.c \ + test_write_disk_sparse.c \ test_write_disk_times.c \ test_write_format_ar.c \ test_write_format_cpio.c \ Added: head/lib/libarchive/test/test_write_disk_sparse.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/lib/libarchive/test/test_write_disk_sparse.c Mon Jan 26 05:44:40 2009 (r187704) @@ -0,0 +1,278 @@ +/*- + * Copyright (c) 2003-2007 Tim Kientzle + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test.h" +__FBSDID("$FreeBSD$"); + +/* + * Write a file using archive_write_data call, read the file + * back and verify the contents. The data written includes large + * blocks of nulls, so it should exercise the sparsification logic + * if ARCHIVE_EXTRACT_SPARSE is enabled. + */ +static void +verify_write_data(struct archive *a, int sparse) +{ + static const char data[]="abcdefghijklmnopqrstuvwxyz"; + struct stat st; + struct archive_entry *ae; + size_t buff_size = 64 * 1024; + char *buff, *p; + const char *msg = sparse ? "sparse" : "non-sparse"; + int fd; + + buff = malloc(buff_size); + assert(buff != NULL); + + ae = archive_entry_new(); + assert(ae != NULL); + archive_entry_set_size(ae, 8 * buff_size); + archive_entry_set_pathname(ae, "test_write_data"); + archive_entry_set_mode(ae, AE_IFREG | 0755); + assertEqualIntA(a, 0, archive_write_header(a, ae)); + + /* Use archive_write_data() to write three relatively sparse blocks. */ + + /* First has non-null data at beginning. */ + memset(buff, 0, buff_size); + memcpy(buff, data, sizeof(data)); + failure("%s", msg); + assertEqualInt(buff_size, archive_write_data(a, buff, buff_size)); + + /* Second has non-null data in the middle. */ + memset(buff, 0, buff_size); + memcpy(buff + buff_size / 2 - 3, data, sizeof(data)); + failure("%s", msg); + assertEqualInt(buff_size, archive_write_data(a, buff, buff_size)); + + /* Third has non-null data at the end. */ + memset(buff, 0, buff_size); + memcpy(buff + buff_size - sizeof(data), data, sizeof(data)); + failure("%s", msg); + assertEqualInt(buff_size, archive_write_data(a, buff, buff_size)); + + failure("%s", msg); + assertEqualIntA(a, 0, archive_write_finish_entry(a)); + + /* Test the entry on disk. */ + assert(0 == stat(archive_entry_pathname(ae), &st)); + assertEqualInt(st.st_size, 8 * buff_size); + fd = open(archive_entry_pathname(ae), O_RDONLY); + if (!assert(fd >= 0)) + return; + + /* Check first block. */ + assertEqualInt(buff_size, read(fd, buff, buff_size)); + failure("%s", msg); + assertEqualMem(buff, data, sizeof(data)); + for (p = buff + sizeof(data); p < buff + buff_size; ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (!assertEqualInt(0, *p)) + break; + } + + /* Check second block. */ + assertEqualInt(buff_size, read(fd, buff, buff_size)); + for (p = buff; p < buff + buff_size; ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (p == buff + buff_size / 2 - 3) { + assertEqualMem(p, data, sizeof(data)); + p += sizeof(data); + } else if (!assertEqualInt(0, *p)) + break; + } + + /* Check third block. */ + assertEqualInt(buff_size, read(fd, buff, buff_size)); + for (p = buff; p < buff + buff_size - sizeof(data); ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (!assertEqualInt(0, *p)) + break; + } + failure("%s", msg); + assertEqualMem(buff + buff_size - sizeof(data), data, sizeof(data)); + + /* XXX more XXX */ + + assertEqualInt(0, close(fd)); + free(buff); +} + +/* + * As above, but using the archive_write_data_block() call. + */ +static void +verify_write_data_block(struct archive *a, int sparse) +{ + static const char data[]="abcdefghijklmnopqrstuvwxyz"; + struct stat st; + struct archive_entry *ae; + size_t buff_size = 64 * 1024; + char *buff, *p; + const char *msg = sparse ? "sparse" : "non-sparse"; + int fd; + + buff = malloc(buff_size); + assert(buff != NULL); + + ae = archive_entry_new(); + assert(ae != NULL); + archive_entry_set_size(ae, 8 * buff_size); + archive_entry_set_pathname(ae, "test_write_data_block"); + archive_entry_set_mode(ae, AE_IFREG | 0755); + assertEqualIntA(a, 0, archive_write_header(a, ae)); + + /* Use archive_write_data_block() to write three + relatively sparse blocks. */ + + /* First has non-null data at beginning. */ + memset(buff, 0, buff_size); + memcpy(buff, data, sizeof(data)); + failure("%s", msg); + assertEqualInt(ARCHIVE_OK, + archive_write_data_block(a, buff, buff_size, 100)); + + /* Second has non-null data in the middle. */ + memset(buff, 0, buff_size); + memcpy(buff + buff_size / 2 - 3, data, sizeof(data)); + failure("%s", msg); + assertEqualInt(ARCHIVE_OK, + archive_write_data_block(a, buff, buff_size, buff_size + 200)); + + /* Third has non-null data at the end. */ + memset(buff, 0, buff_size); + memcpy(buff + buff_size - sizeof(data), data, sizeof(data)); + failure("%s", msg); + assertEqualInt(ARCHIVE_OK, + archive_write_data_block(a, buff, buff_size, buff_size * 2 + 300)); + + failure("%s", msg); + assertEqualIntA(a, 0, archive_write_finish_entry(a)); + + /* Test the entry on disk. */ + assert(0 == stat(archive_entry_pathname(ae), &st)); + assertEqualInt(st.st_size, 8 * buff_size); + fd = open(archive_entry_pathname(ae), O_RDONLY); + if (!assert(fd >= 0)) + return; + + /* Check 100-byte gap at beginning */ + assertEqualInt(100, read(fd, buff, 100)); + failure("%s", msg); + for (p = buff; p < buff + 100; ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (!assertEqualInt(0, *p)) + break; + } + + /* Check first block. */ + assertEqualInt(buff_size, read(fd, buff, buff_size)); + failure("%s", msg); + assertEqualMem(buff, data, sizeof(data)); + for (p = buff + sizeof(data); p < buff + buff_size; ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (!assertEqualInt(0, *p)) + break; + } + + /* Check 100-byte gap */ + assertEqualInt(100, read(fd, buff, 100)); + failure("%s", msg); + for (p = buff; p < buff + 100; ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (!assertEqualInt(0, *p)) + break; + } + + /* Check second block. */ + assertEqualInt(buff_size, read(fd, buff, buff_size)); + for (p = buff; p < buff + buff_size; ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (p == buff + buff_size / 2 - 3) { + assertEqualMem(p, data, sizeof(data)); + p += sizeof(data); + } else if (!assertEqualInt(0, *p)) + break; + } + + /* Check 100-byte gap */ + assertEqualInt(100, read(fd, buff, 100)); + failure("%s", msg); + for (p = buff; p < buff + 100; ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (!assertEqualInt(0, *p)) + break; + } + + /* Check third block. */ + assertEqualInt(buff_size, read(fd, buff, buff_size)); + for (p = buff; p < buff + buff_size - sizeof(data); ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (!assertEqualInt(0, *p)) + break; + } + failure("%s", msg); + assertEqualMem(buff + buff_size - sizeof(data), data, sizeof(data)); + + /* Check another block size beyond last we wrote. */ + assertEqualInt(buff_size, read(fd, buff, buff_size)); + failure("%s", msg); + for (p = buff; p < buff + buff_size; ++p) { + failure("offset: %d, %s", (int)(p - buff), msg); + if (!assertEqualInt(0, *p)) + break; + } + + + /* XXX more XXX */ + + assertEqualInt(0, close(fd)); + free(buff); +} + +DEFINE_TEST(test_write_disk_sparse) +{ + struct archive *ad; + + + /* + * The return values, etc, of the write data functions + * shouldn't change regardless of whether we've requested + * sparsification. (The performance and pattern of actual + * write calls to the disk should vary, of course, but the + * client program shouldn't see any difference.) + */ + assert((ad = archive_write_disk_new()) != NULL); + archive_write_disk_set_options(ad, 0); + verify_write_data(ad, 0); + verify_write_data_block(ad, 0); + assertEqualInt(0, archive_write_finish(ad)); + + assert((ad = archive_write_disk_new()) != NULL); + archive_write_disk_set_options(ad, ARCHIVE_EXTRACT_SPARSE); + verify_write_data(ad, 1); + verify_write_data_block(ad, 1); + assertEqualInt(0, archive_write_finish(ad)); + +}