From owner-svn-src-vendor@freebsd.org Thu Oct 13 11:34:25 2016 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A70CFC0EB38; Thu, 13 Oct 2016 11:34:25 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 7315F178; Thu, 13 Oct 2016 11:34:25 +0000 (UTC) (envelope-from mm@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id u9DBYOQa010546; Thu, 13 Oct 2016 11:34:24 GMT (envelope-from mm@FreeBSD.org) Received: (from mm@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id u9DBYNgs010536; Thu, 13 Oct 2016 11:34:23 GMT (envelope-from mm@FreeBSD.org) Message-Id: <201610131134.u9DBYNgs010536@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mm set sender to mm@FreeBSD.org using -f From: Martin Matuska Date: Thu, 13 Oct 2016 11:34:23 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r307214 - in vendor/libarchive/dist: cpio/test libarchive libarchive/test tar/test X-SVN-Group: vendor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 13 Oct 2016 11:34:25 -0000 Author: mm Date: Thu Oct 13 11:34:23 2016 New Revision: 307214 URL: https://svnweb.freebsd.org/changeset/base/307214 Log: Update vendor/libarchive to git e3bdbbf3475c3abf264e563c753a20972095665e Important vendor bugfixes (relevant to FreeBSD): #801: FreeBSD Coverity report: resource leak in libarchive/tar/test/main.c Modified: vendor/libarchive/dist/cpio/test/main.c vendor/libarchive/dist/cpio/test/test.h vendor/libarchive/dist/libarchive/archive_read_disk_entry_from_file.c vendor/libarchive/dist/libarchive/archive_read_disk_set_standard_lookup.c vendor/libarchive/dist/libarchive/test/main.c vendor/libarchive/dist/libarchive/test/test.h vendor/libarchive/dist/libarchive/test/test_read_set_format.c vendor/libarchive/dist/tar/test/main.c vendor/libarchive/dist/tar/test/test.h vendor/libarchive/dist/tar/test/test_missing_file.c Modified: vendor/libarchive/dist/cpio/test/main.c ============================================================================== --- vendor/libarchive/dist/cpio/test/main.c Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/cpio/test/main.c Thu Oct 13 11:34:23 2016 (r307214) @@ -130,6 +130,13 @@ __FBSDID("$FreeBSD: src/usr.bin/cpio/tes # include #endif +mode_t umasked(mode_t expected_mode) +{ + mode_t mode = umask(0); + umask(mode); + return expected_mode & ~mode; +} + /* Path to working directory for current test */ const char *testworkdir; #ifdef PROGRAM @@ -1294,6 +1301,11 @@ assertion_file_time(const char *file, in switch (type) { case 'a': filet_nsec = st.st_atimespec.tv_nsec; break; case 'b': filet = st.st_birthtime; + /* FreeBSD filesystems that don't support birthtime + * (e.g., UFS1) always return -1 here. */ + if (filet == -1) { + return (1); + } filet_nsec = st.st_birthtimespec.tv_nsec; break; case 'm': filet_nsec = st.st_mtimespec.tv_nsec; break; default: fprintf(stderr, "INTERNAL: Bad type %c for file time", type); @@ -1425,7 +1437,7 @@ assertion_file_nlinks(const char *file, assertion_count(file, line); r = lstat(pathname, &st); if (r == 0 && (int)st.st_nlink == nlinks) - return (1); + return (1); failure_start(file, line, "File %s has %d links, expected %d", pathname, st.st_nlink, nlinks); failure_finish(NULL); @@ -1661,6 +1673,7 @@ assertion_make_file(const char *file, in if (0 != chmod(path, mode)) { failure_start(file, line, "Could not chmod %s", path); failure_finish(NULL); + close(fd); return (0); } if (contents != NULL) { @@ -1675,6 +1688,7 @@ assertion_make_file(const char *file, in failure_start(file, line, "Could not write to %s", path); failure_finish(NULL); + close(fd); return (0); } } Modified: vendor/libarchive/dist/cpio/test/test.h ============================================================================== --- vendor/libarchive/dist/cpio/test/test.h Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/cpio/test/test.h Thu Oct 13 11:34:23 2016 (r307214) @@ -182,6 +182,8 @@ assertion_file_nlinks(__FILE__, __LINE__, pathname, nlinks) #define assertFileSize(pathname, size) \ assertion_file_size(__FILE__, __LINE__, pathname, size) +#define assertFileMode(pathname, mode) \ + assertion_file_mode(__FILE__, __LINE__, pathname, mode) #define assertTextFileContents(text, pathname) \ assertion_text_file_contents(__FILE__, __LINE__, text, pathname) #define assertFileContainsLinesAnyOrder(pathname, lines) \ @@ -327,6 +329,9 @@ void copy_reference_file(const char *); */ void extract_reference_files(const char **); +/* Subtract umask from mode */ +mode_t umasked(mode_t expected_mode); + /* Path to working directory for current test */ extern const char *testworkdir; Modified: vendor/libarchive/dist/libarchive/archive_read_disk_entry_from_file.c ============================================================================== --- vendor/libarchive/dist/libarchive/archive_read_disk_entry_from_file.c Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/libarchive/archive_read_disk_entry_from_file.c Thu Oct 13 11:34:23 2016 (r307214) @@ -628,7 +628,6 @@ translate_acl(struct archive_read_disk * archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Unknown ACL brand"); return (ARCHIVE_WARN); - break; } #endif Modified: vendor/libarchive/dist/libarchive/archive_read_disk_set_standard_lookup.c ============================================================================== --- vendor/libarchive/dist/libarchive/archive_read_disk_set_standard_lookup.c Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/libarchive/archive_read_disk_set_standard_lookup.c Thu Oct 13 11:34:23 2016 (r307214) @@ -232,6 +232,7 @@ static const char * lookup_uname_helper(struct name_cache *cache, id_t id) { struct passwd *result; + (void)cache; /* UNUSED */ result = getpwuid((uid_t)id); @@ -298,6 +299,7 @@ static const char * lookup_gname_helper(struct name_cache *cache, id_t id) { struct group *result; + (void)cache; /* UNUSED */ result = getgrgid((gid_t)id); Modified: vendor/libarchive/dist/libarchive/test/main.c ============================================================================== --- vendor/libarchive/dist/libarchive/test/main.c Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/libarchive/test/main.c Thu Oct 13 11:34:23 2016 (r307214) @@ -128,6 +128,13 @@ __FBSDID("$FreeBSD: head/lib/libarchive/ # include #endif +mode_t umasked(mode_t expected_mode) +{ + mode_t mode = umask(0); + umask(mode); + return expected_mode & ~mode; +} + /* Path to working directory for current test */ const char *testworkdir; #ifdef PROGRAM @@ -1364,6 +1371,31 @@ assertion_file_birthtime_recent(const ch return assertion_file_time(file, line, pathname, 0, 0, 'b', 1); } +/* Verify mode of 'pathname'. */ +int +assertion_file_mode(const char *file, int line, const char *pathname, int expected_mode) +{ + int mode; + int r; + + assertion_count(file, line); +#if defined(_WIN32) && !defined(__CYGWIN__) + failure_start(file, line, "assertFileMode not yet implemented for Windows"); +#else + { + struct stat st; + r = lstat(pathname, &st); + mode = (int)(st.st_mode & 0777); + } + if (r == 0 && mode == expected_mode) + return (1); + failure_start(file, line, "File %s has mode %o, expected %o", + pathname, mode, expected_mode); +#endif + failure_finish(NULL); + return (0); +} + /* Verify mtime of 'pathname'. */ int assertion_file_mtime(const char *file, int line, @@ -1403,7 +1435,7 @@ assertion_file_nlinks(const char *file, assertion_count(file, line); r = lstat(pathname, &st); if (r == 0 && (int)st.st_nlink == nlinks) - return (1); + return (1); failure_start(file, line, "File %s has %d links, expected %d", pathname, st.st_nlink, nlinks); failure_finish(NULL); @@ -1440,31 +1472,6 @@ assertion_file_size(const char *file, in return (0); } -/* Verify mode of 'pathname'. */ -int -assertion_file_mode(const char *file, int line, const char *pathname, int expected_mode) -{ - int mode; - int r; - - assertion_count(file, line); -#if defined(_WIN32) && !defined(__CYGWIN__) - failure_start(file, line, "assertFileMode not yet implemented for Windows"); -#else - { - struct stat st; - r = lstat(pathname, &st); - mode = (int)(st.st_mode & 0777); - } - if (r == 0 && mode == expected_mode) - return (1); - failure_start(file, line, "File %s has mode %o, expected %o", - pathname, mode, expected_mode); -#endif - failure_finish(NULL); - return (0); -} - /* Assert that 'pathname' is a dir. If mode >= 0, verify that too. */ int assertion_is_dir(const char *file, int line, const char *pathname, int mode) @@ -1664,6 +1671,7 @@ assertion_make_file(const char *file, in if (0 != chmod(path, mode)) { failure_start(file, line, "Could not chmod %s", path); failure_finish(NULL); + close(fd); return (0); } if (contents != NULL) { @@ -1678,6 +1686,7 @@ assertion_make_file(const char *file, in failure_start(file, line, "Could not write to %s", path); failure_finish(NULL); + close(fd); return (0); } } Modified: vendor/libarchive/dist/libarchive/test/test.h ============================================================================== --- vendor/libarchive/dist/libarchive/test/test.h Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/libarchive/test/test.h Thu Oct 13 11:34:23 2016 (r307214) @@ -243,12 +243,12 @@ int assertion_file_birthtime_recent(cons int assertion_file_contains_lines_any_order(const char *, int, const char *, const char **); int assertion_file_contents(const char *, int, const void *, int, const char *); int assertion_file_exists(const char *, int, const char *); +int assertion_file_mode(const char *, int, const char *, int); int assertion_file_mtime(const char *, int, const char *, long, long); int assertion_file_mtime_recent(const char *, int, const char *); int assertion_file_nlinks(const char *, int, const char *, int); int assertion_file_not_exists(const char *, int, const char *); int assertion_file_size(const char *, int, const char *, long); -int assertion_file_mode(const char *, int, const char *, int); int assertion_is_dir(const char *, int, const char *, int); int assertion_is_hardlink(const char *, int, const char *, const char *); int assertion_is_not_hardlink(const char *, int, const char *, const char *); @@ -329,6 +329,9 @@ void copy_reference_file(const char *); */ void extract_reference_files(const char **); +/* Subtract umask from mode */ +mode_t umasked(mode_t expected_mode); + /* Path to working directory for current test */ extern const char *testworkdir; Modified: vendor/libarchive/dist/libarchive/test/test_read_set_format.c ============================================================================== --- vendor/libarchive/dist/libarchive/test/test_read_set_format.c Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/libarchive/test/test_read_set_format.c Thu Oct 13 11:34:23 2016 (r307214) @@ -200,6 +200,8 @@ DEFINE_TEST(test_read_append_filter_wron { struct archive_entry *ae; struct archive *a; + int fd; + fpos_t pos; /* * If we have "bunzip2 -q", try using that. @@ -208,6 +210,13 @@ DEFINE_TEST(test_read_append_filter_wron skipping("Can't run bunzip2 program on this platform"); return; } + + /* bunzip2 will write to stderr, redirect it to a file */ + fflush(stderr); + fgetpos(stderr, &pos); + fd = dup(fileno(stderr)); + freopen("stderr1", "w", stderr); + assert((a = archive_read_new()) != NULL); assertA(0 == archive_read_set_format(a, ARCHIVE_FORMAT_TAR)); assertEqualIntA(a, ARCHIVE_OK, @@ -217,4 +226,13 @@ DEFINE_TEST(test_read_append_filter_wron assertA(archive_read_next_header(a, &ae) < (ARCHIVE_WARN)); assertEqualIntA(a, ARCHIVE_WARN, archive_read_close(a)); assertEqualInt(ARCHIVE_OK, archive_read_free(a)); + + /* restore stderr */ + fflush(stderr); + dup2(fd, fileno(stderr)); + close(fd); + clearerr(stderr); + fsetpos(stderr, &pos); + + assertTextFileContents("bunzip2: (stdin) is not a bzip2 file.\n", "stderr1"); } Modified: vendor/libarchive/dist/tar/test/main.c ============================================================================== --- vendor/libarchive/dist/tar/test/main.c Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/tar/test/main.c Thu Oct 13 11:34:23 2016 (r307214) @@ -1164,6 +1164,35 @@ assertion_file_contains_lines_any_order( return (0); } +/* Verify that a text file does not contains the specified strings */ +int +assertion_file_contains_no_invalid_strings(const char *file, int line, + const char *pathname, const char *strings[]) +{ + char *buff; + int i; + + buff = slurpfile(NULL, "%s", pathname); + if (buff == NULL) { + failure_start(file, line, "Can't read file: %s", pathname); + failure_finish(NULL); + return (0); + } + + for (i = 0; strings[i] != NULL; ++i) { + if (strstr(buff, strings[i]) != NULL) { + failure_start(file, line, "Invalid string in %s: %s", pathname, + strings[i]); + failure_finish(NULL); + free(buff); + return(0); + } + } + + free(buff); + return (0); +} + /* Test that two paths point to the same file. */ /* As a side-effect, asserts that both files exist. */ static int @@ -1301,6 +1330,11 @@ assertion_file_time(const char *file, in switch (type) { case 'a': filet_nsec = st.st_atimespec.tv_nsec; break; case 'b': filet = st.st_birthtime; + /* FreeBSD filesystems that don't support birthtime + * (e.g., UFS1) always return -1 here. */ + if (filet == -1) { + return (1); + } filet_nsec = st.st_birthtimespec.tv_nsec; break; case 'm': filet_nsec = st.st_mtimespec.tv_nsec; break; default: fprintf(stderr, "INTERNAL: Bad type %c for file time", type); @@ -1432,7 +1466,7 @@ assertion_file_nlinks(const char *file, assertion_count(file, line); r = lstat(pathname, &st); if (r == 0 && (int)st.st_nlink == nlinks) - return (1); + return (1); failure_start(file, line, "File %s has %d links, expected %d", pathname, st.st_nlink, nlinks); failure_finish(NULL); @@ -1668,6 +1702,7 @@ assertion_make_file(const char *file, in if (0 != chmod(path, mode)) { failure_start(file, line, "Could not chmod %s", path); failure_finish(NULL); + close(fd); return (0); } if (contents != NULL) { @@ -1682,6 +1717,7 @@ assertion_make_file(const char *file, in failure_start(file, line, "Could not write to %s", path); failure_finish(NULL); + close(fd); return (0); } } Modified: vendor/libarchive/dist/tar/test/test.h ============================================================================== --- vendor/libarchive/dist/tar/test/test.h Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/tar/test/test.h Thu Oct 13 11:34:23 2016 (r307214) @@ -174,6 +174,9 @@ /* Assert that file contents match a string. */ #define assertFileContents(data, data_size, pathname) \ assertion_file_contents(__FILE__, __LINE__, data, data_size, pathname) +/* Verify that a file does not contain invalid strings */ +#define assertFileContainsNoInvalidStrings(pathname, strings) \ + assertion_file_contains_no_invalid_strings(__FILE__, __LINE__, pathname, strings) #define assertFileMtime(pathname, sec, nsec) \ assertion_file_mtime(__FILE__, __LINE__, pathname, sec, nsec) #define assertFileMtimeRecent(pathname) \ @@ -182,6 +185,8 @@ assertion_file_nlinks(__FILE__, __LINE__, pathname, nlinks) #define assertFileSize(pathname, size) \ assertion_file_size(__FILE__, __LINE__, pathname, size) +#define assertFileMode(pathname, mode) \ + assertion_file_mode(__FILE__, __LINE__, pathname, mode) #define assertTextFileContents(text, pathname) \ assertion_text_file_contents(__FILE__, __LINE__, text, pathname) #define assertFileContainsLinesAnyOrder(pathname, lines) \ @@ -239,6 +244,7 @@ int assertion_file_atime_recent(const ch int assertion_file_birthtime(const char *, int, const char *, long, long); int assertion_file_birthtime_recent(const char *, int, const char *); int assertion_file_contains_lines_any_order(const char *, int, const char *, const char **); +int assertion_file_contains_no_invalid_strings(const char *, int, const char *, const char **); int assertion_file_contents(const char *, int, const void *, int, const char *); int assertion_file_exists(const char *, int, const char *); int assertion_file_mode(const char *, int, const char *, int); Modified: vendor/libarchive/dist/tar/test/test_missing_file.c ============================================================================== --- vendor/libarchive/dist/tar/test/test_missing_file.c Thu Oct 13 09:37:21 2016 (r307213) +++ vendor/libarchive/dist/tar/test/test_missing_file.c Thu Oct 13 11:34:23 2016 (r307214) @@ -27,11 +27,15 @@ __FBSDID("$FreeBSD$"); DEFINE_TEST(test_missing_file) { + const char * invalid_stderr[] = { "INTERNAL ERROR", NULL }; assertMakeFile("file1", 0644, "file1"); assertMakeFile("file2", 0644, "file2"); assert(0 == systemf("%s -cf archive.tar file1 file2 2>stderr1", testprog)); assertEmptyFile("stderr1"); assert(0 != systemf("%s -cf archive.tar file1 file2 file3 2>stderr2", testprog)); + assertFileContainsNoInvalidStrings("stderr2", invalid_stderr); assert(0 != systemf("%s -cf archive.tar 2>stderr3", testprog)); - assert(0 != systemf("%s -cf archive.tar file3 2>stderr4", testprog)); + assertFileContainsNoInvalidStrings("stderr3", invalid_stderr); + assert(0 != systemf("%s -cf archive.tar file3 file4 2>stderr4", testprog)); + assertFileContainsNoInvalidStrings("stderr4", invalid_stderr); } From owner-svn-src-vendor@freebsd.org Fri Oct 14 11:41:08 2016 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 35E38C11679; Fri, 14 Oct 2016 11:41:08 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id EB1CFFB3; Fri, 14 Oct 2016 11:41:07 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id u9EBf7j9062522; Fri, 14 Oct 2016 11:41:07 GMT (envelope-from mav@FreeBSD.org) Received: (from mav@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id u9EBf6U3062517; Fri, 14 Oct 2016 11:41:06 GMT (envelope-from mav@FreeBSD.org) Message-Id: <201610141141.u9EBf6U3062517@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mav set sender to mav@FreeBSD.org using -f From: Alexander Motin Date: Fri, 14 Oct 2016 11:41:06 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r307313 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Oct 2016 11:41:08 -0000 Author: mav Date: Fri Oct 14 11:41:06 2016 New Revision: 307313 URL: https://svnweb.freebsd.org/changeset/base/307313 Log: 5120 zfs should allow large block/gzip/raidz boot pool (loader project) Reviewed by: George Wilson Reviewed by: Yuri Pankov Reviewed by: Andrew Stormont Reviewed by: Matthew Ahrens Approved by: Robert Mustacchi Author: Toomas Soome openzfs/openzfs@c8811bd3e2427dddbac6c05a59cfe117d8fea370 Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c Fri Oct 14 09:34:48 2016 (r307312) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/spa.c Fri Oct 14 11:41:06 2016 (r307313) @@ -26,6 +26,7 @@ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright 2013 Saso Kiselkov. All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2016 Toomas Soome */ /* @@ -519,12 +520,6 @@ spa_prop_validate(spa_t *spa, nvlist_t * &propval)) == 0 && !BOOTFS_COMPRESS_VALID(propval)) { error = SET_ERROR(ENOTSUP); - } else if ((error = - dsl_prop_get_int_ds(dmu_objset_ds(os), - zfs_prop_to_name(ZFS_PROP_RECORDSIZE), - &propval)) == 0 && - propval > SPA_OLD_MAXBLOCKSIZE) { - error = SET_ERROR(ENOTSUP); } else { objnum = dmu_objset_id(os); } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h Fri Oct 14 09:34:48 2016 (r307312) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zio.h Fri Oct 14 11:41:06 2016 (r307313) @@ -25,6 +25,7 @@ * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. + * Copyright 2016 Toomas Soome */ #ifndef _ZIO_H @@ -141,6 +142,15 @@ enum zio_compress { #define BOOTFS_COMPRESS_VALID(compress) \ ((compress) == ZIO_COMPRESS_LZJB || \ (compress) == ZIO_COMPRESS_LZ4 || \ + (compress) == ZIO_COMPRESS_GZIP_1 || \ + (compress) == ZIO_COMPRESS_GZIP_2 || \ + (compress) == ZIO_COMPRESS_GZIP_3 || \ + (compress) == ZIO_COMPRESS_GZIP_4 || \ + (compress) == ZIO_COMPRESS_GZIP_5 || \ + (compress) == ZIO_COMPRESS_GZIP_6 || \ + (compress) == ZIO_COMPRESS_GZIP_7 || \ + (compress) == ZIO_COMPRESS_GZIP_8 || \ + (compress) == ZIO_COMPRESS_GZIP_9 || \ (compress) == ZIO_COMPRESS_ON || \ (compress) == ZIO_COMPRESS_OFF) Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c Fri Oct 14 09:34:48 2016 (r307312) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/vdev.c Fri Oct 14 11:41:06 2016 (r307313) @@ -24,6 +24,7 @@ * Copyright (c) 2011, 2015 by Delphix. All rights reserved. * Copyright 2015 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2016 Toomas Soome */ #include @@ -3319,9 +3320,8 @@ vdev_set_state(vdev_t *vd, boolean_t iso /* * Check the vdev configuration to ensure that it's capable of supporting - * a root pool. Currently, we do not support RAID-Z or partial configuration. - * In addition, only a single top-level vdev is allowed and none of the leaves - * can be wholedisks. + * a root pool. We do not support partial configuration. + * In addition, only a single top-level vdev is allowed. */ boolean_t vdev_is_bootable(vdev_t *vd) @@ -3332,8 +3332,7 @@ vdev_is_bootable(vdev_t *vd) if (strcmp(vdev_type, VDEV_TYPE_ROOT) == 0 && vd->vdev_children > 1) { return (B_FALSE); - } else if (strcmp(vdev_type, VDEV_TYPE_RAIDZ) == 0 || - strcmp(vdev_type, VDEV_TYPE_MISSING) == 0) { + } else if (strcmp(vdev_type, VDEV_TYPE_MISSING) == 0) { return (B_FALSE); } } Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c Fri Oct 14 09:34:48 2016 (r307312) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_ioctl.c Fri Oct 14 11:41:06 2016 (r307313) @@ -30,6 +30,7 @@ * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013 Steven Hartland. All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2016 Toomas Soome */ /* @@ -3837,16 +3838,6 @@ zfs_check_settable(const char *dsname, n spa_t *spa; /* - * If this is a bootable dataset then - * the we don't allow large (>128K) blocks, - * because GRUB doesn't support them. - */ - if (zfs_is_bootfs(dsname) && - intval > SPA_OLD_MAXBLOCKSIZE) { - return (SET_ERROR(ERANGE)); - } - - /* * We don't allow setting the property above 1MB, * unless the tunable has been changed. */ From owner-svn-src-vendor@freebsd.org Fri Oct 14 11:46:18 2016 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id A7563C118AB; Fri, 14 Oct 2016 11:46:18 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 824F861C; Fri, 14 Oct 2016 11:46:18 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id u9EBkHic066054; Fri, 14 Oct 2016 11:46:17 GMT (envelope-from mav@FreeBSD.org) Received: (from mav@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id u9EBkHMF066053; Fri, 14 Oct 2016 11:46:17 GMT (envelope-from mav@FreeBSD.org) Message-Id: <201610141146.u9EBkHMF066053@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mav set sender to mav@FreeBSD.org using -f From: Alexander Motin Date: Fri, 14 Oct 2016 11:46:17 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r307314 - vendor-sys/illumos/dist/uts/common/fs/zfs X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Oct 2016 11:46:18 -0000 Author: mav Date: Fri Oct 14 11:46:17 2016 New Revision: 307314 URL: https://svnweb.freebsd.org/changeset/base/307314 Log: 6988 spa_sync() spends half its time in dmu_objset_do_userquota_updates Using a benchmark which creates 2 million files in one TXG, I observe that the thread running spa_sync() is on CPU almost the entire time we are syncing, and therefore can be a performance bottleneck. About 50% of the time in spa_sync() is in dmu_objset_do_userquota_updates(). The problem is that dmu_objset_do_userquota_updates() calls zap_increment_int(DMU_USERUSED_OBJECT) once for every file that was modified (or created). In this benchmark, all the files are owned by the same user/group, so all 2 million calls to zap_increment_int() are modifying the same entry in the zap. The same issue exists for the DMU_GROUPUSED_OBJECT. We should keep an in-memory map from user to space delta while we are syncing, and when we finish, iterate over the in-memory map and modify the ZAP once per entry. This reduces the number of calls to zap_increment_int() from "number of objects modified" to "number of owners/groups of modified files". This reduced the time spent in spa_sync() in the file create benchmark by ~33%, from 11 seconds to 7 seconds. Closes #107 Reviewed by: George Wilson Reviewed by: Steve Gonczi Reviewed by: Ned Bass Reviewed by: Jinshan Xiong Author: Matthew Ahrens openzfs/openzfs@5fc46359c569369d87728ca09f8705cdff6cc8e2 Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Oct 14 11:41:06 2016 (r307313) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu_objset.c Fri Oct 14 11:46:17 2016 (r307314) @@ -1207,18 +1207,83 @@ dmu_objset_userused_enabled(objset_t *os DMU_USERUSED_DNODE(os) != NULL); } +typedef struct userquota_node { + uint64_t uqn_id; + int64_t uqn_delta; + avl_node_t uqn_node; +} userquota_node_t; + +typedef struct userquota_cache { + avl_tree_t uqc_user_deltas; + avl_tree_t uqc_group_deltas; +} userquota_cache_t; + +static int +userquota_compare(const void *l, const void *r) +{ + const userquota_node_t *luqn = l; + const userquota_node_t *ruqn = r; + + if (luqn->uqn_id < ruqn->uqn_id) + return (-1); + if (luqn->uqn_id > ruqn->uqn_id) + return (1); + return (0); +} + static void -do_userquota_update(objset_t *os, uint64_t used, uint64_t flags, - uint64_t user, uint64_t group, boolean_t subtract, dmu_tx_t *tx) +do_userquota_cacheflush(objset_t *os, userquota_cache_t *cache, dmu_tx_t *tx) +{ + void *cookie; + userquota_node_t *uqn; + + ASSERT(dmu_tx_is_syncing(tx)); + + cookie = NULL; + while ((uqn = avl_destroy_nodes(&cache->uqc_user_deltas, + &cookie)) != NULL) { + VERIFY0(zap_increment_int(os, DMU_USERUSED_OBJECT, + uqn->uqn_id, uqn->uqn_delta, tx)); + kmem_free(uqn, sizeof (*uqn)); + } + avl_destroy(&cache->uqc_user_deltas); + + cookie = NULL; + while ((uqn = avl_destroy_nodes(&cache->uqc_group_deltas, + &cookie)) != NULL) { + VERIFY0(zap_increment_int(os, DMU_GROUPUSED_OBJECT, + uqn->uqn_id, uqn->uqn_delta, tx)); + kmem_free(uqn, sizeof (*uqn)); + } + avl_destroy(&cache->uqc_group_deltas); +} + +static void +userquota_update_cache(avl_tree_t *avl, uint64_t id, int64_t delta) +{ + userquota_node_t search = { .uqn_id = id }; + avl_index_t idx; + + userquota_node_t *uqn = avl_find(avl, &search, &idx); + if (uqn == NULL) { + uqn = kmem_zalloc(sizeof (*uqn), KM_SLEEP); + uqn->uqn_id = id; + avl_insert(avl, uqn, idx); + } + uqn->uqn_delta += delta; +} + +static void +do_userquota_update(userquota_cache_t *cache, uint64_t used, uint64_t flags, + uint64_t user, uint64_t group, boolean_t subtract) { if ((flags & DNODE_FLAG_USERUSED_ACCOUNTED)) { int64_t delta = DNODE_SIZE + used; if (subtract) delta = -delta; - VERIFY3U(0, ==, zap_increment_int(os, DMU_USERUSED_OBJECT, - user, delta, tx)); - VERIFY3U(0, ==, zap_increment_int(os, DMU_GROUPUSED_OBJECT, - group, delta, tx)); + + userquota_update_cache(&cache->uqc_user_deltas, user, delta); + userquota_update_cache(&cache->uqc_group_deltas, group, delta); } } @@ -1227,9 +1292,15 @@ dmu_objset_do_userquota_updates(objset_t { dnode_t *dn; list_t *list = &os->os_synced_dnodes; + userquota_cache_t cache = { 0 }; ASSERT(list_head(list) == NULL || dmu_objset_userused_enabled(os)); + avl_create(&cache.uqc_user_deltas, userquota_compare, + sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node)); + avl_create(&cache.uqc_group_deltas, userquota_compare, + sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node)); + while (dn = list_head(list)) { int flags; ASSERT(!DMU_OBJECT_IS_SPECIAL(dn->dn_object)); @@ -1239,32 +1310,26 @@ dmu_objset_do_userquota_updates(objset_t /* Allocate the user/groupused objects if necessary. */ if (DMU_USERUSED_DNODE(os)->dn_type == DMU_OT_NONE) { - VERIFY(0 == zap_create_claim(os, + VERIFY0(zap_create_claim(os, DMU_USERUSED_OBJECT, DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx)); - VERIFY(0 == zap_create_claim(os, + VERIFY0(zap_create_claim(os, DMU_GROUPUSED_OBJECT, DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx)); } - /* - * We intentionally modify the zap object even if the - * net delta is zero. Otherwise - * the block of the zap obj could be shared between - * datasets but need to be different between them after - * a bprewrite. - */ - flags = dn->dn_id_flags; ASSERT(flags); if (flags & DN_ID_OLD_EXIST) { - do_userquota_update(os, dn->dn_oldused, dn->dn_oldflags, - dn->dn_olduid, dn->dn_oldgid, B_TRUE, tx); + do_userquota_update(&cache, + dn->dn_oldused, dn->dn_oldflags, + dn->dn_olduid, dn->dn_oldgid, B_TRUE); } if (flags & DN_ID_NEW_EXIST) { - do_userquota_update(os, DN_USED_BYTES(dn->dn_phys), + do_userquota_update(&cache, + DN_USED_BYTES(dn->dn_phys), dn->dn_phys->dn_flags, dn->dn_newuid, - dn->dn_newgid, B_FALSE, tx); + dn->dn_newgid, B_FALSE); } mutex_enter(&dn->dn_mtx); @@ -1285,6 +1350,7 @@ dmu_objset_do_userquota_updates(objset_t list_remove(list, dn); dnode_rele(dn, list); } + do_userquota_cacheflush(os, &cache, tx); } /* From owner-svn-src-vendor@freebsd.org Fri Oct 14 11:49:38 2016 Return-Path: Delivered-To: svn-src-vendor@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 426DAC11957; Fri, 14 Oct 2016 11:49:38 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id EC79D81B; Fri, 14 Oct 2016 11:49:37 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id u9EBnbDk066216; Fri, 14 Oct 2016 11:49:37 GMT (envelope-from mav@FreeBSD.org) Received: (from mav@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id u9EBnaHr066212; Fri, 14 Oct 2016 11:49:36 GMT (envelope-from mav@FreeBSD.org) Message-Id: <201610141149.u9EBnaHr066212@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mav set sender to mav@FreeBSD.org using -f From: Alexander Motin Date: Fri, 14 Oct 2016 11:49:36 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r307315 - in vendor-sys/illumos/dist/uts/common/fs/zfs: . sys X-SVN-Group: vendor-sys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-vendor@freebsd.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: SVN commit messages for the vendor work area tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 14 Oct 2016 11:49:38 -0000 Author: mav Date: Fri Oct 14 11:49:36 2016 New Revision: 307315 URL: https://svnweb.freebsd.org/changeset/base/307315 Log: 7301 zpool export -f should be able to interrupt file freeing Reviewed by: Matthew Ahrens Reviewed by: Sanjay Nadkarni Reviewed by: Saso Kiselkov Reviewed by: John Kennedy Author: Alek Pinchuk Closes #175 Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_znode.h vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c Fri Oct 14 11:46:17 2016 (r307314) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/dmu.c Fri Oct 14 11:49:36 2016 (r307315) @@ -24,7 +24,7 @@ */ /* Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */ /* Copyright (c) 2013, Joyent, Inc. All rights reserved. */ -/* Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved. */ +/* Copyright 2016 Nexenta Systems, Inc. All rights reserved. */ #include #include @@ -695,6 +695,22 @@ get_next_chunk(dnode_t *dn, uint64_t *st return (0); } +/* + * If this objset is of type OST_ZFS return true if vfs's unmounted flag is set, + * otherwise return false. + * Used below in dmu_free_long_range_impl() to enable abort when unmounting + */ +/*ARGSUSED*/ +static boolean_t +dmu_objset_zfs_unmounting(objset_t *os) +{ +#ifdef _KERNEL + if (dmu_objset_type(os) == DMU_OST_ZFS) + return (zfs_get_vfs_flag_unmounted(os)); +#endif + return (B_FALSE); +} + static int dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset, uint64_t length) @@ -711,6 +727,9 @@ dmu_free_long_range_impl(objset_t *os, d while (length != 0) { uint64_t chunk_end, chunk_begin; + if (dmu_objset_zfs_unmounting(dn->dn_objset)) + return (SET_ERROR(EINTR)); + chunk_end = chunk_begin = offset + length; /* move chunk_begin backwards to the beginning of this chunk */ Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_znode.h ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_znode.h Fri Oct 14 11:46:17 2016 (r307314) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/sys/zfs_znode.h Fri Oct 14 11:49:36 2016 (r307315) @@ -22,6 +22,7 @@ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2015 by Delphix. All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2016 Nexenta Systems, Inc. All rights reserved. */ #ifndef _SYS_FS_ZFS_ZNODE_H @@ -302,6 +303,7 @@ extern int zfs_sync(vfs_t *vfsp, short f extern dev_t zfs_cmpldev(uint64_t); extern int zfs_get_zplprop(objset_t *os, zfs_prop_t prop, uint64_t *value); extern int zfs_get_stats(objset_t *os, nvlist_t *nv); +extern boolean_t zfs_get_vfs_flag_unmounted(objset_t *os); extern void zfs_znode_dmu_fini(znode_t *); extern void zfs_log_create(zilog_t *zilog, dmu_tx_t *tx, uint64_t txtype, Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c Fri Oct 14 11:46:17 2016 (r307314) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_dir.c Fri Oct 14 11:49:36 2016 (r307315) @@ -618,8 +618,8 @@ zfs_rmnode(znode_t *zp) error = dmu_free_long_range(os, zp->z_id, 0, DMU_OBJECT_END); if (error) { /* - * Not enough space. Leave the file in the unlinked - * set. + * Not enough space or we were interrupted by unmount. + * Leave the file in the unlinked set. */ zfs_znode_dmu_fini(zp); zfs_znode_free(zp); Modified: vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c ============================================================================== --- vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c Fri Oct 14 11:46:17 2016 (r307314) +++ vendor-sys/illumos/dist/uts/common/fs/zfs/zfs_vfsops.c Fri Oct 14 11:49:36 2016 (r307315) @@ -22,6 +22,7 @@ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2015 by Delphix. All rights reserved. * Copyright (c) 2014 Integros [integros.com] + * Copyright 2016 Nexenta Systems, Inc. All rights reserved. */ /* Portions Copyright 2010 Robert Milkowski */ @@ -1728,7 +1729,7 @@ zfs_root(vfs_t *vfsp, vnode_t **vpp) /* * Teardown the zfsvfs::z_os. * - * Note, if 'unmounting' if FALSE, we return with the 'z_teardown_lock' + * Note, if 'unmounting' is FALSE, we return with the 'z_teardown_lock' * and 'z_teardown_inactive_lock' held. */ static int @@ -1793,8 +1794,8 @@ zfsvfs_teardown(zfsvfs_t *zfsvfs, boolea */ if (unmounting) { zfsvfs->z_unmounted = B_TRUE; - rrm_exit(&zfsvfs->z_teardown_lock, FTAG); rw_exit(&zfsvfs->z_teardown_inactive_lock); + rrm_exit(&zfsvfs->z_teardown_lock, FTAG); } /* @@ -2272,6 +2273,29 @@ zfs_get_zplprop(objset_t *os, zfs_prop_t return (error); } +/* + * Return true if the coresponding vfs's unmounted flag is set. + * Otherwise return false. + * If this function returns true we know VFS unmount has been initiated. + */ +boolean_t +zfs_get_vfs_flag_unmounted(objset_t *os) +{ + zfsvfs_t *zfvp; + boolean_t unmounted = B_FALSE; + + ASSERT(dmu_objset_type(os) == DMU_OST_ZFS); + + mutex_enter(&os->os_user_ptr_lock); + zfvp = dmu_objset_get_user(os); + if (zfvp != NULL && zfvp->z_vfs != NULL && + (zfvp->z_vfs->vfs_flag & VFS_UNMOUNTED)) + unmounted = B_TRUE; + mutex_exit(&os->os_user_ptr_lock); + + return (unmounted); +} + static vfsdef_t vfw = { VFSDEF_VERSION, MNTTYPE_ZFS,