From owner-svn-src-projects@freebsd.org Mon Jun 17 22:01:25 2019 Return-Path: Delivered-To: svn-src-projects@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 34DF315C5B23 for ; Mon, 17 Jun 2019 22:01:25 +0000 (UTC) (envelope-from asomers@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id CFA52849C5; Mon, 17 Jun 2019 22:01:24 +0000 (UTC) (envelope-from asomers@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id AA5943F0A; Mon, 17 Jun 2019 22:01:24 +0000 (UTC) (envelope-from asomers@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id x5HM1OTH061030; Mon, 17 Jun 2019 22:01:24 GMT (envelope-from asomers@FreeBSD.org) Received: (from asomers@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id x5HM1N9R061021; Mon, 17 Jun 2019 22:01:23 GMT (envelope-from asomers@FreeBSD.org) Message-Id: <201906172201.x5HM1N9R061021@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: asomers set sender to asomers@FreeBSD.org using -f From: Alan Somers Date: Mon, 17 Jun 2019 22:01:23 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r349158 - in projects/fuse2: sys/fs/fuse tests/sys/fs/fusefs X-SVN-Group: projects X-SVN-Commit-Author: asomers X-SVN-Commit-Paths: in projects/fuse2: sys/fs/fuse tests/sys/fs/fusefs X-SVN-Commit-Revision: 349158 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-Rspamd-Queue-Id: CFA52849C5 X-Spamd-Bar: -- Authentication-Results: mx1.freebsd.org X-Spamd-Result: default: False [-2.97 / 15.00]; local_wl_from(0.00)[FreeBSD.org]; NEURAL_HAM_MEDIUM(-1.00)[-0.999,0]; NEURAL_HAM_SHORT(-0.97)[-0.973,0]; ASN(0.00)[asn:11403, ipnet:2610:1c1:1::/48, country:US]; NEURAL_HAM_LONG(-1.00)[-1.000,0] X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 17 Jun 2019 22:01:25 -0000 Author: asomers Date: Mon Jun 17 22:01:23 2019 New Revision: 349158 URL: https://svnweb.freebsd.org/changeset/base/349158 Log: fusefs: use cluster_read for more readahead fusefs will now use cluster_read. This allows readahead of more than one cache block. However, it won't yet actually cluster the reads because that requires VOP_BMAP, which fusefs does not yet implement. Sponsored by: The FreeBSD Foundation Modified: projects/fuse2/sys/fs/fuse/fuse_io.c projects/fuse2/tests/sys/fs/fusefs/mockfs.cc projects/fuse2/tests/sys/fs/fusefs/mockfs.hh projects/fuse2/tests/sys/fs/fusefs/read.cc projects/fuse2/tests/sys/fs/fusefs/utils.cc projects/fuse2/tests/sys/fs/fusefs/utils.hh Modified: projects/fuse2/sys/fs/fuse/fuse_io.c ============================================================================== --- projects/fuse2/sys/fs/fuse/fuse_io.c Mon Jun 17 21:46:13 2019 (r349157) +++ projects/fuse2/sys/fs/fuse/fuse_io.c Mon Jun 17 22:01:23 2019 (r349158) @@ -317,7 +317,13 @@ fuse_read_biobackend(struct vnode *vp, struct uio *uio if (bcount < biosize) { /* If near EOF, don't do readahead */ err = bread(vp, lbn, bcount, NOCRED, &bp); - /* TODO: clustered read */ + } else if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) { + /* Try clustered read */ + long totread = uio->uio_resid + on; + seqcount = MIN(seqcount, + data->max_readahead / biosize + 1); + err = cluster_read(vp, filesize, lbn, bcount, NOCRED, + totread, seqcount, 0, &bp); } else if (seqcount > 1 && data->max_readahead >= nextsize) { /* Try non-clustered readahead */ err = breadn(vp, lbn, bcount, &nextlbn, &nextsize, 1, Modified: projects/fuse2/tests/sys/fs/fusefs/mockfs.cc ============================================================================== --- projects/fuse2/tests/sys/fs/fusefs/mockfs.cc Mon Jun 17 21:46:13 2019 (r349157) +++ projects/fuse2/tests/sys/fs/fusefs/mockfs.cc Mon Jun 17 22:01:23 2019 (r349158) @@ -336,7 +336,8 @@ void MockFS::debug_response(const mockfs_buf_out &out) MockFS::MockFS(int max_readahead, bool allow_other, bool default_permissions, bool push_symlinks_in, bool ro, enum poll_method pm, uint32_t flags, - uint32_t kernel_minor_version, uint32_t max_write, bool async) + uint32_t kernel_minor_version, uint32_t max_write, bool async, + bool noclusterr) { struct sigaction sa; struct iovec *iov = NULL; @@ -408,6 +409,10 @@ MockFS::MockFS(int max_readahead, bool allow_other, bo if (async) { build_iovec(&iov, &iovlen, "async", __DECONST(void*, &trueval), sizeof(bool)); + } + if (noclusterr) { + build_iovec(&iov, &iovlen, "noclusterr", + __DECONST(void*, &trueval), sizeof(bool)); } if (nmount(iov, iovlen, 0)) throw(std::system_error(errno, std::system_category(), Modified: projects/fuse2/tests/sys/fs/fusefs/mockfs.hh ============================================================================== --- projects/fuse2/tests/sys/fs/fusefs/mockfs.hh Mon Jun 17 21:46:13 2019 (r349157) +++ projects/fuse2/tests/sys/fs/fusefs/mockfs.hh Mon Jun 17 22:01:23 2019 (r349158) @@ -303,7 +303,8 @@ class MockFS { MockFS(int max_readahead, bool allow_other, bool default_permissions, bool push_symlinks_in, bool ro, enum poll_method pm, uint32_t flags, - uint32_t kernel_minor_version, uint32_t max_write, bool async); + uint32_t kernel_minor_version, uint32_t max_write, bool async, + bool no_clusterr); virtual ~MockFS(); Modified: projects/fuse2/tests/sys/fs/fusefs/read.cc ============================================================================== --- projects/fuse2/tests/sys/fs/fusefs/read.cc Mon Jun 17 21:46:13 2019 (r349157) +++ projects/fuse2/tests/sys/fs/fusefs/read.cc Mon Jun 17 22:01:23 2019 (r349158) @@ -109,9 +109,12 @@ virtual void SetUp() { } }; -class ReadAhead: public ReadCacheable, public WithParamInterface { +class ReadAhead: public ReadCacheable, + public WithParamInterface> +{ virtual void SetUp() { - m_maxreadahead = GetParam(); + m_maxreadahead = get<1>(GetParam()); + m_noclusterr = get<0>(GetParam()); ReadCacheable::SetUp(); } }; @@ -746,6 +749,48 @@ TEST_F(ReadCacheable, DISABLED_sendfile_eio) /* Deliberately leak fd. close(2) will be tested in release.cc */ } +/* Large reads should be clustered, even across cache block boundaries */ +/* + * Disabled because clustered reads requires VOP_BMAP, which fusefs does not + * yet support + */ +TEST_P(ReadAhead, DISABLED_cluster) { + const char FULLPATH[] = "mountpoint/some_file.txt"; + const char RELPATH[] = "some_file.txt"; + uint64_t ino = 42; + int fd, maxcontig; + ssize_t bufsize = 4 * m_maxbcachebuf; + ssize_t filesize = bufsize; + uint64_t len; + char *rbuf, *contents; + off_t offs; + + contents = (char*)malloc(filesize); + ASSERT_NE(NULL, contents); + memset(contents, 'X', filesize); + rbuf = (char*)calloc(1, bufsize); + + expect_lookup(RELPATH, ino, filesize); + expect_open(ino, 0, 1); + maxcontig = m_noclusterr ? m_maxbcachebuf : + m_maxbcachebuf + (int)get<1>(GetParam()); + for (offs = 0; offs < bufsize; offs += maxcontig) { + len = std::min((size_t)maxcontig, (size_t)(filesize - offs)); + expect_read(ino, offs, len, len, contents + offs); + } + + fd = open(FULLPATH, O_RDONLY); + ASSERT_LE(0, fd) << strerror(errno); + + /* Set the internal readahead counter to a "large" value */ + ASSERT_EQ(0, fcntl(fd, F_READAHEAD, 1'000'000'000)) << strerror(errno); + + ASSERT_EQ(bufsize, read(fd, rbuf, bufsize)) << strerror(errno); + ASSERT_EQ(0, memcmp(rbuf, contents, bufsize)); + + /* Deliberately leak fd. close(2) will be tested in release.cc */ +} + /* fuse(4) should honor the filesystem's requested m_readahead parameter */ TEST_P(ReadAhead, readahead) { const char FULLPATH[] = "mountpoint/some_file.txt"; @@ -753,7 +798,7 @@ TEST_P(ReadAhead, readahead) { uint64_t ino = 42; int fd, i; ssize_t bufsize = m_maxbcachebuf; - ssize_t filesize = m_maxbcachebuf * 4; + ssize_t filesize = m_maxbcachebuf * 6; char *rbuf, *contents; contents = (char*)malloc(filesize); @@ -765,7 +810,7 @@ TEST_P(ReadAhead, readahead) { expect_open(ino, 0, 1); /* fuse(4) should only read ahead the allowed amount */ expect_read(ino, 0, m_maxbcachebuf, m_maxbcachebuf, contents); - for (i = 0; i < (int)GetParam() / m_maxbcachebuf; i++) { + for (i = 0; i < (int)get<1>(GetParam()) / m_maxbcachebuf; i++) { off_t offs = (i + 1) * m_maxbcachebuf; expect_read(ino, offs, m_maxbcachebuf, m_maxbcachebuf, contents + offs); @@ -783,4 +828,10 @@ TEST_P(ReadAhead, readahead) { /* Deliberately leak fd. close(2) will be tested in release.cc */ } -INSTANTIATE_TEST_CASE_P(RA, ReadAhead, ::testing::Values(0u, 65536)); +INSTANTIATE_TEST_CASE_P(RA, ReadAhead, + Values(tuple(false, 0u), + tuple(false, 0x10000), + tuple(false, 0x20000), + tuple(false, 0x30000), + tuple(true, 0u), + tuple(true, 0x10000))); Modified: projects/fuse2/tests/sys/fs/fusefs/utils.cc ============================================================================== --- projects/fuse2/tests/sys/fs/fusefs/utils.cc Mon Jun 17 21:46:13 2019 (r349157) +++ projects/fuse2/tests/sys/fs/fusefs/utils.cc Mon Jun 17 22:01:23 2019 (r349158) @@ -113,7 +113,7 @@ void FuseTest::SetUp() { m_mock = new MockFS(m_maxreadahead, m_allow_other, m_default_permissions, m_push_symlinks_in, m_ro, m_pm, m_init_flags, m_kernel_minor_version, - m_maxwrite, m_async); + m_maxwrite, m_async, m_noclusterr); /* * FUSE_ACCESS is called almost universally. Expecting it in * each test case would be super-annoying. Instead, set a Modified: projects/fuse2/tests/sys/fs/fusefs/utils.hh ============================================================================== --- projects/fuse2/tests/sys/fs/fusefs/utils.hh Mon Jun 17 21:46:13 2019 (r349157) +++ projects/fuse2/tests/sys/fs/fusefs/utils.hh Mon Jun 17 22:01:23 2019 (r349158) @@ -54,6 +54,7 @@ class FuseTest : public ::testing::Test { bool m_push_symlinks_in; bool m_ro; bool m_async; + bool m_noclusterr; MockFS *m_mock = NULL; const static uint64_t FH = 0xdeadbeef1a7ebabe; @@ -70,7 +71,8 @@ class FuseTest : public ::testing::Test { m_pm(BLOCKING), m_push_symlinks_in(false), m_ro(false), - m_async(false) + m_async(false), + m_noclusterr(false) {} virtual void SetUp();