From owner-svn-src-all@freebsd.org Wed Sep 16 15:38:24 2020 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 0D3FF3E9B51; Wed, 16 Sep 2020 15:38:24 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4Bs44b6XQSz3fBF; Wed, 16 Sep 2020 15:38:23 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id C3F501B804; Wed, 16 Sep 2020 15:38:23 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 08GFcNiV004775; Wed, 16 Sep 2020 15:38:23 GMT (envelope-from kib@FreeBSD.org) Received: (from kib@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 08GFcMwm004771; Wed, 16 Sep 2020 15:38:22 GMT (envelope-from kib@FreeBSD.org) Message-Id: <202009161538.08GFcMwm004771@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: kib set sender to kib@FreeBSD.org using -f From: Konstantin Belousov Date: Wed, 16 Sep 2020 15:38:22 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r365803 - stable/12/sys/vm X-SVN-Group: stable-12 X-SVN-Commit-Author: kib X-SVN-Commit-Paths: stable/12/sys/vm X-SVN-Commit-Revision: 365803 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.33 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 16 Sep 2020 15:38:24 -0000 Author: kib Date: Wed Sep 16 15:38:22 2020 New Revision: 365803 URL: https://svnweb.freebsd.org/changeset/base/365803 Log: MFC r365488: Allow consumer to customize physical pager. Modified: stable/12/sys/vm/phys_pager.c stable/12/sys/vm/vm_object.c stable/12/sys/vm/vm_object.h stable/12/sys/vm/vm_pager.h Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/vm/phys_pager.c ============================================================================== --- stable/12/sys/vm/phys_pager.c Wed Sep 16 15:10:16 2020 (r365802) +++ stable/12/sys/vm/phys_pager.c Wed Sep 16 15:38:22 2020 (r365803) @@ -51,6 +51,20 @@ static struct pagerlst phys_pager_object_list; /* protect access to phys_pager_object_list */ static struct mtx phys_pager_mtx; +static int default_phys_pager_getpages(vm_object_t object, vm_page_t *m, + int count, int *rbehind, int *rahead); +static int default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx, + int fault_type, vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last); +static boolean_t default_phys_pager_haspage(vm_object_t object, + vm_pindex_t pindex, int *before, int *after); +struct phys_pager_ops default_phys_pg_ops = { + .phys_pg_getpages = default_phys_pager_getpages, + .phys_pg_populate = default_phys_pager_populate, + .phys_pg_haspage = default_phys_pager_haspage, + .phys_pg_ctor = NULL, + .phys_pg_dtor = NULL, +}; + static void phys_pager_init(void) { @@ -59,12 +73,13 @@ phys_pager_init(void) mtx_init(&phys_pager_mtx, "phys_pager list", NULL, MTX_DEF); } -static vm_object_t -phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, - vm_ooffset_t foff, struct ucred *cred) +vm_object_t +phys_pager_allocate(void *handle, struct phys_pager_ops *ops, void *data, + vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred) { vm_object_t object, object1; vm_pindex_t pindex; + bool init; /* * Offset should be page aligned. @@ -73,6 +88,7 @@ phys_pager_alloc(void *handle, vm_ooffset_t size, vm_p return (NULL); pindex = OFF_TO_IDX(foff + PAGE_MASK + size); + init = true; if (handle != NULL) { mtx_lock(&phys_pager_mtx); @@ -97,11 +113,15 @@ phys_pager_alloc(void *handle, vm_ooffset_t size, vm_p */ if (pindex > object->size) object->size = pindex; + init = false; } else { object = object1; object1 = NULL; object->handle = handle; - vm_object_set_flag(object, OBJ_POPULATE); + object->un_pager.phys.ops = ops; + object->un_pager.phys.data_ptr = data; + if (ops->phys_pg_populate != NULL) + vm_object_set_flag(object, OBJ_POPULATE); TAILQ_INSERT_TAIL(&phys_pager_object_list, object, pager_object_list); } @@ -113,12 +133,25 @@ phys_pager_alloc(void *handle, vm_ooffset_t size, vm_p vm_object_deallocate(object1); } else { object = vm_object_allocate(OBJT_PHYS, pindex); - vm_object_set_flag(object, OBJ_POPULATE); + object->un_pager.phys.ops = ops; + object->un_pager.phys.data_ptr = data; + if (ops->phys_pg_populate != NULL) + vm_object_set_flag(object, OBJ_POPULATE); } + if (init && ops->phys_pg_ctor != NULL) + ops->phys_pg_ctor(object, prot, foff, cred); return (object); } +static vm_object_t +phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot, + vm_ooffset_t foff, struct ucred *ucred) +{ + return (phys_pager_allocate(handle, &default_phys_pg_ops, NULL, + size, prot, foff, ucred)); +} + static void phys_pager_dealloc(vm_object_t object) { @@ -130,16 +163,18 @@ phys_pager_dealloc(vm_object_t object) mtx_unlock(&phys_pager_mtx); VM_OBJECT_WLOCK(object); } - object->handle = NULL; object->type = OBJT_DEAD; + if (object->un_pager.phys.ops->phys_pg_dtor != NULL) + object->un_pager.phys.ops->phys_pg_dtor(object); + object->handle = NULL; } /* * Fill as many pages as vm_fault has allocated for us. */ static int -phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, - int *rahead) +default_phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, + int *rbehind, int *rahead) { int i; @@ -162,6 +197,14 @@ phys_pager_getpages(vm_object_t object, vm_page_t *m, return (VM_PAGER_OK); } +static int +phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, + int *rahead) +{ + return (object->un_pager.phys.ops->phys_pg_getpages(object, m, + count, rbehind, rahead)); +} + /* * Implement a pretty aggressive clustered getpages strategy. Hint that * everything in an entire 4MB window should be prefaulted at once. @@ -186,7 +229,7 @@ SYSCTL_INT(_vm, OID_AUTO, phys_pager_cluster, CTLFLAG_ #define PHYSALLOC 16 static int -phys_pager_populate(vm_object_t object, vm_pindex_t pidx, +default_phys_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first, vm_pindex_t *last) { @@ -238,6 +281,14 @@ retry: return (VM_PAGER_OK); } +static int +phys_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type, + vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) +{ + return (object->un_pager.phys.ops->phys_pg_populate(object, pidx, + fault_type, max_prot, first, last)); +} + static void phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync, int *rtvals) @@ -247,7 +298,7 @@ phys_pager_putpages(vm_object_t object, vm_page_t *m, } static boolean_t -phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, +default_phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after) { vm_pindex_t base, end; @@ -259,6 +310,14 @@ phys_pager_haspage(vm_object_t object, vm_pindex_t pin if (after != NULL) *after = end - pindex; return (TRUE); +} + +static boolean_t +phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, + int *after) +{ + return (object->un_pager.phys.ops->phys_pg_haspage(object, pindex, + before, after)); } struct pagerops physpagerops = { Modified: stable/12/sys/vm/vm_object.c ============================================================================== --- stable/12/sys/vm/vm_object.c Wed Sep 16 15:10:16 2020 (r365802) +++ stable/12/sys/vm/vm_object.c Wed Sep 16 15:38:22 2020 (r365803) @@ -307,6 +307,7 @@ vm_object_init(void) kernel_object->flags |= OBJ_COLORED; kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS); #endif + kernel_object->un_pager.phys.ops = &default_phys_pg_ops; /* * The lock portion of struct vm_object must be type stable due Modified: stable/12/sys/vm/vm_object.h ============================================================================== --- stable/12/sys/vm/vm_object.h Wed Sep 16 15:10:16 2020 (r365802) +++ stable/12/sys/vm/vm_object.h Wed Sep 16 15:38:22 2020 (r365803) @@ -169,6 +169,17 @@ struct vm_object { struct pctrie swp_blks; vm_ooffset_t writemappings; } swp; + + /* + * Phys pager + */ + struct { + struct phys_pager_ops *ops; + union { + void *data_ptr; + uintptr_t data_val; + }; + } phys; } un_pager; struct ucred *cred; vm_ooffset_t charge; Modified: stable/12/sys/vm/vm_pager.h ============================================================================== --- stable/12/sys/vm/vm_pager.h Wed Sep 16 15:10:16 2020 (r365802) +++ stable/12/sys/vm/vm_pager.h Wed Sep 16 15:38:22 2020 (r365803) @@ -233,5 +233,22 @@ vm_object_t cdev_pager_allocate(void *handle, enum obj vm_object_t cdev_pager_lookup(void *handle); void cdev_pager_free_page(vm_object_t object, vm_page_t m); +struct phys_pager_ops { + int (*phys_pg_getpages)(vm_object_t vm_obj, vm_page_t *m, int count, + int *rbehind, int *rahead); + int (*phys_pg_populate)(vm_object_t vm_obj, vm_pindex_t pidx, + int fault_type, vm_prot_t max_prot, vm_pindex_t *first, + vm_pindex_t *last); + boolean_t (*phys_pg_haspage)(vm_object_t obj, vm_pindex_t pindex, + int *before, int *after); + void (*phys_pg_ctor)(vm_object_t vm_obj, vm_prot_t prot, + vm_ooffset_t foff, struct ucred *cred); + void (*phys_pg_dtor)(vm_object_t vm_obj); +}; +extern struct phys_pager_ops default_phys_pg_ops; +vm_object_t phys_pager_allocate(void *handle, struct phys_pager_ops *ops, + void *data, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, + struct ucred *cred); + #endif /* _KERNEL */ #endif /* _VM_PAGER_ */