From owner-svn-src-all@freebsd.org Tue Sep 22 23:28:07 2020 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 3D0F03E4497; Tue, 22 Sep 2020 23:28:07 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4BwyCq0vBGz3cT5; Tue, 22 Sep 2020 23:28:07 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 0225227145; Tue, 22 Sep 2020 23:28:07 +0000 (UTC) (envelope-from kib@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 08MNS6SG091360; Tue, 22 Sep 2020 23:28:06 GMT (envelope-from kib@FreeBSD.org) Received: (from kib@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 08MNS6AL091359; Tue, 22 Sep 2020 23:28:06 GMT (envelope-from kib@FreeBSD.org) Message-Id: <202009222328.08MNS6AL091359@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: kib set sender to kib@FreeBSD.org using -f From: Konstantin Belousov Date: Tue, 22 Sep 2020 23:28:06 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r366030 - head/sys/amd64/amd64 X-SVN-Group: head X-SVN-Commit-Author: kib X-SVN-Commit-Paths: head/sys/amd64/amd64 X-SVN-Commit-Revision: 366030 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.33 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 22 Sep 2020 23:28:07 -0000 Author: kib Date: Tue Sep 22 23:28:06 2020 New Revision: 366030 URL: https://svnweb.freebsd.org/changeset/base/366030 Log: amd64 pmap: More unification for psind = 1 vs 2 in pmap_enter_largepage(). Move pkru check wait for page alloc wire accounting update asserting allowed updates for valid mappings out of psind conditions. Also add assert that psind references supported page size. Remove not true comment. Avoid uneccessary page table walks from top level. Reviewed by: alc, markj (previous version) Sponsored by: The FreeBSD Foundation MFC after: 1 week Differential revision: https://reviews.freebsd.org/D26513 Modified: head/sys/amd64/amd64/pmap.c Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Tue Sep 22 23:27:09 2020 (r366029) +++ head/sys/amd64/amd64/pmap.c Tue Sep 22 23:28:06 2020 (r366030) @@ -6479,7 +6479,7 @@ pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_e pt_entry_t origpte, *pml4e, *pdpe, *pde, pten, PG_V; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - KASSERT(psind > 0 && psind < MAXPAGESIZES, + KASSERT(psind > 0 && psind < MAXPAGESIZES && pagesizes[psind] != 0, ("psind %d unexpected", psind)); KASSERT(((newpte & PG_FRAME) & (pagesizes[psind] - 1)) == 0, ("unaligned phys address %#lx newpte %#lx psind %d", @@ -6494,38 +6494,25 @@ pmap_enter_largepage(pmap_t pmap, vm_offset_t va, pt_e PG_V = pmap_valid_bit(pmap); restart: + if (!pmap_pkru_same(pmap, va, va + pagesizes[psind])) + return (KERN_PROTECTION_FAILURE); pten = newpte; if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) pten |= pmap_pkru_get(pmap, va); if (psind == 2) { /* 1G */ - if (!pmap_pkru_same(pmap, va, va + NBPDP)) - return (KERN_PROTECTION_FAILURE); pml4e = pmap_pml4e(pmap, va); if (pml4e == NULL || (*pml4e & PG_V) == 0) { mp = _pmap_allocpte(pmap, pmap_pml4e_pindex(va), NULL, va); - if (mp == NULL) { - if ((flags & PMAP_ENTER_NOSLEEP) != 0) - return (KERN_RESOURCE_SHORTAGE); - PMAP_UNLOCK(pmap); - vm_wait(NULL); - PMAP_LOCK(pmap); - - /* - * Restart at least to recalcuate the pkru - * key. Our caller must keep the map locked - * so no paging structure can be validated - * under us. - */ - goto restart; - } - pdpe = pmap_pdpe(pmap, va); - KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va)); + if (mp == NULL) + goto allocf; + pdpe = (pdp_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); + pdpe = &pdpe[pmap_pdpe_index(va)]; origpte = *pdpe; MPASS(origpte == 0); } else { - pdpe = pmap_pdpe(pmap, va); + pdpe = pmap_pml4e_to_pdpe(pml4e, va); KASSERT(pdpe != NULL, ("va %#lx lost pdpe", va)); origpte = *pdpe; if ((origpte & PG_V) == 0) { @@ -6533,57 +6520,49 @@ restart: mp->ref_count++; } } - KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 && - (origpte & PG_FRAME) == (pten & PG_FRAME)), - ("va %#lx changing 1G phys page pdpe %#lx pten %#lx", - va, origpte, pten)); - if ((pten & PG_W) != 0 && (origpte & PG_W) == 0) - pmap->pm_stats.wired_count += NBPDP / PAGE_SIZE; - else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0) - pmap->pm_stats.wired_count -= NBPDP / PAGE_SIZE; *pdpe = pten; } else /* (psind == 1) */ { /* 2M */ - if (!pmap_pkru_same(pmap, va, va + NBPDR)) - return (KERN_PROTECTION_FAILURE); pde = pmap_pde(pmap, va); if (pde == NULL) { mp = _pmap_allocpte(pmap, pmap_pdpe_pindex(va), NULL, va); - if (mp == NULL) { - if ((flags & PMAP_ENTER_NOSLEEP) != 0) - return (KERN_RESOURCE_SHORTAGE); - PMAP_UNLOCK(pmap); - vm_wait(NULL); - PMAP_LOCK(pmap); - goto restart; - } + if (mp == NULL) + goto allocf; pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mp)); pde = &pde[pmap_pde_index(va)]; origpte = *pde; MPASS(origpte == 0); } else { - pdpe = pmap_pdpe(pmap, va); - MPASS(pdpe != NULL && (*pdpe & PG_V) != 0); origpte = *pde; if ((origpte & PG_V) == 0) { + pdpe = pmap_pdpe(pmap, va); + MPASS(pdpe != NULL && (*pdpe & PG_V) != 0); mp = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); mp->ref_count++; } } - KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 && - (origpte & PG_FRAME) == (pten & PG_FRAME)), - ("va %#lx changing 2M phys page pde %#lx pten %#lx", - va, origpte, pten)); - if ((pten & PG_W) != 0 && (origpte & PG_W) == 0) - pmap->pm_stats.wired_count += NBPDR / PAGE_SIZE; - else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0) - pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE; *pde = pten; } + KASSERT((origpte & PG_V) == 0 || ((origpte & PG_PS) != 0 && + (origpte & PG_PS_FRAME) == (pten & PG_PS_FRAME)), + ("va %#lx changing %s phys page origpte %#lx pten %#lx", + va, psind == 2 ? "1G" : "2M", origpte, pten)); + if ((pten & PG_W) != 0 && (origpte & PG_W) == 0) + pmap->pm_stats.wired_count += pagesizes[psind] / PAGE_SIZE; + else if ((pten & PG_W) == 0 && (origpte & PG_W) != 0) + pmap->pm_stats.wired_count -= pagesizes[psind] / PAGE_SIZE; if ((origpte & PG_V) == 0) pmap_resident_count_inc(pmap, pagesizes[psind] / PAGE_SIZE); return (KERN_SUCCESS); + +allocf: + if ((flags & PMAP_ENTER_NOSLEEP) != 0) + return (KERN_RESOURCE_SHORTAGE); + PMAP_UNLOCK(pmap); + vm_wait(NULL); + PMAP_LOCK(pmap); + goto restart; } /*