Date: Mon, 1 Nov 2021 14:32:56 GMT From: Mark Johnston <markj@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org Subject: git: a3d4c8e21d64 - stable/13 - amd64: Implement a KASAN shadow map Message-ID: <202111011432.1A1EWueM021267@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch stable/13 has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=a3d4c8e21d645bc8022af53232c12e236285de25 commit a3d4c8e21d645bc8022af53232c12e236285de25 Author: Mark Johnston <markj@FreeBSD.org> AuthorDate: 2021-04-13 20:30:05 +0000 Commit: Mark Johnston <markj@FreeBSD.org> CommitDate: 2021-11-01 13:57:30 +0000 amd64: Implement a KASAN shadow map The idea behind KASAN is to use a region of memory to track the validity of buffers in the kernel map. This region is the shadow map. The compiler inserts calls to the KASAN runtime for every emitted load and store, and the runtime uses the shadow map to decide whether the access is valid. Various kernel allocators call kasan_mark() to update the shadow map. Since the shadow map tracks only accesses to the kernel map, accesses to other kernel maps are not validated by KASAN. UMA_MD_SMALL_ALLOC is disabled when KASAN is configured to reduce usage of the direct map. Currently we have no mechanism to completely eliminate uses of the direct map, so KASAN's coverage is not comprehensive. The shadow map uses one byte per eight bytes in the kernel map. In pmap_bootstrap() we create an initial set of page tables for the kernel and preloaded data. When pmap_growkernel() is called, we call kasan_shadow_map() to extend the shadow map. kasan_shadow_map() uses pmap_kasan_enter() to allocate memory for the shadow region and map it. Reviewed by: kib MFC after: 2 weeks Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D29417 (cherry picked from commit 6faf45b34b14da5f138774b43ec14fb5567ac584) --- sys/amd64/amd64/pmap.c | 140 +++++++++++++++++++++++++++++++++++++++++++- sys/amd64/include/asan.h | 71 ++++++++++++++++++++++ sys/amd64/include/pmap.h | 15 ++++- sys/amd64/include/vmparam.h | 8 ++- sys/kern/kern_malloc.c | 10 ++++ sys/kern/vfs_bio.c | 10 ++++ 6 files changed, 251 insertions(+), 3 deletions(-) diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 43ed0f7b7411..62bc1bb22e96 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -112,6 +112,7 @@ __FBSDID("$FreeBSD$"); #include "opt_vm.h" #include <sys/param.h> +#include <sys/asan.h> #include <sys/bitstring.h> #include <sys/bus.h> #include <sys/systm.h> @@ -154,6 +155,7 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_dumpset.h> #include <vm/uma.h> +#include <machine/asan.h> #include <machine/intr_machdep.h> #include <x86/apicvar.h> #include <x86/ifunc.h> @@ -425,6 +427,10 @@ u_int64_t KPML4phys; /* phys addr of kernel level 4 */ u_int64_t KPML5phys; /* phys addr of kernel level 5, if supported */ +#ifdef KASAN +static uint64_t KASANPDPphys; +#endif + static pml4_entry_t *kernel_pml4; static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */ static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */ @@ -1631,6 +1637,12 @@ create_pagetables(vm_paddr_t *firstaddr) pml4_entry_t *p4_p; uint64_t DMPDkernphys; vm_paddr_t pax; +#ifdef KASAN + pt_entry_t *pt_p; + uint64_t KASANPDphys, KASANPTphys, KASANphys; + vm_offset_t kasankernbase; + int kasankpdpi, kasankpdi, nkasanpte; +#endif int i, j, ndm1g, nkpdpe, nkdmpde; /* Allocate page table pages for the direct map */ @@ -1673,6 +1685,10 @@ create_pagetables(vm_paddr_t *firstaddr) /* Allocate pages */ KPML4phys = allocpages(firstaddr, 1); KPDPphys = allocpages(firstaddr, NKPML4E); +#ifdef KASAN + KASANPDPphys = allocpages(firstaddr, NKASANPML4E); + KASANPDphys = allocpages(firstaddr, 1); +#endif /* * Allocate the initial number of kernel page table pages required to @@ -1690,6 +1706,12 @@ create_pagetables(vm_paddr_t *firstaddr) KPTphys = allocpages(firstaddr, nkpt); KPDphys = allocpages(firstaddr, nkpdpe); +#ifdef KASAN + nkasanpte = howmany(nkpt, KASAN_SHADOW_SCALE); + KASANPTphys = allocpages(firstaddr, nkasanpte); + KASANphys = allocpages(firstaddr, nkasanpte * NPTEPG); +#endif + /* * Connect the zero-filled PT pages to their PD entries. This * implicitly maps the PT pages at their correct locations within @@ -1726,6 +1748,25 @@ create_pagetables(vm_paddr_t *firstaddr) for (i = 0; i < nkpdpe; i++) pdp_p[i + KPDPI] = (KPDphys + ptoa(i)) | X86_PG_RW | X86_PG_V; +#ifdef KASAN + kasankernbase = kasan_md_addr_to_shad(KERNBASE); + kasankpdpi = pmap_pdpe_index(kasankernbase); + kasankpdi = pmap_pde_index(kasankernbase); + + pdp_p = (pdp_entry_t *)KASANPDPphys; + pdp_p[kasankpdpi] = (KASANPDphys | X86_PG_RW | X86_PG_V | pg_nx); + + pd_p = (pd_entry_t *)KASANPDphys; + for (i = 0; i < nkasanpte; i++) + pd_p[i + kasankpdi] = (KASANPTphys + ptoa(i)) | X86_PG_RW | + X86_PG_V | pg_nx; + + pt_p = (pt_entry_t *)KASANPTphys; + for (i = 0; i < nkasanpte * NPTEPG; i++) + pt_p[i] = (KASANphys + ptoa(i)) | X86_PG_RW | X86_PG_V | + X86_PG_M | X86_PG_A | pg_nx; +#endif + /* * Now, set up the direct map region using 2MB and/or 1GB pages. If * the end of physical memory is not aligned to a 1GB page boundary, @@ -1777,7 +1818,15 @@ create_pagetables(vm_paddr_t *firstaddr) p4_p[PML4PML4I] = KPML4phys; p4_p[PML4PML4I] |= X86_PG_RW | X86_PG_V | pg_nx; - /* Connect the Direct Map slot(s) up to the PML4. */ +#ifdef KASAN + /* Connect the KASAN shadow map slots up to the PML4. */ + for (i = 0; i < NKASANPML4E; i++) { + p4_p[KASANPML4I + i] = KASANPDPphys + ptoa(i); + p4_p[KASANPML4I + i] |= X86_PG_RW | X86_PG_V | pg_nx; + } +#endif + + /* Connect the Direct Map slots up to the PML4. */ for (i = 0; i < ndmpdpphys; i++) { p4_p[DMPML4I + i] = DMPDPphys + ptoa(i); p4_p[DMPML4I + i] |= X86_PG_RW | X86_PG_V | pg_nx; @@ -4132,6 +4181,12 @@ pmap_pinit_pml4(vm_page_t pml4pg) pm_pml4[KPML4BASE + i] = (KPDPphys + ptoa(i)) | X86_PG_RW | X86_PG_V; } +#ifdef KASAN + for (i = 0; i < NKASANPML4E; i++) { + pm_pml4[KASANPML4I + i] = (KASANPDPphys + ptoa(i)) | X86_PG_RW | + X86_PG_V | pg_nx; + } +#endif for (i = 0; i < ndmpdpphys; i++) { pm_pml4[DMPML4I + i] = (DMPDPphys + ptoa(i)) | X86_PG_RW | X86_PG_V; @@ -4715,6 +4770,10 @@ pmap_release(pmap_t pmap) } else { for (i = 0; i < NKPML4E; i++) /* KVA */ pmap->pm_pmltop[KPML4BASE + i] = 0; +#ifdef KASAN + for (i = 0; i < NKASANPML4E; i++) /* KASAN shadow map */ + pmap->pm_pmltop[KASANPML4I + i] = 0; +#endif for (i = 0; i < ndmpdpphys; i++)/* Direct Map */ pmap->pm_pmltop[DMPML4I + i] = 0; pmap->pm_pmltop[PML4PML4I] = 0; /* Recursive Mapping */ @@ -4832,6 +4891,8 @@ pmap_growkernel(vm_offset_t addr) addr = roundup2(addr, NBPDR); if (addr - 1 >= vm_map_max(kernel_map)) addr = vm_map_max(kernel_map); + if (kernel_vm_end < addr) + kasan_shadow_map((void *)kernel_vm_end, addr - kernel_vm_end); while (kernel_vm_end < addr) { pdpe = pmap_pdpe(kernel_pmap, kernel_vm_end); if ((*pdpe & X86_PG_V) == 0) { @@ -11227,6 +11288,78 @@ pmap_pkru_clear(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) return (error); } +#ifdef KASAN +static vm_page_t +pmap_kasan_enter_alloc_4k(void) +{ + vm_page_t m; + + m = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | + VM_ALLOC_WIRED | VM_ALLOC_ZERO); + if (m == NULL) + panic("%s: no memory to grow shadow map", __func__); + if ((m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + return (m); +} + +static vm_page_t +pmap_kasan_enter_alloc_2m(void) +{ + vm_page_t m; + + m = vm_page_alloc_contig(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | + VM_ALLOC_WIRED, NPTEPG, 0, ~0ul, NBPDR, 0, VM_MEMATTR_DEFAULT); + if (m != NULL) + memset((void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), 0, NBPDR); + return (m); +} + +/* + * Grow the shadow map by at least one 4KB page at the specified address. Use + * 2MB pages when possible. + */ +void +pmap_kasan_enter(vm_offset_t va) +{ + pdp_entry_t *pdpe; + pd_entry_t *pde; + pt_entry_t *pte; + vm_page_t m; + + mtx_assert(&kernel_map->system_mtx, MA_OWNED); + + pdpe = pmap_pdpe(kernel_pmap, va); + if ((*pdpe & X86_PG_V) == 0) { + m = pmap_kasan_enter_alloc_4k(); + *pdpe = (pdp_entry_t)(VM_PAGE_TO_PHYS(m) | X86_PG_RW | + X86_PG_V | pg_nx); + } + pde = pmap_pdpe_to_pde(pdpe, va); + if ((*pde & X86_PG_V) == 0) { + m = pmap_kasan_enter_alloc_2m(); + if (m != NULL) { + *pde = (pd_entry_t)(VM_PAGE_TO_PHYS(m) | X86_PG_RW | + X86_PG_PS | X86_PG_V | pg_nx); + } else { + m = pmap_kasan_enter_alloc_4k(); + *pde = (pd_entry_t)(VM_PAGE_TO_PHYS(m) | X86_PG_RW | + X86_PG_V | pg_nx); + } + } + if ((*pde & X86_PG_PS) != 0) + return; + pte = pmap_pde_to_pte(pde, va); + if ((*pte & X86_PG_V) != 0) + return; + KASSERT((*pte & X86_PG_V) == 0, + ("%s: shadow address %#lx is already mapped", __func__, va)); + m = pmap_kasan_enter_alloc_4k(); + *pte = (pt_entry_t)(VM_PAGE_TO_PHYS(m) | X86_PG_RW | X86_PG_V | + X86_PG_M | X86_PG_A | pg_nx); +} +#endif + /* * Track a range of the kernel's virtual address space that is contiguous * in various mapping attributes. @@ -11404,6 +11537,11 @@ sysctl_kmaps(SYSCTL_HANDLER_ARGS) case DMPML4I: sbuf_printf(sb, "\nDirect map:\n"); break; +#ifdef KASAN + case KASANPML4I: + sbuf_printf(sb, "\nKASAN shadow map:\n"); + break; +#endif case KPML4BASE: sbuf_printf(sb, "\nKernel map:\n"); break; diff --git a/sys/amd64/include/asan.h b/sys/amd64/include/asan.h new file mode 100644 index 000000000000..03d57673d05e --- /dev/null +++ b/sys/amd64/include/asan.h @@ -0,0 +1,71 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2020 The FreeBSD Foundation + * + * This software was developed by Mark Johnston under sponsorship from the + * FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _MACHINE_ASAN_H_ +#define _MACHINE_ASAN_H_ + +#ifdef KASAN + +#include <vm/vm.h> +#include <vm/pmap.h> +#include <vm/vm_page.h> +#include <machine/vmparam.h> + +static inline vm_offset_t +kasan_md_addr_to_shad(vm_offset_t addr) +{ + return (((addr - VM_MIN_KERNEL_ADDRESS) >> KASAN_SHADOW_SCALE_SHIFT) + + KASAN_MIN_ADDRESS); +} + +static inline bool +kasan_md_unsupported(vm_offset_t addr) +{ + vm_offset_t kernmin; + + /* + * The vm_page array is mapped at the beginning of the kernel map, but + * accesses to the array are not validated for now. Handle the fact + * that KASAN must validate accesses before the vm_page array is + * initialized. + */ + kernmin = vm_page_array == NULL ? VM_MIN_KERNEL_ADDRESS : + (vm_offset_t)(vm_page_array + vm_page_array_size); + return (addr < kernmin || addr >= VM_MAX_KERNEL_ADDRESS); +} + +static inline void +kasan_md_init(void) +{ +} + +#endif /* KASAN */ + +#endif /* !_MACHINE_ASAN_H_ */ diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index 0ac91d17792a..c5c1714f2f94 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -195,6 +195,12 @@ */ #define NKPML4E 4 +/* + * Number of PML4 slots for the KASAN shadow map. It requires 1 byte of memory + * for every 8 bytes of the kernel address space. + */ +#define NKASANPML4E ((NKPML4E + 7) / 8) + /* * We use the same numbering of the page table pages for 5-level and * 4-level paging structures. @@ -243,9 +249,11 @@ #define KPML4I (NPML4EPG-1) #define KPDPI (NPDPEPG-2) /* kernbase at -2GB */ +#define KASANPML4I (DMPML4I - NKASANPML4E) /* Below the direct map */ + /* Large map: index of the first and max last pml4 entry */ #define LMSPML4I (PML4PML4I + 1) -#define LMEPML4I (DMPML4I - 1) +#define LMEPML4I (KASANPML4I - 1) /* * XXX doesn't really belong here I guess... @@ -509,6 +517,11 @@ void pmap_thread_init_invl_gen(struct thread *td); int pmap_vmspace_copy(pmap_t dst_pmap, pmap_t src_pmap); void pmap_page_array_startup(long count); vm_page_t pmap_page_alloc_below_4g(bool zeroed); + +#ifdef KASAN +void pmap_kasan_enter(vm_offset_t); +#endif + #endif /* _KERNEL */ /* Return various clipped indexes for a given VA */ diff --git a/sys/amd64/include/vmparam.h b/sys/amd64/include/vmparam.h index b6f79ef8ca84..88fd29b80be3 100644 --- a/sys/amd64/include/vmparam.h +++ b/sys/amd64/include/vmparam.h @@ -75,7 +75,9 @@ * of the direct mapped segment. This uses 2MB pages for reduced * TLB pressure. */ +#ifndef KASAN #define UMA_MD_SMALL_ALLOC +#endif /* * The physical address space is densely populated. @@ -167,7 +169,8 @@ * 0xffff800000000000 - 0xffff804020100fff recursive page table (512GB slot) * 0xffff804020100fff - 0xffff807fffffffff unused * 0xffff808000000000 - 0xffff847fffffffff large map (can be tuned up) - * 0xffff848000000000 - 0xfffff7ffffffffff unused (large map extends there) + * 0xffff848000000000 - 0xfffff77fffffffff unused (large map extends there) + * 0xfffff78000000000 - 0xfffff7ffffffffff 512GB KASAN shadow map * 0xfffff80000000000 - 0xfffffbffffffffff 4TB direct map * 0xfffffc0000000000 - 0xfffffdffffffffff unused * 0xfffffe0000000000 - 0xffffffffffffffff 2TB kernel map @@ -185,6 +188,9 @@ #define DMAP_MIN_ADDRESS KV4ADDR(DMPML4I, 0, 0, 0) #define DMAP_MAX_ADDRESS KV4ADDR(DMPML4I + NDMPML4E, 0, 0, 0) +#define KASAN_MIN_ADDRESS KV4ADDR(KASANPML4I, 0, 0, 0) +#define KASAN_MAX_ADDRESS KV4ADDR(KASANPML4I + NKASANPML4E, 0, 0, 0) + #define LARGEMAP_MIN_ADDRESS KV4ADDR(LMSPML4I, 0, 0, 0) #define LARGEMAP_MAX_ADDRESS KV4ADDR(LMEPML4I + 1, 0, 0, 0) diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c index 0fc4fcbc0539..8f371d35f066 100644 --- a/sys/kern/kern_malloc.c +++ b/sys/kern/kern_malloc.c @@ -1160,6 +1160,16 @@ kmeminit(void) vm_kmem_size = 2 * mem_size * PAGE_SIZE; vm_kmem_size = round_page(vm_kmem_size); + +#ifdef KASAN + /* + * With KASAN enabled, dynamically allocated kernel memory is shadowed. + * Account for this when setting the UMA limit. + */ + vm_kmem_size = (vm_kmem_size * KASAN_SHADOW_SCALE) / + (KASAN_SHADOW_SCALE + 1); +#endif + #ifdef DEBUG_MEMGUARD tmp = memguard_fudge(vm_kmem_size, kernel_map); #else diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 74dc2feb9ea6..d610f391de98 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> +#include <sys/asan.h> #include <sys/bio.h> #include <sys/bitset.h> #include <sys/conf.h> @@ -1043,6 +1044,15 @@ kern_vfs_bio_buffer_alloc(caddr_t v, long physmem_est) int tuned_nbuf; long maxbuf, maxbuf_sz, buf_sz, biotmap_sz; +#ifdef KASAN + /* + * With KASAN enabled, the kernel map is shadowed. Account for this + * when sizing maps based on the amount of physical memory available. + */ + physmem_est = (physmem_est * KASAN_SHADOW_SCALE) / + (KASAN_SHADOW_SCALE + 1); +#endif + /* * physmem_est is in pages. Convert it to kilobytes (assumes * PAGE_SIZE is >= 1K)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202111011432.1A1EWueM021267>