From owner-dev-commits-src-branches@freebsd.org Tue Jul 20 23:46:19 2021 Return-Path: Delivered-To: dev-commits-src-branches@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 851D6655C8A; Tue, 20 Jul 2021 23:46:19 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4GTwMv342sz3PPT; Tue, 20 Jul 2021 23:46:19 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 5133D13ED7; Tue, 20 Jul 2021 23:46:19 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 16KNkJSU054533; Tue, 20 Jul 2021 23:46:19 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 16KNkJ5i054532; Tue, 20 Jul 2021 23:46:19 GMT (envelope-from git) Date: Tue, 20 Jul 2021 23:46:19 GMT Message-Id: <202107202346.16KNkJ5i054532@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: Vladimir Kondratyev Subject: git: 10235ad0567f - stable/13 - LinuxKPI: Allow kmem_cache_free() to be called from critical sections MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: wulf X-Git-Repository: src X-Git-Refname: refs/heads/stable/13 X-Git-Reftype: branch X-Git-Commit: 10235ad0567f7e1faf8949551731410d5f0aad15 Auto-Submitted: auto-generated X-BeenThere: dev-commits-src-branches@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Commits to the stable branches of the FreeBSD src repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 20 Jul 2021 23:46:19 -0000 The branch stable/13 has been updated by wulf: URL: https://cgit.FreeBSD.org/src/commit/?id=10235ad0567f7e1faf8949551731410d5f0aad15 commit 10235ad0567f7e1faf8949551731410d5f0aad15 Author: Vladimir Kondratyev AuthorDate: 2021-07-05 00:18:14 +0000 Commit: Vladimir Kondratyev CommitDate: 2021-07-20 22:43:52 +0000 LinuxKPI: Allow kmem_cache_free() to be called from critical sections as it is required by i915kms driver from Linux kernel v 5.5. This is done with asynchronous freeing of requested memory areas from taskqueue thread. As memory to be freed is reused to store linked list entry, backing UMA zone item size is rounded up to pointer size. While here, make struct linux_kmem_cache private to LKPI to reduce amount of BSD headers included by linux/slab.h and switch RCU code to usage of LKPI's linux_irq_work_tq taskqueue to avoid injection of current into system-wide taskqueue_fast thread context. Submitted by: nc (initial version for drm-kmod) Reviewed by: manu, nc Differential revision: https://reviews.freebsd.org/D30760 (cherry picked from commit a2b83b59db365255461a50ff8fa5e5e4e92e4d51) --- sys/compat/linuxkpi/common/include/linux/slab.h | 46 +++------------ sys/compat/linuxkpi/common/src/linux_rcu.c | 50 ++++++----------- sys/compat/linuxkpi/common/src/linux_slab.c | 75 ++++++++++++++++++++++++- 3 files changed, 99 insertions(+), 72 deletions(-) diff --git a/sys/compat/linuxkpi/common/include/linux/slab.h b/sys/compat/linuxkpi/common/include/linux/slab.h index 8d5eb3e85e96..ecd39d711eb4 100644 --- a/sys/compat/linuxkpi/common/include/linux/slab.h +++ b/sys/compat/linuxkpi/common/include/linux/slab.h @@ -31,12 +31,9 @@ #ifndef _LINUX_SLAB_H_ #define _LINUX_SLAB_H_ -#include -#include +#include #include #include -#include -#include #include #include @@ -65,8 +62,9 @@ MALLOC_DECLARE(M_KMALLOC); */ #define kmem_cache linux_kmem_cache #define kmem_cache_create(...) linux_kmem_cache_create(__VA_ARGS__) -#define kmem_cache_alloc(...) linux_kmem_cache_alloc(__VA_ARGS__) -#define kmem_cache_free(...) linux_kmem_cache_free(__VA_ARGS__) +#define kmem_cache_alloc(...) lkpi_kmem_cache_alloc(__VA_ARGS__) +#define kmem_cache_zalloc(...) lkpi_kmem_cache_zalloc(__VA_ARGS__) +#define kmem_cache_free(...) lkpi_kmem_cache_free(__VA_ARGS__) #define kmem_cache_destroy(...) linux_kmem_cache_destroy(__VA_ARGS__) #define KMEM_CACHE(__struct, flags) \ @@ -75,12 +73,7 @@ MALLOC_DECLARE(M_KMALLOC); typedef void linux_kmem_ctor_t (void *); -struct linux_kmem_cache { - uma_zone_t cache_zone; - linux_kmem_ctor_t *cache_ctor; - unsigned cache_flags; - unsigned cache_size; -}; +struct linux_kmem_cache; #define SLAB_HWCACHE_ALIGN (1 << 0) #define SLAB_TYPESAFE_BY_RCU (1 << 1) @@ -212,32 +205,9 @@ ksize(const void *ptr) extern struct linux_kmem_cache *linux_kmem_cache_create(const char *name, size_t size, size_t align, unsigned flags, linux_kmem_ctor_t *ctor); - -static inline void * -linux_kmem_cache_alloc(struct linux_kmem_cache *c, gfp_t flags) -{ - return (uma_zalloc_arg(c->cache_zone, c, - linux_check_m_flags(flags))); -} - -static inline void * -kmem_cache_zalloc(struct linux_kmem_cache *c, gfp_t flags) -{ - return (uma_zalloc_arg(c->cache_zone, c, - linux_check_m_flags(flags | M_ZERO))); -} - -extern void linux_kmem_cache_free_rcu(struct linux_kmem_cache *, void *); - -static inline void -linux_kmem_cache_free(struct linux_kmem_cache *c, void *m) -{ - if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU)) - linux_kmem_cache_free_rcu(c, m); - else - uma_zfree(c->cache_zone, m); -} - +extern void *lkpi_kmem_cache_alloc(struct linux_kmem_cache *, gfp_t); +extern void *lkpi_kmem_cache_zalloc(struct linux_kmem_cache *, gfp_t); +extern void lkpi_kmem_cache_free(struct linux_kmem_cache *, void *); extern void linux_kmem_cache_destroy(struct linux_kmem_cache *); void linux_kfree_async(void *); diff --git a/sys/compat/linuxkpi/common/src/linux_rcu.c b/sys/compat/linuxkpi/common/src/linux_rcu.c index 404c5cec4ae4..a39949cf5013 100644 --- a/sys/compat/linuxkpi/common/src/linux_rcu.c +++ b/sys/compat/linuxkpi/common/src/linux_rcu.c @@ -48,6 +48,8 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include +#include /* * By defining CONFIG_NO_RCU_SKIP LinuxKPI RCU locks and asserts will @@ -60,13 +62,15 @@ __FBSDID("$FreeBSD$"); #endif struct callback_head { - STAILQ_ENTRY(callback_head) entry; + union { + STAILQ_ENTRY(callback_head) entry; + struct llist_node node; + }; rcu_callback_t func; }; struct linux_epoch_head { - STAILQ_HEAD(, callback_head) cb_head; - struct mtx lock; + struct llist_head cb_head; struct task task; } __aligned(CACHE_LINE_SIZE); @@ -120,9 +124,8 @@ linux_rcu_runtime_init(void *arg __unused) head = &linux_epoch_head[j]; - mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF); TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head); - STAILQ_INIT(&head->cb_head); + init_llist_head(&head->cb_head); CPU_FOREACH(i) { struct linux_epoch_record *record; @@ -139,37 +142,22 @@ linux_rcu_runtime_init(void *arg __unused) } SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL); -static void -linux_rcu_runtime_uninit(void *arg __unused) -{ - struct linux_epoch_head *head; - int j; - - for (j = 0; j != RCU_TYPE_MAX; j++) { - head = &linux_epoch_head[j]; - - mtx_destroy(&head->lock); - } -} -SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL); - static void linux_rcu_cleaner_func(void *context, int pending __unused) { - struct linux_epoch_head *head; + struct linux_epoch_head *head = context; struct callback_head *rcu; STAILQ_HEAD(, callback_head) tmp_head; + struct llist_node *node, *next; uintptr_t offset; - linux_set_current(curthread); - - head = context; - /* move current callbacks into own queue */ - mtx_lock(&head->lock); STAILQ_INIT(&tmp_head); - STAILQ_CONCAT(&tmp_head, &head->cb_head); - mtx_unlock(&head->lock); + llist_for_each_safe(node, next, llist_del_all(&head->cb_head)) { + rcu = container_of(node, struct callback_head, node); + /* re-reverse list to restore chronological order */ + STAILQ_INSERT_HEAD(&tmp_head, rcu, entry); + } /* synchronize */ linux_synchronize_rcu(head - linux_epoch_head); @@ -384,7 +372,7 @@ linux_rcu_barrier(unsigned type) head = &linux_epoch_head[type]; /* wait for callbacks to complete */ - taskqueue_drain(taskqueue_fast, &head->task); + taskqueue_drain(linux_irq_work_tq, &head->task); } void @@ -398,11 +386,9 @@ linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func) rcu = (struct callback_head *)context; head = &linux_epoch_head[type]; - mtx_lock(&head->lock); rcu->func = func; - STAILQ_INSERT_TAIL(&head->cb_head, rcu, entry); - taskqueue_enqueue(taskqueue_fast, &head->task); - mtx_unlock(&head->lock); + llist_add(&rcu->node, &head->cb_head); + taskqueue_enqueue(linux_irq_work_tq, &head->task); } int diff --git a/sys/compat/linuxkpi/common/src/linux_slab.c b/sys/compat/linuxkpi/common/src/linux_slab.c index 3304c34b1dee..5dbd87b66d1a 100644 --- a/sys/compat/linuxkpi/common/src/linux_slab.c +++ b/sys/compat/linuxkpi/common/src/linux_slab.c @@ -35,12 +35,22 @@ __FBSDID("$FreeBSD$"); #include #include +#include struct linux_kmem_rcu { struct rcu_head rcu_head; struct linux_kmem_cache *cache; }; +struct linux_kmem_cache { + uma_zone_t cache_zone; + linux_kmem_ctor_t *cache_ctor; + unsigned cache_flags; + unsigned cache_size; + struct llist_head cache_items; + struct task cache_task; +}; + #define LINUX_KMEM_TO_RCU(c, m) \ ((struct linux_kmem_rcu *)((char *)(m) + \ (c)->cache_size - sizeof(struct linux_kmem_rcu))) @@ -51,6 +61,22 @@ struct linux_kmem_rcu { static LLIST_HEAD(linux_kfree_async_list); +static void lkpi_kmem_cache_free_async_fn(void *, int); + +void * +lkpi_kmem_cache_alloc(struct linux_kmem_cache *c, gfp_t flags) +{ + return (uma_zalloc_arg(c->cache_zone, c, + linux_check_m_flags(flags))); +} + +void * +lkpi_kmem_cache_zalloc(struct linux_kmem_cache *c, gfp_t flags) +{ + return (uma_zalloc_arg(c->cache_zone, c, + linux_check_m_flags(flags | M_ZERO))); +} + static int linux_kmem_ctor(void *mem, int size, void *arg, int flags) { @@ -102,6 +128,9 @@ linux_kmem_cache_create(const char *name, size_t size, size_t align, linux_kmem_ctor, NULL, NULL, NULL, align, UMA_ZONE_ZINIT); } else { + /* make room for async task list items */ + size = MAX(size, sizeof(struct llist_node)); + /* create cache_zone */ c->cache_zone = uma_zcreate(name, size, ctor ? linux_kmem_ctor : NULL, NULL, @@ -111,17 +140,56 @@ linux_kmem_cache_create(const char *name, size_t size, size_t align, c->cache_flags = flags; c->cache_ctor = ctor; c->cache_size = size; + init_llist_head(&c->cache_items); + TASK_INIT(&c->cache_task, 0, lkpi_kmem_cache_free_async_fn, c); return (c); } -void -linux_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m) +static inline void +lkpi_kmem_cache_free_rcu(struct linux_kmem_cache *c, void *m) { struct linux_kmem_rcu *rcu = LINUX_KMEM_TO_RCU(c, m); call_rcu(&rcu->rcu_head, linux_kmem_cache_free_rcu_callback); } +static inline void +lkpi_kmem_cache_free_sync(struct linux_kmem_cache *c, void *m) +{ + uma_zfree(c->cache_zone, m); +} + +static void +lkpi_kmem_cache_free_async_fn(void *context, int pending) +{ + struct linux_kmem_cache *c = context; + struct llist_node *freed, *next; + + llist_for_each_safe(freed, next, llist_del_all(&c->cache_items)) + lkpi_kmem_cache_free_sync(c, freed); +} + +static inline void +lkpi_kmem_cache_free_async(struct linux_kmem_cache *c, void *m) +{ + if (m == NULL) + return; + + llist_add(m, &c->cache_items); + taskqueue_enqueue(linux_irq_work_tq, &c->cache_task); +} + +void +lkpi_kmem_cache_free(struct linux_kmem_cache *c, void *m) +{ + if (unlikely(c->cache_flags & SLAB_TYPESAFE_BY_RCU)) + lkpi_kmem_cache_free_rcu(c, m); + else if (unlikely(curthread->td_critnest != 0)) + lkpi_kmem_cache_free_async(c, m); + else + lkpi_kmem_cache_free_sync(c, m); +} + void linux_kmem_cache_destroy(struct linux_kmem_cache *c) { @@ -130,6 +198,9 @@ linux_kmem_cache_destroy(struct linux_kmem_cache *c) rcu_barrier(); } + if (!llist_empty(&c->cache_items)) + taskqueue_enqueue(linux_irq_work_tq, &c->cache_task); + taskqueue_drain(linux_irq_work_tq, &c->cache_task); uma_zdestroy(c->cache_zone); free(c, M_KMALLOC); }