Date: Wed, 15 Apr 2020 04:56:28 +0000 (UTC) From: Hans Petter Selasky <hselasky@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r359957 - in stable/12/sys: compat/linuxkpi/common/include/linux compat/linuxkpi/common/src sys Message-ID: <202004150456.03F4uST2015407@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: hselasky Date: Wed Apr 15 04:56:27 2020 New Revision: 359957 URL: https://svnweb.freebsd.org/changeset/base/359957 Log: MFC r359727: Clone the RCU interface into a sleepable and a non-sleepable part in the LinuxKPI. This allows synchronize RCU to be used inside a SRCU read section. No functional change intended. Bump the __FreeBSD_version to force recompilation of external kernel modules. PR: 242272 Sponsored by: Mellanox Technologies Modified: stable/12/sys/compat/linuxkpi/common/include/linux/rcupdate.h stable/12/sys/compat/linuxkpi/common/src/linux_rcu.c stable/12/sys/sys/param.h Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/compat/linuxkpi/common/include/linux/rcupdate.h ============================================================================== --- stable/12/sys/compat/linuxkpi/common/include/linux/rcupdate.h Wed Apr 15 04:52:59 2020 (r359956) +++ stable/12/sys/compat/linuxkpi/common/include/linux/rcupdate.h Wed Apr 15 04:56:27 2020 (r359957) @@ -35,6 +35,11 @@ #define LINUX_KFREE_RCU_OFFSET_MAX 4096 /* exclusive */ +/* BSD specific defines */ +#define RCU_TYPE_REGULAR 0 +#define RCU_TYPE_SLEEPABLE 1 +#define RCU_TYPE_MAX 2 + #define RCU_INITIALIZER(v) \ ((__typeof(*(v)) *)(v)) @@ -43,27 +48,27 @@ } while (0) #define call_rcu(ptr, func) do { \ - linux_call_rcu(ptr, func); \ + linux_call_rcu(RCU_TYPE_REGULAR, ptr, func); \ } while (0) #define rcu_barrier(void) do { \ - linux_rcu_barrier(); \ + linux_rcu_barrier(RCU_TYPE_REGULAR); \ } while (0) #define rcu_read_lock(void) do { \ - linux_rcu_read_lock(); \ + linux_rcu_read_lock(RCU_TYPE_REGULAR); \ } while (0) #define rcu_read_unlock(void) do { \ - linux_rcu_read_unlock(); \ + linux_rcu_read_unlock(RCU_TYPE_REGULAR);\ } while (0) #define synchronize_rcu(void) do { \ - linux_synchronize_rcu(); \ + linux_synchronize_rcu(RCU_TYPE_REGULAR); \ } while (0) #define synchronize_rcu_expedited(void) do { \ - linux_synchronize_rcu(); \ + linux_synchronize_rcu(RCU_TYPE_REGULAR); \ } while (0) #define kfree_rcu(ptr, rcu_head) do { \ @@ -94,11 +99,11 @@ /* prototypes */ -extern void linux_call_rcu(struct rcu_head *ptr, rcu_callback_t func); -extern void linux_rcu_barrier(void); -extern void linux_rcu_read_lock(void); -extern void linux_rcu_read_unlock(void); -extern void linux_synchronize_rcu(void); +extern void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func); +extern void linux_rcu_barrier(unsigned type); +extern void linux_rcu_read_lock(unsigned type); +extern void linux_rcu_read_unlock(unsigned type); +extern void linux_synchronize_rcu(unsigned type); /* Empty implementation for !DEBUG */ #define init_rcu_head(...) Modified: stable/12/sys/compat/linuxkpi/common/src/linux_rcu.c ============================================================================== --- stable/12/sys/compat/linuxkpi/common/src/linux_rcu.c Wed Apr 15 04:52:59 2020 (r359956) +++ stable/12/sys/compat/linuxkpi/common/src/linux_rcu.c Wed Apr 15 04:56:27 2020 (r359957) @@ -1,6 +1,6 @@ /*- * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io) - * Copyright (c) 2017 Hans Petter Selasky (hselasky@freebsd.org) + * Copyright (c) 2017-2020 Hans Petter Selasky (hselasky@freebsd.org) * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -90,9 +90,9 @@ CTASSERT(sizeof(struct rcu_head) == sizeof(struct call */ CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0); -static ck_epoch_t linux_epoch; -static struct linux_epoch_head linux_epoch_head; -DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record); +static ck_epoch_t linux_epoch[RCU_TYPE_MAX]; +static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX]; +DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]); static void linux_rcu_cleaner_func(void *, int); @@ -101,23 +101,27 @@ linux_rcu_runtime_init(void *arg __unused) { struct linux_epoch_head *head; int i; + int j; - ck_epoch_init(&linux_epoch); + for (j = 0; j != RCU_TYPE_MAX; j++) { + ck_epoch_init(&linux_epoch[j]); - head = &linux_epoch_head; + head = &linux_epoch_head[j]; - mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF); - TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, NULL); - STAILQ_INIT(&head->cb_head); + mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF); + TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head); + STAILQ_INIT(&head->cb_head); - CPU_FOREACH(i) { - struct linux_epoch_record *record; + CPU_FOREACH(i) { + struct linux_epoch_record *record; - record = &DPCPU_ID_GET(i, linux_epoch_record); + record = &DPCPU_ID_GET(i, linux_epoch_record[j]); - record->cpuid = i; - ck_epoch_register(&linux_epoch, &record->epoch_record, NULL); - TAILQ_INIT(&record->ts_head); + record->cpuid = i; + ck_epoch_register(&linux_epoch[j], + &record->epoch_record, NULL); + TAILQ_INIT(&record->ts_head); + } } } SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL); @@ -126,24 +130,27 @@ static void linux_rcu_runtime_uninit(void *arg __unused) { struct linux_epoch_head *head; + int j; - head = &linux_epoch_head; + for (j = 0; j != RCU_TYPE_MAX; j++) { + head = &linux_epoch_head[j]; - /* destroy head lock */ - mtx_destroy(&head->lock); + mtx_destroy(&head->lock); + } } SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL); static void -linux_rcu_cleaner_func(void *context __unused, int pending __unused) +linux_rcu_cleaner_func(void *context, int pending __unused) { struct linux_epoch_head *head; struct callback_head *rcu; STAILQ_HEAD(, callback_head) tmp_head; + uintptr_t offset; linux_set_current(curthread); - head = &linux_epoch_head; + head = context; /* move current callbacks into own queue */ mtx_lock(&head->lock); @@ -152,11 +159,10 @@ linux_rcu_cleaner_func(void *context __unused, int pen mtx_unlock(&head->lock); /* synchronize */ - linux_synchronize_rcu(); + linux_synchronize_rcu(head - linux_epoch_head); /* dispatch all callbacks, if any */ while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) { - uintptr_t offset; STAILQ_REMOVE_HEAD(&tmp_head, entry); @@ -170,11 +176,13 @@ linux_rcu_cleaner_func(void *context __unused, int pen } void -linux_rcu_read_lock(void) +linux_rcu_read_lock(unsigned type) { struct linux_epoch_record *record; struct task_struct *ts; + MPASS(type < RCU_TYPE_MAX); + if (RCU_SKIP()) return; @@ -184,7 +192,7 @@ linux_rcu_read_lock(void) */ sched_pin(); - record = &DPCPU_GET(linux_epoch_record); + record = &DPCPU_GET(linux_epoch_record[type]); ts = current; /* @@ -200,15 +208,17 @@ linux_rcu_read_lock(void) } void -linux_rcu_read_unlock(void) +linux_rcu_read_unlock(unsigned type) { struct linux_epoch_record *record; struct task_struct *ts; + MPASS(type < RCU_TYPE_MAX); + if (RCU_SKIP()) return; - record = &DPCPU_GET(linux_epoch_record); + record = &DPCPU_GET(linux_epoch_record[type]); ts = current; /* @@ -282,7 +292,7 @@ linux_synchronize_rcu_cb(ck_epoch_t *epoch __unused, c } void -linux_synchronize_rcu(void) +linux_synchronize_rcu(unsigned type) { struct thread *td; int was_bound; @@ -290,6 +300,8 @@ linux_synchronize_rcu(void) int old_pinned; u_char old_prio; + MPASS(type < RCU_TYPE_MAX); + if (RCU_SKIP()) return; @@ -313,7 +325,7 @@ linux_synchronize_rcu(void) td->td_pinned = 0; sched_bind(td, old_cpu); - ck_epoch_synchronize_wait(&linux_epoch, + ck_epoch_synchronize_wait(&linux_epoch[type], &linux_synchronize_rcu_cb, NULL); /* restore CPU binding, if any */ @@ -336,24 +348,31 @@ linux_synchronize_rcu(void) } void -linux_rcu_barrier(void) +linux_rcu_barrier(unsigned type) { struct linux_epoch_head *head; - linux_synchronize_rcu(); + MPASS(type < RCU_TYPE_MAX); - head = &linux_epoch_head; + linux_synchronize_rcu(type); + head = &linux_epoch_head[type]; + /* wait for callbacks to complete */ taskqueue_drain(taskqueue_fast, &head->task); } void -linux_call_rcu(struct rcu_head *context, rcu_callback_t func) +linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func) { - struct callback_head *rcu = (struct callback_head *)context; - struct linux_epoch_head *head = &linux_epoch_head; + struct callback_head *rcu; + struct linux_epoch_head *head; + MPASS(type < RCU_TYPE_MAX); + + rcu = (struct callback_head *)context; + head = &linux_epoch_head[type]; + mtx_lock(&head->lock); rcu->func = func; STAILQ_INSERT_TAIL(&head->cb_head, rcu, entry); @@ -375,24 +394,24 @@ cleanup_srcu_struct(struct srcu_struct *srcu) int srcu_read_lock(struct srcu_struct *srcu) { - linux_rcu_read_lock(); + linux_rcu_read_lock(RCU_TYPE_SLEEPABLE); return (0); } void srcu_read_unlock(struct srcu_struct *srcu, int key __unused) { - linux_rcu_read_unlock(); + linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE); } void synchronize_srcu(struct srcu_struct *srcu) { - linux_synchronize_rcu(); + linux_synchronize_rcu(RCU_TYPE_SLEEPABLE); } void srcu_barrier(struct srcu_struct *srcu) { - linux_rcu_barrier(); + linux_rcu_barrier(RCU_TYPE_SLEEPABLE); } Modified: stable/12/sys/sys/param.h ============================================================================== --- stable/12/sys/sys/param.h Wed Apr 15 04:52:59 2020 (r359956) +++ stable/12/sys/sys/param.h Wed Apr 15 04:56:27 2020 (r359957) @@ -60,7 +60,7 @@ * in the range 5 to 9. */ #undef __FreeBSD_version -#define __FreeBSD_version 1201512 /* Master, propagated to newvers */ +#define __FreeBSD_version 1201513 /* Master, propagated to newvers */ /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202004150456.03F4uST2015407>