Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 15 Apr 2020 04:56:28 +0000 (UTC)
From:      Hans Petter Selasky <hselasky@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org
Subject:   svn commit: r359957 - in stable/12/sys: compat/linuxkpi/common/include/linux compat/linuxkpi/common/src sys
Message-ID:  <202004150456.03F4uST2015407@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: hselasky
Date: Wed Apr 15 04:56:27 2020
New Revision: 359957
URL: https://svnweb.freebsd.org/changeset/base/359957

Log:
  MFC r359727:
  Clone the RCU interface into a sleepable and a non-sleepable part
  in the LinuxKPI.
  
  This allows synchronize RCU to be used inside a SRCU read section.
  No functional change intended.
  
  Bump the __FreeBSD_version to force recompilation of external kernel modules.
  
  PR:		242272
  Sponsored by:	Mellanox Technologies

Modified:
  stable/12/sys/compat/linuxkpi/common/include/linux/rcupdate.h
  stable/12/sys/compat/linuxkpi/common/src/linux_rcu.c
  stable/12/sys/sys/param.h
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/compat/linuxkpi/common/include/linux/rcupdate.h
==============================================================================
--- stable/12/sys/compat/linuxkpi/common/include/linux/rcupdate.h	Wed Apr 15 04:52:59 2020	(r359956)
+++ stable/12/sys/compat/linuxkpi/common/include/linux/rcupdate.h	Wed Apr 15 04:56:27 2020	(r359957)
@@ -35,6 +35,11 @@
 
 #define	LINUX_KFREE_RCU_OFFSET_MAX	4096	/* exclusive */
 
+/* BSD specific defines */
+#define	RCU_TYPE_REGULAR 0
+#define	RCU_TYPE_SLEEPABLE 1
+#define	RCU_TYPE_MAX 2
+
 #define	RCU_INITIALIZER(v)			\
 	((__typeof(*(v)) *)(v))
 
@@ -43,27 +48,27 @@
 } while (0)
 
 #define	call_rcu(ptr, func) do {		\
-	linux_call_rcu(ptr, func);		\
+	linux_call_rcu(RCU_TYPE_REGULAR, ptr, func);	\
 } while (0)
 
 #define	rcu_barrier(void) do {			\
-	linux_rcu_barrier();			\
+	linux_rcu_barrier(RCU_TYPE_REGULAR);	\
 } while (0)
 
 #define	rcu_read_lock(void) do {		\
-	linux_rcu_read_lock();			\
+	linux_rcu_read_lock(RCU_TYPE_REGULAR);	\
 } while (0)
 
 #define	rcu_read_unlock(void) do {		\
-	linux_rcu_read_unlock();		\
+	linux_rcu_read_unlock(RCU_TYPE_REGULAR);\
 } while (0)
 
 #define	synchronize_rcu(void) do {	\
-	linux_synchronize_rcu();	\
+	linux_synchronize_rcu(RCU_TYPE_REGULAR);	\
 } while (0)
 
 #define	synchronize_rcu_expedited(void) do {	\
-	linux_synchronize_rcu();		\
+	linux_synchronize_rcu(RCU_TYPE_REGULAR);	\
 } while (0)
 
 #define	kfree_rcu(ptr, rcu_head) do {				\
@@ -94,11 +99,11 @@
 
 /* prototypes */
 
-extern void linux_call_rcu(struct rcu_head *ptr, rcu_callback_t func);
-extern void linux_rcu_barrier(void);
-extern void linux_rcu_read_lock(void);
-extern void linux_rcu_read_unlock(void);
-extern void linux_synchronize_rcu(void);
+extern void linux_call_rcu(unsigned type, struct rcu_head *ptr, rcu_callback_t func);
+extern void linux_rcu_barrier(unsigned type);
+extern void linux_rcu_read_lock(unsigned type);
+extern void linux_rcu_read_unlock(unsigned type);
+extern void linux_synchronize_rcu(unsigned type);
 
 /* Empty implementation for !DEBUG */
 #define	init_rcu_head(...)

Modified: stable/12/sys/compat/linuxkpi/common/src/linux_rcu.c
==============================================================================
--- stable/12/sys/compat/linuxkpi/common/src/linux_rcu.c	Wed Apr 15 04:52:59 2020	(r359956)
+++ stable/12/sys/compat/linuxkpi/common/src/linux_rcu.c	Wed Apr 15 04:56:27 2020	(r359957)
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2016 Matthew Macy (mmacy@mattmacy.io)
- * Copyright (c) 2017 Hans Petter Selasky (hselasky@freebsd.org)
+ * Copyright (c) 2017-2020 Hans Petter Selasky (hselasky@freebsd.org)
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -90,9 +90,9 @@ CTASSERT(sizeof(struct rcu_head) == sizeof(struct call
  */
 CTASSERT(offsetof(struct linux_epoch_record, epoch_record) == 0);
 
-static ck_epoch_t linux_epoch;
-static struct linux_epoch_head linux_epoch_head;
-DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record);
+static ck_epoch_t linux_epoch[RCU_TYPE_MAX];
+static struct linux_epoch_head linux_epoch_head[RCU_TYPE_MAX];
+DPCPU_DEFINE_STATIC(struct linux_epoch_record, linux_epoch_record[RCU_TYPE_MAX]);
 
 static void linux_rcu_cleaner_func(void *, int);
 
@@ -101,23 +101,27 @@ linux_rcu_runtime_init(void *arg __unused)
 {
 	struct linux_epoch_head *head;
 	int i;
+	int j;
 
-	ck_epoch_init(&linux_epoch);
+	for (j = 0; j != RCU_TYPE_MAX; j++) {
+		ck_epoch_init(&linux_epoch[j]);
 
-	head = &linux_epoch_head;
+		head = &linux_epoch_head[j];
 
-	mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF);
-	TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, NULL);
-	STAILQ_INIT(&head->cb_head);
+		mtx_init(&head->lock, "LRCU-HEAD", NULL, MTX_DEF);
+		TASK_INIT(&head->task, 0, linux_rcu_cleaner_func, head);
+		STAILQ_INIT(&head->cb_head);
 
-	CPU_FOREACH(i) {
-		struct linux_epoch_record *record;
+		CPU_FOREACH(i) {
+			struct linux_epoch_record *record;
 
-		record = &DPCPU_ID_GET(i, linux_epoch_record);
+			record = &DPCPU_ID_GET(i, linux_epoch_record[j]);
 
-		record->cpuid = i;
-		ck_epoch_register(&linux_epoch, &record->epoch_record, NULL);
-		TAILQ_INIT(&record->ts_head);
+			record->cpuid = i;
+			ck_epoch_register(&linux_epoch[j],
+			    &record->epoch_record, NULL);
+			TAILQ_INIT(&record->ts_head);
+		}
 	}
 }
 SYSINIT(linux_rcu_runtime, SI_SUB_CPU, SI_ORDER_ANY, linux_rcu_runtime_init, NULL);
@@ -126,24 +130,27 @@ static void
 linux_rcu_runtime_uninit(void *arg __unused)
 {
 	struct linux_epoch_head *head;
+	int j;
 
-	head = &linux_epoch_head;
+	for (j = 0; j != RCU_TYPE_MAX; j++) {
+		head = &linux_epoch_head[j];
 
-	/* destroy head lock */
-	mtx_destroy(&head->lock);
+		mtx_destroy(&head->lock);
+	}
 }
 SYSUNINIT(linux_rcu_runtime, SI_SUB_LOCK, SI_ORDER_SECOND, linux_rcu_runtime_uninit, NULL);
 
 static void
-linux_rcu_cleaner_func(void *context __unused, int pending __unused)
+linux_rcu_cleaner_func(void *context, int pending __unused)
 {
 	struct linux_epoch_head *head;
 	struct callback_head *rcu;
 	STAILQ_HEAD(, callback_head) tmp_head;
+	uintptr_t offset;
 
 	linux_set_current(curthread);
 
-	head = &linux_epoch_head;
+	head = context;
 
 	/* move current callbacks into own queue */
 	mtx_lock(&head->lock);
@@ -152,11 +159,10 @@ linux_rcu_cleaner_func(void *context __unused, int pen
 	mtx_unlock(&head->lock);
 
 	/* synchronize */
-	linux_synchronize_rcu();
+	linux_synchronize_rcu(head - linux_epoch_head);
 
 	/* dispatch all callbacks, if any */
 	while ((rcu = STAILQ_FIRST(&tmp_head)) != NULL) {
-		uintptr_t offset;
 
 		STAILQ_REMOVE_HEAD(&tmp_head, entry);
 
@@ -170,11 +176,13 @@ linux_rcu_cleaner_func(void *context __unused, int pen
 }
 
 void
-linux_rcu_read_lock(void)
+linux_rcu_read_lock(unsigned type)
 {
 	struct linux_epoch_record *record;
 	struct task_struct *ts;
 
+	MPASS(type < RCU_TYPE_MAX);
+
 	if (RCU_SKIP())
 		return;
 
@@ -184,7 +192,7 @@ linux_rcu_read_lock(void)
 	 */
 	sched_pin();
 
-	record = &DPCPU_GET(linux_epoch_record);
+	record = &DPCPU_GET(linux_epoch_record[type]);
 	ts = current;
 
 	/*
@@ -200,15 +208,17 @@ linux_rcu_read_lock(void)
 }
 
 void
-linux_rcu_read_unlock(void)
+linux_rcu_read_unlock(unsigned type)
 {
 	struct linux_epoch_record *record;
 	struct task_struct *ts;
 
+	MPASS(type < RCU_TYPE_MAX);
+
 	if (RCU_SKIP())
 		return;
 
-	record = &DPCPU_GET(linux_epoch_record);
+	record = &DPCPU_GET(linux_epoch_record[type]);
 	ts = current;
 
 	/*
@@ -282,7 +292,7 @@ linux_synchronize_rcu_cb(ck_epoch_t *epoch __unused, c
 }
 
 void
-linux_synchronize_rcu(void)
+linux_synchronize_rcu(unsigned type)
 {
 	struct thread *td;
 	int was_bound;
@@ -290,6 +300,8 @@ linux_synchronize_rcu(void)
 	int old_pinned;
 	u_char old_prio;
 
+	MPASS(type < RCU_TYPE_MAX);
+
 	if (RCU_SKIP())
 		return;
 
@@ -313,7 +325,7 @@ linux_synchronize_rcu(void)
 	td->td_pinned = 0;
 	sched_bind(td, old_cpu);
 
-	ck_epoch_synchronize_wait(&linux_epoch,
+	ck_epoch_synchronize_wait(&linux_epoch[type],
 	    &linux_synchronize_rcu_cb, NULL);
 
 	/* restore CPU binding, if any */
@@ -336,24 +348,31 @@ linux_synchronize_rcu(void)
 }
 
 void
-linux_rcu_barrier(void)
+linux_rcu_barrier(unsigned type)
 {
 	struct linux_epoch_head *head;
 
-	linux_synchronize_rcu();
+	MPASS(type < RCU_TYPE_MAX);
 
-	head = &linux_epoch_head;
+	linux_synchronize_rcu(type);
 
+	head = &linux_epoch_head[type];
+
 	/* wait for callbacks to complete */
 	taskqueue_drain(taskqueue_fast, &head->task);
 }
 
 void
-linux_call_rcu(struct rcu_head *context, rcu_callback_t func)
+linux_call_rcu(unsigned type, struct rcu_head *context, rcu_callback_t func)
 {
-	struct callback_head *rcu = (struct callback_head *)context;
-	struct linux_epoch_head *head = &linux_epoch_head;
+	struct callback_head *rcu;
+	struct linux_epoch_head *head;
 
+	MPASS(type < RCU_TYPE_MAX);
+
+	rcu = (struct callback_head *)context;
+	head = &linux_epoch_head[type];
+
 	mtx_lock(&head->lock);
 	rcu->func = func;
 	STAILQ_INSERT_TAIL(&head->cb_head, rcu, entry);
@@ -375,24 +394,24 @@ cleanup_srcu_struct(struct srcu_struct *srcu)
 int
 srcu_read_lock(struct srcu_struct *srcu)
 {
-	linux_rcu_read_lock();
+	linux_rcu_read_lock(RCU_TYPE_SLEEPABLE);
 	return (0);
 }
 
 void
 srcu_read_unlock(struct srcu_struct *srcu, int key __unused)
 {
-	linux_rcu_read_unlock();
+	linux_rcu_read_unlock(RCU_TYPE_SLEEPABLE);
 }
 
 void
 synchronize_srcu(struct srcu_struct *srcu)
 {
-	linux_synchronize_rcu();
+	linux_synchronize_rcu(RCU_TYPE_SLEEPABLE);
 }
 
 void
 srcu_barrier(struct srcu_struct *srcu)
 {
-	linux_rcu_barrier();
+	linux_rcu_barrier(RCU_TYPE_SLEEPABLE);
 }

Modified: stable/12/sys/sys/param.h
==============================================================================
--- stable/12/sys/sys/param.h	Wed Apr 15 04:52:59 2020	(r359956)
+++ stable/12/sys/sys/param.h	Wed Apr 15 04:56:27 2020	(r359957)
@@ -60,7 +60,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1201512	/* Master, propagated to newvers */
+#define __FreeBSD_version 1201513	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202004150456.03F4uST2015407>