diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt
index 742f69d18fc8989ae28d9c0662d6bf334109dddd..7673943d30850c2e70f4ebd27596df8a2c8388be 100644
--- a/Documentation/kernel-parameters.txt
+++ b/Documentation/kernel-parameters.txt
@@ -3296,6 +3296,12 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
rcutorture.verbose= [KNL]
Enable additional printk() statements.
+ rcupdate.rcu_cpu_stall_suppress= [KNL]
+ Suppress RCU CPU stall warning messages.
+
+ rcupdate.rcu_cpu_stall_timeout= [KNL]
+ Set timeout for RCU CPU stall warning messages.
+
rcupdate.rcu_expedited= [KNL]
Use expedited grace-period primitives, for
example, synchronize_rcu_expedited() instead
@@ -3303,11 +3309,14 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
but can increase CPU utilization, degrade
real-time latency, and degrade energy efficiency.
- rcupdate.rcu_cpu_stall_suppress= [KNL]
- Suppress RCU CPU stall warning messages.
-
- rcupdate.rcu_cpu_stall_timeout= [KNL]
- Set timeout for RCU CPU stall warning messages.
+ rcupdate.rcu_normal= [KNL]
+ Use only normal grace-period primitives,
+ for example, synchronize_rcu() instead of
+ synchronize_rcu_expedited(). This improves
+ real-time latency, CPU utilization, and energy
+ efficiency, but can expose users to increased
+ grace-period latency. This parameter overrides
+ rcupdate.rcu_expedited.
rcupdate.rcu_task_stall_timeout= [KNL]
Set timeout in jiffies for RCU task stall warning
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index a0189ba67fde721a824cc7c8db4b1f015197059e..98d9f30c02d43ab41154b110795d8bfb8449fd3b 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -49,9 +49,14 @@
#include <asm/barrier.h>
extern int rcu_expedited; /* for sysctl */
+extern int rcu_normal; /* also for sysctl */
#ifdef CONFIG_TINY_RCU
/* Tiny RCU doesn't expedite, as its purpose in life is instead to be tiny. */
+static inline bool rcu_gp_is_normal(void) /* Internal RCU use. */
+{
+ return true;
+}
static inline bool rcu_gp_is_expedited(void) /* Internal RCU use. */
{
return false;
@@ -65,6 +70,7 @@ static inline void rcu_unexpedite_gp(void)
{
}
#else /* #ifdef CONFIG_TINY_RCU */
+bool rcu_gp_is_normal(void); /* Internal RCU use. */
bool rcu_gp_is_expedited(void); /* Internal RCU use. */
void rcu_expedite_gp(void);
void rcu_unexpedite_gp(void);
diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c
index e83b264640615c47c31cce539f31014dc11b0776..b4e2fa52d8bc4feb4feaace94da14aae0b1ed23a 100644
--- a/kernel/ksysfs.c
+++ b/kernel/ksysfs.c
@@ -20,7 +20,7 @@
#include <linux/capability.h>
#include <linux/compiler.h>
-#include <linux/rcupdate.h> /* rcu_expedited */
+#include <linux/rcupdate.h> /* rcu_expedited and rcu_normal */
#define KERNEL_ATTR_RO(_name) \
static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
@@ -148,7 +148,7 @@ int rcu_expedited;
static ssize_t rcu_expedited_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
- return sprintf(buf, "%d\n", rcu_expedited);
+ return sprintf(buf, "%d\n", READ_ONCE(rcu_expedited));
}
static ssize_t rcu_expedited_store(struct kobject *kobj,
struct kobj_attribute *attr,
@@ -161,6 +161,23 @@ static ssize_t rcu_expedited_store(struct kobject *kobj,
}
KERNEL_ATTR_RW(rcu_expedited);
+int rcu_normal;
+static ssize_t rcu_normal_show(struct kobject *kobj,
+ struct kobj_attribute *attr, char *buf)
+{
+ return sprintf(buf, "%d\n", READ_ONCE(rcu_normal));
+}
+static ssize_t rcu_normal_store(struct kobject *kobj,
+ struct kobj_attribute *attr,
+ const char *buf, size_t count)
+{
+ if (kstrtoint(buf, 0, &rcu_normal))
+ return -EINVAL;
+
+ return count;
+}
+KERNEL_ATTR_RW(rcu_normal);
+
/*
* Make /sys/kernel/notes give the raw contents of our kernel .notes section.
*/
@@ -203,6 +220,7 @@ static struct attribute * kernel_attrs[] = {
&vmcoreinfo_attr.attr,
#endif
&rcu_expedited_attr.attr,
+ &rcu_normal_attr.attr,
NULL
};
diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c
index a63a1ea5a41bf450f6b162674149a5928fca3530..9b9cdd549caa848111247b35b19109da7af9099d 100644
--- a/kernel/rcu/srcu.c
+++ b/kernel/rcu/srcu.c
@@ -489,7 +489,7 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
*/
void synchronize_srcu(struct srcu_struct *sp)
{
- __synchronize_srcu(sp, rcu_gp_is_expedited()
+ __synchronize_srcu(sp, (rcu_gp_is_expedited() && !rcu_gp_is_normal())
? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
: SYNCHRONIZE_SRCU_TRYCOUNT);
}
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 6a652d1f3d7f88e3ea9aae5fdadd0d6a5a2b4522..489992997c06eb543a7fb1d97236661bea009530 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3841,6 +3841,12 @@ void synchronize_sched_expedited(void)
if (rcu_blocking_is_gp())
return;
+ /* If expedited grace periods are prohibited, fall back to normal. */
+ if (rcu_gp_is_normal()) {
+ wait_rcu_gp(call_rcu_sched);
+ return;
+ }
+
/* Take a snapshot of the sequence number. */
s = rcu_exp_gp_seq_snap(rsp);
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 57ba873d2f180066ef2d9b8bbd5d333297679c4d..d45df378155166a08c4b44e492ff9f98651b4b74 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -746,6 +746,12 @@ void synchronize_rcu_expedited(void)
struct rcu_state *rsp = rcu_state_p;
unsigned long s;
+ /* If expedited grace periods are prohibited, fall back to normal. */
+ if (rcu_gp_is_normal()) {
+ wait_rcu_gp(call_rcu);
+ return;
+ }
+
s = rcu_exp_gp_seq_snap(rsp);
rnp_unlock = exp_funnel_lock(rsp, s);
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index 5f748c5a40f0756b2d6c97fa615cde551649db4a..8fccda3a794d9069d78dee99a74857af4b1ede95 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -61,6 +61,7 @@ MODULE_ALIAS("rcupdate");
#define MODULE_PARAM_PREFIX "rcupdate."
module_param(rcu_expedited, int, 0);
+module_param(rcu_normal, int, 0);
#if defined(CONFIG_DEBUG_LOCK_ALLOC) && defined(CONFIG_PREEMPT_COUNT)
/**
@@ -113,6 +114,17 @@ EXPORT_SYMBOL(rcu_read_lock_sched_held);
#ifndef CONFIG_TINY_RCU
+/*
+ * Should expedited grace-period primitives always fall back to their
+ * non-expedited counterparts? Intended for use within RCU. Note
+ * that if the user specifies both rcu_expedited and rcu_normal, then
+ * rcu_normal wins.
+ */
+bool rcu_gp_is_normal(void)
+{
+ return READ_ONCE(rcu_normal);
+}
+
static atomic_t rcu_expedited_nesting =
ATOMIC_INIT(IS_ENABLED(CONFIG_RCU_EXPEDITE_BOOT) ? 1 : 0);