diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 0ac817b750dbc7a15f0c69c21ed184d5882e1696..6e43fa355e7127e8e2b10ff33eee5c0ab43ccf90 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -9,6 +9,10 @@
*
* Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
* Nauman Rafique <nauman@google.com>
+ *
+ * For policy-specific per-blkcg data:
+ * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
+ * Arianna Avanzini <avanzini.arianna@gmail.com>
*/
#include <linux/ioprio.h>
#include <linux/kdev_t.h>
@@ -26,8 +30,7 @@
static DEFINE_MUTEX(blkcg_pol_mutex);
-struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT,
- .cfq_leaf_weight = 2 * CFQ_WEIGHT_DEFAULT, };
+struct blkcg blkcg_root;
EXPORT_SYMBOL_GPL(blkcg_root);
static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
@@ -823,6 +826,8 @@ static struct cgroup_subsys_state *
blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
{
struct blkcg *blkcg;
+ struct cgroup_subsys_state *ret;
+ int i;
if (!parent_css) {
blkcg = &blkcg_root;
@@ -830,17 +835,49 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
}
blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
- if (!blkcg)
- return ERR_PTR(-ENOMEM);
+ if (!blkcg) {
+ ret = ERR_PTR(-ENOMEM);
+ goto free_blkcg;
+ }
+
+ for (i = 0; i < BLKCG_MAX_POLS ; i++) {
+ struct blkcg_policy *pol = blkcg_policy[i];
+ struct blkcg_policy_data *cpd;
+
+ /*
+ * If the policy hasn't been attached yet, wait for it
+ * to be attached before doing anything else. Otherwise,
+ * check if the policy requires any specific per-cgroup
+ * data: if it does, allocate and initialize it.
+ */
+ if (!pol || !pol->cpd_size)
+ continue;
+
+ BUG_ON(blkcg->pd[i]);
+ cpd = kzalloc(pol->cpd_size, GFP_KERNEL);
+ if (!cpd) {
+ ret = ERR_PTR(-ENOMEM);
+ goto free_pd_blkcg;
+ }
+ blkcg->pd[i] = cpd;
+ cpd->plid = i;
+ pol->cpd_init_fn(blkcg);
+ }
- blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT;
- blkcg->cfq_leaf_weight = CFQ_WEIGHT_DEFAULT;
done:
spin_lock_init(&blkcg->lock);
INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC);
INIT_HLIST_HEAD(&blkcg->blkg_list);
return &blkcg->css;
+
+free_pd_blkcg:
+ for (i--; i >= 0; i--)
+ kfree(blkcg->pd[i]);
+
+free_blkcg:
+ kfree(blkcg);
+ return ret;
}
/**
@@ -958,8 +995,10 @@ int blkcg_activate_policy(struct request_queue *q,
const struct blkcg_policy *pol)
{
LIST_HEAD(pds);
+ LIST_HEAD(cpds);
struct blkcg_gq *blkg, *new_blkg;
- struct blkg_policy_data *pd, *n;
+ struct blkg_policy_data *pd, *nd;
+ struct blkcg_policy_data *cpd, *cnd;
int cnt = 0, ret;
bool preloaded;
@@ -1003,7 +1042,10 @@ int blkcg_activate_policy(struct request_queue *q,
spin_unlock_irq(q->queue_lock);
- /* allocate policy_data for all existing blkgs */
+ /*
+ * Allocate per-blkg and per-blkcg policy data
+ * for all existing blkgs.
+ */
while (cnt--) {
pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node);
if (!pd) {
@@ -1011,26 +1053,50 @@ int blkcg_activate_policy(struct request_queue *q,
goto out_free;
}
list_add_tail(&pd->alloc_node, &pds);
+
+ if (!pol->cpd_size)
+ continue;
+ cpd = kzalloc_node(pol->cpd_size, GFP_KERNEL, q->node);
+ if (!cpd) {
+ ret = -ENOMEM;
+ goto out_free;
+ }
+ list_add_tail(&cpd->alloc_node, &cpds);
}
/*
- * Install the allocated pds. With @q bypassing, no new blkg
+ * Install the allocated pds and cpds. With @q bypassing, no new blkg
* should have been created while the queue lock was dropped.
*/
spin_lock_irq(q->queue_lock);
list_for_each_entry(blkg, &q->blkg_list, q_node) {
- if (WARN_ON(list_empty(&pds))) {
+ if (WARN_ON(list_empty(&pds)) ||
+ WARN_ON(pol->cpd_size && list_empty(&cpds))) {
/* umm... this shouldn't happen, just abort */
ret = -ENOMEM;
goto out_unlock;
}
+ cpd = list_first_entry(&cpds, struct blkcg_policy_data,
+ alloc_node);
+ list_del_init(&cpd->alloc_node);
pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node);
list_del_init(&pd->alloc_node);
/* grab blkcg lock too while installing @pd on @blkg */
spin_lock(&blkg->blkcg->lock);
+ if (!pol->cpd_size)
+ goto no_cpd;
+ if (!blkg->blkcg->pd[pol->plid]) {
+ /* Per-policy per-blkcg data */
+ blkg->blkcg->pd[pol->plid] = cpd;
+ cpd->plid = pol->plid;
+ pol->cpd_init_fn(blkg->blkcg);
+ } else { /* must free it as it has already been extracted */
+ kfree(cpd);
+ }
+no_cpd:
blkg->pd[pol->plid] = pd;
pd->blkg = blkg;
pd->plid = pol->plid;
@@ -1045,8 +1111,10 @@ int blkcg_activate_policy(struct request_queue *q,
spin_unlock_irq(q->queue_lock);
out_free:
blk_queue_bypass_end(q);
- list_for_each_entry_safe(pd, n, &pds, alloc_node)
+ list_for_each_entry_safe(pd, nd, &pds, alloc_node)
kfree(pd);
+ list_for_each_entry_safe(cpd, cnd, &cpds, alloc_node)
+ kfree(cpd);
return ret;
}
EXPORT_SYMBOL_GPL(blkcg_activate_policy);
@@ -1087,6 +1155,8 @@ void blkcg_deactivate_policy(struct request_queue *q,
kfree(blkg->pd[pol->plid]);
blkg->pd[pol->plid] = NULL;
+ kfree(blkg->blkcg->pd[pol->plid]);
+ blkg->blkcg->pd[pol->plid] = NULL;
spin_unlock(&blkg->blkcg->lock);
}
diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h
index c567865b5f1df6baea2cdabf89663994a67e8ade..74296a78bba16896b5eb8c65a7e618163b766f9f 100644
--- a/block/blk-cgroup.h
+++ b/block/blk-cgroup.h
@@ -23,11 +23,6 @@
/* Max limits for throttle policy */
#define THROTL_IOPS_MAX UINT_MAX
-/* CFQ specific, out here for blkcg->cfq_weight */
-#define CFQ_WEIGHT_MIN 10
-#define CFQ_WEIGHT_MAX 1000
-#define CFQ_WEIGHT_DEFAULT 500
-
#ifdef CONFIG_BLK_CGROUP
enum blkg_rwstat_type {
@@ -50,9 +45,7 @@ struct blkcg {
struct blkcg_gq *blkg_hint;
struct hlist_head blkg_list;
- /* TODO: per-policy storage in blkcg */
- unsigned int cfq_weight; /* belongs to cfq */
- unsigned int cfq_leaf_weight;
+ struct blkcg_policy_data *pd[BLKCG_MAX_POLS];
};
struct blkg_stat {
@@ -87,6 +80,24 @@ struct blkg_policy_data {
struct list_head alloc_node;
};
+/*
+ * Policies that need to keep per-blkcg data which is independent
+ * from any request_queue associated to it must specify its size
+ * with the cpd_size field of the blkcg_policy structure and
+ * embed a blkcg_policy_data in it. blkcg core allocates
+ * policy-specific per-blkcg structures lazily the first time
+ * they are actually needed, so it handles them together with
+ * blkgs. cpd_init() is invoked to let each policy handle
+ * per-blkcg data.
+ */
+struct blkcg_policy_data {
+ /* the policy id this per-policy data belongs to */
+ int plid;
+
+ /* used during policy activation */
+ struct list_head alloc_node;
+};
+
/* association between a blk cgroup and a request queue */
struct blkcg_gq {
/* Pointer to the associated request_queue */
@@ -112,6 +123,7 @@ struct blkcg_gq {
struct rcu_head rcu_head;
};
+typedef void (blkcg_pol_init_cpd_fn)(const struct blkcg *blkcg);
typedef void (blkcg_pol_init_pd_fn)(struct blkcg_gq *blkg);
typedef void (blkcg_pol_online_pd_fn)(struct blkcg_gq *blkg);
typedef void (blkcg_pol_offline_pd_fn)(struct blkcg_gq *blkg);
@@ -122,10 +134,13 @@ struct blkcg_policy {
int plid;
/* policy specific private data size */
size_t pd_size;
+ /* policy specific per-blkcg data size */
+ size_t cpd_size;
/* cgroup files for the policy */
struct cftype *cftypes;
/* operations */
+ blkcg_pol_init_cpd_fn *cpd_init_fn;
blkcg_pol_init_pd_fn *pd_init_fn;
blkcg_pol_online_pd_fn *pd_online_fn;
blkcg_pol_offline_pd_fn *pd_offline_fn;
@@ -218,6 +233,12 @@ static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
return blkg ? blkg->pd[pol->plid] : NULL;
}
+static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg,
+ struct blkcg_policy *pol)
+{
+ return blkcg ? blkcg->pd[pol->plid] : NULL;
+}
+
/**
* pdata_to_blkg - get blkg associated with policy private data
* @pd: policy private data of interest
@@ -564,6 +585,9 @@ struct blkcg;
struct blkg_policy_data {
};
+struct blkcg_policy_data {
+};
+
struct blkcg_gq {
};
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 402be013912203a80df8b6560cec565b78aa8d28..c808ad87652d2818f7770cc6abd59f02110ab4c8 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -67,6 +67,11 @@ static struct kmem_cache *cfq_pool;
#define sample_valid(samples) ((samples) > 80)
#define rb_entry_cfqg(node) rb_entry((node), struct cfq_group, rb_node)
+/* blkio-related constants */
+#define CFQ_WEIGHT_MIN 10
+#define CFQ_WEIGHT_MAX 1000
+#define CFQ_WEIGHT_DEFAULT 500
+
struct cfq_ttime {
unsigned long last_end_request;
@@ -212,6 +217,15 @@ struct cfqg_stats {
#endif /* CONFIG_CFQ_GROUP_IOSCHED */
};
+/* Per-cgroup data */
+struct cfq_group_data {
+ /* must be the first member */
+ struct blkcg_policy_data pd;
+
+ unsigned int weight;
+ unsigned int leaf_weight;
+};
+
/* This is per cgroup per device grouping structure */
struct cfq_group {
/* must be the first member */
@@ -451,6 +465,12 @@ static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd)
return pd ? container_of(pd, struct cfq_group, pd) : NULL;
}
+static struct cfq_group_data
+*cpd_to_cfqgd(struct blkcg_policy_data *cpd)
+{
+ return cpd ? container_of(cpd, struct cfq_group_data, pd) : NULL;
+}
+
static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg)
{
return pd_to_blkg(&cfqg->pd);
@@ -607,6 +627,11 @@ static inline struct cfq_group *blkg_to_cfqg(struct blkcg_gq *blkg)
return pd_to_cfqg(blkg_to_pd(blkg, &blkcg_policy_cfq));
}
+static struct cfq_group_data *blkcg_to_cfqgd(struct blkcg *blkcg)
+{
+ return cpd_to_cfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_cfq));
+}
+
static inline struct cfq_group *cfqg_parent(struct cfq_group *cfqg)
{
struct blkcg_gq *pblkg = cfqg_to_blkg(cfqg)->parent;
@@ -1544,13 +1569,28 @@ static void cfqg_stats_init(struct cfqg_stats *stats)
#endif
}
+static void cfq_cpd_init(const struct blkcg *blkcg)
+{
+ struct cfq_group_data *cgd =
+ cpd_to_cfqgd(blkcg->pd[blkcg_policy_cfq.plid]);
+
+ if (blkcg == &blkcg_root) {
+ cgd->weight = 2 * CFQ_WEIGHT_DEFAULT;
+ cgd->leaf_weight = 2 * CFQ_WEIGHT_DEFAULT;
+ } else {
+ cgd->weight = CFQ_WEIGHT_DEFAULT;
+ cgd->leaf_weight = CFQ_WEIGHT_DEFAULT;
+ }
+}
+
static void cfq_pd_init(struct blkcg_gq *blkg)
{
struct cfq_group *cfqg = blkg_to_cfqg(blkg);
+ struct cfq_group_data *cgd = blkcg_to_cfqgd(blkg->blkcg);
cfq_init_cfqg_base(cfqg);
- cfqg->weight = blkg->blkcg->cfq_weight;
- cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight;
+ cfqg->weight = cgd->weight;
+ cfqg->leaf_weight = cgd->leaf_weight;
cfqg_stats_init(&cfqg->stats);
cfqg_stats_init(&cfqg->dead_stats);
}
@@ -1673,13 +1713,17 @@ static int cfqg_print_leaf_weight_device(struct seq_file *sf, void *v)
static int cfq_print_weight(struct seq_file *sf, void *v)
{
- seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_weight);
+ struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+
+ seq_printf(sf, "%u\n", blkcg_to_cfqgd(blkcg)->weight);
return 0;
}
static int cfq_print_leaf_weight(struct seq_file *sf, void *v)
{
- seq_printf(sf, "%u\n", css_to_blkcg(seq_css(sf))->cfq_leaf_weight);
+ struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+
+ seq_printf(sf, "%u\n", blkcg_to_cfqgd(blkcg)->leaf_weight);
return 0;
}
@@ -1690,6 +1734,7 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of,
struct blkcg *blkcg = css_to_blkcg(of_css(of));
struct blkg_conf_ctx ctx;
struct cfq_group *cfqg;
+ struct cfq_group_data *cfqgd;
int ret;
ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx);
@@ -1698,13 +1743,14 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of,
ret = -EINVAL;
cfqg = blkg_to_cfqg(ctx.blkg);
+ cfqgd = blkcg_to_cfqgd(blkcg);
if (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && ctx.v <= CFQ_WEIGHT_MAX)) {
if (!is_leaf_weight) {
cfqg->dev_weight = ctx.v;
- cfqg->new_weight = ctx.v ?: blkcg->cfq_weight;
+ cfqg->new_weight = ctx.v ?: cfqgd->weight;
} else {
cfqg->dev_leaf_weight = ctx.v;
- cfqg->new_leaf_weight = ctx.v ?: blkcg->cfq_leaf_weight;
+ cfqg->new_leaf_weight = ctx.v ?: cfqgd->leaf_weight;
}
ret = 0;
}
@@ -1730,16 +1776,18 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
{
struct blkcg *blkcg = css_to_blkcg(css);
struct blkcg_gq *blkg;
+ struct cfq_group_data *cfqgd;
if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX)
return -EINVAL;
spin_lock_irq(&blkcg->lock);
+ cfqgd = blkcg_to_cfqgd(blkcg);
if (!is_leaf_weight)
- blkcg->cfq_weight = val;
+ cfqgd->weight = val;
else
- blkcg->cfq_leaf_weight = val;
+ cfqgd->leaf_weight = val;
hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
struct cfq_group *cfqg = blkg_to_cfqg(blkg);
@@ -1749,10 +1797,10 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft,
if (!is_leaf_weight) {
if (!cfqg->dev_weight)
- cfqg->new_weight = blkcg->cfq_weight;
+ cfqg->new_weight = cfqgd->weight;
} else {
if (!cfqg->dev_leaf_weight)
- cfqg->new_leaf_weight = blkcg->cfq_leaf_weight;
+ cfqg->new_leaf_weight = cfqgd->leaf_weight;
}
}
@@ -4603,8 +4651,10 @@ static struct elevator_type iosched_cfq = {
#ifdef CONFIG_CFQ_GROUP_IOSCHED
static struct blkcg_policy blkcg_policy_cfq = {
.pd_size = sizeof(struct cfq_group),
+ .cpd_size = sizeof(struct cfq_group_data),
.cftypes = cfq_blkcg_files,
+ .cpd_init_fn = cfq_cpd_init,
.pd_init_fn = cfq_pd_init,
.pd_offline_fn = cfq_pd_offline,
.pd_reset_stats_fn = cfq_pd_reset_stats,