diff --git a/kernel/drivers/module-bbswitch/pspec.xml b/kernel/drivers/module-bbswitch/pspec.xml index d0651179..d17eb17a 100644 --- a/kernel/drivers/module-bbswitch/pspec.xml +++ b/kernel/drivers/module-bbswitch/pspec.xml @@ -11,7 +11,7 @@ Kernel module allowing to switch dedicated graphics card on Optimus laptops kernel module allowing to switch dedicated graphics card on Optimus laptops - kernel-module-headers + kernel-module-headers https://github.com/Bumblebee-Project/bbswitch/archive/v0.8.tar.gz @@ -20,7 +20,7 @@ module-bbswitch - kernel + kernel /lib/modules @@ -34,6 +34,13 @@ + + 2016-11-02 + 0.8 + Rebuild for new kernel. + Ertuğrul Erata + ertugrulerata@gmail.com + 2016-08-24 0.8 diff --git a/kernel/drivers/module-broadcom-wl/pspec.xml b/kernel/drivers/module-broadcom-wl/pspec.xml index bc8d8f2d..fdf228dc 100644 --- a/kernel/drivers/module-broadcom-wl/pspec.xml +++ b/kernel/drivers/module-broadcom-wl/pspec.xml @@ -15,7 +15,7 @@ http://www.broadcom.com/docs/linux_sta/hybrid-v35-nodebug-pcoem-6_30_223_271.tar.gz http://www.broadcom.com/docs/linux_sta/hybrid-v35_64-nodebug-pcoem-6_30_223_271.tar.gz - kernel-module-headers + kernel-module-headers patch/license.patch @@ -27,7 +27,7 @@ module-broadcom-wl - kernel + kernel module-broadcom-wl-userspace @@ -56,6 +56,13 @@ + + 2016-11-02 + 6.30.223.271 + Rebuild for new kernel. + PisiLinux Community + admin@pisilinux.org + 2016-08-24 6.30.223.271 diff --git a/kernel/drivers/module-virtualbox-guest/pspec.xml b/kernel/drivers/module-virtualbox-guest/pspec.xml index ad67c789..d3c9e3ee 100644 --- a/kernel/drivers/module-virtualbox-guest/pspec.xml +++ b/kernel/drivers/module-virtualbox-guest/pspec.xml @@ -14,14 +14,14 @@ This package provides the kernel modules needed for mouse integration and shared folder support between VirtualBox host and guest systems. https://sourceforge.net/projects/pisi-source/files/kernel-4.7/module-virtualbox-guest-5.1.6.tar.xz - kernel-module-headers + kernel-module-headers module-virtualbox-guest - kernel + kernel baselayout module-virtualbox-guest-userspace @@ -52,6 +52,13 @@ + + 2016-11-02 + 5.1.6 + Rebuild for new kernel. + Ertuğrul Erata + ertugrulerata@gmail.com + 2016-09-26 5.1.6 diff --git a/kernel/drivers/module-virtualbox/pspec.xml b/kernel/drivers/module-virtualbox/pspec.xml index 039bff23..f82470b6 100644 --- a/kernel/drivers/module-virtualbox/pspec.xml +++ b/kernel/drivers/module-virtualbox/pspec.xml @@ -14,7 +14,7 @@ This package provides the kernel support for VirtualBox. https://sourceforge.net/projects/pisi-source/files/kernel-4.7/module-virtualbox-5.1.6.tar.xz - kernel-module-headers + kernel-module-headers non-fhsm */ + au_br_fhsm_fin(br->br_fhsm); -+ kfree(br->br_fhsm); ++ au_delayed_kfree(br->br_fhsm); + br->br_fhsm = NULL; + } + } else if (au_br_fhsm(mod->perm)) @@ -3460,7 +3578,8 @@ index 0000000..baf786e + goto out; /* success */ + +out_bf: -+ kfree(bf); ++ if (bf) ++ au_delayed_kfree(bf); +out: + AuTraceErr(err); + return err; @@ -3787,10 +3906,10 @@ index 0000000..32a4d8f +#endif /* __AUFS_BRANCH_H__ */ diff --git a/fs/aufs/cpup.c b/fs/aufs/cpup.c new file mode 100644 -index 0000000..52f87de +index 0000000..598a4d6 --- /dev/null +++ b/fs/aufs/cpup.c -@@ -0,0 +1,1370 @@ +@@ -0,0 +1,1378 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -4137,9 +4256,9 @@ index 0000000..52f87de + dst->f_pos = 0; + err = au_do_copy_file(dst, src, len, buf, blksize); + if (do_kfree) -+ kfree(buf); ++ au_delayed_kfree(buf); + else -+ free_page((unsigned long)buf); ++ au_delayed_free_page((unsigned long)buf); + +out: + return err; @@ -4174,6 +4293,7 @@ index 0000000..52f87de + } + }; + struct super_block *sb; ++ struct inode *h_src_inode; + struct task_struct *tsk = current; + + /* bsrc branch can be ro/rw. */ @@ -4189,7 +4309,9 @@ index 0000000..52f87de + } + + /* try stopping to update while we copyup */ -+ IMustLock(d_inode(file[SRC].dentry)); ++ h_src_inode = d_inode(file[SRC].dentry); ++ if (!au_test_nfs(h_src_inode->i_sb)) ++ IMustLock(h_src_inode); + err = au_copy_file(file[DST].file, file[SRC].file, cpg->len); + + /* i wonder if we had O_NO_DELAY_FPUT flag */ @@ -4249,8 +4371,13 @@ index 0000000..52f87de + goto out; + } + h_src_attr->valid = 1; -+ err = au_cp_regular(cpg); -+ inode_unlock(h_src_inode); ++ if (!au_test_nfs(h_src_inode->i_sb)) { ++ err = au_cp_regular(cpg); ++ inode_unlock(h_src_inode); ++ } else { ++ inode_unlock(h_src_inode); ++ err = au_cp_regular(cpg); ++ } + rerr = au_pin_hdir_relock(cpg->pin); + if (!err && rerr) + err = rerr; @@ -4299,7 +4426,7 @@ index 0000000..52f87de + sym.k[symlen] = 0; + err = vfsub_symlink(h_dir, h_path, sym.k); + } -+ free_page((unsigned long)sym.k); ++ au_delayed_free_page((unsigned long)sym.k); + +out: + return err; @@ -4670,7 +4797,7 @@ index 0000000..52f87de + } +out_parent: + dput(dst_parent); -+ kfree(a); ++ au_delayed_kfree(a); +out: + return err; +} @@ -5250,10 +5377,10 @@ index 0000000..ccba2c4 +#endif /* __AUFS_CPUP_H__ */ diff --git a/fs/aufs/dbgaufs.c b/fs/aufs/dbgaufs.c new file mode 100644 -index 0000000..21ac773 +index 0000000..4db01e8 --- /dev/null +++ b/fs/aufs/dbgaufs.c -@@ -0,0 +1,419 @@ +@@ -0,0 +1,425 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -5284,7 +5411,7 @@ index 0000000..21ac773 +static int dbgaufs_xi_release(struct inode *inode __maybe_unused, + struct file *file) +{ -+ kfree(file->private_data); ++ au_delayed_kfree(file->private_data); + return 0; +} + @@ -5346,7 +5473,7 @@ index 0000000..21ac773 +static int dbgaufs_plink_release(struct inode *inode __maybe_unused, + struct file *file) +{ -+ free_page((unsigned long)file->private_data); ++ au_delayed_free_page((unsigned long)file->private_data); + return 0; +} + @@ -5410,7 +5537,7 @@ index 0000000..21ac773 + goto out; /* success */ + +out_free: -+ free_page((unsigned long)p); ++ au_delayed_free_page((unsigned long)p); +out: + return err; +} @@ -5512,7 +5639,10 @@ index 0000000..21ac773 + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + xi = &br->br_xino; ++ /* debugfs acquires the parent i_mutex */ ++ lockdep_off(); + debugfs_remove(xi->xi_dbgaufs); ++ lockdep_on(); + xi->xi_dbgaufs = NULL; + } +} @@ -5537,8 +5667,11 @@ index 0000000..21ac773 + br = au_sbr(sb, bindex); + xi = &br->br_xino; + AuDebugOn(xi->xi_dbgaufs); ++ /* debugfs acquires the parent i_mutex */ ++ lockdep_off(); + xi->xi_dbgaufs = debugfs_create_file(name, dbgaufs_mode, parent, + sbinfo, &dbgaufs_xino_fop); ++ lockdep_on(); + /* ignore an error */ + if (unlikely(!xi->xi_dbgaufs)) + AuWarn1("failed %s under debugfs\n", name); @@ -5716,10 +5849,10 @@ index 0000000..81f272e +#endif /* __DBGAUFS_H__ */ diff --git a/fs/aufs/dcsub.c b/fs/aufs/dcsub.c new file mode 100644 -index 0000000..e72acce +index 0000000..5d46398 --- /dev/null +++ b/fs/aufs/dcsub.c -@@ -0,0 +1,211 @@ +@@ -0,0 +1,212 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -5738,7 +5871,7 @@ index 0000000..e72acce + p = dpage->dentries; + for (i = 0; i < dpage->ndentry; i++) + dput(*p++); -+ free_page((unsigned long)dpage->dentries); ++ au_delayed_free_page((unsigned long)dpage->dentries); +} + +int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp) @@ -5761,7 +5894,7 @@ index 0000000..e72acce + return 0; /* success */ + +out_dpages: -+ kfree(dpages->dpages); ++ au_delayed_kfree(dpages->dpages); +out: + return err; +} @@ -5774,7 +5907,7 @@ index 0000000..e72acce + p = dpages->dpages; + for (i = 0; i < dpages->ndpage; i++) + au_dpage_free(p++); -+ kfree(dpages->dpages); ++ au_delayed_kfree(dpages->dpages); +} + +static int au_dpages_append(struct au_dcsub_pages *dpages, @@ -5791,7 +5924,8 @@ index 0000000..e72acce + err = -ENOMEM; + sz = dpages->ndpage * sizeof(*dpages->dpages); + p = au_kzrealloc(dpages->dpages, sz, -+ sz + sizeof(*dpages->dpages), gfp); ++ sz + sizeof(*dpages->dpages), gfp, ++ /*may_shrink*/0); + if (unlikely(!p)) + goto out; + @@ -6062,10 +6196,10 @@ index 0000000..5d2cf66 +#endif /* __AUFS_DCSUB_H__ */ diff --git a/fs/aufs/debug.c b/fs/aufs/debug.c new file mode 100644 -index 0000000..44bdb5f +index 0000000..443fee3 --- /dev/null +++ b/fs/aufs/debug.c -@@ -0,0 +1,428 @@ +@@ -0,0 +1,427 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -6391,7 +6525,7 @@ index 0000000..44bdb5f + au_br_count_init(&a->fake); + err = do_pri_br(-1, &a->fake); + au_br_count_fin(&a->fake); -+ kfree(a); ++ au_delayed_kfree(a); + dpri("dev 0x%x\n", sb->s_dev); + if (err || !au_test_aufs(sb)) + return; @@ -6399,9 +6533,8 @@ index 0000000..44bdb5f + sbinfo = au_sbi(sb); + if (!sbinfo) + return; -+ dpri("nw %lld, gen %u, kobj %d\n", -+ percpu_counter_sum(&sbinfo->si_nowait.nw_len), -+ sbinfo->si_generation, ++ dpri("nw %d, gen %u, kobj %d\n", ++ atomic_read(&sbinfo->si_nowait.nw_len), sbinfo->si_generation, + atomic_read(&sbinfo->si_kobj.kref.refcount)); + for (bindex = 0; bindex <= sbinfo->si_bbot; bindex++) + do_pri_br(bindex, sbinfo->si_branch[0 + bindex]); @@ -6714,10 +6847,10 @@ index 0000000..0567f31 +#endif /* __AUFS_DEBUG_H__ */ diff --git a/fs/aufs/dentry.c b/fs/aufs/dentry.c new file mode 100644 -index 0000000..4bb1530 +index 0000000..5eb4f36 --- /dev/null +++ b/fs/aufs/dentry.c -@@ -0,0 +1,1115 @@ +@@ -0,0 +1,1117 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -6756,7 +6889,7 @@ index 0000000..4bb1530 + br = au_sbr(dentry->d_sb, bindex); + wh_able = !!au_br_whable(br->br_perm); + if (wh_able) -+ wh_found = au_wh_test(h_parent, wh_name, /*try_sio*/0); ++ wh_found = au_wh_test(h_parent, wh_name, ignore_perm); + h_dentry = ERR_PTR(wh_found); + if (!wh_found) + goto real_lookup; @@ -6921,7 +7054,7 @@ index 0000000..4bb1530 + +out_parent: + dput(parent); -+ kfree(whname.name); ++ au_delayed_kfree(whname.name); +out: + return err; +} @@ -7421,7 +7554,7 @@ index 0000000..4bb1530 + +int au_refresh_dentry(struct dentry *dentry, struct dentry *parent) +{ -+ int err, ebrange; ++ int err, ebrange, nbr; + unsigned int sigen; + struct au_dinfo *dinfo, *tmp; + struct super_block *sb; @@ -7437,8 +7570,9 @@ index 0000000..4bb1530 + if (unlikely(err)) + goto out; + ++ nbr = au_sbbot(sb) + 1; + dinfo = au_di(dentry); -+ err = au_di_realloc(dinfo, au_sbbot(sb) + 1); ++ err = au_di_realloc(dinfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + ebrange = au_dbrange_test(dentry); @@ -7481,6 +7615,7 @@ index 0000000..4bb1530 + au_dbg_verify_dinode(dentry); + AuTraceErr(err); + } ++ au_di_realloc(dinfo, nbr, /*may_shrink*/1); /* harmless if err */ + au_rw_write_unlock(&tmp->di_rwsem); + au_di_free(tmp); + if (unlikely(err)) @@ -7835,10 +7970,10 @@ index 0000000..4bb1530 +}; diff --git a/fs/aufs/dentry.h b/fs/aufs/dentry.h new file mode 100644 -index 0000000..bdc1af6 +index 0000000..01211d2 --- /dev/null +++ b/fs/aufs/dentry.h -@@ -0,0 +1,239 @@ +@@ -0,0 +1,242 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -7866,7 +8001,10 @@ index 0000000..bdc1af6 + struct au_rwsem di_rwsem; + aufs_bindex_t di_btop, di_bbot, di_bwh, di_bdiropq; + unsigned char di_tmpfile; /* to allow the different name */ -+ struct au_hdentry *di_hdentry; ++ union { ++ struct au_hdentry *di_hdentry; ++ struct llist_node di_lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ @@ -7904,7 +8042,7 @@ index 0000000..bdc1af6 +void au_di_cp(struct au_dinfo *dst, struct au_dinfo *src); +int au_di_init(struct dentry *dentry); +void au_di_fin(struct dentry *dentry); -+int au_di_realloc(struct au_dinfo *dinfo, int nbr); ++int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink); + +void di_read_lock(struct dentry *d, int flags, unsigned int lsc); +void di_read_unlock(struct dentry *d, int flags); @@ -8080,10 +8218,10 @@ index 0000000..bdc1af6 +#endif /* __AUFS_DENTRY_H__ */ diff --git a/fs/aufs/dinfo.c b/fs/aufs/dinfo.c new file mode 100644 -index 0000000..43b5509 +index 0000000..2626c0c --- /dev/null +++ b/fs/aufs/dinfo.c -@@ -0,0 +1,539 @@ +@@ -0,0 +1,540 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -8126,7 +8264,7 @@ index 0000000..43b5509 + goto out; + } + -+ au_cache_free_dinfo(dinfo); ++ au_cache_dfree_dinfo(dinfo); + dinfo = NULL; + +out: @@ -8146,8 +8284,8 @@ index 0000000..43b5509 + while (bindex++ <= bbot) + au_hdput(p++); + } -+ kfree(dinfo->di_hdentry); -+ au_cache_free_dinfo(dinfo); ++ au_delayed_kfree(dinfo->di_hdentry); ++ au_cache_dfree_dinfo(dinfo); +} + +void au_di_swap(struct au_dinfo *a, struct au_dinfo *b) @@ -8215,7 +8353,7 @@ index 0000000..43b5509 + au_di_free(dinfo); +} + -+int au_di_realloc(struct au_dinfo *dinfo, int nbr) ++int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink) +{ + int err, sz; + struct au_hdentry *hdp; @@ -8226,7 +8364,8 @@ index 0000000..43b5509 + sz = sizeof(*hdp) * (dinfo->di_bbot + 1); + if (!sz) + sz = sizeof(*hdp); -+ hdp = au_kzrealloc(dinfo->di_hdentry, sz, sizeof(*hdp) * nbr, GFP_NOFS); ++ hdp = au_kzrealloc(dinfo->di_hdentry, sz, sizeof(*hdp) * nbr, GFP_NOFS, ++ may_shrink); + if (hdp) { + dinfo->di_hdentry = hdp; + err = 0; @@ -8625,10 +8764,10 @@ index 0000000..43b5509 +} diff --git a/fs/aufs/dir.c b/fs/aufs/dir.c new file mode 100644 -index 0000000..27bef0b +index 0000000..395563e --- /dev/null +++ b/fs/aufs/dir.c -@@ -0,0 +1,743 @@ +@@ -0,0 +1,749 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -8774,7 +8913,7 @@ index 0000000..27bef0b +out: + dput(a->dentry); + au_nwt_done(&au_sbi(sb)->si_nowait); -+ kfree(arg); ++ au_delayed_kfree(arg); +} + +void au_dir_ts(struct inode *dir, aufs_bindex_t bindex) @@ -8810,7 +8949,7 @@ index 0000000..27bef0b + if (unlikely(wkq_err)) { + pr_err("wkq %d\n", wkq_err); + dput(dentry); -+ kfree(arg); ++ au_delayed_kfree(arg); + } + +out: @@ -8929,7 +9068,7 @@ index 0000000..27bef0b + }; + err = au_do_open(file, &args); + if (unlikely(err)) -+ kfree(fidir); ++ au_delayed_kfree(fidir); + } + si_read_unlock(sb); + return err; @@ -8941,8 +9080,11 @@ index 0000000..27bef0b + struct au_vdir *vdir_cache; + struct au_finfo *finfo; + struct au_fidir *fidir; ++ struct au_hfile *hf; + aufs_bindex_t bindex, bbot; ++ int execed, delayed; + ++ delayed = (current->flags & PF_KTHREAD) || in_interrupt(); + finfo = au_fi(file); + fidir = finfo->fi_hdir; + if (fidir) { @@ -8950,22 +9092,25 @@ index 0000000..27bef0b + &au_sbi(file->f_path.dentry->d_sb)->si_files); + vdir_cache = fidir->fd_vdir_cache; /* lock-free */ + if (vdir_cache) -+ au_vdir_free(vdir_cache); ++ au_vdir_free(vdir_cache, delayed); + + bindex = finfo->fi_btop; + if (bindex >= 0) { ++ execed = vfsub_file_execed(file); ++ hf = fidir->fd_hfile + bindex; + /* + * calls fput() instead of filp_close(), + * since no dnotify or lock for the lower file. + */ + bbot = fidir->fd_bbot; -+ for (; bindex <= bbot; bindex++) -+ au_set_h_fptr(file, bindex, NULL); ++ for (; bindex <= bbot; bindex++, hf++) ++ if (hf->hf_file) ++ au_hfput(hf, execed); + } -+ kfree(fidir); ++ au_delayed_kfree(fidir); + finfo->fi_hdir = NULL; + } -+ au_finfo_fin(file); ++ au_finfo_fin(file, delayed); + return 0; +} + @@ -9374,10 +9519,10 @@ index 0000000..27bef0b +}; diff --git a/fs/aufs/dir.h b/fs/aufs/dir.h new file mode 100644 -index 0000000..b0a79d7 +index 0000000..4f3945a --- /dev/null +++ b/fs/aufs/dir.h -@@ -0,0 +1,118 @@ +@@ -0,0 +1,124 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -9409,7 +9554,10 @@ index 0000000..b0a79d7 + +struct au_vdir_dehstr { + struct hlist_node hash; -+ struct au_vdir_destr *str; ++ union { ++ struct au_vdir_destr *str; ++ struct llist_node lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +struct au_vdir_de { @@ -9447,7 +9595,10 @@ index 0000000..b0a79d7 + + unsigned long vd_version; + unsigned int vd_deblk_sz; -+ unsigned long vd_jiffy; ++ union { ++ unsigned long vd_jiffy; ++ struct llist_node vd_lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ @@ -9471,7 +9622,7 @@ index 0000000..b0a79d7 +int au_nhash_append_wh(struct au_nhash *whlist, char *name, int nlen, ino_t ino, + unsigned int d_type, aufs_bindex_t bindex, + unsigned char shwh); -+void au_vdir_free(struct au_vdir *vdir); ++void au_vdir_free(struct au_vdir *vdir, int atonce); +int au_vdir_init(struct file *file); +int au_vdir_fill_de(struct file *file, struct dir_context *ctx); + @@ -9498,10 +9649,10 @@ index 0000000..b0a79d7 +#endif /* __AUFS_DIR_H__ */ diff --git a/fs/aufs/dynop.c b/fs/aufs/dynop.c new file mode 100644 -index 0000000..dfb3a71 +index 0000000..eb0e1a7 --- /dev/null +++ b/fs/aufs/dynop.c -@@ -0,0 +1,356 @@ +@@ -0,0 +1,358 @@ +/* + * Copyright (C) 2010-2016 Junjiro R. Okajima + */ @@ -9518,17 +9669,17 @@ index 0000000..dfb3a71 + * How large will these lists be? + * Usually just a few elements, 20-30 at most for each, I guess. + */ -+static struct au_splhead dynop[AuDyLast]; ++static struct au_sphlhead dynop[AuDyLast]; + -+static struct au_dykey *dy_gfind_get(struct au_splhead *spl, const void *h_op) ++static struct au_dykey *dy_gfind_get(struct au_sphlhead *sphl, const void *h_op) +{ + struct au_dykey *key, *tmp; -+ struct list_head *head; ++ struct hlist_head *head; + + key = NULL; -+ head = &spl->head; ++ head = &sphl->head; + rcu_read_lock(); -+ list_for_each_entry_rcu(tmp, head, dk_list) ++ hlist_for_each_entry_rcu(tmp, head, dk_hnode) + if (tmp->dk_op.dy_hop == h_op) { + key = tmp; + kref_get(&key->dk_kref); @@ -9575,24 +9726,24 @@ index 0000000..dfb3a71 +} + +/* kref_get() if @key is already added */ -+static struct au_dykey *dy_gadd(struct au_splhead *spl, struct au_dykey *key) ++static struct au_dykey *dy_gadd(struct au_sphlhead *sphl, struct au_dykey *key) +{ + struct au_dykey *tmp, *found; -+ struct list_head *head; ++ struct hlist_head *head; + const void *h_op = key->dk_op.dy_hop; + + found = NULL; -+ head = &spl->head; -+ spin_lock(&spl->spin); -+ list_for_each_entry(tmp, head, dk_list) ++ head = &sphl->head; ++ spin_lock(&sphl->spin); ++ hlist_for_each_entry(tmp, head, dk_hnode) + if (tmp->dk_op.dy_hop == h_op) { + kref_get(&tmp->dk_kref); + found = tmp; + break; + } + if (!found) -+ list_add_rcu(&key->dk_list, head); -+ spin_unlock(&spl->spin); ++ hlist_add_head_rcu(&key->dk_hnode, head); ++ spin_unlock(&sphl->spin); + + if (!found) + DyPrSym(key); @@ -9605,17 +9756,17 @@ index 0000000..dfb3a71 + + key = container_of(rcu, struct au_dykey, dk_rcu); + DyPrSym(key); -+ kfree(key); ++ kfree(key); /* not delayed */ +} + +static void dy_free(struct kref *kref) +{ + struct au_dykey *key; -+ struct au_splhead *spl; ++ struct au_sphlhead *sphl; + + key = container_of(kref, struct au_dykey, dk_kref); -+ spl = dynop + key->dk_op.dy_type; -+ au_spl_del_rcu(&key->dk_list, spl); ++ sphl = dynop + key->dk_op.dy_type; ++ au_sphl_del_rcu(&key->dk_hnode, sphl); + call_rcu(&key->dk_rcu, dy_free_rcu); +} + @@ -9680,6 +9831,8 @@ index 0000000..dfb3a71 + /* this one will be changed according to an aufs mount option */ + DySetAop(direct_IO); + DySetAop(migratepage); ++ DySetAop(isolate_page); ++ DySetAop(putback_page); + DySetAop(launder_page); + DySetAop(is_partially_uptodate); + DySetAop(is_dirty_writeback); @@ -9700,7 +9853,7 @@ index 0000000..dfb3a71 +static struct au_dykey *dy_get(struct au_dynop *op, struct au_branch *br) +{ + struct au_dykey *key, *old; -+ struct au_splhead *spl; ++ struct au_sphlhead *sphl; + struct op { + unsigned int sz; + void (*set)(struct au_dykey *key, const void *h_op, @@ -9714,8 +9867,8 @@ index 0000000..dfb3a71 + }; + const struct op *p; + -+ spl = dynop + op->dy_type; -+ key = dy_gfind_get(spl, op->dy_hop); ++ sphl = dynop + op->dy_type; ++ key = dy_gfind_get(sphl, op->dy_hop); + if (key) + goto out_add; /* success */ + @@ -9729,9 +9882,9 @@ index 0000000..dfb3a71 + key->dk_op.dy_hop = op->dy_hop; + kref_init(&key->dk_kref); + p->set(key, op->dy_hop, au_br_sb(br)); -+ old = dy_gadd(spl, key); ++ old = dy_gadd(sphl, key); + if (old) { -+ kfree(key); ++ au_delayed_kfree(key); + key = old; + } + @@ -9826,16 +9979,16 @@ index 0000000..dfb3a71 + +void au_dy_arefresh(int do_dx) +{ -+ struct au_splhead *spl; -+ struct list_head *head; ++ struct au_sphlhead *sphl; ++ struct hlist_head *head; + struct au_dykey *key; + -+ spl = dynop + AuDy_AOP; -+ head = &spl->head; -+ spin_lock(&spl->spin); -+ list_for_each_entry(key, head, dk_list) ++ sphl = dynop + AuDy_AOP; ++ head = &sphl->head; ++ spin_lock(&sphl->spin); ++ hlist_for_each_entry(key, head, dk_hnode) + dy_adx((void *)key, do_dx); -+ spin_unlock(&spl->spin); ++ spin_unlock(&sphl->spin); +} + +/* ---------------------------------------------------------------------- */ @@ -9848,7 +10001,7 @@ index 0000000..dfb3a71 + BUILD_BUG_ON(offsetof(struct au_dyaop, da_key)); + + for (i = 0; i < AuDyLast; i++) -+ au_spl_init(dynop + i); ++ au_sphl_init(dynop + i); +} + +void au_dy_fin(void) @@ -9856,11 +10009,11 @@ index 0000000..dfb3a71 + int i; + + for (i = 0; i < AuDyLast; i++) -+ WARN_ON(!list_empty(&dynop[i].head)); ++ WARN_ON(!hlist_empty(&dynop[i].head)); +} diff --git a/fs/aufs/dynop.h b/fs/aufs/dynop.h new file mode 100644 -index 0000000..8680bfc +index 0000000..5db2da2 --- /dev/null +++ b/fs/aufs/dynop.h @@ -0,0 +1,61 @@ @@ -9892,7 +10045,7 @@ index 0000000..8680bfc + +struct au_dykey { + union { -+ struct list_head dk_list; ++ struct hlist_node dk_hnode; + struct rcu_head dk_rcu; + }; + struct au_dynop dk_op; @@ -9927,7 +10080,7 @@ index 0000000..8680bfc +#endif /* __AUFS_DYNOP_H__ */ diff --git a/fs/aufs/export.c b/fs/aufs/export.c new file mode 100644 -index 0000000..98a7036 +index 0000000..6b5a7be --- /dev/null +++ b/fs/aufs/export.c @@ -0,0 +1,824 @@ @@ -10337,7 +10490,7 @@ index 0000000..98a7036 + } + +out_name: -+ free_page((unsigned long)arg.name); ++ au_delayed_free_page((unsigned long)arg.name); +out_file: + fput(file); +out: @@ -10491,7 +10644,7 @@ index 0000000..98a7036 + dentry = ERR_PTR(-ESTALE); + } +out_pathname: -+ free_page((unsigned long)pathname); ++ au_delayed_free_page((unsigned long)pathname); +out_h_parent: + dput(h_parent); +out: @@ -10757,10 +10910,10 @@ index 0000000..98a7036 +} diff --git a/fs/aufs/f_op.c b/fs/aufs/f_op.c new file mode 100644 -index 0000000..504b767 +index 0000000..00475fb --- /dev/null +++ b/fs/aufs/f_op.c -@@ -0,0 +1,757 @@ +@@ -0,0 +1,759 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -10849,6 +11002,7 @@ index 0000000..504b767 +{ + struct au_finfo *finfo; + aufs_bindex_t bindex; ++ int delayed; + + finfo = au_fi(file); + au_sphl_del(&finfo->fi_hlist, @@ -10857,7 +11011,8 @@ index 0000000..504b767 + if (bindex >= 0) + au_set_h_fptr(file, bindex, NULL); + -+ au_finfo_fin(file); ++ delayed = (current->flags & PF_KTHREAD) || in_interrupt(); ++ au_finfo_fin(file, delayed); + return 0; +} + @@ -11938,10 +12093,10 @@ index 0000000..e3cb6ed +} diff --git a/fs/aufs/file.c b/fs/aufs/file.c new file mode 100644 -index 0000000..4b68aef +index 0000000..bfc1d1b --- /dev/null +++ b/fs/aufs/file.c -@@ -0,0 +1,830 @@ +@@ -0,0 +1,844 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -12204,7 +12359,7 @@ index 0000000..4b68aef + } + if (unlikely(err)) { + finfo->fi_hdir = NULL; -+ au_finfo_fin(file); ++ au_finfo_fin(file, /*atonce*/0); + } + +out: @@ -12524,6 +12679,7 @@ index 0000000..4b68aef + +static void au_do_refresh_dir(struct file *file) +{ ++ int execed; + aufs_bindex_t bindex, bbot, new_bindex, brid; + struct au_hfile *p, tmp, *q; + struct au_finfo *finfo; @@ -12562,6 +12718,7 @@ index 0000000..4b68aef + } + } + ++ execed = vfsub_file_execed(file); + p = fidir->fd_hfile; + if (!au_test_mmapped(file) && !d_unlinked(file->f_path.dentry)) { + bbot = au_sbbot(sb); @@ -12570,14 +12727,14 @@ index 0000000..4b68aef + if (p->hf_file) { + if (file_inode(p->hf_file)) + break; -+ au_hfput(p, file); ++ au_hfput(p, execed); + } + } else { + bbot = au_br_index(sb, brid); + for (finfo->fi_btop = 0; finfo->fi_btop < bbot; + finfo->fi_btop++, p++) + if (p->hf_file) -+ au_hfput(p, file); ++ au_hfput(p, execed); + bbot = au_sbbot(sb); + } + @@ -12587,7 +12744,7 @@ index 0000000..4b68aef + if (p->hf_file) { + if (file_inode(p->hf_file)) + break; -+ au_hfput(p, file); ++ au_hfput(p, execed); + } + AuDebugOn(fidir->fd_bbot < finfo->fi_btop); +} @@ -12597,23 +12754,26 @@ index 0000000..4b68aef + */ +static int refresh_file(struct file *file, int (*reopen)(struct file *file)) +{ -+ int err, need_reopen; ++ int err, need_reopen, nbr; + aufs_bindex_t bbot, bindex; + struct dentry *dentry; ++ struct super_block *sb; + struct au_finfo *finfo; + struct au_hfile *hfile; + + dentry = file->f_path.dentry; ++ sb = dentry->d_sb; ++ nbr = au_sbbot(sb) + 1; + finfo = au_fi(file); + if (!finfo->fi_hdir) { + hfile = &finfo->fi_htop; + AuDebugOn(!hfile->hf_file); -+ bindex = au_br_index(dentry->d_sb, hfile->hf_br->br_id); ++ bindex = au_br_index(sb, hfile->hf_br->br_id); + AuDebugOn(bindex < 0); + if (bindex != finfo->fi_btop) + au_set_fbtop(file, bindex); + } else { -+ err = au_fidir_realloc(finfo, au_sbbot(dentry->d_sb) + 1); ++ err = au_fidir_realloc(finfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + au_do_refresh_dir(file); @@ -12623,6 +12783,9 @@ index 0000000..4b68aef + need_reopen = 1; + if (!au_test_mmapped(file)) + err = au_file_refresh_by_inode(file, &need_reopen); ++ if (finfo->fi_hdir) ++ /* harmless if err */ ++ au_fidir_realloc(finfo, nbr, /*may_shrink*/1); + if (!err && need_reopen && !d_unlinked(dentry)) + err = reopen(file); + if (!err) { @@ -12730,6 +12893,10 @@ index 0000000..4b68aef + struct page *page, enum migrate_mode mode) +{ AuUnsupport(); return 0; } +#endif ++static bool aufs_isolate_page(struct page *page, isolate_mode_t mode) ++{ AuUnsupport(); return true; } ++static void aufs_putback_page(struct page *page) ++{ AuUnsupport(); } +static int aufs_launder_page(struct page *page) +{ AuUnsupport(); return 0; } +static int aufs_is_partially_uptodate(struct page *page, @@ -12764,6 +12931,8 @@ index 0000000..4b68aef + .releasepage = aufs_releasepage, + /* is fallback_migrate_page ok? */ + /* .migratepage = aufs_migratepage, */ ++ .isolate_page = aufs_isolate_page, ++ .putback_page = aufs_putback_page, + .launder_page = aufs_launder_page, + .is_partially_uptodate = aufs_is_partially_uptodate, + .is_dirty_writeback = aufs_is_dirty_writeback, @@ -12774,10 +12943,10 @@ index 0000000..4b68aef +}; diff --git a/fs/aufs/file.h b/fs/aufs/file.h new file mode 100644 -index 0000000..96ab0a0 +index 0000000..f53b381 --- /dev/null +++ b/fs/aufs/file.h -@@ -0,0 +1,278 @@ +@@ -0,0 +1,281 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -12830,7 +12999,10 @@ index 0000000..96ab0a0 + struct au_fidir *fi_hdir; /* for dir only */ + + struct hlist_node fi_hlist; -+ struct file *fi_file; /* very ugly */ ++ union { ++ struct file *fi_file; /* very ugly */ ++ struct llist_node fi_lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ @@ -12881,16 +13053,16 @@ index 0000000..96ab0a0 +struct file *au_read_pre(struct file *file, int keep_fi); + +/* finfo.c */ -+void au_hfput(struct au_hfile *hf, struct file *file); ++void au_hfput(struct au_hfile *hf, int execed); +void au_set_h_fptr(struct file *file, aufs_bindex_t bindex, + struct file *h_file); + +void au_update_figen(struct file *file); +struct au_fidir *au_fidir_alloc(struct super_block *sb); -+int au_fidir_realloc(struct au_finfo *finfo, int nbr); ++int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink); + +void au_fi_init_once(void *_fi); -+void au_finfo_fin(struct file *file); ++void au_finfo_fin(struct file *file, int atonce); +int au_finfo_init(struct file *file, struct au_fidir *fidir); + +/* ioctl.c */ @@ -13058,10 +13230,10 @@ index 0000000..96ab0a0 +#endif /* __AUFS_FILE_H__ */ diff --git a/fs/aufs/finfo.c b/fs/aufs/finfo.c new file mode 100644 -index 0000000..07f6eb3 +index 0000000..0f016ed --- /dev/null +++ b/fs/aufs/finfo.c -@@ -0,0 +1,136 @@ +@@ -0,0 +1,138 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -13072,10 +13244,9 @@ index 0000000..07f6eb3 + +#include "aufs.h" + -+void au_hfput(struct au_hfile *hf, struct file *file) ++void au_hfput(struct au_hfile *hf, int execed) +{ -+ /* todo: direct access f_flags */ -+ if (vfsub_file_flags(file) & __FMODE_EXEC) ++ if (execed) + allow_write_access(hf->hf_file); + fput(hf->hf_file); + hf->hf_file = NULL; @@ -13097,7 +13268,7 @@ index 0000000..07f6eb3 + hf = fidir->fd_hfile + bindex; + + if (hf && hf->hf_file) -+ au_hfput(hf, file); ++ au_hfput(hf, vfsub_file_execed(file)); + if (val) { + FiMustWriteLock(file); + AuDebugOn(IS_ERR_OR_NULL(file->f_path.dentry)); @@ -13131,7 +13302,7 @@ index 0000000..07f6eb3 + return fidir; +} + -+int au_fidir_realloc(struct au_finfo *finfo, int nbr) ++int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink) +{ + int err; + struct au_fidir *fidir, *p; @@ -13142,7 +13313,7 @@ index 0000000..07f6eb3 + + err = -ENOMEM; + p = au_kzrealloc(fidir, au_fidir_sz(fidir->fd_nent), au_fidir_sz(nbr), -+ GFP_NOFS); ++ GFP_NOFS, may_shrink); + if (p) { + p->fd_nent = nbr; + finfo->fi_hdir = p; @@ -13154,7 +13325,7 @@ index 0000000..07f6eb3 + +/* ---------------------------------------------------------------------- */ + -+void au_finfo_fin(struct file *file) ++void au_finfo_fin(struct file *file, int atonce) +{ + struct au_finfo *finfo; + @@ -13163,7 +13334,10 @@ index 0000000..07f6eb3 + finfo = au_fi(file); + AuDebugOn(finfo->fi_hdir); + AuRwDestroy(&finfo->fi_rwsem); -+ au_cache_free_finfo(finfo); ++ if (!atonce) ++ au_cache_dfree_finfo(finfo); ++ else ++ au_cache_free_finfo(finfo); +} + +void au_fi_init_once(void *_finfo) @@ -13200,7 +13374,7 @@ index 0000000..07f6eb3 +} diff --git a/fs/aufs/fstype.h b/fs/aufs/fstype.h new file mode 100644 -index 0000000..725b2ff +index 0000000..95b78b9 --- /dev/null +++ b/fs/aufs/fstype.h @@ -0,0 +1,387 @@ @@ -13234,7 +13408,7 @@ index 0000000..725b2ff + +static inline int au_test_iso9660(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_ISO9660_FS) || defined(CONFIG_ISO9660_FS_MODULE) ++#if IS_ENABLED(CONFIG_ISO9660_FS) + return sb->s_magic == ISOFS_SUPER_MAGIC; +#else + return 0; @@ -13243,7 +13417,7 @@ index 0000000..725b2ff + +static inline int au_test_romfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_ROMFS_FS) || defined(CONFIG_ROMFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_ROMFS_FS) + return sb->s_magic == ROMFS_MAGIC; +#else + return 0; @@ -13252,7 +13426,7 @@ index 0000000..725b2ff + +static inline int au_test_cramfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_CRAMFS) || defined(CONFIG_CRAMFS_MODULE) ++#if IS_ENABLED(CONFIG_CRAMFS) + return sb->s_magic == CRAMFS_MAGIC; +#endif + return 0; @@ -13260,7 +13434,7 @@ index 0000000..725b2ff + +static inline int au_test_nfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_NFS_FS) || defined(CONFIG_NFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_NFS_FS) + return sb->s_magic == NFS_SUPER_MAGIC; +#else + return 0; @@ -13269,7 +13443,7 @@ index 0000000..725b2ff + +static inline int au_test_fuse(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_FUSE_FS) || defined(CONFIG_FUSE_FS_MODULE) ++#if IS_ENABLED(CONFIG_FUSE_FS) + return sb->s_magic == FUSE_SUPER_MAGIC; +#else + return 0; @@ -13278,7 +13452,7 @@ index 0000000..725b2ff + +static inline int au_test_xfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_XFS_FS) || defined(CONFIG_XFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_XFS_FS) + return sb->s_magic == XFS_SB_MAGIC; +#else + return 0; @@ -13296,7 +13470,7 @@ index 0000000..725b2ff + +static inline int au_test_ecryptfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_ECRYPT_FS) || defined(CONFIG_ECRYPT_FS_MODULE) ++#if IS_ENABLED(CONFIG_ECRYPT_FS) + return !strcmp(au_sbtype(sb), "ecryptfs"); +#else + return 0; @@ -13310,7 +13484,7 @@ index 0000000..725b2ff + +static inline int au_test_ubifs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_UBIFS_FS) || defined(CONFIG_UBIFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_UBIFS_FS) + return sb->s_magic == UBIFS_SUPER_MAGIC; +#else + return 0; @@ -13337,7 +13511,7 @@ index 0000000..725b2ff + +static inline int au_test_configfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_CONFIGFS_FS) || defined(CONFIG_CONFIGFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_CONFIGFS_FS) + return sb->s_magic == CONFIGFS_MAGIC; +#else + return 0; @@ -13346,7 +13520,7 @@ index 0000000..725b2ff + +static inline int au_test_minix(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_MINIX_FS) || defined(CONFIG_MINIX_FS_MODULE) ++#if IS_ENABLED(CONFIG_MINIX_FS) + return sb->s_magic == MINIX3_SUPER_MAGIC + || sb->s_magic == MINIX2_SUPER_MAGIC + || sb->s_magic == MINIX2_SUPER_MAGIC2 @@ -13359,7 +13533,7 @@ index 0000000..725b2ff + +static inline int au_test_fat(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_FAT_FS) || defined(CONFIG_FAT_FS_MODULE) ++#if IS_ENABLED(CONFIG_FAT_FS) + return sb->s_magic == MSDOS_SUPER_MAGIC; +#else + return 0; @@ -13387,7 +13561,7 @@ index 0000000..725b2ff + +static inline int au_test_squashfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_SQUASHFS) || defined(CONFIG_SQUASHFS_MODULE) ++#if IS_ENABLED(CONFIG_SQUASHFS) + return sb->s_magic == SQUASHFS_MAGIC; +#else + return 0; @@ -13396,7 +13570,7 @@ index 0000000..725b2ff + +static inline int au_test_btrfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_BTRFS_FS) || defined(CONFIG_BTRFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_BTRFS_FS) + return sb->s_magic == BTRFS_SUPER_MAGIC; +#else + return 0; @@ -13405,7 +13579,7 @@ index 0000000..725b2ff + +static inline int au_test_xenfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_XENFS) || defined(CONFIG_XENFS_MODULE) ++#if IS_ENABLED(CONFIG_XENFS) + return sb->s_magic == XENFS_SUPER_MAGIC; +#else + return 0; @@ -13423,7 +13597,7 @@ index 0000000..725b2ff + +static inline int au_test_nilfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_NILFS) || defined(CONFIG_NILFS_MODULE) ++#if IS_ENABLED(CONFIG_NILFS) + return sb->s_magic == NILFS_SUPER_MAGIC; +#else + return 0; @@ -13432,7 +13606,7 @@ index 0000000..725b2ff + +static inline int au_test_hfsplus(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_HFSPLUS_FS) || defined(CONFIG_HFSPLUS_FS_MODULE) ++#if IS_ENABLED(CONFIG_HFSPLUS_FS) + return sb->s_magic == HFSPLUS_SUPER_MAGIC; +#else + return 0; @@ -13593,7 +13767,7 @@ index 0000000..725b2ff +#endif /* __AUFS_FSTYPE_H__ */ diff --git a/fs/aufs/hfsnotify.c b/fs/aufs/hfsnotify.c new file mode 100644 -index 0000000..6afef3f +index 0000000..110f8f5 --- /dev/null +++ b/fs/aufs/hfsnotify.c @@ -0,0 +1,274 @@ @@ -13618,7 +13792,7 @@ index 0000000..6afef3f + struct au_hnotify *hn = container_of(mark, struct au_hnotify, + hn_mark); + /* AuDbg("here\n"); */ -+ au_cache_free_hnotify(hn); ++ au_cache_dfree_hnotify(hn); + smp_mb__before_atomic(); + if (atomic64_dec_and_test(&au_hfsn_ifree)) + wake_up(&au_hfsn_wq); @@ -13742,7 +13916,7 @@ index 0000000..6afef3f + struct au_br_hfsnotify *hfsn = group->private; + + /* AuDbg("here\n"); */ -+ kfree(hfsn); ++ au_delayed_kfree(hfsn); +} + +static int au_hfsn_handle_event(struct fsnotify_group *group, @@ -13836,7 +14010,7 @@ index 0000000..6afef3f + goto out; /* success */ + +out_hfsn: -+ kfree(hfsn); ++ au_delayed_kfree(hfsn); +out: + return err; +} @@ -13922,10 +14096,10 @@ index 0000000..145c6ac +} diff --git a/fs/aufs/hnotify.c b/fs/aufs/hnotify.c new file mode 100644 -index 0000000..3016c5e +index 0000000..a05aed8 --- /dev/null +++ b/fs/aufs/hnotify.c -@@ -0,0 +1,697 @@ +@@ -0,0 +1,710 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -13950,7 +14124,7 @@ index 0000000..3016c5e + AuTraceErr(err); + if (unlikely(err)) { + hinode->hi_notify = NULL; -+ au_cache_free_hnotify(hn); ++ au_cache_dfree_hnotify(hn); + /* + * The upper dir was removed by udba, but the same named + * dir left. In this case, aufs assignes a new inode @@ -13974,7 +14148,7 @@ index 0000000..3016c5e + if (hn) { + hinode->hi_notify = NULL; + if (au_hnotify_op.free(hinode, hn)) -+ au_cache_free_hnotify(hn); ++ au_cache_dfree_hnotify(hn); + } +} + @@ -14408,7 +14582,7 @@ index 0000000..3016c5e + || au_ftest_hnjob(a->flags[AuHn_CHILD], GEN))) { + inode = lookup_wlock_by_ino(sb, bfound, h_ino); + try_iput = 1; -+ } ++ } + + args.flags = a->flags[AuHn_CHILD]; + args.dentry = dentry; @@ -14447,7 +14621,7 @@ index 0000000..3016c5e + iput(a->dir); + si_write_unlock(sb); + au_nwt_done(&sbinfo->si_nowait); -+ kfree(a); ++ au_delayed_kfree(a); +} + +/* ---------------------------------------------------------------------- */ @@ -14553,7 +14727,7 @@ index 0000000..3016c5e + iput(args->h_child_inode); + iput(args->h_dir); + iput(args->dir); -+ kfree(args); ++ au_delayed_kfree(args); + } + +out: @@ -14594,17 +14768,26 @@ index 0000000..3016c5e + +static void au_hn_destroy_cache(void) +{ -+ kmem_cache_destroy(au_cachep[AuCache_HNOTIFY]); -+ au_cachep[AuCache_HNOTIFY] = NULL; ++ struct au_cache *cp; ++ ++ flush_delayed_work(&au_dfree.dwork); ++ cp = au_dfree.cache + AuCache_HNOTIFY; ++ AuDebugOn(!llist_empty(&cp->llist)); ++ kmem_cache_destroy(cp->cache); ++ cp->cache = NULL; +} + ++AU_CACHE_DFREE_FUNC(hnotify, HNOTIFY, hn_lnode); ++ +int __init au_hnotify_init(void) +{ + int err; ++ struct au_cache *cp; + + err = -ENOMEM; -+ au_cachep[AuCache_HNOTIFY] = AuCache(au_hnotify); -+ if (au_cachep[AuCache_HNOTIFY]) { ++ cp = au_dfree.cache + AuCache_HNOTIFY; ++ cp->cache = AuCache(au_hnotify); ++ if (cp->cache) { + err = 0; + if (au_hnotify_op.init) + err = au_hnotify_op.init(); @@ -14617,18 +14800,22 @@ index 0000000..3016c5e + +void au_hnotify_fin(void) +{ ++ struct au_cache *cp; ++ + if (au_hnotify_op.fin) + au_hnotify_op.fin(); ++ + /* cf. au_cache_fin() */ -+ if (au_cachep[AuCache_HNOTIFY]) ++ cp = au_dfree.cache + AuCache_HNOTIFY; ++ if (cp->cache) + au_hn_destroy_cache(); +} diff --git a/fs/aufs/i_op.c b/fs/aufs/i_op.c new file mode 100644 -index 0000000..cc1e312 +index 0000000..c730ec7 --- /dev/null +++ b/fs/aufs/i_op.c -@@ -0,0 +1,1401 @@ +@@ -0,0 +1,1438 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -14644,17 +14831,20 @@ index 0000000..cc1e312 +#include "aufs.h" + +static int h_permission(struct inode *h_inode, int mask, -+ struct vfsmount *h_mnt, int brperm) ++ struct path *h_path, int brperm) +{ + int err; + const unsigned char write_mask = !!(mask & (MAY_WRITE | MAY_APPEND)); + ++ err = -EPERM; ++ if (write_mask && IS_IMMUTABLE(h_inode)) ++ goto out; ++ + err = -EACCES; -+ if ((write_mask && IS_IMMUTABLE(h_inode)) -+ || ((mask & MAY_EXEC) -+ && S_ISREG(h_inode->i_mode) -+ && ((h_mnt->mnt_flags & MNT_NOEXEC) -+ || !(h_inode->i_mode & S_IXUGO)))) ++ if (((mask & MAY_EXEC) ++ && S_ISREG(h_inode->i_mode) ++ && (path_noexec(h_path) ++ || !(h_inode->i_mode & S_IXUGO)))) + goto out; + + /* @@ -14738,7 +14928,7 @@ index 0000000..cc1e312 + err = 0; + bindex = au_ibtop(inode); + br = au_sbr(sb, bindex); -+ err = h_permission(h_inode, mask, au_br_mnt(br), br->br_perm); ++ err = h_permission(h_inode, mask, &br->br_path, br->br_perm); + if (write_mask + && !err + && !special_file(h_inode->i_mode)) { @@ -14764,7 +14954,7 @@ index 0000000..cc1e312 + break; + + br = au_sbr(sb, bindex); -+ err = h_permission(h_inode, mask, au_br_mnt(br), ++ err = h_permission(h_inode, mask, &br->br_path, + br->br_perm); + } + } @@ -14899,8 +15089,7 @@ index 0000000..cc1e312 +{ + int err, h_opened = *opened; + unsigned int lkup_flags; -+ struct dentry *parent; -+ struct dentry *d; ++ struct dentry *parent, *d; + struct au_sphlhead *aopen; + struct vfsub_aopen_args args = { + .open_flag = open_flag, @@ -14995,10 +15184,10 @@ index 0000000..cc1e312 + di_write_unlock(parent); + aufs_read_unlock(dentry, AuLock_DW); + AuDbgDentry(dentry); -+ if (unlikely(err)) ++ if (unlikely(err < 0)) + goto out; +out_no_open: -+ if (!err && !(*opened & FILE_CREATED)) { ++ if (err >= 0 && !(*opened & FILE_CREATED)) { + AuLabel(out_no_open); + dget(dentry); + err = finish_no_open(file, dentry); @@ -15612,7 +15801,7 @@ index 0000000..cc1e312 +out_si: + si_read_unlock(sb); +out_kfree: -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; @@ -15705,7 +15894,7 @@ index 0000000..cc1e312 + di_write_unlock(dentry); + si_read_unlock(sb); +out_kfree: -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; @@ -15925,20 +16114,55 @@ index 0000000..cc1e312 + +/* ---------------------------------------------------------------------- */ + ++static int au_is_special(struct inode *inode) ++{ ++ return (inode->i_mode & (S_IFBLK | S_IFCHR | S_IFIFO | S_IFSOCK)); ++} ++ +static int aufs_update_time(struct inode *inode, struct timespec *ts, int flags) +{ + int err; ++ aufs_bindex_t bindex; + struct super_block *sb; + struct inode *h_inode; ++ struct vfsmount *h_mnt; + + sb = inode->i_sb; ++ WARN_ONCE((flags & S_ATIME) && !IS_NOATIME(inode), ++ "unexpected s_flags 0x%lx", sb->s_flags); ++ + /* mmap_sem might be acquired already, cf. aufs_mmap() */ + lockdep_off(); + si_read_lock(sb, AuLock_FLUSH); + ii_write_lock_child(inode); + lockdep_on(); -+ h_inode = au_h_iptr(inode, au_ibtop(inode)); -+ err = vfsub_update_time(h_inode, ts, flags); ++ ++ err = 0; ++ bindex = au_ibtop(inode); ++ h_inode = au_h_iptr(inode, bindex); ++ if (!au_test_ro(sb, bindex, inode)) { ++ h_mnt = au_sbr_mnt(sb, bindex); ++ err = vfsub_mnt_want_write(h_mnt); ++ if (!err) { ++ err = vfsub_update_time(h_inode, ts, flags); ++ vfsub_mnt_drop_write(h_mnt); ++ } ++ } else if (au_is_special(h_inode)) { ++ /* ++ * Never copy-up here. ++ * These special files may already be opened and used for ++ * communicating. If we copied it up, then the communication ++ * would be corrupted. ++ */ ++ AuWarn1("timestamps for i%lu are ignored " ++ "since it is on readonly branch (hi%lu).\n", ++ inode->i_ino, h_inode->i_ino); ++ } else if (flags & ~S_ATIME) { ++ err = -EIO; ++ AuIOErr1("unexpected flags 0x%x\n", flags); ++ AuDebugOn(1); ++ } ++ + lockdep_off(); + if (!err) + au_cpup_attr_timesizes(inode); @@ -16032,7 +16256,7 @@ index 0000000..cc1e312 +}; diff --git a/fs/aufs/i_op_add.c b/fs/aufs/i_op_add.c new file mode 100644 -index 0000000..8e3fb61 +index 0000000..2a9aa09 --- /dev/null +++ b/fs/aufs/i_op_add.c @@ -0,0 +1,911 @@ @@ -16370,7 +16594,7 @@ index 0000000..8e3fb61 + if (!try_aopen) + aufs_read_unlock(dentry, AuLock_DW); +out_free: -+ kfree(a); ++ au_delayed_kfree(a); +out: + return err; +} @@ -16834,7 +17058,7 @@ index 0000000..8e3fb61 + } + aufs_read_and_write_unlock2(dentry, src_dentry); +out_kfree: -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; @@ -16943,13 +17167,13 @@ index 0000000..8e3fb61 + } + aufs_read_unlock(dentry, AuLock_DW); +out_free: -+ kfree(a); ++ au_delayed_kfree(a); +out: + return err; +} diff --git a/fs/aufs/i_op_del.c b/fs/aufs/i_op_del.c new file mode 100644 -index 0000000..35b923c +index 0000000..2fe8f07 --- /dev/null +++ b/fs/aufs/i_op_del.c @@ -0,0 +1,498 @@ @@ -17336,7 +17560,7 @@ index 0000000..35b923c +out_unlock: + aufs_read_unlock(dentry, AuLock_DW); +out_free: -+ kfree(a); ++ au_delayed_kfree(a); +out: + return err; +} @@ -17446,14 +17670,14 @@ index 0000000..35b923c +out_unlock: + aufs_read_unlock(dentry, AuLock_DW); +out_free: -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/i_op_ren.c b/fs/aufs/i_op_ren.c new file mode 100644 -index 0000000..df697de +index 0000000..43a0909 --- /dev/null +++ b/fs/aufs/i_op_ren.c @@ -0,0 +1,1002 @@ @@ -18454,17 +18678,17 @@ index 0000000..df697de + iput(a->dst_inode); + if (a->thargs) + au_whtmp_rmdir_free(a->thargs); -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/iinfo.c b/fs/aufs/iinfo.c new file mode 100644 -index 0000000..fae4c27 +index 0000000..975cff3 --- /dev/null +++ b/fs/aufs/iinfo.c -@@ -0,0 +1,271 @@ +@@ -0,0 +1,272 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -18672,7 +18896,7 @@ index 0000000..fae4c27 + return -ENOMEM; +} + -+int au_hinode_realloc(struct au_iinfo *iinfo, int nbr) ++int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink) +{ + int err, i; + struct au_hinode *hip; @@ -18680,7 +18904,8 @@ index 0000000..fae4c27 + AuRwMustWriteLock(&iinfo->ii_rwsem); + + err = -ENOMEM; -+ hip = krealloc(iinfo->ii_hinode, sizeof(*hip) * nbr, GFP_NOFS); ++ hip = au_krealloc(iinfo->ii_hinode, sizeof(*hip) * nbr, GFP_NOFS, ++ may_shrink); + if (hip) { + iinfo->ii_hinode = hip; + i = iinfo->ii_bbot + 1; @@ -18721,7 +18946,7 @@ index 0000000..fae4c27 + + iinfo = au_ii(inode); + if (iinfo->ii_vdir) -+ au_vdir_free(iinfo->ii_vdir); ++ au_vdir_free(iinfo->ii_vdir, /*atonce*/0); + + bindex = iinfo->ii_btop; + if (bindex >= 0) { @@ -18733,15 +18958,15 @@ index 0000000..fae4c27 + hi++; + } + } -+ kfree(iinfo->ii_hinode); ++ au_delayed_kfree(iinfo->ii_hinode); + AuRwDestroy(&iinfo->ii_rwsem); +} diff --git a/fs/aufs/inode.c b/fs/aufs/inode.c new file mode 100644 -index 0000000..a4b994e +index 0000000..8f7446d --- /dev/null +++ b/fs/aufs/inode.c -@@ -0,0 +1,504 @@ +@@ -0,0 +1,506 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -18771,7 +18996,7 @@ index 0000000..a4b994e + +static int au_ii_refresh(struct inode *inode, int *update) +{ -+ int err, e; ++ int err, e, nbr; + umode_t type; + aufs_bindex_t bindex, new_bindex; + struct super_block *sb; @@ -18783,9 +19008,10 @@ index 0000000..a4b994e + + *update = 0; + sb = inode->i_sb; ++ nbr = au_sbbot(sb) + 1; + type = inode->i_mode & S_IFMT; + iinfo = au_ii(inode); -+ err = au_hinode_realloc(iinfo, au_sbbot(sb) + 1); ++ err = au_hinode_realloc(iinfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + @@ -18823,6 +19049,7 @@ index 0000000..a4b994e + } + } + au_update_ibrange(inode, /*do_put_zero*/0); ++ au_hinode_realloc(iinfo, nbr, /*may_shrink*/1); /* harmless if err */ + e = au_dy_irefresh(inode); + if (unlikely(e && !err)) + err = e; @@ -19248,10 +19475,10 @@ index 0000000..a4b994e +} diff --git a/fs/aufs/inode.h b/fs/aufs/inode.h new file mode 100644 -index 0000000..3fc5a58 +index 0000000..33d9f5e --- /dev/null +++ b/fs/aufs/inode.h -@@ -0,0 +1,681 @@ +@@ -0,0 +1,687 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -19276,7 +19503,10 @@ index 0000000..3fc5a58 + /* never use fsnotify_add_vfsmount_mark() */ + struct fsnotify_mark hn_mark; +#endif -+ struct inode *hn_aufs_inode; /* no get/put */ ++ union { ++ struct inode *hn_aufs_inode; /* no get/put */ ++ struct llist_node hn_lnode; /* delayed free */ ++ }; +#endif +} ____cacheline_aligned_in_smp; + @@ -19319,7 +19549,10 @@ index 0000000..3fc5a58 +struct au_icntnr { + struct au_iinfo iinfo; + struct inode vfs_inode; -+ struct hlist_node plink; ++ union { ++ struct hlist_node plink; ++ struct llist_node lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +/* au_pin flags */ @@ -19506,7 +19739,7 @@ index 0000000..3fc5a58 +void au_hinode_init(struct au_hinode *hinode); +int au_iinfo_init(struct inode *inode); +void au_iinfo_fin(struct inode *inode); -+int au_hinode_realloc(struct au_iinfo *iinfo, int nbr); ++int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink); + +#ifdef CONFIG_PROC_FS +/* plink.c */ @@ -20147,10 +20380,10 @@ index 0000000..2ebfdc4 +#endif diff --git a/fs/aufs/loop.c b/fs/aufs/loop.c new file mode 100644 -index 0000000..5711e7a +index 0000000..c3ca50f --- /dev/null +++ b/fs/aufs/loop.c -@@ -0,0 +1,133 @@ +@@ -0,0 +1,134 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -20244,7 +20477,8 @@ index 0000000..5711e7a + new_nelem = au_warn_loopback_nelem + au_warn_loopback_step; + a = au_kzrealloc(au_warn_loopback_array, + au_warn_loopback_nelem * sizeof(unsigned long), -+ new_nelem * sizeof(unsigned long), GFP_ATOMIC); ++ new_nelem * sizeof(unsigned long), GFP_ATOMIC, ++ /*may_shrink*/0); + if (a) { + au_warn_loopback_nelem = new_nelem; + au_warn_loopback_array = a; @@ -20282,7 +20516,7 @@ index 0000000..5711e7a +{ + if (backing_file_func) + symbol_put(loop_backing_file); -+ kfree(au_warn_loopback_array); ++ au_delayed_kfree(au_warn_loopback_array); +} diff --git a/fs/aufs/loop.h b/fs/aufs/loop.h new file mode 100644 @@ -20367,10 +20601,10 @@ index 0000000..4f83bdf +endif diff --git a/fs/aufs/module.c b/fs/aufs/module.c new file mode 100644 -index 0000000..88f8f41 +index 0000000..5092b4b --- /dev/null +++ b/fs/aufs/module.c -@@ -0,0 +1,209 @@ +@@ -0,0 +1,319 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -20383,29 +20617,120 @@ index 0000000..88f8f41 +#include +#include "aufs.h" + -+void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp) ++/* shrinkable realloc */ ++void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink) +{ -+ if (new_sz <= nused) -+ return p; ++ size_t sz; ++ int diff; + -+ p = krealloc(p, new_sz, gfp); -+ if (p) ++ sz = 0; ++ diff = -1; ++ if (p) { ++#if 0 /* unused */ ++ if (!new_sz) { ++ au_delayed_kfree(p); ++ p = NULL; ++ goto out; ++ } ++#else ++ AuDebugOn(!new_sz); ++#endif ++ sz = ksize(p); ++ diff = au_kmidx_sub(sz, new_sz); ++ } ++ if (sz && !diff) ++ goto out; ++ ++ if (sz < new_sz) ++ /* expand or SLOB */ ++ p = krealloc(p, new_sz, gfp); ++ else if (new_sz < sz && may_shrink) { ++ /* shrink */ ++ void *q; ++ ++ q = kmalloc(new_sz, gfp); ++ if (q) { ++ if (p) { ++ memcpy(q, p, new_sz); ++ au_delayed_kfree(p); ++ } ++ p = q; ++ } else ++ p = NULL; ++ } ++ ++out: ++ return p; ++} ++ ++void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp, ++ int may_shrink) ++{ ++ p = au_krealloc(p, new_sz, gfp, may_shrink); ++ if (p && new_sz > nused) + memset(p + nused, 0, new_sz - nused); + return p; +} + +/* ---------------------------------------------------------------------- */ -+ +/* + * aufs caches + */ -+struct kmem_cache *au_cachep[AuCache_Last] = { -+ [0] = NULL -+}; ++ ++struct au_dfree au_dfree; ++ ++/* delayed free */ ++static void au_do_dfree(struct work_struct *work __maybe_unused) ++{ ++ struct llist_head *head; ++ struct llist_node *node, *next; ++ ++#define AU_CACHE_DFREE_DO_BODY(name, idx, lnode) do { \ ++ head = &au_dfree.cache[AuCache_##idx].llist; \ ++ node = llist_del_all(head); \ ++ for (; node; node = next) { \ ++ struct au_##name *p \ ++ = llist_entry(node, struct au_##name, \ ++ lnode); \ ++ next = llist_next(node); \ ++ au_cache_free_##name(p); \ ++ } \ ++ } while (0) ++ ++ AU_CACHE_DFREE_DO_BODY(dinfo, DINFO, di_lnode); ++ AU_CACHE_DFREE_DO_BODY(icntnr, ICNTNR, lnode); ++ AU_CACHE_DFREE_DO_BODY(finfo, FINFO, fi_lnode); ++ AU_CACHE_DFREE_DO_BODY(vdir, VDIR, vd_lnode); ++ AU_CACHE_DFREE_DO_BODY(vdir_dehstr, DEHSTR, lnode); ++#ifdef CONFIG_AUFS_HNOTIFY ++ AU_CACHE_DFREE_DO_BODY(hnotify, HNOTIFY, hn_lnode); ++#endif ++ ++#define AU_DFREE_DO_BODY(llist, func) do { \ ++ node = llist_del_all(llist); \ ++ for (; node; node = next) { \ ++ next = llist_next(node); \ ++ func(node); \ ++ } \ ++ } while (0) ++ ++ AU_DFREE_DO_BODY(au_dfree.llist + AU_DFREE_KFREE, kfree); ++ AU_DFREE_DO_BODY(au_dfree.llist + AU_DFREE_FREE_PAGE, au_free_page); ++ ++#undef AU_CACHE_DFREE_DO_BODY ++#undef AU_DFREE_DO_BODY ++} ++ ++AU_CACHE_DFREE_FUNC(dinfo, DINFO, di_lnode); ++AU_CACHE_DFREE_FUNC(icntnr, ICNTNR, lnode); ++AU_CACHE_DFREE_FUNC(finfo, FINFO, fi_lnode); ++AU_CACHE_DFREE_FUNC(vdir, VDIR, vd_lnode); ++AU_CACHE_DFREE_FUNC(vdir_dehstr, DEHSTR, lnode); + +static void au_cache_fin(void) +{ + int i; ++ struct au_cache *cp; + + /* + * Make sure all delayed rcu free inodes are flushed before we @@ -20415,27 +20740,33 @@ index 0000000..88f8f41 + + /* excluding AuCache_HNOTIFY */ + BUILD_BUG_ON(AuCache_HNOTIFY + 1 != AuCache_Last); ++ flush_delayed_work(&au_dfree.dwork); + for (i = 0; i < AuCache_HNOTIFY; i++) { -+ kmem_cache_destroy(au_cachep[i]); -+ au_cachep[i] = NULL; ++ cp = au_dfree.cache + i; ++ AuDebugOn(!llist_empty(&cp->llist)); ++ kmem_cache_destroy(cp->cache); ++ cp->cache = NULL; + } +} + +static int __init au_cache_init(void) +{ -+ au_cachep[AuCache_DINFO] = AuCacheCtor(au_dinfo, au_di_init_once); -+ if (au_cachep[AuCache_DINFO]) ++ struct au_cache *cp; ++ ++ cp = au_dfree.cache; ++ cp[AuCache_DINFO].cache = AuCacheCtor(au_dinfo, au_di_init_once); ++ if (cp[AuCache_DINFO].cache) + /* SLAB_DESTROY_BY_RCU */ -+ au_cachep[AuCache_ICNTNR] = AuCacheCtor(au_icntnr, -+ au_icntnr_init_once); -+ if (au_cachep[AuCache_ICNTNR]) -+ au_cachep[AuCache_FINFO] = AuCacheCtor(au_finfo, -+ au_fi_init_once); -+ if (au_cachep[AuCache_FINFO]) -+ au_cachep[AuCache_VDIR] = AuCache(au_vdir); -+ if (au_cachep[AuCache_VDIR]) -+ au_cachep[AuCache_DEHSTR] = AuCache(au_vdir_dehstr); -+ if (au_cachep[AuCache_DEHSTR]) ++ cp[AuCache_ICNTNR].cache = AuCacheCtor(au_icntnr, ++ au_icntnr_init_once); ++ if (cp[AuCache_ICNTNR].cache) ++ cp[AuCache_FINFO].cache = AuCacheCtor(au_finfo, ++ au_fi_init_once); ++ if (cp[AuCache_FINFO].cache) ++ cp[AuCache_VDIR].cache = AuCache(au_vdir); ++ if (cp[AuCache_VDIR].cache) ++ cp[AuCache_DEHSTR].cache = AuCache(au_vdir_dehstr); ++ if (cp[AuCache_DEHSTR].cache) + return 0; + + au_cache_fin(); @@ -20497,6 +20828,7 @@ index 0000000..88f8f41 +{ + int err, i; + char *p; ++ struct au_cache *cp; + + p = au_esc_chars; + for (i = 1; i <= ' '; i++) @@ -20511,6 +20843,16 @@ index 0000000..88f8f41 + for (i = 0; i < AuIop_Last; i++) + aufs_iop_nogetattr[i].getattr = NULL; + ++ /* First, initialize au_dfree */ ++ for (i = 0; i < AuCache_Last; i++) { /* including hnotify */ ++ cp = au_dfree.cache + i; ++ cp->cache = NULL; ++ init_llist_head(&cp->llist); ++ } ++ for (i = 0; i < AU_DFREE_Last; i++) ++ init_llist_head(au_dfree.llist + i); ++ INIT_DELAYED_WORK(&au_dfree.dwork, au_do_dfree); ++ + au_sbilist_init(); + sysaufs_brs_init(); + au_debug_init(); @@ -20561,6 +20903,7 @@ index 0000000..88f8f41 +out_sysaufs: + sysaufs_fin(); + au_dy_fin(); ++ flush_delayed_work(&au_dfree.dwork); +out: + return err; +} @@ -20576,16 +20919,17 @@ index 0000000..88f8f41 + au_procfs_fin(); + sysaufs_fin(); + au_dy_fin(); ++ flush_delayed_work(&au_dfree.dwork); +} + +module_init(aufs_init); +module_exit(aufs_exit); diff --git a/fs/aufs/module.h b/fs/aufs/module.h new file mode 100644 -index 0000000..1383e3d +index 0000000..6f968ba --- /dev/null +++ b/fs/aufs/module.h -@@ -0,0 +1,76 @@ +@@ -0,0 +1,143 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -20600,6 +20944,7 @@ index 0000000..1383e3d +#ifdef __KERNEL__ + +#include ++#include "debug.h" + +struct path; +struct seq_file; @@ -20612,7 +20957,19 @@ index 0000000..1383e3d + +extern int au_dir_roflags; + -+void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp); ++void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink); ++void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp, ++ int may_shrink); ++ ++static inline int au_kmidx_sub(size_t sz, size_t new_sz) ++{ ++#ifndef CONFIG_SLOB ++ return kmalloc_index(sz) - kmalloc_index(new_sz); ++#else ++ return -1; /* SLOB is untested */ ++#endif ++} ++ +int au_seq_path(struct seq_file *seq, struct path *path); + +#ifdef CONFIG_PROC_FS @@ -20626,7 +20983,7 @@ index 0000000..1383e3d + +/* ---------------------------------------------------------------------- */ + -+/* kmem cache */ ++/* kmem cache and delayed free */ +enum { + AuCache_DINFO, + AuCache_ICNTNR, @@ -20637,19 +20994,54 @@ index 0000000..1383e3d + AuCache_Last +}; + ++enum { ++ AU_DFREE_KFREE, ++ AU_DFREE_FREE_PAGE, ++ AU_DFREE_Last ++}; ++ ++struct au_cache { ++ struct kmem_cache *cache; ++ struct llist_head llist; /* delayed free */ ++}; ++ ++/* ++ * in order to reduce the cost of the internal timer, consolidate all the ++ * delayed free works into a single delayed_work. ++ */ ++struct au_dfree { ++ struct au_cache cache[AuCache_Last]; ++ struct llist_head llist[AU_DFREE_Last]; ++ struct delayed_work dwork; ++}; ++ ++extern struct au_dfree au_dfree; ++ +#define AuCacheFlags (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD) +#define AuCache(type) KMEM_CACHE(type, AuCacheFlags) +#define AuCacheCtor(type, ctor) \ + kmem_cache_create(#type, sizeof(struct type), \ + __alignof__(struct type), AuCacheFlags, ctor) + -+extern struct kmem_cache *au_cachep[]; ++#define AU_DFREE_DELAY msecs_to_jiffies(10) ++#define AU_DFREE_BODY(lnode, llist) do { \ ++ if (llist_add(lnode, llist)) \ ++ schedule_delayed_work(&au_dfree.dwork, \ ++ AU_DFREE_DELAY); \ ++ } while (0) ++#define AU_CACHE_DFREE_FUNC(name, idx, lnode) \ ++ void au_cache_dfree_##name(struct au_##name *p) \ ++ { \ ++ struct au_cache *cp = au_dfree.cache + AuCache_##idx; \ ++ AU_DFREE_BODY(&p->lnode, &cp->llist); \ ++ } + +#define AuCacheFuncs(name, index) \ +static inline struct au_##name *au_cache_alloc_##name(void) \ -+{ return kmem_cache_alloc(au_cachep[AuCache_##index], GFP_NOFS); } \ ++{ return kmem_cache_alloc(au_dfree.cache[AuCache_##index].cache, GFP_NOFS); } \ +static inline void au_cache_free_##name(struct au_##name *p) \ -+{ kmem_cache_free(au_cachep[AuCache_##index], p); } ++{ kmem_cache_free(au_dfree.cache[AuCache_##index].cache, p); } \ ++void au_cache_dfree_##name(struct au_##name *p) + +AuCacheFuncs(dinfo, DINFO); +AuCacheFuncs(icntnr, ICNTNR); @@ -20660,11 +21052,30 @@ index 0000000..1383e3d +AuCacheFuncs(hnotify, HNOTIFY); +#endif + ++static inline void au_delayed_kfree(const void *p) ++{ ++ AuDebugOn(!p); ++ AuDebugOn(ksize(p) < sizeof(struct llist_node)); ++ ++ AU_DFREE_BODY((void *)p, au_dfree.llist + AU_DFREE_KFREE); ++} ++ ++/* cast only */ ++static inline void au_free_page(void *p) ++{ ++ free_page((unsigned long)p); ++} ++ ++static inline void au_delayed_free_page(unsigned long addr) ++{ ++ AU_DFREE_BODY((void *)addr, au_dfree.llist + AU_DFREE_FREE_PAGE); ++} ++ +#endif /* __KERNEL__ */ +#endif /* __AUFS_MODULE_H__ */ diff --git a/fs/aufs/mvdown.c b/fs/aufs/mvdown.c new file mode 100644 -index 0000000..a48aa1c +index 0000000..b0e5b84 --- /dev/null +++ b/fs/aufs/mvdown.c @@ -0,0 +1,691 @@ @@ -21354,17 +21765,17 @@ index 0000000..a48aa1c + e = copy_to_user(uarg, &args->mvdown, sizeof(args->mvdown)); + if (unlikely(e)) + err = -EFAULT; -+ kfree(args); ++ au_delayed_kfree(args); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/opts.c b/fs/aufs/opts.c new file mode 100644 -index 0000000..79c7d71 +index 0000000..1d21f0d --- /dev/null +++ b/fs/aufs/opts.c -@@ -0,0 +1,1846 @@ +@@ -0,0 +1,1847 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -22610,7 +23021,7 @@ index 0000000..79c7d71 + } + } + -+ kfree(a); ++ au_delayed_kfree(a); + dump_opts(opts); + if (unlikely(err)) + au_opts_free(opts); @@ -23032,7 +23443,8 @@ index 0000000..79c7d71 + au_hn_inode_unlock(hdir); + + if (!err && do_free) { -+ kfree(wbr); ++ if (wbr) ++ au_delayed_kfree(wbr); + br->br_wbr = NULL; + } + } @@ -23417,10 +23829,10 @@ index 0000000..0d6c2e1 +#endif /* __AUFS_OPTS_H__ */ diff --git a/fs/aufs/plink.c b/fs/aufs/plink.c new file mode 100644 -index 0000000..c42734d +index 0000000..d8d75e8 --- /dev/null +++ b/fs/aufs/plink.c -@@ -0,0 +1,489 @@ +@@ -0,0 +1,501 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -23455,6 +23867,7 @@ index 0000000..c42734d +{ + int err; + pid_t pid, ppid; ++ struct task_struct *parent, *prev; + struct au_sbinfo *sbi; + + SiMustAnyLock(sb); @@ -23469,11 +23882,22 @@ index 0000000..c42734d + goto out; + + /* todo: it highly depends upon /sbin/mount.aufs */ ++ prev = NULL; ++ parent = current; ++ ppid = 0; + rcu_read_lock(); -+ ppid = task_pid_vnr(rcu_dereference(current->real_parent)); ++ while (1) { ++ parent = rcu_dereference(parent->real_parent); ++ if (parent == prev) ++ break; ++ ppid = task_pid_vnr(parent); ++ if (pid == ppid) { ++ rcu_read_unlock(); ++ goto out; ++ } ++ prev = parent; ++ } + rcu_read_unlock(); -+ if (pid == ppid) -+ goto out; + + if (au_ftest_lock(flags, NOPLMW)) { + /* if there is no i_mutex lock in VFS, we don't need to wait */ @@ -24775,10 +25199,10 @@ index 0000000..6c0d5a9 +#endif /* __AUFS_RWSEM_H__ */ diff --git a/fs/aufs/sbinfo.c b/fs/aufs/sbinfo.c new file mode 100644 -index 0000000..aa97059 +index 0000000..e113b51 --- /dev/null +++ b/fs/aufs/sbinfo.c -@@ -0,0 +1,340 @@ +@@ -0,0 +1,342 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -24801,7 +25225,7 @@ index 0000000..aa97059 + sbinfo = container_of(kobj, struct au_sbinfo, si_kobj); + for (i = 0; i < AuPlink_NHASH; i++) + AuDebugOn(!hlist_empty(&sbinfo->si_plink[i].head)); -+ au_nwt_fin(&sbinfo->si_nowait); ++ AuDebugOn(atomic_read(&sbinfo->si_nowait.nw_len)); + + AuDebugOn(percpu_counter_sum(&sbinfo->si_ninodes)); + percpu_counter_destroy(&sbinfo->si_ninodes); @@ -24812,14 +25236,15 @@ index 0000000..aa97059 + au_br_free(sbinfo); + au_rw_write_unlock(&sbinfo->si_rwsem); + -+ kfree(sbinfo->si_branch); ++ au_delayed_kfree(sbinfo->si_branch); + for (i = 0; i < AU_NPIDMAP; i++) -+ kfree(sbinfo->au_si_pid.pid_bitmap[i]); ++ if (sbinfo->au_si_pid.pid_bitmap[i]) ++ au_delayed_kfree(sbinfo->au_si_pid.pid_bitmap[i]); + mutex_destroy(&sbinfo->au_si_pid.pid_mtx); + mutex_destroy(&sbinfo->si_xib_mtx); + AuRwDestroy(&sbinfo->si_rwsem); + -+ kfree(sbinfo); ++ au_delayed_kfree(sbinfo); +} + +int au_si_alloc(struct super_block *sb) @@ -24891,14 +25316,14 @@ index 0000000..aa97059 + return 0; /* success */ + +out_br: -+ kfree(sbinfo->si_branch); ++ au_delayed_kfree(sbinfo->si_branch); +out_sbinfo: -+ kfree(sbinfo); ++ au_delayed_kfree(sbinfo); +out: + return err; +} + -+int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr) ++int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink) +{ + int err, sz; + struct au_branch **brp; @@ -24909,7 +25334,8 @@ index 0000000..aa97059 + sz = sizeof(*brp) * (sbinfo->si_bbot + 1); + if (unlikely(!sz)) + sz = sizeof(*brp); -+ brp = au_kzrealloc(sbinfo->si_branch, sz, sizeof(*brp) * nbr, GFP_NOFS); ++ brp = au_kzrealloc(sbinfo->si_branch, sz, sizeof(*brp) * nbr, GFP_NOFS, ++ may_shrink); + if (brp) { + sbinfo->si_branch = brp; + err = 0; @@ -25121,10 +25547,10 @@ index 0000000..aa97059 +} diff --git a/fs/aufs/spl.h b/fs/aufs/spl.h new file mode 100644 -index 0000000..f9b5288 +index 0000000..8f519db3 --- /dev/null +++ b/fs/aufs/spl.h -@@ -0,0 +1,98 @@ +@@ -0,0 +1,100 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -25138,6 +25564,7 @@ index 0000000..f9b5288 + +#ifdef __KERNEL__ + ++#if 0 +struct au_splhead { + spinlock_t spin; + struct list_head head; @@ -25170,6 +25597,7 @@ index 0000000..f9b5288 + list_del_rcu(list); + spin_unlock(&spl->spin); +} ++#endif + +/* ---------------------------------------------------------------------- */ + @@ -25225,10 +25653,10 @@ index 0000000..f9b5288 +#endif /* __AUFS_SPL_H__ */ diff --git a/fs/aufs/super.c b/fs/aufs/super.c new file mode 100644 -index 0000000..8bd2d9c +index 0000000..58a773c --- /dev/null +++ b/fs/aufs/super.c -@@ -0,0 +1,1026 @@ +@@ -0,0 +1,1025 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -25264,8 +25692,7 @@ index 0000000..8bd2d9c +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + -+ INIT_HLIST_HEAD(&inode->i_dentry); -+ au_cache_free_icntnr(container_of(inode, struct au_icntnr, vfs_inode)); ++ au_cache_dfree_icntnr(container_of(inode, struct au_icntnr, vfs_inode)); +} + +static void aufs_destroy_inode(struct inode *inode) @@ -26047,7 +26474,7 @@ index 0000000..8bd2d9c +out_mtx: + inode_unlock(inode); +out_opts: -+ free_page((unsigned long)opts.opt); ++ au_delayed_free_page((unsigned long)opts.opt); +out: + err = cvt_err(err); + AuTraceErr(err); @@ -26188,7 +26615,7 @@ index 0000000..8bd2d9c + kobject_put(&sbinfo->si_kobj); + sb->s_fs_info = NULL; +out_opts: -+ free_page((unsigned long)opts.opt); ++ au_delayed_free_page((unsigned long)opts.opt); +out: + AuTraceErr(err); + err = cvt_err(err); @@ -26257,7 +26684,7 @@ index 0000000..8bd2d9c +}; diff --git a/fs/aufs/super.h b/fs/aufs/super.h new file mode 100644 -index 0000000..b8ca149 +index 0000000..e4ac866 --- /dev/null +++ b/fs/aufs/super.h @@ -0,0 +1,625 @@ @@ -26529,7 +26956,7 @@ index 0000000..b8ca149 +/* sbinfo.c */ +void au_si_free(struct kobject *kobj); +int au_si_alloc(struct super_block *sb); -+int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr); ++int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink); + +unsigned int au_sigen_inc(struct super_block *sb); +aufs_bindex_t au_new_br_id(struct super_block *sb); @@ -27079,7 +27506,7 @@ index 0000000..1f79983 +#endif /* __SYSAUFS_H__ */ diff --git a/fs/aufs/sysfs.c b/fs/aufs/sysfs.c new file mode 100644 -index 0000000..0efb77a +index 0000000..f4c3170 --- /dev/null +++ b/fs/aufs/sysfs.c @@ -0,0 +1,340 @@ @@ -27258,7 +27685,7 @@ index 0000000..0efb77a + if (unlikely(err == PAGE_SIZE)) + err = -EFBIG; + } -+ kfree(seq); ++ au_delayed_kfree(seq); +out_unlock: + si_read_unlock(sb); +out: @@ -27329,9 +27756,9 @@ index 0000000..0efb77a + err = -EFAULT; + +out_seq: -+ kfree(seq); ++ au_delayed_kfree(seq); +out_buf: -+ free_page((unsigned long)buf); ++ au_delayed_free_page((unsigned long)buf); +out: + si_read_unlock(sb); + return err; @@ -27575,10 +28002,10 @@ index 0000000..7d22979 +} diff --git a/fs/aufs/vdir.c b/fs/aufs/vdir.c new file mode 100644 -index 0000000..0a9ff00 +index 0000000..b2eb4c0 --- /dev/null +++ b/fs/aufs/vdir.c -@@ -0,0 +1,876 @@ +@@ -0,0 +1,887 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -27678,7 +28105,7 @@ index 0000000..0a9ff00 + struct hlist_node *node; + + hlist_for_each_entry_safe(pos, node, head, wh_hash) -+ kfree(pos); ++ au_delayed_kfree(pos); +} + +static void au_nhash_de_do_free(struct hlist_head *head) @@ -27687,7 +28114,7 @@ index 0000000..0a9ff00 + struct hlist_node *node; + + hlist_for_each_entry_safe(pos, node, head, hash) -+ au_cache_free_vdir_dehstr(pos); ++ au_cache_dfree_vdir_dehstr(pos); +} + +static void au_nhash_do_free(struct au_nhash *nhash, @@ -27705,7 +28132,7 @@ index 0000000..0a9ff00 + nhash_count(head); + free(head++); + } -+ kfree(nhash->nh_head); ++ au_delayed_kfree(nhash->nh_head); +} + +void au_nhash_wh_free(struct au_nhash *whlist) @@ -27748,6 +28175,8 @@ index 0000000..0a9ff00 + AuDebugOn(!nhash->nh_num || !nhash->nh_head); + + v = 0; ++ if (len > 8) ++ len = 8; + while (len--) + v += *name++; + /* v = hash_long(v, magic_bit); */ @@ -27845,8 +28274,8 @@ index 0000000..0a9ff00 + unsigned char **o; + + err = -ENOMEM; -+ o = krealloc(vdir->vd_deblk, sizeof(*o) * (vdir->vd_nblk + 1), -+ GFP_NOFS); ++ o = au_krealloc(vdir->vd_deblk, sizeof(*o) * (vdir->vd_nblk + 1), ++ GFP_NOFS, /*may_shrink*/0); + if (unlikely(!o)) + goto out; + @@ -27916,15 +28345,23 @@ index 0000000..0a9ff00 + +/* ---------------------------------------------------------------------- */ + -+void au_vdir_free(struct au_vdir *vdir) ++void au_vdir_free(struct au_vdir *vdir, int atonce) +{ + unsigned char **deblk; + + deblk = vdir->vd_deblk; -+ while (vdir->vd_nblk--) -+ kfree(*deblk++); -+ kfree(vdir->vd_deblk); -+ au_cache_free_vdir(vdir); ++ if (!atonce) { ++ while (vdir->vd_nblk--) ++ au_delayed_kfree(*deblk++); ++ au_delayed_kfree(vdir->vd_deblk); ++ au_cache_dfree_vdir(vdir); ++ } else { ++ /* not delayed */ ++ while (vdir->vd_nblk--) ++ kfree(*deblk++); ++ kfree(vdir->vd_deblk); ++ au_cache_free_vdir(vdir); ++ } +} + +static struct au_vdir *alloc_vdir(struct file *file) @@ -27958,10 +28395,10 @@ index 0000000..0a9ff00 + if (!err) + return vdir; /* success */ + -+ kfree(vdir->vd_deblk); ++ au_delayed_kfree(vdir->vd_deblk); + +out_free: -+ au_cache_free_vdir(vdir); ++ au_cache_dfree_vdir(vdir); +out: + vdir = ERR_PTR(err); + return vdir; @@ -27973,7 +28410,7 @@ index 0000000..0a9ff00 + union au_vdir_deblk_p p, deblk_end; + + while (vdir->vd_nblk > 1) { -+ kfree(vdir->vd_deblk[vdir->vd_nblk - 1]); ++ au_delayed_kfree(vdir->vd_deblk[vdir->vd_nblk - 1]); + /* vdir->vd_deblk[vdir->vd_nblk - 1] = NULL; */ + vdir->vd_nblk--; + } @@ -28104,7 +28541,7 @@ index 0000000..0a9ff00 + } + } + -+ free_page((unsigned long)o); ++ au_delayed_free_page((unsigned long)o); + +out: + AuTraceErr(err); @@ -28243,7 +28680,7 @@ index 0000000..0a9ff00 + if (allocated) + au_set_ivdir(inode, allocated); + } else if (allocated) -+ au_vdir_free(allocated); ++ au_vdir_free(allocated, /*atonce*/0); + +out: + return err; @@ -28261,8 +28698,8 @@ index 0000000..0a9ff00 + if (tgt->vd_nblk < src->vd_nblk) { + unsigned char **p; + -+ p = krealloc(tgt->vd_deblk, sizeof(*p) * src->vd_nblk, -+ GFP_NOFS); ++ p = au_krealloc(tgt->vd_deblk, sizeof(*p) * src->vd_nblk, ++ GFP_NOFS, /*may_shrink*/0); + if (unlikely(!p)) + goto out; + tgt->vd_deblk = p; @@ -28272,7 +28709,8 @@ index 0000000..0a9ff00 + unsigned char *p; + + tgt->vd_deblk_sz = deblk_sz; -+ p = krealloc(tgt->vd_deblk[0], deblk_sz, GFP_NOFS); ++ p = au_krealloc(tgt->vd_deblk[0], deblk_sz, GFP_NOFS, ++ /*may_shrink*/1); + if (unlikely(!p)) + goto out; + tgt->vd_deblk[0] = p; @@ -28337,7 +28775,7 @@ index 0000000..0a9ff00 + if (allocated) + au_set_fvdir_cache(file, allocated); + } else if (allocated) -+ au_vdir_free(allocated); ++ au_vdir_free(allocated, /*atonce*/0); + +out: + return err; @@ -29334,10 +29772,10 @@ index 0000000..2e54bad +} diff --git a/fs/aufs/vfsub.h b/fs/aufs/vfsub.h new file mode 100644 -index 0000000..291bfae +index 0000000..64fff0f --- /dev/null +++ b/fs/aufs/vfsub.h -@@ -0,0 +1,297 @@ +@@ -0,0 +1,303 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -29523,6 +29961,12 @@ index 0000000..291bfae + return flags; +} + ++static inline int vfsub_file_execed(struct file *file) ++{ ++ /* todo: direct access f_flags */ ++ return !!(vfsub_file_flags(file) & __FMODE_EXEC); ++} ++ +#if 0 /* reserved */ +static inline void vfsub_file_accessed(struct file *h_file) +{ @@ -29637,7 +30081,7 @@ index 0000000..291bfae +#endif /* __AUFS_VFSUB_H__ */ diff --git a/fs/aufs/wbr_policy.c b/fs/aufs/wbr_policy.c new file mode 100644 -index 0000000..91a010a +index 0000000..79eb9b4 --- /dev/null +++ b/fs/aufs/wbr_policy.c @@ -0,0 +1,752 @@ @@ -30091,7 +30535,7 @@ index 0000000..91a010a + + mfs->mfsrr_bytes = bavail; + AuDbg("b%d\n", mfs->mfs_bindex); -+ kfree(st); ++ au_delayed_kfree(st); +} + +static int au_wbr_create_mfs(struct dentry *dentry, unsigned int flags) @@ -30395,7 +30839,7 @@ index 0000000..91a010a +}; diff --git a/fs/aufs/whout.c b/fs/aufs/whout.c new file mode 100644 -index 0000000..16a96f88 +index 0000000..54feb08 --- /dev/null +++ b/fs/aufs/whout.c @@ -0,0 +1,1047 @@ @@ -30550,7 +30994,7 @@ index 0000000..16a96f88 + +out_name: + if (name != defname) -+ kfree(name); ++ au_delayed_kfree(name); +out: + AuTraceErrPtr(dentry); + return dentry; @@ -30989,7 +31433,7 @@ index 0000000..16a96f88 + au_br_put(a->br); + si_write_unlock(a->sb); + au_nwt_done(&au_sbi(a->sb)->si_nowait); -+ kfree(arg); ++ au_delayed_kfree(arg); + if (unlikely(err)) + AuIOErr("err %d\n", err); +} @@ -31017,7 +31461,7 @@ index 0000000..16a96f88 + if (unlikely(wkq_err)) { + atomic_dec(&br->br_wbr->wbr_wh_running); + au_br_put(br); -+ kfree(arg); ++ au_delayed_kfree(arg); + } + do_dec = 0; + } @@ -31176,7 +31620,7 @@ index 0000000..16a96f88 + wh_dentry = ERR_PTR(err); + if (!err) { + wh_dentry = vfsub_lkup_one(&wh_name, h_parent); -+ kfree(wh_name.name); ++ au_delayed_kfree(wh_name.name); + } + return wh_dentry; +} @@ -31252,7 +31696,7 @@ index 0000000..16a96f88 + break; + } + } -+ free_page((unsigned long)wh_name.name); ++ au_delayed_free_page((unsigned long)wh_name.name); + +out: + return err; @@ -31294,7 +31738,7 @@ index 0000000..16a96f88 + rdhash = AUFS_RDHASH_DEF; + err = au_nhash_alloc(&whtmp->whlist, rdhash, gfp); + if (unlikely(err)) { -+ kfree(whtmp); ++ au_delayed_kfree(whtmp); + whtmp = ERR_PTR(err); + } + @@ -31309,7 +31753,7 @@ index 0000000..16a96f88 + dput(whtmp->wh_dentry); + iput(whtmp->dir); + au_nhash_wh_free(&whtmp->whlist); -+ kfree(whtmp); ++ au_delayed_kfree(whtmp); +} + +/* @@ -31526,10 +31970,10 @@ index 0000000..4077dd1 +#endif /* __AUFS_WHOUT_H__ */ diff --git a/fs/aufs/wkq.c b/fs/aufs/wkq.c new file mode 100644 -index 0000000..65c0137 +index 0000000..216d08f --- /dev/null +++ b/fs/aufs/wkq.c -@@ -0,0 +1,205 @@ +@@ -0,0 +1,200 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -31573,7 +32017,7 @@ index 0000000..65c0137 + else { + kobject_put(wkinfo->kobj); + module_put(THIS_MODULE); /* todo: ?? */ -+ kfree(wkinfo); ++ au_delayed_kfree(wkinfo); + } +} + @@ -31596,7 +32040,7 @@ index 0000000..65c0137 + +static void au_wkq_comp_free(struct completion *comp) +{ -+ kfree(comp); ++ au_delayed_kfree(comp); +} + +#else @@ -31677,7 +32121,7 @@ index 0000000..65c0137 + int err; + struct au_wkinfo *wkinfo; + -+ percpu_counter_inc(&au_sbi(sb)->si_nowait.nw_len); ++ atomic_inc(&au_sbi(sb)->si_nowait.nw_len); + + /* + * wkq_func() must free this wkinfo. @@ -31707,16 +32151,11 @@ index 0000000..65c0137 + +void au_nwt_init(struct au_nowait_tasks *nwt) +{ -+ percpu_counter_init(&nwt->nw_len, 0, GFP_NOFS); ++ atomic_set(&nwt->nw_len, 0); ++ /* smp_mb(); */ /* atomic_set */ + init_waitqueue_head(&nwt->nw_wq); +} + -+void au_nwt_fin(struct au_nowait_tasks *nwt) -+{ -+ AuDebugOn(percpu_counter_sum(&nwt->nw_len)); -+ percpu_counter_destroy(&nwt->nw_len); -+} -+ +void au_wkq_fin(void) +{ + destroy_workqueue(au_wkq); @@ -31737,10 +32176,10 @@ index 0000000..65c0137 +} diff --git a/fs/aufs/wkq.h b/fs/aufs/wkq.h new file mode 100644 -index 0000000..0b870cc +index 0000000..6d63f74 --- /dev/null +++ b/fs/aufs/wkq.h -@@ -0,0 +1,82 @@ +@@ -0,0 +1,80 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -31765,7 +32204,7 @@ index 0000000..0b870cc + * in the next operation, wait for the 'nowait' tasks in system-wide workqueue + */ +struct au_nowait_tasks { -+ struct percpu_counter nw_len; ++ atomic_t nw_len; + wait_queue_head_t nw_wq; +}; + @@ -31792,7 +32231,6 @@ index 0000000..0b870cc +int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb, + unsigned int flags); +void au_nwt_init(struct au_nowait_tasks *nwt); -+void au_nwt_fin(struct au_nowait_tasks *nwt); +int __init au_wkq_init(void); +void au_wkq_fin(void); + @@ -31810,14 +32248,13 @@ index 0000000..0b870cc + +static inline void au_nwt_done(struct au_nowait_tasks *nwt) +{ -+ percpu_counter_dec(&nwt->nw_len); -+ if (!percpu_counter_sum(&nwt->nw_len)) ++ if (atomic_dec_and_test(&nwt->nw_len)) + wake_up_all(&nwt->nw_wq); +} + +static inline int au_nwt_flush(struct au_nowait_tasks *nwt) +{ -+ wait_event(nwt->nw_wq, !percpu_counter_sum(&nwt->nw_len)); ++ wait_event(nwt->nw_wq, !atomic_read(&nwt->nw_len)); + return 0; +} + @@ -31825,10 +32262,10 @@ index 0000000..0b870cc +#endif /* __AUFS_WKQ_H__ */ diff --git a/fs/aufs/xattr.c b/fs/aufs/xattr.c new file mode 100644 -index 0000000..c12518f +index 0000000..2a148c9 --- /dev/null +++ b/fs/aufs/xattr.c -@@ -0,0 +1,332 @@ +@@ -0,0 +1,334 @@ +/* + * Copyright (C) 2014-2016 Junjiro R. Okajima + */ @@ -31994,10 +32431,12 @@ index 0000000..c12518f + AuTraceErr(err); + } + -+ kfree(value); ++ if (value) ++ au_delayed_kfree(value); + +out_free: -+ kfree(o); ++ if (o) ++ au_delayed_kfree(o); +out: + if (!unlocked) + inode_unlock(h_isrc); @@ -32163,10 +32602,10 @@ index 0000000..c12518f +#endif diff --git a/fs/aufs/xino.c b/fs/aufs/xino.c new file mode 100644 -index 0000000..4d1d5d4 +index 0000000..2a411f9 --- /dev/null +++ b/fs/aufs/xino.c -@@ -0,0 +1,1304 @@ +@@ -0,0 +1,1305 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -32494,7 +32933,7 @@ index 0000000..4d1d5d4 + AuErr1("statfs err %d, ignored\n", err); + +out_st: -+ kfree(st); ++ au_delayed_kfree(st); +out: + return err; +} @@ -32529,7 +32968,7 @@ index 0000000..4d1d5d4 + au_br_put(br); + si_write_unlock(sb); + au_nwt_done(&au_sbi(sb)->si_nowait); -+ kfree(args); ++ au_delayed_kfree(args); +} + +static int xino_trunc_test(struct super_block *sb, struct au_branch *br) @@ -32571,7 +33010,7 @@ index 0000000..4d1d5d4 + args = kmalloc(sizeof(*args), GFP_NOFS); + if (unlikely(!args)) { + AuErr1("no memory\n"); -+ goto out_args; ++ goto out; + } + + au_br_get(br); @@ -32583,9 +33022,8 @@ index 0000000..4d1d5d4 + + pr_err("wkq %d\n", wkq_err); + au_br_put(br); ++ au_delayed_kfree(args); + -+out_args: -+ kfree(args); +out: + atomic_dec(&br->br_xino_running); +} @@ -33108,7 +33546,7 @@ index 0000000..4d1d5d4 + (sb, au_sbr(sb, bindex)->br_xino.xi_file, page); + else + AuDbg("b%d\n", bindex); -+ free_page((unsigned long)page); ++ au_delayed_free_page((unsigned long)page); + +out: + return err; @@ -33185,7 +33623,8 @@ index 0000000..4d1d5d4 + if (sbinfo->si_xib) + fput(sbinfo->si_xib); + sbinfo->si_xib = NULL; -+ free_page((unsigned long)sbinfo->si_xib_buf); ++ if (sbinfo->si_xib_buf) ++ au_delayed_free_page((unsigned long)sbinfo->si_xib_buf); + sbinfo->si_xib_buf = NULL; +} + @@ -33228,7 +33667,8 @@ index 0000000..4d1d5d4 + goto out; /* success */ + +out_free: -+ free_page((unsigned long)sbinfo->si_xib_buf); ++ if (sbinfo->si_xib_buf) ++ au_delayed_free_page((unsigned long)sbinfo->si_xib_buf); + sbinfo->si_xib_buf = NULL; + if (err >= 0) + err = -EIO; @@ -33321,7 +33761,7 @@ index 0000000..4d1d5d4 + fput(p->new); + else + break; -+ kfree(fpair); ++ au_delayed_kfree(fpair); +out: + return err; +} @@ -33432,7 +33872,7 @@ index 0000000..4d1d5d4 + if (!IS_ERR(file)) + au_xino_brid_set(sb, br->br_id); + } -+ free_page((unsigned long)page); ++ au_delayed_free_page((unsigned long)page); + } else { + file = au_xino_create(sb, AUFS_XINO_DEFPATH, /*silent*/0); + if (IS_ERR(file)) @@ -33472,10 +33912,10 @@ index 0000000..4d1d5d4 + return err; +} diff --git a/fs/dcache.c b/fs/dcache.c -index d6847d7..c3c0b6d 100644 +index 5c7cc95..df0268c 100644 --- a/fs/dcache.c +++ b/fs/dcache.c -@@ -1202,7 +1202,7 @@ enum d_walk_ret { +@@ -1164,7 +1164,7 @@ enum d_walk_ret { * * The @enter() and @finish() callbacks are called with d_lock held. */ @@ -33507,10 +33947,10 @@ index 350a2c8..6f42279 100644 return error; diff --git a/fs/inode.c b/fs/inode.c -index 4ccbc21..aa6d071 100644 +index 7e3ef3a..675fe84 100644 --- a/fs/inode.c +++ b/fs/inode.c -@@ -1591,7 +1591,7 @@ EXPORT_SYMBOL(generic_update_time); +@@ -1593,7 +1593,7 @@ EXPORT_SYMBOL(generic_update_time); * This does the actual work of updating an inodes time or version. Must have * had called mnt_want_write() before calling this. */ @@ -33520,10 +33960,10 @@ index 4ccbc21..aa6d071 100644 int (*update_time)(struct inode *, struct timespec *, int); diff --git a/fs/proc/base.c b/fs/proc/base.c -index a11eb71..8f10865 100644 +index ac0df4d..42255e5 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c -@@ -1939,7 +1939,7 @@ static int map_files_get_link(struct dentry *dentry, struct path *path) +@@ -1938,7 +1938,7 @@ static int map_files_get_link(struct dentry *dentry, struct path *path) down_read(&mm->mmap_sem); vma = find_exact_vma(mm, vm_start, vm_end); if (vma && vma->vm_file) { @@ -33549,7 +33989,7 @@ index f8595e8..cb8eda0 100644 ino = inode->i_ino; } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c -index 4648c7f..061cb85 100644 +index f6fa99e..2750949 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -298,7 +298,10 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid) @@ -33564,7 +34004,7 @@ index 4648c7f..061cb85 100644 dev = inode->i_sb->s_dev; ino = inode->i_ino; pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; -@@ -1624,7 +1627,7 @@ static int show_numa_map(struct seq_file *m, void *v, int is_pid) +@@ -1634,7 +1637,7 @@ static int show_numa_map(struct seq_file *m, void *v, int is_pid) struct proc_maps_private *proc_priv = &numa_priv->proc_maps; struct vm_area_struct *vma = v; struct numa_maps *md = &numa_priv->md; @@ -33590,7 +34030,7 @@ index faacb0c..17b43be 100644 ino = inode->i_ino; pgoff = (loff_t)vma->vm_pgoff << PAGE_SHIFT; diff --git a/fs/read_write.c b/fs/read_write.c -index 933b53a..2d13282 100644 +index 66215a7..a1da117 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -515,6 +515,28 @@ ssize_t __vfs_write(struct file *file, const char __user *p, size_t count, @@ -33663,10 +34103,10 @@ index 7444f5f..bdac0be 100644 static inline void fput_light(struct file *file, int fput_needed) { diff --git a/include/linux/fs.h b/include/linux/fs.h -index dd28814..b689a48 100644 +index 901e25d..a71aa9e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h -@@ -1306,6 +1306,7 @@ extern void fasync_free(struct fasync_struct *); +@@ -1275,6 +1275,7 @@ extern void fasync_free(struct fasync_struct *); /* can be called from interrupts */ extern void kill_fasync(struct fasync_struct **, int, int); @@ -33674,7 +34114,7 @@ index dd28814..b689a48 100644 extern void __f_setown(struct file *filp, struct pid *, enum pid_type, int force); extern void f_setown(struct file *filp, unsigned long arg, int force); extern void f_delown(struct file *filp); -@@ -1690,6 +1691,7 @@ struct file_operations { +@@ -1699,6 +1700,7 @@ struct file_operations { ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int); unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); int (*check_flags)(int); @@ -33682,7 +34122,7 @@ index dd28814..b689a48 100644 int (*flock) (struct file *, int, struct file_lock *); ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); -@@ -1750,6 +1752,12 @@ ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, +@@ -1759,6 +1761,12 @@ ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, struct iovec *fast_pointer, struct iovec **ret_pointer); @@ -33695,7 +34135,7 @@ index dd28814..b689a48 100644 extern ssize_t __vfs_read(struct file *, char __user *, size_t, loff_t *); extern ssize_t __vfs_write(struct file *, const char __user *, size_t, loff_t *); extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *); -@@ -2105,6 +2113,7 @@ extern int current_umask(void); +@@ -2123,6 +2131,7 @@ extern int current_umask(void); extern void ihold(struct inode * inode); extern void iput(struct inode *); extern int generic_update_time(struct inode *, struct timespec *, int); @@ -33704,10 +34144,10 @@ index dd28814..b689a48 100644 /* /sys/fs */ extern struct kobject *fs_kobj; diff --git a/include/linux/mm.h b/include/linux/mm.h -index ece042d..1e24513 100644 +index ef815b9..a772481 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h -@@ -1239,6 +1239,28 @@ static inline int fixup_user_fault(struct task_struct *tsk, +@@ -1278,6 +1278,28 @@ static inline int fixup_user_fault(struct task_struct *tsk, } #endif @@ -33737,10 +34177,10 @@ index ece042d..1e24513 100644 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, int write); diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index ca3e517..10bc491 100644 +index 903200f..55fc528 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h -@@ -274,6 +274,7 @@ struct vm_region { +@@ -275,6 +275,7 @@ struct vm_region { unsigned long vm_top; /* region allocated to here */ unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */ struct file *vm_file; /* the backing file or NULL */ @@ -33748,7 +34188,7 @@ index ca3e517..10bc491 100644 int vm_usage; /* region usage count (access under nommu_region_sem) */ bool vm_icache_flushed : 1; /* true if the icache has been flushed for -@@ -348,6 +349,7 @@ struct vm_area_struct { +@@ -349,6 +350,7 @@ struct vm_area_struct { unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE units */ struct file * vm_file; /* File we map to (can be NULL). */ @@ -33772,7 +34212,7 @@ index da2751d..2e0fca6 100644 + unsigned int flags); #endif diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild -index 8bdae34..65dbd5f 100644 +index 185f8ea..5deb0d1 100644 --- a/include/uapi/linux/Kbuild +++ b/include/uapi/linux/Kbuild @@ -59,6 +59,7 @@ header-y += atmsvc.h @@ -33785,7 +34225,7 @@ index 8bdae34..65dbd5f 100644 header-y += auxvec.h diff --git a/include/uapi/linux/aufs_type.h b/include/uapi/linux/aufs_type.h new file mode 100644 -index 0000000..4e08bdf +index 0000000..e220283 --- /dev/null +++ b/include/uapi/linux/aufs_type.h @@ -0,0 +1,406 @@ @@ -33817,7 +34257,7 @@ index 0000000..4e08bdf + +#include + -+#define AUFS_VERSION "4.x-rcN" ++#define AUFS_VERSION "4.8" + +/* todo? move this to linux-2.6.19/include/magic.h */ +#define AUFS_SUPER_MAGIC ('a' << 24 | 'u' << 16 | 'f' << 8 | 's') @@ -34196,10 +34636,10 @@ index 0000000..4e08bdf + +#endif /* __AUFS_TYPE_H__ */ diff --git a/kernel/fork.c b/kernel/fork.c -index 4a7ec0c..8c8f7ac 100644 +index beb3172..ad4cfa8 100644 --- a/kernel/fork.c +++ b/kernel/fork.c -@@ -479,7 +479,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) +@@ -477,7 +477,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) struct inode *inode = file_inode(file); struct address_space *mapping = file->f_mapping; @@ -34209,10 +34649,10 @@ index 4a7ec0c..8c8f7ac 100644 atomic_dec(&inode->i_writecount); i_mmap_lock_write(mapping); diff --git a/mm/Makefile b/mm/Makefile -index 78c6f7d..aea4230 100644 +index 2ca1faf..6b9da3f 100644 --- a/mm/Makefile +++ b/mm/Makefile -@@ -37,7 +37,7 @@ obj-y := filemap.o mempool.o oom_kill.o \ +@@ -40,7 +40,7 @@ obj-y := filemap.o mempool.o oom_kill.o \ mm_init.o mmu_context.o percpu.o slab_common.o \ compaction.o vmacache.o \ interval_tree.o list_lru.o workingset.o \ @@ -34222,10 +34662,10 @@ index 78c6f7d..aea4230 100644 obj-y += init-mm.o diff --git a/mm/filemap.c b/mm/filemap.c -index 20f3b1f..ee827ce 100644 +index 2d0986a..4a31bad 100644 --- a/mm/filemap.c +++ b/mm/filemap.c -@@ -2208,7 +2208,7 @@ int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) +@@ -2284,7 +2284,7 @@ int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) int ret = VM_FAULT_LOCKED; sb_start_pagefault(inode->i_sb); @@ -34235,10 +34675,10 @@ index 20f3b1f..ee827ce 100644 if (page->mapping != inode->i_mapping) { unlock_page(page); diff --git a/mm/memory.c b/mm/memory.c -index cd1f29e..f0c204c 100644 +index 793fe0f..45f39f3 100644 --- a/mm/memory.c +++ b/mm/memory.c -@@ -2100,7 +2100,7 @@ static inline int wp_page_reuse(struct mm_struct *mm, +@@ -2113,7 +2113,7 @@ static inline int wp_page_reuse(struct fault_env *fe, pte_t orig_pte, } if (!page_mkwrite) @@ -34248,10 +34688,10 @@ index cd1f29e..f0c204c 100644 return VM_FAULT_WRITE; diff --git a/mm/mmap.c b/mm/mmap.c -index de2c176..b7f391c 100644 +index ca9d91b..f3ebc5a 100644 --- a/mm/mmap.c +++ b/mm/mmap.c -@@ -162,7 +162,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) +@@ -163,7 +163,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) if (vma->vm_ops && vma->vm_ops->close) vma->vm_ops->close(vma); if (vma->vm_file) @@ -34260,7 +34700,7 @@ index de2c176..b7f391c 100644 mpol_put(vma_policy(vma)); kmem_cache_free(vm_area_cachep, vma); return next; -@@ -782,7 +782,7 @@ again: remove_next = 1 + (end > next->vm_end); +@@ -790,7 +790,7 @@ again: if (remove_next) { if (file) { uprobe_munmap(next, next->vm_start, next->vm_end); @@ -34269,7 +34709,7 @@ index de2c176..b7f391c 100644 } if (next->anon_vma) anon_vma_merge(vma, next); -@@ -1563,8 +1563,8 @@ out: +@@ -1574,8 +1574,8 @@ out: return addr; unmap_and_free_vma: @@ -34279,7 +34719,7 @@ index de2c176..b7f391c 100644 /* Undo any partial mapping done by a device driver. */ unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); -@@ -2358,7 +2358,7 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, +@@ -2380,7 +2380,7 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, goto out_free_mpol; if (new->vm_file) @@ -34288,7 +34728,7 @@ index de2c176..b7f391c 100644 if (new->vm_ops && new->vm_ops->open) new->vm_ops->open(new); -@@ -2377,7 +2377,7 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, +@@ -2399,7 +2399,7 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, if (new->vm_ops && new->vm_ops->close) new->vm_ops->close(new); if (new->vm_file) @@ -34297,7 +34737,7 @@ index de2c176..b7f391c 100644 unlink_anon_vmas(new); out_free_mpol: mpol_put(vma_policy(new)); -@@ -2528,7 +2528,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, +@@ -2550,7 +2550,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, struct vm_area_struct *vma; unsigned long populate = 0; unsigned long ret = -EINVAL; @@ -34306,7 +34746,7 @@ index de2c176..b7f391c 100644 pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.txt.\n", current->comm, current->pid); -@@ -2597,10 +2597,27 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, +@@ -2625,10 +2625,27 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, } } @@ -34335,7 +34775,7 @@ index de2c176..b7f391c 100644 out: up_write(&mm->mmap_sem); if (populate) -@@ -2873,7 +2890,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, +@@ -2903,7 +2920,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (anon_vma_clone(new_vma, vma)) goto out_free_mempol; if (new_vma->vm_file) @@ -34345,7 +34785,7 @@ index de2c176..b7f391c 100644 new_vma->vm_ops->open(new_vma); vma_link(mm, new_vma, prev, rb_link, rb_parent); diff --git a/mm/nommu.c b/mm/nommu.c -index c2e588802..c39edc4 100644 +index 95daf81..5086a29 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -644,7 +644,7 @@ static void __put_nommu_region(struct vm_region *region) diff --git a/kernel/kernel/files/patches/mageia/gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch b/kernel/kernel/files/patches/mageia/gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch new file mode 100644 index 00000000..f7d2ac73 --- /dev/null +++ b/kernel/kernel/files/patches/mageia/gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch @@ -0,0 +1,75 @@ +From 5a920b85f2c6e3fd7d9dd9bb3f3345e9085e2360 Mon Sep 17 00:00:00 2001 +From: Paulo Zanoni +Date: Tue, 4 Oct 2016 14:37:32 -0300 +Subject: drm/i915/gen9: fix DDB partitioning for multi-screen cases + +With the previous code we were only recomputing the DDB partitioning +for the CRTCs included in the atomic commit, so any other active CRTCs +would end up having their DDB registers zeroed. In this patch we make +sure that the computed state starts as a copy of the current +partitioning, and then we only zero the DDBs that we're actually +going to recompute. + +How to reproduce the bug: + 1 - Enable the primary plane on pipe A + 2 - Enable the primary plane on pipe B + 3 - Enable the cursor or sprite plane on pipe A + +Step 3 will zero the DDB partitioning for pipe B since it's not +included in the commit that enabled the cursor or sprite for pipe A. + +I expect this to fix many FIFO underrun problems on gen9+. + +v2: + - Mention the cursor on the steps to reproduce the problem (Paulo). + - Add Testcase tag provided by Maarten (Maarten). + +Testcase: kms_cursor_legacy.cursorA-vs-flipB-atomic-transitions +Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=96226 +Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=96828 +Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=97450 +Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=97596 +Bugzilla: https://www.phoronix.com/scan.php?page=news_item&px=Intel-Skylake-Multi-Screen-Woes +Cc: stable@vger.kernel.org +Signed-off-by: Paulo Zanoni +Reviewed-by: Lyude +Link: http://patchwork.freedesktop.org/patch/msgid/1475602652-17326-1-git-send-email-paulo.r.zanoni@intel.com + +diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c +index 7f1748a..fe6c1c6 100644 +--- a/drivers/gpu/drm/i915/intel_pm.c ++++ b/drivers/gpu/drm/i915/intel_pm.c +@@ -3362,13 +3362,15 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, + int num_active; + int id, i; + ++ /* Clear the partitioning for disabled planes. */ ++ memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); ++ memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe])); ++ + if (WARN_ON(!state)) + return 0; + + if (!cstate->base.active) { + ddb->pipe[pipe].start = ddb->pipe[pipe].end = 0; +- memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); +- memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe])); + return 0; + } + +@@ -4054,6 +4056,12 @@ skl_compute_ddb(struct drm_atomic_state *state) + intel_state->wm_results.dirty_pipes = ~0; + } + ++ /* ++ * We're not recomputing for the pipes not included in the commit, so ++ * make sure we start with the current state. ++ */ ++ memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb)); ++ + for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) { + struct intel_crtc_state *cstate; + +-- +cgit v0.10.2 + diff --git a/kernel/kernel/files/patches/mageia/intel_th-Fix-a-deadlock-in-modprobing.patch b/kernel/kernel/files/patches/mageia/intel_th-Fix-a-deadlock-in-modprobing.patch deleted file mode 100644 index 45b63cdd..00000000 --- a/kernel/kernel/files/patches/mageia/intel_th-Fix-a-deadlock-in-modprobing.patch +++ /dev/null @@ -1,95 +0,0 @@ -From a36aa80f3cb2540fb1dbad6240852de4365a2e82 Mon Sep 17 00:00:00 2001 -From: Alexander Shishkin -Date: Thu, 30 Jun 2016 11:51:44 +0300 -Subject: [PATCH] intel_th: Fix a deadlock in modprobing - -Driver initialization tries to request a hub (GTH) driver module from -its probe callback, resulting in a deadlock. - -This patch solves the problem by adding a deferred work for requesting -the hub module. - -Signed-off-by: Alexander Shishkin -Cc: # 4.4.x- ---- - drivers/hwtracing/intel_th/core.c | 35 ++++++++++++++++++++++++++++++++++- - drivers/hwtracing/intel_th/intel_th.h | 3 +++ - 2 files changed, 37 insertions(+), 1 deletion(-) - -diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c -index 1be543e..0b112ae 100644 ---- a/drivers/hwtracing/intel_th/core.c -+++ b/drivers/hwtracing/intel_th/core.c -@@ -465,6 +465,38 @@ static struct intel_th_subdevice { - }, - }; - -+#ifdef CONFIG_MODULES -+static void __intel_th_request_hub_module(struct work_struct *work) -+{ -+ struct intel_th *th = container_of(work, struct intel_th, -+ request_module_work); -+ -+ request_module("intel_th_%s", th->hub->name); -+} -+ -+static int intel_th_request_hub_module(struct intel_th *th) -+{ -+ INIT_WORK(&th->request_module_work, __intel_th_request_hub_module); -+ schedule_work(&th->request_module_work); -+ -+ return 0; -+} -+ -+static void intel_th_request_hub_module_flush(struct intel_th *th) -+{ -+ flush_work(&th->request_module_work); -+} -+#else -+static inline int intel_th_request_hub_module(struct intel_th *th) -+{ -+ return -EINVAL; -+} -+ -+static inline void intel_th_request_hub_module_flush(struct intel_th *th) -+{ -+} -+#endif /* CONFIG_MODULES */ -+ - static int intel_th_populate(struct intel_th *th, struct resource *devres, - unsigned int ndevres, int irq) - { -@@ -535,7 +567,7 @@ static int intel_th_populate(struct intel_th *th, struct resource *devres, - /* need switch driver to be loaded to enumerate the rest */ - if (subdev->type == INTEL_TH_SWITCH && !req) { - th->hub = thdev; -- err = request_module("intel_th_%s", subdev->name); -+ err = intel_th_request_hub_module(th); - if (!err) - req++; - } -@@ -652,6 +684,7 @@ void intel_th_free(struct intel_th *th) - { - int i; - -+ intel_th_request_hub_module_flush(th); - for (i = 0; i < TH_SUBDEVICE_MAX; i++) - if (th->thdev[i] != th->hub) - intel_th_device_remove(th->thdev[i]); -diff --git a/drivers/hwtracing/intel_th/intel_th.h b/drivers/hwtracing/intel_th/intel_th.h -index 0df22e3..0482848 100644 ---- a/drivers/hwtracing/intel_th/intel_th.h -+++ b/drivers/hwtracing/intel_th/intel_th.h -@@ -205,6 +205,9 @@ struct intel_th { - - int id; - int major; -+#ifdef CONFIG_MODULES -+ struct work_struct request_module_work; -+#endif /* CONFIG_MODULES */ - #ifdef CONFIG_INTEL_TH_DEBUG - struct dentry *dbg; - #endif --- -2.9.2 - diff --git a/kernel/kernel/files/patches/mageia/intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch b/kernel/kernel/files/patches/mageia/intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch deleted file mode 100644 index f6d4f341..00000000 --- a/kernel/kernel/files/patches/mageia/intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 7a1a47ce35821b40f5b2ce46379ba14393bc3873 Mon Sep 17 00:00:00 2001 -From: Alexander Shishkin -Date: Tue, 28 Jun 2016 18:55:23 +0300 -Subject: [PATCH] intel_th: pci: Add Kaby Lake PCH-H support - -This adds Intel(R) Trace Hub PCI ID for Kaby Lake PCH-H. - -Signed-off-by: Alexander Shishkin -Cc: ---- - drivers/hwtracing/intel_th/pci.c | 5 +++++ - 1 file changed, 5 insertions(+) - -diff --git a/drivers/hwtracing/intel_th/pci.c b/drivers/hwtracing/intel_th/pci.c -index 5e25c7e..0bba384 100644 ---- a/drivers/hwtracing/intel_th/pci.c -+++ b/drivers/hwtracing/intel_th/pci.c -@@ -80,6 +80,11 @@ static const struct pci_device_id intel_th_pci_id_table[] = { - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x1a8e), - .driver_data = (kernel_ulong_t)0, - }, -+ { -+ /* Kaby Lake PCH-H */ -+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xa2a6), -+ .driver_data = (kernel_ulong_t)0, -+ }, - { 0 }, - }; - --- -2.9.2 - diff --git a/kernel/kernel/files/patches/mageia/kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch b/kernel/kernel/files/patches/mageia/kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch new file mode 100644 index 00000000..05b8cf99 --- /dev/null +++ b/kernel/kernel/files/patches/mageia/kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch @@ -0,0 +1,49 @@ +From 9f692cbe4a01dd9e3c3e954ec6b59662b68f9ce4 Mon Sep 17 00:00:00 2001 +From: Laura Abbott +Date: Fri, 9 Sep 2016 10:19:02 -0700 +Subject: [PATCH] cpupower: Correct return type of cpu_power_is_cpu_online in + cpufreq +To: Thomas Renninger +Cc: linux-pm@vger.kernel.org +Cc: linux-kernel@vger.kernel.org + +When converting to a shared library in ac5a181d065d ("cpupower: Add +cpuidle parts into library"), cpu_freq_cpu_exists was converted to +cpupower_is_cpu_online. cpu_req_cpu_exists returned 0 on success and +-ENOSYS on failure whereas cpupower_is_cpu_online returns 1 on success. +Check for the correct return value in cpufreq-set. + +See https://bugzilla.redhat.com/show_bug.cgi?id=1374212 + +Fixes: ac5a181d065d ("cpupower: Add cpuidle parts into library") +Reported-by: Julian Seward +Signed-off-by: Laura Abbott +--- + tools/power/cpupower/utils/cpufreq-set.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/tools/power/cpupower/utils/cpufreq-set.c b/tools/power/cpupower/utils/cpufreq-set.c +index b4bf769..8971d71 100644 +--- a/tools/power/cpupower/utils/cpufreq-set.c ++++ b/tools/power/cpupower/utils/cpufreq-set.c +@@ -296,7 +296,7 @@ int cmd_freq_set(int argc, char **argv) + struct cpufreq_affected_cpus *cpus; + + if (!bitmask_isbitset(cpus_chosen, cpu) || +- cpupower_is_cpu_online(cpu)) ++ cpupower_is_cpu_online(cpu) != 1) + continue; + + cpus = cpufreq_get_related_cpus(cpu); +@@ -316,7 +316,7 @@ int cmd_freq_set(int argc, char **argv) + cpu <= bitmask_last(cpus_chosen); cpu++) { + + if (!bitmask_isbitset(cpus_chosen, cpu) || +- cpupower_is_cpu_online(cpu)) ++ cpupower_is_cpu_online(cpu) != 1) + continue; + + if (cpupower_is_cpu_online(cpu) != 1) +-- +2.10.0 + diff --git a/kernel/kernel/files/patches/mageia/linux-BFQ-v8r4.patch b/kernel/kernel/files/patches/mageia/linux-BFQ-v8r4.patch new file mode 100644 index 00000000..abb33847 --- /dev/null +++ b/kernel/kernel/files/patches/mageia/linux-BFQ-v8r4.patch @@ -0,0 +1,9004 @@ +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq-cgroup.c linux-4.8-bfq/block/bfq-cgroup.c +--- linux-4.8/block/bfq-cgroup.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq-cgroup.c 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,1196 @@ ++/* ++ * BFQ: CGROUPS support. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2015 Paolo Valente ++ * ++ * Copyright (C) 2016 Paolo Valente ++ * ++ * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ ++ * file. ++ */ ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ ++/* bfqg stats flags */ ++enum bfqg_stats_flags { ++ BFQG_stats_waiting = 0, ++ BFQG_stats_idling, ++ BFQG_stats_empty, ++}; ++ ++#define BFQG_FLAG_FNS(name) \ ++static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \ ++{ \ ++ stats->flags |= (1 << BFQG_stats_##name); \ ++} \ ++static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \ ++{ \ ++ stats->flags &= ~(1 << BFQG_stats_##name); \ ++} \ ++static int bfqg_stats_##name(struct bfqg_stats *stats) \ ++{ \ ++ return (stats->flags & (1 << BFQG_stats_##name)) != 0; \ ++} \ ++ ++BFQG_FLAG_FNS(waiting) ++BFQG_FLAG_FNS(idling) ++BFQG_FLAG_FNS(empty) ++#undef BFQG_FLAG_FNS ++ ++/* This should be called with the queue_lock held. */ ++static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats) ++{ ++ unsigned long long now; ++ ++ if (!bfqg_stats_waiting(stats)) ++ return; ++ ++ now = sched_clock(); ++ if (time_after64(now, stats->start_group_wait_time)) ++ blkg_stat_add(&stats->group_wait_time, ++ now - stats->start_group_wait_time); ++ bfqg_stats_clear_waiting(stats); ++} ++ ++/* This should be called with the queue_lock held. */ ++static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg, ++ struct bfq_group *curr_bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ if (bfqg_stats_waiting(stats)) ++ return; ++ if (bfqg == curr_bfqg) ++ return; ++ stats->start_group_wait_time = sched_clock(); ++ bfqg_stats_mark_waiting(stats); ++} ++ ++/* This should be called with the queue_lock held. */ ++static void bfqg_stats_end_empty_time(struct bfqg_stats *stats) ++{ ++ unsigned long long now; ++ ++ if (!bfqg_stats_empty(stats)) ++ return; ++ ++ now = sched_clock(); ++ if (time_after64(now, stats->start_empty_time)) ++ blkg_stat_add(&stats->empty_time, ++ now - stats->start_empty_time); ++ bfqg_stats_clear_empty(stats); ++} ++ ++static void bfqg_stats_update_dequeue(struct bfq_group *bfqg) ++{ ++ blkg_stat_add(&bfqg->stats.dequeue, 1); ++} ++ ++static void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ if (blkg_rwstat_total(&stats->queued)) ++ return; ++ ++ /* ++ * group is already marked empty. This can happen if bfqq got new ++ * request in parent group and moved to this group while being added ++ * to service tree. Just ignore the event and move on. ++ */ ++ if (bfqg_stats_empty(stats)) ++ return; ++ ++ stats->start_empty_time = sched_clock(); ++ bfqg_stats_mark_empty(stats); ++} ++ ++static void bfqg_stats_update_idle_time(struct bfq_group *bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ if (bfqg_stats_idling(stats)) { ++ unsigned long long now = sched_clock(); ++ ++ if (time_after64(now, stats->start_idle_time)) ++ blkg_stat_add(&stats->idle_time, ++ now - stats->start_idle_time); ++ bfqg_stats_clear_idling(stats); ++ } ++} ++ ++static void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ stats->start_idle_time = sched_clock(); ++ bfqg_stats_mark_idling(stats); ++} ++ ++static void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ blkg_stat_add(&stats->avg_queue_size_sum, ++ blkg_rwstat_total(&stats->queued)); ++ blkg_stat_add(&stats->avg_queue_size_samples, 1); ++ bfqg_stats_update_group_wait_time(stats); ++} ++ ++static struct blkcg_policy blkcg_policy_bfq; ++ ++/* ++ * blk-cgroup policy-related handlers ++ * The following functions help in converting between blk-cgroup ++ * internal structures and BFQ-specific structures. ++ */ ++ ++static struct bfq_group *pd_to_bfqg(struct blkg_policy_data *pd) ++{ ++ return pd ? container_of(pd, struct bfq_group, pd) : NULL; ++} ++ ++static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg) ++{ ++ return pd_to_blkg(&bfqg->pd); ++} ++ ++static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg) ++{ ++ struct blkg_policy_data *pd = blkg_to_pd(blkg, &blkcg_policy_bfq); ++ ++ return pd_to_bfqg(pd); ++} ++ ++/* ++ * bfq_group handlers ++ * The following functions help in navigating the bfq_group hierarchy ++ * by allowing to find the parent of a bfq_group or the bfq_group ++ * associated to a bfq_queue. ++ */ ++ ++static struct bfq_group *bfqg_parent(struct bfq_group *bfqg) ++{ ++ struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent; ++ ++ return pblkg ? blkg_to_bfqg(pblkg) : NULL; ++} ++ ++static struct bfq_group *bfqq_group(struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *group_entity = bfqq->entity.parent; ++ ++ return group_entity ? container_of(group_entity, struct bfq_group, ++ entity) : ++ bfqq->bfqd->root_group; ++} ++ ++/* ++ * The following two functions handle get and put of a bfq_group by ++ * wrapping the related blk-cgroup hooks. ++ */ ++ ++static void bfqg_get(struct bfq_group *bfqg) ++{ ++ return blkg_get(bfqg_to_blkg(bfqg)); ++} ++ ++static void bfqg_put(struct bfq_group *bfqg) ++{ ++ return blkg_put(bfqg_to_blkg(bfqg)); ++} ++ ++static void bfqg_stats_update_io_add(struct bfq_group *bfqg, ++ struct bfq_queue *bfqq, ++ int op, int op_flags) ++{ ++ blkg_rwstat_add(&bfqg->stats.queued, op, op_flags, 1); ++ bfqg_stats_end_empty_time(&bfqg->stats); ++ if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue)) ++ bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq)); ++} ++ ++static void bfqg_stats_update_io_remove(struct bfq_group *bfqg, int op, ++ int op_flags) ++{ ++ blkg_rwstat_add(&bfqg->stats.queued, op, op_flags, -1); ++} ++ ++static void bfqg_stats_update_io_merged(struct bfq_group *bfqg, int op, ++ int op_flags) ++{ ++ blkg_rwstat_add(&bfqg->stats.merged, op, op_flags, 1); ++} ++ ++static void bfqg_stats_update_completion(struct bfq_group *bfqg, ++ uint64_t start_time, uint64_t io_start_time, int op, ++ int op_flags) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ unsigned long long now = sched_clock(); ++ ++ if (time_after64(now, io_start_time)) ++ blkg_rwstat_add(&stats->service_time, op, op_flags, ++ now - io_start_time); ++ if (time_after64(io_start_time, start_time)) ++ blkg_rwstat_add(&stats->wait_time, op, op_flags, ++ io_start_time - start_time); ++} ++ ++/* @stats = 0 */ ++static void bfqg_stats_reset(struct bfqg_stats *stats) ++{ ++ /* queued stats shouldn't be cleared */ ++ blkg_rwstat_reset(&stats->merged); ++ blkg_rwstat_reset(&stats->service_time); ++ blkg_rwstat_reset(&stats->wait_time); ++ blkg_stat_reset(&stats->time); ++ blkg_stat_reset(&stats->avg_queue_size_sum); ++ blkg_stat_reset(&stats->avg_queue_size_samples); ++ blkg_stat_reset(&stats->dequeue); ++ blkg_stat_reset(&stats->group_wait_time); ++ blkg_stat_reset(&stats->idle_time); ++ blkg_stat_reset(&stats->empty_time); ++} ++ ++/* @to += @from */ ++static void bfqg_stats_add_aux(struct bfqg_stats *to, struct bfqg_stats *from) ++{ ++ if (!to || !from) ++ return; ++ ++ /* queued stats shouldn't be cleared */ ++ blkg_rwstat_add_aux(&to->merged, &from->merged); ++ blkg_rwstat_add_aux(&to->service_time, &from->service_time); ++ blkg_rwstat_add_aux(&to->wait_time, &from->wait_time); ++ blkg_stat_add_aux(&from->time, &from->time); ++ blkg_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum); ++ blkg_stat_add_aux(&to->avg_queue_size_samples, ++ &from->avg_queue_size_samples); ++ blkg_stat_add_aux(&to->dequeue, &from->dequeue); ++ blkg_stat_add_aux(&to->group_wait_time, &from->group_wait_time); ++ blkg_stat_add_aux(&to->idle_time, &from->idle_time); ++ blkg_stat_add_aux(&to->empty_time, &from->empty_time); ++} ++ ++/* ++ * Transfer @bfqg's stats to its parent's dead_stats so that the ancestors' ++ * recursive stats can still account for the amount used by this bfqg after ++ * it's gone. ++ */ ++static void bfqg_stats_xfer_dead(struct bfq_group *bfqg) ++{ ++ struct bfq_group *parent; ++ ++ if (!bfqg) /* root_group */ ++ return; ++ ++ parent = bfqg_parent(bfqg); ++ ++ lockdep_assert_held(bfqg_to_blkg(bfqg)->q->queue_lock); ++ ++ if (unlikely(!parent)) ++ return; ++ ++ bfqg_stats_add_aux(&parent->stats, &bfqg->stats); ++ bfqg_stats_reset(&bfqg->stats); ++} ++ ++static void bfq_init_entity(struct bfq_entity *entity, ++ struct bfq_group *bfqg) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ entity->weight = entity->new_weight; ++ entity->orig_weight = entity->new_weight; ++ if (bfqq) { ++ bfqq->ioprio = bfqq->new_ioprio; ++ bfqq->ioprio_class = bfqq->new_ioprio_class; ++ bfqg_get(bfqg); ++ } ++ entity->parent = bfqg->my_entity; ++ entity->sched_data = &bfqg->sched_data; ++} ++ ++static void bfqg_stats_exit(struct bfqg_stats *stats) ++{ ++ blkg_rwstat_exit(&stats->merged); ++ blkg_rwstat_exit(&stats->service_time); ++ blkg_rwstat_exit(&stats->wait_time); ++ blkg_rwstat_exit(&stats->queued); ++ blkg_stat_exit(&stats->time); ++ blkg_stat_exit(&stats->avg_queue_size_sum); ++ blkg_stat_exit(&stats->avg_queue_size_samples); ++ blkg_stat_exit(&stats->dequeue); ++ blkg_stat_exit(&stats->group_wait_time); ++ blkg_stat_exit(&stats->idle_time); ++ blkg_stat_exit(&stats->empty_time); ++} ++ ++static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp) ++{ ++ if (blkg_rwstat_init(&stats->merged, gfp) || ++ blkg_rwstat_init(&stats->service_time, gfp) || ++ blkg_rwstat_init(&stats->wait_time, gfp) || ++ blkg_rwstat_init(&stats->queued, gfp) || ++ blkg_stat_init(&stats->time, gfp) || ++ blkg_stat_init(&stats->avg_queue_size_sum, gfp) || ++ blkg_stat_init(&stats->avg_queue_size_samples, gfp) || ++ blkg_stat_init(&stats->dequeue, gfp) || ++ blkg_stat_init(&stats->group_wait_time, gfp) || ++ blkg_stat_init(&stats->idle_time, gfp) || ++ blkg_stat_init(&stats->empty_time, gfp)) { ++ bfqg_stats_exit(stats); ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++static struct bfq_group_data *cpd_to_bfqgd(struct blkcg_policy_data *cpd) ++{ ++ return cpd ? container_of(cpd, struct bfq_group_data, pd) : NULL; ++} ++ ++static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg) ++{ ++ return cpd_to_bfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_bfq)); ++} ++ ++static struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp) ++{ ++ struct bfq_group_data *bgd; ++ ++ bgd = kzalloc(sizeof(*bgd), GFP_KERNEL); ++ if (!bgd) ++ return NULL; ++ return &bgd->pd; ++} ++ ++static void bfq_cpd_init(struct blkcg_policy_data *cpd) ++{ ++ struct bfq_group_data *d = cpd_to_bfqgd(cpd); ++ ++ d->weight = cgroup_subsys_on_dfl(io_cgrp_subsys) ? ++ CGROUP_WEIGHT_DFL : BFQ_WEIGHT_LEGACY_DFL; ++} ++ ++static void bfq_cpd_free(struct blkcg_policy_data *cpd) ++{ ++ kfree(cpd_to_bfqgd(cpd)); ++} ++ ++static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node) ++{ ++ struct bfq_group *bfqg; ++ ++ bfqg = kzalloc_node(sizeof(*bfqg), gfp, node); ++ if (!bfqg) ++ return NULL; ++ ++ if (bfqg_stats_init(&bfqg->stats, gfp)) { ++ kfree(bfqg); ++ return NULL; ++ } ++ ++ return &bfqg->pd; ++} ++ ++static void bfq_pd_init(struct blkg_policy_data *pd) ++{ ++ struct blkcg_gq *blkg; ++ struct bfq_group *bfqg; ++ struct bfq_data *bfqd; ++ struct bfq_entity *entity; ++ struct bfq_group_data *d; ++ ++ blkg = pd_to_blkg(pd); ++ BUG_ON(!blkg); ++ bfqg = blkg_to_bfqg(blkg); ++ bfqd = blkg->q->elevator->elevator_data; ++ entity = &bfqg->entity; ++ d = blkcg_to_bfqgd(blkg->blkcg); ++ ++ entity->orig_weight = entity->weight = entity->new_weight = d->weight; ++ entity->my_sched_data = &bfqg->sched_data; ++ bfqg->my_entity = entity; /* ++ * the root_group's will be set to NULL ++ * in bfq_init_queue() ++ */ ++ bfqg->bfqd = bfqd; ++ bfqg->active_entities = 0; ++ bfqg->rq_pos_tree = RB_ROOT; ++} ++ ++static void bfq_pd_free(struct blkg_policy_data *pd) ++{ ++ struct bfq_group *bfqg = pd_to_bfqg(pd); ++ ++ bfqg_stats_exit(&bfqg->stats); ++ return kfree(bfqg); ++} ++ ++static void bfq_pd_reset_stats(struct blkg_policy_data *pd) ++{ ++ struct bfq_group *bfqg = pd_to_bfqg(pd); ++ ++ bfqg_stats_reset(&bfqg->stats); ++} ++ ++static void bfq_group_set_parent(struct bfq_group *bfqg, ++ struct bfq_group *parent) ++{ ++ struct bfq_entity *entity; ++ ++ BUG_ON(!parent); ++ BUG_ON(!bfqg); ++ BUG_ON(bfqg == parent); ++ ++ entity = &bfqg->entity; ++ entity->parent = parent->my_entity; ++ entity->sched_data = &parent->sched_data; ++} ++ ++static struct bfq_group *bfq_lookup_bfqg(struct bfq_data *bfqd, ++ struct blkcg *blkcg) ++{ ++ struct blkcg_gq *blkg; ++ ++ blkg = blkg_lookup(blkcg, bfqd->queue); ++ if (likely(blkg)) ++ return blkg_to_bfqg(blkg); ++ return NULL; ++} ++ ++static struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd, ++ struct blkcg *blkcg) ++{ ++ struct bfq_group *bfqg, *parent; ++ struct bfq_entity *entity; ++ ++ assert_spin_locked(bfqd->queue->queue_lock); ++ ++ bfqg = bfq_lookup_bfqg(bfqd, blkcg); ++ ++ if (unlikely(!bfqg)) ++ return NULL; ++ ++ /* ++ * Update chain of bfq_groups as we might be handling a leaf group ++ * which, along with some of its relatives, has not been hooked yet ++ * to the private hierarchy of BFQ. ++ */ ++ entity = &bfqg->entity; ++ for_each_entity(entity) { ++ bfqg = container_of(entity, struct bfq_group, entity); ++ BUG_ON(!bfqg); ++ if (bfqg != bfqd->root_group) { ++ parent = bfqg_parent(bfqg); ++ if (!parent) ++ parent = bfqd->root_group; ++ BUG_ON(!parent); ++ bfq_group_set_parent(bfqg, parent); ++ } ++ } ++ ++ return bfqg; ++} ++ ++static void bfq_pos_tree_add_move(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq); ++ ++static void bfq_bfqq_expire(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ bool compensate, ++ enum bfqq_expiration reason); ++ ++/** ++ * bfq_bfqq_move - migrate @bfqq to @bfqg. ++ * @bfqd: queue descriptor. ++ * @bfqq: the queue to move. ++ * @bfqg: the group to move to. ++ * ++ * Move @bfqq to @bfqg, deactivating it from its old group and reactivating ++ * it on the new one. Avoid putting the entity on the old group idle tree. ++ * ++ * Must be called under the queue lock; the cgroup owning @bfqg must ++ * not disappear (by now this just means that we are called under ++ * rcu_read_lock()). ++ */ ++static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ struct bfq_group *bfqg) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ BUG_ON(!bfq_bfqq_busy(bfqq) && !RB_EMPTY_ROOT(&bfqq->sort_list)); ++ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list) && !entity->on_st); ++ BUG_ON(bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) ++ && entity->on_st && ++ bfqq != bfqd->in_service_queue); ++ BUG_ON(!bfq_bfqq_busy(bfqq) && bfqq == bfqd->in_service_queue); ++ ++ /* If bfqq is empty, then bfq_bfqq_expire also invokes ++ * bfq_del_bfqq_busy, thereby removing bfqq and its entity ++ * from data structures related to current group. Otherwise we ++ * need to remove bfqq explicitly with bfq_deactivate_bfqq, as ++ * we do below. ++ */ ++ if (bfqq == bfqd->in_service_queue) ++ bfq_bfqq_expire(bfqd, bfqd->in_service_queue, ++ false, BFQ_BFQQ_PREEMPTED); ++ ++ BUG_ON(entity->on_st && !bfq_bfqq_busy(bfqq) ++ && &bfq_entity_service_tree(entity)->idle != ++ entity->tree); ++ ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_busy(bfqq)); ++ ++ if (bfq_bfqq_busy(bfqq)) ++ bfq_deactivate_bfqq(bfqd, bfqq, 0); ++ else if (entity->on_st) { ++ BUG_ON(&bfq_entity_service_tree(entity)->idle != ++ entity->tree); ++ bfq_put_idle_entity(bfq_entity_service_tree(entity), entity); ++ } ++ bfqg_put(bfqq_group(bfqq)); ++ ++ /* ++ * Here we use a reference to bfqg. We don't need a refcounter ++ * as the cgroup reference will not be dropped, so that its ++ * destroy() callback will not be invoked. ++ */ ++ entity->parent = bfqg->my_entity; ++ entity->sched_data = &bfqg->sched_data; ++ bfqg_get(bfqg); ++ ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_busy(bfqq)); ++ if (bfq_bfqq_busy(bfqq)) { ++ bfq_pos_tree_add_move(bfqd, bfqq); ++ bfq_activate_bfqq(bfqd, bfqq); ++ } ++ ++ if (!bfqd->in_service_queue && !bfqd->rq_in_driver) ++ bfq_schedule_dispatch(bfqd); ++ BUG_ON(entity->on_st && !bfq_bfqq_busy(bfqq) ++ && &bfq_entity_service_tree(entity)->idle != ++ entity->tree); ++} ++ ++/** ++ * __bfq_bic_change_cgroup - move @bic to @cgroup. ++ * @bfqd: the queue descriptor. ++ * @bic: the bic to move. ++ * @blkcg: the blk-cgroup to move to. ++ * ++ * Move bic to blkcg, assuming that bfqd->queue is locked; the caller ++ * has to make sure that the reference to cgroup is valid across the call. ++ * ++ * NOTE: an alternative approach might have been to store the current ++ * cgroup in bfqq and getting a reference to it, reducing the lookup ++ * time here, at the price of slightly more complex code. ++ */ ++static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd, ++ struct bfq_io_cq *bic, ++ struct blkcg *blkcg) ++{ ++ struct bfq_queue *async_bfqq = bic_to_bfqq(bic, 0); ++ struct bfq_queue *sync_bfqq = bic_to_bfqq(bic, 1); ++ struct bfq_group *bfqg; ++ struct bfq_entity *entity; ++ ++ lockdep_assert_held(bfqd->queue->queue_lock); ++ ++ bfqg = bfq_find_set_group(bfqd, blkcg); ++ ++ if (unlikely(!bfqg)) ++ bfqg = bfqd->root_group; ++ ++ if (async_bfqq) { ++ entity = &async_bfqq->entity; ++ ++ if (entity->sched_data != &bfqg->sched_data) { ++ bic_set_bfqq(bic, NULL, 0); ++ bfq_log_bfqq(bfqd, async_bfqq, ++ "bic_change_group: %p %d", ++ async_bfqq, ++ async_bfqq->ref); ++ bfq_put_queue(async_bfqq); ++ } ++ } ++ ++ if (sync_bfqq) { ++ entity = &sync_bfqq->entity; ++ if (entity->sched_data != &bfqg->sched_data) ++ bfq_bfqq_move(bfqd, sync_bfqq, bfqg); ++ } ++ ++ return bfqg; ++} ++ ++static void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) ++{ ++ struct bfq_data *bfqd = bic_to_bfqd(bic); ++ struct bfq_group *bfqg = NULL; ++ uint64_t serial_nr; ++ ++ rcu_read_lock(); ++ serial_nr = bio_blkcg(bio)->css.serial_nr; ++ ++ /* ++ * Check whether blkcg has changed. The condition may trigger ++ * spuriously on a newly created cic but there's no harm. ++ */ ++ if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr)) ++ goto out; ++ ++ bfqg = __bfq_bic_change_cgroup(bfqd, bic, bio_blkcg(bio)); ++ bic->blkcg_serial_nr = serial_nr; ++out: ++ rcu_read_unlock(); ++} ++ ++/** ++ * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st. ++ * @st: the service tree being flushed. ++ */ ++static void bfq_flush_idle_tree(struct bfq_service_tree *st) ++{ ++ struct bfq_entity *entity = st->first_idle; ++ ++ for (; entity ; entity = st->first_idle) ++ __bfq_deactivate_entity(entity, 0); ++} ++ ++/** ++ * bfq_reparent_leaf_entity - move leaf entity to the root_group. ++ * @bfqd: the device data structure with the root group. ++ * @entity: the entity to move. ++ */ ++static void bfq_reparent_leaf_entity(struct bfq_data *bfqd, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ BUG_ON(!bfqq); ++ bfq_bfqq_move(bfqd, bfqq, bfqd->root_group); ++} ++ ++/** ++ * bfq_reparent_active_entities - move to the root group all active ++ * entities. ++ * @bfqd: the device data structure with the root group. ++ * @bfqg: the group to move from. ++ * @st: the service tree with the entities. ++ * ++ * Needs queue_lock to be taken and reference to be valid over the call. ++ */ ++static void bfq_reparent_active_entities(struct bfq_data *bfqd, ++ struct bfq_group *bfqg, ++ struct bfq_service_tree *st) ++{ ++ struct rb_root *active = &st->active; ++ struct bfq_entity *entity = NULL; ++ ++ if (!RB_EMPTY_ROOT(&st->active)) ++ entity = bfq_entity_of(rb_first(active)); ++ ++ for (; entity ; entity = bfq_entity_of(rb_first(active))) ++ bfq_reparent_leaf_entity(bfqd, entity); ++ ++ if (bfqg->sched_data.in_service_entity) ++ bfq_reparent_leaf_entity(bfqd, ++ bfqg->sched_data.in_service_entity); ++} ++ ++/** ++ * bfq_pd_offline - deactivate the entity associated with @pd, ++ * and reparent its children entities. ++ * @pd: descriptor of the policy going offline. ++ * ++ * blkio already grabs the queue_lock for us, so no need to use ++ * RCU-based magic ++ */ ++static void bfq_pd_offline(struct blkg_policy_data *pd) ++{ ++ struct bfq_service_tree *st; ++ struct bfq_group *bfqg; ++ struct bfq_data *bfqd; ++ struct bfq_entity *entity; ++ int i; ++ ++ BUG_ON(!pd); ++ bfqg = pd_to_bfqg(pd); ++ BUG_ON(!bfqg); ++ bfqd = bfqg->bfqd; ++ BUG_ON(bfqd && !bfqd->root_group); ++ ++ entity = bfqg->my_entity; ++ ++ if (!entity) /* root group */ ++ return; ++ ++ /* ++ * Empty all service_trees belonging to this group before ++ * deactivating the group itself. ++ */ ++ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) { ++ BUG_ON(!bfqg->sched_data.service_tree); ++ st = bfqg->sched_data.service_tree + i; ++ /* ++ * The idle tree may still contain bfq_queues belonging ++ * to exited task because they never migrated to a different ++ * cgroup from the one being destroyed now. No one else ++ * can access them so it's safe to act without any lock. ++ */ ++ bfq_flush_idle_tree(st); ++ ++ /* ++ * It may happen that some queues are still active ++ * (busy) upon group destruction (if the corresponding ++ * processes have been forced to terminate). We move ++ * all the leaf entities corresponding to these queues ++ * to the root_group. ++ * Also, it may happen that the group has an entity ++ * in service, which is disconnected from the active ++ * tree: it must be moved, too. ++ * There is no need to put the sync queues, as the ++ * scheduler has taken no reference. ++ */ ++ bfq_reparent_active_entities(bfqd, bfqg, st); ++ BUG_ON(!RB_EMPTY_ROOT(&st->active)); ++ BUG_ON(!RB_EMPTY_ROOT(&st->idle)); ++ } ++ BUG_ON(bfqg->sched_data.next_in_service); ++ BUG_ON(bfqg->sched_data.in_service_entity); ++ ++ __bfq_deactivate_entity(entity, 0); ++ bfq_put_async_queues(bfqd, bfqg); ++ BUG_ON(entity->tree); ++ ++ /* ++ * @blkg is going offline and will be ignored by ++ * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so ++ * that they don't get lost. If IOs complete after this point, the ++ * stats for them will be lost. Oh well... ++ */ ++ bfqg_stats_xfer_dead(bfqg); ++} ++ ++static void bfq_end_wr_async(struct bfq_data *bfqd) ++{ ++ struct blkcg_gq *blkg; ++ ++ list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) { ++ struct bfq_group *bfqg = blkg_to_bfqg(blkg); ++ BUG_ON(!bfqg); ++ ++ bfq_end_wr_async_queues(bfqd, bfqg); ++ } ++ bfq_end_wr_async_queues(bfqd, bfqd->root_group); ++} ++ ++static int bfq_io_show_weight(struct seq_file *sf, void *v) ++{ ++ struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); ++ struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg); ++ unsigned int val = 0; ++ ++ if (bfqgd) ++ val = bfqgd->weight; ++ ++ seq_printf(sf, "%u\n", val); ++ ++ return 0; ++} ++ ++static int bfq_io_set_weight_legacy(struct cgroup_subsys_state *css, ++ struct cftype *cftype, ++ u64 val) ++{ ++ struct blkcg *blkcg = css_to_blkcg(css); ++ struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg); ++ struct blkcg_gq *blkg; ++ int ret = -ERANGE; ++ ++ if (val < BFQ_MIN_WEIGHT || val > BFQ_MAX_WEIGHT) ++ return ret; ++ ++ ret = 0; ++ spin_lock_irq(&blkcg->lock); ++ bfqgd->weight = (unsigned short)val; ++ hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { ++ struct bfq_group *bfqg = blkg_to_bfqg(blkg); ++ ++ if (!bfqg) ++ continue; ++ /* ++ * Setting the prio_changed flag of the entity ++ * to 1 with new_weight == weight would re-set ++ * the value of the weight to its ioprio mapping. ++ * Set the flag only if necessary. ++ */ ++ if ((unsigned short)val != bfqg->entity.new_weight) { ++ bfqg->entity.new_weight = (unsigned short)val; ++ /* ++ * Make sure that the above new value has been ++ * stored in bfqg->entity.new_weight before ++ * setting the prio_changed flag. In fact, ++ * this flag may be read asynchronously (in ++ * critical sections protected by a different ++ * lock than that held here), and finding this ++ * flag set may cause the execution of the code ++ * for updating parameters whose value may ++ * depend also on bfqg->entity.new_weight (in ++ * __bfq_entity_update_weight_prio). ++ * This barrier makes sure that the new value ++ * of bfqg->entity.new_weight is correctly ++ * seen in that code. ++ */ ++ smp_wmb(); ++ bfqg->entity.prio_changed = 1; ++ } ++ } ++ spin_unlock_irq(&blkcg->lock); ++ ++ return ret; ++} ++ ++static ssize_t bfq_io_set_weight(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, ++ loff_t off) ++{ ++ u64 weight; ++ /* First unsigned long found in the file is used */ ++ int ret = kstrtoull(strim(buf), 0, &weight); ++ ++ if (ret) ++ return ret; ++ ++ return bfq_io_set_weight_legacy(of_css(of), NULL, weight); ++} ++ ++static int bfqg_print_stat(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat, ++ &blkcg_policy_bfq, seq_cft(sf)->private, false); ++ return 0; ++} ++ ++static int bfqg_print_rwstat(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat, ++ &blkcg_policy_bfq, seq_cft(sf)->private, true); ++ return 0; ++} ++ ++static u64 bfqg_prfill_stat_recursive(struct seq_file *sf, ++ struct blkg_policy_data *pd, int off) ++{ ++ u64 sum = blkg_stat_recursive_sum(pd_to_blkg(pd), ++ &blkcg_policy_bfq, off); ++ return __blkg_prfill_u64(sf, pd, sum); ++} ++ ++static u64 bfqg_prfill_rwstat_recursive(struct seq_file *sf, ++ struct blkg_policy_data *pd, int off) ++{ ++ struct blkg_rwstat sum = blkg_rwstat_recursive_sum(pd_to_blkg(pd), ++ &blkcg_policy_bfq, ++ off); ++ return __blkg_prfill_rwstat(sf, pd, &sum); ++} ++ ++static int bfqg_print_stat_recursive(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_stat_recursive, &blkcg_policy_bfq, ++ seq_cft(sf)->private, false); ++ return 0; ++} ++ ++static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq, ++ seq_cft(sf)->private, true); ++ return 0; ++} ++ ++static u64 bfqg_prfill_sectors(struct seq_file *sf, struct blkg_policy_data *pd, ++ int off) ++{ ++ u64 sum = blkg_rwstat_total(&pd->blkg->stat_bytes); ++ ++ return __blkg_prfill_u64(sf, pd, sum >> 9); ++} ++ ++static int bfqg_print_stat_sectors(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_sectors, &blkcg_policy_bfq, 0, false); ++ return 0; ++} ++ ++static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf, ++ struct blkg_policy_data *pd, int off) ++{ ++ struct blkg_rwstat tmp = blkg_rwstat_recursive_sum(pd->blkg, NULL, ++ offsetof(struct blkcg_gq, stat_bytes)); ++ u64 sum = atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) + ++ atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]); ++ ++ return __blkg_prfill_u64(sf, pd, sum >> 9); ++} ++ ++static int bfqg_print_stat_sectors_recursive(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_sectors_recursive, &blkcg_policy_bfq, 0, ++ false); ++ return 0; ++} ++ ++ ++static u64 bfqg_prfill_avg_queue_size(struct seq_file *sf, ++ struct blkg_policy_data *pd, int off) ++{ ++ struct bfq_group *bfqg = pd_to_bfqg(pd); ++ u64 samples = blkg_stat_read(&bfqg->stats.avg_queue_size_samples); ++ u64 v = 0; ++ ++ if (samples) { ++ v = blkg_stat_read(&bfqg->stats.avg_queue_size_sum); ++ v = div64_u64(v, samples); ++ } ++ __blkg_prfill_u64(sf, pd, v); ++ return 0; ++} ++ ++/* print avg_queue_size */ ++static int bfqg_print_avg_queue_size(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_avg_queue_size, &blkcg_policy_bfq, ++ 0, false); ++ return 0; ++} ++ ++static struct bfq_group * ++bfq_create_group_hierarchy(struct bfq_data *bfqd, int node) ++{ ++ int ret; ++ ++ ret = blkcg_activate_policy(bfqd->queue, &blkcg_policy_bfq); ++ if (ret) ++ return NULL; ++ ++ return blkg_to_bfqg(bfqd->queue->root_blkg); ++} ++ ++static struct cftype bfq_blkcg_legacy_files[] = { ++ { ++ .name = "bfq.weight", ++ .flags = CFTYPE_NOT_ON_ROOT, ++ .seq_show = bfq_io_show_weight, ++ .write_u64 = bfq_io_set_weight_legacy, ++ }, ++ ++ /* statistics, covers only the tasks in the bfqg */ ++ { ++ .name = "bfq.time", ++ .private = offsetof(struct bfq_group, stats.time), ++ .seq_show = bfqg_print_stat, ++ }, ++ { ++ .name = "bfq.sectors", ++ .seq_show = bfqg_print_stat_sectors, ++ }, ++ { ++ .name = "bfq.io_service_bytes", ++ .private = (unsigned long)&blkcg_policy_bfq, ++ .seq_show = blkg_print_stat_bytes, ++ }, ++ { ++ .name = "bfq.io_serviced", ++ .private = (unsigned long)&blkcg_policy_bfq, ++ .seq_show = blkg_print_stat_ios, ++ }, ++ { ++ .name = "bfq.io_service_time", ++ .private = offsetof(struct bfq_group, stats.service_time), ++ .seq_show = bfqg_print_rwstat, ++ }, ++ { ++ .name = "bfq.io_wait_time", ++ .private = offsetof(struct bfq_group, stats.wait_time), ++ .seq_show = bfqg_print_rwstat, ++ }, ++ { ++ .name = "bfq.io_merged", ++ .private = offsetof(struct bfq_group, stats.merged), ++ .seq_show = bfqg_print_rwstat, ++ }, ++ { ++ .name = "bfq.io_queued", ++ .private = offsetof(struct bfq_group, stats.queued), ++ .seq_show = bfqg_print_rwstat, ++ }, ++ ++ /* the same statictics which cover the bfqg and its descendants */ ++ { ++ .name = "bfq.time_recursive", ++ .private = offsetof(struct bfq_group, stats.time), ++ .seq_show = bfqg_print_stat_recursive, ++ }, ++ { ++ .name = "bfq.sectors_recursive", ++ .seq_show = bfqg_print_stat_sectors_recursive, ++ }, ++ { ++ .name = "bfq.io_service_bytes_recursive", ++ .private = (unsigned long)&blkcg_policy_bfq, ++ .seq_show = blkg_print_stat_bytes_recursive, ++ }, ++ { ++ .name = "bfq.io_serviced_recursive", ++ .private = (unsigned long)&blkcg_policy_bfq, ++ .seq_show = blkg_print_stat_ios_recursive, ++ }, ++ { ++ .name = "bfq.io_service_time_recursive", ++ .private = offsetof(struct bfq_group, stats.service_time), ++ .seq_show = bfqg_print_rwstat_recursive, ++ }, ++ { ++ .name = "bfq.io_wait_time_recursive", ++ .private = offsetof(struct bfq_group, stats.wait_time), ++ .seq_show = bfqg_print_rwstat_recursive, ++ }, ++ { ++ .name = "bfq.io_merged_recursive", ++ .private = offsetof(struct bfq_group, stats.merged), ++ .seq_show = bfqg_print_rwstat_recursive, ++ }, ++ { ++ .name = "bfq.io_queued_recursive", ++ .private = offsetof(struct bfq_group, stats.queued), ++ .seq_show = bfqg_print_rwstat_recursive, ++ }, ++ { ++ .name = "bfq.avg_queue_size", ++ .seq_show = bfqg_print_avg_queue_size, ++ }, ++ { ++ .name = "bfq.group_wait_time", ++ .private = offsetof(struct bfq_group, stats.group_wait_time), ++ .seq_show = bfqg_print_stat, ++ }, ++ { ++ .name = "bfq.idle_time", ++ .private = offsetof(struct bfq_group, stats.idle_time), ++ .seq_show = bfqg_print_stat, ++ }, ++ { ++ .name = "bfq.empty_time", ++ .private = offsetof(struct bfq_group, stats.empty_time), ++ .seq_show = bfqg_print_stat, ++ }, ++ { ++ .name = "bfq.dequeue", ++ .private = offsetof(struct bfq_group, stats.dequeue), ++ .seq_show = bfqg_print_stat, ++ }, ++ { } /* terminate */ ++}; ++ ++static struct cftype bfq_blkg_files[] = { ++ { ++ .name = "bfq.weight", ++ .flags = CFTYPE_NOT_ON_ROOT, ++ .seq_show = bfq_io_show_weight, ++ .write = bfq_io_set_weight, ++ }, ++ {} /* terminate */ ++}; ++ ++#else /* CONFIG_BFQ_GROUP_IOSCHED */ ++ ++static inline void bfqg_stats_update_io_add(struct bfq_group *bfqg, ++ struct bfq_queue *bfqq, int op, int op_flags) { } ++static inline void ++bfqg_stats_update_io_remove(struct bfq_group *bfqg, int op, int op_flags) { } ++static inline void ++bfqg_stats_update_io_merged(struct bfq_group *bfqg, int op, int op_flags) { } ++static inline void bfqg_stats_update_completion(struct bfq_group *bfqg, ++ uint64_t start_time, uint64_t io_start_time, int op, ++ int op_flags) { } ++static inline void ++bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg, ++ struct bfq_group *curr_bfqg) { } ++static inline void bfqg_stats_end_empty_time(struct bfqg_stats *stats) { } ++static inline void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { } ++static inline void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { } ++static inline void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { } ++static inline void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { } ++static inline void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { } ++ ++static void bfq_init_entity(struct bfq_entity *entity, ++ struct bfq_group *bfqg) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ entity->weight = entity->new_weight; ++ entity->orig_weight = entity->new_weight; ++ if (bfqq) { ++ bfqq->ioprio = bfqq->new_ioprio; ++ bfqq->ioprio_class = bfqq->new_ioprio_class; ++ } ++ entity->sched_data = &bfqg->sched_data; ++} ++ ++static struct bfq_group * ++bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) ++{ ++ struct bfq_data *bfqd = bic_to_bfqd(bic); ++ ++ return bfqd->root_group; ++} ++ ++static void bfq_end_wr_async(struct bfq_data *bfqd) ++{ ++ bfq_end_wr_async_queues(bfqd, bfqd->root_group); ++} ++ ++static struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd, ++ struct blkcg *blkcg) ++{ ++ return bfqd->root_group; ++} ++ ++static struct bfq_group *bfqq_group(struct bfq_queue *bfqq) ++{ ++ return bfqq->bfqd->root_group; ++} ++ ++static struct bfq_group * ++bfq_create_group_hierarchy(struct bfq_data *bfqd, int node) ++{ ++ struct bfq_group *bfqg; ++ int i; ++ ++ bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node); ++ if (!bfqg) ++ return NULL; ++ ++ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) ++ bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT; ++ ++ return bfqg; ++} ++#endif +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq.h linux-4.8-bfq/block/bfq.h +--- linux-4.8/block/bfq.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq.h 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,886 @@ ++/* ++ * BFQ-v8r4 for 4.8.0: data structures and common functions prototypes. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2015 Paolo Valente ++ * ++ * Copyright (C) 2016 Paolo Valente ++ */ ++ ++#ifndef _BFQ_H ++#define _BFQ_H ++ ++#include ++#include ++#include ++#include ++#include ++ ++#define BFQ_IOPRIO_CLASSES 3 ++#define BFQ_CL_IDLE_TIMEOUT (HZ/5) ++ ++#define BFQ_MIN_WEIGHT 1 ++#define BFQ_MAX_WEIGHT 1000 ++#define BFQ_WEIGHT_CONVERSION_COEFF 10 ++ ++#define BFQ_DEFAULT_QUEUE_IOPRIO 4 ++ ++#define BFQ_WEIGHT_LEGACY_DFL 100 ++#define BFQ_DEFAULT_GRP_IOPRIO 0 ++#define BFQ_DEFAULT_GRP_CLASS IOPRIO_CLASS_BE ++ ++/* ++ * Soft real-time applications are extremely more latency sensitive ++ * than interactive ones. Over-raise the weight of the former to ++ * privilege them against the latter. ++ */ ++#define BFQ_SOFTRT_WEIGHT_FACTOR 100 ++ ++struct bfq_entity; ++ ++/** ++ * struct bfq_service_tree - per ioprio_class service tree. ++ * ++ * Each service tree represents a B-WF2Q+ scheduler on its own. Each ++ * ioprio_class has its own independent scheduler, and so its own ++ * bfq_service_tree. All the fields are protected by the queue lock ++ * of the containing bfqd. ++ */ ++struct bfq_service_tree { ++ /* tree for active entities (i.e., those backlogged) */ ++ struct rb_root active; ++ /* tree for idle entities (i.e., not backlogged, with V <= F_i)*/ ++ struct rb_root idle; ++ ++ struct bfq_entity *first_idle; /* idle entity with minimum F_i */ ++ struct bfq_entity *last_idle; /* idle entity with maximum F_i */ ++ ++ u64 vtime; /* scheduler virtual time */ ++ /* scheduler weight sum; active and idle entities contribute to it */ ++ unsigned long wsum; ++}; ++ ++/** ++ * struct bfq_sched_data - multi-class scheduler. ++ * ++ * bfq_sched_data is the basic scheduler queue. It supports three ++ * ioprio_classes, and can be used either as a toplevel queue or as an ++ * intermediate queue on a hierarchical setup. @next_in_service ++ * points to the active entity of the sched_data service trees that ++ * will be scheduled next. It is used to reduce the number of steps ++ * needed for each hierarchical-schedule update. ++ * ++ * The supported ioprio_classes are the same as in CFQ, in descending ++ * priority order, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE. ++ * Requests from higher priority queues are served before all the ++ * requests from lower priority queues; among requests of the same ++ * queue requests are served according to B-WF2Q+. ++ * All the fields are protected by the queue lock of the containing bfqd. ++ */ ++struct bfq_sched_data { ++ struct bfq_entity *in_service_entity; /* entity in service */ ++ /* head-of-the-line entity in the scheduler (see comments above) */ ++ struct bfq_entity *next_in_service; ++ /* array of service trees, one per ioprio_class */ ++ struct bfq_service_tree service_tree[BFQ_IOPRIO_CLASSES]; ++}; ++ ++/** ++ * struct bfq_weight_counter - counter of the number of all active entities ++ * with a given weight. ++ */ ++struct bfq_weight_counter { ++ unsigned int weight; /* weight of the entities this counter refers to */ ++ unsigned int num_active; /* nr of active entities with this weight */ ++ /* ++ * Weights tree member (see bfq_data's @queue_weights_tree and ++ * @group_weights_tree) ++ */ ++ struct rb_node weights_node; ++}; ++ ++/** ++ * struct bfq_entity - schedulable entity. ++ * ++ * A bfq_entity is used to represent either a bfq_queue (leaf node in the ++ * cgroup hierarchy) or a bfq_group into the upper level scheduler. Each ++ * entity belongs to the sched_data of the parent group in the cgroup ++ * hierarchy. Non-leaf entities have also their own sched_data, stored ++ * in @my_sched_data. ++ * ++ * Each entity stores independently its priority values; this would ++ * allow different weights on different devices, but this ++ * functionality is not exported to userspace by now. Priorities and ++ * weights are updated lazily, first storing the new values into the ++ * new_* fields, then setting the @prio_changed flag. As soon as ++ * there is a transition in the entity state that allows the priority ++ * update to take place the effective and the requested priority ++ * values are synchronized. ++ * ++ * Unless cgroups are used, the weight value is calculated from the ++ * ioprio to export the same interface as CFQ. When dealing with ++ * ``well-behaved'' queues (i.e., queues that do not spend too much ++ * time to consume their budget and have true sequential behavior, and ++ * when there are no external factors breaking anticipation) the ++ * relative weights at each level of the cgroups hierarchy should be ++ * guaranteed. All the fields are protected by the queue lock of the ++ * containing bfqd. ++ */ ++struct bfq_entity { ++ struct rb_node rb_node; /* service_tree member */ ++ /* pointer to the weight counter associated with this entity */ ++ struct bfq_weight_counter *weight_counter; ++ ++ /* ++ * flag, true if the entity is on a tree (either the active or ++ * the idle one of its service_tree). ++ */ ++ int on_st; ++ ++ u64 finish; /* B-WF2Q+ finish timestamp (aka F_i) */ ++ u64 start; /* B-WF2Q+ start timestamp (aka S_i) */ ++ ++ /* tree the entity is enqueued into; %NULL if not on a tree */ ++ struct rb_root *tree; ++ ++ /* ++ * minimum start time of the (active) subtree rooted at this ++ * entity; used for O(log N) lookups into active trees ++ */ ++ u64 min_start; ++ ++ /* amount of service received during the last service slot */ ++ int service; ++ ++ /* budget, used also to calculate F_i: F_i = S_i + @budget / @weight */ ++ int budget; ++ ++ unsigned int weight; /* weight of the queue */ ++ unsigned int new_weight; /* next weight if a change is in progress */ ++ ++ /* original weight, used to implement weight boosting */ ++ unsigned int orig_weight; ++ ++ /* parent entity, for hierarchical scheduling */ ++ struct bfq_entity *parent; ++ ++ /* ++ * For non-leaf nodes in the hierarchy, the associated ++ * scheduler queue, %NULL on leaf nodes. ++ */ ++ struct bfq_sched_data *my_sched_data; ++ /* the scheduler queue this entity belongs to */ ++ struct bfq_sched_data *sched_data; ++ ++ /* flag, set to request a weight, ioprio or ioprio_class change */ ++ int prio_changed; ++}; ++ ++struct bfq_group; ++ ++/** ++ * struct bfq_queue - leaf schedulable entity. ++ * ++ * A bfq_queue is a leaf request queue; it can be associated with an ++ * io_context or more, if it is async or shared between cooperating ++ * processes. @cgroup holds a reference to the cgroup, to be sure that it ++ * does not disappear while a bfqq still references it (mostly to avoid ++ * races between request issuing and task migration followed by cgroup ++ * destruction). ++ * All the fields are protected by the queue lock of the containing bfqd. ++ */ ++struct bfq_queue { ++ /* reference counter */ ++ int ref; ++ /* parent bfq_data */ ++ struct bfq_data *bfqd; ++ ++ /* current ioprio and ioprio class */ ++ unsigned short ioprio, ioprio_class; ++ /* next ioprio and ioprio class if a change is in progress */ ++ unsigned short new_ioprio, new_ioprio_class; ++ ++ /* ++ * Shared bfq_queue if queue is cooperating with one or more ++ * other queues. ++ */ ++ struct bfq_queue *new_bfqq; ++ /* request-position tree member (see bfq_group's @rq_pos_tree) */ ++ struct rb_node pos_node; ++ /* request-position tree root (see bfq_group's @rq_pos_tree) */ ++ struct rb_root *pos_root; ++ ++ /* sorted list of pending requests */ ++ struct rb_root sort_list; ++ /* if fifo isn't expired, next request to serve */ ++ struct request *next_rq; ++ /* number of sync and async requests queued */ ++ int queued[2]; ++ /* number of sync and async requests currently allocated */ ++ int allocated[2]; ++ /* number of pending metadata requests */ ++ int meta_pending; ++ /* fifo list of requests in sort_list */ ++ struct list_head fifo; ++ ++ /* entity representing this queue in the scheduler */ ++ struct bfq_entity entity; ++ ++ /* maximum budget allowed from the feedback mechanism */ ++ int max_budget; ++ /* budget expiration (in jiffies) */ ++ unsigned long budget_timeout; ++ ++ /* number of requests on the dispatch list or inside driver */ ++ int dispatched; ++ ++ unsigned int flags; /* status flags.*/ ++ ++ /* node for active/idle bfqq list inside parent bfqd */ ++ struct list_head bfqq_list; ++ ++ /* bit vector: a 1 for each seeky requests in history */ ++ u32 seek_history; ++ ++ /* node for the device's burst list */ ++ struct hlist_node burst_list_node; ++ ++ /* position of the last request enqueued */ ++ sector_t last_request_pos; ++ ++ /* Number of consecutive pairs of request completion and ++ * arrival, such that the queue becomes idle after the ++ * completion, but the next request arrives within an idle ++ * time slice; used only if the queue's IO_bound flag has been ++ * cleared. ++ */ ++ unsigned int requests_within_timer; ++ ++ /* pid of the process owning the queue, used for logging purposes */ ++ pid_t pid; ++ ++ /* ++ * Pointer to the bfq_io_cq owning the bfq_queue, set to %NULL ++ * if the queue is shared. ++ */ ++ struct bfq_io_cq *bic; ++ ++ /* current maximum weight-raising time for this queue */ ++ unsigned long wr_cur_max_time; ++ /* ++ * Minimum time instant such that, only if a new request is ++ * enqueued after this time instant in an idle @bfq_queue with ++ * no outstanding requests, then the task associated with the ++ * queue it is deemed as soft real-time (see the comments on ++ * the function bfq_bfqq_softrt_next_start()) ++ */ ++ unsigned long soft_rt_next_start; ++ /* ++ * Start time of the current weight-raising period if ++ * the @bfq-queue is being weight-raised, otherwise ++ * finish time of the last weight-raising period. ++ */ ++ unsigned long last_wr_start_finish; ++ /* factor by which the weight of this queue is multiplied */ ++ unsigned int wr_coeff; ++ /* ++ * Time of the last transition of the @bfq_queue from idle to ++ * backlogged. ++ */ ++ unsigned long last_idle_bklogged; ++ /* ++ * Cumulative service received from the @bfq_queue since the ++ * last transition from idle to backlogged. ++ */ ++ unsigned long service_from_backlogged; ++ /* ++ * Value of wr start time when switching to soft rt ++ */ ++ unsigned long wr_start_at_switch_to_srt; ++ ++ unsigned long split_time; /* time of last split */ ++}; ++ ++/** ++ * struct bfq_ttime - per process thinktime stats. ++ */ ++struct bfq_ttime { ++ u64 last_end_request; /* completion time of last request */ ++ ++ u64 ttime_total; /* total process thinktime */ ++ unsigned long ttime_samples; /* number of thinktime samples */ ++ u64 ttime_mean; /* average process thinktime */ ++ ++}; ++ ++/** ++ * struct bfq_io_cq - per (request_queue, io_context) structure. ++ */ ++struct bfq_io_cq { ++ /* associated io_cq structure */ ++ struct io_cq icq; /* must be the first member */ ++ /* array of two process queues, the sync and the async */ ++ struct bfq_queue *bfqq[2]; ++ /* associated @bfq_ttime struct */ ++ struct bfq_ttime ttime; ++ /* per (request_queue, blkcg) ioprio */ ++ int ioprio; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ uint64_t blkcg_serial_nr; /* the current blkcg serial */ ++#endif ++ ++ /* ++ * Snapshot of the idle window before merging; taken to ++ * remember this value while the queue is merged, so as to be ++ * able to restore it in case of split. ++ */ ++ bool saved_idle_window; ++ /* ++ * Same purpose as the previous two fields for the I/O bound ++ * classification of a queue. ++ */ ++ bool saved_IO_bound; ++ ++ /* ++ * Same purpose as the previous fields for the value of the ++ * field keeping the queue's belonging to a large burst ++ */ ++ bool saved_in_large_burst; ++ /* ++ * True if the queue belonged to a burst list before its merge ++ * with another cooperating queue. ++ */ ++ bool was_in_burst_list; ++ ++ /* ++ * Similar to previous fields: save wr information. ++ */ ++ unsigned long saved_wr_coeff; ++ unsigned long saved_last_wr_start_finish; ++ unsigned long saved_wr_start_at_switch_to_srt; ++}; ++ ++enum bfq_device_speed { ++ BFQ_BFQD_FAST, ++ BFQ_BFQD_SLOW, ++}; ++ ++/** ++ * struct bfq_data - per-device data structure. ++ * ++ * All the fields are protected by the @queue lock. ++ */ ++struct bfq_data { ++ /* request queue for the device */ ++ struct request_queue *queue; ++ ++ /* root bfq_group for the device */ ++ struct bfq_group *root_group; ++ ++ /* ++ * rbtree of weight counters of @bfq_queues, sorted by ++ * weight. Used to keep track of whether all @bfq_queues have ++ * the same weight. The tree contains one counter for each ++ * distinct weight associated to some active and not ++ * weight-raised @bfq_queue (see the comments to the functions ++ * bfq_weights_tree_[add|remove] for further details). ++ */ ++ struct rb_root queue_weights_tree; ++ /* ++ * rbtree of non-queue @bfq_entity weight counters, sorted by ++ * weight. Used to keep track of whether all @bfq_groups have ++ * the same weight. The tree contains one counter for each ++ * distinct weight associated to some active @bfq_group (see ++ * the comments to the functions bfq_weights_tree_[add|remove] ++ * for further details). ++ */ ++ struct rb_root group_weights_tree; ++ ++ /* ++ * Number of bfq_queues containing requests (including the ++ * queue in service, even if it is idling). ++ */ ++ int busy_queues; ++ /* number of weight-raised busy @bfq_queues */ ++ int wr_busy_queues; ++ /* number of queued requests */ ++ int queued; ++ /* number of requests dispatched and waiting for completion */ ++ int rq_in_driver; ++ ++ /* ++ * Maximum number of requests in driver in the last ++ * @hw_tag_samples completed requests. ++ */ ++ int max_rq_in_driver; ++ /* number of samples used to calculate hw_tag */ ++ int hw_tag_samples; ++ /* flag set to one if the driver is showing a queueing behavior */ ++ int hw_tag; ++ ++ /* number of budgets assigned */ ++ int budgets_assigned; ++ ++ /* ++ * Timer set when idling (waiting) for the next request from ++ * the queue in service. ++ */ ++ struct hrtimer idle_slice_timer; ++ /* delayed work to restart dispatching on the request queue */ ++ struct work_struct unplug_work; ++ ++ /* bfq_queue in service */ ++ struct bfq_queue *in_service_queue; ++ /* bfq_io_cq (bic) associated with the @in_service_queue */ ++ struct bfq_io_cq *in_service_bic; ++ ++ /* on-disk position of the last served request */ ++ sector_t last_position; ++ ++ /* time of last request completion (ns) */ ++ u64 last_completion; ++ ++ /* time of first rq dispatch in current observation interval (ns) */ ++ u64 first_dispatch; ++ /* time of last rq dispatch in current observation interval (ns) */ ++ u64 last_dispatch; ++ ++ /* beginning of the last budget */ ++ ktime_t last_budget_start; ++ /* beginning of the last idle slice */ ++ ktime_t last_idling_start; ++ ++ /* number of samples in current observation interval */ ++ int peak_rate_samples; ++ /* num of samples of seq dispatches in current observation interval */ ++ u32 sequential_samples; ++ /* total num of sectors transferred in current observation interval */ ++ u64 tot_sectors_dispatched; ++ /* max rq size seen during current observation interval (sectors) */ ++ u32 last_rq_max_size; ++ /* time elapsed from first dispatch in current observ. interval (us) */ ++ u64 delta_from_first; ++ /* current estimate of device peak rate */ ++ u32 peak_rate; ++ ++ /* maximum budget allotted to a bfq_queue before rescheduling */ ++ int bfq_max_budget; ++ ++ /* list of all the bfq_queues active on the device */ ++ struct list_head active_list; ++ /* list of all the bfq_queues idle on the device */ ++ struct list_head idle_list; ++ ++ /* ++ * Timeout for async/sync requests; when it fires, requests ++ * are served in fifo order. ++ */ ++ u64 bfq_fifo_expire[2]; ++ /* weight of backward seeks wrt forward ones */ ++ unsigned int bfq_back_penalty; ++ /* maximum allowed backward seek */ ++ unsigned int bfq_back_max; ++ /* maximum idling time */ ++ u32 bfq_slice_idle; ++ /* last time CLASS_IDLE was served */ ++ u64 bfq_class_idle_last_service; ++ ++ /* user-configured max budget value (0 for auto-tuning) */ ++ int bfq_user_max_budget; ++ /* ++ * Timeout for bfq_queues to consume their budget; used to ++ * prevent seeky queues from imposing long latencies to ++ * sequential or quasi-sequential ones (this also implies that ++ * seeky queues cannot receive guarantees in the service ++ * domain; after a timeout they are charged for the time they ++ * have been in service, to preserve fairness among them, but ++ * without service-domain guarantees). ++ */ ++ unsigned int bfq_timeout; ++ ++ /* ++ * Number of consecutive requests that must be issued within ++ * the idle time slice to set again idling to a queue which ++ * was marked as non-I/O-bound (see the definition of the ++ * IO_bound flag for further details). ++ */ ++ unsigned int bfq_requests_within_timer; ++ ++ /* ++ * Force device idling whenever needed to provide accurate ++ * service guarantees, without caring about throughput ++ * issues. CAVEAT: this may even increase latencies, in case ++ * of useless idling for processes that did stop doing I/O. ++ */ ++ bool strict_guarantees; ++ ++ /* ++ * Last time at which a queue entered the current burst of ++ * queues being activated shortly after each other; for more ++ * details about this and the following parameters related to ++ * a burst of activations, see the comments on the function ++ * bfq_handle_burst. ++ */ ++ unsigned long last_ins_in_burst; ++ /* ++ * Reference time interval used to decide whether a queue has ++ * been activated shortly after @last_ins_in_burst. ++ */ ++ unsigned long bfq_burst_interval; ++ /* number of queues in the current burst of queue activations */ ++ int burst_size; ++ ++ /* common parent entity for the queues in the burst */ ++ struct bfq_entity *burst_parent_entity; ++ /* Maximum burst size above which the current queue-activation ++ * burst is deemed as 'large'. ++ */ ++ unsigned long bfq_large_burst_thresh; ++ /* true if a large queue-activation burst is in progress */ ++ bool large_burst; ++ /* ++ * Head of the burst list (as for the above fields, more ++ * details in the comments on the function bfq_handle_burst). ++ */ ++ struct hlist_head burst_list; ++ ++ /* if set to true, low-latency heuristics are enabled */ ++ bool low_latency; ++ /* ++ * Maximum factor by which the weight of a weight-raised queue ++ * is multiplied. ++ */ ++ unsigned int bfq_wr_coeff; ++ /* maximum duration of a weight-raising period (jiffies) */ ++ unsigned int bfq_wr_max_time; ++ ++ /* Maximum weight-raising duration for soft real-time processes */ ++ unsigned int bfq_wr_rt_max_time; ++ /* ++ * Minimum idle period after which weight-raising may be ++ * reactivated for a queue (in jiffies). ++ */ ++ unsigned int bfq_wr_min_idle_time; ++ /* ++ * Minimum period between request arrivals after which ++ * weight-raising may be reactivated for an already busy async ++ * queue (in jiffies). ++ */ ++ unsigned long bfq_wr_min_inter_arr_async; ++ ++ /* Max service-rate for a soft real-time queue, in sectors/sec */ ++ unsigned int bfq_wr_max_softrt_rate; ++ /* ++ * Cached value of the product R*T, used for computing the ++ * maximum duration of weight raising automatically. ++ */ ++ u64 RT_prod; ++ /* device-speed class for the low-latency heuristic */ ++ enum bfq_device_speed device_speed; ++ ++ /* fallback dummy bfqq for extreme OOM conditions */ ++ struct bfq_queue oom_bfqq; ++}; ++ ++enum bfqq_state_flags { ++ BFQ_BFQQ_FLAG_just_created = 0, /* queue just allocated */ ++ BFQ_BFQQ_FLAG_busy, /* has requests or is in service */ ++ BFQ_BFQQ_FLAG_wait_request, /* waiting for a request */ ++ BFQ_BFQQ_FLAG_non_blocking_wait_rq, /* ++ * waiting for a request ++ * without idling the device ++ */ ++ BFQ_BFQQ_FLAG_must_alloc, /* must be allowed rq alloc */ ++ BFQ_BFQQ_FLAG_fifo_expire, /* FIFO checked in this slice */ ++ BFQ_BFQQ_FLAG_idle_window, /* slice idling enabled */ ++ BFQ_BFQQ_FLAG_sync, /* synchronous queue */ ++ BFQ_BFQQ_FLAG_IO_bound, /* ++ * bfqq has timed-out at least once ++ * having consumed at most 2/10 of ++ * its budget ++ */ ++ BFQ_BFQQ_FLAG_in_large_burst, /* ++ * bfqq activated in a large burst, ++ * see comments to bfq_handle_burst. ++ */ ++ BFQ_BFQQ_FLAG_softrt_update, /* ++ * may need softrt-next-start ++ * update ++ */ ++ BFQ_BFQQ_FLAG_coop, /* bfqq is shared */ ++ BFQ_BFQQ_FLAG_split_coop /* shared bfqq will be split */ ++}; ++ ++#define BFQ_BFQQ_FNS(name) \ ++static void bfq_mark_bfqq_##name(struct bfq_queue *bfqq) \ ++{ \ ++ (bfqq)->flags |= (1 << BFQ_BFQQ_FLAG_##name); \ ++} \ ++static void bfq_clear_bfqq_##name(struct bfq_queue *bfqq) \ ++{ \ ++ (bfqq)->flags &= ~(1 << BFQ_BFQQ_FLAG_##name); \ ++} \ ++static int bfq_bfqq_##name(const struct bfq_queue *bfqq) \ ++{ \ ++ return ((bfqq)->flags & (1 << BFQ_BFQQ_FLAG_##name)) != 0; \ ++} ++ ++BFQ_BFQQ_FNS(just_created); ++BFQ_BFQQ_FNS(busy); ++BFQ_BFQQ_FNS(wait_request); ++BFQ_BFQQ_FNS(non_blocking_wait_rq); ++BFQ_BFQQ_FNS(must_alloc); ++BFQ_BFQQ_FNS(fifo_expire); ++BFQ_BFQQ_FNS(idle_window); ++BFQ_BFQQ_FNS(sync); ++BFQ_BFQQ_FNS(IO_bound); ++BFQ_BFQQ_FNS(in_large_burst); ++BFQ_BFQQ_FNS(coop); ++BFQ_BFQQ_FNS(split_coop); ++BFQ_BFQQ_FNS(softrt_update); ++#undef BFQ_BFQQ_FNS ++ ++/* Logging facilities. */ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static struct bfq_group *bfqq_group(struct bfq_queue *bfqq); ++static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg); ++ ++#define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \ ++ char __pbuf[128]; \ ++ \ ++ assert_spin_locked((bfqd)->queue->queue_lock); \ ++ blkg_path(bfqg_to_blkg(bfqq_group(bfqq)), __pbuf, sizeof(__pbuf)); \ ++ blk_add_trace_msg((bfqd)->queue, "bfq%d%c %s " fmt, \ ++ (bfqq)->pid, \ ++ bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ ++ __pbuf, ##args); \ ++} while (0) ++ ++#define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do { \ ++ char __pbuf[128]; \ ++ \ ++ blkg_path(bfqg_to_blkg(bfqg), __pbuf, sizeof(__pbuf)); \ ++ blk_add_trace_msg((bfqd)->queue, "%s " fmt, __pbuf, ##args); \ ++} while (0) ++ ++#else /* CONFIG_BFQ_GROUP_IOSCHED */ ++ ++#define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \ ++ blk_add_trace_msg((bfqd)->queue, "bfq%d%c " fmt, (bfqq)->pid, \ ++ bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ ++ ##args) ++#define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do {} while (0) ++ ++#endif /* CONFIG_BFQ_GROUP_IOSCHED */ ++ ++#define bfq_log(bfqd, fmt, args...) \ ++ blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args) ++ ++/* Expiration reasons. */ ++enum bfqq_expiration { ++ BFQ_BFQQ_TOO_IDLE = 0, /* ++ * queue has been idling for ++ * too long ++ */ ++ BFQ_BFQQ_BUDGET_TIMEOUT, /* budget took too long to be used */ ++ BFQ_BFQQ_BUDGET_EXHAUSTED, /* budget consumed */ ++ BFQ_BFQQ_NO_MORE_REQUESTS, /* the queue has no more requests */ ++ BFQ_BFQQ_PREEMPTED /* preemption in progress */ ++}; ++ ++ ++struct bfqg_stats { ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ /* number of ios merged */ ++ struct blkg_rwstat merged; ++ /* total time spent on device in ns, may not be accurate w/ queueing */ ++ struct blkg_rwstat service_time; ++ /* total time spent waiting in scheduler queue in ns */ ++ struct blkg_rwstat wait_time; ++ /* number of IOs queued up */ ++ struct blkg_rwstat queued; ++ /* total disk time and nr sectors dispatched by this group */ ++ struct blkg_stat time; ++ /* sum of number of ios queued across all samples */ ++ struct blkg_stat avg_queue_size_sum; ++ /* count of samples taken for average */ ++ struct blkg_stat avg_queue_size_samples; ++ /* how many times this group has been removed from service tree */ ++ struct blkg_stat dequeue; ++ /* total time spent waiting for it to be assigned a timeslice. */ ++ struct blkg_stat group_wait_time; ++ /* time spent idling for this blkcg_gq */ ++ struct blkg_stat idle_time; ++ /* total time with empty current active q with other requests queued */ ++ struct blkg_stat empty_time; ++ /* fields after this shouldn't be cleared on stat reset */ ++ uint64_t start_group_wait_time; ++ uint64_t start_idle_time; ++ uint64_t start_empty_time; ++ uint16_t flags; ++#endif ++}; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++/* ++ * struct bfq_group_data - per-blkcg storage for the blkio subsystem. ++ * ++ * @ps: @blkcg_policy_storage that this structure inherits ++ * @weight: weight of the bfq_group ++ */ ++struct bfq_group_data { ++ /* must be the first member */ ++ struct blkcg_policy_data pd; ++ ++ unsigned int weight; ++}; ++ ++/** ++ * struct bfq_group - per (device, cgroup) data structure. ++ * @entity: schedulable entity to insert into the parent group sched_data. ++ * @sched_data: own sched_data, to contain child entities (they may be ++ * both bfq_queues and bfq_groups). ++ * @bfqd: the bfq_data for the device this group acts upon. ++ * @async_bfqq: array of async queues for all the tasks belonging to ++ * the group, one queue per ioprio value per ioprio_class, ++ * except for the idle class that has only one queue. ++ * @async_idle_bfqq: async queue for the idle class (ioprio is ignored). ++ * @my_entity: pointer to @entity, %NULL for the toplevel group; used ++ * to avoid too many special cases during group creation/ ++ * migration. ++ * @active_entities: number of active entities belonging to the group; ++ * unused for the root group. Used to know whether there ++ * are groups with more than one active @bfq_entity ++ * (see the comments to the function ++ * bfq_bfqq_may_idle()). ++ * @rq_pos_tree: rbtree sorted by next_request position, used when ++ * determining if two or more queues have interleaving ++ * requests (see bfq_find_close_cooperator()). ++ * ++ * Each (device, cgroup) pair has its own bfq_group, i.e., for each cgroup ++ * there is a set of bfq_groups, each one collecting the lower-level ++ * entities belonging to the group that are acting on the same device. ++ * ++ * Locking works as follows: ++ * o @bfqd is protected by the queue lock, RCU is used to access it ++ * from the readers. ++ * o All the other fields are protected by the @bfqd queue lock. ++ */ ++struct bfq_group { ++ /* must be the first member */ ++ struct blkg_policy_data pd; ++ ++ struct bfq_entity entity; ++ struct bfq_sched_data sched_data; ++ ++ void *bfqd; ++ ++ struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR]; ++ struct bfq_queue *async_idle_bfqq; ++ ++ struct bfq_entity *my_entity; ++ ++ int active_entities; ++ ++ struct rb_root rq_pos_tree; ++ ++ struct bfqg_stats stats; ++}; ++ ++#else ++struct bfq_group { ++ struct bfq_sched_data sched_data; ++ ++ struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR]; ++ struct bfq_queue *async_idle_bfqq; ++ ++ struct rb_root rq_pos_tree; ++}; ++#endif ++ ++static struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity); ++ ++static struct bfq_service_tree * ++bfq_entity_service_tree(struct bfq_entity *entity) ++{ ++ struct bfq_sched_data *sched_data = entity->sched_data; ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ unsigned int idx = bfqq ? bfqq->ioprio_class - 1 : ++ BFQ_DEFAULT_GRP_CLASS - 1; ++ ++ BUG_ON(idx >= BFQ_IOPRIO_CLASSES); ++ BUG_ON(sched_data == NULL); ++ ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "entity_service_tree %p %d", ++ sched_data->service_tree + idx, idx); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "entity_service_tree %p %d", ++ sched_data->service_tree + idx, idx); ++ } ++#endif ++ return sched_data->service_tree + idx; ++} ++ ++static struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync) ++{ ++ return bic->bfqq[is_sync]; ++} ++ ++static void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, ++ bool is_sync) ++{ ++ bic->bfqq[is_sync] = bfqq; ++} ++ ++static struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic) ++{ ++ return bic->icq.q->elevator->elevator_data; ++} ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ ++static struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *group_entity = bfqq->entity.parent; ++ ++ if (!group_entity) ++ group_entity = &bfqq->bfqd->root_group->entity; ++ ++ return container_of(group_entity, struct bfq_group, entity); ++} ++ ++#else ++ ++static struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq) ++{ ++ return bfqq->bfqd->root_group; ++} ++ ++#endif ++ ++static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio); ++static void bfq_put_queue(struct bfq_queue *bfqq); ++static void bfq_dispatch_insert(struct request_queue *q, struct request *rq); ++static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, ++ struct bio *bio, bool is_sync, ++ struct bfq_io_cq *bic); ++static void bfq_end_wr_async_queues(struct bfq_data *bfqd, ++ struct bfq_group *bfqg); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); ++#endif ++static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); ++ ++#endif /* _BFQ_H */ +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq-ioc.c linux-4.8-bfq/block/bfq-ioc.c +--- linux-4.8/block/bfq-ioc.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq-ioc.c 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,36 @@ ++/* ++ * BFQ: I/O context handling. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2010 Paolo Valente ++ */ ++ ++/** ++ * icq_to_bic - convert iocontext queue structure to bfq_io_cq. ++ * @icq: the iocontext queue. ++ */ ++static struct bfq_io_cq *icq_to_bic(struct io_cq *icq) ++{ ++ /* bic->icq is the first member, %NULL will convert to %NULL */ ++ return container_of(icq, struct bfq_io_cq, icq); ++} ++ ++/** ++ * bfq_bic_lookup - search into @ioc a bic associated to @bfqd. ++ * @bfqd: the lookup key. ++ * @ioc: the io_context of the process doing I/O. ++ * ++ * Queue lock must be held. ++ */ ++static struct bfq_io_cq *bfq_bic_lookup(struct bfq_data *bfqd, ++ struct io_context *ioc) ++{ ++ if (ioc) ++ return icq_to_bic(ioc_lookup_icq(ioc, bfqd->queue)); ++ return NULL; ++} +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq-iosched.c linux-4.8-bfq/block/bfq-iosched.c +--- linux-4.8/block/bfq-iosched.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq-iosched.c 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,5288 @@ ++/* ++ * Budget Fair Queueing (BFQ) disk scheduler. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2015 Paolo Valente ++ * ++ * Copyright (C) 2016 Paolo Valente ++ * ++ * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ ++ * file. ++ * ++ * BFQ is a proportional-share storage-I/O scheduling algorithm based ++ * on the slice-by-slice service scheme of CFQ. But BFQ assigns ++ * budgets, measured in number of sectors, to processes instead of ++ * time slices. The device is not granted to the in-service process ++ * for a given time slice, but until it has exhausted its assigned ++ * budget. This change from the time to the service domain enables BFQ ++ * to distribute the device throughput among processes as desired, ++ * without any distortion due to throughput fluctuations, or to device ++ * internal queueing. BFQ uses an ad hoc internal scheduler, called ++ * B-WF2Q+, to schedule processes according to their budgets. More ++ * precisely, BFQ schedules queues associated with processes. Thanks to ++ * the accurate policy of B-WF2Q+, BFQ can afford to assign high ++ * budgets to I/O-bound processes issuing sequential requests (to ++ * boost the throughput), and yet guarantee a low latency to ++ * interactive and soft real-time applications. ++ * ++ * BFQ is described in [1], where also a reference to the initial, more ++ * theoretical paper on BFQ can be found. The interested reader can find ++ * in the latter paper full details on the main algorithm, as well as ++ * formulas of the guarantees and formal proofs of all the properties. ++ * With respect to the version of BFQ presented in these papers, this ++ * implementation adds a few more heuristics, such as the one that ++ * guarantees a low latency to soft real-time applications, and a ++ * hierarchical extension based on H-WF2Q+. ++ * ++ * B-WF2Q+ is based on WF2Q+, that is described in [2], together with ++ * H-WF2Q+, while the augmented tree used to implement B-WF2Q+ with O(log N) ++ * complexity derives from the one introduced with EEVDF in [3]. ++ * ++ * [1] P. Valente and M. Andreolini, ``Improving Application Responsiveness ++ * with the BFQ Disk I/O Scheduler'', ++ * Proceedings of the 5th Annual International Systems and Storage ++ * Conference (SYSTOR '12), June 2012. ++ * ++ * http://algogroup.unimo.it/people/paolo/disk_sched/bf1-v1-suite-results.pdf ++ * ++ * [2] Jon C.R. Bennett and H. Zhang, ``Hierarchical Packet Fair Queueing ++ * Algorithms,'' IEEE/ACM Transactions on Networking, 5(5):675-689, ++ * Oct 1997. ++ * ++ * http://www.cs.cmu.edu/~hzhang/papers/TON-97-Oct.ps.gz ++ * ++ * [3] I. Stoica and H. Abdel-Wahab, ``Earliest Eligible Virtual Deadline ++ * First: A Flexible and Accurate Mechanism for Proportional Share ++ * Resource Allocation,'' technical report. ++ * ++ * http://www.cs.berkeley.edu/~istoica/papers/eevdf-tr-95.pdf ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "bfq.h" ++#include "blk.h" ++ ++/* Expiration time of sync (0) and async (1) requests, in ns. */ ++static const u64 bfq_fifo_expire[2] = { NSEC_PER_SEC / 4, NSEC_PER_SEC / 8 }; ++ ++/* Maximum backwards seek, in KiB. */ ++static const int bfq_back_max = 16 * 1024; ++ ++/* Penalty of a backwards seek, in number of sectors. */ ++static const int bfq_back_penalty = 2; ++ ++/* Idling period duration, in ns. */ ++static u32 bfq_slice_idle = NSEC_PER_SEC / 125; ++ ++/* Minimum number of assigned budgets for which stats are safe to compute. */ ++static const int bfq_stats_min_budgets = 194; ++ ++/* Default maximum budget values, in sectors and number of requests. */ ++static const int bfq_default_max_budget = 16 * 1024; ++ ++/* ++ * Async to sync throughput distribution is controlled as follows: ++ * when an async request is served, the entity is charged the number ++ * of sectors of the request, multiplied by the factor below ++ */ ++static const int bfq_async_charge_factor = 10; ++ ++/* Default timeout values, in jiffies, approximating CFQ defaults. */ ++static const int bfq_timeout = HZ / 8; ++ ++struct kmem_cache *bfq_pool; ++ ++/* Below this threshold (in ns), we consider thinktime immediate. */ ++#define BFQ_MIN_TT (2 * NSEC_PER_MSEC) ++ ++/* hw_tag detection: parallel requests threshold and min samples needed. */ ++#define BFQ_HW_QUEUE_THRESHOLD 4 ++#define BFQ_HW_QUEUE_SAMPLES 32 ++ ++#define BFQQ_SEEK_THR (sector_t)(8 * 100) ++#define BFQQ_CLOSE_THR (sector_t)(8 * 1024) ++#define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 32/8) ++ ++/* Min number of samples required to perform peak-rate update */ ++#define BFQ_RATE_MIN_SAMPLES 32 ++/* Min observation time interval required to perform a peak-rate update (ns) */ ++#define BFQ_RATE_MIN_INTERVAL 300*NSEC_PER_MSEC ++/* Target observation time interval for a peak-rate update (ns) */ ++#define BFQ_RATE_REF_INTERVAL NSEC_PER_SEC ++ ++/* Shift used for peak rate fixed precision calculations. */ ++#define BFQ_RATE_SHIFT 16 ++ ++/* ++ * By default, BFQ computes the duration of the weight raising for ++ * interactive applications automatically, using the following formula: ++ * duration = (R / r) * T, where r is the peak rate of the device, and ++ * R and T are two reference parameters. ++ * In particular, R is the peak rate of the reference device (see below), ++ * and T is a reference time: given the systems that are likely to be ++ * installed on the reference device according to its speed class, T is ++ * about the maximum time needed, under BFQ and while reading two files in ++ * parallel, to load typical large applications on these systems. ++ * In practice, the slower/faster the device at hand is, the more/less it ++ * takes to load applications with respect to the reference device. ++ * Accordingly, the longer/shorter BFQ grants weight raising to interactive ++ * applications. ++ * ++ * BFQ uses four different reference pairs (R, T), depending on: ++ * . whether the device is rotational or non-rotational; ++ * . whether the device is slow, such as old or portable HDDs, as well as ++ * SD cards, or fast, such as newer HDDs and SSDs. ++ * ++ * The device's speed class is dynamically (re)detected in ++ * bfq_update_peak_rate() every time the estimated peak rate is updated. ++ * ++ * In the following definitions, R_slow[0]/R_fast[0] and ++ * T_slow[0]/T_fast[0] are the reference values for a slow/fast ++ * rotational device, whereas R_slow[1]/R_fast[1] and ++ * T_slow[1]/T_fast[1] are the reference values for a slow/fast ++ * non-rotational device. Finally, device_speed_thresh are the ++ * thresholds used to switch between speed classes. The reference ++ * rates are not the actual peak rates of the devices used as a ++ * reference, but slightly lower values. The reason for using these ++ * slightly lower values is that the peak-rate estimator tends to ++ * yield slightly lower values than the actual peak rate (it can yield ++ * the actual peak rate only if there is only one process doing I/O, ++ * and the process does sequential I/O). ++ * ++ * Both the reference peak rates and the thresholds are measured in ++ * sectors/usec, left-shifted by BFQ_RATE_SHIFT. ++ */ ++static int R_slow[2] = {1000, 10700}; ++static int R_fast[2] = {14000, 33000}; ++/* ++ * To improve readability, a conversion function is used to initialize the ++ * following arrays, which entails that they can be initialized only in a ++ * function. ++ */ ++static int T_slow[2]; ++static int T_fast[2]; ++static int device_speed_thresh[2]; ++ ++#define BFQ_SERVICE_TREE_INIT ((struct bfq_service_tree) \ ++ { RB_ROOT, RB_ROOT, NULL, NULL, 0, 0 }) ++ ++#define RQ_BIC(rq) ((struct bfq_io_cq *) (rq)->elv.priv[0]) ++#define RQ_BFQQ(rq) ((rq)->elv.priv[1]) ++ ++static void bfq_schedule_dispatch(struct bfq_data *bfqd); ++ ++#include "bfq-ioc.c" ++#include "bfq-sched.c" ++#include "bfq-cgroup.c" ++ ++#define bfq_class_idle(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_IDLE) ++#define bfq_class_rt(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_RT) ++ ++#define bfq_sample_valid(samples) ((samples) > 80) ++ ++/* ++ * We regard a request as SYNC, if either it's a read or has the SYNC bit ++ * set (in which case it could also be a direct WRITE). ++ */ ++static int bfq_bio_sync(struct bio *bio) ++{ ++ return bio_data_dir(bio) == READ || (bio->bi_opf & REQ_SYNC); ++} ++ ++/* ++ * Scheduler run of queue, if there are requests pending and no one in the ++ * driver that will restart queueing. ++ */ ++static void bfq_schedule_dispatch(struct bfq_data *bfqd) ++{ ++ if (bfqd->queued != 0) { ++ bfq_log(bfqd, "schedule dispatch"); ++ kblockd_schedule_work(&bfqd->unplug_work); ++ } ++} ++ ++/* ++ * Lifted from AS - choose which of rq1 and rq2 that is best served now. ++ * We choose the request that is closesr to the head right now. Distance ++ * behind the head is penalized and only allowed to a certain extent. ++ */ ++static struct request *bfq_choose_req(struct bfq_data *bfqd, ++ struct request *rq1, ++ struct request *rq2, ++ sector_t last) ++{ ++ sector_t s1, s2, d1 = 0, d2 = 0; ++ unsigned long back_max; ++#define BFQ_RQ1_WRAP 0x01 /* request 1 wraps */ ++#define BFQ_RQ2_WRAP 0x02 /* request 2 wraps */ ++ unsigned int wrap = 0; /* bit mask: requests behind the disk head? */ ++ ++ if (!rq1 || rq1 == rq2) ++ return rq2; ++ if (!rq2) ++ return rq1; ++ ++ if (rq_is_sync(rq1) && !rq_is_sync(rq2)) ++ return rq1; ++ else if (rq_is_sync(rq2) && !rq_is_sync(rq1)) ++ return rq2; ++ if ((rq1->cmd_flags & REQ_META) && !(rq2->cmd_flags & REQ_META)) ++ return rq1; ++ else if ((rq2->cmd_flags & REQ_META) && !(rq1->cmd_flags & REQ_META)) ++ return rq2; ++ ++ s1 = blk_rq_pos(rq1); ++ s2 = blk_rq_pos(rq2); ++ ++ /* ++ * By definition, 1KiB is 2 sectors. ++ */ ++ back_max = bfqd->bfq_back_max * 2; ++ ++ /* ++ * Strict one way elevator _except_ in the case where we allow ++ * short backward seeks which are biased as twice the cost of a ++ * similar forward seek. ++ */ ++ if (s1 >= last) ++ d1 = s1 - last; ++ else if (s1 + back_max >= last) ++ d1 = (last - s1) * bfqd->bfq_back_penalty; ++ else ++ wrap |= BFQ_RQ1_WRAP; ++ ++ if (s2 >= last) ++ d2 = s2 - last; ++ else if (s2 + back_max >= last) ++ d2 = (last - s2) * bfqd->bfq_back_penalty; ++ else ++ wrap |= BFQ_RQ2_WRAP; ++ ++ /* Found required data */ ++ ++ /* ++ * By doing switch() on the bit mask "wrap" we avoid having to ++ * check two variables for all permutations: --> faster! ++ */ ++ switch (wrap) { ++ case 0: /* common case for CFQ: rq1 and rq2 not wrapped */ ++ if (d1 < d2) ++ return rq1; ++ else if (d2 < d1) ++ return rq2; ++ ++ if (s1 >= s2) ++ return rq1; ++ else ++ return rq2; ++ ++ case BFQ_RQ2_WRAP: ++ return rq1; ++ case BFQ_RQ1_WRAP: ++ return rq2; ++ case (BFQ_RQ1_WRAP|BFQ_RQ2_WRAP): /* both rqs wrapped */ ++ default: ++ /* ++ * Since both rqs are wrapped, ++ * start with the one that's further behind head ++ * (--> only *one* back seek required), ++ * since back seek takes more time than forward. ++ */ ++ if (s1 <= s2) ++ return rq1; ++ else ++ return rq2; ++ } ++} ++ ++static struct bfq_queue * ++bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, ++ sector_t sector, struct rb_node **ret_parent, ++ struct rb_node ***rb_link) ++{ ++ struct rb_node **p, *parent; ++ struct bfq_queue *bfqq = NULL; ++ ++ parent = NULL; ++ p = &root->rb_node; ++ while (*p) { ++ struct rb_node **n; ++ ++ parent = *p; ++ bfqq = rb_entry(parent, struct bfq_queue, pos_node); ++ ++ /* ++ * Sort strictly based on sector. Smallest to the left, ++ * largest to the right. ++ */ ++ if (sector > blk_rq_pos(bfqq->next_rq)) ++ n = &(*p)->rb_right; ++ else if (sector < blk_rq_pos(bfqq->next_rq)) ++ n = &(*p)->rb_left; ++ else ++ break; ++ p = n; ++ bfqq = NULL; ++ } ++ ++ *ret_parent = parent; ++ if (rb_link) ++ *rb_link = p; ++ ++ bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d", ++ (unsigned long long) sector, ++ bfqq ? bfqq->pid : 0); ++ ++ return bfqq; ++} ++ ++static void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ struct rb_node **p, *parent; ++ struct bfq_queue *__bfqq; ++ ++ if (bfqq->pos_root) { ++ rb_erase(&bfqq->pos_node, bfqq->pos_root); ++ bfqq->pos_root = NULL; ++ } ++ ++ if (bfq_class_idle(bfqq)) ++ return; ++ if (!bfqq->next_rq) ++ return; ++ ++ bfqq->pos_root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; ++ __bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root, ++ blk_rq_pos(bfqq->next_rq), &parent, &p); ++ if (!__bfqq) { ++ rb_link_node(&bfqq->pos_node, parent, p); ++ rb_insert_color(&bfqq->pos_node, bfqq->pos_root); ++ } else ++ bfqq->pos_root = NULL; ++} ++ ++/* ++ * Tell whether there are active queues or groups with differentiated weights. ++ */ ++static bool bfq_differentiated_weights(struct bfq_data *bfqd) ++{ ++ /* ++ * For weights to differ, at least one of the trees must contain ++ * at least two nodes. ++ */ ++ return (!RB_EMPTY_ROOT(&bfqd->queue_weights_tree) && ++ (bfqd->queue_weights_tree.rb_node->rb_left || ++ bfqd->queue_weights_tree.rb_node->rb_right) ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ ) || ++ (!RB_EMPTY_ROOT(&bfqd->group_weights_tree) && ++ (bfqd->group_weights_tree.rb_node->rb_left || ++ bfqd->group_weights_tree.rb_node->rb_right) ++#endif ++ ); ++} ++ ++/* ++ * The following function returns true if every queue must receive the ++ * same share of the throughput (this condition is used when deciding ++ * whether idling may be disabled, see the comments in the function ++ * bfq_bfqq_may_idle()). ++ * ++ * Such a scenario occurs when: ++ * 1) all active queues have the same weight, ++ * 2) all active groups at the same level in the groups tree have the same ++ * weight, ++ * 3) all active groups at the same level in the groups tree have the same ++ * number of children. ++ * ++ * Unfortunately, keeping the necessary state for evaluating exactly the ++ * above symmetry conditions would be quite complex and time-consuming. ++ * Therefore this function evaluates, instead, the following stronger ++ * sub-conditions, for which it is much easier to maintain the needed ++ * state: ++ * 1) all active queues have the same weight, ++ * 2) all active groups have the same weight, ++ * 3) all active groups have at most one active child each. ++ * In particular, the last two conditions are always true if hierarchical ++ * support and the cgroups interface are not enabled, thus no state needs ++ * to be maintained in this case. ++ */ ++static bool bfq_symmetric_scenario(struct bfq_data *bfqd) ++{ ++ return !bfq_differentiated_weights(bfqd); ++} ++ ++/* ++ * If the weight-counter tree passed as input contains no counter for ++ * the weight of the input entity, then add that counter; otherwise just ++ * increment the existing counter. ++ * ++ * Note that weight-counter trees contain few nodes in mostly symmetric ++ * scenarios. For example, if all queues have the same weight, then the ++ * weight-counter tree for the queues may contain at most one node. ++ * This holds even if low_latency is on, because weight-raised queues ++ * are not inserted in the tree. ++ * In most scenarios, the rate at which nodes are created/destroyed ++ * should be low too. ++ */ ++static void bfq_weights_tree_add(struct bfq_data *bfqd, ++ struct bfq_entity *entity, ++ struct rb_root *root) ++{ ++ struct rb_node **new = &(root->rb_node), *parent = NULL; ++ ++ /* ++ * Do not insert if the entity is already associated with a ++ * counter, which happens if: ++ * 1) the entity is associated with a queue, ++ * 2) a request arrival has caused the queue to become both ++ * non-weight-raised, and hence change its weight, and ++ * backlogged; in this respect, each of the two events ++ * causes an invocation of this function, ++ * 3) this is the invocation of this function caused by the ++ * second event. This second invocation is actually useless, ++ * and we handle this fact by exiting immediately. More ++ * efficient or clearer solutions might possibly be adopted. ++ */ ++ if (entity->weight_counter) ++ return; ++ ++ while (*new) { ++ struct bfq_weight_counter *__counter = container_of(*new, ++ struct bfq_weight_counter, ++ weights_node); ++ parent = *new; ++ ++ if (entity->weight == __counter->weight) { ++ entity->weight_counter = __counter; ++ goto inc_counter; ++ } ++ if (entity->weight < __counter->weight) ++ new = &((*new)->rb_left); ++ else ++ new = &((*new)->rb_right); ++ } ++ ++ entity->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), ++ GFP_ATOMIC); ++ entity->weight_counter->weight = entity->weight; ++ rb_link_node(&entity->weight_counter->weights_node, parent, new); ++ rb_insert_color(&entity->weight_counter->weights_node, root); ++ ++inc_counter: ++ entity->weight_counter->num_active++; ++} ++ ++/* ++ * Decrement the weight counter associated with the entity, and, if the ++ * counter reaches 0, remove the counter from the tree. ++ * See the comments to the function bfq_weights_tree_add() for considerations ++ * about overhead. ++ */ ++static void bfq_weights_tree_remove(struct bfq_data *bfqd, ++ struct bfq_entity *entity, ++ struct rb_root *root) ++{ ++ if (!entity->weight_counter) ++ return; ++ ++ BUG_ON(RB_EMPTY_ROOT(root)); ++ BUG_ON(entity->weight_counter->weight != entity->weight); ++ ++ BUG_ON(!entity->weight_counter->num_active); ++ entity->weight_counter->num_active--; ++ if (entity->weight_counter->num_active > 0) ++ goto reset_entity_pointer; ++ ++ rb_erase(&entity->weight_counter->weights_node, root); ++ kfree(entity->weight_counter); ++ ++reset_entity_pointer: ++ entity->weight_counter = NULL; ++} ++ ++static struct request *bfq_find_next_rq(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ struct request *last) ++{ ++ struct rb_node *rbnext = rb_next(&last->rb_node); ++ struct rb_node *rbprev = rb_prev(&last->rb_node); ++ struct request *next = NULL, *prev = NULL; ++ ++ BUG_ON(RB_EMPTY_NODE(&last->rb_node)); ++ ++ if (rbprev) ++ prev = rb_entry_rq(rbprev); ++ ++ if (rbnext) ++ next = rb_entry_rq(rbnext); ++ else { ++ rbnext = rb_first(&bfqq->sort_list); ++ if (rbnext && rbnext != &last->rb_node) ++ next = rb_entry_rq(rbnext); ++ } ++ ++ return bfq_choose_req(bfqd, next, prev, blk_rq_pos(last)); ++} ++ ++/* see the definition of bfq_async_charge_factor for details */ ++static unsigned long bfq_serv_to_charge(struct request *rq, ++ struct bfq_queue *bfqq) ++{ ++ if (bfq_bfqq_sync(bfqq) || bfqq->wr_coeff > 1) ++ return blk_rq_sectors(rq); ++ ++ /* ++ * If there are no weight-raised queues, then amplify service ++ * by just the async charge factor; otherwise amplify service ++ * by twice the async charge factor, to further reduce latency ++ * for weight-raised queues. ++ */ ++ if (bfqq->bfqd->wr_busy_queues == 0) ++ return blk_rq_sectors(rq) * bfq_async_charge_factor; ++ ++ return blk_rq_sectors(rq) * 2 * bfq_async_charge_factor; ++} ++ ++/** ++ * bfq_updated_next_req - update the queue after a new next_rq selection. ++ * @bfqd: the device data the queue belongs to. ++ * @bfqq: the queue to update. ++ * ++ * If the first request of a queue changes we make sure that the queue ++ * has enough budget to serve at least its first request (if the ++ * request has grown). We do this because if the queue has not enough ++ * budget for its first request, it has to go through two dispatch ++ * rounds to actually get it dispatched. ++ */ ++static void bfq_updated_next_req(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ struct bfq_service_tree *st = bfq_entity_service_tree(entity); ++ struct request *next_rq = bfqq->next_rq; ++ unsigned long new_budget; ++ ++ if (!next_rq) ++ return; ++ ++ if (bfqq == bfqd->in_service_queue) ++ /* ++ * In order not to break guarantees, budgets cannot be ++ * changed after an entity has been selected. ++ */ ++ return; ++ ++ BUG_ON(entity->tree != &st->active); ++ BUG_ON(entity == entity->sched_data->in_service_entity); ++ ++ new_budget = max_t(unsigned long, bfqq->max_budget, ++ bfq_serv_to_charge(next_rq, bfqq)); ++ if (entity->budget != new_budget) { ++ entity->budget = new_budget; ++ bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", ++ new_budget); ++ bfq_activate_bfqq(bfqd, bfqq); ++ } ++} ++ ++static unsigned int bfq_wr_duration(struct bfq_data *bfqd) ++{ ++ u64 dur; ++ ++ if (bfqd->bfq_wr_max_time > 0) ++ return bfqd->bfq_wr_max_time; ++ ++ dur = bfqd->RT_prod; ++ do_div(dur, bfqd->peak_rate); ++ ++ /* ++ * Limit duration between 3 and 13 seconds. Tests show that ++ * higher values than 13 seconds often yield the opposite of ++ * the desired result, i.e., worsen responsiveness by letting ++ * non-interactive and non-soft-real-time applications ++ * preserve weight raising for a too long time interval. ++ * ++ * On the other end, lower values than 3 seconds make it ++ * difficult for most interactive tasks to complete their jobs ++ * before weight-raising finishes. ++ */ ++ if (dur > msecs_to_jiffies(13000)) ++ dur = msecs_to_jiffies(13000); ++ else if (dur < msecs_to_jiffies(3000)) ++ dur = msecs_to_jiffies(3000); ++ ++ return dur; ++} ++ ++static void ++bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_io_cq *bic) ++{ ++ if (bic->saved_idle_window) ++ bfq_mark_bfqq_idle_window(bfqq); ++ else ++ bfq_clear_bfqq_idle_window(bfqq); ++ ++ if (bic->saved_IO_bound) ++ bfq_mark_bfqq_IO_bound(bfqq); ++ else ++ bfq_clear_bfqq_IO_bound(bfqq); ++ ++ bfqq->wr_coeff = bic->saved_wr_coeff; ++ bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt; ++ BUG_ON(time_is_after_jiffies(bfqq->wr_start_at_switch_to_srt)); ++ bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish; ++ BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish)); ++ ++ if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) || ++ time_is_before_jiffies(bfqq->last_wr_start_finish + ++ bfqq->wr_cur_max_time))) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "resume state: switching off wr"); ++ ++ bfqq->wr_coeff = 1; ++ } ++ /* make sure weight will be updated, however we got here */ ++ bfqq->entity.prio_changed = 1; ++} ++ ++static int bfqq_process_refs(struct bfq_queue *bfqq) ++{ ++ int process_refs, io_refs; ++ ++ lockdep_assert_held(bfqq->bfqd->queue->queue_lock); ++ ++ io_refs = bfqq->allocated[READ] + bfqq->allocated[WRITE]; ++ process_refs = bfqq->ref - io_refs - bfqq->entity.on_st; ++ BUG_ON(process_refs < 0); ++ return process_refs; ++} ++ ++/* Empty burst list and add just bfqq (see comments to bfq_handle_burst) */ ++static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ struct bfq_queue *item; ++ struct hlist_node *n; ++ ++ hlist_for_each_entry_safe(item, n, &bfqd->burst_list, burst_list_node) ++ hlist_del_init(&item->burst_list_node); ++ hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); ++ bfqd->burst_size = 1; ++ bfqd->burst_parent_entity = bfqq->entity.parent; ++} ++ ++/* Add bfqq to the list of queues in current burst (see bfq_handle_burst) */ ++static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ /* Increment burst size to take into account also bfqq */ ++ bfqd->burst_size++; ++ ++ bfq_log_bfqq(bfqd, bfqq, "add_to_burst %d", bfqd->burst_size); ++ ++ BUG_ON(bfqd->burst_size > bfqd->bfq_large_burst_thresh); ++ ++ if (bfqd->burst_size == bfqd->bfq_large_burst_thresh) { ++ struct bfq_queue *pos, *bfqq_item; ++ struct hlist_node *n; ++ ++ /* ++ * Enough queues have been activated shortly after each ++ * other to consider this burst as large. ++ */ ++ bfqd->large_burst = true; ++ bfq_log_bfqq(bfqd, bfqq, "add_to_burst: large burst started"); ++ ++ /* ++ * We can now mark all queues in the burst list as ++ * belonging to a large burst. ++ */ ++ hlist_for_each_entry(bfqq_item, &bfqd->burst_list, ++ burst_list_node) { ++ bfq_mark_bfqq_in_large_burst(bfqq_item); ++ bfq_log_bfqq(bfqd, bfqq_item, "marked in large burst"); ++ } ++ bfq_mark_bfqq_in_large_burst(bfqq); ++ bfq_log_bfqq(bfqd, bfqq, "marked in large burst"); ++ ++ /* ++ * From now on, and until the current burst finishes, any ++ * new queue being activated shortly after the last queue ++ * was inserted in the burst can be immediately marked as ++ * belonging to a large burst. So the burst list is not ++ * needed any more. Remove it. ++ */ ++ hlist_for_each_entry_safe(pos, n, &bfqd->burst_list, ++ burst_list_node) ++ hlist_del_init(&pos->burst_list_node); ++ } else /* ++ * Burst not yet large: add bfqq to the burst list. Do ++ * not increment the ref counter for bfqq, because bfqq ++ * is removed from the burst list before freeing bfqq ++ * in put_queue. ++ */ ++ hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); ++} ++ ++/* ++ * If many queues belonging to the same group happen to be created ++ * shortly after each other, then the processes associated with these ++ * queues have typically a common goal. In particular, bursts of queue ++ * creations are usually caused by services or applications that spawn ++ * many parallel threads/processes. Examples are systemd during boot, ++ * or git grep. To help these processes get their job done as soon as ++ * possible, it is usually better to not grant either weight-raising ++ * or device idling to their queues. ++ * ++ * In this comment we describe, firstly, the reasons why this fact ++ * holds, and, secondly, the next function, which implements the main ++ * steps needed to properly mark these queues so that they can then be ++ * treated in a different way. ++ * ++ * The above services or applications benefit mostly from a high ++ * throughput: the quicker the requests of the activated queues are ++ * cumulatively served, the sooner the target job of these queues gets ++ * completed. As a consequence, weight-raising any of these queues, ++ * which also implies idling the device for it, is almost always ++ * counterproductive. In most cases it just lowers throughput. ++ * ++ * On the other hand, a burst of queue creations may be caused also by ++ * the start of an application that does not consist of a lot of ++ * parallel I/O-bound threads. In fact, with a complex application, ++ * several short processes may need to be executed to start-up the ++ * application. In this respect, to start an application as quickly as ++ * possible, the best thing to do is in any case to privilege the I/O ++ * related to the application with respect to all other ++ * I/O. Therefore, the best strategy to start as quickly as possible ++ * an application that causes a burst of queue creations is to ++ * weight-raise all the queues created during the burst. This is the ++ * exact opposite of the best strategy for the other type of bursts. ++ * ++ * In the end, to take the best action for each of the two cases, the ++ * two types of bursts need to be distinguished. Fortunately, this ++ * seems relatively easy, by looking at the sizes of the bursts. In ++ * particular, we found a threshold such that only bursts with a ++ * larger size than that threshold are apparently caused by ++ * services or commands such as systemd or git grep. For brevity, ++ * hereafter we call just 'large' these bursts. BFQ *does not* ++ * weight-raise queues whose creation occurs in a large burst. In ++ * addition, for each of these queues BFQ performs or does not perform ++ * idling depending on which choice boosts the throughput more. The ++ * exact choice depends on the device and request pattern at ++ * hand. ++ * ++ * Unfortunately, false positives may occur while an interactive task ++ * is starting (e.g., an application is being started). The ++ * consequence is that the queues associated with the task do not ++ * enjoy weight raising as expected. Fortunately these false positives ++ * are very rare. They typically occur if some service happens to ++ * start doing I/O exactly when the interactive task starts. ++ * ++ * Turning back to the next function, it implements all the steps ++ * needed to detect the occurrence of a large burst and to properly ++ * mark all the queues belonging to it (so that they can then be ++ * treated in a different way). This goal is achieved by maintaining a ++ * "burst list" that holds, temporarily, the queues that belong to the ++ * burst in progress. The list is then used to mark these queues as ++ * belonging to a large burst if the burst does become large. The main ++ * steps are the following. ++ * ++ * . when the very first queue is created, the queue is inserted into the ++ * list (as it could be the first queue in a possible burst) ++ * ++ * . if the current burst has not yet become large, and a queue Q that does ++ * not yet belong to the burst is activated shortly after the last time ++ * at which a new queue entered the burst list, then the function appends ++ * Q to the burst list ++ * ++ * . if, as a consequence of the previous step, the burst size reaches ++ * the large-burst threshold, then ++ * ++ * . all the queues in the burst list are marked as belonging to a ++ * large burst ++ * ++ * . the burst list is deleted; in fact, the burst list already served ++ * its purpose (keeping temporarily track of the queues in a burst, ++ * so as to be able to mark them as belonging to a large burst in the ++ * previous sub-step), and now is not needed any more ++ * ++ * . the device enters a large-burst mode ++ * ++ * . if a queue Q that does not belong to the burst is created while ++ * the device is in large-burst mode and shortly after the last time ++ * at which a queue either entered the burst list or was marked as ++ * belonging to the current large burst, then Q is immediately marked ++ * as belonging to a large burst. ++ * ++ * . if a queue Q that does not belong to the burst is created a while ++ * later, i.e., not shortly after, than the last time at which a queue ++ * either entered the burst list or was marked as belonging to the ++ * current large burst, then the current burst is deemed as finished and: ++ * ++ * . the large-burst mode is reset if set ++ * ++ * . the burst list is emptied ++ * ++ * . Q is inserted in the burst list, as Q may be the first queue ++ * in a possible new burst (then the burst list contains just Q ++ * after this step). ++ */ ++static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ /* ++ * If bfqq is already in the burst list or is part of a large ++ * burst, or finally has just been split, then there is ++ * nothing else to do. ++ */ ++ if (!hlist_unhashed(&bfqq->burst_list_node) || ++ bfq_bfqq_in_large_burst(bfqq) || ++ time_is_after_eq_jiffies(bfqq->split_time + ++ msecs_to_jiffies(10))) ++ return; ++ ++ /* ++ * If bfqq's creation happens late enough, or bfqq belongs to ++ * a different group than the burst group, then the current ++ * burst is finished, and related data structures must be ++ * reset. ++ * ++ * In this respect, consider the special case where bfqq is ++ * the very first queue created after BFQ is selected for this ++ * device. In this case, last_ins_in_burst and ++ * burst_parent_entity are not yet significant when we get ++ * here. But it is easy to verify that, whether or not the ++ * following condition is true, bfqq will end up being ++ * inserted into the burst list. In particular the list will ++ * happen to contain only bfqq. And this is exactly what has ++ * to happen, as bfqq may be the first queue of the first ++ * burst. ++ */ ++ if (time_is_before_jiffies(bfqd->last_ins_in_burst + ++ bfqd->bfq_burst_interval) || ++ bfqq->entity.parent != bfqd->burst_parent_entity) { ++ bfqd->large_burst = false; ++ bfq_reset_burst_list(bfqd, bfqq); ++ bfq_log_bfqq(bfqd, bfqq, ++ "handle_burst: late activation or different group"); ++ goto end; ++ } ++ ++ /* ++ * If we get here, then bfqq is being activated shortly after the ++ * last queue. So, if the current burst is also large, we can mark ++ * bfqq as belonging to this large burst immediately. ++ */ ++ if (bfqd->large_burst) { ++ bfq_log_bfqq(bfqd, bfqq, "handle_burst: marked in burst"); ++ bfq_mark_bfqq_in_large_burst(bfqq); ++ goto end; ++ } ++ ++ /* ++ * If we get here, then a large-burst state has not yet been ++ * reached, but bfqq is being activated shortly after the last ++ * queue. Then we add bfqq to the burst. ++ */ ++ bfq_add_to_burst(bfqd, bfqq); ++end: ++ /* ++ * At this point, bfqq either has been added to the current ++ * burst or has caused the current burst to terminate and a ++ * possible new burst to start. In particular, in the second ++ * case, bfqq has become the first queue in the possible new ++ * burst. In both cases last_ins_in_burst needs to be moved ++ * forward. ++ */ ++ bfqd->last_ins_in_burst = jiffies; ++ ++} ++ ++static int bfq_bfqq_budget_left(struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ return entity->budget - entity->service; ++} ++ ++/* ++ * If enough samples have been computed, return the current max budget ++ * stored in bfqd, which is dynamically updated according to the ++ * estimated disk peak rate; otherwise return the default max budget ++ */ ++static int bfq_max_budget(struct bfq_data *bfqd) ++{ ++ if (bfqd->budgets_assigned < bfq_stats_min_budgets) ++ return bfq_default_max_budget; ++ else ++ return bfqd->bfq_max_budget; ++} ++ ++/* ++ * Return min budget, which is a fraction of the current or default ++ * max budget (trying with 1/32) ++ */ ++static int bfq_min_budget(struct bfq_data *bfqd) ++{ ++ if (bfqd->budgets_assigned < bfq_stats_min_budgets) ++ return bfq_default_max_budget / 32; ++ else ++ return bfqd->bfq_max_budget / 32; ++} ++ ++static void bfq_bfqq_expire(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ bool compensate, ++ enum bfqq_expiration reason); ++ ++/* ++ * The next function, invoked after the input queue bfqq switches from ++ * idle to busy, updates the budget of bfqq. The function also tells ++ * whether the in-service queue should be expired, by returning ++ * true. The purpose of expiring the in-service queue is to give bfqq ++ * the chance to possibly preempt the in-service queue, and the reason ++ * for preempting the in-service queue is to achieve one of the two ++ * goals below. ++ * ++ * 1. Guarantee to bfqq its reserved bandwidth even if bfqq has ++ * expired because it has remained idle. In particular, bfqq may have ++ * expired for one of the following two reasons: ++ * ++ * - BFQ_BFQQ_NO_MORE_REQUEST bfqq did not enjoy any device idling and ++ * did not make it to issue a new request before its last request ++ * was served; ++ * ++ * - BFQ_BFQQ_TOO_IDLE bfqq did enjoy device idling, but did not issue ++ * a new request before the expiration of the idling-time. ++ * ++ * Even if bfqq has expired for one of the above reasons, the process ++ * associated with the queue may be however issuing requests greedily, ++ * and thus be sensitive to the bandwidth it receives (bfqq may have ++ * remained idle for other reasons: CPU high load, bfqq not enjoying ++ * idling, I/O throttling somewhere in the path from the process to ++ * the I/O scheduler, ...). But if, after every expiration for one of ++ * the above two reasons, bfqq has to wait for the service of at least ++ * one full budget of another queue before being served again, then ++ * bfqq is likely to get a much lower bandwidth or resource time than ++ * its reserved ones. To address this issue, two countermeasures need ++ * to be taken. ++ * ++ * First, the budget and the timestamps of bfqq need to be updated in ++ * a special way on bfqq reactivation: they need to be updated as if ++ * bfqq did not remain idle and did not expire. In fact, if they are ++ * computed as if bfqq expired and remained idle until reactivation, ++ * then the process associated with bfqq is treated as if, instead of ++ * being greedy, it stopped issuing requests when bfqq remained idle, ++ * and restarts issuing requests only on this reactivation. In other ++ * words, the scheduler does not help the process recover the "service ++ * hole" between bfqq expiration and reactivation. As a consequence, ++ * the process receives a lower bandwidth than its reserved one. In ++ * contrast, to recover this hole, the budget must be updated as if ++ * bfqq was not expired at all before this reactivation, i.e., it must ++ * be set to the value of the remaining budget when bfqq was ++ * expired. Along the same line, timestamps need to be assigned the ++ * value they had the last time bfqq was selected for service, i.e., ++ * before last expiration. Thus timestamps need to be back-shifted ++ * with respect to their normal computation (see [1] for more details ++ * on this tricky aspect). ++ * ++ * Secondly, to allow the process to recover the hole, the in-service ++ * queue must be expired too, to give bfqq the chance to preempt it ++ * immediately. In fact, if bfqq has to wait for a full budget of the ++ * in-service queue to be completed, then it may become impossible to ++ * let the process recover the hole, even if the back-shifted ++ * timestamps of bfqq are lower than those of the in-service queue. If ++ * this happens for most or all of the holes, then the process may not ++ * receive its reserved bandwidth. In this respect, it is worth noting ++ * that, being the service of outstanding requests unpreemptible, a ++ * little fraction of the holes may however be unrecoverable, thereby ++ * causing a little loss of bandwidth. ++ * ++ * The last important point is detecting whether bfqq does need this ++ * bandwidth recovery. In this respect, the next function deems the ++ * process associated with bfqq greedy, and thus allows it to recover ++ * the hole, if: 1) the process is waiting for the arrival of a new ++ * request (which implies that bfqq expired for one of the above two ++ * reasons), and 2) such a request has arrived soon. The first ++ * condition is controlled through the flag non_blocking_wait_rq, ++ * while the second through the flag arrived_in_time. If both ++ * conditions hold, then the function computes the budget in the ++ * above-described special way, and signals that the in-service queue ++ * should be expired. Timestamp back-shifting is done later in ++ * __bfq_activate_entity. ++ * ++ * 2. Reduce latency. Even if timestamps are not backshifted to let ++ * the process associated with bfqq recover a service hole, bfqq may ++ * however happen to have, after being (re)activated, a lower finish ++ * timestamp than the in-service queue. That is, the next budget of ++ * bfqq may have to be completed before the one of the in-service ++ * queue. If this is the case, then preempting the in-service queue ++ * allows this goal to be achieved, apart from the unpreemptible, ++ * outstanding requests mentioned above. ++ * ++ * Unfortunately, regardless of which of the above two goals one wants ++ * to achieve, service trees need first to be updated to know whether ++ * the in-service queue must be preempted. To have service trees ++ * correctly updated, the in-service queue must be expired and ++ * rescheduled, and bfqq must be scheduled too. This is one of the ++ * most costly operations (in future versions, the scheduling ++ * mechanism may be re-designed in such a way to make it possible to ++ * know whether preemption is needed without needing to update service ++ * trees). In addition, queue preemptions almost always cause random ++ * I/O, and thus loss of throughput. Because of these facts, the next ++ * function adopts the following simple scheme to avoid both costly ++ * operations and too frequent preemptions: it requests the expiration ++ * of the in-service queue (unconditionally) only for queues that need ++ * to recover a hole, or that either are weight-raised or deserve to ++ * be weight-raised. ++ */ ++static bool bfq_bfqq_update_budg_for_activation(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ bool arrived_in_time, ++ bool wr_or_deserves_wr) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ if (bfq_bfqq_non_blocking_wait_rq(bfqq) && arrived_in_time) { ++ /* ++ * We do not clear the flag non_blocking_wait_rq here, as ++ * the latter is used in bfq_activate_bfqq to signal ++ * that timestamps need to be back-shifted (and is ++ * cleared right after). ++ */ ++ ++ /* ++ * In next assignment we rely on that either ++ * entity->service or entity->budget are not updated ++ * on expiration if bfqq is empty (see ++ * __bfq_bfqq_recalc_budget). Thus both quantities ++ * remain unchanged after such an expiration, and the ++ * following statement therefore assigns to ++ * entity->budget the remaining budget on such an ++ * expiration. For clarity, entity->service is not ++ * updated on expiration in any case, and, in normal ++ * operation, is reset only when bfqq is selected for ++ * service (see bfq_get_next_queue). ++ */ ++ BUG_ON(bfqq->max_budget < 0); ++ entity->budget = min_t(unsigned long, ++ bfq_bfqq_budget_left(bfqq), ++ bfqq->max_budget); ++ ++ BUG_ON(entity->budget < 0); ++ return true; ++ } ++ ++ BUG_ON(bfqq->max_budget < 0); ++ entity->budget = max_t(unsigned long, bfqq->max_budget, ++ bfq_serv_to_charge(bfqq->next_rq, bfqq)); ++ BUG_ON(entity->budget < 0); ++ ++ bfq_clear_bfqq_non_blocking_wait_rq(bfqq); ++ return wr_or_deserves_wr; ++} ++ ++static void bfq_update_bfqq_wr_on_rq_arrival(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ unsigned int old_wr_coeff, ++ bool wr_or_deserves_wr, ++ bool interactive, ++ bool in_burst, ++ bool soft_rt) ++{ ++ if (old_wr_coeff == 1 && wr_or_deserves_wr) { ++ /* start a weight-raising period */ ++ if (interactive) { ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff; ++ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); ++ } else { ++ bfqq->wr_start_at_switch_to_srt = jiffies; ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff * ++ BFQ_SOFTRT_WEIGHT_FACTOR; ++ bfqq->wr_cur_max_time = ++ bfqd->bfq_wr_rt_max_time; ++ } ++ /* ++ * If needed, further reduce budget to make sure it is ++ * close to bfqq's backlog, so as to reduce the ++ * scheduling-error component due to a too large ++ * budget. Do not care about throughput consequences, ++ * but only about latency. Finally, do not assign a ++ * too small budget either, to avoid increasing ++ * latency by causing too frequent expirations. ++ */ ++ bfqq->entity.budget = min_t(unsigned long, ++ bfqq->entity.budget, ++ 2 * bfq_min_budget(bfqd)); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "wrais starting at %lu, rais_max_time %u", ++ jiffies, ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ } else if (old_wr_coeff > 1) { ++ if (interactive) { /* update wr coeff and duration */ ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff; ++ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); ++ } else if (in_burst) { ++ bfqq->wr_coeff = 1; ++ bfq_log_bfqq(bfqd, bfqq, ++ "wrais ending at %lu, rais_max_time %u", ++ jiffies, ++ jiffies_to_msecs(bfqq-> ++ wr_cur_max_time)); ++ } else if (soft_rt) { ++ /* ++ * The application is now or still meeting the ++ * requirements for being deemed soft rt. We ++ * can then correctly and safely (re)charge ++ * the weight-raising duration for the ++ * application with the weight-raising ++ * duration for soft rt applications. ++ * ++ * In particular, doing this recharge now, i.e., ++ * before the weight-raising period for the ++ * application finishes, reduces the probability ++ * of the following negative scenario: ++ * 1) the weight of a soft rt application is ++ * raised at startup (as for any newly ++ * created application), ++ * 2) since the application is not interactive, ++ * at a certain time weight-raising is ++ * stopped for the application, ++ * 3) at that time the application happens to ++ * still have pending requests, and hence ++ * is destined to not have a chance to be ++ * deemed soft rt before these requests are ++ * completed (see the comments to the ++ * function bfq_bfqq_softrt_next_start() ++ * for details on soft rt detection), ++ * 4) these pending requests experience a high ++ * latency because the application is not ++ * weight-raised while they are pending. ++ */ ++ if (bfqq->wr_cur_max_time != ++ bfqd->bfq_wr_rt_max_time) { ++ bfqq->wr_start_at_switch_to_srt = ++ bfqq->last_wr_start_finish; ++ BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish)); ++ ++ bfqq->wr_cur_max_time = ++ bfqd->bfq_wr_rt_max_time; ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff * ++ BFQ_SOFTRT_WEIGHT_FACTOR; ++ bfq_log_bfqq(bfqd, bfqq, ++ "switching to soft_rt wr"); ++ } else ++ bfq_log_bfqq(bfqd, bfqq, ++ "moving forward soft_rt wr duration"); ++ bfqq->last_wr_start_finish = jiffies; ++ } ++ } ++} ++ ++static bool bfq_bfqq_idle_for_long_time(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ return bfqq->dispatched == 0 && ++ time_is_before_jiffies( ++ bfqq->budget_timeout + ++ bfqd->bfq_wr_min_idle_time); ++} ++ ++static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ int old_wr_coeff, ++ struct request *rq, ++ bool *interactive) ++{ ++ bool soft_rt, in_burst, wr_or_deserves_wr, ++ bfqq_wants_to_preempt, ++ idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq), ++ /* ++ * See the comments on ++ * bfq_bfqq_update_budg_for_activation for ++ * details on the usage of the next variable. ++ */ ++ arrived_in_time = ktime_get_ns() <= ++ RQ_BIC(rq)->ttime.last_end_request + ++ bfqd->bfq_slice_idle * 3; ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "bfq_add_request non-busy: " ++ "jiffies %lu, in_time %d, idle_long %d busyw %d " ++ "wr_coeff %u", ++ jiffies, arrived_in_time, ++ idle_for_long_time, ++ bfq_bfqq_non_blocking_wait_rq(bfqq), ++ old_wr_coeff); ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ bfqg_stats_update_io_add(bfqq_group(RQ_BFQQ(rq)), bfqq, ++ req_op(rq), rq->cmd_flags); ++ ++ /* ++ * bfqq deserves to be weight-raised if: ++ * - it is sync, ++ * - it does not belong to a large burst, ++ * - it has been idle for enough time or is soft real-time, ++ * - is linked to a bfq_io_cq (it is not shared in any sense) ++ */ ++ in_burst = bfq_bfqq_in_large_burst(bfqq); ++ soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 && ++ !in_burst && ++ time_is_before_jiffies(bfqq->soft_rt_next_start); ++ *interactive = ++ !in_burst && ++ idle_for_long_time; ++ wr_or_deserves_wr = bfqd->low_latency && ++ (bfqq->wr_coeff > 1 || ++ (bfq_bfqq_sync(bfqq) && ++ bfqq->bic && (*interactive || soft_rt))); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "bfq_add_request: " ++ "in_burst %d, " ++ "soft_rt %d (next %lu), inter %d, bic %p", ++ bfq_bfqq_in_large_burst(bfqq), soft_rt, ++ bfqq->soft_rt_next_start, ++ *interactive, ++ bfqq->bic); ++ ++ /* ++ * Using the last flag, update budget and check whether bfqq ++ * may want to preempt the in-service queue. ++ */ ++ bfqq_wants_to_preempt = ++ bfq_bfqq_update_budg_for_activation(bfqd, bfqq, ++ arrived_in_time, ++ wr_or_deserves_wr); ++ ++ /* ++ * If bfqq happened to be activated in a burst, but has been ++ * idle for much more than an interactive queue, then we ++ * assume that, in the overall I/O initiated in the burst, the ++ * I/O associated with bfqq is finished. So bfqq does not need ++ * to be treated as a queue belonging to a burst ++ * anymore. Accordingly, we reset bfqq's in_large_burst flag ++ * if set, and remove bfqq from the burst list if it's ++ * there. We do not decrement burst_size, because the fact ++ * that bfqq does not need to belong to the burst list any ++ * more does not invalidate the fact that bfqq was created in ++ * a burst. ++ */ ++ if (likely(!bfq_bfqq_just_created(bfqq)) && ++ idle_for_long_time && ++ time_is_before_jiffies( ++ bfqq->budget_timeout + ++ msecs_to_jiffies(10000))) { ++ hlist_del_init(&bfqq->burst_list_node); ++ bfq_clear_bfqq_in_large_burst(bfqq); ++ } ++ ++ bfq_clear_bfqq_just_created(bfqq); ++ ++ if (!bfq_bfqq_IO_bound(bfqq)) { ++ if (arrived_in_time) { ++ bfqq->requests_within_timer++; ++ if (bfqq->requests_within_timer >= ++ bfqd->bfq_requests_within_timer) ++ bfq_mark_bfqq_IO_bound(bfqq); ++ } else ++ bfqq->requests_within_timer = 0; ++ bfq_log_bfqq(bfqd, bfqq, "requests in time %d", ++ bfqq->requests_within_timer); ++ } ++ ++ if (bfqd->low_latency) { ++ if (unlikely(time_is_after_jiffies(bfqq->split_time))) ++ /* wraparound */ ++ bfqq->split_time = ++ jiffies - bfqd->bfq_wr_min_idle_time - 1; ++ ++ if (time_is_before_jiffies(bfqq->split_time + ++ bfqd->bfq_wr_min_idle_time)) { ++ bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq, ++ old_wr_coeff, ++ wr_or_deserves_wr, ++ *interactive, ++ in_burst, ++ soft_rt); ++ ++ if (old_wr_coeff != bfqq->wr_coeff) ++ bfqq->entity.prio_changed = 1; ++ } ++ } ++ ++ bfqq->last_idle_bklogged = jiffies; ++ bfqq->service_from_backlogged = 0; ++ bfq_clear_bfqq_softrt_update(bfqq); ++ ++ bfq_add_bfqq_busy(bfqd, bfqq); ++ ++ /* ++ * Expire in-service queue only if preemption may be needed ++ * for guarantees. In this respect, the function ++ * next_queue_may_preempt just checks a simple, necessary ++ * condition, and not a sufficient condition based on ++ * timestamps. In fact, for the latter condition to be ++ * evaluated, timestamps would need first to be updated, and ++ * this operation is quite costly (see the comments on the ++ * function bfq_bfqq_update_budg_for_activation). ++ */ ++ if (bfqd->in_service_queue && bfqq_wants_to_preempt && ++ bfqd->in_service_queue->wr_coeff < bfqq->wr_coeff && ++ next_queue_may_preempt(bfqd)) { ++ struct bfq_queue *in_serv = ++ bfqd->in_service_queue; ++ BUG_ON(in_serv == bfqq); ++ ++ bfq_bfqq_expire(bfqd, bfqd->in_service_queue, ++ false, BFQ_BFQQ_PREEMPTED); ++ BUG_ON(in_serv->entity.budget < 0); ++ } ++} ++ ++static void bfq_add_request(struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ struct bfq_data *bfqd = bfqq->bfqd; ++ struct request *next_rq, *prev; ++ unsigned int old_wr_coeff = bfqq->wr_coeff; ++ bool interactive = false; ++ ++ bfq_log_bfqq(bfqd, bfqq, "add_request: size %u %s", ++ blk_rq_sectors(rq), rq_is_sync(rq) ? "S" : "A"); ++ ++ if (bfqq->wr_coeff > 1) /* queue is being weight-raised */ ++ bfq_log_bfqq(bfqd, bfqq, ++ "raising period dur %u/%u msec, old coeff %u, w %d(%d)", ++ jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), ++ jiffies_to_msecs(bfqq->wr_cur_max_time), ++ bfqq->wr_coeff, ++ bfqq->entity.weight, bfqq->entity.orig_weight); ++ ++ bfqq->queued[rq_is_sync(rq)]++; ++ bfqd->queued++; ++ ++ elv_rb_add(&bfqq->sort_list, rq); ++ ++ /* ++ * Check if this request is a better next-to-serve candidate. ++ */ ++ prev = bfqq->next_rq; ++ next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position); ++ BUG_ON(!next_rq); ++ bfqq->next_rq = next_rq; ++ ++ /* ++ * Adjust priority tree position, if next_rq changes. ++ */ ++ if (prev != bfqq->next_rq) ++ bfq_pos_tree_add_move(bfqd, bfqq); ++ ++ if (!bfq_bfqq_busy(bfqq)) /* switching to busy ... */ ++ bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff, ++ rq, &interactive); ++ else { ++ if (bfqd->low_latency && old_wr_coeff == 1 && !rq_is_sync(rq) && ++ time_is_before_jiffies( ++ bfqq->last_wr_start_finish + ++ bfqd->bfq_wr_min_inter_arr_async)) { ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff; ++ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); ++ ++ bfqd->wr_busy_queues++; ++ bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqd, bfqq, ++ "non-idle wrais starting, " ++ "wr_max_time %u wr_busy %d", ++ jiffies_to_msecs(bfqq->wr_cur_max_time), ++ bfqd->wr_busy_queues); ++ } ++ if (prev != bfqq->next_rq) ++ bfq_updated_next_req(bfqd, bfqq); ++ } ++ ++ /* ++ * Assign jiffies to last_wr_start_finish in the following ++ * cases: ++ * ++ * . if bfqq is not going to be weight-raised, because, for ++ * non weight-raised queues, last_wr_start_finish stores the ++ * arrival time of the last request; as of now, this piece ++ * of information is used only for deciding whether to ++ * weight-raise async queues ++ * ++ * . if bfqq is not weight-raised, because, if bfqq is now ++ * switching to weight-raised, then last_wr_start_finish ++ * stores the time when weight-raising starts ++ * ++ * . if bfqq is interactive, because, regardless of whether ++ * bfqq is currently weight-raised, the weight-raising ++ * period must start or restart (this case is considered ++ * separately because it is not detected by the above ++ * conditions, if bfqq is already weight-raised) ++ * ++ * last_wr_start_finish has to be updated also if bfqq is soft ++ * real-time, because the weight-raising period is constantly ++ * restarted on idle-to-busy transitions for these queues, but ++ * this is already done in bfq_bfqq_handle_idle_busy_switch if ++ * needed. ++ */ ++ if (bfqd->low_latency && ++ (old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive)) ++ bfqq->last_wr_start_finish = jiffies; ++} ++ ++static struct request *bfq_find_rq_fmerge(struct bfq_data *bfqd, ++ struct bio *bio) ++{ ++ struct task_struct *tsk = current; ++ struct bfq_io_cq *bic; ++ struct bfq_queue *bfqq; ++ ++ bic = bfq_bic_lookup(bfqd, tsk->io_context); ++ if (!bic) ++ return NULL; ++ ++ bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio)); ++ if (bfqq) ++ return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio)); ++ ++ return NULL; ++} ++ ++static sector_t get_sdist(sector_t last_pos, struct request *rq) ++{ ++ sector_t sdist = 0; ++ ++ if (last_pos) { ++ if (last_pos < blk_rq_pos(rq)) ++ sdist = blk_rq_pos(rq) - last_pos; ++ else ++ sdist = last_pos - blk_rq_pos(rq); ++ } ++ ++ return sdist; ++} ++ ++static void bfq_activate_request(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ bfqd->rq_in_driver++; ++} ++ ++static void bfq_deactivate_request(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ ++ BUG_ON(bfqd->rq_in_driver == 0); ++ bfqd->rq_in_driver--; ++} ++ ++static void bfq_remove_request(struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ struct bfq_data *bfqd = bfqq->bfqd; ++ const int sync = rq_is_sync(rq); ++ ++ BUG_ON(bfqq->entity.service > bfqq->entity.budget && ++ bfqq == bfqd->in_service_queue); ++ ++ if (bfqq->next_rq == rq) { ++ bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); ++ bfq_updated_next_req(bfqd, bfqq); ++ } ++ ++ if (rq->queuelist.prev != &rq->queuelist) ++ list_del_init(&rq->queuelist); ++ BUG_ON(bfqq->queued[sync] == 0); ++ bfqq->queued[sync]--; ++ bfqd->queued--; ++ elv_rb_del(&bfqq->sort_list, rq); ++ ++ if (RB_EMPTY_ROOT(&bfqq->sort_list)) { ++ BUG_ON(bfqq->entity.budget < 0); ++ ++ if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { ++ bfq_del_bfqq_busy(bfqd, bfqq, 1); ++ ++ /* bfqq emptied. In normal operation, when ++ * bfqq is empty, bfqq->entity.service and ++ * bfqq->entity.budget must contain, ++ * respectively, the service received and the ++ * budget used last time bfqq emptied. These ++ * facts do not hold in this case, as at least ++ * this last removal occurred while bfqq is ++ * not in service. To avoid inconsistencies, ++ * reset both bfqq->entity.service and ++ * bfqq->entity.budget. ++ */ ++ bfqq->entity.budget = bfqq->entity.service = 0; ++ } ++ ++ /* ++ * Remove queue from request-position tree as it is empty. ++ */ ++ if (bfqq->pos_root) { ++ rb_erase(&bfqq->pos_node, bfqq->pos_root); ++ bfqq->pos_root = NULL; ++ } ++ } ++ ++ if (rq->cmd_flags & REQ_META) { ++ BUG_ON(bfqq->meta_pending == 0); ++ bfqq->meta_pending--; ++ } ++ bfqg_stats_update_io_remove(bfqq_group(bfqq), req_op(rq), ++ rq->cmd_flags); ++} ++ ++static int bfq_merge(struct request_queue *q, struct request **req, ++ struct bio *bio) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct request *__rq; ++ ++ __rq = bfq_find_rq_fmerge(bfqd, bio); ++ if (__rq && elv_bio_merge_ok(__rq, bio)) { ++ *req = __rq; ++ return ELEVATOR_FRONT_MERGE; ++ } ++ ++ return ELEVATOR_NO_MERGE; ++} ++ ++static void bfq_merged_request(struct request_queue *q, struct request *req, ++ int type) ++{ ++ if (type == ELEVATOR_FRONT_MERGE && ++ rb_prev(&req->rb_node) && ++ blk_rq_pos(req) < ++ blk_rq_pos(container_of(rb_prev(&req->rb_node), ++ struct request, rb_node))) { ++ struct bfq_queue *bfqq = RQ_BFQQ(req); ++ struct bfq_data *bfqd = bfqq->bfqd; ++ struct request *prev, *next_rq; ++ ++ /* Reposition request in its sort_list */ ++ elv_rb_del(&bfqq->sort_list, req); ++ elv_rb_add(&bfqq->sort_list, req); ++ /* Choose next request to be served for bfqq */ ++ prev = bfqq->next_rq; ++ next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req, ++ bfqd->last_position); ++ BUG_ON(!next_rq); ++ bfqq->next_rq = next_rq; ++ /* ++ * If next_rq changes, update both the queue's budget to ++ * fit the new request and the queue's position in its ++ * rq_pos_tree. ++ */ ++ if (prev != bfqq->next_rq) { ++ bfq_updated_next_req(bfqd, bfqq); ++ bfq_pos_tree_add_move(bfqd, bfqq); ++ } ++ } ++} ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static void bfq_bio_merged(struct request_queue *q, struct request *req, ++ struct bio *bio) ++{ ++ bfqg_stats_update_io_merged(bfqq_group(RQ_BFQQ(req)), bio_op(bio), ++ bio->bi_opf); ++} ++#endif ++ ++static void bfq_merged_requests(struct request_queue *q, struct request *rq, ++ struct request *next) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq), *next_bfqq = RQ_BFQQ(next); ++ ++ /* ++ * If next and rq belong to the same bfq_queue and next is older ++ * than rq, then reposition rq in the fifo (by substituting next ++ * with rq). Otherwise, if next and rq belong to different ++ * bfq_queues, never reposition rq: in fact, we would have to ++ * reposition it with respect to next's position in its own fifo, ++ * which would most certainly be too expensive with respect to ++ * the benefits. ++ */ ++ if (bfqq == next_bfqq && ++ !list_empty(&rq->queuelist) && !list_empty(&next->queuelist) && ++ next->fifo_time < rq->fifo_time) { ++ list_del_init(&rq->queuelist); ++ list_replace_init(&next->queuelist, &rq->queuelist); ++ rq->fifo_time = next->fifo_time; ++ } ++ ++ if (bfqq->next_rq == next) ++ bfqq->next_rq = rq; ++ ++ bfq_remove_request(next); ++ bfqg_stats_update_io_merged(bfqq_group(bfqq), req_op(next), ++ next->cmd_flags); ++} ++ ++/* Must be called with bfqq != NULL */ ++static void bfq_bfqq_end_wr(struct bfq_queue *bfqq) ++{ ++ BUG_ON(!bfqq); ++ ++ if (bfq_bfqq_busy(bfqq)) ++ bfqq->bfqd->wr_busy_queues--; ++ bfqq->wr_coeff = 1; ++ bfqq->wr_cur_max_time = 0; ++ bfqq->last_wr_start_finish = jiffies; ++ /* ++ * Trigger a weight change on the next invocation of ++ * __bfq_entity_update_weight_prio. ++ */ ++ bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "end_wr: wrais ending at %lu, rais_max_time %u", ++ bfqq->last_wr_start_finish, ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "end_wr: wr_busy %d", ++ bfqq->bfqd->wr_busy_queues); ++} ++ ++static void bfq_end_wr_async_queues(struct bfq_data *bfqd, ++ struct bfq_group *bfqg) ++{ ++ int i, j; ++ ++ for (i = 0; i < 2; i++) ++ for (j = 0; j < IOPRIO_BE_NR; j++) ++ if (bfqg->async_bfqq[i][j]) ++ bfq_bfqq_end_wr(bfqg->async_bfqq[i][j]); ++ if (bfqg->async_idle_bfqq) ++ bfq_bfqq_end_wr(bfqg->async_idle_bfqq); ++} ++ ++static void bfq_end_wr(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq; ++ ++ spin_lock_irq(bfqd->queue->queue_lock); ++ ++ list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) ++ bfq_bfqq_end_wr(bfqq); ++ list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) ++ bfq_bfqq_end_wr(bfqq); ++ bfq_end_wr_async(bfqd); ++ ++ spin_unlock_irq(bfqd->queue->queue_lock); ++} ++ ++static sector_t bfq_io_struct_pos(void *io_struct, bool request) ++{ ++ if (request) ++ return blk_rq_pos(io_struct); ++ else ++ return ((struct bio *)io_struct)->bi_iter.bi_sector; ++} ++ ++static int bfq_rq_close_to_sector(void *io_struct, bool request, ++ sector_t sector) ++{ ++ return abs(bfq_io_struct_pos(io_struct, request) - sector) <= ++ BFQQ_CLOSE_THR; ++} ++ ++static struct bfq_queue *bfqq_find_close(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ sector_t sector) ++{ ++ struct rb_root *root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; ++ struct rb_node *parent, *node; ++ struct bfq_queue *__bfqq; ++ ++ if (RB_EMPTY_ROOT(root)) ++ return NULL; ++ ++ /* ++ * First, if we find a request starting at the end of the last ++ * request, choose it. ++ */ ++ __bfqq = bfq_rq_pos_tree_lookup(bfqd, root, sector, &parent, NULL); ++ if (__bfqq) ++ return __bfqq; ++ ++ /* ++ * If the exact sector wasn't found, the parent of the NULL leaf ++ * will contain the closest sector (rq_pos_tree sorted by ++ * next_request position). ++ */ ++ __bfqq = rb_entry(parent, struct bfq_queue, pos_node); ++ if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) ++ return __bfqq; ++ ++ if (blk_rq_pos(__bfqq->next_rq) < sector) ++ node = rb_next(&__bfqq->pos_node); ++ else ++ node = rb_prev(&__bfqq->pos_node); ++ if (!node) ++ return NULL; ++ ++ __bfqq = rb_entry(node, struct bfq_queue, pos_node); ++ if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) ++ return __bfqq; ++ ++ return NULL; ++} ++ ++static struct bfq_queue *bfq_find_close_cooperator(struct bfq_data *bfqd, ++ struct bfq_queue *cur_bfqq, ++ sector_t sector) ++{ ++ struct bfq_queue *bfqq; ++ ++ /* ++ * We shall notice if some of the queues are cooperating, ++ * e.g., working closely on the same area of the device. In ++ * that case, we can group them together and: 1) don't waste ++ * time idling, and 2) serve the union of their requests in ++ * the best possible order for throughput. ++ */ ++ bfqq = bfqq_find_close(bfqd, cur_bfqq, sector); ++ if (!bfqq || bfqq == cur_bfqq) ++ return NULL; ++ ++ return bfqq; ++} ++ ++static struct bfq_queue * ++bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) ++{ ++ int process_refs, new_process_refs; ++ struct bfq_queue *__bfqq; ++ ++ /* ++ * If there are no process references on the new_bfqq, then it is ++ * unsafe to follow the ->new_bfqq chain as other bfqq's in the chain ++ * may have dropped their last reference (not just their last process ++ * reference). ++ */ ++ if (!bfqq_process_refs(new_bfqq)) ++ return NULL; ++ ++ /* Avoid a circular list and skip interim queue merges. */ ++ while ((__bfqq = new_bfqq->new_bfqq)) { ++ if (__bfqq == bfqq) ++ return NULL; ++ new_bfqq = __bfqq; ++ } ++ ++ process_refs = bfqq_process_refs(bfqq); ++ new_process_refs = bfqq_process_refs(new_bfqq); ++ /* ++ * If the process for the bfqq has gone away, there is no ++ * sense in merging the queues. ++ */ ++ if (process_refs == 0 || new_process_refs == 0) ++ return NULL; ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", ++ new_bfqq->pid); ++ ++ /* ++ * Merging is just a redirection: the requests of the process ++ * owning one of the two queues are redirected to the other queue. ++ * The latter queue, in its turn, is set as shared if this is the ++ * first time that the requests of some process are redirected to ++ * it. ++ * ++ * We redirect bfqq to new_bfqq and not the opposite, because we ++ * are in the context of the process owning bfqq, hence we have ++ * the io_cq of this process. So we can immediately configure this ++ * io_cq to redirect the requests of the process to new_bfqq. ++ * ++ * NOTE, even if new_bfqq coincides with the in-service queue, the ++ * io_cq of new_bfqq is not available, because, if the in-service ++ * queue is shared, bfqd->in_service_bic may not point to the ++ * io_cq of the in-service queue. ++ * Redirecting the requests of the process owning bfqq to the ++ * currently in-service queue is in any case the best option, as ++ * we feed the in-service queue with new requests close to the ++ * last request served and, by doing so, hopefully increase the ++ * throughput. ++ */ ++ bfqq->new_bfqq = new_bfqq; ++ new_bfqq->ref += process_refs; ++ return new_bfqq; ++} ++ ++static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, ++ struct bfq_queue *new_bfqq) ++{ ++ if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) || ++ (bfqq->ioprio_class != new_bfqq->ioprio_class)) ++ return false; ++ ++ /* ++ * If either of the queues has already been detected as seeky, ++ * then merging it with the other queue is unlikely to lead to ++ * sequential I/O. ++ */ ++ if (BFQQ_SEEKY(bfqq) || BFQQ_SEEKY(new_bfqq)) ++ return false; ++ ++ /* ++ * Interleaved I/O is known to be done by (some) applications ++ * only for reads, so it does not make sense to merge async ++ * queues. ++ */ ++ if (!bfq_bfqq_sync(bfqq) || !bfq_bfqq_sync(new_bfqq)) ++ return false; ++ ++ return true; ++} ++ ++/* ++ * If this function returns true, then bfqq cannot be merged. The idea ++ * is that true cooperation happens very early after processes start ++ * to do I/O. Usually, late cooperations are just accidental false ++ * positives. In case bfqq is weight-raised, such false positives ++ * would evidently degrade latency guarantees for bfqq. ++ */ ++bool wr_from_too_long(struct bfq_queue *bfqq) ++{ ++ return bfqq->wr_coeff > 1 && ++ time_is_before_jiffies(bfqq->last_wr_start_finish + ++ msecs_to_jiffies(100)); ++} ++ ++/* ++ * Attempt to schedule a merge of bfqq with the currently in-service ++ * queue or with a close queue among the scheduled queues. Return ++ * NULL if no merge was scheduled, a pointer to the shared bfq_queue ++ * structure otherwise. ++ * ++ * The OOM queue is not allowed to participate to cooperation: in fact, since ++ * the requests temporarily redirected to the OOM queue could be redirected ++ * again to dedicated queues at any time, the state needed to correctly ++ * handle merging with the OOM queue would be quite complex and expensive ++ * to maintain. Besides, in such a critical condition as an out of memory, ++ * the benefits of queue merging may be little relevant, or even negligible. ++ * ++ * Weight-raised queues can be merged only if their weight-raising ++ * period has just started. In fact cooperating processes are usually ++ * started together. Thus, with this filter we avoid false positives ++ * that would jeopardize low-latency guarantees. ++ * ++ * WARNING: queue merging may impair fairness among non-weight raised ++ * queues, for at least two reasons: 1) the original weight of a ++ * merged queue may change during the merged state, 2) even being the ++ * weight the same, a merged queue may be bloated with many more ++ * requests than the ones produced by its originally-associated ++ * process. ++ */ ++static struct bfq_queue * ++bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ void *io_struct, bool request) ++{ ++ struct bfq_queue *in_service_bfqq, *new_bfqq; ++ ++ if (bfqq->new_bfqq) ++ return bfqq->new_bfqq; ++ ++ if (io_struct && wr_from_too_long(bfqq) && ++ likely(bfqq != &bfqd->oom_bfqq)) ++ bfq_log_bfqq(bfqd, bfqq, ++ "would have looked for coop, but bfq%d wr", ++ bfqq->pid); ++ ++ if (!io_struct || ++ wr_from_too_long(bfqq) || ++ unlikely(bfqq == &bfqd->oom_bfqq)) ++ return NULL; ++ ++ /* If there is only one backlogged queue, don't search. */ ++ if (bfqd->busy_queues == 1) ++ return NULL; ++ ++ in_service_bfqq = bfqd->in_service_queue; ++ ++ if (in_service_bfqq && in_service_bfqq != bfqq && ++ bfqd->in_service_bic && wr_from_too_long(in_service_bfqq) ++ && likely(in_service_bfqq == &bfqd->oom_bfqq)) ++ bfq_log_bfqq(bfqd, bfqq, ++ "would have tried merge with in-service-queue, but wr"); ++ ++ if (!in_service_bfqq || in_service_bfqq == bfqq || ++ !bfqd->in_service_bic || wr_from_too_long(in_service_bfqq) || ++ unlikely(in_service_bfqq == &bfqd->oom_bfqq)) ++ goto check_scheduled; ++ ++ if (bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) && ++ bfqq->entity.parent == in_service_bfqq->entity.parent && ++ bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) { ++ new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq); ++ if (new_bfqq) ++ return new_bfqq; ++ } ++ /* ++ * Check whether there is a cooperator among currently scheduled ++ * queues. The only thing we need is that the bio/request is not ++ * NULL, as we need it to establish whether a cooperator exists. ++ */ ++check_scheduled: ++ new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, ++ bfq_io_struct_pos(io_struct, request)); ++ ++ BUG_ON(new_bfqq && bfqq->entity.parent != new_bfqq->entity.parent); ++ ++ if (new_bfqq && wr_from_too_long(new_bfqq) && ++ likely(new_bfqq != &bfqd->oom_bfqq) && ++ bfq_may_be_close_cooperator(bfqq, new_bfqq)) ++ bfq_log_bfqq(bfqd, bfqq, ++ "would have merged with bfq%d, but wr", ++ new_bfqq->pid); ++ ++ if (new_bfqq && !wr_from_too_long(new_bfqq) && ++ likely(new_bfqq != &bfqd->oom_bfqq) && ++ bfq_may_be_close_cooperator(bfqq, new_bfqq)) ++ return bfq_setup_merge(bfqq, new_bfqq); ++ ++ return NULL; ++} ++ ++static void bfq_bfqq_save_state(struct bfq_queue *bfqq) ++{ ++ struct bfq_io_cq *bic = bfqq->bic; ++ ++ /* ++ * If !bfqq->bic, the queue is already shared or its requests ++ * have already been redirected to a shared queue; both idle window ++ * and weight raising state have already been saved. Do nothing. ++ */ ++ if (!bic) ++ return; ++ ++ bic->saved_idle_window = bfq_bfqq_idle_window(bfqq); ++ bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); ++ bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); ++ bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node); ++ bic->saved_wr_coeff = bfqq->wr_coeff; ++ bic->saved_wr_start_at_switch_to_srt = bfqq->wr_start_at_switch_to_srt; ++ bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish; ++ BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish)); ++} ++ ++static void bfq_get_bic_reference(struct bfq_queue *bfqq) ++{ ++ /* ++ * If bfqq->bic has a non-NULL value, the bic to which it belongs ++ * is about to begin using a shared bfq_queue. ++ */ ++ if (bfqq->bic) ++ atomic_long_inc(&bfqq->bic->icq.ioc->refcount); ++} ++ ++static void ++bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, ++ struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) ++{ ++ bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", ++ (unsigned long) new_bfqq->pid); ++ /* Save weight raising and idle window of the merged queues */ ++ bfq_bfqq_save_state(bfqq); ++ bfq_bfqq_save_state(new_bfqq); ++ if (bfq_bfqq_IO_bound(bfqq)) ++ bfq_mark_bfqq_IO_bound(new_bfqq); ++ bfq_clear_bfqq_IO_bound(bfqq); ++ ++ /* ++ * If bfqq is weight-raised, then let new_bfqq inherit ++ * weight-raising. To reduce false positives, neglect the case ++ * where bfqq has just been created, but has not yet made it ++ * to be weight-raised (which may happen because EQM may merge ++ * bfqq even before bfq_add_request is executed for the first ++ * time for bfqq). Handling this case would however be very ++ * easy, thanks to the flag just_created. ++ */ ++ if (new_bfqq->wr_coeff == 1 && bfqq->wr_coeff > 1) { ++ new_bfqq->wr_coeff = bfqq->wr_coeff; ++ new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time; ++ new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish; ++ new_bfqq->wr_start_at_switch_to_srt = bfqq->wr_start_at_switch_to_srt; ++ if (bfq_bfqq_busy(new_bfqq)) ++ bfqd->wr_busy_queues++; ++ new_bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqd, new_bfqq, ++ "wr start after merge with %d, rais_max_time %u", ++ bfqq->pid, ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ } ++ ++ if (bfqq->wr_coeff > 1) { /* bfqq has given its wr to new_bfqq */ ++ bfqq->wr_coeff = 1; ++ bfqq->entity.prio_changed = 1; ++ if (bfq_bfqq_busy(bfqq)) ++ bfqd->wr_busy_queues--; ++ } ++ ++ bfq_log_bfqq(bfqd, new_bfqq, "merge_bfqqs: wr_busy %d", ++ bfqd->wr_busy_queues); ++ ++ /* ++ * Grab a reference to the bic, to prevent it from being destroyed ++ * before being possibly touched by a bfq_split_bfqq(). ++ */ ++ bfq_get_bic_reference(bfqq); ++ bfq_get_bic_reference(new_bfqq); ++ /* ++ * Merge queues (that is, let bic redirect its requests to new_bfqq) ++ */ ++ bic_set_bfqq(bic, new_bfqq, 1); ++ bfq_mark_bfqq_coop(new_bfqq); ++ /* ++ * new_bfqq now belongs to at least two bics (it is a shared queue): ++ * set new_bfqq->bic to NULL. bfqq either: ++ * - does not belong to any bic any more, and hence bfqq->bic must ++ * be set to NULL, or ++ * - is a queue whose owning bics have already been redirected to a ++ * different queue, hence the queue is destined to not belong to ++ * any bic soon and bfqq->bic is already NULL (therefore the next ++ * assignment causes no harm). ++ */ ++ new_bfqq->bic = NULL; ++ bfqq->bic = NULL; ++ bfq_put_queue(bfqq); ++} ++ ++static int bfq_allow_bio_merge(struct request_queue *q, struct request *rq, ++ struct bio *bio) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct bfq_io_cq *bic; ++ struct bfq_queue *bfqq, *new_bfqq; ++ ++ /* ++ * Disallow merge of a sync bio into an async request. ++ */ ++ if (bfq_bio_sync(bio) && !rq_is_sync(rq)) ++ return false; ++ ++ /* ++ * Lookup the bfqq that this bio will be queued with. Allow ++ * merge only if rq is queued there. ++ * Queue lock is held here. ++ */ ++ bic = bfq_bic_lookup(bfqd, current->io_context); ++ if (!bic) ++ return false; ++ ++ bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio)); ++ /* ++ * We take advantage of this function to perform an early merge ++ * of the queues of possible cooperating processes. ++ */ ++ if (bfqq) { ++ new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false); ++ if (new_bfqq) { ++ bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq); ++ /* ++ * If we get here, the bio will be queued in the ++ * shared queue, i.e., new_bfqq, so use new_bfqq ++ * to decide whether bio and rq can be merged. ++ */ ++ bfqq = new_bfqq; ++ } ++ } ++ ++ return bfqq == RQ_BFQQ(rq); ++} ++ ++static int bfq_allow_rq_merge(struct request_queue *q, struct request *rq, ++ struct request *next) ++{ ++ return RQ_BFQQ(rq) == RQ_BFQQ(next); ++} ++ ++/* ++ * Set the maximum time for the in-service queue to consume its ++ * budget. This prevents seeky processes from lowering the throughput. ++ * In practice, a time-slice service scheme is used with seeky ++ * processes. ++ */ ++static void bfq_set_budget_timeout(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ unsigned int timeout_coeff; ++ ++ if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) ++ timeout_coeff = 1; ++ else ++ timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight; ++ ++ bfqd->last_budget_start = ktime_get(); ++ ++ bfqq->budget_timeout = jiffies + ++ bfqd->bfq_timeout * timeout_coeff; ++ ++ bfq_log_bfqq(bfqd, bfqq, "set budget_timeout %u", ++ jiffies_to_msecs(bfqd->bfq_timeout * timeout_coeff)); ++} ++ ++static void __bfq_set_in_service_queue(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ if (bfqq) { ++ bfqg_stats_update_avg_queue_size(bfqq_group(bfqq)); ++ bfq_mark_bfqq_must_alloc(bfqq); ++ bfq_clear_bfqq_fifo_expire(bfqq); ++ ++ bfqd->budgets_assigned = (bfqd->budgets_assigned*7 + 256) / 8; ++ ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list)); ++ ++ if (time_is_before_jiffies(bfqq->last_wr_start_finish) && ++ bfqq->wr_coeff > 1 && ++ bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && ++ time_is_before_jiffies(bfqq->budget_timeout)) { ++ /* ++ * For soft real-time queues, move the start ++ * of the weight-raising period forward by the ++ * time the queue has not received any ++ * service. Otherwise, a relatively long ++ * service delay is likely to cause the ++ * weight-raising period of the queue to end, ++ * because of the short duration of the ++ * weight-raising period of a soft real-time ++ * queue. It is worth noting that this move ++ * is not so dangerous for the other queues, ++ * because soft real-time queues are not ++ * greedy. ++ * ++ * To not add a further variable, we use the ++ * overloaded field budget_timeout to ++ * determine for how long the queue has not ++ * received service, i.e., how much time has ++ * elapsed since the queue expired. However, ++ * this is a little imprecise, because ++ * budget_timeout is set to jiffies if bfqq ++ * not only expires, but also remains with no ++ * request. ++ */ ++ bfqq->last_wr_start_finish += jiffies - ++ max_t(unsigned long, bfqq->last_wr_start_finish, ++ bfqq->budget_timeout); ++ if (time_is_after_jiffies(bfqq->last_wr_start_finish)) { ++ pr_crit( ++ "BFQ WARNING:last %lu budget %lu jiffies %lu", ++ bfqq->last_wr_start_finish, ++ bfqq->budget_timeout, ++ jiffies); ++ pr_crit("diff %lu", jiffies - ++ max_t(unsigned long, ++ bfqq->last_wr_start_finish, ++ bfqq->budget_timeout)); ++ bfqq->last_wr_start_finish = jiffies; ++ } ++ } ++ ++ bfq_set_budget_timeout(bfqd, bfqq); ++ bfq_log_bfqq(bfqd, bfqq, ++ "set_in_service_queue, cur-budget = %d", ++ bfqq->entity.budget); ++ } else ++ bfq_log(bfqd, "set_in_service_queue: NULL"); ++ ++ bfqd->in_service_queue = bfqq; ++} ++ ++/* ++ * Get and set a new queue for service. ++ */ ++static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); ++ ++ __bfq_set_in_service_queue(bfqd, bfqq); ++ return bfqq; ++} ++ ++static void bfq_arm_slice_timer(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq = bfqd->in_service_queue; ++ struct bfq_io_cq *bic; ++ u32 sl; ++ ++ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list)); ++ ++ /* Processes have exited, don't wait. */ ++ bic = bfqd->in_service_bic; ++ if (!bic || atomic_read(&bic->icq.ioc->active_ref) == 0) ++ return; ++ ++ bfq_mark_bfqq_wait_request(bfqq); ++ ++ /* ++ * We don't want to idle for seeks, but we do want to allow ++ * fair distribution of slice time for a process doing back-to-back ++ * seeks. So allow a little bit of time for him to submit a new rq. ++ * ++ * To prevent processes with (partly) seeky workloads from ++ * being too ill-treated, grant them a small fraction of the ++ * assigned budget before reducing the waiting time to ++ * BFQ_MIN_TT. This happened to help reduce latency. ++ */ ++ sl = bfqd->bfq_slice_idle; ++ /* ++ * Unless the queue is being weight-raised or the scenario is ++ * asymmetric, grant only minimum idle time if the queue ++ * is seeky. A long idling is preserved for a weight-raised ++ * queue, or, more in general, in an asymemtric scenario, ++ * because a long idling is needed for guaranteeing to a queue ++ * its reserved share of the throughput (in particular, it is ++ * needed if the queue has a higher weight than some other ++ * queue). ++ */ ++ if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && ++ bfq_symmetric_scenario(bfqd)) ++ sl = min_t(u32, sl, BFQ_MIN_TT); ++ ++ bfqd->last_idling_start = ktime_get(); ++ hrtimer_start(&bfqd->idle_slice_timer, ns_to_ktime(sl), ++ HRTIMER_MODE_REL); ++ bfqg_stats_set_start_idle_time(bfqq_group(bfqq)); ++ bfq_log(bfqd, "arm idle: %ld/%ld ms", ++ sl / NSEC_PER_MSEC, bfqd->bfq_slice_idle / NSEC_PER_MSEC); ++} ++ ++/* ++ * In autotuning mode, max_budget is dynamically recomputed as the ++ * amount of sectors transferred in timeout at the estimated peak ++ * rate. This enables BFQ to utilize a full timeslice with a full ++ * budget, even if the in-service queue is served at peak rate. And ++ * this maximises throughput with sequential workloads. ++ */ ++static unsigned long bfq_calc_max_budget(struct bfq_data *bfqd) ++{ ++ return (u64)bfqd->peak_rate * USEC_PER_MSEC * ++ jiffies_to_msecs(bfqd->bfq_timeout)>>BFQ_RATE_SHIFT; ++} ++ ++/* ++ * Update parameters related to throughput and responsiveness, as a ++ * function of the estimated peak rate. See comments on ++ * bfq_calc_max_budget(), and on T_slow and T_fast arrays. ++ */ ++void update_thr_responsiveness_params(struct bfq_data *bfqd) ++{ ++ int dev_type = blk_queue_nonrot(bfqd->queue); ++ ++ if (bfqd->bfq_user_max_budget == 0) { ++ bfqd->bfq_max_budget = ++ bfq_calc_max_budget(bfqd); ++ BUG_ON(bfqd->bfq_max_budget < 0); ++ bfq_log(bfqd, "new max_budget = %d", ++ bfqd->bfq_max_budget); ++ } ++ ++ if (bfqd->device_speed == BFQ_BFQD_FAST && ++ bfqd->peak_rate < device_speed_thresh[dev_type]) { ++ bfqd->device_speed = BFQ_BFQD_SLOW; ++ bfqd->RT_prod = R_slow[dev_type] * ++ T_slow[dev_type]; ++ } else if (bfqd->device_speed == BFQ_BFQD_SLOW && ++ bfqd->peak_rate > device_speed_thresh[dev_type]) { ++ bfqd->device_speed = BFQ_BFQD_FAST; ++ bfqd->RT_prod = R_fast[dev_type] * ++ T_fast[dev_type]; ++ } ++ ++ bfq_log(bfqd, ++"dev_type %s dev_speed_class = %s (%llu sects/sec), thresh %llu setcs/sec", ++ dev_type == 0 ? "ROT" : "NONROT", ++ bfqd->device_speed == BFQ_BFQD_FAST ? "FAST" : "SLOW", ++ bfqd->device_speed == BFQ_BFQD_FAST ? ++ (USEC_PER_SEC*(u64)R_fast[dev_type])>>BFQ_RATE_SHIFT : ++ (USEC_PER_SEC*(u64)R_slow[dev_type])>>BFQ_RATE_SHIFT, ++ (USEC_PER_SEC*(u64)device_speed_thresh[dev_type])>> ++ BFQ_RATE_SHIFT); ++} ++ ++void bfq_reset_rate_computation(struct bfq_data *bfqd, struct request *rq) ++{ ++ if (rq != NULL) { /* new rq dispatch now, reset accordingly */ ++ bfqd->last_dispatch = bfqd->first_dispatch = ktime_get_ns() ; ++ bfqd->peak_rate_samples = 1; ++ bfqd->sequential_samples = 0; ++ bfqd->tot_sectors_dispatched = bfqd->last_rq_max_size = ++ blk_rq_sectors(rq); ++ } else /* no new rq dispatched, just reset the number of samples */ ++ bfqd->peak_rate_samples = 0; /* full re-init on next disp. */ ++ ++ bfq_log(bfqd, ++ "reset_rate_computation at end, sample %u/%u tot_sects %llu", ++ bfqd->peak_rate_samples, bfqd->sequential_samples, ++ bfqd->tot_sectors_dispatched); ++} ++ ++void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) ++{ ++ u32 rate, weight, divisor; ++ ++ /* ++ * For the convergence property to hold (see comments on ++ * bfq_update_peak_rate()) and for the assessment to be ++ * reliable, a minimum number of samples must be present, and ++ * a minimum amount of time must have elapsed. If not so, do ++ * not compute new rate. Just reset parameters, to get ready ++ * for a new evaluation attempt. ++ */ ++ if (bfqd->peak_rate_samples < BFQ_RATE_MIN_SAMPLES || ++ bfqd->delta_from_first < BFQ_RATE_MIN_INTERVAL) { ++ bfq_log(bfqd, ++ "update_rate_reset: only resetting, delta_first %lluus samples %d", ++ bfqd->delta_from_first>>10, bfqd->peak_rate_samples); ++ goto reset_computation; ++ } ++ ++ /* ++ * If a new request completion has occurred after last ++ * dispatch, then, to approximate the rate at which requests ++ * have been served by the device, it is more precise to ++ * extend the observation interval to the last completion. ++ */ ++ bfqd->delta_from_first = ++ max_t(u64, bfqd->delta_from_first, ++ bfqd->last_completion - bfqd->first_dispatch); ++ ++ BUG_ON(bfqd->delta_from_first == 0); ++ /* ++ * Rate computed in sects/usec, and not sects/nsec, for ++ * precision issues. ++ */ ++ rate = div64_ul(bfqd->tot_sectors_dispatched<delta_from_first, NSEC_PER_USEC)); ++ ++ bfq_log(bfqd, ++"update_rate_reset: tot_sects %llu delta_first %lluus rate %llu sects/s (%d)", ++ bfqd->tot_sectors_dispatched, bfqd->delta_from_first>>10, ++ ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), ++ rate > 20< 20M sectors/sec) ++ */ ++ if ((bfqd->peak_rate_samples > (3 * bfqd->sequential_samples)>>2 && ++ rate <= bfqd->peak_rate) || ++ rate > 20<peak_rate_samples, bfqd->sequential_samples, ++ ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), ++ ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); ++ goto reset_computation; ++ } else { ++ bfq_log(bfqd, ++ "update_rate_reset: do update, samples %u/%u rate/peak %llu/%llu", ++ bfqd->peak_rate_samples, bfqd->sequential_samples, ++ ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), ++ ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); ++ } ++ ++ /* ++ * We have to update the peak rate, at last! To this purpose, ++ * we use a low-pass filter. We compute the smoothing constant ++ * of the filter as a function of the 'weight' of the new ++ * measured rate. ++ * ++ * As can be seen in next formulas, we define this weight as a ++ * quantity proportional to how sequential the workload is, ++ * and to how long the observation time interval is. ++ * ++ * The weight runs from 0 to 8. The maximum value of the ++ * weight, 8, yields the minimum value for the smoothing ++ * constant. At this minimum value for the smoothing constant, ++ * the measured rate contributes for half of the next value of ++ * the estimated peak rate. ++ * ++ * So, the first step is to compute the weight as a function ++ * of how sequential the workload is. Note that the weight ++ * cannot reach 9, because bfqd->sequential_samples cannot ++ * become equal to bfqd->peak_rate_samples, which, in its ++ * turn, holds true because bfqd->sequential_samples is not ++ * incremented for the first sample. ++ */ ++ weight = (9 * bfqd->sequential_samples) / bfqd->peak_rate_samples; ++ ++ /* ++ * Second step: further refine the weight as a function of the ++ * duration of the observation interval. ++ */ ++ weight = min_t(u32, 8, ++ div_u64(weight * bfqd->delta_from_first, ++ BFQ_RATE_REF_INTERVAL)); ++ ++ /* ++ * Divisor ranging from 10, for minimum weight, to 2, for ++ * maximum weight. ++ */ ++ divisor = 10 - weight; ++ BUG_ON(divisor == 0); ++ ++ /* ++ * Finally, update peak rate: ++ * ++ * peak_rate = peak_rate * (divisor-1) / divisor + rate / divisor ++ */ ++ bfqd->peak_rate *= divisor-1; ++ bfqd->peak_rate /= divisor; ++ rate /= divisor; /* smoothing constant alpha = 1/divisor */ ++ ++ bfq_log(bfqd, ++ "update_rate_reset: divisor %d tmp_peak_rate %llu tmp_rate %u", ++ divisor, ++ ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT), ++ (u32)((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT)); ++ ++ BUG_ON(bfqd->peak_rate == 0); ++ BUG_ON(bfqd->peak_rate > 20<peak_rate += rate; ++ update_thr_responsiveness_params(bfqd); ++ BUG_ON(bfqd->peak_rate > 20<peak_rate_samples == 0) { /* first dispatch */ ++ bfq_log(bfqd, ++ "update_peak_rate: goto reset, samples %d", ++ bfqd->peak_rate_samples) ; ++ bfq_reset_rate_computation(bfqd, rq); ++ goto update_last_values; /* will add one sample */ ++ } ++ ++ /* ++ * Device idle for very long: the observation interval lasting ++ * up to this dispatch cannot be a valid observation interval ++ * for computing a new peak rate (similarly to the late- ++ * completion event in bfq_completed_request()). Go to ++ * update_rate_and_reset to have the following three steps ++ * taken: ++ * - close the observation interval at the last (previous) ++ * request dispatch or completion ++ * - compute rate, if possible, for that observation interval ++ * - start a new observation interval with this dispatch ++ */ ++ if (now_ns - bfqd->last_dispatch > 100*NSEC_PER_MSEC && ++ bfqd->rq_in_driver == 0) { ++ bfq_log(bfqd, ++"update_peak_rate: jumping to updating&resetting delta_last %lluus samples %d", ++ (now_ns - bfqd->last_dispatch)>>10, ++ bfqd->peak_rate_samples) ; ++ goto update_rate_and_reset; ++ } ++ ++ /* Update sampling information */ ++ bfqd->peak_rate_samples++; ++ ++ if ((bfqd->rq_in_driver > 0 || ++ now_ns - bfqd->last_completion < BFQ_MIN_TT) ++ && get_sdist(bfqd->last_position, rq) < BFQQ_SEEK_THR) ++ bfqd->sequential_samples++; ++ ++ bfqd->tot_sectors_dispatched += blk_rq_sectors(rq); ++ ++ /* Reset max observed rq size every 32 dispatches */ ++ if (likely(bfqd->peak_rate_samples % 32)) ++ bfqd->last_rq_max_size = ++ max_t(u32, blk_rq_sectors(rq), bfqd->last_rq_max_size); ++ else ++ bfqd->last_rq_max_size = blk_rq_sectors(rq); ++ ++ bfqd->delta_from_first = now_ns - bfqd->first_dispatch; ++ ++ bfq_log(bfqd, ++ "update_peak_rate: added samples %u/%u tot_sects %llu delta_first %lluus", ++ bfqd->peak_rate_samples, bfqd->sequential_samples, ++ bfqd->tot_sectors_dispatched, ++ bfqd->delta_from_first>>10); ++ ++ /* Target observation interval not yet reached, go on sampling */ ++ if (bfqd->delta_from_first < BFQ_RATE_REF_INTERVAL) ++ goto update_last_values; ++ ++update_rate_and_reset: ++ bfq_update_rate_reset(bfqd, rq); ++update_last_values: ++ bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq); ++ bfqd->last_dispatch = now_ns; ++ ++ bfq_log(bfqd, ++ "update_peak_rate: delta_first %lluus last_pos %llu peak_rate %llu", ++ (now_ns - bfqd->first_dispatch)>>10, ++ (unsigned long long) bfqd->last_position, ++ ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); ++ bfq_log(bfqd, ++ "update_peak_rate: samples at end %d", bfqd->peak_rate_samples); ++} ++ ++/* ++ * Move request from internal lists to the dispatch list of the request queue ++ */ ++static void bfq_dispatch_insert(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ ++ /* ++ * For consistency, the next instruction should have been executed ++ * after removing the request from the queue and dispatching it. ++ * We execute instead this instruction before bfq_remove_request() ++ * (and hence introduce a temporary inconsistency), for efficiency. ++ * In fact, in a forced_dispatch, this prevents two counters related ++ * to bfqq->dispatched to risk to be uselessly decremented if bfqq ++ * is not in service, and then to be incremented again after ++ * incrementing bfqq->dispatched. ++ */ ++ bfqq->dispatched++; ++ bfq_update_peak_rate(q->elevator->elevator_data, rq); ++ ++ bfq_remove_request(rq); ++ elv_dispatch_sort(q, rq); ++} ++ ++/* ++ * Return expired entry, or NULL to just start from scratch in rbtree. ++ */ ++static struct request *bfq_check_fifo(struct bfq_queue *bfqq) ++{ ++ struct request *rq = NULL; ++ ++ if (bfq_bfqq_fifo_expire(bfqq)) ++ return NULL; ++ ++ bfq_mark_bfqq_fifo_expire(bfqq); ++ ++ if (list_empty(&bfqq->fifo)) ++ return NULL; ++ ++ rq = rq_entry_fifo(bfqq->fifo.next); ++ ++ if (ktime_get_ns() < rq->fifo_time) ++ return NULL; ++ ++ return rq; ++} ++ ++static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ BUG_ON(bfqq != bfqd->in_service_queue); ++ ++ __bfq_bfqd_reset_in_service(bfqd); ++ ++ /* ++ * If this bfqq is shared between multiple processes, check ++ * to make sure that those processes are still issuing I/Os ++ * within the mean seek distance. If not, it may be time to ++ * break the queues apart again. ++ */ ++ if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq)) ++ bfq_mark_bfqq_split_coop(bfqq); ++ ++ if (RB_EMPTY_ROOT(&bfqq->sort_list)) { ++ if (bfqq->dispatched == 0) ++ /* ++ * Overloading budget_timeout field to store ++ * the time at which the queue remains with no ++ * backlog and no outstanding request; used by ++ * the weight-raising mechanism. ++ */ ++ bfqq->budget_timeout = jiffies; ++ ++ bfq_del_bfqq_busy(bfqd, bfqq, 1); ++ } else { ++ bfq_activate_bfqq(bfqd, bfqq); ++ /* ++ * Resort priority tree of potential close cooperators. ++ */ ++ bfq_pos_tree_add_move(bfqd, bfqq); ++ } ++} ++ ++/** ++ * __bfq_bfqq_recalc_budget - try to adapt the budget to the @bfqq behavior. ++ * @bfqd: device data. ++ * @bfqq: queue to update. ++ * @reason: reason for expiration. ++ * ++ * Handle the feedback on @bfqq budget at queue expiration. ++ * See the body for detailed comments. ++ */ ++static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ enum bfqq_expiration reason) ++{ ++ struct request *next_rq; ++ int budget, min_budget; ++ ++ BUG_ON(bfqq != bfqd->in_service_queue); ++ ++ min_budget = bfq_min_budget(bfqd); ++ ++ if (bfqq->wr_coeff == 1) ++ budget = bfqq->max_budget; ++ else /* ++ * Use a constant, low budget for weight-raised queues, ++ * to help achieve a low latency. Keep it slightly higher ++ * than the minimum possible budget, to cause a little ++ * bit fewer expirations. ++ */ ++ budget = 2 * min_budget; ++ ++ bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d", ++ bfqq->entity.budget, bfq_bfqq_budget_left(bfqq)); ++ bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d", ++ budget, bfq_min_budget(bfqd)); ++ bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", ++ bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); ++ ++ if (bfq_bfqq_sync(bfqq) && bfqq->wr_coeff == 1) { ++ switch (reason) { ++ /* ++ * Caveat: in all the following cases we trade latency ++ * for throughput. ++ */ ++ case BFQ_BFQQ_TOO_IDLE: ++ /* ++ * This is the only case where we may reduce ++ * the budget: if there is no request of the ++ * process still waiting for completion, then ++ * we assume (tentatively) that the timer has ++ * expired because the batch of requests of ++ * the process could have been served with a ++ * smaller budget. Hence, betting that ++ * process will behave in the same way when it ++ * becomes backlogged again, we reduce its ++ * next budget. As long as we guess right, ++ * this budget cut reduces the latency ++ * experienced by the process. ++ * ++ * However, if there are still outstanding ++ * requests, then the process may have not yet ++ * issued its next request just because it is ++ * still waiting for the completion of some of ++ * the still outstanding ones. So in this ++ * subcase we do not reduce its budget, on the ++ * contrary we increase it to possibly boost ++ * the throughput, as discussed in the ++ * comments to the BUDGET_TIMEOUT case. ++ */ ++ if (bfqq->dispatched > 0) /* still outstanding reqs */ ++ budget = min(budget * 2, bfqd->bfq_max_budget); ++ else { ++ if (budget > 5 * min_budget) ++ budget -= 4 * min_budget; ++ else ++ budget = min_budget; ++ } ++ break; ++ case BFQ_BFQQ_BUDGET_TIMEOUT: ++ /* ++ * We double the budget here because it gives ++ * the chance to boost the throughput if this ++ * is not a seeky process (and has bumped into ++ * this timeout because of, e.g., ZBR). ++ */ ++ budget = min(budget * 2, bfqd->bfq_max_budget); ++ break; ++ case BFQ_BFQQ_BUDGET_EXHAUSTED: ++ /* ++ * The process still has backlog, and did not ++ * let either the budget timeout or the disk ++ * idling timeout expire. Hence it is not ++ * seeky, has a short thinktime and may be ++ * happy with a higher budget too. So ++ * definitely increase the budget of this good ++ * candidate to boost the disk throughput. ++ */ ++ budget = min(budget * 4, bfqd->bfq_max_budget); ++ break; ++ case BFQ_BFQQ_NO_MORE_REQUESTS: ++ /* ++ * For queues that expire for this reason, it ++ * is particularly important to keep the ++ * budget close to the actual service they ++ * need. Doing so reduces the timestamp ++ * misalignment problem described in the ++ * comments in the body of ++ * __bfq_activate_entity. In fact, suppose ++ * that a queue systematically expires for ++ * BFQ_BFQQ_NO_MORE_REQUESTS and presents a ++ * new request in time to enjoy timestamp ++ * back-shifting. The larger the budget of the ++ * queue is with respect to the service the ++ * queue actually requests in each service ++ * slot, the more times the queue can be ++ * reactivated with the same virtual finish ++ * time. It follows that, even if this finish ++ * time is pushed to the system virtual time ++ * to reduce the consequent timestamp ++ * misalignment, the queue unjustly enjoys for ++ * many re-activations a lower finish time ++ * than all newly activated queues. ++ * ++ * The service needed by bfqq is measured ++ * quite precisely by bfqq->entity.service. ++ * Since bfqq does not enjoy device idling, ++ * bfqq->entity.service is equal to the number ++ * of sectors that the process associated with ++ * bfqq requested to read/write before waiting ++ * for request completions, or blocking for ++ * other reasons. ++ */ ++ budget = max_t(int, bfqq->entity.service, min_budget); ++ break; ++ default: ++ return; ++ } ++ } else if (!bfq_bfqq_sync(bfqq)) ++ /* ++ * Async queues get always the maximum possible ++ * budget, as for them we do not care about latency ++ * (in addition, their ability to dispatch is limited ++ * by the charging factor). ++ */ ++ budget = bfqd->bfq_max_budget; ++ ++ bfqq->max_budget = budget; ++ ++ if (bfqd->budgets_assigned >= bfq_stats_min_budgets && ++ !bfqd->bfq_user_max_budget) ++ bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget); ++ ++ /* ++ * If there is still backlog, then assign a new budget, making ++ * sure that it is large enough for the next request. Since ++ * the finish time of bfqq must be kept in sync with the ++ * budget, be sure to call __bfq_bfqq_expire() *after* this ++ * update. ++ * ++ * If there is no backlog, then no need to update the budget; ++ * it will be updated on the arrival of a new request. ++ */ ++ next_rq = bfqq->next_rq; ++ if (next_rq) { ++ BUG_ON(reason == BFQ_BFQQ_TOO_IDLE || ++ reason == BFQ_BFQQ_NO_MORE_REQUESTS); ++ bfqq->entity.budget = max_t(unsigned long, bfqq->max_budget, ++ bfq_serv_to_charge(next_rq, bfqq)); ++ BUG_ON(!bfq_bfqq_busy(bfqq)); ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list)); ++ } ++ ++ bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %d", ++ next_rq ? blk_rq_sectors(next_rq) : 0, ++ bfqq->entity.budget); ++} ++ ++/* ++ * Return true if the process associated with bfqq is "slow". The slow ++ * flag is used, in addition to the budget timeout, to reduce the ++ * amount of service provided to seeky processes, and thus reduce ++ * their chances to lower the throughput. More details in the comments ++ * on the function bfq_bfqq_expire(). ++ * ++ * An important observation is in order: as discussed in the comments ++ * on the function bfq_update_peak_rate(), with devices with internal ++ * queues, it is hard if ever possible to know when and for how long ++ * an I/O request is processed by the device (apart from the trivial ++ * I/O pattern where a new request is dispatched only after the ++ * previous one has been completed). This makes it hard to evaluate ++ * the real rate at which the I/O requests of each bfq_queue are ++ * served. In fact, for an I/O scheduler like BFQ, serving a ++ * bfq_queue means just dispatching its requests during its service ++ * slot (i.e., until the budget of the queue is exhausted, or the ++ * queue remains idle, or, finally, a timeout fires). But, during the ++ * service slot of a bfq_queue, around 100 ms at most, the device may ++ * be even still processing requests of bfq_queues served in previous ++ * service slots. On the opposite end, the requests of the in-service ++ * bfq_queue may be completed after the service slot of the queue ++ * finishes. ++ * ++ * Anyway, unless more sophisticated solutions are used ++ * (where possible), the sum of the sizes of the requests dispatched ++ * during the service slot of a bfq_queue is probably the only ++ * approximation available for the service received by the bfq_queue ++ * during its service slot. And this sum is the quantity used in this ++ * function to evaluate the I/O speed of a process. ++ */ ++static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ bool compensate, enum bfqq_expiration reason, ++ unsigned long *delta_ms) ++{ ++ ktime_t delta_ktime; ++ u32 delta_usecs; ++ bool slow = BFQQ_SEEKY(bfqq); /* if delta too short, use seekyness */ ++ ++ if (!bfq_bfqq_sync(bfqq)) ++ return false; ++ ++ if (compensate) ++ delta_ktime = bfqd->last_idling_start; ++ else ++ delta_ktime = ktime_get(); ++ delta_ktime = ktime_sub(delta_ktime, bfqd->last_budget_start); ++ delta_usecs = ktime_to_us(delta_ktime); ++ ++ /* don't trust short/unrealistic values. */ ++ if (delta_usecs < 1000 || delta_usecs >= LONG_MAX) { ++ if (blk_queue_nonrot(bfqd->queue)) ++ /* ++ * give same worst-case guarantees as idling ++ * for seeky ++ */ ++ *delta_ms = BFQ_MIN_TT / NSEC_PER_MSEC; ++ else /* charge at least one seek */ ++ *delta_ms = bfq_slice_idle / NSEC_PER_MSEC; ++ ++ bfq_log(bfqd, "bfq_bfqq_is_slow: unrealistic %u", delta_usecs); ++ ++ return slow; ++ } ++ ++ *delta_ms = delta_usecs / USEC_PER_MSEC; ++ ++ /* ++ * Use only long (> 20ms) intervals to filter out excessive ++ * spikes in service rate estimation. ++ */ ++ if (delta_usecs > 20000) { ++ /* ++ * Caveat for rotational devices: processes doing I/O ++ * in the slower disk zones tend to be slow(er) even ++ * if not seeky. In this respect, the estimated peak ++ * rate is likely to be an average over the disk ++ * surface. Accordingly, to not be too harsh with ++ * unlucky processes, a process is deemed slow only if ++ * its rate has been lower than half of the estimated ++ * peak rate. ++ */ ++ slow = bfqq->entity.service < bfqd->bfq_max_budget / 2; ++ bfq_log(bfqd, "bfq_bfqq_is_slow: relative rate %d/%d", ++ bfqq->entity.service, bfqd->bfq_max_budget); ++ } ++ ++ bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow); ++ ++ return slow; ++} ++ ++/* ++ * To be deemed as soft real-time, an application must meet two ++ * requirements. First, the application must not require an average ++ * bandwidth higher than the approximate bandwidth required to playback or ++ * record a compressed high-definition video. ++ * The next function is invoked on the completion of the last request of a ++ * batch, to compute the next-start time instant, soft_rt_next_start, such ++ * that, if the next request of the application does not arrive before ++ * soft_rt_next_start, then the above requirement on the bandwidth is met. ++ * ++ * The second requirement is that the request pattern of the application is ++ * isochronous, i.e., that, after issuing a request or a batch of requests, ++ * the application stops issuing new requests until all its pending requests ++ * have been completed. After that, the application may issue a new batch, ++ * and so on. ++ * For this reason the next function is invoked to compute ++ * soft_rt_next_start only for applications that meet this requirement, ++ * whereas soft_rt_next_start is set to infinity for applications that do ++ * not. ++ * ++ * Unfortunately, even a greedy application may happen to behave in an ++ * isochronous way if the CPU load is high. In fact, the application may ++ * stop issuing requests while the CPUs are busy serving other processes, ++ * then restart, then stop again for a while, and so on. In addition, if ++ * the disk achieves a low enough throughput with the request pattern ++ * issued by the application (e.g., because the request pattern is random ++ * and/or the device is slow), then the application may meet the above ++ * bandwidth requirement too. To prevent such a greedy application to be ++ * deemed as soft real-time, a further rule is used in the computation of ++ * soft_rt_next_start: soft_rt_next_start must be higher than the current ++ * time plus the maximum time for which the arrival of a request is waited ++ * for when a sync queue becomes idle, namely bfqd->bfq_slice_idle. ++ * This filters out greedy applications, as the latter issue instead their ++ * next request as soon as possible after the last one has been completed ++ * (in contrast, when a batch of requests is completed, a soft real-time ++ * application spends some time processing data). ++ * ++ * Unfortunately, the last filter may easily generate false positives if ++ * only bfqd->bfq_slice_idle is used as a reference time interval and one ++ * or both the following cases occur: ++ * 1) HZ is so low that the duration of a jiffy is comparable to or higher ++ * than bfqd->bfq_slice_idle. This happens, e.g., on slow devices with ++ * HZ=100. ++ * 2) jiffies, instead of increasing at a constant rate, may stop increasing ++ * for a while, then suddenly 'jump' by several units to recover the lost ++ * increments. This seems to happen, e.g., inside virtual machines. ++ * To address this issue, we do not use as a reference time interval just ++ * bfqd->bfq_slice_idle, but bfqd->bfq_slice_idle plus a few jiffies. In ++ * particular we add the minimum number of jiffies for which the filter ++ * seems to be quite precise also in embedded systems and KVM/QEMU virtual ++ * machines. ++ */ ++static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ bfq_log_bfqq(bfqd, bfqq, ++"softrt_next_start: service_blkg %lu soft_rate %u sects/sec interval %u", ++ bfqq->service_from_backlogged, ++ bfqd->bfq_wr_max_softrt_rate, ++ jiffies_to_msecs(HZ * bfqq->service_from_backlogged / ++ bfqd->bfq_wr_max_softrt_rate)); ++ ++ return max(bfqq->last_idle_bklogged + ++ HZ * bfqq->service_from_backlogged / ++ bfqd->bfq_wr_max_softrt_rate, ++ jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); ++} ++ ++/* ++ * Return the farthest future time instant according to jiffies ++ * macros. ++ */ ++static unsigned long bfq_greatest_from_now(void) ++{ ++ return jiffies + MAX_JIFFY_OFFSET; ++} ++ ++/* ++ * Return the farthest past time instant according to jiffies ++ * macros. ++ */ ++static unsigned long bfq_smallest_from_now(void) ++{ ++ return jiffies - MAX_JIFFY_OFFSET; ++} ++ ++/** ++ * bfq_bfqq_expire - expire a queue. ++ * @bfqd: device owning the queue. ++ * @bfqq: the queue to expire. ++ * @compensate: if true, compensate for the time spent idling. ++ * @reason: the reason causing the expiration. ++ * ++ * If the process associated with bfqq does slow I/O (e.g., because it ++ * issues random requests), we charge bfqq with the time it has been ++ * in service instead of the service it has received (see ++ * bfq_bfqq_charge_time for details on how this goal is achieved). As ++ * a consequence, bfqq will typically get higher timestamps upon ++ * reactivation, and hence it will be rescheduled as if it had ++ * received more service than what it has actually received. In the ++ * end, bfqq receives less service in proportion to how slowly its ++ * associated process consumes its budgets (and hence how seriously it ++ * tends to lower the throughput). In addition, this time-charging ++ * strategy guarantees time fairness among slow processes. In ++ * contrast, if the process associated with bfqq is not slow, we ++ * charge bfqq exactly with the service it has received. ++ * ++ * Charging time to the first type of queues and the exact service to ++ * the other has the effect of using the WF2Q+ policy to schedule the ++ * former on a timeslice basis, without violating service domain ++ * guarantees among the latter. ++ */ ++static void bfq_bfqq_expire(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ bool compensate, ++ enum bfqq_expiration reason) ++{ ++ bool slow; ++ unsigned long delta = 0; ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ BUG_ON(bfqq != bfqd->in_service_queue); ++ ++ /* ++ * Check whether the process is slow (see bfq_bfqq_is_slow). ++ */ ++ slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta); ++ ++ /* ++ * Increase service_from_backlogged before next statement, ++ * because the possible next invocation of ++ * bfq_bfqq_charge_time would likely inflate ++ * entity->service. In contrast, service_from_backlogged must ++ * contain real service, to enable the soft real-time ++ * heuristic to correctly compute the bandwidth consumed by ++ * bfqq. ++ */ ++ bfqq->service_from_backlogged += entity->service; ++ ++ /* ++ * As above explained, charge slow (typically seeky) and ++ * timed-out queues with the time and not the service ++ * received, to favor sequential workloads. ++ * ++ * Processes doing I/O in the slower disk zones will tend to ++ * be slow(er) even if not seeky. Therefore, since the ++ * estimated peak rate is actually an average over the disk ++ * surface, these processes may timeout just for bad luck. To ++ * avoid punishing them, do not charge time to processes that ++ * succeeded in consuming at least 2/3 of their budget. This ++ * allows BFQ to preserve enough elasticity to still perform ++ * bandwidth, and not time, distribution with little unlucky ++ * or quasi-sequential processes. ++ */ ++ if (bfqq->wr_coeff == 1 && ++ (slow || ++ (reason == BFQ_BFQQ_BUDGET_TIMEOUT && ++ bfq_bfqq_budget_left(bfqq) >= entity->budget / 3))) ++ bfq_bfqq_charge_time(bfqd, bfqq, delta); ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ ++ if (reason == BFQ_BFQQ_TOO_IDLE && ++ entity->service <= 2 * entity->budget / 10) ++ bfq_clear_bfqq_IO_bound(bfqq); ++ ++ if (bfqd->low_latency && bfqq->wr_coeff == 1) ++ bfqq->last_wr_start_finish = jiffies; ++ ++ if (bfqd->low_latency && bfqd->bfq_wr_max_softrt_rate > 0 && ++ RB_EMPTY_ROOT(&bfqq->sort_list)) { ++ /* ++ * If we get here, and there are no outstanding ++ * requests, then the request pattern is isochronous ++ * (see the comments on the function ++ * bfq_bfqq_softrt_next_start()). Thus we can compute ++ * soft_rt_next_start. If, instead, the queue still ++ * has outstanding requests, then we have to wait for ++ * the completion of all the outstanding requests to ++ * discover whether the request pattern is actually ++ * isochronous. ++ */ ++ BUG_ON(bfqd->busy_queues < 1); ++ if (bfqq->dispatched == 0) { ++ bfqq->soft_rt_next_start = ++ bfq_bfqq_softrt_next_start(bfqd, bfqq); ++ bfq_log_bfqq(bfqd, bfqq, "new soft_rt_next %lu", ++ bfqq->soft_rt_next_start); ++ } else { ++ /* ++ * The application is still waiting for the ++ * completion of one or more requests: ++ * prevent it from possibly being incorrectly ++ * deemed as soft real-time by setting its ++ * soft_rt_next_start to infinity. In fact, ++ * without this assignment, the application ++ * would be incorrectly deemed as soft ++ * real-time if: ++ * 1) it issued a new request before the ++ * completion of all its in-flight ++ * requests, and ++ * 2) at that time, its soft_rt_next_start ++ * happened to be in the past. ++ */ ++ bfqq->soft_rt_next_start = ++ bfq_greatest_from_now(); ++ /* ++ * Schedule an update of soft_rt_next_start to when ++ * the task may be discovered to be isochronous. ++ */ ++ bfq_mark_bfqq_softrt_update(bfqq); ++ } ++ } ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "expire (%d, slow %d, num_disp %d, idle_win %d, weight %d)", ++ reason, slow, bfqq->dispatched, ++ bfq_bfqq_idle_window(bfqq), entity->weight); ++ ++ /* ++ * Increase, decrease or leave budget unchanged according to ++ * reason. ++ */ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ __bfq_bfqq_recalc_budget(bfqd, bfqq, reason); ++ BUG_ON(bfqq->next_rq == NULL && ++ bfqq->entity.budget < bfqq->entity.service); ++ __bfq_bfqq_expire(bfqd, bfqq); ++ ++ BUG_ON(!bfq_bfqq_busy(bfqq) && reason == BFQ_BFQQ_BUDGET_EXHAUSTED && ++ !bfq_class_idle(bfqq)); ++ ++ if (!bfq_bfqq_busy(bfqq) && ++ reason != BFQ_BFQQ_BUDGET_TIMEOUT && ++ reason != BFQ_BFQQ_BUDGET_EXHAUSTED) ++ bfq_mark_bfqq_non_blocking_wait_rq(bfqq); ++} ++ ++/* ++ * Budget timeout is not implemented through a dedicated timer, but ++ * just checked on request arrivals and completions, as well as on ++ * idle timer expirations. ++ */ ++static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq) ++{ ++ return time_is_before_eq_jiffies(bfqq->budget_timeout); ++} ++ ++/* ++ * If we expire a queue that is actively waiting (i.e., with the ++ * device idled) for the arrival of a new request, then we may incur ++ * the timestamp misalignment problem described in the body of the ++ * function __bfq_activate_entity. Hence we return true only if this ++ * condition does not hold, or if the queue is slow enough to deserve ++ * only to be kicked off for preserving a high throughput. ++ */ ++static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) ++{ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "may_budget_timeout: wait_request %d left %d timeout %d", ++ bfq_bfqq_wait_request(bfqq), ++ bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3, ++ bfq_bfqq_budget_timeout(bfqq)); ++ ++ return (!bfq_bfqq_wait_request(bfqq) || ++ bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3) ++ && ++ bfq_bfqq_budget_timeout(bfqq); ++} ++ ++/* ++ * For a queue that becomes empty, device idling is allowed only if ++ * this function returns true for that queue. As a consequence, since ++ * device idling plays a critical role for both throughput boosting ++ * and service guarantees, the return value of this function plays a ++ * critical role as well. ++ * ++ * In a nutshell, this function returns true only if idling is ++ * beneficial for throughput or, even if detrimental for throughput, ++ * idling is however necessary to preserve service guarantees (low ++ * latency, desired throughput distribution, ...). In particular, on ++ * NCQ-capable devices, this function tries to return false, so as to ++ * help keep the drives' internal queues full, whenever this helps the ++ * device boost the throughput without causing any service-guarantee ++ * issue. ++ * ++ * In more detail, the return value of this function is obtained by, ++ * first, computing a number of boolean variables that take into ++ * account throughput and service-guarantee issues, and, then, ++ * combining these variables in a logical expression. Most of the ++ * issues taken into account are not trivial. We discuss these issues ++ * while introducing the variables. ++ */ ++static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq) ++{ ++ struct bfq_data *bfqd = bfqq->bfqd; ++ bool idling_boosts_thr, idling_boosts_thr_without_issues, ++ idling_needed_for_service_guarantees, ++ asymmetric_scenario; ++ ++ if (bfqd->strict_guarantees) ++ return true; ++ ++ /* ++ * The next variable takes into account the cases where idling ++ * boosts the throughput. ++ * ++ * The value of the variable is computed considering, first, that ++ * idling is virtually always beneficial for the throughput if: ++ * (a) the device is not NCQ-capable, or ++ * (b) regardless of the presence of NCQ, the device is rotational ++ * and the request pattern for bfqq is I/O-bound and sequential. ++ * ++ * Secondly, and in contrast to the above item (b), idling an ++ * NCQ-capable flash-based device would not boost the ++ * throughput even with sequential I/O; rather it would lower ++ * the throughput in proportion to how fast the device ++ * is. Accordingly, the next variable is true if any of the ++ * above conditions (a) and (b) is true, and, in particular, ++ * happens to be false if bfqd is an NCQ-capable flash-based ++ * device. ++ */ ++ idling_boosts_thr = !bfqd->hw_tag || ++ (!blk_queue_nonrot(bfqd->queue) && bfq_bfqq_IO_bound(bfqq) && ++ bfq_bfqq_idle_window(bfqq)); ++ ++ /* ++ * The value of the next variable, ++ * idling_boosts_thr_without_issues, is equal to that of ++ * idling_boosts_thr, unless a special case holds. In this ++ * special case, described below, idling may cause problems to ++ * weight-raised queues. ++ * ++ * When the request pool is saturated (e.g., in the presence ++ * of write hogs), if the processes associated with ++ * non-weight-raised queues ask for requests at a lower rate, ++ * then processes associated with weight-raised queues have a ++ * higher probability to get a request from the pool ++ * immediately (or at least soon) when they need one. Thus ++ * they have a higher probability to actually get a fraction ++ * of the device throughput proportional to their high ++ * weight. This is especially true with NCQ-capable drives, ++ * which enqueue several requests in advance, and further ++ * reorder internally-queued requests. ++ * ++ * For this reason, we force to false the value of ++ * idling_boosts_thr_without_issues if there are weight-raised ++ * busy queues. In this case, and if bfqq is not weight-raised, ++ * this guarantees that the device is not idled for bfqq (if, ++ * instead, bfqq is weight-raised, then idling will be ++ * guaranteed by another variable, see below). Combined with ++ * the timestamping rules of BFQ (see [1] for details), this ++ * behavior causes bfqq, and hence any sync non-weight-raised ++ * queue, to get a lower number of requests served, and thus ++ * to ask for a lower number of requests from the request ++ * pool, before the busy weight-raised queues get served ++ * again. This often mitigates starvation problems in the ++ * presence of heavy write workloads and NCQ, thereby ++ * guaranteeing a higher application and system responsiveness ++ * in these hostile scenarios. ++ */ ++ idling_boosts_thr_without_issues = idling_boosts_thr && ++ bfqd->wr_busy_queues == 0; ++ ++ /* ++ * There is then a case where idling must be performed not ++ * for throughput concerns, but to preserve service ++ * guarantees. ++ * ++ * To introduce this case, we can note that allowing the drive ++ * to enqueue more than one request at a time, and hence ++ * delegating de facto final scheduling decisions to the ++ * drive's internal scheduler, entails loss of control on the ++ * actual request service order. In particular, the critical ++ * situation is when requests from different processes happen ++ * to be present, at the same time, in the internal queue(s) ++ * of the drive. In such a situation, the drive, by deciding ++ * the service order of the internally-queued requests, does ++ * determine also the actual throughput distribution among ++ * these processes. But the drive typically has no notion or ++ * concern about per-process throughput distribution, and ++ * makes its decisions only on a per-request basis. Therefore, ++ * the service distribution enforced by the drive's internal ++ * scheduler is likely to coincide with the desired ++ * device-throughput distribution only in a completely ++ * symmetric scenario where: ++ * (i) each of these processes must get the same throughput as ++ * the others; ++ * (ii) all these processes have the same I/O pattern ++ * (either sequential or random). ++ * In fact, in such a scenario, the drive will tend to treat ++ * the requests of each of these processes in about the same ++ * way as the requests of the others, and thus to provide ++ * each of these processes with about the same throughput ++ * (which is exactly the desired throughput distribution). In ++ * contrast, in any asymmetric scenario, device idling is ++ * certainly needed to guarantee that bfqq receives its ++ * assigned fraction of the device throughput (see [1] for ++ * details). ++ * ++ * We address this issue by controlling, actually, only the ++ * symmetry sub-condition (i), i.e., provided that ++ * sub-condition (i) holds, idling is not performed, ++ * regardless of whether sub-condition (ii) holds. In other ++ * words, only if sub-condition (i) holds, then idling is ++ * allowed, and the device tends to be prevented from queueing ++ * many requests, possibly of several processes. The reason ++ * for not controlling also sub-condition (ii) is that we ++ * exploit preemption to preserve guarantees in case of ++ * symmetric scenarios, even if (ii) does not hold, as ++ * explained in the next two paragraphs. ++ * ++ * Even if a queue, say Q, is expired when it remains idle, Q ++ * can still preempt the new in-service queue if the next ++ * request of Q arrives soon (see the comments on ++ * bfq_bfqq_update_budg_for_activation). If all queues and ++ * groups have the same weight, this form of preemption, ++ * combined with the hole-recovery heuristic described in the ++ * comments on function bfq_bfqq_update_budg_for_activation, ++ * are enough to preserve a correct bandwidth distribution in ++ * the mid term, even without idling. In fact, even if not ++ * idling allows the internal queues of the device to contain ++ * many requests, and thus to reorder requests, we can rather ++ * safely assume that the internal scheduler still preserves a ++ * minimum of mid-term fairness. The motivation for using ++ * preemption instead of idling is that, by not idling, ++ * service guarantees are preserved without minimally ++ * sacrificing throughput. In other words, both a high ++ * throughput and its desired distribution are obtained. ++ * ++ * More precisely, this preemption-based, idleless approach ++ * provides fairness in terms of IOPS, and not sectors per ++ * second. This can be seen with a simple example. Suppose ++ * that there are two queues with the same weight, but that ++ * the first queue receives requests of 8 sectors, while the ++ * second queue receives requests of 1024 sectors. In ++ * addition, suppose that each of the two queues contains at ++ * most one request at a time, which implies that each queue ++ * always remains idle after it is served. Finally, after ++ * remaining idle, each queue receives very quickly a new ++ * request. It follows that the two queues are served ++ * alternatively, preempting each other if needed. This ++ * implies that, although both queues have the same weight, ++ * the queue with large requests receives a service that is ++ * 1024/8 times as high as the service received by the other ++ * queue. ++ * ++ * On the other hand, device idling is performed, and thus ++ * pure sector-domain guarantees are provided, for the ++ * following queues, which are likely to need stronger ++ * throughput guarantees: weight-raised queues, and queues ++ * with a higher weight than other queues. When such queues ++ * are active, sub-condition (i) is false, which triggers ++ * device idling. ++ * ++ * According to the above considerations, the next variable is ++ * true (only) if sub-condition (i) holds. To compute the ++ * value of this variable, we not only use the return value of ++ * the function bfq_symmetric_scenario(), but also check ++ * whether bfqq is being weight-raised, because ++ * bfq_symmetric_scenario() does not take into account also ++ * weight-raised queues (see comments on ++ * bfq_weights_tree_add()). ++ * ++ * As a side note, it is worth considering that the above ++ * device-idling countermeasures may however fail in the ++ * following unlucky scenario: if idling is (correctly) ++ * disabled in a time period during which all symmetry ++ * sub-conditions hold, and hence the device is allowed to ++ * enqueue many requests, but at some later point in time some ++ * sub-condition stops to hold, then it may become impossible ++ * to let requests be served in the desired order until all ++ * the requests already queued in the device have been served. ++ */ ++ asymmetric_scenario = bfqq->wr_coeff > 1 || ++ !bfq_symmetric_scenario(bfqd); ++ ++ /* ++ * Finally, there is a case where maximizing throughput is the ++ * best choice even if it may cause unfairness toward ++ * bfqq. Such a case is when bfqq became active in a burst of ++ * queue activations. Queues that became active during a large ++ * burst benefit only from throughput, as discussed in the ++ * comments on bfq_handle_burst. Thus, if bfqq became active ++ * in a burst and not idling the device maximizes throughput, ++ * then the device must no be idled, because not idling the ++ * device provides bfqq and all other queues in the burst with ++ * maximum benefit. Combining this and the above case, we can ++ * now establish when idling is actually needed to preserve ++ * service guarantees. ++ */ ++ idling_needed_for_service_guarantees = ++ asymmetric_scenario && !bfq_bfqq_in_large_burst(bfqq); ++ ++ /* ++ * We have now all the components we need to compute the return ++ * value of the function, which is true only if both the following ++ * conditions hold: ++ * 1) bfqq is sync, because idling make sense only for sync queues; ++ * 2) idling either boosts the throughput (without issues), or ++ * is necessary to preserve service guarantees. ++ */ ++ bfq_log_bfqq(bfqd, bfqq, "may_idle: sync %d idling_boosts_thr %d", ++ bfq_bfqq_sync(bfqq), idling_boosts_thr); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "may_idle: wr_busy %d boosts %d IO-bound %d guar %d", ++ bfqd->wr_busy_queues, ++ idling_boosts_thr_without_issues, ++ bfq_bfqq_IO_bound(bfqq), ++ idling_needed_for_service_guarantees); ++ ++ return bfq_bfqq_sync(bfqq) && ++ (idling_boosts_thr_without_issues || ++ idling_needed_for_service_guarantees); ++} ++ ++/* ++ * If the in-service queue is empty but the function bfq_bfqq_may_idle ++ * returns true, then: ++ * 1) the queue must remain in service and cannot be expired, and ++ * 2) the device must be idled to wait for the possible arrival of a new ++ * request for the queue. ++ * See the comments on the function bfq_bfqq_may_idle for the reasons ++ * why performing device idling is the best choice to boost the throughput ++ * and preserve service guarantees when bfq_bfqq_may_idle itself ++ * returns true. ++ */ ++static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) ++{ ++ struct bfq_data *bfqd = bfqq->bfqd; ++ ++ return RB_EMPTY_ROOT(&bfqq->sort_list) && bfqd->bfq_slice_idle != 0 && ++ bfq_bfqq_may_idle(bfqq); ++} ++ ++/* ++ * Select a queue for service. If we have a current queue in service, ++ * check whether to continue servicing it, or retrieve and set a new one. ++ */ ++static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq; ++ struct request *next_rq; ++ enum bfqq_expiration reason = BFQ_BFQQ_BUDGET_TIMEOUT; ++ ++ bfqq = bfqd->in_service_queue; ++ if (!bfqq) ++ goto new_queue; ++ ++ bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); ++ ++ if (bfq_may_expire_for_budg_timeout(bfqq) && ++ !hrtimer_active(&bfqd->idle_slice_timer) && ++ !bfq_bfqq_must_idle(bfqq)) ++ goto expire; ++ ++ next_rq = bfqq->next_rq; ++ /* ++ * If bfqq has requests queued and it has enough budget left to ++ * serve them, keep the queue, otherwise expire it. ++ */ ++ if (next_rq) { ++ if (bfq_serv_to_charge(next_rq, bfqq) > ++ bfq_bfqq_budget_left(bfqq)) { ++ reason = BFQ_BFQQ_BUDGET_EXHAUSTED; ++ goto expire; ++ } else { ++ /* ++ * The idle timer may be pending because we may ++ * not disable disk idling even when a new request ++ * arrives. ++ */ ++ if (bfq_bfqq_wait_request(bfqq)) { ++ BUG_ON(!hrtimer_active(&bfqd->idle_slice_timer)); ++ /* ++ * If we get here: 1) at least a new request ++ * has arrived but we have not disabled the ++ * timer because the request was too small, ++ * 2) then the block layer has unplugged ++ * the device, causing the dispatch to be ++ * invoked. ++ * ++ * Since the device is unplugged, now the ++ * requests are probably large enough to ++ * provide a reasonable throughput. ++ * So we disable idling. ++ */ ++ bfq_clear_bfqq_wait_request(bfqq); ++ hrtimer_try_to_cancel(&bfqd->idle_slice_timer); ++ bfqg_stats_update_idle_time(bfqq_group(bfqq)); ++ } ++ goto keep_queue; ++ } ++ } ++ ++ /* ++ * No requests pending. However, if the in-service queue is idling ++ * for a new request, or has requests waiting for a completion and ++ * may idle after their completion, then keep it anyway. ++ */ ++ if (hrtimer_active(&bfqd->idle_slice_timer) || ++ (bfqq->dispatched != 0 && bfq_bfqq_may_idle(bfqq))) { ++ bfqq = NULL; ++ goto keep_queue; ++ } ++ ++ reason = BFQ_BFQQ_NO_MORE_REQUESTS; ++expire: ++ bfq_bfqq_expire(bfqd, bfqq, false, reason); ++new_queue: ++ bfqq = bfq_set_in_service_queue(bfqd); ++ bfq_log(bfqd, "select_queue: new queue %d returned", ++ bfqq ? bfqq->pid : 0); ++keep_queue: ++ return bfqq; ++} ++ ++static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */ ++ BUG_ON(bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && ++ time_is_after_jiffies(bfqq->last_wr_start_finish)); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "raising period dur %u/%u msec, old coeff %u, w %d(%d)", ++ jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), ++ jiffies_to_msecs(bfqq->wr_cur_max_time), ++ bfqq->wr_coeff, ++ bfqq->entity.weight, bfqq->entity.orig_weight); ++ ++ BUG_ON(bfqq != bfqd->in_service_queue && entity->weight != ++ entity->orig_weight * bfqq->wr_coeff); ++ if (entity->prio_changed) ++ bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change"); ++ ++ /* ++ * If the queue was activated in a burst, or too much ++ * time has elapsed from the beginning of this ++ * weight-raising period, then end weight raising. ++ */ ++ if (bfq_bfqq_in_large_burst(bfqq)) ++ bfq_bfqq_end_wr(bfqq); ++ else if (time_is_before_jiffies(bfqq->last_wr_start_finish + ++ bfqq->wr_cur_max_time)) { ++ if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time || ++ time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt + ++ bfq_wr_duration(bfqd))) ++ bfq_bfqq_end_wr(bfqq); ++ else { ++ /* switch back to interactive wr */ ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff; ++ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); ++ bfqq->last_wr_start_finish = ++ bfqq->wr_start_at_switch_to_srt; ++ BUG_ON(time_is_after_jiffies( ++ bfqq->last_wr_start_finish)); ++ bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqd, bfqq, ++ "back to interactive wr"); ++ } ++ } ++ } ++ /* Update weight both if it must be raised and if it must be lowered */ ++ if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1)) ++ __bfq_entity_update_weight_prio( ++ bfq_entity_service_tree(entity), ++ entity); ++} ++ ++/* ++ * Dispatch one request from bfqq, moving it to the request queue ++ * dispatch list. ++ */ ++static int bfq_dispatch_request(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ int dispatched = 0; ++ struct request *rq; ++ unsigned long service_to_charge; ++ ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list)); ++ ++ /* Follow expired path, else get first next available. */ ++ rq = bfq_check_fifo(bfqq); ++ if (!rq) ++ rq = bfqq->next_rq; ++ service_to_charge = bfq_serv_to_charge(rq, bfqq); ++ ++ if (service_to_charge > bfq_bfqq_budget_left(bfqq)) { ++ /* ++ * This may happen if the next rq is chosen in fifo order ++ * instead of sector order. The budget is properly ++ * dimensioned to be always sufficient to serve the next ++ * request only if it is chosen in sector order. The reason ++ * is that it would be quite inefficient and little useful ++ * to always make sure that the budget is large enough to ++ * serve even the possible next rq in fifo order. ++ * In fact, requests are seldom served in fifo order. ++ * ++ * Expire the queue for budget exhaustion, and make sure ++ * that the next act_budget is enough to serve the next ++ * request, even if it comes from the fifo expired path. ++ */ ++ bfqq->next_rq = rq; ++ /* ++ * Since this dispatch is failed, make sure that ++ * a new one will be performed ++ */ ++ if (!bfqd->rq_in_driver) ++ bfq_schedule_dispatch(bfqd); ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ goto expire; ++ } ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ /* Finally, insert request into driver dispatch list. */ ++ bfq_bfqq_served(bfqq, service_to_charge); ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ ++ bfq_dispatch_insert(bfqd->queue, rq); ++ ++ /* ++ * If weight raising has to terminate for bfqq, then next ++ * function causes an immediate update of bfqq's weight, ++ * without waiting for next activation. As a consequence, on ++ * expiration, bfqq will be timestamped as if has never been ++ * weight-raised during this service slot, even if it has ++ * received part or even most of the service as a ++ * weight-raised queue. This inflates bfqq's timestamps, which ++ * is beneficial, as bfqq is then more willing to leave the ++ * device immediately to possible other weight-raised queues. ++ */ ++ bfq_update_wr_data(bfqd, bfqq); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "dispatched %u sec req (%llu), budg left %d", ++ blk_rq_sectors(rq), ++ (unsigned long long) blk_rq_pos(rq), ++ bfq_bfqq_budget_left(bfqq)); ++ ++ dispatched++; ++ ++ if (!bfqd->in_service_bic) { ++ atomic_long_inc(&RQ_BIC(rq)->icq.ioc->refcount); ++ bfqd->in_service_bic = RQ_BIC(rq); ++ } ++ ++ if (bfqd->busy_queues > 1 && bfq_class_idle(bfqq)) ++ goto expire; ++ ++ return dispatched; ++ ++expire: ++ bfq_bfqq_expire(bfqd, bfqq, false, BFQ_BFQQ_BUDGET_EXHAUSTED); ++ return dispatched; ++} ++ ++static int __bfq_forced_dispatch_bfqq(struct bfq_queue *bfqq) ++{ ++ int dispatched = 0; ++ ++ while (bfqq->next_rq) { ++ bfq_dispatch_insert(bfqq->bfqd->queue, bfqq->next_rq); ++ dispatched++; ++ } ++ ++ BUG_ON(!list_empty(&bfqq->fifo)); ++ return dispatched; ++} ++ ++/* ++ * Drain our current requests. ++ * Used for barriers and when switching io schedulers on-the-fly. ++ */ ++static int bfq_forced_dispatch(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq, *n; ++ struct bfq_service_tree *st; ++ int dispatched = 0; ++ ++ bfqq = bfqd->in_service_queue; ++ if (bfqq) ++ __bfq_bfqq_expire(bfqd, bfqq); ++ ++ /* ++ * Loop through classes, and be careful to leave the scheduler ++ * in a consistent state, as feedback mechanisms and vtime ++ * updates cannot be disabled during the process. ++ */ ++ list_for_each_entry_safe(bfqq, n, &bfqd->active_list, bfqq_list) { ++ st = bfq_entity_service_tree(&bfqq->entity); ++ ++ dispatched += __bfq_forced_dispatch_bfqq(bfqq); ++ ++ bfqq->max_budget = bfq_max_budget(bfqd); ++ bfq_forget_idle(st); ++ } ++ ++ BUG_ON(bfqd->busy_queues != 0); ++ ++ return dispatched; ++} ++ ++static int bfq_dispatch_requests(struct request_queue *q, int force) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct bfq_queue *bfqq; ++ ++ bfq_log(bfqd, "dispatch requests: %d busy queues", bfqd->busy_queues); ++ ++ if (bfqd->busy_queues == 0) ++ return 0; ++ ++ if (unlikely(force)) ++ return bfq_forced_dispatch(bfqd); ++ ++ /* ++ * Force device to serve one request at a time if ++ * strict_guarantees is true. Forcing this service scheme is ++ * currently the ONLY way to guarantee that the request ++ * service order enforced by the scheduler is respected by a ++ * queueing device. Otherwise the device is free even to make ++ * some unlucky request wait for as long as the device ++ * wishes. ++ * ++ * Of course, serving one request at at time may cause loss of ++ * throughput. ++ */ ++ if (bfqd->strict_guarantees && bfqd->rq_in_driver > 0) ++ return 0; ++ ++ bfqq = bfq_select_queue(bfqd); ++ if (!bfqq) ++ return 0; ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ ++ BUG_ON(bfq_bfqq_wait_request(bfqq)); ++ ++ if (!bfq_dispatch_request(bfqd, bfqq)) ++ return 0; ++ ++ bfq_log_bfqq(bfqd, bfqq, "dispatched %s request", ++ bfq_bfqq_sync(bfqq) ? "sync" : "async"); ++ ++ BUG_ON(bfqq->next_rq == NULL && ++ bfqq->entity.budget < bfqq->entity.service); ++ return 1; ++} ++ ++/* ++ * Task holds one reference to the queue, dropped when task exits. Each rq ++ * in-flight on this queue also holds a reference, dropped when rq is freed. ++ * ++ * Queue lock must be held here. ++ */ ++static void bfq_put_queue(struct bfq_queue *bfqq) ++{ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ struct bfq_group *bfqg = bfqq_group(bfqq); ++#endif ++ ++ BUG_ON(bfqq->ref <= 0); ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); ++ bfqq->ref--; ++ if (bfqq->ref) ++ return; ++ ++ BUG_ON(rb_first(&bfqq->sort_list)); ++ BUG_ON(bfqq->allocated[READ] + bfqq->allocated[WRITE] != 0); ++ BUG_ON(bfqq->entity.tree); ++ BUG_ON(bfq_bfqq_busy(bfqq)); ++ BUG_ON(bfqq->bfqd->in_service_queue == bfqq); ++ ++ if (bfq_bfqq_sync(bfqq)) ++ /* ++ * The fact that this queue is being destroyed does not ++ * invalidate the fact that this queue may have been ++ * activated during the current burst. As a consequence, ++ * although the queue does not exist anymore, and hence ++ * needs to be removed from the burst list if there, ++ * the burst size has not to be decremented. ++ */ ++ hlist_del_init(&bfqq->burst_list_node); ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p freed", bfqq); ++ ++ kmem_cache_free(bfq_pool, bfqq); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ bfqg_put(bfqg); ++#endif ++} ++ ++static void bfq_put_cooperator(struct bfq_queue *bfqq) ++{ ++ struct bfq_queue *__bfqq, *next; ++ ++ /* ++ * If this queue was scheduled to merge with another queue, be ++ * sure to drop the reference taken on that queue (and others in ++ * the merge chain). See bfq_setup_merge and bfq_merge_bfqqs. ++ */ ++ __bfqq = bfqq->new_bfqq; ++ while (__bfqq) { ++ if (__bfqq == bfqq) ++ break; ++ next = __bfqq->new_bfqq; ++ bfq_put_queue(__bfqq); ++ __bfqq = next; ++ } ++} ++ ++static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ if (bfqq == bfqd->in_service_queue) { ++ __bfq_bfqq_expire(bfqd, bfqq); ++ bfq_schedule_dispatch(bfqd); ++ } ++ ++ bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref); ++ ++ bfq_put_cooperator(bfqq); ++ ++ bfq_put_queue(bfqq); ++} ++ ++static void bfq_init_icq(struct io_cq *icq) ++{ ++ icq_to_bic(icq)->ttime.last_end_request = ktime_get_ns() - (1ULL<<32); ++} ++ ++static void bfq_exit_icq(struct io_cq *icq) ++{ ++ struct bfq_io_cq *bic = icq_to_bic(icq); ++ struct bfq_data *bfqd = bic_to_bfqd(bic); ++ ++ if (bic_to_bfqq(bic, false)) { ++ bfq_exit_bfqq(bfqd, bic_to_bfqq(bic, false)); ++ bic_set_bfqq(bic, NULL, false); ++ } ++ ++ if (bic_to_bfqq(bic, true)) { ++ /* ++ * If the bic is using a shared queue, put the reference ++ * taken on the io_context when the bic started using a ++ * shared bfq_queue. ++ */ ++ if (bfq_bfqq_coop(bic_to_bfqq(bic, true))) ++ put_io_context(icq->ioc); ++ bfq_exit_bfqq(bfqd, bic_to_bfqq(bic, true)); ++ bic_set_bfqq(bic, NULL, true); ++ } ++} ++ ++/* ++ * Update the entity prio values; note that the new values will not ++ * be used until the next (re)activation. ++ */ ++static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, ++ struct bfq_io_cq *bic) ++{ ++ struct task_struct *tsk = current; ++ int ioprio_class; ++ ++ ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio); ++ switch (ioprio_class) { ++ default: ++ dev_err(bfqq->bfqd->queue->backing_dev_info.dev, ++ "bfq: bad prio class %d\n", ioprio_class); ++ case IOPRIO_CLASS_NONE: ++ /* ++ * No prio set, inherit CPU scheduling settings. ++ */ ++ bfqq->new_ioprio = task_nice_ioprio(tsk); ++ bfqq->new_ioprio_class = task_nice_ioclass(tsk); ++ break; ++ case IOPRIO_CLASS_RT: ++ bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); ++ bfqq->new_ioprio_class = IOPRIO_CLASS_RT; ++ break; ++ case IOPRIO_CLASS_BE: ++ bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); ++ bfqq->new_ioprio_class = IOPRIO_CLASS_BE; ++ break; ++ case IOPRIO_CLASS_IDLE: ++ bfqq->new_ioprio_class = IOPRIO_CLASS_IDLE; ++ bfqq->new_ioprio = 7; ++ bfq_clear_bfqq_idle_window(bfqq); ++ break; ++ } ++ ++ if (bfqq->new_ioprio >= IOPRIO_BE_NR) { ++ pr_crit("bfq_set_next_ioprio_data: new_ioprio %d\n", ++ bfqq->new_ioprio); ++ BUG(); ++ } ++ ++ bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio); ++ bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "set_next_ioprio_data: bic_class %d prio %d class %d", ++ ioprio_class, bfqq->new_ioprio, bfqq->new_ioprio_class); ++} ++ ++static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio) ++{ ++ struct bfq_data *bfqd = bic_to_bfqd(bic); ++ struct bfq_queue *bfqq; ++ unsigned long uninitialized_var(flags); ++ int ioprio = bic->icq.ioc->ioprio; ++ ++ /* ++ * This condition may trigger on a newly created bic, be sure to ++ * drop the lock before returning. ++ */ ++ if (unlikely(!bfqd) || likely(bic->ioprio == ioprio)) ++ return; ++ ++ bic->ioprio = ioprio; ++ ++ bfqq = bic_to_bfqq(bic, false); ++ if (bfqq) { ++ bfq_put_queue(bfqq); ++ bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic); ++ bic_set_bfqq(bic, bfqq, false); ++ bfq_log_bfqq(bfqd, bfqq, ++ "check_ioprio_change: bfqq %p %d", ++ bfqq, bfqq->ref); ++ } ++ ++ bfqq = bic_to_bfqq(bic, true); ++ if (bfqq) ++ bfq_set_next_ioprio_data(bfqq, bic); ++} ++ ++static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ struct bfq_io_cq *bic, pid_t pid, int is_sync) ++{ ++ RB_CLEAR_NODE(&bfqq->entity.rb_node); ++ INIT_LIST_HEAD(&bfqq->fifo); ++ INIT_HLIST_NODE(&bfqq->burst_list_node); ++ BUG_ON(!hlist_unhashed(&bfqq->burst_list_node)); ++ ++ bfqq->ref = 0; ++ bfqq->bfqd = bfqd; ++ ++ if (bic) ++ bfq_set_next_ioprio_data(bfqq, bic); ++ ++ if (is_sync) { ++ if (!bfq_class_idle(bfqq)) ++ bfq_mark_bfqq_idle_window(bfqq); ++ bfq_mark_bfqq_sync(bfqq); ++ bfq_mark_bfqq_just_created(bfqq); ++ } else ++ bfq_clear_bfqq_sync(bfqq); ++ bfq_mark_bfqq_IO_bound(bfqq); ++ ++ /* Tentative initial value to trade off between thr and lat */ ++ bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3; ++ bfqq->pid = pid; ++ ++ bfqq->wr_coeff = 1; ++ bfqq->last_wr_start_finish = jiffies; ++ bfqq->wr_start_at_switch_to_srt = bfq_smallest_from_now(); ++ bfqq->budget_timeout = bfq_smallest_from_now(); ++ bfqq->split_time = bfq_smallest_from_now(); ++ ++ /* ++ * Set to the value for which bfqq will not be deemed as ++ * soft rt when it becomes backlogged. ++ */ ++ bfqq->soft_rt_next_start = bfq_greatest_from_now(); ++ ++ /* first request is almost certainly seeky */ ++ bfqq->seek_history = 1; ++} ++ ++static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd, ++ struct bfq_group *bfqg, ++ int ioprio_class, int ioprio) ++{ ++ switch (ioprio_class) { ++ case IOPRIO_CLASS_RT: ++ return &bfqg->async_bfqq[0][ioprio]; ++ case IOPRIO_CLASS_NONE: ++ ioprio = IOPRIO_NORM; ++ /* fall through */ ++ case IOPRIO_CLASS_BE: ++ return &bfqg->async_bfqq[1][ioprio]; ++ case IOPRIO_CLASS_IDLE: ++ return &bfqg->async_idle_bfqq; ++ default: ++ BUG(); ++ } ++} ++ ++static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, ++ struct bio *bio, bool is_sync, ++ struct bfq_io_cq *bic) ++{ ++ const int ioprio = IOPRIO_PRIO_DATA(bic->ioprio); ++ const int ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio); ++ struct bfq_queue **async_bfqq = NULL; ++ struct bfq_queue *bfqq; ++ struct bfq_group *bfqg; ++ ++ rcu_read_lock(); ++ ++ bfqg = bfq_find_set_group(bfqd, bio_blkcg(bio)); ++ if (!bfqg) { ++ bfqq = &bfqd->oom_bfqq; ++ goto out; ++ } ++ ++ if (!is_sync) { ++ async_bfqq = bfq_async_queue_prio(bfqd, bfqg, ioprio_class, ++ ioprio); ++ bfqq = *async_bfqq; ++ if (bfqq) ++ goto out; ++ } ++ ++ bfqq = kmem_cache_alloc_node(bfq_pool, GFP_NOWAIT | __GFP_ZERO, ++ bfqd->queue->node); ++ ++ if (bfqq) { ++ bfq_init_bfqq(bfqd, bfqq, bic, current->pid, ++ is_sync); ++ bfq_init_entity(&bfqq->entity, bfqg); ++ bfq_log_bfqq(bfqd, bfqq, "allocated"); ++ } else { ++ bfqq = &bfqd->oom_bfqq; ++ bfq_log_bfqq(bfqd, bfqq, "using oom bfqq"); ++ goto out; ++ } ++ ++ /* ++ * Pin the queue now that it's allocated, scheduler exit will ++ * prune it. ++ */ ++ if (async_bfqq) { ++ bfqq->ref++; ++ bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", ++ bfqq, bfqq->ref); ++ *async_bfqq = bfqq; ++ } ++ ++out: ++ bfqq->ref++; ++ bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, bfqq->ref); ++ rcu_read_unlock(); ++ return bfqq; ++} ++ ++static void bfq_update_io_thinktime(struct bfq_data *bfqd, ++ struct bfq_io_cq *bic) ++{ ++ struct bfq_ttime *ttime = &bic->ttime; ++ u64 elapsed = ktime_get_ns() - bic->ttime.last_end_request; ++ ++ elapsed = min_t(u64, elapsed, 2 * bfqd->bfq_slice_idle); ++ ++ ttime->ttime_samples = (7*bic->ttime.ttime_samples + 256) / 8; ++ ttime->ttime_total = div_u64(7*ttime->ttime_total + 256*elapsed, 8); ++ ttime->ttime_mean = div64_ul(ttime->ttime_total + 128, ++ ttime->ttime_samples); ++} ++ ++static void ++bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ struct request *rq) ++{ ++ bfqq->seek_history <<= 1; ++ bfqq->seek_history |= ++ get_sdist(bfqq->last_request_pos, rq) > BFQQ_SEEK_THR; ++} ++ ++/* ++ * Disable idle window if the process thinks too long or seeks so much that ++ * it doesn't matter. ++ */ ++static void bfq_update_idle_window(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ struct bfq_io_cq *bic) ++{ ++ int enable_idle; ++ ++ /* Don't idle for async or idle io prio class. */ ++ if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq)) ++ return; ++ ++ /* Idle window just restored, statistics are meaningless. */ ++ if (time_is_after_eq_jiffies(bfqq->split_time + ++ bfqd->bfq_wr_min_idle_time)) ++ return; ++ ++ enable_idle = bfq_bfqq_idle_window(bfqq); ++ ++ if (atomic_read(&bic->icq.ioc->active_ref) == 0 || ++ bfqd->bfq_slice_idle == 0 || ++ (bfqd->hw_tag && BFQQ_SEEKY(bfqq) && ++ bfqq->wr_coeff == 1)) ++ enable_idle = 0; ++ else if (bfq_sample_valid(bic->ttime.ttime_samples)) { ++ if (bic->ttime.ttime_mean > bfqd->bfq_slice_idle && ++ bfqq->wr_coeff == 1) ++ enable_idle = 0; ++ else ++ enable_idle = 1; ++ } ++ bfq_log_bfqq(bfqd, bfqq, "update_idle_window: enable_idle %d", ++ enable_idle); ++ ++ if (enable_idle) ++ bfq_mark_bfqq_idle_window(bfqq); ++ else ++ bfq_clear_bfqq_idle_window(bfqq); ++} ++ ++/* ++ * Called when a new fs request (rq) is added to bfqq. Check if there's ++ * something we should do about it. ++ */ ++static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ struct request *rq) ++{ ++ struct bfq_io_cq *bic = RQ_BIC(rq); ++ ++ if (rq->cmd_flags & REQ_META) ++ bfqq->meta_pending++; ++ ++ bfq_update_io_thinktime(bfqd, bic); ++ bfq_update_io_seektime(bfqd, bfqq, rq); ++ if (bfqq->entity.service > bfq_max_budget(bfqd) / 8 || ++ !BFQQ_SEEKY(bfqq)) ++ bfq_update_idle_window(bfqd, bfqq, bic); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "rq_enqueued: idle_window=%d (seeky %d)", ++ bfq_bfqq_idle_window(bfqq), BFQQ_SEEKY(bfqq)); ++ ++ bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); ++ ++ if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) { ++ bool small_req = bfqq->queued[rq_is_sync(rq)] == 1 && ++ blk_rq_sectors(rq) < 32; ++ bool budget_timeout = bfq_bfqq_budget_timeout(bfqq); ++ ++ /* ++ * There is just this request queued: if the request ++ * is small and the queue is not to be expired, then ++ * just exit. ++ * ++ * In this way, if the device is being idled to wait ++ * for a new request from the in-service queue, we ++ * avoid unplugging the device and committing the ++ * device to serve just a small request. On the ++ * contrary, we wait for the block layer to decide ++ * when to unplug the device: hopefully, new requests ++ * will be merged to this one quickly, then the device ++ * will be unplugged and larger requests will be ++ * dispatched. ++ */ ++ if (small_req && !budget_timeout) ++ return; ++ ++ /* ++ * A large enough request arrived, or the queue is to ++ * be expired: in both cases disk idling is to be ++ * stopped, so clear wait_request flag and reset ++ * timer. ++ */ ++ bfq_clear_bfqq_wait_request(bfqq); ++ hrtimer_try_to_cancel(&bfqd->idle_slice_timer); ++ bfqg_stats_update_idle_time(bfqq_group(bfqq)); ++ ++ /* ++ * The queue is not empty, because a new request just ++ * arrived. Hence we can safely expire the queue, in ++ * case of budget timeout, without risking that the ++ * timestamps of the queue are not updated correctly. ++ * See [1] for more details. ++ */ ++ if (budget_timeout) ++ bfq_bfqq_expire(bfqd, bfqq, false, ++ BFQ_BFQQ_BUDGET_TIMEOUT); ++ ++ /* ++ * Let the request rip immediately, or let a new queue be ++ * selected if bfqq has just been expired. ++ */ ++ __blk_run_queue(bfqd->queue); ++ } ++} ++ ++static void bfq_insert_request(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct bfq_queue *bfqq = RQ_BFQQ(rq), *new_bfqq; ++ ++ assert_spin_locked(bfqd->queue->queue_lock); ++ ++ /* ++ * An unplug may trigger a requeue of a request from the device ++ * driver: make sure we are in process context while trying to ++ * merge two bfq_queues. ++ */ ++ if (!in_interrupt()) { ++ new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true); ++ if (new_bfqq) { ++ if (bic_to_bfqq(RQ_BIC(rq), 1) != bfqq) ++ new_bfqq = bic_to_bfqq(RQ_BIC(rq), 1); ++ /* ++ * Release the request's reference to the old bfqq ++ * and make sure one is taken to the shared queue. ++ */ ++ new_bfqq->allocated[rq_data_dir(rq)]++; ++ bfqq->allocated[rq_data_dir(rq)]--; ++ new_bfqq->ref++; ++ bfq_clear_bfqq_just_created(bfqq); ++ bfq_put_queue(bfqq); ++ if (bic_to_bfqq(RQ_BIC(rq), 1) == bfqq) ++ bfq_merge_bfqqs(bfqd, RQ_BIC(rq), ++ bfqq, new_bfqq); ++ rq->elv.priv[1] = new_bfqq; ++ bfqq = new_bfqq; ++ } ++ } ++ ++ bfq_add_request(rq); ++ ++ rq->fifo_time = ktime_get_ns() + ++ jiffies_to_nsecs(bfqd->bfq_fifo_expire[rq_is_sync(rq)]); ++ list_add_tail(&rq->queuelist, &bfqq->fifo); ++ ++ bfq_rq_enqueued(bfqd, bfqq, rq); ++} ++ ++static void bfq_update_hw_tag(struct bfq_data *bfqd) ++{ ++ bfqd->max_rq_in_driver = max_t(int, bfqd->max_rq_in_driver, ++ bfqd->rq_in_driver); ++ ++ if (bfqd->hw_tag == 1) ++ return; ++ ++ /* ++ * This sample is valid if the number of outstanding requests ++ * is large enough to allow a queueing behavior. Note that the ++ * sum is not exact, as it's not taking into account deactivated ++ * requests. ++ */ ++ if (bfqd->rq_in_driver + bfqd->queued < BFQ_HW_QUEUE_THRESHOLD) ++ return; ++ ++ if (bfqd->hw_tag_samples++ < BFQ_HW_QUEUE_SAMPLES) ++ return; ++ ++ bfqd->hw_tag = bfqd->max_rq_in_driver > BFQ_HW_QUEUE_THRESHOLD; ++ bfqd->max_rq_in_driver = 0; ++ bfqd->hw_tag_samples = 0; ++} ++ ++static void bfq_completed_request(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ struct bfq_data *bfqd = bfqq->bfqd; ++ u64 now_ns; ++ u32 delta_us; ++ ++ bfq_log_bfqq(bfqd, bfqq, "completed one req with %u sects left", ++ blk_rq_sectors(rq)); ++ ++ assert_spin_locked(bfqd->queue->queue_lock); ++ bfq_update_hw_tag(bfqd); ++ ++ BUG_ON(!bfqd->rq_in_driver); ++ BUG_ON(!bfqq->dispatched); ++ bfqd->rq_in_driver--; ++ bfqq->dispatched--; ++ bfqg_stats_update_completion(bfqq_group(bfqq), ++ rq_start_time_ns(rq), ++ rq_io_start_time_ns(rq), req_op(rq), ++ rq->cmd_flags); ++ ++ if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) { ++ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list)); ++ /* ++ * Set budget_timeout (which we overload to store the ++ * time at which the queue remains with no backlog and ++ * no outstanding request; used by the weight-raising ++ * mechanism). ++ */ ++ bfqq->budget_timeout = jiffies; ++ ++ bfq_weights_tree_remove(bfqd, &bfqq->entity, ++ &bfqd->queue_weights_tree); ++ } ++ ++ now_ns = ktime_get_ns(); ++ ++ RQ_BIC(rq)->ttime.last_end_request = now_ns; ++ ++ /* ++ * Using us instead of ns, to get a reasonable precision in ++ * computing rate in next check. ++ */ ++ delta_us = div_u64(now_ns - bfqd->last_completion, NSEC_PER_USEC); ++ ++ bfq_log(bfqd, "rq_completed: delta %uus/%luus max_size %u rate %llu/%llu", ++ delta_us, BFQ_MIN_TT/NSEC_PER_USEC, bfqd->last_rq_max_size, ++ (USEC_PER_SEC* ++ (u64)((bfqd->last_rq_max_size<>BFQ_RATE_SHIFT, ++ (USEC_PER_SEC*(u64)(1UL<<(BFQ_RATE_SHIFT-10)))>>BFQ_RATE_SHIFT); ++ ++ /* ++ * If the request took rather long to complete, and, according ++ * to the maximum request size recorded, this completion latency ++ * implies that the request was certainly served at a very low ++ * rate (less than 1M sectors/sec), then the whole observation ++ * interval that lasts up to this time instant cannot be a ++ * valid time interval for computing a new peak rate. Invoke ++ * bfq_update_rate_reset to have the following three steps ++ * taken: ++ * - close the observation interval at the last (previous) ++ * request dispatch or completion ++ * - compute rate, if possible, for that observation interval ++ * - reset to zero samples, which will trigger a proper ++ * re-initialization of the observation interval on next ++ * dispatch ++ */ ++ if (delta_us > BFQ_MIN_TT/NSEC_PER_USEC && ++ (bfqd->last_rq_max_size<last_completion = now_ns; ++ ++ /* ++ * If we are waiting to discover whether the request pattern ++ * of the task associated with the queue is actually ++ * isochronous, and both requisites for this condition to hold ++ * are now satisfied, then compute soft_rt_next_start (see the ++ * comments on the function bfq_bfqq_softrt_next_start()). We ++ * schedule this delayed check when bfqq expires, if it still ++ * has in-flight requests. ++ */ ++ if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 && ++ RB_EMPTY_ROOT(&bfqq->sort_list)) ++ bfqq->soft_rt_next_start = ++ bfq_bfqq_softrt_next_start(bfqd, bfqq); ++ ++ /* ++ * If this is the in-service queue, check if it needs to be expired, ++ * or if we want to idle in case it has no pending requests. ++ */ ++ if (bfqd->in_service_queue == bfqq) { ++ if (bfqq->dispatched == 0 && bfq_bfqq_must_idle(bfqq)) { ++ bfq_arm_slice_timer(bfqd); ++ goto out; ++ } else if (bfq_may_expire_for_budg_timeout(bfqq)) ++ bfq_bfqq_expire(bfqd, bfqq, false, ++ BFQ_BFQQ_BUDGET_TIMEOUT); ++ else if (RB_EMPTY_ROOT(&bfqq->sort_list) && ++ (bfqq->dispatched == 0 || ++ !bfq_bfqq_may_idle(bfqq))) ++ bfq_bfqq_expire(bfqd, bfqq, false, ++ BFQ_BFQQ_NO_MORE_REQUESTS); ++ } ++ ++ if (!bfqd->rq_in_driver) ++ bfq_schedule_dispatch(bfqd); ++ ++out: ++ return; ++} ++ ++static int __bfq_may_queue(struct bfq_queue *bfqq) ++{ ++ if (bfq_bfqq_wait_request(bfqq) && bfq_bfqq_must_alloc(bfqq)) { ++ bfq_clear_bfqq_must_alloc(bfqq); ++ return ELV_MQUEUE_MUST; ++ } ++ ++ return ELV_MQUEUE_MAY; ++} ++ ++static int bfq_may_queue(struct request_queue *q, int op, int op_flags) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct task_struct *tsk = current; ++ struct bfq_io_cq *bic; ++ struct bfq_queue *bfqq; ++ ++ /* ++ * Don't force setup of a queue from here, as a call to may_queue ++ * does not necessarily imply that a request actually will be ++ * queued. So just lookup a possibly existing queue, or return ++ * 'may queue' if that fails. ++ */ ++ bic = bfq_bic_lookup(bfqd, tsk->io_context); ++ if (!bic) ++ return ELV_MQUEUE_MAY; ++ ++ bfqq = bic_to_bfqq(bic, rw_is_sync(op, op_flags)); ++ if (bfqq) ++ return __bfq_may_queue(bfqq); ++ ++ return ELV_MQUEUE_MAY; ++} ++ ++/* ++ * Queue lock held here. ++ */ ++static void bfq_put_request(struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ ++ if (bfqq) { ++ const int rw = rq_data_dir(rq); ++ ++ BUG_ON(!bfqq->allocated[rw]); ++ bfqq->allocated[rw]--; ++ ++ rq->elv.priv[0] = NULL; ++ rq->elv.priv[1] = NULL; ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_request %p, %d", ++ bfqq, bfqq->ref); ++ bfq_put_queue(bfqq); ++ } ++} ++ ++/* ++ * Returns NULL if a new bfqq should be allocated, or the old bfqq if this ++ * was the last process referring to that bfqq. ++ */ ++static struct bfq_queue * ++bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq) ++{ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); ++ ++ put_io_context(bic->icq.ioc); ++ ++ if (bfqq_process_refs(bfqq) == 1) { ++ bfqq->pid = current->pid; ++ bfq_clear_bfqq_coop(bfqq); ++ bfq_clear_bfqq_split_coop(bfqq); ++ return bfqq; ++ } ++ ++ bic_set_bfqq(bic, NULL, 1); ++ ++ bfq_put_cooperator(bfqq); ++ ++ bfq_put_queue(bfqq); ++ return NULL; ++} ++ ++/* ++ * Allocate bfq data structures associated with this request. ++ */ ++static int bfq_set_request(struct request_queue *q, struct request *rq, ++ struct bio *bio, gfp_t gfp_mask) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct bfq_io_cq *bic = icq_to_bic(rq->elv.icq); ++ const int rw = rq_data_dir(rq); ++ const int is_sync = rq_is_sync(rq); ++ struct bfq_queue *bfqq; ++ unsigned long flags; ++ bool split = false; ++ ++ spin_lock_irqsave(q->queue_lock, flags); ++ bfq_check_ioprio_change(bic, bio); ++ ++ if (!bic) ++ goto queue_fail; ++ ++ bfq_bic_update_cgroup(bic, bio); ++ ++new_queue: ++ bfqq = bic_to_bfqq(bic, is_sync); ++ if (!bfqq || bfqq == &bfqd->oom_bfqq) { ++ if (bfqq) ++ bfq_put_queue(bfqq); ++ bfqq = bfq_get_queue(bfqd, bio, is_sync, bic); ++ BUG_ON(!hlist_unhashed(&bfqq->burst_list_node)); ++ ++ bic_set_bfqq(bic, bfqq, is_sync); ++ if (split && is_sync) { ++ bfq_log_bfqq(bfqd, bfqq, ++ "set_request: was_in_list %d " ++ "was_in_large_burst %d " ++ "large burst in progress %d", ++ bic->was_in_burst_list, ++ bic->saved_in_large_burst, ++ bfqd->large_burst); ++ ++ if ((bic->was_in_burst_list && bfqd->large_burst) || ++ bic->saved_in_large_burst) { ++ bfq_log_bfqq(bfqd, bfqq, ++ "set_request: marking in " ++ "large burst"); ++ bfq_mark_bfqq_in_large_burst(bfqq); ++ } else { ++ bfq_log_bfqq(bfqd, bfqq, ++ "set_request: clearing in " ++ "large burst"); ++ bfq_clear_bfqq_in_large_burst(bfqq); ++ if (bic->was_in_burst_list) ++ hlist_add_head(&bfqq->burst_list_node, ++ &bfqd->burst_list); ++ } ++ bfqq->split_time = jiffies; ++ } ++ } else { ++ /* If the queue was seeky for too long, break it apart. */ ++ if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) { ++ bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq"); ++ ++ /* Update bic before losing reference to bfqq */ ++ if (bfq_bfqq_in_large_burst(bfqq)) ++ bic->saved_in_large_burst = true; ++ ++ bfqq = bfq_split_bfqq(bic, bfqq); ++ split = true; ++ if (!bfqq) ++ goto new_queue; ++ } ++ } ++ ++ bfqq->allocated[rw]++; ++ bfqq->ref++; ++ bfq_log_bfqq(bfqd, bfqq, "set_request: bfqq %p, %d", bfqq, bfqq->ref); ++ ++ rq->elv.priv[0] = bic; ++ rq->elv.priv[1] = bfqq; ++ ++ /* ++ * If a bfq_queue has only one process reference, it is owned ++ * by only one bfq_io_cq: we can set the bic field of the ++ * bfq_queue to the address of that structure. Also, if the ++ * queue has just been split, mark a flag so that the ++ * information is available to the other scheduler hooks. ++ */ ++ if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) { ++ bfqq->bic = bic; ++ if (split) { ++ /* ++ * If the queue has just been split from a shared ++ * queue, restore the idle window and the possible ++ * weight raising period. ++ */ ++ bfq_bfqq_resume_state(bfqq, bic); ++ } ++ } ++ ++ if (unlikely(bfq_bfqq_just_created(bfqq))) ++ bfq_handle_burst(bfqd, bfqq); ++ ++ spin_unlock_irqrestore(q->queue_lock, flags); ++ ++ return 0; ++ ++queue_fail: ++ bfq_schedule_dispatch(bfqd); ++ spin_unlock_irqrestore(q->queue_lock, flags); ++ ++ return 1; ++} ++ ++static void bfq_kick_queue(struct work_struct *work) ++{ ++ struct bfq_data *bfqd = ++ container_of(work, struct bfq_data, unplug_work); ++ struct request_queue *q = bfqd->queue; ++ ++ spin_lock_irq(q->queue_lock); ++ __blk_run_queue(q); ++ spin_unlock_irq(q->queue_lock); ++} ++ ++/* ++ * Handler of the expiration of the timer running if the in-service queue ++ * is idling inside its time slice. ++ */ ++static enum hrtimer_restart bfq_idle_slice_timer(struct hrtimer *timer) ++{ ++ struct bfq_data *bfqd = container_of(timer, struct bfq_data, ++ idle_slice_timer); ++ struct bfq_queue *bfqq; ++ unsigned long flags; ++ enum bfqq_expiration reason; ++ ++ spin_lock_irqsave(bfqd->queue->queue_lock, flags); ++ ++ bfqq = bfqd->in_service_queue; ++ /* ++ * Theoretical race here: the in-service queue can be NULL or ++ * different from the queue that was idling if the timer handler ++ * spins on the queue_lock and a new request arrives for the ++ * current queue and there is a full dispatch cycle that changes ++ * the in-service queue. This can hardly happen, but in the worst ++ * case we just expire a queue too early. ++ */ ++ if (bfqq) { ++ bfq_log_bfqq(bfqd, bfqq, "slice_timer expired"); ++ bfq_clear_bfqq_wait_request(bfqq); ++ ++ if (bfq_bfqq_budget_timeout(bfqq)) ++ /* ++ * Also here the queue can be safely expired ++ * for budget timeout without wasting ++ * guarantees ++ */ ++ reason = BFQ_BFQQ_BUDGET_TIMEOUT; ++ else if (bfqq->queued[0] == 0 && bfqq->queued[1] == 0) ++ /* ++ * The queue may not be empty upon timer expiration, ++ * because we may not disable the timer when the ++ * first request of the in-service queue arrives ++ * during disk idling. ++ */ ++ reason = BFQ_BFQQ_TOO_IDLE; ++ else ++ goto schedule_dispatch; ++ ++ bfq_bfqq_expire(bfqd, bfqq, true, reason); ++ } ++ ++schedule_dispatch: ++ bfq_schedule_dispatch(bfqd); ++ ++ spin_unlock_irqrestore(bfqd->queue->queue_lock, flags); ++ return HRTIMER_NORESTART; ++} ++ ++static void bfq_shutdown_timer_wq(struct bfq_data *bfqd) ++{ ++ hrtimer_cancel(&bfqd->idle_slice_timer); ++ cancel_work_sync(&bfqd->unplug_work); ++} ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static void __bfq_put_async_bfqq(struct bfq_data *bfqd, ++ struct bfq_queue **bfqq_ptr) ++{ ++ struct bfq_group *root_group = bfqd->root_group; ++ struct bfq_queue *bfqq = *bfqq_ptr; ++ ++ bfq_log(bfqd, "put_async_bfqq: %p", bfqq); ++ if (bfqq) { ++ bfq_bfqq_move(bfqd, bfqq, root_group); ++ bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", ++ bfqq, bfqq->ref); ++ bfq_put_queue(bfqq); ++ *bfqq_ptr = NULL; ++ } ++} ++ ++/* ++ * Release all the bfqg references to its async queues. If we are ++ * deallocating the group these queues may still contain requests, so ++ * we reparent them to the root cgroup (i.e., the only one that will ++ * exist for sure until all the requests on a device are gone). ++ */ ++static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg) ++{ ++ int i, j; ++ ++ for (i = 0; i < 2; i++) ++ for (j = 0; j < IOPRIO_BE_NR; j++) ++ __bfq_put_async_bfqq(bfqd, &bfqg->async_bfqq[i][j]); ++ ++ __bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq); ++} ++#endif ++ ++static void bfq_exit_queue(struct elevator_queue *e) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ struct request_queue *q = bfqd->queue; ++ struct bfq_queue *bfqq, *n; ++ ++ bfq_shutdown_timer_wq(bfqd); ++ ++ spin_lock_irq(q->queue_lock); ++ ++ BUG_ON(bfqd->in_service_queue); ++ list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list) ++ bfq_deactivate_bfqq(bfqd, bfqq, 0); ++ ++ spin_unlock_irq(q->queue_lock); ++ ++ bfq_shutdown_timer_wq(bfqd); ++ ++ BUG_ON(hrtimer_active(&bfqd->idle_slice_timer)); ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ blkcg_deactivate_policy(q, &blkcg_policy_bfq); ++#else ++ kfree(bfqd->root_group); ++#endif ++ ++ kfree(bfqd); ++} ++ ++static void bfq_init_root_group(struct bfq_group *root_group, ++ struct bfq_data *bfqd) ++{ ++ int i; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ root_group->entity.parent = NULL; ++ root_group->my_entity = NULL; ++ root_group->bfqd = bfqd; ++#endif ++ root_group->rq_pos_tree = RB_ROOT; ++ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) ++ root_group->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT; ++} ++ ++static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) ++{ ++ struct bfq_data *bfqd; ++ struct elevator_queue *eq; ++ ++ eq = elevator_alloc(q, e); ++ if (!eq) ++ return -ENOMEM; ++ ++ bfqd = kzalloc_node(sizeof(*bfqd), GFP_KERNEL, q->node); ++ if (!bfqd) { ++ kobject_put(&eq->kobj); ++ return -ENOMEM; ++ } ++ eq->elevator_data = bfqd; ++ ++ /* ++ * Our fallback bfqq if bfq_find_alloc_queue() runs into OOM issues. ++ * Grab a permanent reference to it, so that the normal code flow ++ * will not attempt to free it. ++ */ ++ bfq_init_bfqq(bfqd, &bfqd->oom_bfqq, NULL, 1, 0); ++ bfqd->oom_bfqq.ref++; ++ bfqd->oom_bfqq.new_ioprio = BFQ_DEFAULT_QUEUE_IOPRIO; ++ bfqd->oom_bfqq.new_ioprio_class = IOPRIO_CLASS_BE; ++ bfqd->oom_bfqq.entity.new_weight = ++ bfq_ioprio_to_weight(bfqd->oom_bfqq.new_ioprio); ++ ++ /* oom_bfqq does not participate to bursts */ ++ bfq_clear_bfqq_just_created(&bfqd->oom_bfqq); ++ /* ++ * Trigger weight initialization, according to ioprio, at the ++ * oom_bfqq's first activation. The oom_bfqq's ioprio and ioprio ++ * class won't be changed any more. ++ */ ++ bfqd->oom_bfqq.entity.prio_changed = 1; ++ ++ bfqd->queue = q; ++ ++ spin_lock_irq(q->queue_lock); ++ q->elevator = eq; ++ spin_unlock_irq(q->queue_lock); ++ ++ bfqd->root_group = bfq_create_group_hierarchy(bfqd, q->node); ++ if (!bfqd->root_group) ++ goto out_free; ++ bfq_init_root_group(bfqd->root_group, bfqd); ++ bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group); ++ ++ hrtimer_init(&bfqd->idle_slice_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ bfqd->idle_slice_timer.function = bfq_idle_slice_timer; ++ ++ bfqd->queue_weights_tree = RB_ROOT; ++ bfqd->group_weights_tree = RB_ROOT; ++ ++ INIT_WORK(&bfqd->unplug_work, bfq_kick_queue); ++ ++ INIT_LIST_HEAD(&bfqd->active_list); ++ INIT_LIST_HEAD(&bfqd->idle_list); ++ INIT_HLIST_HEAD(&bfqd->burst_list); ++ ++ bfqd->hw_tag = -1; ++ ++ bfqd->bfq_max_budget = bfq_default_max_budget; ++ ++ bfqd->bfq_fifo_expire[0] = bfq_fifo_expire[0]; ++ bfqd->bfq_fifo_expire[1] = bfq_fifo_expire[1]; ++ bfqd->bfq_back_max = bfq_back_max; ++ bfqd->bfq_back_penalty = bfq_back_penalty; ++ bfqd->bfq_slice_idle = bfq_slice_idle; ++ bfqd->bfq_class_idle_last_service = 0; ++ bfqd->bfq_timeout = bfq_timeout; ++ ++ bfqd->bfq_requests_within_timer = 120; ++ ++ bfqd->bfq_large_burst_thresh = 8; ++ bfqd->bfq_burst_interval = msecs_to_jiffies(180); ++ ++ bfqd->low_latency = true; ++ ++ /* ++ * Trade-off between responsiveness and fairness. ++ */ ++ bfqd->bfq_wr_coeff = 30; ++ bfqd->bfq_wr_rt_max_time = msecs_to_jiffies(300); ++ bfqd->bfq_wr_max_time = 0; ++ bfqd->bfq_wr_min_idle_time = msecs_to_jiffies(2000); ++ bfqd->bfq_wr_min_inter_arr_async = msecs_to_jiffies(500); ++ bfqd->bfq_wr_max_softrt_rate = 7000; /* ++ * Approximate rate required ++ * to playback or record a ++ * high-definition compressed ++ * video. ++ */ ++ bfqd->wr_busy_queues = 0; ++ ++ /* ++ * Begin by assuming, optimistically, that the device is a ++ * high-speed one, and that its peak rate is equal to 2/3 of ++ * the highest reference rate. ++ */ ++ bfqd->RT_prod = R_fast[blk_queue_nonrot(bfqd->queue)] * ++ T_fast[blk_queue_nonrot(bfqd->queue)]; ++ bfqd->peak_rate = R_fast[blk_queue_nonrot(bfqd->queue)] * 2 / 3; ++ bfqd->device_speed = BFQ_BFQD_FAST; ++ ++ return 0; ++ ++out_free: ++ kfree(bfqd); ++ kobject_put(&eq->kobj); ++ return -ENOMEM; ++} ++ ++static void bfq_slab_kill(void) ++{ ++ kmem_cache_destroy(bfq_pool); ++} ++ ++static int __init bfq_slab_setup(void) ++{ ++ bfq_pool = KMEM_CACHE(bfq_queue, 0); ++ if (!bfq_pool) ++ return -ENOMEM; ++ return 0; ++} ++ ++static ssize_t bfq_var_show(unsigned int var, char *page) ++{ ++ return sprintf(page, "%u\n", var); ++} ++ ++static ssize_t bfq_var_store(unsigned long *var, const char *page, ++ size_t count) ++{ ++ unsigned long new_val; ++ int ret = kstrtoul(page, 10, &new_val); ++ ++ if (ret == 0) ++ *var = new_val; ++ ++ return count; ++} ++ ++static ssize_t bfq_wr_max_time_show(struct elevator_queue *e, char *page) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ ++ return sprintf(page, "%d\n", bfqd->bfq_wr_max_time > 0 ? ++ jiffies_to_msecs(bfqd->bfq_wr_max_time) : ++ jiffies_to_msecs(bfq_wr_duration(bfqd))); ++} ++ ++static ssize_t bfq_weights_show(struct elevator_queue *e, char *page) ++{ ++ struct bfq_queue *bfqq; ++ struct bfq_data *bfqd = e->elevator_data; ++ ssize_t num_char = 0; ++ ++ num_char += sprintf(page + num_char, "Tot reqs queued %d\n\n", ++ bfqd->queued); ++ ++ spin_lock_irq(bfqd->queue->queue_lock); ++ ++ num_char += sprintf(page + num_char, "Active:\n"); ++ list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) { ++ num_char += sprintf(page + num_char, ++ "pid%d: weight %hu, nr_queued %d %d, ", ++ bfqq->pid, ++ bfqq->entity.weight, ++ bfqq->queued[0], ++ bfqq->queued[1]); ++ num_char += sprintf(page + num_char, ++ "dur %d/%u\n", ++ jiffies_to_msecs( ++ jiffies - ++ bfqq->last_wr_start_finish), ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ } ++ ++ num_char += sprintf(page + num_char, "Idle:\n"); ++ list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) { ++ num_char += sprintf(page + num_char, ++ "pid%d: weight %hu, dur %d/%u\n", ++ bfqq->pid, ++ bfqq->entity.weight, ++ jiffies_to_msecs(jiffies - ++ bfqq->last_wr_start_finish), ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ } ++ ++ spin_unlock_irq(bfqd->queue->queue_lock); ++ ++ return num_char; ++} ++ ++#define SHOW_FUNCTION(__FUNC, __VAR, __CONV) \ ++static ssize_t __FUNC(struct elevator_queue *e, char *page) \ ++{ \ ++ struct bfq_data *bfqd = e->elevator_data; \ ++ u64 __data = __VAR; \ ++ if (__CONV == 1) \ ++ __data = jiffies_to_msecs(__data); \ ++ else if (__CONV == 2) \ ++ __data = div_u64(__data, NSEC_PER_MSEC); \ ++ return bfq_var_show(__data, (page)); \ ++} ++SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 2); ++SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 2); ++SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0); ++SHOW_FUNCTION(bfq_back_seek_penalty_show, bfqd->bfq_back_penalty, 0); ++SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 2); ++SHOW_FUNCTION(bfq_max_budget_show, bfqd->bfq_user_max_budget, 0); ++SHOW_FUNCTION(bfq_timeout_sync_show, bfqd->bfq_timeout, 1); ++SHOW_FUNCTION(bfq_strict_guarantees_show, bfqd->strict_guarantees, 0); ++SHOW_FUNCTION(bfq_low_latency_show, bfqd->low_latency, 0); ++SHOW_FUNCTION(bfq_wr_coeff_show, bfqd->bfq_wr_coeff, 0); ++SHOW_FUNCTION(bfq_wr_rt_max_time_show, bfqd->bfq_wr_rt_max_time, 1); ++SHOW_FUNCTION(bfq_wr_min_idle_time_show, bfqd->bfq_wr_min_idle_time, 1); ++SHOW_FUNCTION(bfq_wr_min_inter_arr_async_show, bfqd->bfq_wr_min_inter_arr_async, ++ 1); ++SHOW_FUNCTION(bfq_wr_max_softrt_rate_show, bfqd->bfq_wr_max_softrt_rate, 0); ++#undef SHOW_FUNCTION ++ ++#define USEC_SHOW_FUNCTION(__FUNC, __VAR) \ ++static ssize_t __FUNC(struct elevator_queue *e, char *page) \ ++{ \ ++ struct bfq_data *bfqd = e->elevator_data; \ ++ u64 __data = __VAR; \ ++ __data = div_u64(__data, NSEC_PER_USEC); \ ++ return bfq_var_show(__data, (page)); \ ++} ++USEC_SHOW_FUNCTION(bfq_slice_idle_us_show, bfqd->bfq_slice_idle); ++#undef USEC_SHOW_FUNCTION ++ ++#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \ ++static ssize_t \ ++__FUNC(struct elevator_queue *e, const char *page, size_t count) \ ++{ \ ++ struct bfq_data *bfqd = e->elevator_data; \ ++ unsigned long uninitialized_var(__data); \ ++ int ret = bfq_var_store(&__data, (page), count); \ ++ if (__data < (MIN)) \ ++ __data = (MIN); \ ++ else if (__data > (MAX)) \ ++ __data = (MAX); \ ++ if (__CONV == 1) \ ++ *(__PTR) = msecs_to_jiffies(__data); \ ++ else if (__CONV == 2) \ ++ *(__PTR) = (u64)__data * NSEC_PER_MSEC; \ ++ else \ ++ *(__PTR) = __data; \ ++ return ret; \ ++} ++STORE_FUNCTION(bfq_fifo_expire_sync_store, &bfqd->bfq_fifo_expire[1], 1, ++ INT_MAX, 2); ++STORE_FUNCTION(bfq_fifo_expire_async_store, &bfqd->bfq_fifo_expire[0], 1, ++ INT_MAX, 2); ++STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0); ++STORE_FUNCTION(bfq_back_seek_penalty_store, &bfqd->bfq_back_penalty, 1, ++ INT_MAX, 0); ++STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 2); ++STORE_FUNCTION(bfq_wr_coeff_store, &bfqd->bfq_wr_coeff, 1, INT_MAX, 0); ++STORE_FUNCTION(bfq_wr_max_time_store, &bfqd->bfq_wr_max_time, 0, INT_MAX, 1); ++STORE_FUNCTION(bfq_wr_rt_max_time_store, &bfqd->bfq_wr_rt_max_time, 0, INT_MAX, ++ 1); ++STORE_FUNCTION(bfq_wr_min_idle_time_store, &bfqd->bfq_wr_min_idle_time, 0, ++ INT_MAX, 1); ++STORE_FUNCTION(bfq_wr_min_inter_arr_async_store, ++ &bfqd->bfq_wr_min_inter_arr_async, 0, INT_MAX, 1); ++STORE_FUNCTION(bfq_wr_max_softrt_rate_store, &bfqd->bfq_wr_max_softrt_rate, 0, ++ INT_MAX, 0); ++#undef STORE_FUNCTION ++ ++#define USEC_STORE_FUNCTION(__FUNC, __PTR, MIN, MAX) \ ++static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count)\ ++{ \ ++ struct bfq_data *bfqd = e->elevator_data; \ ++ unsigned long __data; \ ++ int ret = bfq_var_store(&__data, (page), count); \ ++ if (__data < (MIN)) \ ++ __data = (MIN); \ ++ else if (__data > (MAX)) \ ++ __data = (MAX); \ ++ *(__PTR) = (u64)__data * NSEC_PER_USEC; \ ++ return ret; \ ++} ++USEC_STORE_FUNCTION(bfq_slice_idle_us_store, &bfqd->bfq_slice_idle, 0, ++ UINT_MAX); ++#undef USEC_STORE_FUNCTION ++ ++/* do nothing for the moment */ ++static ssize_t bfq_weights_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ return count; ++} ++ ++static ssize_t bfq_max_budget_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ unsigned long uninitialized_var(__data); ++ int ret = bfq_var_store(&__data, (page), count); ++ ++ if (__data == 0) ++ bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); ++ else { ++ if (__data > INT_MAX) ++ __data = INT_MAX; ++ bfqd->bfq_max_budget = __data; ++ } ++ ++ bfqd->bfq_user_max_budget = __data; ++ ++ return ret; ++} ++ ++/* ++ * Leaving this name to preserve name compatibility with cfq ++ * parameters, but this timeout is used for both sync and async. ++ */ ++static ssize_t bfq_timeout_sync_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ unsigned long uninitialized_var(__data); ++ int ret = bfq_var_store(&__data, (page), count); ++ ++ if (__data < 1) ++ __data = 1; ++ else if (__data > INT_MAX) ++ __data = INT_MAX; ++ ++ bfqd->bfq_timeout = msecs_to_jiffies(__data); ++ if (bfqd->bfq_user_max_budget == 0) ++ bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); ++ ++ return ret; ++} ++ ++static ssize_t bfq_strict_guarantees_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ unsigned long uninitialized_var(__data); ++ int ret = bfq_var_store(&__data, (page), count); ++ ++ if (__data > 1) ++ __data = 1; ++ if (!bfqd->strict_guarantees && __data == 1 ++ && bfqd->bfq_slice_idle < 8 * NSEC_PER_MSEC) ++ bfqd->bfq_slice_idle = 8 * NSEC_PER_MSEC; ++ ++ bfqd->strict_guarantees = __data; ++ ++ return ret; ++} ++ ++static ssize_t bfq_low_latency_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ unsigned long uninitialized_var(__data); ++ int ret = bfq_var_store(&__data, (page), count); ++ ++ if (__data > 1) ++ __data = 1; ++ if (__data == 0 && bfqd->low_latency != 0) ++ bfq_end_wr(bfqd); ++ bfqd->low_latency = __data; ++ ++ return ret; ++} ++ ++#define BFQ_ATTR(name) \ ++ __ATTR(name, S_IRUGO|S_IWUSR, bfq_##name##_show, bfq_##name##_store) ++ ++static struct elv_fs_entry bfq_attrs[] = { ++ BFQ_ATTR(fifo_expire_sync), ++ BFQ_ATTR(fifo_expire_async), ++ BFQ_ATTR(back_seek_max), ++ BFQ_ATTR(back_seek_penalty), ++ BFQ_ATTR(slice_idle), ++ BFQ_ATTR(slice_idle_us), ++ BFQ_ATTR(max_budget), ++ BFQ_ATTR(timeout_sync), ++ BFQ_ATTR(strict_guarantees), ++ BFQ_ATTR(low_latency), ++ BFQ_ATTR(wr_coeff), ++ BFQ_ATTR(wr_max_time), ++ BFQ_ATTR(wr_rt_max_time), ++ BFQ_ATTR(wr_min_idle_time), ++ BFQ_ATTR(wr_min_inter_arr_async), ++ BFQ_ATTR(wr_max_softrt_rate), ++ BFQ_ATTR(weights), ++ __ATTR_NULL ++}; ++ ++static struct elevator_type iosched_bfq = { ++ .ops = { ++ .elevator_merge_fn = bfq_merge, ++ .elevator_merged_fn = bfq_merged_request, ++ .elevator_merge_req_fn = bfq_merged_requests, ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ .elevator_bio_merged_fn = bfq_bio_merged, ++#endif ++ .elevator_allow_bio_merge_fn = bfq_allow_bio_merge, ++ .elevator_allow_rq_merge_fn = bfq_allow_rq_merge, ++ .elevator_dispatch_fn = bfq_dispatch_requests, ++ .elevator_add_req_fn = bfq_insert_request, ++ .elevator_activate_req_fn = bfq_activate_request, ++ .elevator_deactivate_req_fn = bfq_deactivate_request, ++ .elevator_completed_req_fn = bfq_completed_request, ++ .elevator_former_req_fn = elv_rb_former_request, ++ .elevator_latter_req_fn = elv_rb_latter_request, ++ .elevator_init_icq_fn = bfq_init_icq, ++ .elevator_exit_icq_fn = bfq_exit_icq, ++ .elevator_set_req_fn = bfq_set_request, ++ .elevator_put_req_fn = bfq_put_request, ++ .elevator_may_queue_fn = bfq_may_queue, ++ .elevator_init_fn = bfq_init_queue, ++ .elevator_exit_fn = bfq_exit_queue, ++ }, ++ .icq_size = sizeof(struct bfq_io_cq), ++ .icq_align = __alignof__(struct bfq_io_cq), ++ .elevator_attrs = bfq_attrs, ++ .elevator_name = "bfq", ++ .elevator_owner = THIS_MODULE, ++}; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static struct blkcg_policy blkcg_policy_bfq = { ++ .dfl_cftypes = bfq_blkg_files, ++ .legacy_cftypes = bfq_blkcg_legacy_files, ++ ++ .cpd_alloc_fn = bfq_cpd_alloc, ++ .cpd_init_fn = bfq_cpd_init, ++ .cpd_bind_fn = bfq_cpd_init, ++ .cpd_free_fn = bfq_cpd_free, ++ ++ .pd_alloc_fn = bfq_pd_alloc, ++ .pd_init_fn = bfq_pd_init, ++ .pd_offline_fn = bfq_pd_offline, ++ .pd_free_fn = bfq_pd_free, ++ .pd_reset_stats_fn = bfq_pd_reset_stats, ++}; ++#endif ++ ++static int __init bfq_init(void) ++{ ++ int ret; ++ char msg[50] = "BFQ I/O-scheduler: v8r4"; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ ret = blkcg_policy_register(&blkcg_policy_bfq); ++ if (ret) ++ return ret; ++#endif ++ ++ ret = -ENOMEM; ++ if (bfq_slab_setup()) ++ goto err_pol_unreg; ++ ++ /* ++ * Times to load large popular applications for the typical ++ * systems installed on the reference devices (see the ++ * comments before the definitions of the next two ++ * arrays). Actually, we use slightly slower values, as the ++ * estimated peak rate tends to be smaller than the actual ++ * peak rate. The reason for this last fact is that estimates ++ * are computed over much shorter time intervals than the long ++ * intervals typically used for benchmarking. Why? First, to ++ * adapt more quickly to variations. Second, because an I/O ++ * scheduler cannot rely on a peak-rate-evaluation workload to ++ * be run for a long time. ++ */ ++ T_slow[0] = msecs_to_jiffies(3500); /* actually 4 sec */ ++ T_slow[1] = msecs_to_jiffies(1000); /* actually 1.5 sec */ ++ T_fast[0] = msecs_to_jiffies(7000); /* actually 8 sec */ ++ T_fast[1] = msecs_to_jiffies(2500); /* actually 3 sec */ ++ ++ /* ++ * Thresholds that determine the switch between speed classes ++ * (see the comments before the definition of the array ++ * device_speed_thresh). These thresholds are biased towards ++ * transitions to the fast class. This is safer than the ++ * opposite bias. In fact, a wrong transition to the slow ++ * class results in short weight-raising periods, because the ++ * speed of the device then tends to be higher that the ++ * reference peak rate. On the opposite end, a wrong ++ * transition to the fast class tends to increase ++ * weight-raising periods, because of the opposite reason. ++ */ ++ device_speed_thresh[0] = (4 * R_slow[0]) / 3; ++ device_speed_thresh[1] = (4 * R_slow[1]) / 3; ++ ++ ret = elv_register(&iosched_bfq); ++ if (ret) ++ goto err_pol_unreg; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ strcat(msg, " (with cgroups support)"); ++#endif ++ pr_info("%s", msg); ++ ++ return 0; ++ ++err_pol_unreg: ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ blkcg_policy_unregister(&blkcg_policy_bfq); ++#endif ++ return ret; ++} ++ ++static void __exit bfq_exit(void) ++{ ++ elv_unregister(&iosched_bfq); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ blkcg_policy_unregister(&blkcg_policy_bfq); ++#endif ++ bfq_slab_kill(); ++} ++ ++module_init(bfq_init); ++module_exit(bfq_exit); ++ ++MODULE_AUTHOR("Arianna Avanzini, Fabio Checconi, Paolo Valente"); ++MODULE_LICENSE("GPL"); +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq-sched.c linux-4.8-bfq/block/bfq-sched.c +--- linux-4.8/block/bfq-sched.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq-sched.c 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,1501 @@ ++/* ++ * BFQ: Hierarchical B-WF2Q+ scheduler. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2015 Paolo Valente ++ * ++ * Copyright (C) 2016 Paolo Valente ++ */ ++ ++static struct bfq_group *bfqq_group(struct bfq_queue *bfqq); ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++#define for_each_entity(entity) \ ++ for (; entity ; entity = entity->parent) ++ ++#define for_each_entity_safe(entity, parent) \ ++ for (; entity && ({ parent = entity->parent; 1; }); entity = parent) ++ ++ ++static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, ++ int extract, ++ struct bfq_data *bfqd); ++ ++static void bfq_update_budget(struct bfq_entity *next_in_service) ++{ ++ struct bfq_entity *bfqg_entity; ++ struct bfq_group *bfqg; ++ struct bfq_sched_data *group_sd; ++ ++ BUG_ON(!next_in_service); ++ ++ group_sd = next_in_service->sched_data; ++ ++ bfqg = container_of(group_sd, struct bfq_group, sched_data); ++ /* ++ * bfq_group's my_entity field is not NULL only if the group ++ * is not the root group. We must not touch the root entity ++ * as it must never become an in-service entity. ++ */ ++ bfqg_entity = bfqg->my_entity; ++ if (bfqg_entity) ++ bfqg_entity->budget = next_in_service->budget; ++} ++ ++static int bfq_update_next_in_service(struct bfq_sched_data *sd) ++{ ++ struct bfq_entity *next_in_service; ++ struct bfq_queue *bfqq; ++ ++ if (sd->in_service_entity) ++ /* will update/requeue at the end of service */ ++ return 0; ++ ++ /* ++ * NOTE: this can be improved in many ways, such as returning ++ * 1 (and thus propagating upwards the update) only when the ++ * budget changes, or caching the bfqq that will be scheduled ++ * next from this subtree. By now we worry more about ++ * correctness than about performance... ++ */ ++ next_in_service = bfq_lookup_next_entity(sd, 0, NULL); ++ sd->next_in_service = next_in_service; ++ ++ if (next_in_service) ++ bfq_update_budget(next_in_service); ++ else ++ goto exit; ++ ++ bfqq = bfq_entity_to_bfqq(next_in_service); ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "update_next_in_service: chosen this queue"); ++ else { ++ struct bfq_group *bfqg = ++ container_of(next_in_service, ++ struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "update_next_in_service: chosen this entity"); ++ } ++exit: ++ return 1; ++} ++ ++static void bfq_check_next_in_service(struct bfq_sched_data *sd, ++ struct bfq_entity *entity) ++{ ++ WARN_ON(sd->next_in_service != entity); ++} ++#else ++#define for_each_entity(entity) \ ++ for (; entity ; entity = NULL) ++ ++#define for_each_entity_safe(entity, parent) \ ++ for (parent = NULL; entity ; entity = parent) ++ ++static int bfq_update_next_in_service(struct bfq_sched_data *sd) ++{ ++ return 0; ++} ++ ++static void bfq_check_next_in_service(struct bfq_sched_data *sd, ++ struct bfq_entity *entity) ++{ ++} ++ ++static void bfq_update_budget(struct bfq_entity *next_in_service) ++{ ++} ++#endif ++ ++/* ++ * Shift for timestamp calculations. This actually limits the maximum ++ * service allowed in one timestamp delta (small shift values increase it), ++ * the maximum total weight that can be used for the queues in the system ++ * (big shift values increase it), and the period of virtual time ++ * wraparounds. ++ */ ++#define WFQ_SERVICE_SHIFT 22 ++ ++/** ++ * bfq_gt - compare two timestamps. ++ * @a: first ts. ++ * @b: second ts. ++ * ++ * Return @a > @b, dealing with wrapping correctly. ++ */ ++static int bfq_gt(u64 a, u64 b) ++{ ++ return (s64)(a - b) > 0; ++} ++ ++static struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = NULL; ++ ++ BUG_ON(!entity); ++ ++ if (!entity->my_sched_data) ++ bfqq = container_of(entity, struct bfq_queue, entity); ++ ++ return bfqq; ++} ++ ++ ++/** ++ * bfq_delta - map service into the virtual time domain. ++ * @service: amount of service. ++ * @weight: scale factor (weight of an entity or weight sum). ++ */ ++static u64 bfq_delta(unsigned long service, unsigned long weight) ++{ ++ u64 d = (u64)service << WFQ_SERVICE_SHIFT; ++ ++ do_div(d, weight); ++ return d; ++} ++ ++/** ++ * bfq_calc_finish - assign the finish time to an entity. ++ * @entity: the entity to act upon. ++ * @service: the service to be charged to the entity. ++ */ ++static void bfq_calc_finish(struct bfq_entity *entity, unsigned long service) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ unsigned long long start, finish, delta; ++ ++ BUG_ON(entity->weight == 0); ++ ++ entity->finish = entity->start + ++ bfq_delta(service, entity->weight); ++ ++ start = ((entity->start>>10)*1000)>>12; ++ finish = ((entity->finish>>10)*1000)>>12; ++ delta = ((bfq_delta(service, entity->weight)>>10)*1000)>>12; ++ ++ if (bfqq) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "calc_finish: serv %lu, w %d", ++ service, entity->weight); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "calc_finish: start %llu, finish %llu, delta %llu", ++ start, finish, delta); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ } else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "calc_finish group: serv %lu, w %d", ++ service, entity->weight); ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "calc_finish group: start %llu, finish %llu, delta %llu", ++ start, finish, delta); ++#endif ++ } ++} ++ ++/** ++ * bfq_entity_of - get an entity from a node. ++ * @node: the node field of the entity. ++ * ++ * Convert a node pointer to the relative entity. This is used only ++ * to simplify the logic of some functions and not as the generic ++ * conversion mechanism because, e.g., in the tree walking functions, ++ * the check for a %NULL value would be redundant. ++ */ ++static struct bfq_entity *bfq_entity_of(struct rb_node *node) ++{ ++ struct bfq_entity *entity = NULL; ++ ++ if (node) ++ entity = rb_entry(node, struct bfq_entity, rb_node); ++ ++ return entity; ++} ++ ++/** ++ * bfq_extract - remove an entity from a tree. ++ * @root: the tree root. ++ * @entity: the entity to remove. ++ */ ++static void bfq_extract(struct rb_root *root, struct bfq_entity *entity) ++{ ++ BUG_ON(entity->tree != root); ++ ++ entity->tree = NULL; ++ rb_erase(&entity->rb_node, root); ++} ++ ++/** ++ * bfq_idle_extract - extract an entity from the idle tree. ++ * @st: the service tree of the owning @entity. ++ * @entity: the entity being removed. ++ */ ++static void bfq_idle_extract(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct rb_node *next; ++ ++ BUG_ON(entity->tree != &st->idle); ++ ++ if (entity == st->first_idle) { ++ next = rb_next(&entity->rb_node); ++ st->first_idle = bfq_entity_of(next); ++ } ++ ++ if (entity == st->last_idle) { ++ next = rb_prev(&entity->rb_node); ++ st->last_idle = bfq_entity_of(next); ++ } ++ ++ bfq_extract(&st->idle, entity); ++ ++ if (bfqq) ++ list_del(&bfqq->bfqq_list); ++} ++ ++/** ++ * bfq_insert - generic tree insertion. ++ * @root: tree root. ++ * @entity: entity to insert. ++ * ++ * This is used for the idle and the active tree, since they are both ++ * ordered by finish time. ++ */ ++static void bfq_insert(struct rb_root *root, struct bfq_entity *entity) ++{ ++ struct bfq_entity *entry; ++ struct rb_node **node = &root->rb_node; ++ struct rb_node *parent = NULL; ++ ++ BUG_ON(entity->tree); ++ ++ while (*node) { ++ parent = *node; ++ entry = rb_entry(parent, struct bfq_entity, rb_node); ++ ++ if (bfq_gt(entry->finish, entity->finish)) ++ node = &parent->rb_left; ++ else ++ node = &parent->rb_right; ++ } ++ ++ rb_link_node(&entity->rb_node, parent, node); ++ rb_insert_color(&entity->rb_node, root); ++ ++ entity->tree = root; ++} ++ ++/** ++ * bfq_update_min - update the min_start field of a entity. ++ * @entity: the entity to update. ++ * @node: one of its children. ++ * ++ * This function is called when @entity may store an invalid value for ++ * min_start due to updates to the active tree. The function assumes ++ * that the subtree rooted at @node (which may be its left or its right ++ * child) has a valid min_start value. ++ */ ++static void bfq_update_min(struct bfq_entity *entity, struct rb_node *node) ++{ ++ struct bfq_entity *child; ++ ++ if (node) { ++ child = rb_entry(node, struct bfq_entity, rb_node); ++ if (bfq_gt(entity->min_start, child->min_start)) ++ entity->min_start = child->min_start; ++ } ++} ++ ++/** ++ * bfq_update_active_node - recalculate min_start. ++ * @node: the node to update. ++ * ++ * @node may have changed position or one of its children may have moved, ++ * this function updates its min_start value. The left and right subtrees ++ * are assumed to hold a correct min_start value. ++ */ ++static void bfq_update_active_node(struct rb_node *node) ++{ ++ struct bfq_entity *entity = rb_entry(node, struct bfq_entity, rb_node); ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ entity->min_start = entity->start; ++ bfq_update_min(entity, node->rb_right); ++ bfq_update_min(entity, node->rb_left); ++ ++ if (bfqq) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "update_active_node: new min_start %llu", ++ ((entity->min_start>>10)*1000)>>12); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ } else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "update_active_node: new min_start %llu", ++ ((entity->min_start>>10)*1000)>>12); ++#endif ++ } ++} ++ ++/** ++ * bfq_update_active_tree - update min_start for the whole active tree. ++ * @node: the starting node. ++ * ++ * @node must be the deepest modified node after an update. This function ++ * updates its min_start using the values held by its children, assuming ++ * that they did not change, and then updates all the nodes that may have ++ * changed in the path to the root. The only nodes that may have changed ++ * are the ones in the path or their siblings. ++ */ ++static void bfq_update_active_tree(struct rb_node *node) ++{ ++ struct rb_node *parent; ++ ++up: ++ bfq_update_active_node(node); ++ ++ parent = rb_parent(node); ++ if (!parent) ++ return; ++ ++ if (node == parent->rb_left && parent->rb_right) ++ bfq_update_active_node(parent->rb_right); ++ else if (parent->rb_left) ++ bfq_update_active_node(parent->rb_left); ++ ++ node = parent; ++ goto up; ++} ++ ++static void bfq_weights_tree_add(struct bfq_data *bfqd, ++ struct bfq_entity *entity, ++ struct rb_root *root); ++ ++static void bfq_weights_tree_remove(struct bfq_data *bfqd, ++ struct bfq_entity *entity, ++ struct rb_root *root); ++ ++ ++/** ++ * bfq_active_insert - insert an entity in the active tree of its ++ * group/device. ++ * @st: the service tree of the entity. ++ * @entity: the entity being inserted. ++ * ++ * The active tree is ordered by finish time, but an extra key is kept ++ * per each node, containing the minimum value for the start times of ++ * its children (and the node itself), so it's possible to search for ++ * the eligible node with the lowest finish time in logarithmic time. ++ */ ++static void bfq_active_insert(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct rb_node *node = &entity->rb_node; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ struct bfq_sched_data *sd = NULL; ++ struct bfq_group *bfqg = NULL; ++ struct bfq_data *bfqd = NULL; ++#endif ++ ++ bfq_insert(&st->active, entity); ++ ++ if (node->rb_left) ++ node = node->rb_left; ++ else if (node->rb_right) ++ node = node->rb_right; ++ ++ bfq_update_active_tree(node); ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ sd = entity->sched_data; ++ bfqg = container_of(sd, struct bfq_group, sched_data); ++ BUG_ON(!bfqg); ++ bfqd = (struct bfq_data *)bfqg->bfqd; ++#endif ++ if (bfqq) ++ list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { /* bfq_group */ ++ BUG_ON(!bfqd); ++ bfq_weights_tree_add(bfqd, entity, &bfqd->group_weights_tree); ++ } ++ if (bfqg != bfqd->root_group) { ++ BUG_ON(!bfqg); ++ BUG_ON(!bfqd); ++ bfqg->active_entities++; ++ } ++#endif ++} ++ ++/** ++ * bfq_ioprio_to_weight - calc a weight from an ioprio. ++ * @ioprio: the ioprio value to convert. ++ */ ++static unsigned short bfq_ioprio_to_weight(int ioprio) ++{ ++ BUG_ON(ioprio < 0 || ioprio >= IOPRIO_BE_NR); ++ return (IOPRIO_BE_NR - ioprio) * BFQ_WEIGHT_CONVERSION_COEFF; ++} ++ ++/** ++ * bfq_weight_to_ioprio - calc an ioprio from a weight. ++ * @weight: the weight value to convert. ++ * ++ * To preserve as much as possible the old only-ioprio user interface, ++ * 0 is used as an escape ioprio value for weights (numerically) equal or ++ * larger than IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF. ++ */ ++static unsigned short bfq_weight_to_ioprio(int weight) ++{ ++ BUG_ON(weight < BFQ_MIN_WEIGHT || weight > BFQ_MAX_WEIGHT); ++ return IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - weight < 0 ? ++ 0 : IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - weight; ++} ++ ++static void bfq_get_entity(struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ if (bfqq) { ++ bfqq->ref++; ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "get_entity: %p %d", ++ bfqq, bfqq->ref); ++ } ++} ++ ++/** ++ * bfq_find_deepest - find the deepest node that an extraction can modify. ++ * @node: the node being removed. ++ * ++ * Do the first step of an extraction in an rb tree, looking for the ++ * node that will replace @node, and returning the deepest node that ++ * the following modifications to the tree can touch. If @node is the ++ * last node in the tree return %NULL. ++ */ ++static struct rb_node *bfq_find_deepest(struct rb_node *node) ++{ ++ struct rb_node *deepest; ++ ++ if (!node->rb_right && !node->rb_left) ++ deepest = rb_parent(node); ++ else if (!node->rb_right) ++ deepest = node->rb_left; ++ else if (!node->rb_left) ++ deepest = node->rb_right; ++ else { ++ deepest = rb_next(node); ++ if (deepest->rb_right) ++ deepest = deepest->rb_right; ++ else if (rb_parent(deepest) != node) ++ deepest = rb_parent(deepest); ++ } ++ ++ return deepest; ++} ++ ++/** ++ * bfq_active_extract - remove an entity from the active tree. ++ * @st: the service_tree containing the tree. ++ * @entity: the entity being removed. ++ */ ++static void bfq_active_extract(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct rb_node *node; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ struct bfq_sched_data *sd = NULL; ++ struct bfq_group *bfqg = NULL; ++ struct bfq_data *bfqd = NULL; ++#endif ++ ++ node = bfq_find_deepest(&entity->rb_node); ++ bfq_extract(&st->active, entity); ++ ++ if (node) ++ bfq_update_active_tree(node); ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ sd = entity->sched_data; ++ bfqg = container_of(sd, struct bfq_group, sched_data); ++ BUG_ON(!bfqg); ++ bfqd = (struct bfq_data *)bfqg->bfqd; ++#endif ++ if (bfqq) ++ list_del(&bfqq->bfqq_list); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { /* bfq_group */ ++ BUG_ON(!bfqd); ++ bfq_weights_tree_remove(bfqd, entity, ++ &bfqd->group_weights_tree); ++ } ++ if (bfqg != bfqd->root_group) { ++ BUG_ON(!bfqg); ++ BUG_ON(!bfqd); ++ BUG_ON(!bfqg->active_entities); ++ bfqg->active_entities--; ++ } ++#endif ++} ++ ++/** ++ * bfq_idle_insert - insert an entity into the idle tree. ++ * @st: the service tree containing the tree. ++ * @entity: the entity to insert. ++ */ ++static void bfq_idle_insert(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct bfq_entity *first_idle = st->first_idle; ++ struct bfq_entity *last_idle = st->last_idle; ++ ++ if (!first_idle || bfq_gt(first_idle->finish, entity->finish)) ++ st->first_idle = entity; ++ if (!last_idle || bfq_gt(entity->finish, last_idle->finish)) ++ st->last_idle = entity; ++ ++ bfq_insert(&st->idle, entity); ++ ++ if (bfqq) ++ list_add(&bfqq->bfqq_list, &bfqq->bfqd->idle_list); ++} ++ ++/** ++ * bfq_forget_entity - remove an entity from the wfq trees. ++ * @st: the service tree. ++ * @entity: the entity being removed. ++ * ++ * Update the device status and forget everything about @entity, putting ++ * the device reference to it, if it is a queue. Entities belonging to ++ * groups are not refcounted. ++ */ ++static void bfq_forget_entity(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct bfq_sched_data *sd; ++ ++ BUG_ON(!entity->on_st); ++ ++ entity->on_st = 0; ++ st->wsum -= entity->weight; ++ if (bfqq) { ++ sd = entity->sched_data; ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "forget_entity: %p %d", ++ bfqq, bfqq->ref); ++ bfq_put_queue(bfqq); ++ } ++} ++ ++/** ++ * bfq_put_idle_entity - release the idle tree ref of an entity. ++ * @st: service tree for the entity. ++ * @entity: the entity being released. ++ */ ++static void bfq_put_idle_entity(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ bfq_idle_extract(st, entity); ++ bfq_forget_entity(st, entity); ++} ++ ++/** ++ * bfq_forget_idle - update the idle tree if necessary. ++ * @st: the service tree to act upon. ++ * ++ * To preserve the global O(log N) complexity we only remove one entry here; ++ * as the idle tree will not grow indefinitely this can be done safely. ++ */ ++static void bfq_forget_idle(struct bfq_service_tree *st) ++{ ++ struct bfq_entity *first_idle = st->first_idle; ++ struct bfq_entity *last_idle = st->last_idle; ++ ++ if (RB_EMPTY_ROOT(&st->active) && last_idle && ++ !bfq_gt(last_idle->finish, st->vtime)) { ++ /* ++ * Forget the whole idle tree, increasing the vtime past ++ * the last finish time of idle entities. ++ */ ++ st->vtime = last_idle->finish; ++ } ++ ++ if (first_idle && !bfq_gt(first_idle->finish, st->vtime)) ++ bfq_put_idle_entity(st, first_idle); ++} ++ ++static struct bfq_service_tree * ++__bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_service_tree *new_st = old_st; ++ ++ if (entity->prio_changed) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ unsigned int prev_weight, new_weight; ++ struct bfq_data *bfqd = NULL; ++ struct rb_root *root; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ struct bfq_sched_data *sd; ++ struct bfq_group *bfqg; ++#endif ++ ++ if (bfqq) ++ bfqd = bfqq->bfqd; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ sd = entity->my_sched_data; ++ bfqg = container_of(sd, struct bfq_group, sched_data); ++ BUG_ON(!bfqg); ++ bfqd = (struct bfq_data *)bfqg->bfqd; ++ BUG_ON(!bfqd); ++ } ++#endif ++ ++ BUG_ON(old_st->wsum < entity->weight); ++ old_st->wsum -= entity->weight; ++ ++ if (entity->new_weight != entity->orig_weight) { ++ if (entity->new_weight < BFQ_MIN_WEIGHT || ++ entity->new_weight > BFQ_MAX_WEIGHT) { ++ pr_crit("update_weight_prio: new_weight %d\n", ++ entity->new_weight); ++ if (entity->new_weight < BFQ_MIN_WEIGHT) ++ entity->new_weight = BFQ_MIN_WEIGHT; ++ else ++ entity->new_weight = BFQ_MAX_WEIGHT; ++ } ++ entity->orig_weight = entity->new_weight; ++ if (bfqq) ++ bfqq->ioprio = ++ bfq_weight_to_ioprio(entity->orig_weight); ++ } ++ ++ if (bfqq) ++ bfqq->ioprio_class = bfqq->new_ioprio_class; ++ entity->prio_changed = 0; ++ ++ /* ++ * NOTE: here we may be changing the weight too early, ++ * this will cause unfairness. The correct approach ++ * would have required additional complexity to defer ++ * weight changes to the proper time instants (i.e., ++ * when entity->finish <= old_st->vtime). ++ */ ++ new_st = bfq_entity_service_tree(entity); ++ ++ prev_weight = entity->weight; ++ new_weight = entity->orig_weight * ++ (bfqq ? bfqq->wr_coeff : 1); ++ /* ++ * If the weight of the entity changes, remove the entity ++ * from its old weight counter (if there is a counter ++ * associated with the entity), and add it to the counter ++ * associated with its new weight. ++ */ ++ if (prev_weight != new_weight) { ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "weight changed %d %d(%d %d)", ++ prev_weight, new_weight, ++ entity->orig_weight, ++ bfqq->wr_coeff); ++ ++ root = bfqq ? &bfqd->queue_weights_tree : ++ &bfqd->group_weights_tree; ++ bfq_weights_tree_remove(bfqd, entity, root); ++ } ++ entity->weight = new_weight; ++ /* ++ * Add the entity to its weights tree only if it is ++ * not associated with a weight-raised queue. ++ */ ++ if (prev_weight != new_weight && ++ (bfqq ? bfqq->wr_coeff == 1 : 1)) ++ /* If we get here, root has been initialized. */ ++ bfq_weights_tree_add(bfqd, entity, root); ++ ++ new_st->wsum += entity->weight; ++ ++ if (new_st != old_st) ++ entity->start = new_st->vtime; ++ } ++ ++ return new_st; ++} ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); ++#endif ++ ++/** ++ * bfq_bfqq_served - update the scheduler status after selection for ++ * service. ++ * @bfqq: the queue being served. ++ * @served: bytes to transfer. ++ * ++ * NOTE: this can be optimized, as the timestamps of upper level entities ++ * are synchronized every time a new bfqq is selected for service. By now, ++ * we keep it to better check consistency. ++ */ ++static void bfq_bfqq_served(struct bfq_queue *bfqq, int served) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ struct bfq_service_tree *st; ++ ++ for_each_entity(entity) { ++ st = bfq_entity_service_tree(entity); ++ ++ entity->service += served; ++ ++ BUG_ON(st->wsum == 0); ++ ++ st->vtime += bfq_delta(served, st->wsum); ++ bfq_forget_idle(st); ++ } ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ bfqg_stats_set_start_empty_time(bfqq_group(bfqq)); ++#endif ++ st = bfq_entity_service_tree(&bfqq->entity); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "bfqq_served %d secs, vtime %llu on %p", ++ served, ((st->vtime>>10)*1000)>>12, st); ++} ++ ++/** ++ * bfq_bfqq_charge_time - charge an amount of service equivalent to the length ++ * of the time interval during which bfqq has been in ++ * service. ++ * @bfqd: the device ++ * @bfqq: the queue that needs a service update. ++ * @time_ms: the amount of time during which the queue has received service ++ * ++ * If a queue does not consume its budget fast enough, then providing ++ * the queue with service fairness may impair throughput, more or less ++ * severely. For this reason, queues that consume their budget slowly ++ * are provided with time fairness instead of service fairness. This ++ * goal is achieved through the BFQ scheduling engine, even if such an ++ * engine works in the service, and not in the time domain. The trick ++ * is charging these queues with an inflated amount of service, equal ++ * to the amount of service that they would have received during their ++ * service slot if they had been fast, i.e., if their requests had ++ * been dispatched at a rate equal to the estimated peak rate. ++ * ++ * It is worth noting that time fairness can cause important ++ * distortions in terms of bandwidth distribution, on devices with ++ * internal queueing. The reason is that I/O requests dispatched ++ * during the service slot of a queue may be served after that service ++ * slot is finished, and may have a total processing time loosely ++ * correlated with the duration of the service slot. This is ++ * especially true for short service slots. ++ */ ++static void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ unsigned long time_ms) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ int tot_serv_to_charge = entity->service; ++ unsigned int timeout_ms = jiffies_to_msecs(bfq_timeout); ++ ++ if (time_ms > 0 && time_ms < timeout_ms) ++ tot_serv_to_charge = ++ (bfqd->bfq_max_budget * time_ms) / timeout_ms; ++ ++ if (tot_serv_to_charge < entity->service) ++ tot_serv_to_charge = entity->service; ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "charge_time: %lu/%u ms, %d/%d/%d sectors", ++ time_ms, timeout_ms, entity->service, ++ tot_serv_to_charge, entity->budget); ++ ++ /* Increase budget to avoid inconsistencies */ ++ if (tot_serv_to_charge > entity->budget) ++ entity->budget = tot_serv_to_charge; ++ ++ bfq_bfqq_served(bfqq, ++ max_t(int, 0, tot_serv_to_charge - entity->service)); ++} ++ ++/** ++ * __bfq_activate_entity - activate an entity. ++ * @entity: the entity being activated. ++ * @non_blocking_wait_rq: true if this entity was waiting for a request ++ * ++ * Called whenever an entity is activated, i.e., it is not active and one ++ * of its children receives a new request, or has to be reactivated due to ++ * budget exhaustion. It uses the current budget of the entity (and the ++ * service received if @entity is active) of the queue to calculate its ++ * timestamps. ++ */ ++static void __bfq_activate_entity(struct bfq_entity *entity, ++ bool non_blocking_wait_rq) ++{ ++ struct bfq_sched_data *sd = entity->sched_data; ++ struct bfq_service_tree *st = bfq_entity_service_tree(entity); ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ bool backshifted = false; ++ ++ BUG_ON(!sd); ++ BUG_ON(!st); ++ if (entity == sd->in_service_entity) { ++ BUG_ON(entity->tree); ++ /* ++ * If we are requeueing the current entity we have ++ * to take care of not charging to it service it has ++ * not received. ++ */ ++ bfq_calc_finish(entity, entity->service); ++ entity->start = entity->finish; ++ sd->in_service_entity = NULL; ++ } else if (entity->tree == &st->active) { ++ /* ++ * Requeueing an entity due to a change of some ++ * next_in_service entity below it. We reuse the ++ * old start time. ++ */ ++ bfq_active_extract(st, entity); ++ } else { ++ unsigned long long min_vstart; ++ ++ /* See comments on bfq_fqq_update_budg_for_activation */ ++ if (non_blocking_wait_rq && bfq_gt(st->vtime, entity->finish)) { ++ backshifted = true; ++ min_vstart = entity->finish; ++ } else ++ min_vstart = st->vtime; ++ ++ if (entity->tree == &st->idle) { ++ /* ++ * Must be on the idle tree, bfq_idle_extract() will ++ * check for that. ++ */ ++ bfq_idle_extract(st, entity); ++ entity->start = bfq_gt(min_vstart, entity->finish) ? ++ min_vstart : entity->finish; ++ } else { ++ /* ++ * The finish time of the entity may be invalid, and ++ * it is in the past for sure, otherwise the queue ++ * would have been on the idle tree. ++ */ ++ entity->start = min_vstart; ++ st->wsum += entity->weight; ++ bfq_get_entity(entity); ++ ++ BUG_ON(entity->on_st); ++ entity->on_st = 1; ++ } ++ } ++ ++ st = __bfq_entity_update_weight_prio(st, entity); ++ bfq_calc_finish(entity, entity->budget); ++ ++ /* ++ * If some queues enjoy backshifting for a while, then their ++ * (virtual) finish timestamps may happen to become lower and ++ * lower than the system virtual time. In particular, if ++ * these queues often happen to be idle for short time ++ * periods, and during such time periods other queues with ++ * higher timestamps happen to be busy, then the backshifted ++ * timestamps of the former queues can become much lower than ++ * the system virtual time. In fact, to serve the queues with ++ * higher timestamps while the ones with lower timestamps are ++ * idle, the system virtual time may be pushed-up to much ++ * higher values than the finish timestamps of the idle ++ * queues. As a consequence, the finish timestamps of all new ++ * or newly activated queues may end up being much larger than ++ * those of lucky queues with backshifted timestamps. The ++ * latter queues may then monopolize the device for a lot of ++ * time. This would simply break service guarantees. ++ * ++ * To reduce this problem, push up a little bit the ++ * backshifted timestamps of the queue associated with this ++ * entity (only a queue can happen to have the backshifted ++ * flag set): just enough to let the finish timestamp of the ++ * queue be equal to the current value of the system virtual ++ * time. This may introduce a little unfairness among queues ++ * with backshifted timestamps, but it does not break ++ * worst-case fairness guarantees. ++ * ++ * As a special case, if bfqq is weight-raised, push up ++ * timestamps much less, to keep very low the probability that ++ * this push up causes the backshifted finish timestamps of ++ * weight-raised queues to become higher than the backshifted ++ * finish timestamps of non weight-raised queues. ++ */ ++ if (backshifted && bfq_gt(st->vtime, entity->finish)) { ++ unsigned long delta = st->vtime - entity->finish; ++ ++ if (bfqq) ++ delta /= bfqq->wr_coeff; ++ ++ entity->start += delta; ++ entity->finish += delta; ++ ++ if (bfqq) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "__activate_entity: new queue finish %llu", ++ ((entity->finish>>10)*1000)>>12); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ } else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "__activate_entity: new group finish %llu", ++ ((entity->finish>>10)*1000)>>12); ++#endif ++ } ++ } ++ ++ bfq_active_insert(st, entity); ++ ++ if (bfqq) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "__activate_entity: queue %seligible in st %p", ++ entity->start <= st->vtime ? "" : "non ", st); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ } else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "__activate_entity: group %seligible in st %p", ++ entity->start <= st->vtime ? "" : "non ", st); ++#endif ++ } ++} ++ ++/** ++ * bfq_activate_entity - activate an entity and its ancestors if necessary. ++ * @entity: the entity to activate. ++ * @non_blocking_wait_rq: true if this entity was waiting for a request ++ * ++ * Activate @entity and all the entities on the path from it to the root. ++ */ ++static void bfq_activate_entity(struct bfq_entity *entity, ++ bool non_blocking_wait_rq) ++{ ++ struct bfq_sched_data *sd; ++ ++ for_each_entity(entity) { ++ BUG_ON(!entity); ++ __bfq_activate_entity(entity, non_blocking_wait_rq); ++ ++ sd = entity->sched_data; ++ if (!bfq_update_next_in_service(sd)) ++ /* ++ * No need to propagate the activation to the ++ * upper entities, as they will be updated when ++ * the in-service entity is rescheduled. ++ */ ++ break; ++ } ++} ++ ++/** ++ * __bfq_deactivate_entity - deactivate an entity from its service tree. ++ * @entity: the entity to deactivate. ++ * @requeue: if false, the entity will not be put into the idle tree. ++ * ++ * Deactivate an entity, independently from its previous state. If the ++ * entity was not on a service tree just return, otherwise if it is on ++ * any scheduler tree, extract it from that tree, and if necessary ++ * and if the caller did not specify @requeue, put it on the idle tree. ++ * ++ * Return %1 if the caller should update the entity hierarchy, i.e., ++ * if the entity was in service or if it was the next_in_service for ++ * its sched_data; return %0 otherwise. ++ */ ++static int __bfq_deactivate_entity(struct bfq_entity *entity, int requeue) ++{ ++ struct bfq_sched_data *sd = entity->sched_data; ++ struct bfq_service_tree *st; ++ int was_in_service; ++ int ret = 0; ++ ++ if (sd == NULL || !entity->on_st) /* never activated, or inactive */ ++ return 0; ++ ++ st = bfq_entity_service_tree(entity); ++ was_in_service = entity == sd->in_service_entity; ++ ++ BUG_ON(was_in_service && entity->tree); ++ ++ if (was_in_service) { ++ bfq_calc_finish(entity, entity->service); ++ sd->in_service_entity = NULL; ++ } else if (entity->tree == &st->active) ++ bfq_active_extract(st, entity); ++ else if (entity->tree == &st->idle) ++ bfq_idle_extract(st, entity); ++ else if (entity->tree) ++ BUG(); ++ ++ if (was_in_service || sd->next_in_service == entity) ++ ret = bfq_update_next_in_service(sd); ++ ++ if (!requeue || !bfq_gt(entity->finish, st->vtime)) ++ bfq_forget_entity(st, entity); ++ else ++ bfq_idle_insert(st, entity); ++ ++ BUG_ON(sd->in_service_entity == entity); ++ BUG_ON(sd->next_in_service == entity); ++ ++ return ret; ++} ++ ++/** ++ * bfq_deactivate_entity - deactivate an entity. ++ * @entity: the entity to deactivate. ++ * @requeue: true if the entity can be put on the idle tree ++ */ ++static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue) ++{ ++ struct bfq_sched_data *sd; ++ struct bfq_entity *parent; ++ ++ for_each_entity_safe(entity, parent) { ++ sd = entity->sched_data; ++ ++ if (!__bfq_deactivate_entity(entity, requeue)) ++ /* ++ * next_in_service has not been changed, so ++ * no upwards update is needed ++ */ ++ break; ++ ++ if (sd->next_in_service) ++ /* ++ * The parent entity is still backlogged, ++ * because next_in_service is not NULL, and ++ * next_in_service has been updated (see ++ * comment on the body of the above if): ++ * upwards update of the schedule is needed. ++ */ ++ goto update; ++ ++ /* ++ * If we get here, then the parent is no more backlogged and ++ * we want to propagate the deactivation upwards. ++ */ ++ requeue = 1; ++ } ++ ++ return; ++ ++update: ++ entity = parent; ++ for_each_entity(entity) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ __bfq_activate_entity(entity, false); ++ ++ sd = entity->sched_data; ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "invoking udpdate_next for this queue"); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, ++ struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "invoking udpdate_next for this entity"); ++ } ++#endif ++ if (!bfq_update_next_in_service(sd)) ++ break; ++ } ++} ++ ++/** ++ * bfq_update_vtime - update vtime if necessary. ++ * @st: the service tree to act upon. ++ * ++ * If necessary update the service tree vtime to have at least one ++ * eligible entity, skipping to its start time. Assumes that the ++ * active tree of the device is not empty. ++ * ++ * NOTE: this hierarchical implementation updates vtimes quite often, ++ * we may end up with reactivated processes getting timestamps after a ++ * vtime skip done because we needed a ->first_active entity on some ++ * intermediate node. ++ */ ++static void bfq_update_vtime(struct bfq_service_tree *st) ++{ ++ struct bfq_entity *entry; ++ struct rb_node *node = st->active.rb_node; ++ ++ entry = rb_entry(node, struct bfq_entity, rb_node); ++ if (bfq_gt(entry->min_start, st->vtime)) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entry); ++ st->vtime = entry->min_start; ++ ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "update_vtime: new vtime %llu %p", ++ ((st->vtime>>10)*1000)>>12, st); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entry, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "update_vtime: new vtime %llu %p", ++ ((st->vtime>>10)*1000)>>12, st); ++ } ++#endif ++ bfq_forget_idle(st); ++ } ++} ++ ++/** ++ * bfq_first_active_entity - find the eligible entity with ++ * the smallest finish time ++ * @st: the service tree to select from. ++ * ++ * This function searches the first schedulable entity, starting from the ++ * root of the tree and going on the left every time on this side there is ++ * a subtree with at least one eligible (start >= vtime) entity. The path on ++ * the right is followed only if a) the left subtree contains no eligible ++ * entities and b) no eligible entity has been found yet. ++ */ ++static struct bfq_entity *bfq_first_active_entity(struct bfq_service_tree *st) ++{ ++ struct bfq_entity *entry, *first = NULL; ++ struct rb_node *node = st->active.rb_node; ++ ++ while (node) { ++ entry = rb_entry(node, struct bfq_entity, rb_node); ++left: ++ if (!bfq_gt(entry->start, st->vtime)) ++ first = entry; ++ ++ BUG_ON(bfq_gt(entry->min_start, st->vtime)); ++ ++ if (node->rb_left) { ++ entry = rb_entry(node->rb_left, ++ struct bfq_entity, rb_node); ++ if (!bfq_gt(entry->min_start, st->vtime)) { ++ node = node->rb_left; ++ goto left; ++ } ++ } ++ if (first) ++ break; ++ node = node->rb_right; ++ } ++ ++ BUG_ON(!first && !RB_EMPTY_ROOT(&st->active)); ++ return first; ++} ++ ++/** ++ * __bfq_lookup_next_entity - return the first eligible entity in @st. ++ * @st: the service tree. ++ * ++ * Update the virtual time in @st and return the first eligible entity ++ * it contains. ++ */ ++static struct bfq_entity * ++__bfq_lookup_next_entity(struct bfq_service_tree *st, bool force) ++{ ++ struct bfq_entity *entity, *new_next_in_service = NULL; ++ struct bfq_queue *bfqq; ++ ++ if (RB_EMPTY_ROOT(&st->active)) ++ return NULL; ++ ++ bfq_update_vtime(st); ++ entity = bfq_first_active_entity(st); ++ BUG_ON(bfq_gt(entity->start, st->vtime)); ++ ++ bfqq = bfq_entity_to_bfqq(entity); ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "__lookup_next: start %llu vtime %llu st %p", ++ ((entity->start>>10)*1000)>>12, ++ ((st->vtime>>10)*1000)>>12, st); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "__lookup_next: start %llu vtime %llu st %p", ++ ((entity->start>>10)*1000)>>12, ++ ((st->vtime>>10)*1000)>>12, st); ++ } ++#endif ++ ++ /* ++ * If the chosen entity does not match with the sched_data's ++ * next_in_service and we are forcedly serving the IDLE priority ++ * class tree, bubble up budget update. ++ */ ++ if (unlikely(force && entity != entity->sched_data->next_in_service)) { ++ new_next_in_service = entity; ++ for_each_entity(new_next_in_service) ++ bfq_update_budget(new_next_in_service); ++ } ++ ++ return entity; ++} ++ ++/** ++ * bfq_lookup_next_entity - return the first eligible entity in @sd. ++ * @sd: the sched_data. ++ * @extract: if true the returned entity will be also extracted from @sd. ++ * ++ * NOTE: since we cache the next_in_service entity at each level of the ++ * hierarchy, the complexity of the lookup can be decreased with ++ * absolutely no effort just returning the cached next_in_service value; ++ * we prefer to do full lookups to test the consistency of * the data ++ * structures. ++ */ ++static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, ++ int extract, ++ struct bfq_data *bfqd) ++{ ++ struct bfq_service_tree *st = sd->service_tree; ++ struct bfq_entity *entity; ++ int i = 0; ++ ++ BUG_ON(sd->in_service_entity); ++ ++ /* ++ * Choose from idle class, if needed to guarantee a minimum ++ * bandwidth to this class. This should also mitigate ++ * priority-inversion problems in case a low priority task is ++ * holding file system resources. ++ */ ++ if (bfqd && ++ jiffies - bfqd->bfq_class_idle_last_service > ++ BFQ_CL_IDLE_TIMEOUT) { ++ entity = __bfq_lookup_next_entity(st + BFQ_IOPRIO_CLASSES - 1, ++ true); ++ if (entity) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ if (bfqq) ++ bfq_log_bfqq(bfqd, bfqq, ++ "idle chosen from st %p %d", ++ st + BFQ_IOPRIO_CLASSES - 1, ++ BFQ_IOPRIO_CLASSES - 1); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg(bfqd, bfqg, ++ "idle chosen from st %p %d", ++ st + BFQ_IOPRIO_CLASSES - 1, ++ BFQ_IOPRIO_CLASSES - 1); ++ } ++#endif ++ i = BFQ_IOPRIO_CLASSES - 1; ++ bfqd->bfq_class_idle_last_service = jiffies; ++ sd->next_in_service = entity; ++ } ++ } ++ for (; i < BFQ_IOPRIO_CLASSES; i++) { ++ entity = __bfq_lookup_next_entity(st + i, false); ++ if (entity) { ++ if (bfqd != NULL) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ if (bfqq) ++ bfq_log_bfqq(bfqd, bfqq, ++ "chosen from st %p %d", ++ st + i, i); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg(bfqd, bfqg, ++ "chosen from st %p %d", ++ st + i, i); ++ } ++#endif ++ } ++ ++ if (extract) { ++ bfq_check_next_in_service(sd, entity); ++ bfq_active_extract(st + i, entity); ++ sd->in_service_entity = entity; ++ sd->next_in_service = NULL; ++ } ++ break; ++ } ++ } ++ ++ return entity; ++} ++ ++static bool next_queue_may_preempt(struct bfq_data *bfqd) ++{ ++ struct bfq_sched_data *sd = &bfqd->root_group->sched_data; ++ ++ return sd->next_in_service != sd->in_service_entity; ++} ++ ++/* ++ * Get next queue for service. ++ */ ++static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd) ++{ ++ struct bfq_entity *entity = NULL; ++ struct bfq_sched_data *sd; ++ struct bfq_queue *bfqq; ++ ++ BUG_ON(bfqd->in_service_queue); ++ ++ if (bfqd->busy_queues == 0) ++ return NULL; ++ ++ sd = &bfqd->root_group->sched_data; ++ for (; sd ; sd = entity->my_sched_data) { ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ if (entity) { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg(bfqd, bfqg, ++ "get_next_queue: lookup in this group"); ++ } else ++ bfq_log_bfqg(bfqd, bfqd->root_group, ++ "get_next_queue: lookup in root group"); ++#endif ++ ++ entity = bfq_lookup_next_entity(sd, 1, bfqd); ++ ++ bfqq = bfq_entity_to_bfqq(entity); ++ if (bfqq) ++ bfq_log_bfqq(bfqd, bfqq, ++ "get_next_queue: this queue, finish %llu", ++ (((entity->finish>>10)*1000)>>10)>>2); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg(bfqd, bfqg, ++ "get_next_queue: this entity, finish %llu", ++ (((entity->finish>>10)*1000)>>10)>>2); ++ } ++#endif ++ ++ BUG_ON(!entity); ++ entity->service = 0; ++ } ++ ++ bfqq = bfq_entity_to_bfqq(entity); ++ BUG_ON(!bfqq); ++ ++ return bfqq; ++} ++ ++static void __bfq_bfqd_reset_in_service(struct bfq_data *bfqd) ++{ ++ if (bfqd->in_service_bic) { ++ put_io_context(bfqd->in_service_bic->icq.ioc); ++ bfqd->in_service_bic = NULL; ++ } ++ ++ bfq_clear_bfqq_wait_request(bfqd->in_service_queue); ++ hrtimer_try_to_cancel(&bfqd->idle_slice_timer); ++ bfqd->in_service_queue = NULL; ++} ++ ++static void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ int requeue) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ bfq_deactivate_entity(entity, requeue); ++} ++ ++static void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ bfq_activate_entity(entity, bfq_bfqq_non_blocking_wait_rq(bfqq)); ++ bfq_clear_bfqq_non_blocking_wait_rq(bfqq); ++} ++ ++static void bfqg_stats_update_dequeue(struct bfq_group *bfqg); ++ ++/* ++ * Called when the bfqq no longer has requests pending, remove it from ++ * the service tree. ++ */ ++static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ int requeue) ++{ ++ BUG_ON(!bfq_bfqq_busy(bfqq)); ++ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list)); ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ ++ bfq_log_bfqq(bfqd, bfqq, "del from busy"); ++ ++ bfq_clear_bfqq_busy(bfqq); ++ ++ BUG_ON(bfqd->busy_queues == 0); ++ bfqd->busy_queues--; ++ ++ if (!bfqq->dispatched) ++ bfq_weights_tree_remove(bfqd, &bfqq->entity, ++ &bfqd->queue_weights_tree); ++ ++ if (bfqq->wr_coeff > 1) ++ bfqd->wr_busy_queues--; ++ ++ bfqg_stats_update_dequeue(bfqq_group(bfqq)); ++ ++ BUG_ON(bfqq->entity.budget < 0); ++ ++ bfq_deactivate_bfqq(bfqd, bfqq, requeue); ++ ++ BUG_ON(bfqq->entity.budget < 0); ++} ++ ++/* ++ * Called when an inactive queue receives a new request. ++ */ ++static void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ BUG_ON(bfq_bfqq_busy(bfqq)); ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ ++ bfq_log_bfqq(bfqd, bfqq, "add to busy"); ++ ++ bfq_activate_bfqq(bfqd, bfqq); ++ ++ bfq_mark_bfqq_busy(bfqq); ++ bfqd->busy_queues++; ++ ++ if (!bfqq->dispatched) ++ if (bfqq->wr_coeff == 1) ++ bfq_weights_tree_add(bfqd, &bfqq->entity, ++ &bfqd->queue_weights_tree); ++ ++ if (bfqq->wr_coeff > 1) ++ bfqd->wr_busy_queues++; ++} +diff -w -uNr '--exclude=.git' linux-4.8/block/Kconfig.iosched linux-4.8-bfq/block/Kconfig.iosched +--- linux-4.8/block/Kconfig.iosched 2016-10-03 01:24:33.000000000 +0200 ++++ linux-4.8-bfq/block/Kconfig.iosched 2016-10-18 12:57:57.691164881 +0200 +@@ -39,6 +39,25 @@ + ---help--- + Enable group IO scheduling in CFQ. + ++config IOSCHED_BFQ ++ tristate "BFQ I/O scheduler" ++ default n ++ ---help--- ++ The BFQ I/O scheduler tries to distribute bandwidth among ++ all processes according to their weights. ++ It aims at distributing the bandwidth as desired, independently of ++ the disk parameters and with any workload. It also tries to ++ guarantee low latency to interactive and soft real-time ++ applications. If compiled built-in (saying Y here), BFQ can ++ be configured to support hierarchical scheduling. ++ ++config BFQ_GROUP_IOSCHED ++ bool "BFQ hierarchical scheduling support" ++ depends on IOSCHED_BFQ && BLK_CGROUP ++ default n ++ ---help--- ++ Enable hierarchical scheduling in BFQ, using the blkio controller. ++ + choice + prompt "Default I/O scheduler" + default DEFAULT_CFQ +@@ -52,6 +71,16 @@ + config DEFAULT_CFQ + bool "CFQ" if IOSCHED_CFQ=y + ++ config DEFAULT_BFQ ++ bool "BFQ" if IOSCHED_BFQ=y ++ help ++ Selects BFQ as the default I/O scheduler which will be ++ used by default for all block devices. ++ The BFQ I/O scheduler aims at distributing the bandwidth ++ as desired, independently of the disk parameters and with ++ any workload. It also tries to guarantee low latency to ++ interactive and soft real-time applications. ++ + config DEFAULT_NOOP + bool "No-op" + +@@ -61,6 +90,7 @@ + string + default "deadline" if DEFAULT_DEADLINE + default "cfq" if DEFAULT_CFQ ++ default "bfq" if DEFAULT_BFQ + default "noop" if DEFAULT_NOOP + + endmenu +diff -w -uNr '--exclude=.git' linux-4.8/block/Makefile linux-4.8-bfq/block/Makefile +--- linux-4.8/block/Makefile 2016-10-03 01:24:33.000000000 +0200 ++++ linux-4.8-bfq/block/Makefile 2016-10-18 12:57:57.691164881 +0200 +@@ -18,6 +18,7 @@ + obj-$(CONFIG_IOSCHED_NOOP) += noop-iosched.o + obj-$(CONFIG_IOSCHED_DEADLINE) += deadline-iosched.o + obj-$(CONFIG_IOSCHED_CFQ) += cfq-iosched.o ++obj-$(CONFIG_IOSCHED_BFQ) += bfq-iosched.o + + obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o + obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o +diff -w -uNr '--exclude=.git' linux-4.8/include/linux/blkdev.h linux-4.8-bfq/include/linux/blkdev.h +--- linux-4.8/include/linux/blkdev.h 2016-10-03 01:24:33.000000000 +0200 ++++ linux-4.8-bfq/include/linux/blkdev.h 2016-10-18 12:57:59.951179412 +0200 +@@ -45,7 +45,7 @@ + * Maximum number of blkcg policies allowed to be registered concurrently. + * Defined here to simplify include dependency. + */ +-#define BLKCG_MAX_POLS 2 ++#define BLKCG_MAX_POLS 3 + + typedef void (rq_end_io_fn)(struct request *, int); + diff --git a/kernel/kernel/files/patches/mageia/net-add-recursion-limit-to-GRO.patch b/kernel/kernel/files/patches/mageia/net-add-recursion-limit-to-GRO.patch new file mode 100644 index 00000000..a3e3aa52 --- /dev/null +++ b/kernel/kernel/files/patches/mageia/net-add-recursion-limit-to-GRO.patch @@ -0,0 +1,242 @@ +From fcd91dd449867c6bfe56a81cabba76b829fd05cd Mon Sep 17 00:00:00 2001 +From: Sabrina Dubroca +Date: Thu, 20 Oct 2016 15:58:02 +0200 +Subject: [PATCH] net: add recursion limit to GRO +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Currently, GRO can do unlimited recursion through the gro_receive +handlers. This was fixed for tunneling protocols by limiting tunnel GRO +to one level with encap_mark, but both VLAN and TEB still have this +problem. Thus, the kernel is vulnerable to a stack overflow, if we +receive a packet composed entirely of VLAN headers. + +This patch adds a recursion counter to the GRO layer to prevent stack +overflow. When a gro_receive function hits the recursion limit, GRO is +aborted for this skb and it is processed normally. This recursion +counter is put in the GRO CB, but could be turned into a percpu counter +if we run out of space in the CB. + +Thanks to Vladimír Beneš for the initial bug report. + +Fixes: CVE-2016-7039 +Fixes: 9b174d88c257 ("net: Add Transparent Ethernet Bridging GRO support.") +Fixes: 66e5133f19e9 ("vlan: Add GRO support for non hardware accelerated vlan") +Signed-off-by: Sabrina Dubroca +Reviewed-by: Jiri Benc +Acked-by: Hannes Frederic Sowa +Acked-by: Tom Herbert +Signed-off-by: David S. Miller +--- + drivers/net/geneve.c | 2 +- + drivers/net/vxlan.c | 2 +- + include/linux/netdevice.h | 39 ++++++++++++++++++++++++++++++++++++++- + net/8021q/vlan.c | 2 +- + net/core/dev.c | 1 + + net/ethernet/eth.c | 2 +- + net/ipv4/af_inet.c | 2 +- + net/ipv4/fou.c | 4 ++-- + net/ipv4/gre_offload.c | 2 +- + net/ipv4/udp_offload.c | 2 +- + net/ipv6/ip6_offload.c | 2 +- + 11 files changed, 49 insertions(+), 11 deletions(-) + +diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c +index 3c20e87..16af1ce 100644 +--- a/drivers/net/geneve.c ++++ b/drivers/net/geneve.c +@@ -453,7 +453,7 @@ static struct sk_buff **geneve_gro_receive(struct sock *sk, + + skb_gro_pull(skb, gh_len); + skb_gro_postpull_rcsum(skb, gh, gh_len); +- pp = ptype->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + flush = 0; + + out_unlock: +diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c +index e7d1668..c1639a3 100644 +--- a/drivers/net/vxlan.c ++++ b/drivers/net/vxlan.c +@@ -583,7 +583,7 @@ static struct sk_buff **vxlan_gro_receive(struct sock *sk, + } + } + +- pp = eth_gro_receive(head, skb); ++ pp = call_gro_receive(eth_gro_receive, head, skb); + flush = 0; + + out: +diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h +index 465e128..91ee364 100644 +--- a/include/linux/netdevice.h ++++ b/include/linux/netdevice.h +@@ -2169,7 +2169,10 @@ struct napi_gro_cb { + /* Used to determine if flush_id can be ignored */ + u8 is_atomic:1; + +- /* 5 bit hole */ ++ /* Number of gro_receive callbacks this packet already went through */ ++ u8 recursion_counter:4; ++ ++ /* 1 bit hole */ + + /* used to support CHECKSUM_COMPLETE for tunneling protocols */ + __wsum csum; +@@ -2180,6 +2183,40 @@ struct napi_gro_cb { + + #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) + ++#define GRO_RECURSION_LIMIT 15 ++static inline int gro_recursion_inc_test(struct sk_buff *skb) ++{ ++ return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; ++} ++ ++typedef struct sk_buff **(*gro_receive_t)(struct sk_buff **, struct sk_buff *); ++static inline struct sk_buff **call_gro_receive(gro_receive_t cb, ++ struct sk_buff **head, ++ struct sk_buff *skb) ++{ ++ if (unlikely(gro_recursion_inc_test(skb))) { ++ NAPI_GRO_CB(skb)->flush |= 1; ++ return NULL; ++ } ++ ++ return cb(head, skb); ++} ++ ++typedef struct sk_buff **(*gro_receive_sk_t)(struct sock *, struct sk_buff **, ++ struct sk_buff *); ++static inline struct sk_buff **call_gro_receive_sk(gro_receive_sk_t cb, ++ struct sock *sk, ++ struct sk_buff **head, ++ struct sk_buff *skb) ++{ ++ if (unlikely(gro_recursion_inc_test(skb))) { ++ NAPI_GRO_CB(skb)->flush |= 1; ++ return NULL; ++ } ++ ++ return cb(sk, head, skb); ++} ++ + struct packet_type { + __be16 type; /* This is really htons(ether_type). */ + struct net_device *dev; /* NULL is wildcarded here */ +diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c +index 8de138d..f2531ad 100644 +--- a/net/8021q/vlan.c ++++ b/net/8021q/vlan.c +@@ -664,7 +664,7 @@ static struct sk_buff **vlan_gro_receive(struct sk_buff **head, + + skb_gro_pull(skb, sizeof(*vhdr)); + skb_gro_postpull_rcsum(skb, vhdr, sizeof(*vhdr)); +- pp = ptype->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +diff --git a/net/core/dev.c b/net/core/dev.c +index b09ac57..dbc8713 100644 +--- a/net/core/dev.c ++++ b/net/core/dev.c +@@ -4511,6 +4511,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff + NAPI_GRO_CB(skb)->flush = 0; + NAPI_GRO_CB(skb)->free = 0; + NAPI_GRO_CB(skb)->encap_mark = 0; ++ NAPI_GRO_CB(skb)->recursion_counter = 0; + NAPI_GRO_CB(skb)->is_fou = 0; + NAPI_GRO_CB(skb)->is_atomic = 1; + NAPI_GRO_CB(skb)->gro_remcsum_start = 0; +diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c +index 66dff5e..02acfff 100644 +--- a/net/ethernet/eth.c ++++ b/net/ethernet/eth.c +@@ -439,7 +439,7 @@ struct sk_buff **eth_gro_receive(struct sk_buff **head, + + skb_gro_pull(skb, sizeof(*eh)); + skb_gro_postpull_rcsum(skb, eh, sizeof(*eh)); +- pp = ptype->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c +index 1effc98..9648c97 100644 +--- a/net/ipv4/af_inet.c ++++ b/net/ipv4/af_inet.c +@@ -1391,7 +1391,7 @@ struct sk_buff **inet_gro_receive(struct sk_buff **head, struct sk_buff *skb) + skb_gro_pull(skb, sizeof(*iph)); + skb_set_transport_header(skb, skb_gro_offset(skb)); + +- pp = ops->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c +index cf50f7e..030d153 100644 +--- a/net/ipv4/fou.c ++++ b/net/ipv4/fou.c +@@ -249,7 +249,7 @@ static struct sk_buff **fou_gro_receive(struct sock *sk, + if (!ops || !ops->callbacks.gro_receive) + goto out_unlock; + +- pp = ops->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +@@ -441,7 +441,7 @@ static struct sk_buff **gue_gro_receive(struct sock *sk, + if (WARN_ON_ONCE(!ops || !ops->callbacks.gro_receive)) + goto out_unlock; + +- pp = ops->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); + flush = 0; + + out_unlock: +diff --git a/net/ipv4/gre_offload.c b/net/ipv4/gre_offload.c +index 96e0efe..d5cac99 100644 +--- a/net/ipv4/gre_offload.c ++++ b/net/ipv4/gre_offload.c +@@ -229,7 +229,7 @@ static struct sk_buff **gre_gro_receive(struct sk_buff **head, + /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/ + skb_gro_postpull_rcsum(skb, greh, grehlen); + +- pp = ptype->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + flush = 0; + + out_unlock: +diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c +index f9333c9..b2be1d9 100644 +--- a/net/ipv4/udp_offload.c ++++ b/net/ipv4/udp_offload.c +@@ -295,7 +295,7 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, + + skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ + skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); +- pp = udp_sk(sk)->gro_receive(sk, head, skb); ++ pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb); + + out_unlock: + rcu_read_unlock(); +diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c +index e7bfd55..1fcf61f 100644 +--- a/net/ipv6/ip6_offload.c ++++ b/net/ipv6/ip6_offload.c +@@ -246,7 +246,7 @@ static struct sk_buff **ipv6_gro_receive(struct sk_buff **head, + + skb_gro_postpull_rcsum(skb, iph, nlen); + +- pp = ops->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +-- +2.10.1 + diff --git a/kernel/kernel/files/patches/mageia/net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch b/kernel/kernel/files/patches/mageia/net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch new file mode 100644 index 00000000..c251e081 --- /dev/null +++ b/kernel/kernel/files/patches/mageia/net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch @@ -0,0 +1,429 @@ +From cf4747d7535a936105f0abe8d8109d3fe339162b Mon Sep 17 00:00:00 2001 +From: Larry Finger +Date: Wed, 12 Oct 2016 13:54:04 -0500 +Subject: [PATCH] rtlwifi: Fix regression caused by commit d86e64768859 + +In commit d86e64768859 ("rtlwifi: rtl818x: constify local structures"), +the configuration struct for most of the drivers was changed to be +constant. The problem is that five of the modified drivers need to be +able to update the firmware name based on the exact model of the card. +As the file names were stored in one of the members of that struct, +these drivers would fail with a kernel BUG splat when they tried to +update the firmware name. + +Rather than reverting the previous commit, I used a suggestion by +Johannes Berg and made the firmware file name pointers be local to +the routines that update the software variables. + +The configuration struct of rtl8192cu, which was not touched in the +previous patch, is now constantfied. + +Fixes: d86e64768859 ("rtlwifi: rtl818x: constify local structures") +Suggested-by: Johannes Berg +Signed-off-by: Larry Finger +Cc: Stable # 4.8 +Cc: Julia Lawall +Signed-off-by: Kalle Valo +--- + drivers/net/wireless/realtek/rtlwifi/core.c | 2 +- + drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c | 8 ++++---- + drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c | 13 +++++-------- + drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c | 12 ++++++------ + drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c | 6 +++--- + drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c | 8 ++++---- + drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c | 9 +++++---- + drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c | 12 +++++------- + drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c | 6 +++--- + drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c | 18 +++++++++--------- + drivers/net/wireless/realtek/rtlwifi/wifi.h | 2 -- + 11 files changed, 45 insertions(+), 51 deletions(-) + +diff --git a/drivers/net/wireless/realtek/rtlwifi/core.c b/drivers/net/wireless/realtek/rtlwifi/core.c +index f95760c..8e7f23c 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/core.c ++++ b/drivers/net/wireless/realtek/rtlwifi/core.c +@@ -111,7 +111,7 @@ static void rtl_fw_do_work(const struct firmware *firmware, void *context, + if (!err) + goto found_alt; + } +- pr_err("Firmware %s not available\n", rtlpriv->cfg->fw_name); ++ pr_err("Selected firmware is not available\n"); + rtlpriv->max_fw_size = 0; + return; + } +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c +index e7b11b4..f361808 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c +@@ -86,6 +86,7 @@ int rtl88e_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + u8 tid; ++ char *fw_name; + + rtl8188ee_bt_reg_init(hw); + rtlpriv->dm.dm_initialgain_enable = 1; +@@ -169,10 +170,10 @@ int rtl88e_init_sw_vars(struct ieee80211_hw *hw) + return 1; + } + +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8188efw.bin"; ++ fw_name = "rtlwifi/rtl8188efw.bin"; + rtlpriv->max_fw_size = 0x8000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -284,7 +285,6 @@ static const struct rtl_hal_cfg rtl88ee_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl88e_pci", +- .fw_name = "rtlwifi/rtl8188efw.bin", + .ops = &rtl8188ee_hal_ops, + .mod_params = &rtl88ee_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c +index 87aa209..8b6e37c 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c +@@ -96,6 +96,7 @@ int rtl92c_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); ++ char *fw_name = "rtlwifi/rtl8192cfwU.bin"; + + rtl8192ce_bt_reg_init(hw); + +@@ -167,15 +168,12 @@ int rtl92c_init_sw_vars(struct ieee80211_hw *hw) + } + + /* request fw */ +- if (IS_VENDOR_UMC_A_CUT(rtlhal->version) && +- !IS_92C_SERIAL(rtlhal->version)) +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cfwU.bin"; +- else if (IS_81XXC_VENDOR_UMC_B_CUT(rtlhal->version)) +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cfwU_B.bin"; ++ if (IS_81XXC_VENDOR_UMC_B_CUT(rtlhal->version)) ++ fw_name = "rtlwifi/rtl8192cfwU_B.bin"; + + rtlpriv->max_fw_size = 0x4000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -262,7 +260,6 @@ static const struct rtl_hal_cfg rtl92ce_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl92c_pci", +- .fw_name = "rtlwifi/rtl8192cfw.bin", + .ops = &rtl8192ce_hal_ops, + .mod_params = &rtl92ce_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c +index 7c6f7f0..f953320 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c +@@ -59,6 +59,7 @@ static int rtl92cu_init_sw_vars(struct ieee80211_hw *hw) + { + struct rtl_priv *rtlpriv = rtl_priv(hw); + int err; ++ char *fw_name; + + rtlpriv->dm.dm_initialgain_enable = true; + rtlpriv->dm.dm_flag = 0; +@@ -77,18 +78,18 @@ static int rtl92cu_init_sw_vars(struct ieee80211_hw *hw) + } + if (IS_VENDOR_UMC_A_CUT(rtlpriv->rtlhal.version) && + !IS_92C_SERIAL(rtlpriv->rtlhal.version)) { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cufw_A.bin"; ++ fw_name = "rtlwifi/rtl8192cufw_A.bin"; + } else if (IS_81XXC_VENDOR_UMC_B_CUT(rtlpriv->rtlhal.version)) { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cufw_B.bin"; ++ fw_name = "rtlwifi/rtl8192cufw_B.bin"; + } else { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cufw_TMSC.bin"; ++ fw_name = "rtlwifi/rtl8192cufw_TMSC.bin"; + } + /* provide name of alternative file */ + rtlpriv->cfg->alt_fw_name = "rtlwifi/rtl8192cufw.bin"; +- pr_info("Loading firmware %s\n", rtlpriv->cfg->fw_name); ++ pr_info("Loading firmware %s\n", fw_name); + rtlpriv->max_fw_size = 0x4000; + err = request_firmware_nowait(THIS_MODULE, 1, +- rtlpriv->cfg->fw_name, rtlpriv->io.dev, ++ fw_name, rtlpriv->io.dev, + GFP_KERNEL, hw, rtl_fw_cb); + return err; + } +@@ -187,7 +188,6 @@ static struct rtl_hal_usbint_cfg rtl92cu_interface_cfg = { + + static struct rtl_hal_cfg rtl92cu_hal_cfg = { + .name = "rtl92c_usb", +- .fw_name = "rtlwifi/rtl8192cufw.bin", + .ops = &rtl8192cu_hal_ops, + .mod_params = &rtl92cu_mod_params, + .usb_interface_cfg = &rtl92cu_interface_cfg, +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c +index 0538a4d..1ebfee1 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c +@@ -92,6 +92,7 @@ static int rtl92d_init_sw_vars(struct ieee80211_hw *hw) + u8 tid; + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); ++ char *fw_name = "rtlwifi/rtl8192defw.bin"; + + rtlpriv->dm.dm_initialgain_enable = true; + rtlpriv->dm.dm_flag = 0; +@@ -181,10 +182,10 @@ static int rtl92d_init_sw_vars(struct ieee80211_hw *hw) + + rtlpriv->max_fw_size = 0x8000; + pr_info("Driver for Realtek RTL8192DE WLAN interface\n"); +- pr_info("Loading firmware file %s\n", rtlpriv->cfg->fw_name); ++ pr_info("Loading firmware file %s\n", fw_name); + + /* request fw */ +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -266,7 +267,6 @@ static const struct rtl_hal_cfg rtl92de_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl8192de", +- .fw_name = "rtlwifi/rtl8192defw.bin", + .ops = &rtl8192de_hal_ops, + .mod_params = &rtl92de_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c +index ac299cb..46b605de3 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c +@@ -91,6 +91,7 @@ int rtl92ee_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + int err = 0; ++ char *fw_name; + + rtl92ee_bt_reg_init(hw); + rtlpci->msi_support = rtlpriv->cfg->mod_params->msi_support; +@@ -170,11 +171,11 @@ int rtl92ee_init_sw_vars(struct ieee80211_hw *hw) + } + + /* request fw */ +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192eefw.bin"; ++ fw_name = "rtlwifi/rtl8192eefw.bin"; + + rtlpriv->max_fw_size = 0x8000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -266,7 +267,6 @@ static const struct rtl_hal_cfg rtl92ee_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl92ee_pci", +- .fw_name = "rtlwifi/rtl8192eefw.bin", + .ops = &rtl8192ee_hal_ops, + .mod_params = &rtl92ee_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c +index 5e8e02d..3e1eaea 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c +@@ -89,12 +89,13 @@ static void rtl92se_fw_cb(const struct firmware *firmware, void *context) + struct ieee80211_hw *hw = context; + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rt_firmware *pfirmware = NULL; ++ char *fw_name = "rtlwifi/rtl8192sefw.bin"; + + RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD, + "Firmware callback routine entered!\n"); + complete(&rtlpriv->firmware_loading_complete); + if (!firmware) { +- pr_err("Firmware %s not available\n", rtlpriv->cfg->fw_name); ++ pr_err("Firmware %s not available\n", fw_name); + rtlpriv->max_fw_size = 0; + return; + } +@@ -117,6 +118,7 @@ static int rtl92s_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + int err = 0; + u16 earlyrxthreshold = 7; ++ char *fw_name = "rtlwifi/rtl8192sefw.bin"; + + rtlpriv->dm.dm_initialgain_enable = true; + rtlpriv->dm.dm_flag = 0; +@@ -214,9 +216,9 @@ static int rtl92s_init_sw_vars(struct ieee80211_hw *hw) + rtlpriv->max_fw_size = RTL8190_MAX_FIRMWARE_CODE_SIZE*2 + + sizeof(struct fw_hdr); + pr_info("Driver for Realtek RTL8192SE/RTL8191SE\n" +- "Loading firmware %s\n", rtlpriv->cfg->fw_name); ++ "Loading firmware %s\n", fw_name); + /* request fw */ +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl92se_fw_cb); + if (err) { +@@ -310,7 +312,6 @@ static const struct rtl_hal_cfg rtl92se_hal_cfg = { + .bar_id = 1, + .write_readback = false, + .name = "rtl92s_pci", +- .fw_name = "rtlwifi/rtl8192sefw.bin", + .ops = &rtl8192se_hal_ops, + .mod_params = &rtl92se_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c +index 89c828a..c51a9e8 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c +@@ -94,6 +94,7 @@ int rtl8723e_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); + int err = 0; ++ char *fw_name = "rtlwifi/rtl8723fw.bin"; + + rtl8723e_bt_reg_init(hw); + +@@ -176,14 +177,12 @@ int rtl8723e_init_sw_vars(struct ieee80211_hw *hw) + return 1; + } + +- if (IS_VENDOR_8723_A_CUT(rtlhal->version)) +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8723fw.bin"; +- else if (IS_81xxC_VENDOR_UMC_B_CUT(rtlhal->version)) +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8723fw_B.bin"; ++ if (IS_81xxC_VENDOR_UMC_B_CUT(rtlhal->version)) ++ fw_name = "rtlwifi/rtl8723fw_B.bin"; + + rtlpriv->max_fw_size = 0x6000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -280,7 +279,6 @@ static const struct rtl_hal_cfg rtl8723e_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl8723e_pci", +- .fw_name = "rtlwifi/rtl8723efw.bin", + .ops = &rtl8723e_hal_ops, + .mod_params = &rtl8723e_mod_params, + .maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL, +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c +index 20b53f0..847644d 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c +@@ -91,6 +91,7 @@ int rtl8723be_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + struct rtl_mac *mac = rtl_mac(rtl_priv(hw)); ++ char *fw_name = "rtlwifi/rtl8723befw.bin"; + + rtl8723be_bt_reg_init(hw); + rtlpriv->btcoexist.btc_ops = rtl_btc_get_ops_pointer(); +@@ -184,8 +185,8 @@ int rtl8723be_init_sw_vars(struct ieee80211_hw *hw) + } + + rtlpriv->max_fw_size = 0x8000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -280,7 +281,6 @@ static const struct rtl_hal_cfg rtl8723be_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl8723be_pci", +- .fw_name = "rtlwifi/rtl8723befw.bin", + .ops = &rtl8723be_hal_ops, + .mod_params = &rtl8723be_mod_params, + .maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL, +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c +index 22f687b1..297938e 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c +@@ -93,6 +93,7 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + struct rtl_mac *mac = rtl_mac(rtl_priv(hw)); + struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); ++ char *fw_name, *wowlan_fw_name; + + rtl8821ae_bt_reg_init(hw); + rtlpriv->btcoexist.btc_ops = rtl_btc_get_ops_pointer(); +@@ -203,17 +204,17 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw) + } + + if (rtlhal->hw_type == HARDWARE_TYPE_RTL8812AE) { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8812aefw.bin"; +- rtlpriv->cfg->wowlan_fw_name = "rtlwifi/rtl8812aefw_wowlan.bin"; ++ fw_name = "rtlwifi/rtl8812aefw.bin"; ++ wowlan_fw_name = "rtlwifi/rtl8812aefw_wowlan.bin"; + } else { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8821aefw.bin"; +- rtlpriv->cfg->wowlan_fw_name = "rtlwifi/rtl8821aefw_wowlan.bin"; ++ fw_name = "rtlwifi/rtl8821aefw.bin"; ++ wowlan_fw_name = "rtlwifi/rtl8821aefw_wowlan.bin"; + } + + rtlpriv->max_fw_size = 0x8000; + /*load normal firmware*/ +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -222,9 +223,9 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw) + return 1; + } + /*load wowlan firmware*/ +- pr_info("Using firmware %s\n", rtlpriv->cfg->wowlan_fw_name); ++ pr_info("Using firmware %s\n", wowlan_fw_name); + err = request_firmware_nowait(THIS_MODULE, 1, +- rtlpriv->cfg->wowlan_fw_name, ++ wowlan_fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_wowlan_fw_cb); + if (err) { +@@ -320,7 +321,6 @@ static const struct rtl_hal_cfg rtl8821ae_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl8821ae_pci", +- .fw_name = "rtlwifi/rtl8821aefw.bin", + .ops = &rtl8821ae_hal_ops, + .mod_params = &rtl8821ae_mod_params, + .maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL, +diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h +index 595f7d5..dafe486 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/wifi.h ++++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h +@@ -2278,9 +2278,7 @@ struct rtl_hal_cfg { + u8 bar_id; + bool write_readback; + char *name; +- char *fw_name; + char *alt_fw_name; +- char *wowlan_fw_name; + struct rtl_hal_ops *ops; + struct rtl_mod_params *mod_params; + struct rtl_hal_usbint_cfg *usb_interface_cfg; +-- +2.10.1 + diff --git a/kernel/kernel/files/patches/mageia/nvmepatch1-V4.patch b/kernel/kernel/files/patches/mageia/nvmepatch1-V4.patch new file mode 100644 index 00000000..9bcefef0 --- /dev/null +++ b/kernel/kernel/files/patches/mageia/nvmepatch1-V4.patch @@ -0,0 +1,111 @@ +diff --git a/drivers/nvme/host/scsi.c b/drivers/nvme/host/scsi.c +index e947e298a737..44009105f8c8 100644 +--- a/drivers/nvme/host/scsi.c ++++ b/drivers/nvme/host/scsi.c +@@ -72,15 +72,6 @@ static int sg_version_num = 30534; /* 2 digits for each component */ + #define ALL_LUNS_RETURNED 0x02 + #define ALL_WELL_KNOWN_LUNS_RETURNED 0x01 + #define RESTRICTED_LUNS_RETURNED 0x00 +-#define NVME_POWER_STATE_START_VALID 0x00 +-#define NVME_POWER_STATE_ACTIVE 0x01 +-#define NVME_POWER_STATE_IDLE 0x02 +-#define NVME_POWER_STATE_STANDBY 0x03 +-#define NVME_POWER_STATE_LU_CONTROL 0x07 +-#define POWER_STATE_0 0 +-#define POWER_STATE_1 1 +-#define POWER_STATE_2 2 +-#define POWER_STATE_3 3 + #define DOWNLOAD_SAVE_ACTIVATE 0x05 + #define DOWNLOAD_SAVE_DEFER_ACTIVATE 0x0E + #define ACTIVATE_DEFERRED_MICROCODE 0x0F +@@ -1229,64 +1220,6 @@ static void nvme_trans_fill_read_cap(u8 *response, struct nvme_id_ns *id_ns, + + /* Start Stop Unit Helper Functions */ + +-static int nvme_trans_power_state(struct nvme_ns *ns, struct sg_io_hdr *hdr, +- u8 pc, u8 pcmod, u8 start) +-{ +- int res; +- int nvme_sc; +- struct nvme_id_ctrl *id_ctrl; +- int lowest_pow_st; /* max npss = lowest power consumption */ +- unsigned ps_desired = 0; +- +- nvme_sc = nvme_identify_ctrl(ns->ctrl, &id_ctrl); +- res = nvme_trans_status_code(hdr, nvme_sc); +- if (res) +- return res; +- +- lowest_pow_st = max(POWER_STATE_0, (int)(id_ctrl->npss - 1)); +- kfree(id_ctrl); +- +- switch (pc) { +- case NVME_POWER_STATE_START_VALID: +- /* Action unspecified if POWER CONDITION MODIFIER != 0 */ +- if (pcmod == 0 && start == 0x1) +- ps_desired = POWER_STATE_0; +- if (pcmod == 0 && start == 0x0) +- ps_desired = lowest_pow_st; +- break; +- case NVME_POWER_STATE_ACTIVE: +- /* Action unspecified if POWER CONDITION MODIFIER != 0 */ +- if (pcmod == 0) +- ps_desired = POWER_STATE_0; +- break; +- case NVME_POWER_STATE_IDLE: +- /* Action unspecified if POWER CONDITION MODIFIER != [0,1,2] */ +- if (pcmod == 0x0) +- ps_desired = POWER_STATE_1; +- else if (pcmod == 0x1) +- ps_desired = POWER_STATE_2; +- else if (pcmod == 0x2) +- ps_desired = POWER_STATE_3; +- break; +- case NVME_POWER_STATE_STANDBY: +- /* Action unspecified if POWER CONDITION MODIFIER != [0,1] */ +- if (pcmod == 0x0) +- ps_desired = max(POWER_STATE_0, (lowest_pow_st - 2)); +- else if (pcmod == 0x1) +- ps_desired = max(POWER_STATE_0, (lowest_pow_st - 1)); +- break; +- case NVME_POWER_STATE_LU_CONTROL: +- default: +- res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION, +- ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB, +- SCSI_ASCQ_CAUSE_NOT_REPORTABLE); +- break; +- } +- nvme_sc = nvme_set_features(ns->ctrl, NVME_FEAT_POWER_MGMT, ps_desired, 0, +- NULL); +- return nvme_trans_status_code(hdr, nvme_sc); +-} +- + static int nvme_trans_send_activate_fw_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr, + u8 buffer_id) + { +@@ -2235,11 +2168,10 @@ static int nvme_trans_synchronize_cache(struct nvme_ns *ns, + static int nvme_trans_start_stop(struct nvme_ns *ns, struct sg_io_hdr *hdr, + u8 *cmd) + { +- u8 immed, pcmod, pc, no_flush, start; ++ u8 immed, pcmod, no_flush, start; + + immed = cmd[1] & 0x01; + pcmod = cmd[3] & 0x0f; +- pc = (cmd[4] & 0xf0) >> 4; + no_flush = cmd[4] & 0x04; + start = cmd[4] & 0x01; + +@@ -2254,8 +2186,8 @@ static int nvme_trans_start_stop(struct nvme_ns *ns, struct sg_io_hdr *hdr, + if (res) + return res; + } +- /* Setup the expected power state transition */ +- return nvme_trans_power_state(ns, hdr, pc, pcmod, start); ++ ++ return 0; + } + } + +-- +2.7.4 diff --git a/kernel/kernel/files/patches/mageia/nvmepatch2-V4.patch b/kernel/kernel/files/patches/mageia/nvmepatch2-V4.patch new file mode 100644 index 00000000..454cb4a5 --- /dev/null +++ b/kernel/kernel/files/patches/mageia/nvmepatch2-V4.patch @@ -0,0 +1,107 @@ +diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c +index 2feacc70bf61..b92a7f767bfc 100644 +--- a/drivers/nvme/host/core.c ++++ b/drivers/nvme/host/core.c +@@ -597,7 +597,7 @@ int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid, + } + + int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, +- dma_addr_t dma_addr, u32 *result) ++ void *buffer, size_t buflen, u32 *result) + { + struct nvme_command c; + struct nvme_completion cqe; +@@ -606,10 +606,9 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, + memset(&c, 0, sizeof(c)); + c.features.opcode = nvme_admin_get_features; + c.features.nsid = cpu_to_le32(nsid); +- c.features.dptr.prp1 = cpu_to_le64(dma_addr); + c.features.fid = cpu_to_le32(fid); + +- ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, NULL, 0, 0, ++ ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, buffer, buflen, 0, + NVME_QID_ANY, 0, 0); + if (ret >= 0 && result) + *result = le32_to_cpu(cqe.result); +@@ -617,7 +616,7 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, + } + + int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, +- dma_addr_t dma_addr, u32 *result) ++ void *buffer, size_t buflen, u32 *result) + { + struct nvme_command c; + struct nvme_completion cqe; +@@ -625,12 +624,11 @@ int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, + + memset(&c, 0, sizeof(c)); + c.features.opcode = nvme_admin_set_features; +- c.features.dptr.prp1 = cpu_to_le64(dma_addr); + c.features.fid = cpu_to_le32(fid); + c.features.dword11 = cpu_to_le32(dword11); + +- ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, NULL, 0, 0, +- NVME_QID_ANY, 0, 0); ++ ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, ++ buffer, buflen, 0, NVME_QID_ANY, 0, 0); + if (ret >= 0 && result) + *result = le32_to_cpu(cqe.result); + return ret; +@@ -664,7 +662,7 @@ int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count) + u32 result; + int status, nr_io_queues; + +- status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, 0, ++ status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0, + &result); + if (status < 0) + return status; +diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h +index ab18b78102bf..c2151761ec5f 100644 +--- a/drivers/nvme/host/nvme.h ++++ b/drivers/nvme/host/nvme.h +@@ -292,9 +292,9 @@ int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid, + struct nvme_id_ns **id); + int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log); + int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, +- dma_addr_t dma_addr, u32 *result); ++ void *buffer, size_t buflen, u32 *result); + int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, +- dma_addr_t dma_addr, u32 *result); ++ void *buffer, size_t buflen, u32 *result); + int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count); + void nvme_start_keep_alive(struct nvme_ctrl *ctrl); + void nvme_stop_keep_alive(struct nvme_ctrl *ctrl); +diff --git a/drivers/nvme/host/scsi.c b/drivers/nvme/host/scsi.c +index 44009105f8c8..c2a0a1c7d05d 100644 +--- a/drivers/nvme/host/scsi.c ++++ b/drivers/nvme/host/scsi.c +@@ -906,7 +906,7 @@ static int nvme_trans_log_temperature(struct nvme_ns *ns, struct sg_io_hdr *hdr, + kfree(smart_log); + + /* Get Features for Temp Threshold */ +- res = nvme_get_features(ns->ctrl, NVME_FEAT_TEMP_THRESH, 0, 0, ++ res = nvme_get_features(ns->ctrl, NVME_FEAT_TEMP_THRESH, 0, NULL, 0, + &feature_resp); + if (res != NVME_SC_SUCCESS) + temp_c_thresh = LOG_TEMP_UNKNOWN; +@@ -1039,7 +1039,7 @@ static int nvme_trans_fill_caching_page(struct nvme_ns *ns, + if (len < MODE_PAGE_CACHING_LEN) + return -EINVAL; + +- nvme_sc = nvme_get_features(ns->ctrl, NVME_FEAT_VOLATILE_WC, 0, 0, ++ nvme_sc = nvme_get_features(ns->ctrl, NVME_FEAT_VOLATILE_WC, 0, NULL, 0, + &feature_resp); + res = nvme_trans_status_code(hdr, nvme_sc); + if (res) +@@ -1328,7 +1328,7 @@ static int nvme_trans_modesel_get_mp(struct nvme_ns *ns, struct sg_io_hdr *hdr, + case MODE_PAGE_CACHING: + dword11 = ((mode_page[2] & CACHING_MODE_PAGE_WCE_MASK) ? 1 : 0); + nvme_sc = nvme_set_features(ns->ctrl, NVME_FEAT_VOLATILE_WC, +- dword11, 0, NULL); ++ dword11, NULL, 0, NULL); + res = nvme_trans_status_code(hdr, nvme_sc); + break; + case MODE_PAGE_CONTROL: +-- +2.7.4 diff --git a/kernel/kernel/files/patches/mageia/nvmepatch3-V4.patch b/kernel/kernel/files/patches/mageia/nvmepatch3-V4.patch new file mode 100644 index 00000000..797c4681 --- /dev/null +++ b/kernel/kernel/files/patches/mageia/nvmepatch3-V4.patch @@ -0,0 +1,264 @@ +diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c +index b92a7f767bfc..a143c72c3d0d 100644 +--- a/drivers/nvme/host/core.c ++++ b/drivers/nvme/host/core.c +@@ -56,6 +56,11 @@ EXPORT_SYMBOL_GPL(nvme_max_retries); + static int nvme_char_major; + module_param(nvme_char_major, int, 0); + ++static unsigned long default_ps_max_latency_us = 25000; ++module_param(default_ps_max_latency_us, ulong, 0644); ++MODULE_PARM_DESC(ps_max_latency_us, ++ "default max power saving latency; overridden per device in sysfs"); ++ + static LIST_HEAD(nvme_ctrl_list); + static DEFINE_SPINLOCK(dev_list_lock); + +@@ -1205,6 +1210,98 @@ static void nvme_set_queue_limits(struct nvme_ctrl *ctrl, + blk_queue_write_cache(q, vwc, vwc); + } + ++static void nvme_configure_apst(struct nvme_ctrl *ctrl) ++{ ++ /* ++ * APST (Autonomous Power State Transition) lets us program a ++ * table of power state transitions that the controller will ++ * perform automatically. We configure it with a simple ++ * heuristic: we are willing to spend at most 2% of the time ++ * transitioning between power states. Therefore, when running ++ * in any given state, we will enter the next lower-power ++ * non-operational state after waiting 100 * (enlat + exlat) ++ * microseconds, as long as that state's total latency is under ++ * the requested maximum latency. ++ * ++ * We will not autonomously enter any non-operational state for ++ * which the total latency exceeds ps_max_latency_us. Users ++ * can set ps_max_latency_us to zero to turn off APST. ++ */ ++ ++ unsigned apste; ++ struct nvme_feat_auto_pst *table; ++ int ret; ++ ++ if (!ctrl->apsta) ++ return; /* APST isn't supported. */ ++ ++ if (ctrl->npss > 31) { ++ dev_warn(ctrl->device, "NPSS is invalid; not using APST\n"); ++ return; ++ } ++ ++ table = kzalloc(sizeof(*table), GFP_KERNEL); ++ if (!table) ++ return; ++ ++ if (ctrl->ps_max_latency_us == 0) { ++ /* Turn off APST. */ ++ apste = 0; ++ } else { ++ __le64 target = cpu_to_le64(0); ++ int state; ++ ++ /* ++ * Walk through all states from lowest- to highest-power. ++ * According to the spec, lower-numbered states use more ++ * power. NPSS, despite the name, is the index of the ++ * lowest-power state, not the number of states. ++ */ ++ for (state = (int)ctrl->npss; state >= 0; state--) { ++ u64 total_latency_us, transition_ms; ++ ++ if (target) ++ table->entries[state] = target; ++ ++ /* ++ * Is this state a useful non-operational state for ++ * higher-power states to autonomously transition to? ++ */ ++ if (!(ctrl->psd[state].flags & 2)) ++ continue; /* It's an operational state. */ ++ ++ total_latency_us = ++ (u64)cpu_to_le32(ctrl->psd[state].entry_lat) + ++ + cpu_to_le32(ctrl->psd[state].exit_lat); ++ if (total_latency_us > ctrl->ps_max_latency_us) ++ continue; ++ ++ /* ++ * This state is good. Use it as the APST idle ++ * target for higher power states. ++ */ ++ transition_ms = total_latency_us + 19; ++ do_div(transition_ms, 20); ++ if (transition_ms >= (1 << 24)) ++ transition_ms = (1 << 24); ++ ++ target = cpu_to_le64((state << 3) | ++ (transition_ms << 8)); ++ } ++ ++ apste = 1; ++ } ++ ++ ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste, ++ table, sizeof(*table), NULL); ++ if (ret) ++ dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret); ++ ++ kfree(table); ++} ++ ++static struct attribute_group nvme_dev_dynamic_attrs_group; ++ + /* + * Initialize the cached copies of the Identify data and various controller + * register in our nvme_ctrl structure. This should be called as soon as +@@ -1271,6 +1368,10 @@ int nvme_init_identify(struct nvme_ctrl *ctrl) + ctrl->sgls = le32_to_cpu(id->sgls); + ctrl->kas = le16_to_cpu(id->kas); + ++ ctrl->npss = id->npss; ++ ctrl->apsta = id->apsta; ++ memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd)); ++ + if (ctrl->ops->is_fabrics) { + ctrl->icdoff = le16_to_cpu(id->icdoff); + ctrl->ioccsz = le32_to_cpu(id->ioccsz); +@@ -1294,6 +1395,10 @@ int nvme_init_identify(struct nvme_ctrl *ctrl) + } + + kfree(id); ++ ++ nvme_configure_apst(ctrl); ++ ++ sysfs_update_group(&ctrl->device->kobj, &nvme_dev_dynamic_attrs_group); + return ret; + } + EXPORT_SYMBOL_GPL(nvme_init_identify); +@@ -1564,6 +1669,41 @@ static ssize_t nvme_sysfs_show_address(struct device *dev, + } + static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL); + ++static ssize_t nvme_sysfs_show_ps_max_latency_us( ++ struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct nvme_ctrl *ctrl = dev_get_drvdata(dev); ++ ++ return snprintf(buf, PAGE_SIZE, "%llu\n", ctrl->ps_max_latency_us); ++} ++ ++static ssize_t nvme_sysfs_store_ps_max_latency_us( ++ struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t size) ++{ ++ struct nvme_ctrl *ctrl = dev_get_drvdata(dev); ++ int ret; ++ u64 val; ++ ++ ret = kstrtoull(buf, 10, &val); ++ if (ret) ++ return ret; ++ ++ if (ctrl->ps_max_latency_us != val) { ++ ctrl->ps_max_latency_us = val; ++ nvme_configure_apst(ctrl); ++ } ++ ++ return size; ++} ++ ++static DEVICE_ATTR(ps_max_latency_us, 0644, ++ nvme_sysfs_show_ps_max_latency_us, ++ nvme_sysfs_store_ps_max_latency_us); ++ + static struct attribute *nvme_dev_attrs[] = { + &dev_attr_reset_controller.attr, + &dev_attr_rescan_controller.attr, +@@ -1605,8 +1745,33 @@ static struct attribute_group nvme_dev_attrs_group = { + .is_visible = nvme_dev_attrs_are_visible, + }; + ++static struct attribute *nvme_dev_dynamic_attrs[] = { ++ &dev_attr_ps_max_latency_us.attr, ++ NULL ++}; ++ ++static umode_t nvme_dev_dynamic_attrs_are_visible(struct kobject *kobj, ++ struct attribute *a, int n) ++{ ++ struct device *dev = container_of(kobj, struct device, kobj); ++ struct nvme_ctrl *ctrl = dev_get_drvdata(dev); ++ ++ if (a == &dev_attr_ps_max_latency_us.attr) { ++ if (!ctrl->apsta) ++ return 0; ++ } ++ ++ return a->mode; ++} ++ ++static struct attribute_group nvme_dev_dynamic_attrs_group = { ++ .attrs = nvme_dev_dynamic_attrs, ++ .is_visible = nvme_dev_dynamic_attrs_are_visible, ++}; ++ + static const struct attribute_group *nvme_dev_attr_groups[] = { + &nvme_dev_attrs_group, ++ &nvme_dev_dynamic_attrs_group, + NULL, + }; + +@@ -1991,6 +2156,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, + ctrl->quirks = quirks; + INIT_WORK(&ctrl->scan_work, nvme_scan_work); + INIT_WORK(&ctrl->async_event_work, nvme_async_event_work); ++ ctrl->ps_max_latency_us = default_ps_max_latency_us; + + ret = nvme_set_instance(ctrl); + if (ret) +diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h +index c2151761ec5f..16d51006afc0 100644 +--- a/drivers/nvme/host/nvme.h ++++ b/drivers/nvme/host/nvme.h +@@ -129,13 +129,19 @@ struct nvme_ctrl { + u32 vs; + u32 sgls; + u16 kas; ++ u8 npss; ++ u8 apsta; + unsigned int kato; + bool subsystem; + unsigned long quirks; ++ struct nvme_id_power_state psd[32]; + struct work_struct scan_work; + struct work_struct async_event_work; + struct delayed_work ka_work; + ++ /* Power saving configuration */ ++ u64 ps_max_latency_us; ++ + /* Fabrics only */ + u16 sqsize; + u32 ioccsz; +diff --git a/include/linux/nvme.h b/include/linux/nvme.h +index d8b37bab2887..a76237dac4b0 100644 +--- a/include/linux/nvme.h ++++ b/include/linux/nvme.h +@@ -543,6 +543,12 @@ struct nvme_dsm_range { + __le64 slba; + }; + ++/* Features */ ++ ++struct nvme_feat_auto_pst { ++ __le64 entries[32]; ++}; ++ + /* Admin commands */ + + enum nvme_admin_opcode { +-- +2.7.4 diff --git a/kernel/kernel/files/patches/mageia/series b/kernel/kernel/files/patches/mageia/series index 6452397c..e7a005b7 100644 --- a/kernel/kernel/files/patches/mageia/series +++ b/kernel/kernel/files/patches/mageia/series @@ -41,11 +41,19 @@ Revert-cpufreq-pcc-Enable-autoload-of-pcc-cpufreq-fo.patch # breaks nvidia304 Revert-x86-mm-mtrr-Remove-kernel-internal-MTRR-inter.patch +# fix non-booting systems +x86-smpboot-Init-apic-mapping-before-usage.patch + ### ### Core ### base-cacheinfo-silence-DT-warnings.patch +# Patch for having the cpupower setting the governor +# (from https://bugzilla.kernel.org/show_bug.cgi?id=135391, same as +# https://bugzilla.redhat.com/show_bug.cgi?id=1374212) +kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch + ### ### i2c ### @@ -119,8 +127,20 @@ ata-prefer-ata-drivers-over-ide-drivers-when-both-are-built.patch # Nice SSD speedup block-Make-CFQ-default-to-IOPS-mode-on-SSDs.patch -# ahci ids -ahci-add-new-Intel-device-IDs.patch +# Include new BFQ I/O scheduler +# (See http://algo.ing.unimo.it/people/paolo/disk_sched) +# +# Release of BFQ I/O scheduler v8r4 (rediffed from git https://github.com/linusw/linux-bfq) +linux-BFQ-v8r4.patch + +# Let NVME drives enter in power state +# (see https://github.com/damige/linux-nvme/tree/master/src) +nvmepatch1-V4.patch +nvmepatch2-V4.patch +nvmepatch3-V4.patch + +# fix feature check, reported on dev@ ml +dm-raid-fix-compat_features-validation.patch ### ### Char @@ -134,9 +154,8 @@ Revert-ipmi-Start-the-timer-and-thread-on-internal-m.patch ### # aufs from: http://aufs.sourceforge.net/ (mga#8314) -fs-aufs-4.7.patch -fs-aufs-4.7-modular.patch -fs-aufs-4.7-ver-fix.patch +fs-aufs-4.8.patch +fs-aufs-4.8-modular.patch ### ### FireWire @@ -171,6 +190,14 @@ gpu-drm-mach64-linux-3.14-buildfix.patch gpu-drm-mach64-3.17-buildfix.patch gpu-drm-mach64-3.18-buildfix.patch +# Attempt to fix FIFO underrun problems: +# (from https://patchwork.freedesktop.org/patch/113642/ and +# also from https://bugs.freedesktop.org/attachment.cgi?id=127264, first reference from +# https://bugs.freedesktop.org/show_bug.cgi?id=97450). Does not completely resolve the +# issues but reduce. +# now in drm-next: https://cgit.freedesktop.org/~airlied/linux/commit?h=drm-next&id=5a920b85f2c6e3fd7d9dd9bb3f3345e9085e2360 +gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch + ### ### Hardware Monitoring ### @@ -222,6 +249,12 @@ net-netfilter-psd.patch net-netfilter-psd-mdv.patch net-netfilter-psd-2.6.35-buildfix.patch +# rtlwifi regression +net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch + +# CVE-2016-7039 +net-add-recursion-limit-to-GRO.patch + ### ### Platform drivers ### @@ -232,10 +265,6 @@ platform-x86-add-shuttle-wmi-driver.patch platform-x86-shuttle-wmi-drop-devinit-exit.patch platform-x86-shuttle-wmi-4.2-buildfix.patch -# -intel_th-Fix-a-deadlock-in-modprobing.patch -intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch - ### ### RTC ### @@ -331,6 +360,7 @@ video-mageia-logo.patch # add rtl8723bs support (mga#15874) 3rd-rtl8723bs.patch 3rd-rtl8723bs-4.7-buildfix.patch +3rd-rtl8723bs-4.8-buildfix.patch ### ### Security @@ -348,10 +378,6 @@ video-mageia-logo.patch ### ARM ### -# RPi2 support -# Read MAC address from DT -arm-0001-ARM-bcm2835-dt-Add-the-ethernet-to-the-device-trees.patch - ### ### IA64 ### @@ -360,6 +386,11 @@ arm-0001-ARM-bcm2835-dt-Add-the-ethernet-to-the-device-trees.patch ### Tools ### +# wipe powerpc refrence so we can nuke dangling symlinks (mga#17676) +tools-testing-selftest-Makefile-remove-powerpc-reference.patch + ### ### UAPI ### +# breaks linux-arm build +Revert-include-uapi-linux-atm_zatm.h-include-linux-t.patch diff --git a/kernel/kernel/files/patches/mageia/tools-testing-selftest-Makefile-remove-powerpc-reference.patch b/kernel/kernel/files/patches/mageia/tools-testing-selftest-Makefile-remove-powerpc-reference.patch new file mode 100644 index 00000000..677e97df --- /dev/null +++ b/kernel/kernel/files/patches/mageia/tools-testing-selftest-Makefile-remove-powerpc-reference.patch @@ -0,0 +1,10 @@ +--- linux/tools/testing/selftests/Makefile.orig ++++ linux/tools/testing/selftests/Makefile +@@ -15,7 +15,6 @@ TARGETS += memory-hotplug + TARGETS += mount + TARGETS += mqueue + TARGETS += net +-TARGETS += powerpc + TARGETS += pstore + TARGETS += ptrace + TARGETS += seccomp diff --git a/kernel/kernel/files/patches/mageia/x86-smpboot-Init-apic-mapping-before-usage.patch b/kernel/kernel/files/patches/mageia/x86-smpboot-Init-apic-mapping-before-usage.patch new file mode 100644 index 00000000..6c2e766a --- /dev/null +++ b/kernel/kernel/files/patches/mageia/x86-smpboot-Init-apic-mapping-before-usage.patch @@ -0,0 +1,61 @@ +From 1e90a13d0c3dc94512af1ccb2b6563e8297838fa Mon Sep 17 00:00:00 2001 +From: Thomas Gleixner +Date: Sat, 29 Oct 2016 13:42:42 +0200 +Subject: [PATCH] x86/smpboot: Init apic mapping before usage + +The recent changes, which forced the registration of the boot cpu on UP +systems, which do not have ACPI tables, have been fixed for systems w/o +local APIC, but left a wreckage for systems which have neither ACPI nor +mptables, but the CPU has an APIC, e.g. virtualbox. + +The boot process crashes in prefill_possible_map() as it wants to register +the boot cpu, which needs to access the local apic, but the local APIC is +not yet mapped. + +There is no reason why init_apic_mapping() can't be invoked before +prefill_possible_map(). So instead of playing another silly early mapping +game, as the ACPI/mptables code does, we just move init_apic_mapping() +before the call to prefill_possible_map(). + +In hindsight, I should have noticed that combination earlier. + +Sorry for the churn (also in stable)! + +Fixes: ff8560512b8d ("x86/boot/smp: Don't try to poke disabled/non-existent APIC") +Reported-and-debugged-by: Michal Necasek +Reported-and-tested-by: Wolfgang Bauer +Cc: prarit@redhat.com +Cc: ville.syrjala@linux.intel.com +Cc: michael.thayer@oracle.com +Cc: knut.osmundsen@oracle.com +Cc: frank.mehnert@oracle.com +Cc: Borislav Petkov +Cc: stable@vger.kernel.org +Link: http://lkml.kernel.org/r/alpine.DEB.2.20.1610282114380.5053@nanos +Signed-off-by: Thomas Gleixner +--- + arch/x86/kernel/setup.c | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c +index bbfbca5..9c337b0 100644 +--- a/arch/x86/kernel/setup.c ++++ b/arch/x86/kernel/setup.c +@@ -1222,11 +1222,16 @@ void __init setup_arch(char **cmdline_p) + if (smp_found_config) + get_smp_config(); + ++ /* ++ * Systems w/o ACPI and mptables might not have it mapped the local ++ * APIC yet, but prefill_possible_map() might need to access it. ++ */ ++ init_apic_mappings(); ++ + prefill_possible_map(); + + init_cpu_to_node(); + +- init_apic_mappings(); + io_apic_init_mappings(); + + kvm_guest_init(); diff --git a/kernel/kernel/pspec.xml b/kernel/kernel/pspec.xml index 694e8c67..e505d91d 100644 --- a/kernel/kernel/pspec.xml +++ b/kernel/kernel/pspec.xml @@ -12,7 +12,7 @@ kernel The Linux kernel (the core of the Linux operating system) for Pisi Linux kernel contains the Linux kernel, the core of any Linux operating system. The kernel handles the basic functions of the operating system: memory allocation, process allocation, device input and output, etc. - https://www.kernel.org/pub/linux/kernel/v4.x/linux-4.7.tar.gz + https://www.kernel.org/pub/linux/kernel/v4.x/linux-4.8.tar.gz configs/kernel-i686-config configs/kernel-x86_64-config @@ -28,8 +28,8 @@ - patches/linux/patch-4.7.2.xz - + patches/linux/patch-4.8.6.xz + patches/mageia/Revert-ipmi-Start-the-timer-and-thread-on-internal-m.patch @@ -37,9 +37,11 @@ patches/mageia/x86-boot-video-80x25-if-break.patch patches/mageia/x86-default_poweroff_up_machines.patch patches/mageia/x86-increase-default-minimum-vmalloc-area-by-64MB-to-192MB.patch - patches/mageia/Revert-cpufreq-pcc-Enable-autoload-of-pcc-cpufreq-fo.patch + patches/mageia/Revert-cpufreq-pcc-Enable-autoload-of-pcc-cpufreq-fo.patch + patches/mageia/x86-smpboot-Init-apic-mapping-before-usage.patch patches/mageia/Revert-x86-mm-mtrr-Remove-kernel-internal-MTRR-inter.patch patches/mageia/base-cacheinfo-silence-DT-warnings.patch + patches/mageia/kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch patches/mageia/pci-add-ALI-M5229-ide-compatibility-mode-quirk.patch patches/mageia/pci-quirks-drop-devinit-exit.patch patches/mageia/acpi-CLEVO-M360S-disable_acpi_irq.patch @@ -58,9 +60,13 @@ patches/mageia/block-floppy-disable-pnp-modalias.patch patches/mageia/ata-prefer-ata-drivers-over-ide-drivers-when-both-are-built.patch patches/mageia/block-Make-CFQ-default-to-IOPS-mode-on-SSDs.patch - patches/mageia/fs-aufs-4.7.patch - patches/mageia/fs-aufs-4.7-modular.patch - patches/mageia/fs-aufs-4.7-ver-fix.patch + patches/mageia/linux-BFQ-v8r4.patch + patches/mageia/nvmepatch1-V4.patch + patches/mageia/nvmepatch2-V4.patch + patches/mageia/nvmepatch3-V4.patch + patches/mageia/dm-raid-fix-compat_features-validation.patch + patches/mageia/fs-aufs-4.8.patch + patches/mageia/fs-aufs-4.8-modular.patch patches/mageia/firewire-ieee1394-module-aliases.patch patches/mageia/char-agp-intel-new-Q57-id.patch @@ -81,6 +87,7 @@ patches/mageia/gpu-drm-mach64-linux-3.14-buildfix.patch patches/mageia/gpu-drm-mach64-3.17-buildfix.patch patches/mageia/gpu-drm-mach64-3.18-buildfix.patch + patches/mageia/gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch patches/mageia/input-i8042-quirks-for-Fujitsu-Lifebook-A544-and-Lif.patch patches/mageia/net-sis190-fix-list-usage.patch patches/mageia/net-netfilter-IFWLOG.patch @@ -93,11 +100,11 @@ patches/mageia/net-netfilter-psd.patch patches/mageia/net-netfilter-psd-mdv.patch patches/mageia/net-netfilter-psd-2.6.35-buildfix.patch + patches/mageia/net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch + patches/mageia/net-add-recursion-limit-to-GRO.patch patches/mageia/platform-x86-add-shuttle-wmi-driver.patch patches/mageia/platform-x86-shuttle-wmi-drop-devinit-exit.patch patches/mageia/platform-x86-shuttle-wmi-4.2-buildfix.patch - patches/mageia/intel_th-Fix-a-deadlock-in-modprobing.patch - patches/mageia/intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch patches/mageia/include-kbuild-export-pci_ids.patch patches/mageia/hid-usbhid-IBM-BladeCenterHS20-quirk.patch patches/mageia/usb-storage-unusual_devs-add-id.patch @@ -130,8 +137,9 @@ patches/mageia/3rd-viahss-3.0-buildfix.patch patches/mageia/3rd-rtl8723bs.patch patches/mageia/3rd-rtl8723bs-4.7-buildfix.patch - patches/mageia/ahci-add-new-Intel-device-IDs.patch - patches/mageia/arm-0001-ARM-bcm2835-dt-Add-the-ethernet-to-the-device-trees.patch + patches/mageia/3rd-rtl8723bs-4.8-buildfix.patch + patches/mageia/tools-testing-selftest-Makefile-remove-powerpc-reference.patch + patches/mageia/Revert-include-uapi-linux-atm_zatm.h-include-linux-t.patch @@ -187,6 +195,17 @@ + + 2016-11-01 + 4.8.6 + Version Bump. + security + + systemRestart + + Ertuğrul Erata + ertugrulerata@gmail.com + 2016-08-24 4.7.2 diff --git a/kernel/tools/cpupowertools/files/patches/linux/patch-4.8.6.xz b/kernel/tools/cpupowertools/files/patches/linux/patch-4.8.6.xz new file mode 100644 index 00000000..0438de2b Binary files /dev/null and b/kernel/tools/cpupowertools/files/patches/linux/patch-4.8.6.xz differ diff --git a/kernel/tools/cpupowertools/files/patches/mageia/3rd-rtl8723bs-4.8-buildfix.patch b/kernel/tools/cpupowertools/files/patches/mageia/3rd-rtl8723bs-4.8-buildfix.patch new file mode 100644 index 00000000..fa695774 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/3rd-rtl8723bs-4.8-buildfix.patch @@ -0,0 +1,19 @@ + +Adjust for cfg80211_scan_done() changes in kernel 4.8 + +Signed-off-by: Thomas Backlund + +--- linux/3rdparty/rtl8723bs/os_dep/ioctl_cfg80211.c.orig ++++ linux/3rdparty/rtl8723bs/os_dep/ioctl_cfg80211.c +@@ -1413,7 +1413,10 @@ void rtw_cfg80211_indicate_scan_done(str + } + else + { +- cfg80211_scan_done(pwdev_priv->scan_request, aborted); ++ struct cfg80211_scan_info info = { ++ .aborted = aborted, ++ }; ++ cfg80211_scan_done(pwdev_priv->scan_request, &info); + } + + pwdev_priv->scan_request = NULL; diff --git a/kernel/tools/cpupowertools/files/patches/mageia/Revert-include-uapi-linux-atm_zatm.h-include-linux-t.patch b/kernel/tools/cpupowertools/files/patches/mageia/Revert-include-uapi-linux-atm_zatm.h-include-linux-t.patch new file mode 100644 index 00000000..a30f81df --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/Revert-include-uapi-linux-atm_zatm.h-include-linux-t.patch @@ -0,0 +1,37 @@ +From e461958e7a7eb8df2e022e02f782dff5f60a44d0 Mon Sep 17 00:00:00 2001 +From: Thomas Backlund +Date: Sun, 16 Oct 2016 20:09:27 +0300 +Subject: [PATCH] Revert "include/uapi/linux/atm_zatm.h: include linux/time.h" + +This reverts commit cf00713a655d3019be7faa184402f16c43a0fed3. + +It causes linux-atm build breakage between glibc headers and kernel headers... + +Fixws are in progress upstream as of: +https://patchwork.kernel.org/patch/9294305/ + +But it's not accepted yet so it might still change, and since its uapi +we dont want to expose changes that might never land upstream. + +Reported by: Pascal Terjam +Signed-off-by: Thomas Backlund + +--- + include/uapi/linux/atm_zatm.h | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/include/uapi/linux/atm_zatm.h b/include/uapi/linux/atm_zatm.h +index 5cd4d4d..9c9c6ad 100644 +--- a/include/uapi/linux/atm_zatm.h ++++ b/include/uapi/linux/atm_zatm.h +@@ -14,7 +14,6 @@ + + #include + #include +-#include + + #define ZATM_GETPOOL _IOW('a',ATMIOC_SARPRV+1,struct atmif_sioc) + /* get pool statistics */ +-- +2.10.1 + diff --git a/kernel/tools/cpupowertools/files/patches/mageia/ahci-add-new-Intel-device-IDs.patch b/kernel/tools/cpupowertools/files/patches/mageia/ahci-add-new-Intel-device-IDs.patch deleted file mode 100644 index 27ba34e7..00000000 --- a/kernel/tools/cpupowertools/files/patches/mageia/ahci-add-new-Intel-device-IDs.patch +++ /dev/null @@ -1,37 +0,0 @@ -From 56e74338a535cbcc2f2da08b1ea1a92920194364 Mon Sep 17 00:00:00 2001 -From: Alexandra Yates -Date: Tue, 3 Nov 2015 14:14:18 -0800 -Subject: [PATCH] ahci: add new Intel device IDs - -Adding Intel codename Lewisburg platform device IDs for SATA. - -Signed-off-by: Alexandra Yates -Signed-off-by: Tejun Heo ---- - drivers/ata/ahci.c | 10 ++++++++++ - 1 file changed, 10 insertions(+) - -diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c -index ff343c4..ff02bb4 100644 ---- a/drivers/ata/ahci.c -+++ b/drivers/ata/ahci.c -@@ -314,6 +314,16 @@ static const struct pci_device_id ahci_pci_tbl[] = { - { PCI_VDEVICE(INTEL, 0x1f37), board_ahci_avn }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f3e), board_ahci_avn }, /* Avoton RAID */ - { PCI_VDEVICE(INTEL, 0x1f3f), board_ahci_avn }, /* Avoton RAID */ -+ { PCI_VDEVICE(INTEL, 0xa182), board_ahci }, /* Lewisburg AHCI*/ -+ { PCI_VDEVICE(INTEL, 0xa202), board_ahci }, /* Lewisburg AHCI*/ -+ { PCI_VDEVICE(INTEL, 0xa184), board_ahci }, /* Lewisburg RAID*/ -+ { PCI_VDEVICE(INTEL, 0xa204), board_ahci }, /* Lewisburg RAID*/ -+ { PCI_VDEVICE(INTEL, 0xa186), board_ahci }, /* Lewisburg RAID*/ -+ { PCI_VDEVICE(INTEL, 0xa206), board_ahci }, /* Lewisburg RAID*/ -+ { PCI_VDEVICE(INTEL, 0x2822), board_ahci }, /* Lewisburg RAID*/ -+ { PCI_VDEVICE(INTEL, 0x2826), board_ahci }, /* Lewisburg RAID*/ -+ { PCI_VDEVICE(INTEL, 0xa18e), board_ahci }, /* Lewisburg RAID*/ -+ { PCI_VDEVICE(INTEL, 0xa20e), board_ahci }, /* Lewisburg RAID*/ - { PCI_VDEVICE(INTEL, 0x2823), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x2827), board_ahci }, /* Wellsburg RAID */ - { PCI_VDEVICE(INTEL, 0x8d02), board_ahci }, /* Wellsburg AHCI */ --- -2.3.10 - diff --git a/kernel/tools/cpupowertools/files/patches/mageia/arm-0001-ARM-bcm2835-dt-Add-the-ethernet-to-the-device-trees.patch b/kernel/tools/cpupowertools/files/patches/mageia/arm-0001-ARM-bcm2835-dt-Add-the-ethernet-to-the-device-trees.patch deleted file mode 100644 index 0c141707..00000000 --- a/kernel/tools/cpupowertools/files/patches/mageia/arm-0001-ARM-bcm2835-dt-Add-the-ethernet-to-the-device-trees.patch +++ /dev/null @@ -1,145 +0,0 @@ -From f7c8d3a509e51f4a555371ded4957b225f1818ad Mon Sep 17 00:00:00 2001 -From: Lubomir Rintel -Date: Mon, 2 May 2016 09:06:51 +0200 -Subject: [PATCH] ARM: bcm2835: dt: Add the ethernet to the device trees - -The hub and the ethernet in its port 1 are hardwired on the board. - -Compared to the adapters that can be plugged into the USB ports, this -one has no serial EEPROM to store its MAC. Nevertheless, the Raspberry Pi -has the MAC address for this adapter in its ROM, accessible from its -firmware. - -U-Boot can read out the address and set the local-mac-address property of the -node with "ethernet" alias. Let's add the node so that U-Boot can do its -business. - -Model B rev2 and Model B+ entries were verified by me, the hierarchy and -pid/vid pair for the Version 2 was provided by Peter Chen. Original -Model B is a blind shot, though very likely correct. - -Signed-off-by: Lubomir Rintel -Acked-by: Stephen Warren ---- - arch/arm/boot/dts/bcm2835-rpi-b-plus.dts | 1 + - arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts | 1 + - arch/arm/boot/dts/bcm2835-rpi-b.dts | 1 + - arch/arm/boot/dts/bcm2836-rpi-2-b.dts | 1 + - arch/arm/boot/dts/bcm283x-rpi-smsc9512.dtsi | 19 +++++++++++++++++++ - arch/arm/boot/dts/bcm283x-rpi-smsc9514.dtsi | 19 +++++++++++++++++++ - arch/arm/boot/dts/bcm283x.dtsi | 2 ++ - 7 files changed, 44 insertions(+) - create mode 100644 arch/arm/boot/dts/bcm283x-rpi-smsc9512.dtsi - create mode 100644 arch/arm/boot/dts/bcm283x-rpi-smsc9514.dtsi - -diff --git a/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts b/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts -index ef54050..c3e01ce 100644 ---- a/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts -+++ b/arch/arm/boot/dts/bcm2835-rpi-b-plus.dts -@@ -1,6 +1,7 @@ - /dts-v1/; - #include "bcm2835.dtsi" - #include "bcm2835-rpi.dtsi" -+#include "bcm283x-rpi-smsc9514.dtsi" - - / { - compatible = "raspberrypi,model-b-plus", "brcm,bcm2835"; -diff --git a/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts b/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts -index 86f1f2f..364086a 100644 ---- a/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts -+++ b/arch/arm/boot/dts/bcm2835-rpi-b-rev2.dts -@@ -1,6 +1,7 @@ - /dts-v1/; - #include "bcm2835.dtsi" - #include "bcm2835-rpi.dtsi" -+#include "bcm283x-rpi-smsc9512.dtsi" - - / { - compatible = "raspberrypi,model-b-rev2", "brcm,bcm2835"; -diff --git a/arch/arm/boot/dts/bcm2835-rpi-b.dts b/arch/arm/boot/dts/bcm2835-rpi-b.dts -index 4859e9d..4420b09 100644 ---- a/arch/arm/boot/dts/bcm2835-rpi-b.dts -+++ b/arch/arm/boot/dts/bcm2835-rpi-b.dts -@@ -1,6 +1,7 @@ - /dts-v1/; - #include "bcm2835.dtsi" - #include "bcm2835-rpi.dtsi" -+#include "bcm283x-rpi-smsc9512.dtsi" - - / { - compatible = "raspberrypi,model-b", "brcm,bcm2835"; -diff --git a/arch/arm/boot/dts/bcm2836-rpi-2-b.dts b/arch/arm/boot/dts/bcm2836-rpi-2-b.dts -index ff94666..b975f29 100644 ---- a/arch/arm/boot/dts/bcm2836-rpi-2-b.dts -+++ b/arch/arm/boot/dts/bcm2836-rpi-2-b.dts -@@ -1,6 +1,7 @@ - /dts-v1/; - #include "bcm2836.dtsi" - #include "bcm2835-rpi.dtsi" -+#include "bcm283x-rpi-smsc9514.dtsi" - - / { - compatible = "raspberrypi,2-model-b", "brcm,bcm2836"; -diff --git a/arch/arm/boot/dts/bcm283x-rpi-smsc9512.dtsi b/arch/arm/boot/dts/bcm283x-rpi-smsc9512.dtsi -new file mode 100644 -index 0000000..12c981e ---- /dev/null -+++ b/arch/arm/boot/dts/bcm283x-rpi-smsc9512.dtsi -@@ -0,0 +1,19 @@ -+/ { -+ aliases { -+ ethernet = ðernet; -+ }; -+}; -+ -+&usb { -+ usb1@1 { -+ compatible = "usb424,9512"; -+ reg = <1>; -+ #address-cells = <1>; -+ #size-cells = <0>; -+ -+ ethernet: usbether@1 { -+ compatible = "usb424,ec00"; -+ reg = <1>; -+ }; -+ }; -+}; -diff --git a/arch/arm/boot/dts/bcm283x-rpi-smsc9514.dtsi b/arch/arm/boot/dts/bcm283x-rpi-smsc9514.dtsi -new file mode 100644 -index 0000000..3f0a56e ---- /dev/null -+++ b/arch/arm/boot/dts/bcm283x-rpi-smsc9514.dtsi -@@ -0,0 +1,19 @@ -+/ { -+ aliases { -+ ethernet = ðernet; -+ }; -+}; -+ -+&usb { -+ usb1@1 { -+ compatible = "usb424,9514"; -+ reg = <1>; -+ #address-cells = <1>; -+ #size-cells = <0>; -+ -+ ethernet: usbether@1 { -+ compatible = "usb424,ec00"; -+ reg = <1>; -+ }; -+ }; -+}; -diff --git a/arch/arm/boot/dts/bcm283x.dtsi b/arch/arm/boot/dts/bcm283x.dtsi -index 8aaf193..04504b7 100644 ---- a/arch/arm/boot/dts/bcm283x.dtsi -+++ b/arch/arm/boot/dts/bcm283x.dtsi -@@ -287,6 +287,8 @@ - compatible = "brcm,bcm2835-usb"; - reg = <0x7e980000 0x10000>; - interrupts = <1 9>; -+ #address-cells = <1>; -+ #size-cells = <0>; - }; - - v3d: v3d@7ec00000 { diff --git a/kernel/tools/cpupowertools/files/patches/mageia/dm-raid-aliases.patch b/kernel/tools/cpupowertools/files/patches/mageia/dm-raid-aliases.patch index febdb720..4bb20cf1 100644 --- a/kernel/tools/cpupowertools/files/patches/mageia/dm-raid-aliases.patch +++ b/kernel/tools/cpupowertools/files/patches/mageia/dm-raid-aliases.patch @@ -8,13 +8,14 @@ Signed-off-by: Thomas Backlund drivers/md/dm-raid.c | 2 ++ 1 file changed, 2 insertions(+) ---- linux-2.6.38.2-mga2/drivers/md/dm-raid.c.orig 2011-03-15 03:20:32.000000000 +0200 -+++ linux-2.6.38.2-mga2/drivers/md/dm-raid.c 2011-04-02 21:52:21.331302590 +0300 -@@ -693,5 +693,7 @@ MODULE_DESCRIPTION(DM_NAME " raid4/5/6 t +--- linux/drivers/md/dm-raid.c.orig ++++ linux/drivers/md/dm-raid.c +@@ -3689,6 +3689,8 @@ MODULE_ALIAS("dm-raid10"); MODULE_ALIAS("dm-raid4"); MODULE_ALIAS("dm-raid5"); MODULE_ALIAS("dm-raid6"); +MODULE_ALIAS("dm-raid45"); +MODULE_ALIAS("dm-raid4-5"); MODULE_AUTHOR("Neil Brown "); + MODULE_AUTHOR("Heinz Mauelshagen "); MODULE_LICENSE("GPL"); diff --git a/kernel/tools/cpupowertools/files/patches/mageia/dm-raid-fix-compat_features-validation.patch b/kernel/tools/cpupowertools/files/patches/mageia/dm-raid-fix-compat_features-validation.patch new file mode 100644 index 00000000..f2261039 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/dm-raid-fix-compat_features-validation.patch @@ -0,0 +1,39 @@ +From 5c33677c87cbe44ae04df69c4a29c1750a9ec4e5 Mon Sep 17 00:00:00 2001 +From: Andy Whitcroft +Date: Tue, 11 Oct 2016 15:16:57 +0100 +Subject: [PATCH] dm raid: fix compat_features validation + +In ecbfb9f118bce4 ("dm raid: add raid level takeover support") a new +compatible feature flag was added. Validation for these compat_features +was added but this only passes for new raid mappings with this feature +flag. This causes previously created raid mappings to be failed at +import. + +Check compat_features for the only valid combination. + +Fixes: ecbfb9f118bce4 ("dm raid: add raid level takeover support") +Cc: stable@vger.kernel.org # v4.8 +Signed-off-by: Andy Whitcroft +Signed-off-by: Heinz Mauelshagen +Signed-off-by: Mike Snitzer +--- + drivers/md/dm-raid.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c +index 8abde6b..2a39700 100644 +--- a/drivers/md/dm-raid.c ++++ b/drivers/md/dm-raid.c +@@ -2258,7 +2258,8 @@ static int super_validate(struct raid_set *rs, struct md_rdev *rdev) + if (!mddev->events && super_init_validation(rs, rdev)) + return -EINVAL; + +- if (le32_to_cpu(sb->compat_features) != FEATURE_FLAG_SUPPORTS_V190) { ++ if (le32_to_cpu(sb->compat_features) && ++ le32_to_cpu(sb->compat_features) != FEATURE_FLAG_SUPPORTS_V190) { + rs->ti->error = "Unable to assemble array: Unknown flag(s) in compatible feature flags"; + return -EINVAL; + } +-- +2.10.1 + diff --git a/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.7-ver-fix.patch b/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.7-ver-fix.patch deleted file mode 100644 index 7ffd27e4..00000000 --- a/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.7-ver-fix.patch +++ /dev/null @@ -1,16 +0,0 @@ - -Mark as 4.7 so we can rebuild aufs tools with matching 4.7 check - -Signed-off-by: Thomas Backlund - ---- linux/include/uapi/linux/aufs_type.h.orig -+++ linux/include/uapi/linux/aufs_type.h -@@ -26,7 +26,7 @@ - - #include - --#define AUFS_VERSION "4.x-rcN" -+#define AUFS_VERSION "4.7" - - /* todo? move this to linux-2.6.19/include/magic.h */ - #define AUFS_SUPER_MAGIC ('a' << 24 | 'u' << 16 | 'f' << 8 | 's') diff --git a/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.7-modular.patch b/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.8-modular.patch similarity index 71% rename from kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.7-modular.patch rename to kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.8-modular.patch index faedd6af..62a3b4f0 100644 --- a/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.7-modular.patch +++ b/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.8-modular.patch @@ -18,9 +18,9 @@ security/security.c | 10 ++++++++++ 17 files changed, 41 insertions(+), 1 deletion(-) -diff -Nurp linux-4.7-rc6-aufs/fs/aufs/Kconfig linux-4.7-rc6-aufs-mod/fs/aufs/Kconfig ---- linux-4.7-rc6-aufs/fs/aufs/Kconfig 2016-07-04 19:02:29.246286120 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/aufs/Kconfig 2016-07-04 19:03:53.653714965 +0300 +diff -Nurp linux-4.8-aufs/fs/aufs/Kconfig linux-4.8-aufs-mod/fs/aufs/Kconfig +--- linux-4.8-aufs/fs/aufs/Kconfig 2016-07-04 19:02:29.246286120 +0300 ++++ linux-4.8-aufs-mod/fs/aufs/Kconfig 2016-07-04 19:03:53.653714965 +0300 @@ -1,5 +1,5 @@ config AUFS_FS - bool "Aufs (Advanced multi layered unification filesystem) support" @@ -28,9 +28,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/aufs/Kconfig linux-4.7-rc6-aufs-mod/fs/aufs/Kco help Aufs is a stackable unification filesystem such as Unionfs, which unifies several directories and provides a merged single -diff -Nurp linux-4.7-rc6-aufs/fs/dcache.c linux-4.7-rc6-aufs-mod/fs/dcache.c ---- linux-4.7-rc6-aufs/fs/dcache.c 2016-07-04 19:02:29.260286196 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/dcache.c 2016-07-04 19:03:53.653714965 +0300 +diff -Nurp linux-4.8-aufs/fs/dcache.c linux-4.8-aufs-mod/fs/dcache.c +--- linux-4.8-aufs/fs/dcache.c 2016-07-04 19:02:29.260286196 +0300 ++++ linux-4.8-aufs-mod/fs/dcache.c 2016-07-04 19:03:53.653714965 +0300 @@ -1310,6 +1310,7 @@ rename_retry: seq = 1; goto again; @@ -39,9 +39,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/dcache.c linux-4.7-rc6-aufs-mod/fs/dcache.c /* * Search for at least 1 mount point in the dentry's subdirs. -diff -Nurp linux-4.7-rc6-aufs/fs/exec.c linux-4.7-rc6-aufs-mod/fs/exec.c ---- linux-4.7-rc6-aufs/fs/exec.c 2016-07-04 17:58:15.707773651 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/exec.c 2016-07-04 19:03:53.654714970 +0300 +diff -Nurp linux-4.8-aufs/fs/exec.c linux-4.8-aufs-mod/fs/exec.c +--- linux-4.8-aufs/fs/exec.c 2016-07-04 17:58:15.707773651 +0300 ++++ linux-4.8-aufs-mod/fs/exec.c 2016-07-04 19:03:53.654714970 +0300 @@ -104,6 +104,7 @@ bool path_noexec(const struct path *path return (path->mnt->mnt_flags & MNT_NOEXEC) || (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); @@ -50,9 +50,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/exec.c linux-4.7-rc6-aufs-mod/fs/exec.c #ifdef CONFIG_USELIB /* -diff -Nurp linux-4.7-rc6-aufs/fs/fcntl.c linux-4.7-rc6-aufs-mod/fs/fcntl.c ---- linux-4.7-rc6-aufs/fs/fcntl.c 2016-07-04 19:02:29.260286196 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/fcntl.c 2016-07-04 19:03:53.654714970 +0300 +diff -Nurp linux-4.8-aufs/fs/fcntl.c linux-4.8-aufs-mod/fs/fcntl.c +--- linux-4.8-aufs/fs/fcntl.c 2016-07-04 19:02:29.260286196 +0300 ++++ linux-4.8-aufs-mod/fs/fcntl.c 2016-07-04 19:03:53.654714970 +0300 @@ -82,6 +82,7 @@ int setfl(int fd, struct file * filp, un out: return error; @@ -61,9 +61,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/fcntl.c linux-4.7-rc6-aufs-mod/fs/fcntl.c static void f_modown(struct file *filp, struct pid *pid, enum pid_type type, int force) -diff -Nurp linux-4.7-rc6-aufs/fs/file_table.c linux-4.7-rc6-aufs-mod/fs/file_table.c ---- linux-4.7-rc6-aufs/fs/file_table.c 2016-05-16 01:43:13.000000000 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/file_table.c 2016-07-04 19:03:53.654714970 +0300 +diff -Nurp linux-4.8-aufs/fs/file_table.c linux-4.8-aufs-mod/fs/file_table.c +--- linux-4.8-aufs/fs/file_table.c 2016-05-16 01:43:13.000000000 +0300 ++++ linux-4.8-aufs-mod/fs/file_table.c 2016-07-04 19:03:53.654714970 +0300 @@ -147,6 +147,7 @@ over: } return ERR_PTR(-ENFILE); @@ -96,9 +96,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/file_table.c linux-4.7-rc6-aufs-mod/fs/file_tab void __init files_init(void) { -diff -Nurp linux-4.7-rc6-aufs/fs/inode.c linux-4.7-rc6-aufs-mod/fs/inode.c ---- linux-4.7-rc6-aufs/fs/inode.c 2016-07-04 19:02:29.261286202 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/inode.c 2016-07-04 20:51:39.805168596 +0300 +diff -Nurp linux-4.8-aufs/fs/inode.c linux-4.8-aufs-mod/fs/inode.c +--- linux-4.8-aufs/fs/inode.c 2016-07-04 19:02:29.261286202 +0300 ++++ linux-4.8-aufs-mod/fs/inode.c 2016-07-04 20:51:39.805168596 +0300 @@ -1600,6 +1600,7 @@ int update_time(struct inode *inode, str return update_time(inode, time, flags); @@ -107,9 +107,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/inode.c linux-4.7-rc6-aufs-mod/fs/inode.c /** * touch_atime - update the access time -diff -Nurp linux-4.7-rc6-aufs/fs/namespace.c linux-4.7-rc6-aufs-mod/fs/namespace.c ---- linux-4.7-rc6-aufs/fs/namespace.c 2016-07-04 17:58:15.729773645 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/namespace.c 2016-07-04 19:03:53.654714970 +0300 +diff -Nurp linux-4.8-aufs/fs/namespace.c linux-4.8-aufs-mod/fs/namespace.c +--- linux-4.8-aufs/fs/namespace.c 2016-07-04 17:58:15.729773645 +0300 ++++ linux-4.8-aufs-mod/fs/namespace.c 2016-07-04 19:03:53.654714970 +0300 @@ -463,6 +463,7 @@ void __mnt_drop_write(struct vfsmount *m mnt_dec_writers(real_mount(mnt)); preempt_enable(); @@ -126,9 +126,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/namespace.c linux-4.7-rc6-aufs-mod/fs/namespace static void cleanup_group_ids(struct mount *mnt, struct mount *end) { -diff -Nurp linux-4.7-rc6-aufs/fs/notify/group.c linux-4.7-rc6-aufs-mod/fs/notify/group.c ---- linux-4.7-rc6-aufs/fs/notify/group.c 2016-07-04 17:58:15.742773641 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/notify/group.c 2016-07-04 19:03:53.655714975 +0300 +diff -Nurp linux-4.8-aufs/fs/notify/group.c linux-4.8-aufs-mod/fs/notify/group.c +--- linux-4.8-aufs/fs/notify/group.c 2016-07-04 17:58:15.742773641 +0300 ++++ linux-4.8-aufs-mod/fs/notify/group.c 2016-07-04 19:03:53.655714975 +0300 @@ -22,6 +22,7 @@ #include #include @@ -161,9 +161,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/notify/group.c linux-4.7-rc6-aufs-mod/fs/notify int fsnotify_fasync(int fd, struct file *file, int on) { -diff -Nurp linux-4.7-rc6-aufs/fs/notify/mark.c linux-4.7-rc6-aufs-mod/fs/notify/mark.c ---- linux-4.7-rc6-aufs/fs/notify/mark.c 2016-07-04 17:58:15.742773641 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/notify/mark.c 2016-07-04 19:03:53.655714975 +0300 +diff -Nurp linux-4.8-aufs/fs/notify/mark.c linux-4.8-aufs-mod/fs/notify/mark.c +--- linux-4.8-aufs/fs/notify/mark.c 2016-07-04 17:58:15.742773641 +0300 ++++ linux-4.8-aufs-mod/fs/notify/mark.c 2016-07-04 19:03:53.655714975 +0300 @@ -113,6 +113,7 @@ void fsnotify_put_mark(struct fsnotify_m mark->free_mark(mark); } @@ -196,9 +196,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/notify/mark.c linux-4.7-rc6-aufs-mod/fs/notify/ /* * Nothing fancy, just initialize lists and locks and counters. -diff -Nurp linux-4.7-rc6-aufs/fs/open.c linux-4.7-rc6-aufs-mod/fs/open.c ---- linux-4.7-rc6-aufs/fs/open.c 2016-07-04 17:58:15.746773640 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/open.c 2016-07-04 19:03:53.655714975 +0300 +diff -Nurp linux-4.8-aufs/fs/open.c linux-4.8-aufs-mod/fs/open.c +--- linux-4.8-aufs/fs/open.c 2016-07-04 17:58:15.746773640 +0300 ++++ linux-4.8-aufs-mod/fs/open.c 2016-07-04 19:03:53.655714975 +0300 @@ -64,6 +64,7 @@ int do_truncate(struct dentry *dentry, l inode_unlock(dentry->d_inode); return ret; @@ -215,9 +215,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/open.c linux-4.7-rc6-aufs-mod/fs/open.c static int do_dentry_open(struct file *f, struct inode *inode, -diff -Nurp linux-4.7-rc6-aufs/fs/read_write.c linux-4.7-rc6-aufs-mod/fs/read_write.c ---- linux-4.7-rc6-aufs/fs/read_write.c 2016-07-04 19:02:29.262286207 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/read_write.c 2016-07-04 19:03:53.655714975 +0300 +diff -Nurp linux-4.8-aufs/fs/read_write.c linux-4.8-aufs-mod/fs/read_write.c +--- linux-4.8-aufs/fs/read_write.c 2016-07-04 19:02:29.262286207 +0300 ++++ linux-4.8-aufs-mod/fs/read_write.c 2016-07-04 19:03:53.655714975 +0300 @@ -525,6 +525,7 @@ vfs_readf_t vfs_readf(struct file *file) return new_sync_read; return ERR_PTR(-ENOSYS); @@ -234,9 +234,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/read_write.c linux-4.7-rc6-aufs-mod/fs/read_wri ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos) { -diff -Nurp linux-4.7-rc6-aufs/fs/splice.c linux-4.7-rc6-aufs-mod/fs/splice.c ---- linux-4.7-rc6-aufs/fs/splice.c 2016-07-04 19:02:29.262286207 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/splice.c 2016-07-04 19:03:53.656714979 +0300 +diff -Nurp linux-4.8-aufs/fs/splice.c linux-4.8-aufs-mod/fs/splice.c +--- linux-4.8-aufs/fs/splice.c 2016-07-04 19:02:29.262286207 +0300 ++++ linux-4.8-aufs-mod/fs/splice.c 2016-07-04 19:03:53.656714979 +0300 @@ -1124,6 +1124,7 @@ long do_splice_from(struct pipe_inode_in return splice_write(pipe, out, ppos, len, flags); @@ -253,9 +253,9 @@ diff -Nurp linux-4.7-rc6-aufs/fs/splice.c linux-4.7-rc6-aufs-mod/fs/splice.c /** * splice_direct_to_actor - splices data directly between two non-pipes -diff -Nurp linux-4.7-rc6-aufs/fs/xattr.c linux-4.7-rc6-aufs-mod/fs/xattr.c ---- linux-4.7-rc6-aufs/fs/xattr.c 2016-07-04 17:58:15.754773638 +0300 -+++ linux-4.7-rc6-aufs-mod/fs/xattr.c 2016-07-04 19:03:53.656714979 +0300 +diff -Nurp linux-4.8-aufs/fs/xattr.c linux-4.8-aufs-mod/fs/xattr.c +--- linux-4.8-aufs/fs/xattr.c 2016-07-04 17:58:15.754773638 +0300 ++++ linux-4.8-aufs-mod/fs/xattr.c 2016-07-04 19:03:53.656714979 +0300 @@ -207,6 +207,7 @@ vfs_getxattr_alloc(struct dentry *dentry *xattr_value = value; return error; @@ -264,17 +264,17 @@ diff -Nurp linux-4.7-rc6-aufs/fs/xattr.c linux-4.7-rc6-aufs-mod/fs/xattr.c ssize_t vfs_getxattr(struct dentry *dentry, const char *name, void *value, size_t size) -diff -Nurp linux-4.7-rc6-aufs/kernel/task_work.c linux-4.7-rc6-aufs-mod/kernel/task_work.c ---- linux-4.7-rc6-aufs/kernel/task_work.c 2016-05-16 01:43:13.000000000 +0300 -+++ linux-4.7-rc6-aufs-mod/kernel/task_work.c 2016-07-04 19:03:53.656714979 +0300 +diff -Nurp linux-4.8-aufs/kernel/task_work.c linux-4.8-aufs-mod/kernel/task_work.c +--- linux-4.8-aufs/kernel/task_work.c 2016-05-16 01:43:13.000000000 +0300 ++++ linux-4.8-aufs-mod/kernel/task_work.c 2016-07-04 19:03:53.656714979 +0300 @@ -118,3 +118,4 @@ void task_work_run(void) } while (work); } } +EXPORT_SYMBOL_GPL(task_work_run); -diff -Nurp linux-4.7-rc6-aufs/security/commoncap.c linux-4.7-rc6-aufs-mod/security/commoncap.c ---- linux-4.7-rc6-aufs/security/commoncap.c 2016-07-04 17:58:15.945773585 +0300 -+++ linux-4.7-rc6-aufs-mod/security/commoncap.c 2016-07-04 19:03:53.656714979 +0300 +diff -Nurp linux-4.8-aufs/security/commoncap.c linux-4.8-aufs-mod/security/commoncap.c +--- linux-4.8-aufs/security/commoncap.c 2016-07-04 17:58:15.945773585 +0300 ++++ linux-4.8-aufs-mod/security/commoncap.c 2016-07-04 19:03:53.656714979 +0300 @@ -1058,12 +1058,14 @@ int cap_mmap_addr(unsigned long addr) } return ret; @@ -290,9 +290,9 @@ diff -Nurp linux-4.7-rc6-aufs/security/commoncap.c linux-4.7-rc6-aufs-mod/securi #ifdef CONFIG_SECURITY -diff -Nurp linux-4.7-rc6-aufs/security/device_cgroup.c linux-4.7-rc6-aufs-mod/security/device_cgroup.c ---- linux-4.7-rc6-aufs/security/device_cgroup.c 2016-05-16 01:43:13.000000000 +0300 -+++ linux-4.7-rc6-aufs-mod/security/device_cgroup.c 2016-07-04 19:03:53.656714979 +0300 +diff -Nurp linux-4.8-aufs/security/device_cgroup.c linux-4.8-aufs-mod/security/device_cgroup.c +--- linux-4.8-aufs/security/device_cgroup.c 2016-05-16 01:43:13.000000000 +0300 ++++ linux-4.8-aufs-mod/security/device_cgroup.c 2016-07-04 19:03:53.656714979 +0300 @@ -7,6 +7,7 @@ #include #include @@ -309,9 +309,9 @@ diff -Nurp linux-4.7-rc6-aufs/security/device_cgroup.c linux-4.7-rc6-aufs-mod/se int devcgroup_inode_mknod(int mode, dev_t dev) { -diff -Nurp linux-4.7-rc6-aufs/security/security.c linux-4.7-rc6-aufs-mod/security/security.c ---- linux-4.7-rc6-aufs/security/security.c 2016-07-04 17:58:15.948773584 +0300 -+++ linux-4.7-rc6-aufs-mod/security/security.c 2016-07-04 19:03:53.657714984 +0300 +diff -Nurp linux-4.8-aufs/security/security.c linux-4.8-aufs-mod/security/security.c +--- linux-4.8-aufs/security/security.c 2016-07-04 17:58:15.948773584 +0300 ++++ linux-4.8-aufs-mod/security/security.c 2016-07-04 19:03:53.657714984 +0300 @@ -434,6 +434,7 @@ int security_path_rmdir(const struct pat return 0; return call_int_hook(path_rmdir, 0, dir, dentry); diff --git a/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.7.patch b/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.8.patch similarity index 96% rename from kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.7.patch rename to kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.8.patch index d2d09b83..1efb3528 100644 --- a/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.7.patch +++ b/kernel/tools/cpupowertools/files/patches/mageia/fs-aufs-4.8.patch @@ -1,4 +1,117 @@ + Documentation/ABI/testing/debugfs-aufs | 50 + Documentation/ABI/testing/sysfs-aufs | 31 + Documentation/filesystems/aufs/README | 392 +++ + Documentation/filesystems/aufs/design/01intro.txt | 157 + + Documentation/filesystems/aufs/design/02struct.txt | 245 ++ + Documentation/filesystems/aufs/design/03atomic_open.txt | 72 + Documentation/filesystems/aufs/design/03lookup.txt | 100 + Documentation/filesystems/aufs/design/04branch.txt | 61 + Documentation/filesystems/aufs/design/05wbr_policy.txt | 51 + Documentation/filesystems/aufs/design/06fhsm.txt | 105 + Documentation/filesystems/aufs/design/06mmap.txt | 59 + Documentation/filesystems/aufs/design/06xattr.txt | 81 + Documentation/filesystems/aufs/design/07export.txt | 45 + Documentation/filesystems/aufs/design/08shwh.txt | 39 + Documentation/filesystems/aufs/design/10dynop.txt | 34 + MAINTAINERS | 13 + drivers/block/loop.c | 18 + fs/Kconfig | 1 + fs/Makefile | 1 + fs/aufs/Kconfig | 185 + + fs/aufs/Makefile | 36 + fs/aufs/aufs.h | 46 + fs/aufs/branch.c | 1399 ++++++++++++ + fs/aufs/branch.h | 296 ++ + fs/aufs/cpup.c | 1378 +++++++++++ + fs/aufs/cpup.h | 81 + fs/aufs/dbgaufs.c | 425 +++ + fs/aufs/dbgaufs.h | 35 + fs/aufs/dcsub.c | 212 + + fs/aufs/dcsub.h | 123 + + fs/aufs/debug.c | 427 +++ + fs/aufs/debug.h | 212 + + fs/aufs/dentry.c | 1117 +++++++++ + fs/aufs/dentry.h | 242 ++ + fs/aufs/dinfo.c | 540 ++++ + fs/aufs/dir.c | 749 ++++++ + fs/aufs/dir.h | 124 + + fs/aufs/dynop.c | 358 +++ + fs/aufs/dynop.h | 61 + fs/aufs/export.c | 824 +++++++ + fs/aufs/f_op.c | 759 ++++++ + fs/aufs/fhsm.c | 412 +++ + fs/aufs/file.c | 844 +++++++ + fs/aufs/file.h | 281 ++ + fs/aufs/finfo.c | 138 + + fs/aufs/fstype.h | 387 +++ + fs/aufs/hfsnotify.c | 274 ++ + fs/aufs/hfsplus.c | 43 + fs/aufs/hnotify.c | 710 ++++++ + fs/aufs/i_op.c | 1438 ++++++++++++ + fs/aufs/i_op_add.c | 911 +++++++ + fs/aufs/i_op_del.c | 498 ++++ + fs/aufs/i_op_ren.c | 1002 ++++++++ + fs/aufs/iinfo.c | 272 ++ + fs/aufs/inode.c | 506 ++++ + fs/aufs/inode.h | 687 +++++ + fs/aufs/ioctl.c | 206 + + fs/aufs/loop.c | 134 + + fs/aufs/loop.h | 39 + fs/aufs/magic.mk | 30 + fs/aufs/module.c | 319 ++ + fs/aufs/module.h | 143 + + fs/aufs/mvdown.c | 691 +++++ + fs/aufs/opts.c | 1847 ++++++++++++++++ + fs/aufs/opts.h | 198 + + fs/aufs/plink.c | 501 ++++ + fs/aufs/poll.c | 39 + fs/aufs/posix_acl.c | 85 + fs/aufs/procfs.c | 156 + + fs/aufs/rdu.c | 368 +++ + fs/aufs/rwsem.h | 185 + + fs/aufs/sbinfo.c | 342 ++ + fs/aufs/spl.h | 100 + fs/aufs/super.c | 1025 ++++++++ + fs/aufs/super.h | 625 +++++ + fs/aufs/sysaufs.c | 91 + fs/aufs/sysaufs.h | 88 + fs/aufs/sysfs.c | 340 ++ + fs/aufs/sysrq.c | 144 + + fs/aufs/vdir.c | 887 +++++++ + fs/aufs/vfsub.c | 871 +++++++ + fs/aufs/vfsub.h | 303 ++ + fs/aufs/wbr_policy.c | 752 ++++++ + fs/aufs/whout.c | 1047 +++++++++ + fs/aufs/whout.h | 72 + fs/aufs/wkq.c | 200 + + fs/aufs/wkq.h | 80 + fs/aufs/xattr.c | 334 ++ + fs/aufs/xino.c | 1305 +++++++++++ + fs/dcache.c | 2 + fs/fcntl.c | 4 + fs/inode.c | 2 + fs/proc/base.c | 2 + fs/proc/nommu.c | 5 + fs/proc/task_mmu.c | 7 + fs/proc/task_nommu.c | 5 + fs/read_write.c | 22 + fs/splice.c | 10 + include/linux/file.h | 1 + include/linux/fs.h | 9 + include/linux/mm.h | 22 + include/linux/mm_types.h | 2 + include/linux/splice.h | 6 + include/uapi/linux/Kbuild | 1 + include/uapi/linux/aufs_type.h | 406 +++ + kernel/fork.c | 2 + mm/Makefile | 2 + mm/filemap.c | 2 + mm/memory.c | 2 + mm/mmap.c | 33 + mm/nommu.c | 10 + mm/prfile.c | 86 + 112 files changed, 33747 insertions(+), 30 deletions(-) Documentation/ABI/testing/debugfs-aufs | 50 + Documentation/ABI/testing/sysfs-aufs | 31 + Documentation/filesystems/aufs/README | 392 +++++ @@ -21,73 +134,73 @@ fs/aufs/Kconfig | 185 ++ fs/aufs/Makefile | 36 + fs/aufs/aufs.h | 46 + - fs/aufs/branch.c | 1393 +++++++++++++++ + fs/aufs/branch.c | 1399 +++++++++++++++ fs/aufs/branch.h | 296 ++++ - fs/aufs/cpup.c | 1370 +++++++++++++++ + fs/aufs/cpup.c | 1378 +++++++++++++++ fs/aufs/cpup.h | 81 + - fs/aufs/dbgaufs.c | 419 +++++ + fs/aufs/dbgaufs.c | 425 +++++ fs/aufs/dbgaufs.h | 35 + - fs/aufs/dcsub.c | 211 +++ + fs/aufs/dcsub.c | 212 +++ fs/aufs/dcsub.h | 123 ++ - fs/aufs/debug.c | 428 +++++ + fs/aufs/debug.c | 427 +++++ fs/aufs/debug.h | 212 +++ - fs/aufs/dentry.c | 1115 ++++++++++++ - fs/aufs/dentry.h | 239 +++ - fs/aufs/dinfo.c | 539 ++++++ - fs/aufs/dir.c | 743 ++++++++ - fs/aufs/dir.h | 118 ++ - fs/aufs/dynop.c | 356 ++++ + fs/aufs/dentry.c | 1117 ++++++++++++ + fs/aufs/dentry.h | 242 +++ + fs/aufs/dinfo.c | 540 ++++++ + fs/aufs/dir.c | 749 ++++++++ + fs/aufs/dir.h | 124 ++ + fs/aufs/dynop.c | 358 ++++ fs/aufs/dynop.h | 61 + fs/aufs/export.c | 824 +++++++++ - fs/aufs/f_op.c | 757 ++++++++ + fs/aufs/f_op.c | 759 ++++++++ fs/aufs/fhsm.c | 412 +++++ - fs/aufs/file.c | 830 +++++++++ - fs/aufs/file.h | 278 +++ - fs/aufs/finfo.c | 136 ++ + fs/aufs/file.c | 844 +++++++++ + fs/aufs/file.h | 281 +++ + fs/aufs/finfo.c | 138 ++ fs/aufs/fstype.h | 387 ++++ fs/aufs/hfsnotify.c | 274 +++ fs/aufs/hfsplus.c | 43 + - fs/aufs/hnotify.c | 697 ++++++++ - fs/aufs/i_op.c | 1401 +++++++++++++++ + fs/aufs/hnotify.c | 710 ++++++++ + fs/aufs/i_op.c | 1438 +++++++++++++++ fs/aufs/i_op_add.c | 911 ++++++++++ fs/aufs/i_op_del.c | 498 ++++++ fs/aufs/i_op_ren.c | 1002 +++++++++++ - fs/aufs/iinfo.c | 271 +++ - fs/aufs/inode.c | 504 ++++++ - fs/aufs/inode.h | 681 ++++++++ + fs/aufs/iinfo.c | 272 +++ + fs/aufs/inode.c | 506 ++++++ + fs/aufs/inode.h | 687 ++++++++ fs/aufs/ioctl.c | 206 +++ - fs/aufs/loop.c | 133 ++ + fs/aufs/loop.c | 134 ++ fs/aufs/loop.h | 39 + fs/aufs/magic.mk | 30 + - fs/aufs/module.c | 209 +++ - fs/aufs/module.h | 76 + + fs/aufs/module.c | 319 ++++ + fs/aufs/module.h | 143 ++ fs/aufs/mvdown.c | 691 ++++++++ - fs/aufs/opts.c | 1846 ++++++++++++++++++++ + fs/aufs/opts.c | 1847 ++++++++++++++++++++ fs/aufs/opts.h | 198 +++ - fs/aufs/plink.c | 489 ++++++ + fs/aufs/plink.c | 501 ++++++ fs/aufs/poll.c | 39 + fs/aufs/posix_acl.c | 85 + fs/aufs/procfs.c | 156 ++ fs/aufs/rdu.c | 368 ++++ fs/aufs/rwsem.h | 185 ++ - fs/aufs/sbinfo.c | 340 ++++ - fs/aufs/spl.h | 98 ++ - fs/aufs/super.c | 1026 +++++++++++ + fs/aufs/sbinfo.c | 342 ++++ + fs/aufs/spl.h | 100 ++ + fs/aufs/super.c | 1025 +++++++++++ fs/aufs/super.h | 625 +++++++ fs/aufs/sysaufs.c | 91 + fs/aufs/sysaufs.h | 88 + fs/aufs/sysfs.c | 340 ++++ fs/aufs/sysrq.c | 144 ++ - fs/aufs/vdir.c | 876 ++++++++++ + fs/aufs/vdir.c | 887 ++++++++++ fs/aufs/vfsub.c | 871 +++++++++ - fs/aufs/vfsub.h | 297 ++++ + fs/aufs/vfsub.h | 303 ++++ fs/aufs/wbr_policy.c | 752 ++++++++ fs/aufs/whout.c | 1047 +++++++++++ fs/aufs/whout.h | 72 + - fs/aufs/wkq.c | 205 +++ - fs/aufs/wkq.h | 82 + - fs/aufs/xattr.c | 332 ++++ - fs/aufs/xino.c | 1304 ++++++++++++++ + fs/aufs/wkq.c | 200 +++ + fs/aufs/wkq.h | 80 + + fs/aufs/xattr.c | 334 ++++ + fs/aufs/xino.c | 1305 ++++++++++++++ fs/dcache.c | 2 +- fs/fcntl.c | 4 +- fs/inode.c | 2 +- @@ -111,7 +224,7 @@ mm/mmap.c | 33 +- mm/nommu.c | 10 +- mm/prfile.c | 86 + - 112 files changed, 33420 insertions(+), 30 deletions(-) + 112 files changed, 33747 insertions(+), 30 deletions(-) diff --git a/Documentation/ABI/testing/debugfs-aufs b/Documentation/ABI/testing/debugfs-aufs new file mode 100644 @@ -1726,10 +1839,10 @@ index 0000000..49afc58 +Currently this approach is applied to address_space_operations for +regular files only. diff --git a/MAINTAINERS b/MAINTAINERS -index 952fd2a..6a8f0f8 100644 +index f593300..8a17054 100644 --- a/MAINTAINERS +++ b/MAINTAINERS -@@ -2210,6 +2210,19 @@ F: include/linux/audit.h +@@ -2256,6 +2256,19 @@ F: include/linux/audit.h F: include/uapi/linux/audit.h F: kernel/audit* @@ -1750,10 +1863,10 @@ index 952fd2a..6a8f0f8 100644 M: Miguel Ojeda Sandonis W: http://miguelojeda.es/auxdisplay.htm diff --git a/drivers/block/loop.c b/drivers/block/loop.c -index 1fa8cc2..7339e65 100644 +index c9f2107..005e292 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c -@@ -712,6 +712,24 @@ static inline int is_loop_device(struct file *file) +@@ -701,6 +701,24 @@ static inline int is_loop_device(struct file *file) return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR; } @@ -1779,10 +1892,10 @@ index 1fa8cc2..7339e65 100644 static ssize_t loop_attr_show(struct device *dev, char *page, diff --git a/fs/Kconfig b/fs/Kconfig -index b8fcb41..78adefb 100644 +index 2bc7ad7..3049386 100644 --- a/fs/Kconfig +++ b/fs/Kconfig -@@ -236,6 +236,7 @@ source "fs/pstore/Kconfig" +@@ -245,6 +245,7 @@ source "fs/pstore/Kconfig" source "fs/sysv/Kconfig" source "fs/ufs/Kconfig" source "fs/exofs/Kconfig" @@ -1791,10 +1904,10 @@ index b8fcb41..78adefb 100644 endif # MISC_FILESYSTEMS diff --git a/fs/Makefile b/fs/Makefile -index 85b6e13..e7bb164 100644 +index ed2b632..aa6d14b 100644 --- a/fs/Makefile +++ b/fs/Makefile -@@ -128,3 +128,4 @@ obj-y += exofs/ # Multiple modules +@@ -129,3 +129,4 @@ obj-y += exofs/ # Multiple modules obj-$(CONFIG_CEPH_FS) += ceph/ obj-$(CONFIG_PSTORE) += pstore/ obj-$(CONFIG_EFIVAR_FS) += efivarfs/ @@ -2086,10 +2199,10 @@ index 0000000..49f43b4 +#endif /* __AUFS_H__ */ diff --git a/fs/aufs/branch.c b/fs/aufs/branch.c new file mode 100644 -index 0000000..baf786e +index 0000000..3bfbe5b --- /dev/null +++ b/fs/aufs/branch.c -@@ -0,0 +1,1393 @@ +@@ -0,0 +1,1399 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -2130,7 +2243,7 @@ index 0000000..baf786e + + if (br->br_fhsm) { + au_br_fhsm_fin(br->br_fhsm); -+ kfree(br->br_fhsm); ++ au_delayed_kfree(br->br_fhsm); + } + + key = br->br_dykey; @@ -2144,8 +2257,9 @@ index 0000000..baf786e + lockdep_off(); + path_put(&br->br_path); + lockdep_on(); -+ kfree(wbr); -+ kfree(br); ++ if (wbr) ++ au_delayed_kfree(wbr); ++ au_delayed_kfree(br); +} + +/* @@ -2232,22 +2346,23 @@ index 0000000..baf786e + goto out_wbr; + } + -+ err = au_sbr_realloc(au_sbi(sb), new_nbranch); ++ err = au_sbr_realloc(au_sbi(sb), new_nbranch, /*may_shrink*/0); + if (!err) -+ err = au_di_realloc(au_di(root), new_nbranch); ++ err = au_di_realloc(au_di(root), new_nbranch, /*may_shrink*/0); + if (!err) { + inode = d_inode(root); -+ err = au_hinode_realloc(au_ii(inode), new_nbranch); ++ err = au_hinode_realloc(au_ii(inode), new_nbranch, /*may_shrink*/0); + } + if (!err) + return add_branch; /* success */ + +out_wbr: -+ kfree(add_branch->br_wbr); ++ if (add_branch->br_wbr) ++ au_delayed_kfree(add_branch->br_wbr); +out_hnotify: + au_hnotify_fin_br(add_branch); +out_br: -+ kfree(add_branch); ++ au_delayed_kfree(add_branch); +out: + return ERR_PTR(err); +} @@ -2413,7 +2528,7 @@ index 0000000..baf786e + br->br_perm = old_perm; + + if (!err && wbr && !au_br_writable(new_perm)) { -+ kfree(wbr); ++ au_delayed_kfree(wbr); + br->br_wbr = NULL; + } + @@ -2980,7 +3095,8 @@ index 0000000..baf786e + sbinfo->si_branch[0 + bbot] = NULL; + sbinfo->si_bbot--; + -+ p = krealloc(sbinfo->si_branch, sizeof(*p) * bbot, AuGFP_SBILIST); ++ p = au_krealloc(sbinfo->si_branch, sizeof(*p) * bbot, AuGFP_SBILIST, ++ /*may_shrink*/1); + if (p) + sbinfo->si_branch = p; + /* harmless error */ @@ -2999,7 +3115,8 @@ index 0000000..baf786e + /* au_h_dentry_init(au_hdentry(dinfo, bbot); */ + dinfo->di_bbot--; + -+ p = krealloc(dinfo->di_hdentry, sizeof(*p) * bbot, AuGFP_SBILIST); ++ p = au_krealloc(dinfo->di_hdentry, sizeof(*p) * bbot, AuGFP_SBILIST, ++ /*may_shrink*/1); + if (p) + dinfo->di_hdentry = p; + /* harmless error */ @@ -3018,7 +3135,8 @@ index 0000000..baf786e + /* au_hinode_init(au_hinode(iinfo, bbot)); */ + iinfo->ii_bbot--; + -+ p = krealloc(iinfo->ii_hinode, sizeof(*p) * bbot, AuGFP_SBILIST); ++ p = au_krealloc(iinfo->ii_hinode, sizeof(*p) * bbot, AuGFP_SBILIST, ++ /*may_shrink*/1); + if (p) + iinfo->ii_hinode = p; + /* harmless error */ @@ -3436,7 +3554,7 @@ index 0000000..baf786e + if (br->br_wbr) { + err = au_wbr_init(br, sb, mod->perm); + if (unlikely(err)) { -+ kfree(br->br_wbr); ++ au_delayed_kfree(br->br_wbr); + br->br_wbr = NULL; + } + } @@ -3448,7 +3566,7 @@ index 0000000..baf786e + if (!au_br_fhsm(mod->perm)) { + /* fhsm --> non-fhsm */ + au_br_fhsm_fin(br->br_fhsm); -+ kfree(br->br_fhsm); ++ au_delayed_kfree(br->br_fhsm); + br->br_fhsm = NULL; + } + } else if (au_br_fhsm(mod->perm)) @@ -3460,7 +3578,8 @@ index 0000000..baf786e + goto out; /* success */ + +out_bf: -+ kfree(bf); ++ if (bf) ++ au_delayed_kfree(bf); +out: + AuTraceErr(err); + return err; @@ -3787,10 +3906,10 @@ index 0000000..32a4d8f +#endif /* __AUFS_BRANCH_H__ */ diff --git a/fs/aufs/cpup.c b/fs/aufs/cpup.c new file mode 100644 -index 0000000..52f87de +index 0000000..598a4d6 --- /dev/null +++ b/fs/aufs/cpup.c -@@ -0,0 +1,1370 @@ +@@ -0,0 +1,1378 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -4137,9 +4256,9 @@ index 0000000..52f87de + dst->f_pos = 0; + err = au_do_copy_file(dst, src, len, buf, blksize); + if (do_kfree) -+ kfree(buf); ++ au_delayed_kfree(buf); + else -+ free_page((unsigned long)buf); ++ au_delayed_free_page((unsigned long)buf); + +out: + return err; @@ -4174,6 +4293,7 @@ index 0000000..52f87de + } + }; + struct super_block *sb; ++ struct inode *h_src_inode; + struct task_struct *tsk = current; + + /* bsrc branch can be ro/rw. */ @@ -4189,7 +4309,9 @@ index 0000000..52f87de + } + + /* try stopping to update while we copyup */ -+ IMustLock(d_inode(file[SRC].dentry)); ++ h_src_inode = d_inode(file[SRC].dentry); ++ if (!au_test_nfs(h_src_inode->i_sb)) ++ IMustLock(h_src_inode); + err = au_copy_file(file[DST].file, file[SRC].file, cpg->len); + + /* i wonder if we had O_NO_DELAY_FPUT flag */ @@ -4249,8 +4371,13 @@ index 0000000..52f87de + goto out; + } + h_src_attr->valid = 1; -+ err = au_cp_regular(cpg); -+ inode_unlock(h_src_inode); ++ if (!au_test_nfs(h_src_inode->i_sb)) { ++ err = au_cp_regular(cpg); ++ inode_unlock(h_src_inode); ++ } else { ++ inode_unlock(h_src_inode); ++ err = au_cp_regular(cpg); ++ } + rerr = au_pin_hdir_relock(cpg->pin); + if (!err && rerr) + err = rerr; @@ -4299,7 +4426,7 @@ index 0000000..52f87de + sym.k[symlen] = 0; + err = vfsub_symlink(h_dir, h_path, sym.k); + } -+ free_page((unsigned long)sym.k); ++ au_delayed_free_page((unsigned long)sym.k); + +out: + return err; @@ -4670,7 +4797,7 @@ index 0000000..52f87de + } +out_parent: + dput(dst_parent); -+ kfree(a); ++ au_delayed_kfree(a); +out: + return err; +} @@ -5250,10 +5377,10 @@ index 0000000..ccba2c4 +#endif /* __AUFS_CPUP_H__ */ diff --git a/fs/aufs/dbgaufs.c b/fs/aufs/dbgaufs.c new file mode 100644 -index 0000000..21ac773 +index 0000000..4db01e8 --- /dev/null +++ b/fs/aufs/dbgaufs.c -@@ -0,0 +1,419 @@ +@@ -0,0 +1,425 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -5284,7 +5411,7 @@ index 0000000..21ac773 +static int dbgaufs_xi_release(struct inode *inode __maybe_unused, + struct file *file) +{ -+ kfree(file->private_data); ++ au_delayed_kfree(file->private_data); + return 0; +} + @@ -5346,7 +5473,7 @@ index 0000000..21ac773 +static int dbgaufs_plink_release(struct inode *inode __maybe_unused, + struct file *file) +{ -+ free_page((unsigned long)file->private_data); ++ au_delayed_free_page((unsigned long)file->private_data); + return 0; +} + @@ -5410,7 +5537,7 @@ index 0000000..21ac773 + goto out; /* success */ + +out_free: -+ free_page((unsigned long)p); ++ au_delayed_free_page((unsigned long)p); +out: + return err; +} @@ -5512,7 +5639,10 @@ index 0000000..21ac773 + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + xi = &br->br_xino; ++ /* debugfs acquires the parent i_mutex */ ++ lockdep_off(); + debugfs_remove(xi->xi_dbgaufs); ++ lockdep_on(); + xi->xi_dbgaufs = NULL; + } +} @@ -5537,8 +5667,11 @@ index 0000000..21ac773 + br = au_sbr(sb, bindex); + xi = &br->br_xino; + AuDebugOn(xi->xi_dbgaufs); ++ /* debugfs acquires the parent i_mutex */ ++ lockdep_off(); + xi->xi_dbgaufs = debugfs_create_file(name, dbgaufs_mode, parent, + sbinfo, &dbgaufs_xino_fop); ++ lockdep_on(); + /* ignore an error */ + if (unlikely(!xi->xi_dbgaufs)) + AuWarn1("failed %s under debugfs\n", name); @@ -5716,10 +5849,10 @@ index 0000000..81f272e +#endif /* __DBGAUFS_H__ */ diff --git a/fs/aufs/dcsub.c b/fs/aufs/dcsub.c new file mode 100644 -index 0000000..e72acce +index 0000000..5d46398 --- /dev/null +++ b/fs/aufs/dcsub.c -@@ -0,0 +1,211 @@ +@@ -0,0 +1,212 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -5738,7 +5871,7 @@ index 0000000..e72acce + p = dpage->dentries; + for (i = 0; i < dpage->ndentry; i++) + dput(*p++); -+ free_page((unsigned long)dpage->dentries); ++ au_delayed_free_page((unsigned long)dpage->dentries); +} + +int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp) @@ -5761,7 +5894,7 @@ index 0000000..e72acce + return 0; /* success */ + +out_dpages: -+ kfree(dpages->dpages); ++ au_delayed_kfree(dpages->dpages); +out: + return err; +} @@ -5774,7 +5907,7 @@ index 0000000..e72acce + p = dpages->dpages; + for (i = 0; i < dpages->ndpage; i++) + au_dpage_free(p++); -+ kfree(dpages->dpages); ++ au_delayed_kfree(dpages->dpages); +} + +static int au_dpages_append(struct au_dcsub_pages *dpages, @@ -5791,7 +5924,8 @@ index 0000000..e72acce + err = -ENOMEM; + sz = dpages->ndpage * sizeof(*dpages->dpages); + p = au_kzrealloc(dpages->dpages, sz, -+ sz + sizeof(*dpages->dpages), gfp); ++ sz + sizeof(*dpages->dpages), gfp, ++ /*may_shrink*/0); + if (unlikely(!p)) + goto out; + @@ -6062,10 +6196,10 @@ index 0000000..5d2cf66 +#endif /* __AUFS_DCSUB_H__ */ diff --git a/fs/aufs/debug.c b/fs/aufs/debug.c new file mode 100644 -index 0000000..44bdb5f +index 0000000..443fee3 --- /dev/null +++ b/fs/aufs/debug.c -@@ -0,0 +1,428 @@ +@@ -0,0 +1,427 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -6391,7 +6525,7 @@ index 0000000..44bdb5f + au_br_count_init(&a->fake); + err = do_pri_br(-1, &a->fake); + au_br_count_fin(&a->fake); -+ kfree(a); ++ au_delayed_kfree(a); + dpri("dev 0x%x\n", sb->s_dev); + if (err || !au_test_aufs(sb)) + return; @@ -6399,9 +6533,8 @@ index 0000000..44bdb5f + sbinfo = au_sbi(sb); + if (!sbinfo) + return; -+ dpri("nw %lld, gen %u, kobj %d\n", -+ percpu_counter_sum(&sbinfo->si_nowait.nw_len), -+ sbinfo->si_generation, ++ dpri("nw %d, gen %u, kobj %d\n", ++ atomic_read(&sbinfo->si_nowait.nw_len), sbinfo->si_generation, + atomic_read(&sbinfo->si_kobj.kref.refcount)); + for (bindex = 0; bindex <= sbinfo->si_bbot; bindex++) + do_pri_br(bindex, sbinfo->si_branch[0 + bindex]); @@ -6714,10 +6847,10 @@ index 0000000..0567f31 +#endif /* __AUFS_DEBUG_H__ */ diff --git a/fs/aufs/dentry.c b/fs/aufs/dentry.c new file mode 100644 -index 0000000..4bb1530 +index 0000000..5eb4f36 --- /dev/null +++ b/fs/aufs/dentry.c -@@ -0,0 +1,1115 @@ +@@ -0,0 +1,1117 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -6756,7 +6889,7 @@ index 0000000..4bb1530 + br = au_sbr(dentry->d_sb, bindex); + wh_able = !!au_br_whable(br->br_perm); + if (wh_able) -+ wh_found = au_wh_test(h_parent, wh_name, /*try_sio*/0); ++ wh_found = au_wh_test(h_parent, wh_name, ignore_perm); + h_dentry = ERR_PTR(wh_found); + if (!wh_found) + goto real_lookup; @@ -6921,7 +7054,7 @@ index 0000000..4bb1530 + +out_parent: + dput(parent); -+ kfree(whname.name); ++ au_delayed_kfree(whname.name); +out: + return err; +} @@ -7421,7 +7554,7 @@ index 0000000..4bb1530 + +int au_refresh_dentry(struct dentry *dentry, struct dentry *parent) +{ -+ int err, ebrange; ++ int err, ebrange, nbr; + unsigned int sigen; + struct au_dinfo *dinfo, *tmp; + struct super_block *sb; @@ -7437,8 +7570,9 @@ index 0000000..4bb1530 + if (unlikely(err)) + goto out; + ++ nbr = au_sbbot(sb) + 1; + dinfo = au_di(dentry); -+ err = au_di_realloc(dinfo, au_sbbot(sb) + 1); ++ err = au_di_realloc(dinfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + ebrange = au_dbrange_test(dentry); @@ -7481,6 +7615,7 @@ index 0000000..4bb1530 + au_dbg_verify_dinode(dentry); + AuTraceErr(err); + } ++ au_di_realloc(dinfo, nbr, /*may_shrink*/1); /* harmless if err */ + au_rw_write_unlock(&tmp->di_rwsem); + au_di_free(tmp); + if (unlikely(err)) @@ -7835,10 +7970,10 @@ index 0000000..4bb1530 +}; diff --git a/fs/aufs/dentry.h b/fs/aufs/dentry.h new file mode 100644 -index 0000000..bdc1af6 +index 0000000..01211d2 --- /dev/null +++ b/fs/aufs/dentry.h -@@ -0,0 +1,239 @@ +@@ -0,0 +1,242 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -7866,7 +8001,10 @@ index 0000000..bdc1af6 + struct au_rwsem di_rwsem; + aufs_bindex_t di_btop, di_bbot, di_bwh, di_bdiropq; + unsigned char di_tmpfile; /* to allow the different name */ -+ struct au_hdentry *di_hdentry; ++ union { ++ struct au_hdentry *di_hdentry; ++ struct llist_node di_lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ @@ -7904,7 +8042,7 @@ index 0000000..bdc1af6 +void au_di_cp(struct au_dinfo *dst, struct au_dinfo *src); +int au_di_init(struct dentry *dentry); +void au_di_fin(struct dentry *dentry); -+int au_di_realloc(struct au_dinfo *dinfo, int nbr); ++int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink); + +void di_read_lock(struct dentry *d, int flags, unsigned int lsc); +void di_read_unlock(struct dentry *d, int flags); @@ -8080,10 +8218,10 @@ index 0000000..bdc1af6 +#endif /* __AUFS_DENTRY_H__ */ diff --git a/fs/aufs/dinfo.c b/fs/aufs/dinfo.c new file mode 100644 -index 0000000..43b5509 +index 0000000..2626c0c --- /dev/null +++ b/fs/aufs/dinfo.c -@@ -0,0 +1,539 @@ +@@ -0,0 +1,540 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -8126,7 +8264,7 @@ index 0000000..43b5509 + goto out; + } + -+ au_cache_free_dinfo(dinfo); ++ au_cache_dfree_dinfo(dinfo); + dinfo = NULL; + +out: @@ -8146,8 +8284,8 @@ index 0000000..43b5509 + while (bindex++ <= bbot) + au_hdput(p++); + } -+ kfree(dinfo->di_hdentry); -+ au_cache_free_dinfo(dinfo); ++ au_delayed_kfree(dinfo->di_hdentry); ++ au_cache_dfree_dinfo(dinfo); +} + +void au_di_swap(struct au_dinfo *a, struct au_dinfo *b) @@ -8215,7 +8353,7 @@ index 0000000..43b5509 + au_di_free(dinfo); +} + -+int au_di_realloc(struct au_dinfo *dinfo, int nbr) ++int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink) +{ + int err, sz; + struct au_hdentry *hdp; @@ -8226,7 +8364,8 @@ index 0000000..43b5509 + sz = sizeof(*hdp) * (dinfo->di_bbot + 1); + if (!sz) + sz = sizeof(*hdp); -+ hdp = au_kzrealloc(dinfo->di_hdentry, sz, sizeof(*hdp) * nbr, GFP_NOFS); ++ hdp = au_kzrealloc(dinfo->di_hdentry, sz, sizeof(*hdp) * nbr, GFP_NOFS, ++ may_shrink); + if (hdp) { + dinfo->di_hdentry = hdp; + err = 0; @@ -8625,10 +8764,10 @@ index 0000000..43b5509 +} diff --git a/fs/aufs/dir.c b/fs/aufs/dir.c new file mode 100644 -index 0000000..27bef0b +index 0000000..395563e --- /dev/null +++ b/fs/aufs/dir.c -@@ -0,0 +1,743 @@ +@@ -0,0 +1,749 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -8774,7 +8913,7 @@ index 0000000..27bef0b +out: + dput(a->dentry); + au_nwt_done(&au_sbi(sb)->si_nowait); -+ kfree(arg); ++ au_delayed_kfree(arg); +} + +void au_dir_ts(struct inode *dir, aufs_bindex_t bindex) @@ -8810,7 +8949,7 @@ index 0000000..27bef0b + if (unlikely(wkq_err)) { + pr_err("wkq %d\n", wkq_err); + dput(dentry); -+ kfree(arg); ++ au_delayed_kfree(arg); + } + +out: @@ -8929,7 +9068,7 @@ index 0000000..27bef0b + }; + err = au_do_open(file, &args); + if (unlikely(err)) -+ kfree(fidir); ++ au_delayed_kfree(fidir); + } + si_read_unlock(sb); + return err; @@ -8941,8 +9080,11 @@ index 0000000..27bef0b + struct au_vdir *vdir_cache; + struct au_finfo *finfo; + struct au_fidir *fidir; ++ struct au_hfile *hf; + aufs_bindex_t bindex, bbot; ++ int execed, delayed; + ++ delayed = (current->flags & PF_KTHREAD) || in_interrupt(); + finfo = au_fi(file); + fidir = finfo->fi_hdir; + if (fidir) { @@ -8950,22 +9092,25 @@ index 0000000..27bef0b + &au_sbi(file->f_path.dentry->d_sb)->si_files); + vdir_cache = fidir->fd_vdir_cache; /* lock-free */ + if (vdir_cache) -+ au_vdir_free(vdir_cache); ++ au_vdir_free(vdir_cache, delayed); + + bindex = finfo->fi_btop; + if (bindex >= 0) { ++ execed = vfsub_file_execed(file); ++ hf = fidir->fd_hfile + bindex; + /* + * calls fput() instead of filp_close(), + * since no dnotify or lock for the lower file. + */ + bbot = fidir->fd_bbot; -+ for (; bindex <= bbot; bindex++) -+ au_set_h_fptr(file, bindex, NULL); ++ for (; bindex <= bbot; bindex++, hf++) ++ if (hf->hf_file) ++ au_hfput(hf, execed); + } -+ kfree(fidir); ++ au_delayed_kfree(fidir); + finfo->fi_hdir = NULL; + } -+ au_finfo_fin(file); ++ au_finfo_fin(file, delayed); + return 0; +} + @@ -9374,10 +9519,10 @@ index 0000000..27bef0b +}; diff --git a/fs/aufs/dir.h b/fs/aufs/dir.h new file mode 100644 -index 0000000..b0a79d7 +index 0000000..4f3945a --- /dev/null +++ b/fs/aufs/dir.h -@@ -0,0 +1,118 @@ +@@ -0,0 +1,124 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -9409,7 +9554,10 @@ index 0000000..b0a79d7 + +struct au_vdir_dehstr { + struct hlist_node hash; -+ struct au_vdir_destr *str; ++ union { ++ struct au_vdir_destr *str; ++ struct llist_node lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +struct au_vdir_de { @@ -9447,7 +9595,10 @@ index 0000000..b0a79d7 + + unsigned long vd_version; + unsigned int vd_deblk_sz; -+ unsigned long vd_jiffy; ++ union { ++ unsigned long vd_jiffy; ++ struct llist_node vd_lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ @@ -9471,7 +9622,7 @@ index 0000000..b0a79d7 +int au_nhash_append_wh(struct au_nhash *whlist, char *name, int nlen, ino_t ino, + unsigned int d_type, aufs_bindex_t bindex, + unsigned char shwh); -+void au_vdir_free(struct au_vdir *vdir); ++void au_vdir_free(struct au_vdir *vdir, int atonce); +int au_vdir_init(struct file *file); +int au_vdir_fill_de(struct file *file, struct dir_context *ctx); + @@ -9498,10 +9649,10 @@ index 0000000..b0a79d7 +#endif /* __AUFS_DIR_H__ */ diff --git a/fs/aufs/dynop.c b/fs/aufs/dynop.c new file mode 100644 -index 0000000..dfb3a71 +index 0000000..eb0e1a7 --- /dev/null +++ b/fs/aufs/dynop.c -@@ -0,0 +1,356 @@ +@@ -0,0 +1,358 @@ +/* + * Copyright (C) 2010-2016 Junjiro R. Okajima + */ @@ -9518,17 +9669,17 @@ index 0000000..dfb3a71 + * How large will these lists be? + * Usually just a few elements, 20-30 at most for each, I guess. + */ -+static struct au_splhead dynop[AuDyLast]; ++static struct au_sphlhead dynop[AuDyLast]; + -+static struct au_dykey *dy_gfind_get(struct au_splhead *spl, const void *h_op) ++static struct au_dykey *dy_gfind_get(struct au_sphlhead *sphl, const void *h_op) +{ + struct au_dykey *key, *tmp; -+ struct list_head *head; ++ struct hlist_head *head; + + key = NULL; -+ head = &spl->head; ++ head = &sphl->head; + rcu_read_lock(); -+ list_for_each_entry_rcu(tmp, head, dk_list) ++ hlist_for_each_entry_rcu(tmp, head, dk_hnode) + if (tmp->dk_op.dy_hop == h_op) { + key = tmp; + kref_get(&key->dk_kref); @@ -9575,24 +9726,24 @@ index 0000000..dfb3a71 +} + +/* kref_get() if @key is already added */ -+static struct au_dykey *dy_gadd(struct au_splhead *spl, struct au_dykey *key) ++static struct au_dykey *dy_gadd(struct au_sphlhead *sphl, struct au_dykey *key) +{ + struct au_dykey *tmp, *found; -+ struct list_head *head; ++ struct hlist_head *head; + const void *h_op = key->dk_op.dy_hop; + + found = NULL; -+ head = &spl->head; -+ spin_lock(&spl->spin); -+ list_for_each_entry(tmp, head, dk_list) ++ head = &sphl->head; ++ spin_lock(&sphl->spin); ++ hlist_for_each_entry(tmp, head, dk_hnode) + if (tmp->dk_op.dy_hop == h_op) { + kref_get(&tmp->dk_kref); + found = tmp; + break; + } + if (!found) -+ list_add_rcu(&key->dk_list, head); -+ spin_unlock(&spl->spin); ++ hlist_add_head_rcu(&key->dk_hnode, head); ++ spin_unlock(&sphl->spin); + + if (!found) + DyPrSym(key); @@ -9605,17 +9756,17 @@ index 0000000..dfb3a71 + + key = container_of(rcu, struct au_dykey, dk_rcu); + DyPrSym(key); -+ kfree(key); ++ kfree(key); /* not delayed */ +} + +static void dy_free(struct kref *kref) +{ + struct au_dykey *key; -+ struct au_splhead *spl; ++ struct au_sphlhead *sphl; + + key = container_of(kref, struct au_dykey, dk_kref); -+ spl = dynop + key->dk_op.dy_type; -+ au_spl_del_rcu(&key->dk_list, spl); ++ sphl = dynop + key->dk_op.dy_type; ++ au_sphl_del_rcu(&key->dk_hnode, sphl); + call_rcu(&key->dk_rcu, dy_free_rcu); +} + @@ -9680,6 +9831,8 @@ index 0000000..dfb3a71 + /* this one will be changed according to an aufs mount option */ + DySetAop(direct_IO); + DySetAop(migratepage); ++ DySetAop(isolate_page); ++ DySetAop(putback_page); + DySetAop(launder_page); + DySetAop(is_partially_uptodate); + DySetAop(is_dirty_writeback); @@ -9700,7 +9853,7 @@ index 0000000..dfb3a71 +static struct au_dykey *dy_get(struct au_dynop *op, struct au_branch *br) +{ + struct au_dykey *key, *old; -+ struct au_splhead *spl; ++ struct au_sphlhead *sphl; + struct op { + unsigned int sz; + void (*set)(struct au_dykey *key, const void *h_op, @@ -9714,8 +9867,8 @@ index 0000000..dfb3a71 + }; + const struct op *p; + -+ spl = dynop + op->dy_type; -+ key = dy_gfind_get(spl, op->dy_hop); ++ sphl = dynop + op->dy_type; ++ key = dy_gfind_get(sphl, op->dy_hop); + if (key) + goto out_add; /* success */ + @@ -9729,9 +9882,9 @@ index 0000000..dfb3a71 + key->dk_op.dy_hop = op->dy_hop; + kref_init(&key->dk_kref); + p->set(key, op->dy_hop, au_br_sb(br)); -+ old = dy_gadd(spl, key); ++ old = dy_gadd(sphl, key); + if (old) { -+ kfree(key); ++ au_delayed_kfree(key); + key = old; + } + @@ -9826,16 +9979,16 @@ index 0000000..dfb3a71 + +void au_dy_arefresh(int do_dx) +{ -+ struct au_splhead *spl; -+ struct list_head *head; ++ struct au_sphlhead *sphl; ++ struct hlist_head *head; + struct au_dykey *key; + -+ spl = dynop + AuDy_AOP; -+ head = &spl->head; -+ spin_lock(&spl->spin); -+ list_for_each_entry(key, head, dk_list) ++ sphl = dynop + AuDy_AOP; ++ head = &sphl->head; ++ spin_lock(&sphl->spin); ++ hlist_for_each_entry(key, head, dk_hnode) + dy_adx((void *)key, do_dx); -+ spin_unlock(&spl->spin); ++ spin_unlock(&sphl->spin); +} + +/* ---------------------------------------------------------------------- */ @@ -9848,7 +10001,7 @@ index 0000000..dfb3a71 + BUILD_BUG_ON(offsetof(struct au_dyaop, da_key)); + + for (i = 0; i < AuDyLast; i++) -+ au_spl_init(dynop + i); ++ au_sphl_init(dynop + i); +} + +void au_dy_fin(void) @@ -9856,11 +10009,11 @@ index 0000000..dfb3a71 + int i; + + for (i = 0; i < AuDyLast; i++) -+ WARN_ON(!list_empty(&dynop[i].head)); ++ WARN_ON(!hlist_empty(&dynop[i].head)); +} diff --git a/fs/aufs/dynop.h b/fs/aufs/dynop.h new file mode 100644 -index 0000000..8680bfc +index 0000000..5db2da2 --- /dev/null +++ b/fs/aufs/dynop.h @@ -0,0 +1,61 @@ @@ -9892,7 +10045,7 @@ index 0000000..8680bfc + +struct au_dykey { + union { -+ struct list_head dk_list; ++ struct hlist_node dk_hnode; + struct rcu_head dk_rcu; + }; + struct au_dynop dk_op; @@ -9927,7 +10080,7 @@ index 0000000..8680bfc +#endif /* __AUFS_DYNOP_H__ */ diff --git a/fs/aufs/export.c b/fs/aufs/export.c new file mode 100644 -index 0000000..98a7036 +index 0000000..6b5a7be --- /dev/null +++ b/fs/aufs/export.c @@ -0,0 +1,824 @@ @@ -10337,7 +10490,7 @@ index 0000000..98a7036 + } + +out_name: -+ free_page((unsigned long)arg.name); ++ au_delayed_free_page((unsigned long)arg.name); +out_file: + fput(file); +out: @@ -10491,7 +10644,7 @@ index 0000000..98a7036 + dentry = ERR_PTR(-ESTALE); + } +out_pathname: -+ free_page((unsigned long)pathname); ++ au_delayed_free_page((unsigned long)pathname); +out_h_parent: + dput(h_parent); +out: @@ -10757,10 +10910,10 @@ index 0000000..98a7036 +} diff --git a/fs/aufs/f_op.c b/fs/aufs/f_op.c new file mode 100644 -index 0000000..504b767 +index 0000000..00475fb --- /dev/null +++ b/fs/aufs/f_op.c -@@ -0,0 +1,757 @@ +@@ -0,0 +1,759 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -10849,6 +11002,7 @@ index 0000000..504b767 +{ + struct au_finfo *finfo; + aufs_bindex_t bindex; ++ int delayed; + + finfo = au_fi(file); + au_sphl_del(&finfo->fi_hlist, @@ -10857,7 +11011,8 @@ index 0000000..504b767 + if (bindex >= 0) + au_set_h_fptr(file, bindex, NULL); + -+ au_finfo_fin(file); ++ delayed = (current->flags & PF_KTHREAD) || in_interrupt(); ++ au_finfo_fin(file, delayed); + return 0; +} + @@ -11938,10 +12093,10 @@ index 0000000..e3cb6ed +} diff --git a/fs/aufs/file.c b/fs/aufs/file.c new file mode 100644 -index 0000000..4b68aef +index 0000000..bfc1d1b --- /dev/null +++ b/fs/aufs/file.c -@@ -0,0 +1,830 @@ +@@ -0,0 +1,844 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -12204,7 +12359,7 @@ index 0000000..4b68aef + } + if (unlikely(err)) { + finfo->fi_hdir = NULL; -+ au_finfo_fin(file); ++ au_finfo_fin(file, /*atonce*/0); + } + +out: @@ -12524,6 +12679,7 @@ index 0000000..4b68aef + +static void au_do_refresh_dir(struct file *file) +{ ++ int execed; + aufs_bindex_t bindex, bbot, new_bindex, brid; + struct au_hfile *p, tmp, *q; + struct au_finfo *finfo; @@ -12562,6 +12718,7 @@ index 0000000..4b68aef + } + } + ++ execed = vfsub_file_execed(file); + p = fidir->fd_hfile; + if (!au_test_mmapped(file) && !d_unlinked(file->f_path.dentry)) { + bbot = au_sbbot(sb); @@ -12570,14 +12727,14 @@ index 0000000..4b68aef + if (p->hf_file) { + if (file_inode(p->hf_file)) + break; -+ au_hfput(p, file); ++ au_hfput(p, execed); + } + } else { + bbot = au_br_index(sb, brid); + for (finfo->fi_btop = 0; finfo->fi_btop < bbot; + finfo->fi_btop++, p++) + if (p->hf_file) -+ au_hfput(p, file); ++ au_hfput(p, execed); + bbot = au_sbbot(sb); + } + @@ -12587,7 +12744,7 @@ index 0000000..4b68aef + if (p->hf_file) { + if (file_inode(p->hf_file)) + break; -+ au_hfput(p, file); ++ au_hfput(p, execed); + } + AuDebugOn(fidir->fd_bbot < finfo->fi_btop); +} @@ -12597,23 +12754,26 @@ index 0000000..4b68aef + */ +static int refresh_file(struct file *file, int (*reopen)(struct file *file)) +{ -+ int err, need_reopen; ++ int err, need_reopen, nbr; + aufs_bindex_t bbot, bindex; + struct dentry *dentry; ++ struct super_block *sb; + struct au_finfo *finfo; + struct au_hfile *hfile; + + dentry = file->f_path.dentry; ++ sb = dentry->d_sb; ++ nbr = au_sbbot(sb) + 1; + finfo = au_fi(file); + if (!finfo->fi_hdir) { + hfile = &finfo->fi_htop; + AuDebugOn(!hfile->hf_file); -+ bindex = au_br_index(dentry->d_sb, hfile->hf_br->br_id); ++ bindex = au_br_index(sb, hfile->hf_br->br_id); + AuDebugOn(bindex < 0); + if (bindex != finfo->fi_btop) + au_set_fbtop(file, bindex); + } else { -+ err = au_fidir_realloc(finfo, au_sbbot(dentry->d_sb) + 1); ++ err = au_fidir_realloc(finfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + au_do_refresh_dir(file); @@ -12623,6 +12783,9 @@ index 0000000..4b68aef + need_reopen = 1; + if (!au_test_mmapped(file)) + err = au_file_refresh_by_inode(file, &need_reopen); ++ if (finfo->fi_hdir) ++ /* harmless if err */ ++ au_fidir_realloc(finfo, nbr, /*may_shrink*/1); + if (!err && need_reopen && !d_unlinked(dentry)) + err = reopen(file); + if (!err) { @@ -12730,6 +12893,10 @@ index 0000000..4b68aef + struct page *page, enum migrate_mode mode) +{ AuUnsupport(); return 0; } +#endif ++static bool aufs_isolate_page(struct page *page, isolate_mode_t mode) ++{ AuUnsupport(); return true; } ++static void aufs_putback_page(struct page *page) ++{ AuUnsupport(); } +static int aufs_launder_page(struct page *page) +{ AuUnsupport(); return 0; } +static int aufs_is_partially_uptodate(struct page *page, @@ -12764,6 +12931,8 @@ index 0000000..4b68aef + .releasepage = aufs_releasepage, + /* is fallback_migrate_page ok? */ + /* .migratepage = aufs_migratepage, */ ++ .isolate_page = aufs_isolate_page, ++ .putback_page = aufs_putback_page, + .launder_page = aufs_launder_page, + .is_partially_uptodate = aufs_is_partially_uptodate, + .is_dirty_writeback = aufs_is_dirty_writeback, @@ -12774,10 +12943,10 @@ index 0000000..4b68aef +}; diff --git a/fs/aufs/file.h b/fs/aufs/file.h new file mode 100644 -index 0000000..96ab0a0 +index 0000000..f53b381 --- /dev/null +++ b/fs/aufs/file.h -@@ -0,0 +1,278 @@ +@@ -0,0 +1,281 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -12830,7 +12999,10 @@ index 0000000..96ab0a0 + struct au_fidir *fi_hdir; /* for dir only */ + + struct hlist_node fi_hlist; -+ struct file *fi_file; /* very ugly */ ++ union { ++ struct file *fi_file; /* very ugly */ ++ struct llist_node fi_lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ @@ -12881,16 +13053,16 @@ index 0000000..96ab0a0 +struct file *au_read_pre(struct file *file, int keep_fi); + +/* finfo.c */ -+void au_hfput(struct au_hfile *hf, struct file *file); ++void au_hfput(struct au_hfile *hf, int execed); +void au_set_h_fptr(struct file *file, aufs_bindex_t bindex, + struct file *h_file); + +void au_update_figen(struct file *file); +struct au_fidir *au_fidir_alloc(struct super_block *sb); -+int au_fidir_realloc(struct au_finfo *finfo, int nbr); ++int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink); + +void au_fi_init_once(void *_fi); -+void au_finfo_fin(struct file *file); ++void au_finfo_fin(struct file *file, int atonce); +int au_finfo_init(struct file *file, struct au_fidir *fidir); + +/* ioctl.c */ @@ -13058,10 +13230,10 @@ index 0000000..96ab0a0 +#endif /* __AUFS_FILE_H__ */ diff --git a/fs/aufs/finfo.c b/fs/aufs/finfo.c new file mode 100644 -index 0000000..07f6eb3 +index 0000000..0f016ed --- /dev/null +++ b/fs/aufs/finfo.c -@@ -0,0 +1,136 @@ +@@ -0,0 +1,138 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -13072,10 +13244,9 @@ index 0000000..07f6eb3 + +#include "aufs.h" + -+void au_hfput(struct au_hfile *hf, struct file *file) ++void au_hfput(struct au_hfile *hf, int execed) +{ -+ /* todo: direct access f_flags */ -+ if (vfsub_file_flags(file) & __FMODE_EXEC) ++ if (execed) + allow_write_access(hf->hf_file); + fput(hf->hf_file); + hf->hf_file = NULL; @@ -13097,7 +13268,7 @@ index 0000000..07f6eb3 + hf = fidir->fd_hfile + bindex; + + if (hf && hf->hf_file) -+ au_hfput(hf, file); ++ au_hfput(hf, vfsub_file_execed(file)); + if (val) { + FiMustWriteLock(file); + AuDebugOn(IS_ERR_OR_NULL(file->f_path.dentry)); @@ -13131,7 +13302,7 @@ index 0000000..07f6eb3 + return fidir; +} + -+int au_fidir_realloc(struct au_finfo *finfo, int nbr) ++int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink) +{ + int err; + struct au_fidir *fidir, *p; @@ -13142,7 +13313,7 @@ index 0000000..07f6eb3 + + err = -ENOMEM; + p = au_kzrealloc(fidir, au_fidir_sz(fidir->fd_nent), au_fidir_sz(nbr), -+ GFP_NOFS); ++ GFP_NOFS, may_shrink); + if (p) { + p->fd_nent = nbr; + finfo->fi_hdir = p; @@ -13154,7 +13325,7 @@ index 0000000..07f6eb3 + +/* ---------------------------------------------------------------------- */ + -+void au_finfo_fin(struct file *file) ++void au_finfo_fin(struct file *file, int atonce) +{ + struct au_finfo *finfo; + @@ -13163,7 +13334,10 @@ index 0000000..07f6eb3 + finfo = au_fi(file); + AuDebugOn(finfo->fi_hdir); + AuRwDestroy(&finfo->fi_rwsem); -+ au_cache_free_finfo(finfo); ++ if (!atonce) ++ au_cache_dfree_finfo(finfo); ++ else ++ au_cache_free_finfo(finfo); +} + +void au_fi_init_once(void *_finfo) @@ -13200,7 +13374,7 @@ index 0000000..07f6eb3 +} diff --git a/fs/aufs/fstype.h b/fs/aufs/fstype.h new file mode 100644 -index 0000000..725b2ff +index 0000000..95b78b9 --- /dev/null +++ b/fs/aufs/fstype.h @@ -0,0 +1,387 @@ @@ -13234,7 +13408,7 @@ index 0000000..725b2ff + +static inline int au_test_iso9660(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_ISO9660_FS) || defined(CONFIG_ISO9660_FS_MODULE) ++#if IS_ENABLED(CONFIG_ISO9660_FS) + return sb->s_magic == ISOFS_SUPER_MAGIC; +#else + return 0; @@ -13243,7 +13417,7 @@ index 0000000..725b2ff + +static inline int au_test_romfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_ROMFS_FS) || defined(CONFIG_ROMFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_ROMFS_FS) + return sb->s_magic == ROMFS_MAGIC; +#else + return 0; @@ -13252,7 +13426,7 @@ index 0000000..725b2ff + +static inline int au_test_cramfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_CRAMFS) || defined(CONFIG_CRAMFS_MODULE) ++#if IS_ENABLED(CONFIG_CRAMFS) + return sb->s_magic == CRAMFS_MAGIC; +#endif + return 0; @@ -13260,7 +13434,7 @@ index 0000000..725b2ff + +static inline int au_test_nfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_NFS_FS) || defined(CONFIG_NFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_NFS_FS) + return sb->s_magic == NFS_SUPER_MAGIC; +#else + return 0; @@ -13269,7 +13443,7 @@ index 0000000..725b2ff + +static inline int au_test_fuse(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_FUSE_FS) || defined(CONFIG_FUSE_FS_MODULE) ++#if IS_ENABLED(CONFIG_FUSE_FS) + return sb->s_magic == FUSE_SUPER_MAGIC; +#else + return 0; @@ -13278,7 +13452,7 @@ index 0000000..725b2ff + +static inline int au_test_xfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_XFS_FS) || defined(CONFIG_XFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_XFS_FS) + return sb->s_magic == XFS_SB_MAGIC; +#else + return 0; @@ -13296,7 +13470,7 @@ index 0000000..725b2ff + +static inline int au_test_ecryptfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_ECRYPT_FS) || defined(CONFIG_ECRYPT_FS_MODULE) ++#if IS_ENABLED(CONFIG_ECRYPT_FS) + return !strcmp(au_sbtype(sb), "ecryptfs"); +#else + return 0; @@ -13310,7 +13484,7 @@ index 0000000..725b2ff + +static inline int au_test_ubifs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_UBIFS_FS) || defined(CONFIG_UBIFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_UBIFS_FS) + return sb->s_magic == UBIFS_SUPER_MAGIC; +#else + return 0; @@ -13337,7 +13511,7 @@ index 0000000..725b2ff + +static inline int au_test_configfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_CONFIGFS_FS) || defined(CONFIG_CONFIGFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_CONFIGFS_FS) + return sb->s_magic == CONFIGFS_MAGIC; +#else + return 0; @@ -13346,7 +13520,7 @@ index 0000000..725b2ff + +static inline int au_test_minix(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_MINIX_FS) || defined(CONFIG_MINIX_FS_MODULE) ++#if IS_ENABLED(CONFIG_MINIX_FS) + return sb->s_magic == MINIX3_SUPER_MAGIC + || sb->s_magic == MINIX2_SUPER_MAGIC + || sb->s_magic == MINIX2_SUPER_MAGIC2 @@ -13359,7 +13533,7 @@ index 0000000..725b2ff + +static inline int au_test_fat(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_FAT_FS) || defined(CONFIG_FAT_FS_MODULE) ++#if IS_ENABLED(CONFIG_FAT_FS) + return sb->s_magic == MSDOS_SUPER_MAGIC; +#else + return 0; @@ -13387,7 +13561,7 @@ index 0000000..725b2ff + +static inline int au_test_squashfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_SQUASHFS) || defined(CONFIG_SQUASHFS_MODULE) ++#if IS_ENABLED(CONFIG_SQUASHFS) + return sb->s_magic == SQUASHFS_MAGIC; +#else + return 0; @@ -13396,7 +13570,7 @@ index 0000000..725b2ff + +static inline int au_test_btrfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_BTRFS_FS) || defined(CONFIG_BTRFS_FS_MODULE) ++#if IS_ENABLED(CONFIG_BTRFS_FS) + return sb->s_magic == BTRFS_SUPER_MAGIC; +#else + return 0; @@ -13405,7 +13579,7 @@ index 0000000..725b2ff + +static inline int au_test_xenfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_XENFS) || defined(CONFIG_XENFS_MODULE) ++#if IS_ENABLED(CONFIG_XENFS) + return sb->s_magic == XENFS_SUPER_MAGIC; +#else + return 0; @@ -13423,7 +13597,7 @@ index 0000000..725b2ff + +static inline int au_test_nilfs(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_NILFS) || defined(CONFIG_NILFS_MODULE) ++#if IS_ENABLED(CONFIG_NILFS) + return sb->s_magic == NILFS_SUPER_MAGIC; +#else + return 0; @@ -13432,7 +13606,7 @@ index 0000000..725b2ff + +static inline int au_test_hfsplus(struct super_block *sb __maybe_unused) +{ -+#if defined(CONFIG_HFSPLUS_FS) || defined(CONFIG_HFSPLUS_FS_MODULE) ++#if IS_ENABLED(CONFIG_HFSPLUS_FS) + return sb->s_magic == HFSPLUS_SUPER_MAGIC; +#else + return 0; @@ -13593,7 +13767,7 @@ index 0000000..725b2ff +#endif /* __AUFS_FSTYPE_H__ */ diff --git a/fs/aufs/hfsnotify.c b/fs/aufs/hfsnotify.c new file mode 100644 -index 0000000..6afef3f +index 0000000..110f8f5 --- /dev/null +++ b/fs/aufs/hfsnotify.c @@ -0,0 +1,274 @@ @@ -13618,7 +13792,7 @@ index 0000000..6afef3f + struct au_hnotify *hn = container_of(mark, struct au_hnotify, + hn_mark); + /* AuDbg("here\n"); */ -+ au_cache_free_hnotify(hn); ++ au_cache_dfree_hnotify(hn); + smp_mb__before_atomic(); + if (atomic64_dec_and_test(&au_hfsn_ifree)) + wake_up(&au_hfsn_wq); @@ -13742,7 +13916,7 @@ index 0000000..6afef3f + struct au_br_hfsnotify *hfsn = group->private; + + /* AuDbg("here\n"); */ -+ kfree(hfsn); ++ au_delayed_kfree(hfsn); +} + +static int au_hfsn_handle_event(struct fsnotify_group *group, @@ -13836,7 +14010,7 @@ index 0000000..6afef3f + goto out; /* success */ + +out_hfsn: -+ kfree(hfsn); ++ au_delayed_kfree(hfsn); +out: + return err; +} @@ -13922,10 +14096,10 @@ index 0000000..145c6ac +} diff --git a/fs/aufs/hnotify.c b/fs/aufs/hnotify.c new file mode 100644 -index 0000000..3016c5e +index 0000000..a05aed8 --- /dev/null +++ b/fs/aufs/hnotify.c -@@ -0,0 +1,697 @@ +@@ -0,0 +1,710 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -13950,7 +14124,7 @@ index 0000000..3016c5e + AuTraceErr(err); + if (unlikely(err)) { + hinode->hi_notify = NULL; -+ au_cache_free_hnotify(hn); ++ au_cache_dfree_hnotify(hn); + /* + * The upper dir was removed by udba, but the same named + * dir left. In this case, aufs assignes a new inode @@ -13974,7 +14148,7 @@ index 0000000..3016c5e + if (hn) { + hinode->hi_notify = NULL; + if (au_hnotify_op.free(hinode, hn)) -+ au_cache_free_hnotify(hn); ++ au_cache_dfree_hnotify(hn); + } +} + @@ -14408,7 +14582,7 @@ index 0000000..3016c5e + || au_ftest_hnjob(a->flags[AuHn_CHILD], GEN))) { + inode = lookup_wlock_by_ino(sb, bfound, h_ino); + try_iput = 1; -+ } ++ } + + args.flags = a->flags[AuHn_CHILD]; + args.dentry = dentry; @@ -14447,7 +14621,7 @@ index 0000000..3016c5e + iput(a->dir); + si_write_unlock(sb); + au_nwt_done(&sbinfo->si_nowait); -+ kfree(a); ++ au_delayed_kfree(a); +} + +/* ---------------------------------------------------------------------- */ @@ -14553,7 +14727,7 @@ index 0000000..3016c5e + iput(args->h_child_inode); + iput(args->h_dir); + iput(args->dir); -+ kfree(args); ++ au_delayed_kfree(args); + } + +out: @@ -14594,17 +14768,26 @@ index 0000000..3016c5e + +static void au_hn_destroy_cache(void) +{ -+ kmem_cache_destroy(au_cachep[AuCache_HNOTIFY]); -+ au_cachep[AuCache_HNOTIFY] = NULL; ++ struct au_cache *cp; ++ ++ flush_delayed_work(&au_dfree.dwork); ++ cp = au_dfree.cache + AuCache_HNOTIFY; ++ AuDebugOn(!llist_empty(&cp->llist)); ++ kmem_cache_destroy(cp->cache); ++ cp->cache = NULL; +} + ++AU_CACHE_DFREE_FUNC(hnotify, HNOTIFY, hn_lnode); ++ +int __init au_hnotify_init(void) +{ + int err; ++ struct au_cache *cp; + + err = -ENOMEM; -+ au_cachep[AuCache_HNOTIFY] = AuCache(au_hnotify); -+ if (au_cachep[AuCache_HNOTIFY]) { ++ cp = au_dfree.cache + AuCache_HNOTIFY; ++ cp->cache = AuCache(au_hnotify); ++ if (cp->cache) { + err = 0; + if (au_hnotify_op.init) + err = au_hnotify_op.init(); @@ -14617,18 +14800,22 @@ index 0000000..3016c5e + +void au_hnotify_fin(void) +{ ++ struct au_cache *cp; ++ + if (au_hnotify_op.fin) + au_hnotify_op.fin(); ++ + /* cf. au_cache_fin() */ -+ if (au_cachep[AuCache_HNOTIFY]) ++ cp = au_dfree.cache + AuCache_HNOTIFY; ++ if (cp->cache) + au_hn_destroy_cache(); +} diff --git a/fs/aufs/i_op.c b/fs/aufs/i_op.c new file mode 100644 -index 0000000..cc1e312 +index 0000000..c730ec7 --- /dev/null +++ b/fs/aufs/i_op.c -@@ -0,0 +1,1401 @@ +@@ -0,0 +1,1438 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -14644,17 +14831,20 @@ index 0000000..cc1e312 +#include "aufs.h" + +static int h_permission(struct inode *h_inode, int mask, -+ struct vfsmount *h_mnt, int brperm) ++ struct path *h_path, int brperm) +{ + int err; + const unsigned char write_mask = !!(mask & (MAY_WRITE | MAY_APPEND)); + ++ err = -EPERM; ++ if (write_mask && IS_IMMUTABLE(h_inode)) ++ goto out; ++ + err = -EACCES; -+ if ((write_mask && IS_IMMUTABLE(h_inode)) -+ || ((mask & MAY_EXEC) -+ && S_ISREG(h_inode->i_mode) -+ && ((h_mnt->mnt_flags & MNT_NOEXEC) -+ || !(h_inode->i_mode & S_IXUGO)))) ++ if (((mask & MAY_EXEC) ++ && S_ISREG(h_inode->i_mode) ++ && (path_noexec(h_path) ++ || !(h_inode->i_mode & S_IXUGO)))) + goto out; + + /* @@ -14738,7 +14928,7 @@ index 0000000..cc1e312 + err = 0; + bindex = au_ibtop(inode); + br = au_sbr(sb, bindex); -+ err = h_permission(h_inode, mask, au_br_mnt(br), br->br_perm); ++ err = h_permission(h_inode, mask, &br->br_path, br->br_perm); + if (write_mask + && !err + && !special_file(h_inode->i_mode)) { @@ -14764,7 +14954,7 @@ index 0000000..cc1e312 + break; + + br = au_sbr(sb, bindex); -+ err = h_permission(h_inode, mask, au_br_mnt(br), ++ err = h_permission(h_inode, mask, &br->br_path, + br->br_perm); + } + } @@ -14899,8 +15089,7 @@ index 0000000..cc1e312 +{ + int err, h_opened = *opened; + unsigned int lkup_flags; -+ struct dentry *parent; -+ struct dentry *d; ++ struct dentry *parent, *d; + struct au_sphlhead *aopen; + struct vfsub_aopen_args args = { + .open_flag = open_flag, @@ -14995,10 +15184,10 @@ index 0000000..cc1e312 + di_write_unlock(parent); + aufs_read_unlock(dentry, AuLock_DW); + AuDbgDentry(dentry); -+ if (unlikely(err)) ++ if (unlikely(err < 0)) + goto out; +out_no_open: -+ if (!err && !(*opened & FILE_CREATED)) { ++ if (err >= 0 && !(*opened & FILE_CREATED)) { + AuLabel(out_no_open); + dget(dentry); + err = finish_no_open(file, dentry); @@ -15612,7 +15801,7 @@ index 0000000..cc1e312 +out_si: + si_read_unlock(sb); +out_kfree: -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; @@ -15705,7 +15894,7 @@ index 0000000..cc1e312 + di_write_unlock(dentry); + si_read_unlock(sb); +out_kfree: -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; @@ -15925,20 +16114,55 @@ index 0000000..cc1e312 + +/* ---------------------------------------------------------------------- */ + ++static int au_is_special(struct inode *inode) ++{ ++ return (inode->i_mode & (S_IFBLK | S_IFCHR | S_IFIFO | S_IFSOCK)); ++} ++ +static int aufs_update_time(struct inode *inode, struct timespec *ts, int flags) +{ + int err; ++ aufs_bindex_t bindex; + struct super_block *sb; + struct inode *h_inode; ++ struct vfsmount *h_mnt; + + sb = inode->i_sb; ++ WARN_ONCE((flags & S_ATIME) && !IS_NOATIME(inode), ++ "unexpected s_flags 0x%lx", sb->s_flags); ++ + /* mmap_sem might be acquired already, cf. aufs_mmap() */ + lockdep_off(); + si_read_lock(sb, AuLock_FLUSH); + ii_write_lock_child(inode); + lockdep_on(); -+ h_inode = au_h_iptr(inode, au_ibtop(inode)); -+ err = vfsub_update_time(h_inode, ts, flags); ++ ++ err = 0; ++ bindex = au_ibtop(inode); ++ h_inode = au_h_iptr(inode, bindex); ++ if (!au_test_ro(sb, bindex, inode)) { ++ h_mnt = au_sbr_mnt(sb, bindex); ++ err = vfsub_mnt_want_write(h_mnt); ++ if (!err) { ++ err = vfsub_update_time(h_inode, ts, flags); ++ vfsub_mnt_drop_write(h_mnt); ++ } ++ } else if (au_is_special(h_inode)) { ++ /* ++ * Never copy-up here. ++ * These special files may already be opened and used for ++ * communicating. If we copied it up, then the communication ++ * would be corrupted. ++ */ ++ AuWarn1("timestamps for i%lu are ignored " ++ "since it is on readonly branch (hi%lu).\n", ++ inode->i_ino, h_inode->i_ino); ++ } else if (flags & ~S_ATIME) { ++ err = -EIO; ++ AuIOErr1("unexpected flags 0x%x\n", flags); ++ AuDebugOn(1); ++ } ++ + lockdep_off(); + if (!err) + au_cpup_attr_timesizes(inode); @@ -16032,7 +16256,7 @@ index 0000000..cc1e312 +}; diff --git a/fs/aufs/i_op_add.c b/fs/aufs/i_op_add.c new file mode 100644 -index 0000000..8e3fb61 +index 0000000..2a9aa09 --- /dev/null +++ b/fs/aufs/i_op_add.c @@ -0,0 +1,911 @@ @@ -16370,7 +16594,7 @@ index 0000000..8e3fb61 + if (!try_aopen) + aufs_read_unlock(dentry, AuLock_DW); +out_free: -+ kfree(a); ++ au_delayed_kfree(a); +out: + return err; +} @@ -16834,7 +17058,7 @@ index 0000000..8e3fb61 + } + aufs_read_and_write_unlock2(dentry, src_dentry); +out_kfree: -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; @@ -16943,13 +17167,13 @@ index 0000000..8e3fb61 + } + aufs_read_unlock(dentry, AuLock_DW); +out_free: -+ kfree(a); ++ au_delayed_kfree(a); +out: + return err; +} diff --git a/fs/aufs/i_op_del.c b/fs/aufs/i_op_del.c new file mode 100644 -index 0000000..35b923c +index 0000000..2fe8f07 --- /dev/null +++ b/fs/aufs/i_op_del.c @@ -0,0 +1,498 @@ @@ -17336,7 +17560,7 @@ index 0000000..35b923c +out_unlock: + aufs_read_unlock(dentry, AuLock_DW); +out_free: -+ kfree(a); ++ au_delayed_kfree(a); +out: + return err; +} @@ -17446,14 +17670,14 @@ index 0000000..35b923c +out_unlock: + aufs_read_unlock(dentry, AuLock_DW); +out_free: -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/i_op_ren.c b/fs/aufs/i_op_ren.c new file mode 100644 -index 0000000..df697de +index 0000000..43a0909 --- /dev/null +++ b/fs/aufs/i_op_ren.c @@ -0,0 +1,1002 @@ @@ -18454,17 +18678,17 @@ index 0000000..df697de + iput(a->dst_inode); + if (a->thargs) + au_whtmp_rmdir_free(a->thargs); -+ kfree(a); ++ au_delayed_kfree(a); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/iinfo.c b/fs/aufs/iinfo.c new file mode 100644 -index 0000000..fae4c27 +index 0000000..975cff3 --- /dev/null +++ b/fs/aufs/iinfo.c -@@ -0,0 +1,271 @@ +@@ -0,0 +1,272 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -18672,7 +18896,7 @@ index 0000000..fae4c27 + return -ENOMEM; +} + -+int au_hinode_realloc(struct au_iinfo *iinfo, int nbr) ++int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink) +{ + int err, i; + struct au_hinode *hip; @@ -18680,7 +18904,8 @@ index 0000000..fae4c27 + AuRwMustWriteLock(&iinfo->ii_rwsem); + + err = -ENOMEM; -+ hip = krealloc(iinfo->ii_hinode, sizeof(*hip) * nbr, GFP_NOFS); ++ hip = au_krealloc(iinfo->ii_hinode, sizeof(*hip) * nbr, GFP_NOFS, ++ may_shrink); + if (hip) { + iinfo->ii_hinode = hip; + i = iinfo->ii_bbot + 1; @@ -18721,7 +18946,7 @@ index 0000000..fae4c27 + + iinfo = au_ii(inode); + if (iinfo->ii_vdir) -+ au_vdir_free(iinfo->ii_vdir); ++ au_vdir_free(iinfo->ii_vdir, /*atonce*/0); + + bindex = iinfo->ii_btop; + if (bindex >= 0) { @@ -18733,15 +18958,15 @@ index 0000000..fae4c27 + hi++; + } + } -+ kfree(iinfo->ii_hinode); ++ au_delayed_kfree(iinfo->ii_hinode); + AuRwDestroy(&iinfo->ii_rwsem); +} diff --git a/fs/aufs/inode.c b/fs/aufs/inode.c new file mode 100644 -index 0000000..a4b994e +index 0000000..8f7446d --- /dev/null +++ b/fs/aufs/inode.c -@@ -0,0 +1,504 @@ +@@ -0,0 +1,506 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -18771,7 +18996,7 @@ index 0000000..a4b994e + +static int au_ii_refresh(struct inode *inode, int *update) +{ -+ int err, e; ++ int err, e, nbr; + umode_t type; + aufs_bindex_t bindex, new_bindex; + struct super_block *sb; @@ -18783,9 +19008,10 @@ index 0000000..a4b994e + + *update = 0; + sb = inode->i_sb; ++ nbr = au_sbbot(sb) + 1; + type = inode->i_mode & S_IFMT; + iinfo = au_ii(inode); -+ err = au_hinode_realloc(iinfo, au_sbbot(sb) + 1); ++ err = au_hinode_realloc(iinfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + @@ -18823,6 +19049,7 @@ index 0000000..a4b994e + } + } + au_update_ibrange(inode, /*do_put_zero*/0); ++ au_hinode_realloc(iinfo, nbr, /*may_shrink*/1); /* harmless if err */ + e = au_dy_irefresh(inode); + if (unlikely(e && !err)) + err = e; @@ -19248,10 +19475,10 @@ index 0000000..a4b994e +} diff --git a/fs/aufs/inode.h b/fs/aufs/inode.h new file mode 100644 -index 0000000..3fc5a58 +index 0000000..33d9f5e --- /dev/null +++ b/fs/aufs/inode.h -@@ -0,0 +1,681 @@ +@@ -0,0 +1,687 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -19276,7 +19503,10 @@ index 0000000..3fc5a58 + /* never use fsnotify_add_vfsmount_mark() */ + struct fsnotify_mark hn_mark; +#endif -+ struct inode *hn_aufs_inode; /* no get/put */ ++ union { ++ struct inode *hn_aufs_inode; /* no get/put */ ++ struct llist_node hn_lnode; /* delayed free */ ++ }; +#endif +} ____cacheline_aligned_in_smp; + @@ -19319,7 +19549,10 @@ index 0000000..3fc5a58 +struct au_icntnr { + struct au_iinfo iinfo; + struct inode vfs_inode; -+ struct hlist_node plink; ++ union { ++ struct hlist_node plink; ++ struct llist_node lnode; /* delayed free */ ++ }; +} ____cacheline_aligned_in_smp; + +/* au_pin flags */ @@ -19506,7 +19739,7 @@ index 0000000..3fc5a58 +void au_hinode_init(struct au_hinode *hinode); +int au_iinfo_init(struct inode *inode); +void au_iinfo_fin(struct inode *inode); -+int au_hinode_realloc(struct au_iinfo *iinfo, int nbr); ++int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink); + +#ifdef CONFIG_PROC_FS +/* plink.c */ @@ -20147,10 +20380,10 @@ index 0000000..2ebfdc4 +#endif diff --git a/fs/aufs/loop.c b/fs/aufs/loop.c new file mode 100644 -index 0000000..5711e7a +index 0000000..c3ca50f --- /dev/null +++ b/fs/aufs/loop.c -@@ -0,0 +1,133 @@ +@@ -0,0 +1,134 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -20244,7 +20477,8 @@ index 0000000..5711e7a + new_nelem = au_warn_loopback_nelem + au_warn_loopback_step; + a = au_kzrealloc(au_warn_loopback_array, + au_warn_loopback_nelem * sizeof(unsigned long), -+ new_nelem * sizeof(unsigned long), GFP_ATOMIC); ++ new_nelem * sizeof(unsigned long), GFP_ATOMIC, ++ /*may_shrink*/0); + if (a) { + au_warn_loopback_nelem = new_nelem; + au_warn_loopback_array = a; @@ -20282,7 +20516,7 @@ index 0000000..5711e7a +{ + if (backing_file_func) + symbol_put(loop_backing_file); -+ kfree(au_warn_loopback_array); ++ au_delayed_kfree(au_warn_loopback_array); +} diff --git a/fs/aufs/loop.h b/fs/aufs/loop.h new file mode 100644 @@ -20367,10 +20601,10 @@ index 0000000..4f83bdf +endif diff --git a/fs/aufs/module.c b/fs/aufs/module.c new file mode 100644 -index 0000000..88f8f41 +index 0000000..5092b4b --- /dev/null +++ b/fs/aufs/module.c -@@ -0,0 +1,209 @@ +@@ -0,0 +1,319 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -20383,29 +20617,120 @@ index 0000000..88f8f41 +#include +#include "aufs.h" + -+void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp) ++/* shrinkable realloc */ ++void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink) +{ -+ if (new_sz <= nused) -+ return p; ++ size_t sz; ++ int diff; + -+ p = krealloc(p, new_sz, gfp); -+ if (p) ++ sz = 0; ++ diff = -1; ++ if (p) { ++#if 0 /* unused */ ++ if (!new_sz) { ++ au_delayed_kfree(p); ++ p = NULL; ++ goto out; ++ } ++#else ++ AuDebugOn(!new_sz); ++#endif ++ sz = ksize(p); ++ diff = au_kmidx_sub(sz, new_sz); ++ } ++ if (sz && !diff) ++ goto out; ++ ++ if (sz < new_sz) ++ /* expand or SLOB */ ++ p = krealloc(p, new_sz, gfp); ++ else if (new_sz < sz && may_shrink) { ++ /* shrink */ ++ void *q; ++ ++ q = kmalloc(new_sz, gfp); ++ if (q) { ++ if (p) { ++ memcpy(q, p, new_sz); ++ au_delayed_kfree(p); ++ } ++ p = q; ++ } else ++ p = NULL; ++ } ++ ++out: ++ return p; ++} ++ ++void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp, ++ int may_shrink) ++{ ++ p = au_krealloc(p, new_sz, gfp, may_shrink); ++ if (p && new_sz > nused) + memset(p + nused, 0, new_sz - nused); + return p; +} + +/* ---------------------------------------------------------------------- */ -+ +/* + * aufs caches + */ -+struct kmem_cache *au_cachep[AuCache_Last] = { -+ [0] = NULL -+}; ++ ++struct au_dfree au_dfree; ++ ++/* delayed free */ ++static void au_do_dfree(struct work_struct *work __maybe_unused) ++{ ++ struct llist_head *head; ++ struct llist_node *node, *next; ++ ++#define AU_CACHE_DFREE_DO_BODY(name, idx, lnode) do { \ ++ head = &au_dfree.cache[AuCache_##idx].llist; \ ++ node = llist_del_all(head); \ ++ for (; node; node = next) { \ ++ struct au_##name *p \ ++ = llist_entry(node, struct au_##name, \ ++ lnode); \ ++ next = llist_next(node); \ ++ au_cache_free_##name(p); \ ++ } \ ++ } while (0) ++ ++ AU_CACHE_DFREE_DO_BODY(dinfo, DINFO, di_lnode); ++ AU_CACHE_DFREE_DO_BODY(icntnr, ICNTNR, lnode); ++ AU_CACHE_DFREE_DO_BODY(finfo, FINFO, fi_lnode); ++ AU_CACHE_DFREE_DO_BODY(vdir, VDIR, vd_lnode); ++ AU_CACHE_DFREE_DO_BODY(vdir_dehstr, DEHSTR, lnode); ++#ifdef CONFIG_AUFS_HNOTIFY ++ AU_CACHE_DFREE_DO_BODY(hnotify, HNOTIFY, hn_lnode); ++#endif ++ ++#define AU_DFREE_DO_BODY(llist, func) do { \ ++ node = llist_del_all(llist); \ ++ for (; node; node = next) { \ ++ next = llist_next(node); \ ++ func(node); \ ++ } \ ++ } while (0) ++ ++ AU_DFREE_DO_BODY(au_dfree.llist + AU_DFREE_KFREE, kfree); ++ AU_DFREE_DO_BODY(au_dfree.llist + AU_DFREE_FREE_PAGE, au_free_page); ++ ++#undef AU_CACHE_DFREE_DO_BODY ++#undef AU_DFREE_DO_BODY ++} ++ ++AU_CACHE_DFREE_FUNC(dinfo, DINFO, di_lnode); ++AU_CACHE_DFREE_FUNC(icntnr, ICNTNR, lnode); ++AU_CACHE_DFREE_FUNC(finfo, FINFO, fi_lnode); ++AU_CACHE_DFREE_FUNC(vdir, VDIR, vd_lnode); ++AU_CACHE_DFREE_FUNC(vdir_dehstr, DEHSTR, lnode); + +static void au_cache_fin(void) +{ + int i; ++ struct au_cache *cp; + + /* + * Make sure all delayed rcu free inodes are flushed before we @@ -20415,27 +20740,33 @@ index 0000000..88f8f41 + + /* excluding AuCache_HNOTIFY */ + BUILD_BUG_ON(AuCache_HNOTIFY + 1 != AuCache_Last); ++ flush_delayed_work(&au_dfree.dwork); + for (i = 0; i < AuCache_HNOTIFY; i++) { -+ kmem_cache_destroy(au_cachep[i]); -+ au_cachep[i] = NULL; ++ cp = au_dfree.cache + i; ++ AuDebugOn(!llist_empty(&cp->llist)); ++ kmem_cache_destroy(cp->cache); ++ cp->cache = NULL; + } +} + +static int __init au_cache_init(void) +{ -+ au_cachep[AuCache_DINFO] = AuCacheCtor(au_dinfo, au_di_init_once); -+ if (au_cachep[AuCache_DINFO]) ++ struct au_cache *cp; ++ ++ cp = au_dfree.cache; ++ cp[AuCache_DINFO].cache = AuCacheCtor(au_dinfo, au_di_init_once); ++ if (cp[AuCache_DINFO].cache) + /* SLAB_DESTROY_BY_RCU */ -+ au_cachep[AuCache_ICNTNR] = AuCacheCtor(au_icntnr, -+ au_icntnr_init_once); -+ if (au_cachep[AuCache_ICNTNR]) -+ au_cachep[AuCache_FINFO] = AuCacheCtor(au_finfo, -+ au_fi_init_once); -+ if (au_cachep[AuCache_FINFO]) -+ au_cachep[AuCache_VDIR] = AuCache(au_vdir); -+ if (au_cachep[AuCache_VDIR]) -+ au_cachep[AuCache_DEHSTR] = AuCache(au_vdir_dehstr); -+ if (au_cachep[AuCache_DEHSTR]) ++ cp[AuCache_ICNTNR].cache = AuCacheCtor(au_icntnr, ++ au_icntnr_init_once); ++ if (cp[AuCache_ICNTNR].cache) ++ cp[AuCache_FINFO].cache = AuCacheCtor(au_finfo, ++ au_fi_init_once); ++ if (cp[AuCache_FINFO].cache) ++ cp[AuCache_VDIR].cache = AuCache(au_vdir); ++ if (cp[AuCache_VDIR].cache) ++ cp[AuCache_DEHSTR].cache = AuCache(au_vdir_dehstr); ++ if (cp[AuCache_DEHSTR].cache) + return 0; + + au_cache_fin(); @@ -20497,6 +20828,7 @@ index 0000000..88f8f41 +{ + int err, i; + char *p; ++ struct au_cache *cp; + + p = au_esc_chars; + for (i = 1; i <= ' '; i++) @@ -20511,6 +20843,16 @@ index 0000000..88f8f41 + for (i = 0; i < AuIop_Last; i++) + aufs_iop_nogetattr[i].getattr = NULL; + ++ /* First, initialize au_dfree */ ++ for (i = 0; i < AuCache_Last; i++) { /* including hnotify */ ++ cp = au_dfree.cache + i; ++ cp->cache = NULL; ++ init_llist_head(&cp->llist); ++ } ++ for (i = 0; i < AU_DFREE_Last; i++) ++ init_llist_head(au_dfree.llist + i); ++ INIT_DELAYED_WORK(&au_dfree.dwork, au_do_dfree); ++ + au_sbilist_init(); + sysaufs_brs_init(); + au_debug_init(); @@ -20561,6 +20903,7 @@ index 0000000..88f8f41 +out_sysaufs: + sysaufs_fin(); + au_dy_fin(); ++ flush_delayed_work(&au_dfree.dwork); +out: + return err; +} @@ -20576,16 +20919,17 @@ index 0000000..88f8f41 + au_procfs_fin(); + sysaufs_fin(); + au_dy_fin(); ++ flush_delayed_work(&au_dfree.dwork); +} + +module_init(aufs_init); +module_exit(aufs_exit); diff --git a/fs/aufs/module.h b/fs/aufs/module.h new file mode 100644 -index 0000000..1383e3d +index 0000000..6f968ba --- /dev/null +++ b/fs/aufs/module.h -@@ -0,0 +1,76 @@ +@@ -0,0 +1,143 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -20600,6 +20944,7 @@ index 0000000..1383e3d +#ifdef __KERNEL__ + +#include ++#include "debug.h" + +struct path; +struct seq_file; @@ -20612,7 +20957,19 @@ index 0000000..1383e3d + +extern int au_dir_roflags; + -+void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp); ++void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink); ++void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp, ++ int may_shrink); ++ ++static inline int au_kmidx_sub(size_t sz, size_t new_sz) ++{ ++#ifndef CONFIG_SLOB ++ return kmalloc_index(sz) - kmalloc_index(new_sz); ++#else ++ return -1; /* SLOB is untested */ ++#endif ++} ++ +int au_seq_path(struct seq_file *seq, struct path *path); + +#ifdef CONFIG_PROC_FS @@ -20626,7 +20983,7 @@ index 0000000..1383e3d + +/* ---------------------------------------------------------------------- */ + -+/* kmem cache */ ++/* kmem cache and delayed free */ +enum { + AuCache_DINFO, + AuCache_ICNTNR, @@ -20637,19 +20994,54 @@ index 0000000..1383e3d + AuCache_Last +}; + ++enum { ++ AU_DFREE_KFREE, ++ AU_DFREE_FREE_PAGE, ++ AU_DFREE_Last ++}; ++ ++struct au_cache { ++ struct kmem_cache *cache; ++ struct llist_head llist; /* delayed free */ ++}; ++ ++/* ++ * in order to reduce the cost of the internal timer, consolidate all the ++ * delayed free works into a single delayed_work. ++ */ ++struct au_dfree { ++ struct au_cache cache[AuCache_Last]; ++ struct llist_head llist[AU_DFREE_Last]; ++ struct delayed_work dwork; ++}; ++ ++extern struct au_dfree au_dfree; ++ +#define AuCacheFlags (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD) +#define AuCache(type) KMEM_CACHE(type, AuCacheFlags) +#define AuCacheCtor(type, ctor) \ + kmem_cache_create(#type, sizeof(struct type), \ + __alignof__(struct type), AuCacheFlags, ctor) + -+extern struct kmem_cache *au_cachep[]; ++#define AU_DFREE_DELAY msecs_to_jiffies(10) ++#define AU_DFREE_BODY(lnode, llist) do { \ ++ if (llist_add(lnode, llist)) \ ++ schedule_delayed_work(&au_dfree.dwork, \ ++ AU_DFREE_DELAY); \ ++ } while (0) ++#define AU_CACHE_DFREE_FUNC(name, idx, lnode) \ ++ void au_cache_dfree_##name(struct au_##name *p) \ ++ { \ ++ struct au_cache *cp = au_dfree.cache + AuCache_##idx; \ ++ AU_DFREE_BODY(&p->lnode, &cp->llist); \ ++ } + +#define AuCacheFuncs(name, index) \ +static inline struct au_##name *au_cache_alloc_##name(void) \ -+{ return kmem_cache_alloc(au_cachep[AuCache_##index], GFP_NOFS); } \ ++{ return kmem_cache_alloc(au_dfree.cache[AuCache_##index].cache, GFP_NOFS); } \ +static inline void au_cache_free_##name(struct au_##name *p) \ -+{ kmem_cache_free(au_cachep[AuCache_##index], p); } ++{ kmem_cache_free(au_dfree.cache[AuCache_##index].cache, p); } \ ++void au_cache_dfree_##name(struct au_##name *p) + +AuCacheFuncs(dinfo, DINFO); +AuCacheFuncs(icntnr, ICNTNR); @@ -20660,11 +21052,30 @@ index 0000000..1383e3d +AuCacheFuncs(hnotify, HNOTIFY); +#endif + ++static inline void au_delayed_kfree(const void *p) ++{ ++ AuDebugOn(!p); ++ AuDebugOn(ksize(p) < sizeof(struct llist_node)); ++ ++ AU_DFREE_BODY((void *)p, au_dfree.llist + AU_DFREE_KFREE); ++} ++ ++/* cast only */ ++static inline void au_free_page(void *p) ++{ ++ free_page((unsigned long)p); ++} ++ ++static inline void au_delayed_free_page(unsigned long addr) ++{ ++ AU_DFREE_BODY((void *)addr, au_dfree.llist + AU_DFREE_FREE_PAGE); ++} ++ +#endif /* __KERNEL__ */ +#endif /* __AUFS_MODULE_H__ */ diff --git a/fs/aufs/mvdown.c b/fs/aufs/mvdown.c new file mode 100644 -index 0000000..a48aa1c +index 0000000..b0e5b84 --- /dev/null +++ b/fs/aufs/mvdown.c @@ -0,0 +1,691 @@ @@ -21354,17 +21765,17 @@ index 0000000..a48aa1c + e = copy_to_user(uarg, &args->mvdown, sizeof(args->mvdown)); + if (unlikely(e)) + err = -EFAULT; -+ kfree(args); ++ au_delayed_kfree(args); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/opts.c b/fs/aufs/opts.c new file mode 100644 -index 0000000..79c7d71 +index 0000000..1d21f0d --- /dev/null +++ b/fs/aufs/opts.c -@@ -0,0 +1,1846 @@ +@@ -0,0 +1,1847 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -22610,7 +23021,7 @@ index 0000000..79c7d71 + } + } + -+ kfree(a); ++ au_delayed_kfree(a); + dump_opts(opts); + if (unlikely(err)) + au_opts_free(opts); @@ -23032,7 +23443,8 @@ index 0000000..79c7d71 + au_hn_inode_unlock(hdir); + + if (!err && do_free) { -+ kfree(wbr); ++ if (wbr) ++ au_delayed_kfree(wbr); + br->br_wbr = NULL; + } + } @@ -23417,10 +23829,10 @@ index 0000000..0d6c2e1 +#endif /* __AUFS_OPTS_H__ */ diff --git a/fs/aufs/plink.c b/fs/aufs/plink.c new file mode 100644 -index 0000000..c42734d +index 0000000..d8d75e8 --- /dev/null +++ b/fs/aufs/plink.c -@@ -0,0 +1,489 @@ +@@ -0,0 +1,501 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -23455,6 +23867,7 @@ index 0000000..c42734d +{ + int err; + pid_t pid, ppid; ++ struct task_struct *parent, *prev; + struct au_sbinfo *sbi; + + SiMustAnyLock(sb); @@ -23469,11 +23882,22 @@ index 0000000..c42734d + goto out; + + /* todo: it highly depends upon /sbin/mount.aufs */ ++ prev = NULL; ++ parent = current; ++ ppid = 0; + rcu_read_lock(); -+ ppid = task_pid_vnr(rcu_dereference(current->real_parent)); ++ while (1) { ++ parent = rcu_dereference(parent->real_parent); ++ if (parent == prev) ++ break; ++ ppid = task_pid_vnr(parent); ++ if (pid == ppid) { ++ rcu_read_unlock(); ++ goto out; ++ } ++ prev = parent; ++ } + rcu_read_unlock(); -+ if (pid == ppid) -+ goto out; + + if (au_ftest_lock(flags, NOPLMW)) { + /* if there is no i_mutex lock in VFS, we don't need to wait */ @@ -24775,10 +25199,10 @@ index 0000000..6c0d5a9 +#endif /* __AUFS_RWSEM_H__ */ diff --git a/fs/aufs/sbinfo.c b/fs/aufs/sbinfo.c new file mode 100644 -index 0000000..aa97059 +index 0000000..e113b51 --- /dev/null +++ b/fs/aufs/sbinfo.c -@@ -0,0 +1,340 @@ +@@ -0,0 +1,342 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -24801,7 +25225,7 @@ index 0000000..aa97059 + sbinfo = container_of(kobj, struct au_sbinfo, si_kobj); + for (i = 0; i < AuPlink_NHASH; i++) + AuDebugOn(!hlist_empty(&sbinfo->si_plink[i].head)); -+ au_nwt_fin(&sbinfo->si_nowait); ++ AuDebugOn(atomic_read(&sbinfo->si_nowait.nw_len)); + + AuDebugOn(percpu_counter_sum(&sbinfo->si_ninodes)); + percpu_counter_destroy(&sbinfo->si_ninodes); @@ -24812,14 +25236,15 @@ index 0000000..aa97059 + au_br_free(sbinfo); + au_rw_write_unlock(&sbinfo->si_rwsem); + -+ kfree(sbinfo->si_branch); ++ au_delayed_kfree(sbinfo->si_branch); + for (i = 0; i < AU_NPIDMAP; i++) -+ kfree(sbinfo->au_si_pid.pid_bitmap[i]); ++ if (sbinfo->au_si_pid.pid_bitmap[i]) ++ au_delayed_kfree(sbinfo->au_si_pid.pid_bitmap[i]); + mutex_destroy(&sbinfo->au_si_pid.pid_mtx); + mutex_destroy(&sbinfo->si_xib_mtx); + AuRwDestroy(&sbinfo->si_rwsem); + -+ kfree(sbinfo); ++ au_delayed_kfree(sbinfo); +} + +int au_si_alloc(struct super_block *sb) @@ -24891,14 +25316,14 @@ index 0000000..aa97059 + return 0; /* success */ + +out_br: -+ kfree(sbinfo->si_branch); ++ au_delayed_kfree(sbinfo->si_branch); +out_sbinfo: -+ kfree(sbinfo); ++ au_delayed_kfree(sbinfo); +out: + return err; +} + -+int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr) ++int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink) +{ + int err, sz; + struct au_branch **brp; @@ -24909,7 +25334,8 @@ index 0000000..aa97059 + sz = sizeof(*brp) * (sbinfo->si_bbot + 1); + if (unlikely(!sz)) + sz = sizeof(*brp); -+ brp = au_kzrealloc(sbinfo->si_branch, sz, sizeof(*brp) * nbr, GFP_NOFS); ++ brp = au_kzrealloc(sbinfo->si_branch, sz, sizeof(*brp) * nbr, GFP_NOFS, ++ may_shrink); + if (brp) { + sbinfo->si_branch = brp; + err = 0; @@ -25121,10 +25547,10 @@ index 0000000..aa97059 +} diff --git a/fs/aufs/spl.h b/fs/aufs/spl.h new file mode 100644 -index 0000000..f9b5288 +index 0000000..8f519db3 --- /dev/null +++ b/fs/aufs/spl.h -@@ -0,0 +1,98 @@ +@@ -0,0 +1,100 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -25138,6 +25564,7 @@ index 0000000..f9b5288 + +#ifdef __KERNEL__ + ++#if 0 +struct au_splhead { + spinlock_t spin; + struct list_head head; @@ -25170,6 +25597,7 @@ index 0000000..f9b5288 + list_del_rcu(list); + spin_unlock(&spl->spin); +} ++#endif + +/* ---------------------------------------------------------------------- */ + @@ -25225,10 +25653,10 @@ index 0000000..f9b5288 +#endif /* __AUFS_SPL_H__ */ diff --git a/fs/aufs/super.c b/fs/aufs/super.c new file mode 100644 -index 0000000..8bd2d9c +index 0000000..58a773c --- /dev/null +++ b/fs/aufs/super.c -@@ -0,0 +1,1026 @@ +@@ -0,0 +1,1025 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -25264,8 +25692,7 @@ index 0000000..8bd2d9c +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + -+ INIT_HLIST_HEAD(&inode->i_dentry); -+ au_cache_free_icntnr(container_of(inode, struct au_icntnr, vfs_inode)); ++ au_cache_dfree_icntnr(container_of(inode, struct au_icntnr, vfs_inode)); +} + +static void aufs_destroy_inode(struct inode *inode) @@ -26047,7 +26474,7 @@ index 0000000..8bd2d9c +out_mtx: + inode_unlock(inode); +out_opts: -+ free_page((unsigned long)opts.opt); ++ au_delayed_free_page((unsigned long)opts.opt); +out: + err = cvt_err(err); + AuTraceErr(err); @@ -26188,7 +26615,7 @@ index 0000000..8bd2d9c + kobject_put(&sbinfo->si_kobj); + sb->s_fs_info = NULL; +out_opts: -+ free_page((unsigned long)opts.opt); ++ au_delayed_free_page((unsigned long)opts.opt); +out: + AuTraceErr(err); + err = cvt_err(err); @@ -26257,7 +26684,7 @@ index 0000000..8bd2d9c +}; diff --git a/fs/aufs/super.h b/fs/aufs/super.h new file mode 100644 -index 0000000..b8ca149 +index 0000000..e4ac866 --- /dev/null +++ b/fs/aufs/super.h @@ -0,0 +1,625 @@ @@ -26529,7 +26956,7 @@ index 0000000..b8ca149 +/* sbinfo.c */ +void au_si_free(struct kobject *kobj); +int au_si_alloc(struct super_block *sb); -+int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr); ++int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink); + +unsigned int au_sigen_inc(struct super_block *sb); +aufs_bindex_t au_new_br_id(struct super_block *sb); @@ -27079,7 +27506,7 @@ index 0000000..1f79983 +#endif /* __SYSAUFS_H__ */ diff --git a/fs/aufs/sysfs.c b/fs/aufs/sysfs.c new file mode 100644 -index 0000000..0efb77a +index 0000000..f4c3170 --- /dev/null +++ b/fs/aufs/sysfs.c @@ -0,0 +1,340 @@ @@ -27258,7 +27685,7 @@ index 0000000..0efb77a + if (unlikely(err == PAGE_SIZE)) + err = -EFBIG; + } -+ kfree(seq); ++ au_delayed_kfree(seq); +out_unlock: + si_read_unlock(sb); +out: @@ -27329,9 +27756,9 @@ index 0000000..0efb77a + err = -EFAULT; + +out_seq: -+ kfree(seq); ++ au_delayed_kfree(seq); +out_buf: -+ free_page((unsigned long)buf); ++ au_delayed_free_page((unsigned long)buf); +out: + si_read_unlock(sb); + return err; @@ -27575,10 +28002,10 @@ index 0000000..7d22979 +} diff --git a/fs/aufs/vdir.c b/fs/aufs/vdir.c new file mode 100644 -index 0000000..0a9ff00 +index 0000000..b2eb4c0 --- /dev/null +++ b/fs/aufs/vdir.c -@@ -0,0 +1,876 @@ +@@ -0,0 +1,887 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -27678,7 +28105,7 @@ index 0000000..0a9ff00 + struct hlist_node *node; + + hlist_for_each_entry_safe(pos, node, head, wh_hash) -+ kfree(pos); ++ au_delayed_kfree(pos); +} + +static void au_nhash_de_do_free(struct hlist_head *head) @@ -27687,7 +28114,7 @@ index 0000000..0a9ff00 + struct hlist_node *node; + + hlist_for_each_entry_safe(pos, node, head, hash) -+ au_cache_free_vdir_dehstr(pos); ++ au_cache_dfree_vdir_dehstr(pos); +} + +static void au_nhash_do_free(struct au_nhash *nhash, @@ -27705,7 +28132,7 @@ index 0000000..0a9ff00 + nhash_count(head); + free(head++); + } -+ kfree(nhash->nh_head); ++ au_delayed_kfree(nhash->nh_head); +} + +void au_nhash_wh_free(struct au_nhash *whlist) @@ -27748,6 +28175,8 @@ index 0000000..0a9ff00 + AuDebugOn(!nhash->nh_num || !nhash->nh_head); + + v = 0; ++ if (len > 8) ++ len = 8; + while (len--) + v += *name++; + /* v = hash_long(v, magic_bit); */ @@ -27845,8 +28274,8 @@ index 0000000..0a9ff00 + unsigned char **o; + + err = -ENOMEM; -+ o = krealloc(vdir->vd_deblk, sizeof(*o) * (vdir->vd_nblk + 1), -+ GFP_NOFS); ++ o = au_krealloc(vdir->vd_deblk, sizeof(*o) * (vdir->vd_nblk + 1), ++ GFP_NOFS, /*may_shrink*/0); + if (unlikely(!o)) + goto out; + @@ -27916,15 +28345,23 @@ index 0000000..0a9ff00 + +/* ---------------------------------------------------------------------- */ + -+void au_vdir_free(struct au_vdir *vdir) ++void au_vdir_free(struct au_vdir *vdir, int atonce) +{ + unsigned char **deblk; + + deblk = vdir->vd_deblk; -+ while (vdir->vd_nblk--) -+ kfree(*deblk++); -+ kfree(vdir->vd_deblk); -+ au_cache_free_vdir(vdir); ++ if (!atonce) { ++ while (vdir->vd_nblk--) ++ au_delayed_kfree(*deblk++); ++ au_delayed_kfree(vdir->vd_deblk); ++ au_cache_dfree_vdir(vdir); ++ } else { ++ /* not delayed */ ++ while (vdir->vd_nblk--) ++ kfree(*deblk++); ++ kfree(vdir->vd_deblk); ++ au_cache_free_vdir(vdir); ++ } +} + +static struct au_vdir *alloc_vdir(struct file *file) @@ -27958,10 +28395,10 @@ index 0000000..0a9ff00 + if (!err) + return vdir; /* success */ + -+ kfree(vdir->vd_deblk); ++ au_delayed_kfree(vdir->vd_deblk); + +out_free: -+ au_cache_free_vdir(vdir); ++ au_cache_dfree_vdir(vdir); +out: + vdir = ERR_PTR(err); + return vdir; @@ -27973,7 +28410,7 @@ index 0000000..0a9ff00 + union au_vdir_deblk_p p, deblk_end; + + while (vdir->vd_nblk > 1) { -+ kfree(vdir->vd_deblk[vdir->vd_nblk - 1]); ++ au_delayed_kfree(vdir->vd_deblk[vdir->vd_nblk - 1]); + /* vdir->vd_deblk[vdir->vd_nblk - 1] = NULL; */ + vdir->vd_nblk--; + } @@ -28104,7 +28541,7 @@ index 0000000..0a9ff00 + } + } + -+ free_page((unsigned long)o); ++ au_delayed_free_page((unsigned long)o); + +out: + AuTraceErr(err); @@ -28243,7 +28680,7 @@ index 0000000..0a9ff00 + if (allocated) + au_set_ivdir(inode, allocated); + } else if (allocated) -+ au_vdir_free(allocated); ++ au_vdir_free(allocated, /*atonce*/0); + +out: + return err; @@ -28261,8 +28698,8 @@ index 0000000..0a9ff00 + if (tgt->vd_nblk < src->vd_nblk) { + unsigned char **p; + -+ p = krealloc(tgt->vd_deblk, sizeof(*p) * src->vd_nblk, -+ GFP_NOFS); ++ p = au_krealloc(tgt->vd_deblk, sizeof(*p) * src->vd_nblk, ++ GFP_NOFS, /*may_shrink*/0); + if (unlikely(!p)) + goto out; + tgt->vd_deblk = p; @@ -28272,7 +28709,8 @@ index 0000000..0a9ff00 + unsigned char *p; + + tgt->vd_deblk_sz = deblk_sz; -+ p = krealloc(tgt->vd_deblk[0], deblk_sz, GFP_NOFS); ++ p = au_krealloc(tgt->vd_deblk[0], deblk_sz, GFP_NOFS, ++ /*may_shrink*/1); + if (unlikely(!p)) + goto out; + tgt->vd_deblk[0] = p; @@ -28337,7 +28775,7 @@ index 0000000..0a9ff00 + if (allocated) + au_set_fvdir_cache(file, allocated); + } else if (allocated) -+ au_vdir_free(allocated); ++ au_vdir_free(allocated, /*atonce*/0); + +out: + return err; @@ -29334,10 +29772,10 @@ index 0000000..2e54bad +} diff --git a/fs/aufs/vfsub.h b/fs/aufs/vfsub.h new file mode 100644 -index 0000000..291bfae +index 0000000..64fff0f --- /dev/null +++ b/fs/aufs/vfsub.h -@@ -0,0 +1,297 @@ +@@ -0,0 +1,303 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -29523,6 +29961,12 @@ index 0000000..291bfae + return flags; +} + ++static inline int vfsub_file_execed(struct file *file) ++{ ++ /* todo: direct access f_flags */ ++ return !!(vfsub_file_flags(file) & __FMODE_EXEC); ++} ++ +#if 0 /* reserved */ +static inline void vfsub_file_accessed(struct file *h_file) +{ @@ -29637,7 +30081,7 @@ index 0000000..291bfae +#endif /* __AUFS_VFSUB_H__ */ diff --git a/fs/aufs/wbr_policy.c b/fs/aufs/wbr_policy.c new file mode 100644 -index 0000000..91a010a +index 0000000..79eb9b4 --- /dev/null +++ b/fs/aufs/wbr_policy.c @@ -0,0 +1,752 @@ @@ -30091,7 +30535,7 @@ index 0000000..91a010a + + mfs->mfsrr_bytes = bavail; + AuDbg("b%d\n", mfs->mfs_bindex); -+ kfree(st); ++ au_delayed_kfree(st); +} + +static int au_wbr_create_mfs(struct dentry *dentry, unsigned int flags) @@ -30395,7 +30839,7 @@ index 0000000..91a010a +}; diff --git a/fs/aufs/whout.c b/fs/aufs/whout.c new file mode 100644 -index 0000000..16a96f88 +index 0000000..54feb08 --- /dev/null +++ b/fs/aufs/whout.c @@ -0,0 +1,1047 @@ @@ -30550,7 +30994,7 @@ index 0000000..16a96f88 + +out_name: + if (name != defname) -+ kfree(name); ++ au_delayed_kfree(name); +out: + AuTraceErrPtr(dentry); + return dentry; @@ -30989,7 +31433,7 @@ index 0000000..16a96f88 + au_br_put(a->br); + si_write_unlock(a->sb); + au_nwt_done(&au_sbi(a->sb)->si_nowait); -+ kfree(arg); ++ au_delayed_kfree(arg); + if (unlikely(err)) + AuIOErr("err %d\n", err); +} @@ -31017,7 +31461,7 @@ index 0000000..16a96f88 + if (unlikely(wkq_err)) { + atomic_dec(&br->br_wbr->wbr_wh_running); + au_br_put(br); -+ kfree(arg); ++ au_delayed_kfree(arg); + } + do_dec = 0; + } @@ -31176,7 +31620,7 @@ index 0000000..16a96f88 + wh_dentry = ERR_PTR(err); + if (!err) { + wh_dentry = vfsub_lkup_one(&wh_name, h_parent); -+ kfree(wh_name.name); ++ au_delayed_kfree(wh_name.name); + } + return wh_dentry; +} @@ -31252,7 +31696,7 @@ index 0000000..16a96f88 + break; + } + } -+ free_page((unsigned long)wh_name.name); ++ au_delayed_free_page((unsigned long)wh_name.name); + +out: + return err; @@ -31294,7 +31738,7 @@ index 0000000..16a96f88 + rdhash = AUFS_RDHASH_DEF; + err = au_nhash_alloc(&whtmp->whlist, rdhash, gfp); + if (unlikely(err)) { -+ kfree(whtmp); ++ au_delayed_kfree(whtmp); + whtmp = ERR_PTR(err); + } + @@ -31309,7 +31753,7 @@ index 0000000..16a96f88 + dput(whtmp->wh_dentry); + iput(whtmp->dir); + au_nhash_wh_free(&whtmp->whlist); -+ kfree(whtmp); ++ au_delayed_kfree(whtmp); +} + +/* @@ -31526,10 +31970,10 @@ index 0000000..4077dd1 +#endif /* __AUFS_WHOUT_H__ */ diff --git a/fs/aufs/wkq.c b/fs/aufs/wkq.c new file mode 100644 -index 0000000..65c0137 +index 0000000..216d08f --- /dev/null +++ b/fs/aufs/wkq.c -@@ -0,0 +1,205 @@ +@@ -0,0 +1,200 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -31573,7 +32017,7 @@ index 0000000..65c0137 + else { + kobject_put(wkinfo->kobj); + module_put(THIS_MODULE); /* todo: ?? */ -+ kfree(wkinfo); ++ au_delayed_kfree(wkinfo); + } +} + @@ -31596,7 +32040,7 @@ index 0000000..65c0137 + +static void au_wkq_comp_free(struct completion *comp) +{ -+ kfree(comp); ++ au_delayed_kfree(comp); +} + +#else @@ -31677,7 +32121,7 @@ index 0000000..65c0137 + int err; + struct au_wkinfo *wkinfo; + -+ percpu_counter_inc(&au_sbi(sb)->si_nowait.nw_len); ++ atomic_inc(&au_sbi(sb)->si_nowait.nw_len); + + /* + * wkq_func() must free this wkinfo. @@ -31707,16 +32151,11 @@ index 0000000..65c0137 + +void au_nwt_init(struct au_nowait_tasks *nwt) +{ -+ percpu_counter_init(&nwt->nw_len, 0, GFP_NOFS); ++ atomic_set(&nwt->nw_len, 0); ++ /* smp_mb(); */ /* atomic_set */ + init_waitqueue_head(&nwt->nw_wq); +} + -+void au_nwt_fin(struct au_nowait_tasks *nwt) -+{ -+ AuDebugOn(percpu_counter_sum(&nwt->nw_len)); -+ percpu_counter_destroy(&nwt->nw_len); -+} -+ +void au_wkq_fin(void) +{ + destroy_workqueue(au_wkq); @@ -31737,10 +32176,10 @@ index 0000000..65c0137 +} diff --git a/fs/aufs/wkq.h b/fs/aufs/wkq.h new file mode 100644 -index 0000000..0b870cc +index 0000000..6d63f74 --- /dev/null +++ b/fs/aufs/wkq.h -@@ -0,0 +1,82 @@ +@@ -0,0 +1,80 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -31765,7 +32204,7 @@ index 0000000..0b870cc + * in the next operation, wait for the 'nowait' tasks in system-wide workqueue + */ +struct au_nowait_tasks { -+ struct percpu_counter nw_len; ++ atomic_t nw_len; + wait_queue_head_t nw_wq; +}; + @@ -31792,7 +32231,6 @@ index 0000000..0b870cc +int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb, + unsigned int flags); +void au_nwt_init(struct au_nowait_tasks *nwt); -+void au_nwt_fin(struct au_nowait_tasks *nwt); +int __init au_wkq_init(void); +void au_wkq_fin(void); + @@ -31810,14 +32248,13 @@ index 0000000..0b870cc + +static inline void au_nwt_done(struct au_nowait_tasks *nwt) +{ -+ percpu_counter_dec(&nwt->nw_len); -+ if (!percpu_counter_sum(&nwt->nw_len)) ++ if (atomic_dec_and_test(&nwt->nw_len)) + wake_up_all(&nwt->nw_wq); +} + +static inline int au_nwt_flush(struct au_nowait_tasks *nwt) +{ -+ wait_event(nwt->nw_wq, !percpu_counter_sum(&nwt->nw_len)); ++ wait_event(nwt->nw_wq, !atomic_read(&nwt->nw_len)); + return 0; +} + @@ -31825,10 +32262,10 @@ index 0000000..0b870cc +#endif /* __AUFS_WKQ_H__ */ diff --git a/fs/aufs/xattr.c b/fs/aufs/xattr.c new file mode 100644 -index 0000000..c12518f +index 0000000..2a148c9 --- /dev/null +++ b/fs/aufs/xattr.c -@@ -0,0 +1,332 @@ +@@ -0,0 +1,334 @@ +/* + * Copyright (C) 2014-2016 Junjiro R. Okajima + */ @@ -31994,10 +32431,12 @@ index 0000000..c12518f + AuTraceErr(err); + } + -+ kfree(value); ++ if (value) ++ au_delayed_kfree(value); + +out_free: -+ kfree(o); ++ if (o) ++ au_delayed_kfree(o); +out: + if (!unlocked) + inode_unlock(h_isrc); @@ -32163,10 +32602,10 @@ index 0000000..c12518f +#endif diff --git a/fs/aufs/xino.c b/fs/aufs/xino.c new file mode 100644 -index 0000000..4d1d5d4 +index 0000000..2a411f9 --- /dev/null +++ b/fs/aufs/xino.c -@@ -0,0 +1,1304 @@ +@@ -0,0 +1,1305 @@ +/* + * Copyright (C) 2005-2016 Junjiro R. Okajima + */ @@ -32494,7 +32933,7 @@ index 0000000..4d1d5d4 + AuErr1("statfs err %d, ignored\n", err); + +out_st: -+ kfree(st); ++ au_delayed_kfree(st); +out: + return err; +} @@ -32529,7 +32968,7 @@ index 0000000..4d1d5d4 + au_br_put(br); + si_write_unlock(sb); + au_nwt_done(&au_sbi(sb)->si_nowait); -+ kfree(args); ++ au_delayed_kfree(args); +} + +static int xino_trunc_test(struct super_block *sb, struct au_branch *br) @@ -32571,7 +33010,7 @@ index 0000000..4d1d5d4 + args = kmalloc(sizeof(*args), GFP_NOFS); + if (unlikely(!args)) { + AuErr1("no memory\n"); -+ goto out_args; ++ goto out; + } + + au_br_get(br); @@ -32583,9 +33022,8 @@ index 0000000..4d1d5d4 + + pr_err("wkq %d\n", wkq_err); + au_br_put(br); ++ au_delayed_kfree(args); + -+out_args: -+ kfree(args); +out: + atomic_dec(&br->br_xino_running); +} @@ -33108,7 +33546,7 @@ index 0000000..4d1d5d4 + (sb, au_sbr(sb, bindex)->br_xino.xi_file, page); + else + AuDbg("b%d\n", bindex); -+ free_page((unsigned long)page); ++ au_delayed_free_page((unsigned long)page); + +out: + return err; @@ -33185,7 +33623,8 @@ index 0000000..4d1d5d4 + if (sbinfo->si_xib) + fput(sbinfo->si_xib); + sbinfo->si_xib = NULL; -+ free_page((unsigned long)sbinfo->si_xib_buf); ++ if (sbinfo->si_xib_buf) ++ au_delayed_free_page((unsigned long)sbinfo->si_xib_buf); + sbinfo->si_xib_buf = NULL; +} + @@ -33228,7 +33667,8 @@ index 0000000..4d1d5d4 + goto out; /* success */ + +out_free: -+ free_page((unsigned long)sbinfo->si_xib_buf); ++ if (sbinfo->si_xib_buf) ++ au_delayed_free_page((unsigned long)sbinfo->si_xib_buf); + sbinfo->si_xib_buf = NULL; + if (err >= 0) + err = -EIO; @@ -33321,7 +33761,7 @@ index 0000000..4d1d5d4 + fput(p->new); + else + break; -+ kfree(fpair); ++ au_delayed_kfree(fpair); +out: + return err; +} @@ -33432,7 +33872,7 @@ index 0000000..4d1d5d4 + if (!IS_ERR(file)) + au_xino_brid_set(sb, br->br_id); + } -+ free_page((unsigned long)page); ++ au_delayed_free_page((unsigned long)page); + } else { + file = au_xino_create(sb, AUFS_XINO_DEFPATH, /*silent*/0); + if (IS_ERR(file)) @@ -33472,10 +33912,10 @@ index 0000000..4d1d5d4 + return err; +} diff --git a/fs/dcache.c b/fs/dcache.c -index d6847d7..c3c0b6d 100644 +index 5c7cc95..df0268c 100644 --- a/fs/dcache.c +++ b/fs/dcache.c -@@ -1202,7 +1202,7 @@ enum d_walk_ret { +@@ -1164,7 +1164,7 @@ enum d_walk_ret { * * The @enter() and @finish() callbacks are called with d_lock held. */ @@ -33507,10 +33947,10 @@ index 350a2c8..6f42279 100644 return error; diff --git a/fs/inode.c b/fs/inode.c -index 4ccbc21..aa6d071 100644 +index 7e3ef3a..675fe84 100644 --- a/fs/inode.c +++ b/fs/inode.c -@@ -1591,7 +1591,7 @@ EXPORT_SYMBOL(generic_update_time); +@@ -1593,7 +1593,7 @@ EXPORT_SYMBOL(generic_update_time); * This does the actual work of updating an inodes time or version. Must have * had called mnt_want_write() before calling this. */ @@ -33520,10 +33960,10 @@ index 4ccbc21..aa6d071 100644 int (*update_time)(struct inode *, struct timespec *, int); diff --git a/fs/proc/base.c b/fs/proc/base.c -index a11eb71..8f10865 100644 +index ac0df4d..42255e5 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c -@@ -1939,7 +1939,7 @@ static int map_files_get_link(struct dentry *dentry, struct path *path) +@@ -1938,7 +1938,7 @@ static int map_files_get_link(struct dentry *dentry, struct path *path) down_read(&mm->mmap_sem); vma = find_exact_vma(mm, vm_start, vm_end); if (vma && vma->vm_file) { @@ -33549,7 +33989,7 @@ index f8595e8..cb8eda0 100644 ino = inode->i_ino; } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c -index 4648c7f..061cb85 100644 +index f6fa99e..2750949 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -298,7 +298,10 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma, int is_pid) @@ -33564,7 +34004,7 @@ index 4648c7f..061cb85 100644 dev = inode->i_sb->s_dev; ino = inode->i_ino; pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; -@@ -1624,7 +1627,7 @@ static int show_numa_map(struct seq_file *m, void *v, int is_pid) +@@ -1634,7 +1637,7 @@ static int show_numa_map(struct seq_file *m, void *v, int is_pid) struct proc_maps_private *proc_priv = &numa_priv->proc_maps; struct vm_area_struct *vma = v; struct numa_maps *md = &numa_priv->md; @@ -33590,7 +34030,7 @@ index faacb0c..17b43be 100644 ino = inode->i_ino; pgoff = (loff_t)vma->vm_pgoff << PAGE_SHIFT; diff --git a/fs/read_write.c b/fs/read_write.c -index 933b53a..2d13282 100644 +index 66215a7..a1da117 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -515,6 +515,28 @@ ssize_t __vfs_write(struct file *file, const char __user *p, size_t count, @@ -33663,10 +34103,10 @@ index 7444f5f..bdac0be 100644 static inline void fput_light(struct file *file, int fput_needed) { diff --git a/include/linux/fs.h b/include/linux/fs.h -index dd28814..b689a48 100644 +index 901e25d..a71aa9e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h -@@ -1306,6 +1306,7 @@ extern void fasync_free(struct fasync_struct *); +@@ -1275,6 +1275,7 @@ extern void fasync_free(struct fasync_struct *); /* can be called from interrupts */ extern void kill_fasync(struct fasync_struct **, int, int); @@ -33674,7 +34114,7 @@ index dd28814..b689a48 100644 extern void __f_setown(struct file *filp, struct pid *, enum pid_type, int force); extern void f_setown(struct file *filp, unsigned long arg, int force); extern void f_delown(struct file *filp); -@@ -1690,6 +1691,7 @@ struct file_operations { +@@ -1699,6 +1700,7 @@ struct file_operations { ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int); unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); int (*check_flags)(int); @@ -33682,7 +34122,7 @@ index dd28814..b689a48 100644 int (*flock) (struct file *, int, struct file_lock *); ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); -@@ -1750,6 +1752,12 @@ ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, +@@ -1759,6 +1761,12 @@ ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, struct iovec *fast_pointer, struct iovec **ret_pointer); @@ -33695,7 +34135,7 @@ index dd28814..b689a48 100644 extern ssize_t __vfs_read(struct file *, char __user *, size_t, loff_t *); extern ssize_t __vfs_write(struct file *, const char __user *, size_t, loff_t *); extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *); -@@ -2105,6 +2113,7 @@ extern int current_umask(void); +@@ -2123,6 +2131,7 @@ extern int current_umask(void); extern void ihold(struct inode * inode); extern void iput(struct inode *); extern int generic_update_time(struct inode *, struct timespec *, int); @@ -33704,10 +34144,10 @@ index dd28814..b689a48 100644 /* /sys/fs */ extern struct kobject *fs_kobj; diff --git a/include/linux/mm.h b/include/linux/mm.h -index ece042d..1e24513 100644 +index ef815b9..a772481 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h -@@ -1239,6 +1239,28 @@ static inline int fixup_user_fault(struct task_struct *tsk, +@@ -1278,6 +1278,28 @@ static inline int fixup_user_fault(struct task_struct *tsk, } #endif @@ -33737,10 +34177,10 @@ index ece042d..1e24513 100644 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, int write); diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index ca3e517..10bc491 100644 +index 903200f..55fc528 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h -@@ -274,6 +274,7 @@ struct vm_region { +@@ -275,6 +275,7 @@ struct vm_region { unsigned long vm_top; /* region allocated to here */ unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */ struct file *vm_file; /* the backing file or NULL */ @@ -33748,7 +34188,7 @@ index ca3e517..10bc491 100644 int vm_usage; /* region usage count (access under nommu_region_sem) */ bool vm_icache_flushed : 1; /* true if the icache has been flushed for -@@ -348,6 +349,7 @@ struct vm_area_struct { +@@ -349,6 +350,7 @@ struct vm_area_struct { unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE units */ struct file * vm_file; /* File we map to (can be NULL). */ @@ -33772,7 +34212,7 @@ index da2751d..2e0fca6 100644 + unsigned int flags); #endif diff --git a/include/uapi/linux/Kbuild b/include/uapi/linux/Kbuild -index 8bdae34..65dbd5f 100644 +index 185f8ea..5deb0d1 100644 --- a/include/uapi/linux/Kbuild +++ b/include/uapi/linux/Kbuild @@ -59,6 +59,7 @@ header-y += atmsvc.h @@ -33785,7 +34225,7 @@ index 8bdae34..65dbd5f 100644 header-y += auxvec.h diff --git a/include/uapi/linux/aufs_type.h b/include/uapi/linux/aufs_type.h new file mode 100644 -index 0000000..4e08bdf +index 0000000..e220283 --- /dev/null +++ b/include/uapi/linux/aufs_type.h @@ -0,0 +1,406 @@ @@ -33817,7 +34257,7 @@ index 0000000..4e08bdf + +#include + -+#define AUFS_VERSION "4.x-rcN" ++#define AUFS_VERSION "4.8" + +/* todo? move this to linux-2.6.19/include/magic.h */ +#define AUFS_SUPER_MAGIC ('a' << 24 | 'u' << 16 | 'f' << 8 | 's') @@ -34196,10 +34636,10 @@ index 0000000..4e08bdf + +#endif /* __AUFS_TYPE_H__ */ diff --git a/kernel/fork.c b/kernel/fork.c -index 4a7ec0c..8c8f7ac 100644 +index beb3172..ad4cfa8 100644 --- a/kernel/fork.c +++ b/kernel/fork.c -@@ -479,7 +479,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) +@@ -477,7 +477,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) struct inode *inode = file_inode(file); struct address_space *mapping = file->f_mapping; @@ -34209,10 +34649,10 @@ index 4a7ec0c..8c8f7ac 100644 atomic_dec(&inode->i_writecount); i_mmap_lock_write(mapping); diff --git a/mm/Makefile b/mm/Makefile -index 78c6f7d..aea4230 100644 +index 2ca1faf..6b9da3f 100644 --- a/mm/Makefile +++ b/mm/Makefile -@@ -37,7 +37,7 @@ obj-y := filemap.o mempool.o oom_kill.o \ +@@ -40,7 +40,7 @@ obj-y := filemap.o mempool.o oom_kill.o \ mm_init.o mmu_context.o percpu.o slab_common.o \ compaction.o vmacache.o \ interval_tree.o list_lru.o workingset.o \ @@ -34222,10 +34662,10 @@ index 78c6f7d..aea4230 100644 obj-y += init-mm.o diff --git a/mm/filemap.c b/mm/filemap.c -index 20f3b1f..ee827ce 100644 +index 2d0986a..4a31bad 100644 --- a/mm/filemap.c +++ b/mm/filemap.c -@@ -2208,7 +2208,7 @@ int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) +@@ -2284,7 +2284,7 @@ int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf) int ret = VM_FAULT_LOCKED; sb_start_pagefault(inode->i_sb); @@ -34235,10 +34675,10 @@ index 20f3b1f..ee827ce 100644 if (page->mapping != inode->i_mapping) { unlock_page(page); diff --git a/mm/memory.c b/mm/memory.c -index cd1f29e..f0c204c 100644 +index 793fe0f..45f39f3 100644 --- a/mm/memory.c +++ b/mm/memory.c -@@ -2100,7 +2100,7 @@ static inline int wp_page_reuse(struct mm_struct *mm, +@@ -2113,7 +2113,7 @@ static inline int wp_page_reuse(struct fault_env *fe, pte_t orig_pte, } if (!page_mkwrite) @@ -34248,10 +34688,10 @@ index cd1f29e..f0c204c 100644 return VM_FAULT_WRITE; diff --git a/mm/mmap.c b/mm/mmap.c -index de2c176..b7f391c 100644 +index ca9d91b..f3ebc5a 100644 --- a/mm/mmap.c +++ b/mm/mmap.c -@@ -162,7 +162,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) +@@ -163,7 +163,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) if (vma->vm_ops && vma->vm_ops->close) vma->vm_ops->close(vma); if (vma->vm_file) @@ -34260,7 +34700,7 @@ index de2c176..b7f391c 100644 mpol_put(vma_policy(vma)); kmem_cache_free(vm_area_cachep, vma); return next; -@@ -782,7 +782,7 @@ again: remove_next = 1 + (end > next->vm_end); +@@ -790,7 +790,7 @@ again: if (remove_next) { if (file) { uprobe_munmap(next, next->vm_start, next->vm_end); @@ -34269,7 +34709,7 @@ index de2c176..b7f391c 100644 } if (next->anon_vma) anon_vma_merge(vma, next); -@@ -1563,8 +1563,8 @@ out: +@@ -1574,8 +1574,8 @@ out: return addr; unmap_and_free_vma: @@ -34279,7 +34719,7 @@ index de2c176..b7f391c 100644 /* Undo any partial mapping done by a device driver. */ unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); -@@ -2358,7 +2358,7 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, +@@ -2380,7 +2380,7 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, goto out_free_mpol; if (new->vm_file) @@ -34288,7 +34728,7 @@ index de2c176..b7f391c 100644 if (new->vm_ops && new->vm_ops->open) new->vm_ops->open(new); -@@ -2377,7 +2377,7 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, +@@ -2399,7 +2399,7 @@ static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, if (new->vm_ops && new->vm_ops->close) new->vm_ops->close(new); if (new->vm_file) @@ -34297,7 +34737,7 @@ index de2c176..b7f391c 100644 unlink_anon_vmas(new); out_free_mpol: mpol_put(vma_policy(new)); -@@ -2528,7 +2528,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, +@@ -2550,7 +2550,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, struct vm_area_struct *vma; unsigned long populate = 0; unsigned long ret = -EINVAL; @@ -34306,7 +34746,7 @@ index de2c176..b7f391c 100644 pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.txt.\n", current->comm, current->pid); -@@ -2597,10 +2597,27 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, +@@ -2625,10 +2625,27 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, } } @@ -34335,7 +34775,7 @@ index de2c176..b7f391c 100644 out: up_write(&mm->mmap_sem); if (populate) -@@ -2873,7 +2890,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, +@@ -2903,7 +2920,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (anon_vma_clone(new_vma, vma)) goto out_free_mempol; if (new_vma->vm_file) @@ -34345,7 +34785,7 @@ index de2c176..b7f391c 100644 new_vma->vm_ops->open(new_vma); vma_link(mm, new_vma, prev, rb_link, rb_parent); diff --git a/mm/nommu.c b/mm/nommu.c -index c2e588802..c39edc4 100644 +index 95daf81..5086a29 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -644,7 +644,7 @@ static void __put_nommu_region(struct vm_region *region) diff --git a/kernel/tools/cpupowertools/files/patches/mageia/gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch b/kernel/tools/cpupowertools/files/patches/mageia/gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch new file mode 100644 index 00000000..f7d2ac73 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch @@ -0,0 +1,75 @@ +From 5a920b85f2c6e3fd7d9dd9bb3f3345e9085e2360 Mon Sep 17 00:00:00 2001 +From: Paulo Zanoni +Date: Tue, 4 Oct 2016 14:37:32 -0300 +Subject: drm/i915/gen9: fix DDB partitioning for multi-screen cases + +With the previous code we were only recomputing the DDB partitioning +for the CRTCs included in the atomic commit, so any other active CRTCs +would end up having their DDB registers zeroed. In this patch we make +sure that the computed state starts as a copy of the current +partitioning, and then we only zero the DDBs that we're actually +going to recompute. + +How to reproduce the bug: + 1 - Enable the primary plane on pipe A + 2 - Enable the primary plane on pipe B + 3 - Enable the cursor or sprite plane on pipe A + +Step 3 will zero the DDB partitioning for pipe B since it's not +included in the commit that enabled the cursor or sprite for pipe A. + +I expect this to fix many FIFO underrun problems on gen9+. + +v2: + - Mention the cursor on the steps to reproduce the problem (Paulo). + - Add Testcase tag provided by Maarten (Maarten). + +Testcase: kms_cursor_legacy.cursorA-vs-flipB-atomic-transitions +Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=96226 +Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=96828 +Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=97450 +Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=97596 +Bugzilla: https://www.phoronix.com/scan.php?page=news_item&px=Intel-Skylake-Multi-Screen-Woes +Cc: stable@vger.kernel.org +Signed-off-by: Paulo Zanoni +Reviewed-by: Lyude +Link: http://patchwork.freedesktop.org/patch/msgid/1475602652-17326-1-git-send-email-paulo.r.zanoni@intel.com + +diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c +index 7f1748a..fe6c1c6 100644 +--- a/drivers/gpu/drm/i915/intel_pm.c ++++ b/drivers/gpu/drm/i915/intel_pm.c +@@ -3362,13 +3362,15 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, + int num_active; + int id, i; + ++ /* Clear the partitioning for disabled planes. */ ++ memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); ++ memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe])); ++ + if (WARN_ON(!state)) + return 0; + + if (!cstate->base.active) { + ddb->pipe[pipe].start = ddb->pipe[pipe].end = 0; +- memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); +- memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe])); + return 0; + } + +@@ -4054,6 +4056,12 @@ skl_compute_ddb(struct drm_atomic_state *state) + intel_state->wm_results.dirty_pipes = ~0; + } + ++ /* ++ * We're not recomputing for the pipes not included in the commit, so ++ * make sure we start with the current state. ++ */ ++ memcpy(ddb, &dev_priv->wm.skl_hw.ddb, sizeof(*ddb)); ++ + for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) { + struct intel_crtc_state *cstate; + +-- +cgit v0.10.2 + diff --git a/kernel/tools/cpupowertools/files/patches/mageia/intel_th-Fix-a-deadlock-in-modprobing.patch b/kernel/tools/cpupowertools/files/patches/mageia/intel_th-Fix-a-deadlock-in-modprobing.patch deleted file mode 100644 index 45b63cdd..00000000 --- a/kernel/tools/cpupowertools/files/patches/mageia/intel_th-Fix-a-deadlock-in-modprobing.patch +++ /dev/null @@ -1,95 +0,0 @@ -From a36aa80f3cb2540fb1dbad6240852de4365a2e82 Mon Sep 17 00:00:00 2001 -From: Alexander Shishkin -Date: Thu, 30 Jun 2016 11:51:44 +0300 -Subject: [PATCH] intel_th: Fix a deadlock in modprobing - -Driver initialization tries to request a hub (GTH) driver module from -its probe callback, resulting in a deadlock. - -This patch solves the problem by adding a deferred work for requesting -the hub module. - -Signed-off-by: Alexander Shishkin -Cc: # 4.4.x- ---- - drivers/hwtracing/intel_th/core.c | 35 ++++++++++++++++++++++++++++++++++- - drivers/hwtracing/intel_th/intel_th.h | 3 +++ - 2 files changed, 37 insertions(+), 1 deletion(-) - -diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c -index 1be543e..0b112ae 100644 ---- a/drivers/hwtracing/intel_th/core.c -+++ b/drivers/hwtracing/intel_th/core.c -@@ -465,6 +465,38 @@ static struct intel_th_subdevice { - }, - }; - -+#ifdef CONFIG_MODULES -+static void __intel_th_request_hub_module(struct work_struct *work) -+{ -+ struct intel_th *th = container_of(work, struct intel_th, -+ request_module_work); -+ -+ request_module("intel_th_%s", th->hub->name); -+} -+ -+static int intel_th_request_hub_module(struct intel_th *th) -+{ -+ INIT_WORK(&th->request_module_work, __intel_th_request_hub_module); -+ schedule_work(&th->request_module_work); -+ -+ return 0; -+} -+ -+static void intel_th_request_hub_module_flush(struct intel_th *th) -+{ -+ flush_work(&th->request_module_work); -+} -+#else -+static inline int intel_th_request_hub_module(struct intel_th *th) -+{ -+ return -EINVAL; -+} -+ -+static inline void intel_th_request_hub_module_flush(struct intel_th *th) -+{ -+} -+#endif /* CONFIG_MODULES */ -+ - static int intel_th_populate(struct intel_th *th, struct resource *devres, - unsigned int ndevres, int irq) - { -@@ -535,7 +567,7 @@ static int intel_th_populate(struct intel_th *th, struct resource *devres, - /* need switch driver to be loaded to enumerate the rest */ - if (subdev->type == INTEL_TH_SWITCH && !req) { - th->hub = thdev; -- err = request_module("intel_th_%s", subdev->name); -+ err = intel_th_request_hub_module(th); - if (!err) - req++; - } -@@ -652,6 +684,7 @@ void intel_th_free(struct intel_th *th) - { - int i; - -+ intel_th_request_hub_module_flush(th); - for (i = 0; i < TH_SUBDEVICE_MAX; i++) - if (th->thdev[i] != th->hub) - intel_th_device_remove(th->thdev[i]); -diff --git a/drivers/hwtracing/intel_th/intel_th.h b/drivers/hwtracing/intel_th/intel_th.h -index 0df22e3..0482848 100644 ---- a/drivers/hwtracing/intel_th/intel_th.h -+++ b/drivers/hwtracing/intel_th/intel_th.h -@@ -205,6 +205,9 @@ struct intel_th { - - int id; - int major; -+#ifdef CONFIG_MODULES -+ struct work_struct request_module_work; -+#endif /* CONFIG_MODULES */ - #ifdef CONFIG_INTEL_TH_DEBUG - struct dentry *dbg; - #endif --- -2.9.2 - diff --git a/kernel/tools/cpupowertools/files/patches/mageia/intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch b/kernel/tools/cpupowertools/files/patches/mageia/intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch deleted file mode 100644 index f6d4f341..00000000 --- a/kernel/tools/cpupowertools/files/patches/mageia/intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 7a1a47ce35821b40f5b2ce46379ba14393bc3873 Mon Sep 17 00:00:00 2001 -From: Alexander Shishkin -Date: Tue, 28 Jun 2016 18:55:23 +0300 -Subject: [PATCH] intel_th: pci: Add Kaby Lake PCH-H support - -This adds Intel(R) Trace Hub PCI ID for Kaby Lake PCH-H. - -Signed-off-by: Alexander Shishkin -Cc: ---- - drivers/hwtracing/intel_th/pci.c | 5 +++++ - 1 file changed, 5 insertions(+) - -diff --git a/drivers/hwtracing/intel_th/pci.c b/drivers/hwtracing/intel_th/pci.c -index 5e25c7e..0bba384 100644 ---- a/drivers/hwtracing/intel_th/pci.c -+++ b/drivers/hwtracing/intel_th/pci.c -@@ -80,6 +80,11 @@ static const struct pci_device_id intel_th_pci_id_table[] = { - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x1a8e), - .driver_data = (kernel_ulong_t)0, - }, -+ { -+ /* Kaby Lake PCH-H */ -+ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xa2a6), -+ .driver_data = (kernel_ulong_t)0, -+ }, - { 0 }, - }; - --- -2.9.2 - diff --git a/kernel/tools/cpupowertools/files/patches/mageia/kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch b/kernel/tools/cpupowertools/files/patches/mageia/kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch new file mode 100644 index 00000000..05b8cf99 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch @@ -0,0 +1,49 @@ +From 9f692cbe4a01dd9e3c3e954ec6b59662b68f9ce4 Mon Sep 17 00:00:00 2001 +From: Laura Abbott +Date: Fri, 9 Sep 2016 10:19:02 -0700 +Subject: [PATCH] cpupower: Correct return type of cpu_power_is_cpu_online in + cpufreq +To: Thomas Renninger +Cc: linux-pm@vger.kernel.org +Cc: linux-kernel@vger.kernel.org + +When converting to a shared library in ac5a181d065d ("cpupower: Add +cpuidle parts into library"), cpu_freq_cpu_exists was converted to +cpupower_is_cpu_online. cpu_req_cpu_exists returned 0 on success and +-ENOSYS on failure whereas cpupower_is_cpu_online returns 1 on success. +Check for the correct return value in cpufreq-set. + +See https://bugzilla.redhat.com/show_bug.cgi?id=1374212 + +Fixes: ac5a181d065d ("cpupower: Add cpuidle parts into library") +Reported-by: Julian Seward +Signed-off-by: Laura Abbott +--- + tools/power/cpupower/utils/cpufreq-set.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/tools/power/cpupower/utils/cpufreq-set.c b/tools/power/cpupower/utils/cpufreq-set.c +index b4bf769..8971d71 100644 +--- a/tools/power/cpupower/utils/cpufreq-set.c ++++ b/tools/power/cpupower/utils/cpufreq-set.c +@@ -296,7 +296,7 @@ int cmd_freq_set(int argc, char **argv) + struct cpufreq_affected_cpus *cpus; + + if (!bitmask_isbitset(cpus_chosen, cpu) || +- cpupower_is_cpu_online(cpu)) ++ cpupower_is_cpu_online(cpu) != 1) + continue; + + cpus = cpufreq_get_related_cpus(cpu); +@@ -316,7 +316,7 @@ int cmd_freq_set(int argc, char **argv) + cpu <= bitmask_last(cpus_chosen); cpu++) { + + if (!bitmask_isbitset(cpus_chosen, cpu) || +- cpupower_is_cpu_online(cpu)) ++ cpupower_is_cpu_online(cpu) != 1) + continue; + + if (cpupower_is_cpu_online(cpu) != 1) +-- +2.10.0 + diff --git a/kernel/tools/cpupowertools/files/patches/mageia/linux-BFQ-v8r4.patch b/kernel/tools/cpupowertools/files/patches/mageia/linux-BFQ-v8r4.patch new file mode 100644 index 00000000..abb33847 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/linux-BFQ-v8r4.patch @@ -0,0 +1,9004 @@ +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq-cgroup.c linux-4.8-bfq/block/bfq-cgroup.c +--- linux-4.8/block/bfq-cgroup.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq-cgroup.c 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,1196 @@ ++/* ++ * BFQ: CGROUPS support. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2015 Paolo Valente ++ * ++ * Copyright (C) 2016 Paolo Valente ++ * ++ * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ ++ * file. ++ */ ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ ++/* bfqg stats flags */ ++enum bfqg_stats_flags { ++ BFQG_stats_waiting = 0, ++ BFQG_stats_idling, ++ BFQG_stats_empty, ++}; ++ ++#define BFQG_FLAG_FNS(name) \ ++static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \ ++{ \ ++ stats->flags |= (1 << BFQG_stats_##name); \ ++} \ ++static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \ ++{ \ ++ stats->flags &= ~(1 << BFQG_stats_##name); \ ++} \ ++static int bfqg_stats_##name(struct bfqg_stats *stats) \ ++{ \ ++ return (stats->flags & (1 << BFQG_stats_##name)) != 0; \ ++} \ ++ ++BFQG_FLAG_FNS(waiting) ++BFQG_FLAG_FNS(idling) ++BFQG_FLAG_FNS(empty) ++#undef BFQG_FLAG_FNS ++ ++/* This should be called with the queue_lock held. */ ++static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats) ++{ ++ unsigned long long now; ++ ++ if (!bfqg_stats_waiting(stats)) ++ return; ++ ++ now = sched_clock(); ++ if (time_after64(now, stats->start_group_wait_time)) ++ blkg_stat_add(&stats->group_wait_time, ++ now - stats->start_group_wait_time); ++ bfqg_stats_clear_waiting(stats); ++} ++ ++/* This should be called with the queue_lock held. */ ++static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg, ++ struct bfq_group *curr_bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ if (bfqg_stats_waiting(stats)) ++ return; ++ if (bfqg == curr_bfqg) ++ return; ++ stats->start_group_wait_time = sched_clock(); ++ bfqg_stats_mark_waiting(stats); ++} ++ ++/* This should be called with the queue_lock held. */ ++static void bfqg_stats_end_empty_time(struct bfqg_stats *stats) ++{ ++ unsigned long long now; ++ ++ if (!bfqg_stats_empty(stats)) ++ return; ++ ++ now = sched_clock(); ++ if (time_after64(now, stats->start_empty_time)) ++ blkg_stat_add(&stats->empty_time, ++ now - stats->start_empty_time); ++ bfqg_stats_clear_empty(stats); ++} ++ ++static void bfqg_stats_update_dequeue(struct bfq_group *bfqg) ++{ ++ blkg_stat_add(&bfqg->stats.dequeue, 1); ++} ++ ++static void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ if (blkg_rwstat_total(&stats->queued)) ++ return; ++ ++ /* ++ * group is already marked empty. This can happen if bfqq got new ++ * request in parent group and moved to this group while being added ++ * to service tree. Just ignore the event and move on. ++ */ ++ if (bfqg_stats_empty(stats)) ++ return; ++ ++ stats->start_empty_time = sched_clock(); ++ bfqg_stats_mark_empty(stats); ++} ++ ++static void bfqg_stats_update_idle_time(struct bfq_group *bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ if (bfqg_stats_idling(stats)) { ++ unsigned long long now = sched_clock(); ++ ++ if (time_after64(now, stats->start_idle_time)) ++ blkg_stat_add(&stats->idle_time, ++ now - stats->start_idle_time); ++ bfqg_stats_clear_idling(stats); ++ } ++} ++ ++static void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ stats->start_idle_time = sched_clock(); ++ bfqg_stats_mark_idling(stats); ++} ++ ++static void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ ++ blkg_stat_add(&stats->avg_queue_size_sum, ++ blkg_rwstat_total(&stats->queued)); ++ blkg_stat_add(&stats->avg_queue_size_samples, 1); ++ bfqg_stats_update_group_wait_time(stats); ++} ++ ++static struct blkcg_policy blkcg_policy_bfq; ++ ++/* ++ * blk-cgroup policy-related handlers ++ * The following functions help in converting between blk-cgroup ++ * internal structures and BFQ-specific structures. ++ */ ++ ++static struct bfq_group *pd_to_bfqg(struct blkg_policy_data *pd) ++{ ++ return pd ? container_of(pd, struct bfq_group, pd) : NULL; ++} ++ ++static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg) ++{ ++ return pd_to_blkg(&bfqg->pd); ++} ++ ++static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg) ++{ ++ struct blkg_policy_data *pd = blkg_to_pd(blkg, &blkcg_policy_bfq); ++ ++ return pd_to_bfqg(pd); ++} ++ ++/* ++ * bfq_group handlers ++ * The following functions help in navigating the bfq_group hierarchy ++ * by allowing to find the parent of a bfq_group or the bfq_group ++ * associated to a bfq_queue. ++ */ ++ ++static struct bfq_group *bfqg_parent(struct bfq_group *bfqg) ++{ ++ struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent; ++ ++ return pblkg ? blkg_to_bfqg(pblkg) : NULL; ++} ++ ++static struct bfq_group *bfqq_group(struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *group_entity = bfqq->entity.parent; ++ ++ return group_entity ? container_of(group_entity, struct bfq_group, ++ entity) : ++ bfqq->bfqd->root_group; ++} ++ ++/* ++ * The following two functions handle get and put of a bfq_group by ++ * wrapping the related blk-cgroup hooks. ++ */ ++ ++static void bfqg_get(struct bfq_group *bfqg) ++{ ++ return blkg_get(bfqg_to_blkg(bfqg)); ++} ++ ++static void bfqg_put(struct bfq_group *bfqg) ++{ ++ return blkg_put(bfqg_to_blkg(bfqg)); ++} ++ ++static void bfqg_stats_update_io_add(struct bfq_group *bfqg, ++ struct bfq_queue *bfqq, ++ int op, int op_flags) ++{ ++ blkg_rwstat_add(&bfqg->stats.queued, op, op_flags, 1); ++ bfqg_stats_end_empty_time(&bfqg->stats); ++ if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue)) ++ bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq)); ++} ++ ++static void bfqg_stats_update_io_remove(struct bfq_group *bfqg, int op, ++ int op_flags) ++{ ++ blkg_rwstat_add(&bfqg->stats.queued, op, op_flags, -1); ++} ++ ++static void bfqg_stats_update_io_merged(struct bfq_group *bfqg, int op, ++ int op_flags) ++{ ++ blkg_rwstat_add(&bfqg->stats.merged, op, op_flags, 1); ++} ++ ++static void bfqg_stats_update_completion(struct bfq_group *bfqg, ++ uint64_t start_time, uint64_t io_start_time, int op, ++ int op_flags) ++{ ++ struct bfqg_stats *stats = &bfqg->stats; ++ unsigned long long now = sched_clock(); ++ ++ if (time_after64(now, io_start_time)) ++ blkg_rwstat_add(&stats->service_time, op, op_flags, ++ now - io_start_time); ++ if (time_after64(io_start_time, start_time)) ++ blkg_rwstat_add(&stats->wait_time, op, op_flags, ++ io_start_time - start_time); ++} ++ ++/* @stats = 0 */ ++static void bfqg_stats_reset(struct bfqg_stats *stats) ++{ ++ /* queued stats shouldn't be cleared */ ++ blkg_rwstat_reset(&stats->merged); ++ blkg_rwstat_reset(&stats->service_time); ++ blkg_rwstat_reset(&stats->wait_time); ++ blkg_stat_reset(&stats->time); ++ blkg_stat_reset(&stats->avg_queue_size_sum); ++ blkg_stat_reset(&stats->avg_queue_size_samples); ++ blkg_stat_reset(&stats->dequeue); ++ blkg_stat_reset(&stats->group_wait_time); ++ blkg_stat_reset(&stats->idle_time); ++ blkg_stat_reset(&stats->empty_time); ++} ++ ++/* @to += @from */ ++static void bfqg_stats_add_aux(struct bfqg_stats *to, struct bfqg_stats *from) ++{ ++ if (!to || !from) ++ return; ++ ++ /* queued stats shouldn't be cleared */ ++ blkg_rwstat_add_aux(&to->merged, &from->merged); ++ blkg_rwstat_add_aux(&to->service_time, &from->service_time); ++ blkg_rwstat_add_aux(&to->wait_time, &from->wait_time); ++ blkg_stat_add_aux(&from->time, &from->time); ++ blkg_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum); ++ blkg_stat_add_aux(&to->avg_queue_size_samples, ++ &from->avg_queue_size_samples); ++ blkg_stat_add_aux(&to->dequeue, &from->dequeue); ++ blkg_stat_add_aux(&to->group_wait_time, &from->group_wait_time); ++ blkg_stat_add_aux(&to->idle_time, &from->idle_time); ++ blkg_stat_add_aux(&to->empty_time, &from->empty_time); ++} ++ ++/* ++ * Transfer @bfqg's stats to its parent's dead_stats so that the ancestors' ++ * recursive stats can still account for the amount used by this bfqg after ++ * it's gone. ++ */ ++static void bfqg_stats_xfer_dead(struct bfq_group *bfqg) ++{ ++ struct bfq_group *parent; ++ ++ if (!bfqg) /* root_group */ ++ return; ++ ++ parent = bfqg_parent(bfqg); ++ ++ lockdep_assert_held(bfqg_to_blkg(bfqg)->q->queue_lock); ++ ++ if (unlikely(!parent)) ++ return; ++ ++ bfqg_stats_add_aux(&parent->stats, &bfqg->stats); ++ bfqg_stats_reset(&bfqg->stats); ++} ++ ++static void bfq_init_entity(struct bfq_entity *entity, ++ struct bfq_group *bfqg) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ entity->weight = entity->new_weight; ++ entity->orig_weight = entity->new_weight; ++ if (bfqq) { ++ bfqq->ioprio = bfqq->new_ioprio; ++ bfqq->ioprio_class = bfqq->new_ioprio_class; ++ bfqg_get(bfqg); ++ } ++ entity->parent = bfqg->my_entity; ++ entity->sched_data = &bfqg->sched_data; ++} ++ ++static void bfqg_stats_exit(struct bfqg_stats *stats) ++{ ++ blkg_rwstat_exit(&stats->merged); ++ blkg_rwstat_exit(&stats->service_time); ++ blkg_rwstat_exit(&stats->wait_time); ++ blkg_rwstat_exit(&stats->queued); ++ blkg_stat_exit(&stats->time); ++ blkg_stat_exit(&stats->avg_queue_size_sum); ++ blkg_stat_exit(&stats->avg_queue_size_samples); ++ blkg_stat_exit(&stats->dequeue); ++ blkg_stat_exit(&stats->group_wait_time); ++ blkg_stat_exit(&stats->idle_time); ++ blkg_stat_exit(&stats->empty_time); ++} ++ ++static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp) ++{ ++ if (blkg_rwstat_init(&stats->merged, gfp) || ++ blkg_rwstat_init(&stats->service_time, gfp) || ++ blkg_rwstat_init(&stats->wait_time, gfp) || ++ blkg_rwstat_init(&stats->queued, gfp) || ++ blkg_stat_init(&stats->time, gfp) || ++ blkg_stat_init(&stats->avg_queue_size_sum, gfp) || ++ blkg_stat_init(&stats->avg_queue_size_samples, gfp) || ++ blkg_stat_init(&stats->dequeue, gfp) || ++ blkg_stat_init(&stats->group_wait_time, gfp) || ++ blkg_stat_init(&stats->idle_time, gfp) || ++ blkg_stat_init(&stats->empty_time, gfp)) { ++ bfqg_stats_exit(stats); ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++static struct bfq_group_data *cpd_to_bfqgd(struct blkcg_policy_data *cpd) ++{ ++ return cpd ? container_of(cpd, struct bfq_group_data, pd) : NULL; ++} ++ ++static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg) ++{ ++ return cpd_to_bfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_bfq)); ++} ++ ++static struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp) ++{ ++ struct bfq_group_data *bgd; ++ ++ bgd = kzalloc(sizeof(*bgd), GFP_KERNEL); ++ if (!bgd) ++ return NULL; ++ return &bgd->pd; ++} ++ ++static void bfq_cpd_init(struct blkcg_policy_data *cpd) ++{ ++ struct bfq_group_data *d = cpd_to_bfqgd(cpd); ++ ++ d->weight = cgroup_subsys_on_dfl(io_cgrp_subsys) ? ++ CGROUP_WEIGHT_DFL : BFQ_WEIGHT_LEGACY_DFL; ++} ++ ++static void bfq_cpd_free(struct blkcg_policy_data *cpd) ++{ ++ kfree(cpd_to_bfqgd(cpd)); ++} ++ ++static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node) ++{ ++ struct bfq_group *bfqg; ++ ++ bfqg = kzalloc_node(sizeof(*bfqg), gfp, node); ++ if (!bfqg) ++ return NULL; ++ ++ if (bfqg_stats_init(&bfqg->stats, gfp)) { ++ kfree(bfqg); ++ return NULL; ++ } ++ ++ return &bfqg->pd; ++} ++ ++static void bfq_pd_init(struct blkg_policy_data *pd) ++{ ++ struct blkcg_gq *blkg; ++ struct bfq_group *bfqg; ++ struct bfq_data *bfqd; ++ struct bfq_entity *entity; ++ struct bfq_group_data *d; ++ ++ blkg = pd_to_blkg(pd); ++ BUG_ON(!blkg); ++ bfqg = blkg_to_bfqg(blkg); ++ bfqd = blkg->q->elevator->elevator_data; ++ entity = &bfqg->entity; ++ d = blkcg_to_bfqgd(blkg->blkcg); ++ ++ entity->orig_weight = entity->weight = entity->new_weight = d->weight; ++ entity->my_sched_data = &bfqg->sched_data; ++ bfqg->my_entity = entity; /* ++ * the root_group's will be set to NULL ++ * in bfq_init_queue() ++ */ ++ bfqg->bfqd = bfqd; ++ bfqg->active_entities = 0; ++ bfqg->rq_pos_tree = RB_ROOT; ++} ++ ++static void bfq_pd_free(struct blkg_policy_data *pd) ++{ ++ struct bfq_group *bfqg = pd_to_bfqg(pd); ++ ++ bfqg_stats_exit(&bfqg->stats); ++ return kfree(bfqg); ++} ++ ++static void bfq_pd_reset_stats(struct blkg_policy_data *pd) ++{ ++ struct bfq_group *bfqg = pd_to_bfqg(pd); ++ ++ bfqg_stats_reset(&bfqg->stats); ++} ++ ++static void bfq_group_set_parent(struct bfq_group *bfqg, ++ struct bfq_group *parent) ++{ ++ struct bfq_entity *entity; ++ ++ BUG_ON(!parent); ++ BUG_ON(!bfqg); ++ BUG_ON(bfqg == parent); ++ ++ entity = &bfqg->entity; ++ entity->parent = parent->my_entity; ++ entity->sched_data = &parent->sched_data; ++} ++ ++static struct bfq_group *bfq_lookup_bfqg(struct bfq_data *bfqd, ++ struct blkcg *blkcg) ++{ ++ struct blkcg_gq *blkg; ++ ++ blkg = blkg_lookup(blkcg, bfqd->queue); ++ if (likely(blkg)) ++ return blkg_to_bfqg(blkg); ++ return NULL; ++} ++ ++static struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd, ++ struct blkcg *blkcg) ++{ ++ struct bfq_group *bfqg, *parent; ++ struct bfq_entity *entity; ++ ++ assert_spin_locked(bfqd->queue->queue_lock); ++ ++ bfqg = bfq_lookup_bfqg(bfqd, blkcg); ++ ++ if (unlikely(!bfqg)) ++ return NULL; ++ ++ /* ++ * Update chain of bfq_groups as we might be handling a leaf group ++ * which, along with some of its relatives, has not been hooked yet ++ * to the private hierarchy of BFQ. ++ */ ++ entity = &bfqg->entity; ++ for_each_entity(entity) { ++ bfqg = container_of(entity, struct bfq_group, entity); ++ BUG_ON(!bfqg); ++ if (bfqg != bfqd->root_group) { ++ parent = bfqg_parent(bfqg); ++ if (!parent) ++ parent = bfqd->root_group; ++ BUG_ON(!parent); ++ bfq_group_set_parent(bfqg, parent); ++ } ++ } ++ ++ return bfqg; ++} ++ ++static void bfq_pos_tree_add_move(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq); ++ ++static void bfq_bfqq_expire(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ bool compensate, ++ enum bfqq_expiration reason); ++ ++/** ++ * bfq_bfqq_move - migrate @bfqq to @bfqg. ++ * @bfqd: queue descriptor. ++ * @bfqq: the queue to move. ++ * @bfqg: the group to move to. ++ * ++ * Move @bfqq to @bfqg, deactivating it from its old group and reactivating ++ * it on the new one. Avoid putting the entity on the old group idle tree. ++ * ++ * Must be called under the queue lock; the cgroup owning @bfqg must ++ * not disappear (by now this just means that we are called under ++ * rcu_read_lock()). ++ */ ++static void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ struct bfq_group *bfqg) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ BUG_ON(!bfq_bfqq_busy(bfqq) && !RB_EMPTY_ROOT(&bfqq->sort_list)); ++ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list) && !entity->on_st); ++ BUG_ON(bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) ++ && entity->on_st && ++ bfqq != bfqd->in_service_queue); ++ BUG_ON(!bfq_bfqq_busy(bfqq) && bfqq == bfqd->in_service_queue); ++ ++ /* If bfqq is empty, then bfq_bfqq_expire also invokes ++ * bfq_del_bfqq_busy, thereby removing bfqq and its entity ++ * from data structures related to current group. Otherwise we ++ * need to remove bfqq explicitly with bfq_deactivate_bfqq, as ++ * we do below. ++ */ ++ if (bfqq == bfqd->in_service_queue) ++ bfq_bfqq_expire(bfqd, bfqd->in_service_queue, ++ false, BFQ_BFQQ_PREEMPTED); ++ ++ BUG_ON(entity->on_st && !bfq_bfqq_busy(bfqq) ++ && &bfq_entity_service_tree(entity)->idle != ++ entity->tree); ++ ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_busy(bfqq)); ++ ++ if (bfq_bfqq_busy(bfqq)) ++ bfq_deactivate_bfqq(bfqd, bfqq, 0); ++ else if (entity->on_st) { ++ BUG_ON(&bfq_entity_service_tree(entity)->idle != ++ entity->tree); ++ bfq_put_idle_entity(bfq_entity_service_tree(entity), entity); ++ } ++ bfqg_put(bfqq_group(bfqq)); ++ ++ /* ++ * Here we use a reference to bfqg. We don't need a refcounter ++ * as the cgroup reference will not be dropped, so that its ++ * destroy() callback will not be invoked. ++ */ ++ entity->parent = bfqg->my_entity; ++ entity->sched_data = &bfqg->sched_data; ++ bfqg_get(bfqg); ++ ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_bfqq_busy(bfqq)); ++ if (bfq_bfqq_busy(bfqq)) { ++ bfq_pos_tree_add_move(bfqd, bfqq); ++ bfq_activate_bfqq(bfqd, bfqq); ++ } ++ ++ if (!bfqd->in_service_queue && !bfqd->rq_in_driver) ++ bfq_schedule_dispatch(bfqd); ++ BUG_ON(entity->on_st && !bfq_bfqq_busy(bfqq) ++ && &bfq_entity_service_tree(entity)->idle != ++ entity->tree); ++} ++ ++/** ++ * __bfq_bic_change_cgroup - move @bic to @cgroup. ++ * @bfqd: the queue descriptor. ++ * @bic: the bic to move. ++ * @blkcg: the blk-cgroup to move to. ++ * ++ * Move bic to blkcg, assuming that bfqd->queue is locked; the caller ++ * has to make sure that the reference to cgroup is valid across the call. ++ * ++ * NOTE: an alternative approach might have been to store the current ++ * cgroup in bfqq and getting a reference to it, reducing the lookup ++ * time here, at the price of slightly more complex code. ++ */ ++static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd, ++ struct bfq_io_cq *bic, ++ struct blkcg *blkcg) ++{ ++ struct bfq_queue *async_bfqq = bic_to_bfqq(bic, 0); ++ struct bfq_queue *sync_bfqq = bic_to_bfqq(bic, 1); ++ struct bfq_group *bfqg; ++ struct bfq_entity *entity; ++ ++ lockdep_assert_held(bfqd->queue->queue_lock); ++ ++ bfqg = bfq_find_set_group(bfqd, blkcg); ++ ++ if (unlikely(!bfqg)) ++ bfqg = bfqd->root_group; ++ ++ if (async_bfqq) { ++ entity = &async_bfqq->entity; ++ ++ if (entity->sched_data != &bfqg->sched_data) { ++ bic_set_bfqq(bic, NULL, 0); ++ bfq_log_bfqq(bfqd, async_bfqq, ++ "bic_change_group: %p %d", ++ async_bfqq, ++ async_bfqq->ref); ++ bfq_put_queue(async_bfqq); ++ } ++ } ++ ++ if (sync_bfqq) { ++ entity = &sync_bfqq->entity; ++ if (entity->sched_data != &bfqg->sched_data) ++ bfq_bfqq_move(bfqd, sync_bfqq, bfqg); ++ } ++ ++ return bfqg; ++} ++ ++static void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) ++{ ++ struct bfq_data *bfqd = bic_to_bfqd(bic); ++ struct bfq_group *bfqg = NULL; ++ uint64_t serial_nr; ++ ++ rcu_read_lock(); ++ serial_nr = bio_blkcg(bio)->css.serial_nr; ++ ++ /* ++ * Check whether blkcg has changed. The condition may trigger ++ * spuriously on a newly created cic but there's no harm. ++ */ ++ if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr)) ++ goto out; ++ ++ bfqg = __bfq_bic_change_cgroup(bfqd, bic, bio_blkcg(bio)); ++ bic->blkcg_serial_nr = serial_nr; ++out: ++ rcu_read_unlock(); ++} ++ ++/** ++ * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st. ++ * @st: the service tree being flushed. ++ */ ++static void bfq_flush_idle_tree(struct bfq_service_tree *st) ++{ ++ struct bfq_entity *entity = st->first_idle; ++ ++ for (; entity ; entity = st->first_idle) ++ __bfq_deactivate_entity(entity, 0); ++} ++ ++/** ++ * bfq_reparent_leaf_entity - move leaf entity to the root_group. ++ * @bfqd: the device data structure with the root group. ++ * @entity: the entity to move. ++ */ ++static void bfq_reparent_leaf_entity(struct bfq_data *bfqd, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ BUG_ON(!bfqq); ++ bfq_bfqq_move(bfqd, bfqq, bfqd->root_group); ++} ++ ++/** ++ * bfq_reparent_active_entities - move to the root group all active ++ * entities. ++ * @bfqd: the device data structure with the root group. ++ * @bfqg: the group to move from. ++ * @st: the service tree with the entities. ++ * ++ * Needs queue_lock to be taken and reference to be valid over the call. ++ */ ++static void bfq_reparent_active_entities(struct bfq_data *bfqd, ++ struct bfq_group *bfqg, ++ struct bfq_service_tree *st) ++{ ++ struct rb_root *active = &st->active; ++ struct bfq_entity *entity = NULL; ++ ++ if (!RB_EMPTY_ROOT(&st->active)) ++ entity = bfq_entity_of(rb_first(active)); ++ ++ for (; entity ; entity = bfq_entity_of(rb_first(active))) ++ bfq_reparent_leaf_entity(bfqd, entity); ++ ++ if (bfqg->sched_data.in_service_entity) ++ bfq_reparent_leaf_entity(bfqd, ++ bfqg->sched_data.in_service_entity); ++} ++ ++/** ++ * bfq_pd_offline - deactivate the entity associated with @pd, ++ * and reparent its children entities. ++ * @pd: descriptor of the policy going offline. ++ * ++ * blkio already grabs the queue_lock for us, so no need to use ++ * RCU-based magic ++ */ ++static void bfq_pd_offline(struct blkg_policy_data *pd) ++{ ++ struct bfq_service_tree *st; ++ struct bfq_group *bfqg; ++ struct bfq_data *bfqd; ++ struct bfq_entity *entity; ++ int i; ++ ++ BUG_ON(!pd); ++ bfqg = pd_to_bfqg(pd); ++ BUG_ON(!bfqg); ++ bfqd = bfqg->bfqd; ++ BUG_ON(bfqd && !bfqd->root_group); ++ ++ entity = bfqg->my_entity; ++ ++ if (!entity) /* root group */ ++ return; ++ ++ /* ++ * Empty all service_trees belonging to this group before ++ * deactivating the group itself. ++ */ ++ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) { ++ BUG_ON(!bfqg->sched_data.service_tree); ++ st = bfqg->sched_data.service_tree + i; ++ /* ++ * The idle tree may still contain bfq_queues belonging ++ * to exited task because they never migrated to a different ++ * cgroup from the one being destroyed now. No one else ++ * can access them so it's safe to act without any lock. ++ */ ++ bfq_flush_idle_tree(st); ++ ++ /* ++ * It may happen that some queues are still active ++ * (busy) upon group destruction (if the corresponding ++ * processes have been forced to terminate). We move ++ * all the leaf entities corresponding to these queues ++ * to the root_group. ++ * Also, it may happen that the group has an entity ++ * in service, which is disconnected from the active ++ * tree: it must be moved, too. ++ * There is no need to put the sync queues, as the ++ * scheduler has taken no reference. ++ */ ++ bfq_reparent_active_entities(bfqd, bfqg, st); ++ BUG_ON(!RB_EMPTY_ROOT(&st->active)); ++ BUG_ON(!RB_EMPTY_ROOT(&st->idle)); ++ } ++ BUG_ON(bfqg->sched_data.next_in_service); ++ BUG_ON(bfqg->sched_data.in_service_entity); ++ ++ __bfq_deactivate_entity(entity, 0); ++ bfq_put_async_queues(bfqd, bfqg); ++ BUG_ON(entity->tree); ++ ++ /* ++ * @blkg is going offline and will be ignored by ++ * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so ++ * that they don't get lost. If IOs complete after this point, the ++ * stats for them will be lost. Oh well... ++ */ ++ bfqg_stats_xfer_dead(bfqg); ++} ++ ++static void bfq_end_wr_async(struct bfq_data *bfqd) ++{ ++ struct blkcg_gq *blkg; ++ ++ list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) { ++ struct bfq_group *bfqg = blkg_to_bfqg(blkg); ++ BUG_ON(!bfqg); ++ ++ bfq_end_wr_async_queues(bfqd, bfqg); ++ } ++ bfq_end_wr_async_queues(bfqd, bfqd->root_group); ++} ++ ++static int bfq_io_show_weight(struct seq_file *sf, void *v) ++{ ++ struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); ++ struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg); ++ unsigned int val = 0; ++ ++ if (bfqgd) ++ val = bfqgd->weight; ++ ++ seq_printf(sf, "%u\n", val); ++ ++ return 0; ++} ++ ++static int bfq_io_set_weight_legacy(struct cgroup_subsys_state *css, ++ struct cftype *cftype, ++ u64 val) ++{ ++ struct blkcg *blkcg = css_to_blkcg(css); ++ struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg); ++ struct blkcg_gq *blkg; ++ int ret = -ERANGE; ++ ++ if (val < BFQ_MIN_WEIGHT || val > BFQ_MAX_WEIGHT) ++ return ret; ++ ++ ret = 0; ++ spin_lock_irq(&blkcg->lock); ++ bfqgd->weight = (unsigned short)val; ++ hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { ++ struct bfq_group *bfqg = blkg_to_bfqg(blkg); ++ ++ if (!bfqg) ++ continue; ++ /* ++ * Setting the prio_changed flag of the entity ++ * to 1 with new_weight == weight would re-set ++ * the value of the weight to its ioprio mapping. ++ * Set the flag only if necessary. ++ */ ++ if ((unsigned short)val != bfqg->entity.new_weight) { ++ bfqg->entity.new_weight = (unsigned short)val; ++ /* ++ * Make sure that the above new value has been ++ * stored in bfqg->entity.new_weight before ++ * setting the prio_changed flag. In fact, ++ * this flag may be read asynchronously (in ++ * critical sections protected by a different ++ * lock than that held here), and finding this ++ * flag set may cause the execution of the code ++ * for updating parameters whose value may ++ * depend also on bfqg->entity.new_weight (in ++ * __bfq_entity_update_weight_prio). ++ * This barrier makes sure that the new value ++ * of bfqg->entity.new_weight is correctly ++ * seen in that code. ++ */ ++ smp_wmb(); ++ bfqg->entity.prio_changed = 1; ++ } ++ } ++ spin_unlock_irq(&blkcg->lock); ++ ++ return ret; ++} ++ ++static ssize_t bfq_io_set_weight(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, ++ loff_t off) ++{ ++ u64 weight; ++ /* First unsigned long found in the file is used */ ++ int ret = kstrtoull(strim(buf), 0, &weight); ++ ++ if (ret) ++ return ret; ++ ++ return bfq_io_set_weight_legacy(of_css(of), NULL, weight); ++} ++ ++static int bfqg_print_stat(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat, ++ &blkcg_policy_bfq, seq_cft(sf)->private, false); ++ return 0; ++} ++ ++static int bfqg_print_rwstat(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat, ++ &blkcg_policy_bfq, seq_cft(sf)->private, true); ++ return 0; ++} ++ ++static u64 bfqg_prfill_stat_recursive(struct seq_file *sf, ++ struct blkg_policy_data *pd, int off) ++{ ++ u64 sum = blkg_stat_recursive_sum(pd_to_blkg(pd), ++ &blkcg_policy_bfq, off); ++ return __blkg_prfill_u64(sf, pd, sum); ++} ++ ++static u64 bfqg_prfill_rwstat_recursive(struct seq_file *sf, ++ struct blkg_policy_data *pd, int off) ++{ ++ struct blkg_rwstat sum = blkg_rwstat_recursive_sum(pd_to_blkg(pd), ++ &blkcg_policy_bfq, ++ off); ++ return __blkg_prfill_rwstat(sf, pd, &sum); ++} ++ ++static int bfqg_print_stat_recursive(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_stat_recursive, &blkcg_policy_bfq, ++ seq_cft(sf)->private, false); ++ return 0; ++} ++ ++static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq, ++ seq_cft(sf)->private, true); ++ return 0; ++} ++ ++static u64 bfqg_prfill_sectors(struct seq_file *sf, struct blkg_policy_data *pd, ++ int off) ++{ ++ u64 sum = blkg_rwstat_total(&pd->blkg->stat_bytes); ++ ++ return __blkg_prfill_u64(sf, pd, sum >> 9); ++} ++ ++static int bfqg_print_stat_sectors(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_sectors, &blkcg_policy_bfq, 0, false); ++ return 0; ++} ++ ++static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf, ++ struct blkg_policy_data *pd, int off) ++{ ++ struct blkg_rwstat tmp = blkg_rwstat_recursive_sum(pd->blkg, NULL, ++ offsetof(struct blkcg_gq, stat_bytes)); ++ u64 sum = atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) + ++ atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]); ++ ++ return __blkg_prfill_u64(sf, pd, sum >> 9); ++} ++ ++static int bfqg_print_stat_sectors_recursive(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_sectors_recursive, &blkcg_policy_bfq, 0, ++ false); ++ return 0; ++} ++ ++ ++static u64 bfqg_prfill_avg_queue_size(struct seq_file *sf, ++ struct blkg_policy_data *pd, int off) ++{ ++ struct bfq_group *bfqg = pd_to_bfqg(pd); ++ u64 samples = blkg_stat_read(&bfqg->stats.avg_queue_size_samples); ++ u64 v = 0; ++ ++ if (samples) { ++ v = blkg_stat_read(&bfqg->stats.avg_queue_size_sum); ++ v = div64_u64(v, samples); ++ } ++ __blkg_prfill_u64(sf, pd, v); ++ return 0; ++} ++ ++/* print avg_queue_size */ ++static int bfqg_print_avg_queue_size(struct seq_file *sf, void *v) ++{ ++ blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), ++ bfqg_prfill_avg_queue_size, &blkcg_policy_bfq, ++ 0, false); ++ return 0; ++} ++ ++static struct bfq_group * ++bfq_create_group_hierarchy(struct bfq_data *bfqd, int node) ++{ ++ int ret; ++ ++ ret = blkcg_activate_policy(bfqd->queue, &blkcg_policy_bfq); ++ if (ret) ++ return NULL; ++ ++ return blkg_to_bfqg(bfqd->queue->root_blkg); ++} ++ ++static struct cftype bfq_blkcg_legacy_files[] = { ++ { ++ .name = "bfq.weight", ++ .flags = CFTYPE_NOT_ON_ROOT, ++ .seq_show = bfq_io_show_weight, ++ .write_u64 = bfq_io_set_weight_legacy, ++ }, ++ ++ /* statistics, covers only the tasks in the bfqg */ ++ { ++ .name = "bfq.time", ++ .private = offsetof(struct bfq_group, stats.time), ++ .seq_show = bfqg_print_stat, ++ }, ++ { ++ .name = "bfq.sectors", ++ .seq_show = bfqg_print_stat_sectors, ++ }, ++ { ++ .name = "bfq.io_service_bytes", ++ .private = (unsigned long)&blkcg_policy_bfq, ++ .seq_show = blkg_print_stat_bytes, ++ }, ++ { ++ .name = "bfq.io_serviced", ++ .private = (unsigned long)&blkcg_policy_bfq, ++ .seq_show = blkg_print_stat_ios, ++ }, ++ { ++ .name = "bfq.io_service_time", ++ .private = offsetof(struct bfq_group, stats.service_time), ++ .seq_show = bfqg_print_rwstat, ++ }, ++ { ++ .name = "bfq.io_wait_time", ++ .private = offsetof(struct bfq_group, stats.wait_time), ++ .seq_show = bfqg_print_rwstat, ++ }, ++ { ++ .name = "bfq.io_merged", ++ .private = offsetof(struct bfq_group, stats.merged), ++ .seq_show = bfqg_print_rwstat, ++ }, ++ { ++ .name = "bfq.io_queued", ++ .private = offsetof(struct bfq_group, stats.queued), ++ .seq_show = bfqg_print_rwstat, ++ }, ++ ++ /* the same statictics which cover the bfqg and its descendants */ ++ { ++ .name = "bfq.time_recursive", ++ .private = offsetof(struct bfq_group, stats.time), ++ .seq_show = bfqg_print_stat_recursive, ++ }, ++ { ++ .name = "bfq.sectors_recursive", ++ .seq_show = bfqg_print_stat_sectors_recursive, ++ }, ++ { ++ .name = "bfq.io_service_bytes_recursive", ++ .private = (unsigned long)&blkcg_policy_bfq, ++ .seq_show = blkg_print_stat_bytes_recursive, ++ }, ++ { ++ .name = "bfq.io_serviced_recursive", ++ .private = (unsigned long)&blkcg_policy_bfq, ++ .seq_show = blkg_print_stat_ios_recursive, ++ }, ++ { ++ .name = "bfq.io_service_time_recursive", ++ .private = offsetof(struct bfq_group, stats.service_time), ++ .seq_show = bfqg_print_rwstat_recursive, ++ }, ++ { ++ .name = "bfq.io_wait_time_recursive", ++ .private = offsetof(struct bfq_group, stats.wait_time), ++ .seq_show = bfqg_print_rwstat_recursive, ++ }, ++ { ++ .name = "bfq.io_merged_recursive", ++ .private = offsetof(struct bfq_group, stats.merged), ++ .seq_show = bfqg_print_rwstat_recursive, ++ }, ++ { ++ .name = "bfq.io_queued_recursive", ++ .private = offsetof(struct bfq_group, stats.queued), ++ .seq_show = bfqg_print_rwstat_recursive, ++ }, ++ { ++ .name = "bfq.avg_queue_size", ++ .seq_show = bfqg_print_avg_queue_size, ++ }, ++ { ++ .name = "bfq.group_wait_time", ++ .private = offsetof(struct bfq_group, stats.group_wait_time), ++ .seq_show = bfqg_print_stat, ++ }, ++ { ++ .name = "bfq.idle_time", ++ .private = offsetof(struct bfq_group, stats.idle_time), ++ .seq_show = bfqg_print_stat, ++ }, ++ { ++ .name = "bfq.empty_time", ++ .private = offsetof(struct bfq_group, stats.empty_time), ++ .seq_show = bfqg_print_stat, ++ }, ++ { ++ .name = "bfq.dequeue", ++ .private = offsetof(struct bfq_group, stats.dequeue), ++ .seq_show = bfqg_print_stat, ++ }, ++ { } /* terminate */ ++}; ++ ++static struct cftype bfq_blkg_files[] = { ++ { ++ .name = "bfq.weight", ++ .flags = CFTYPE_NOT_ON_ROOT, ++ .seq_show = bfq_io_show_weight, ++ .write = bfq_io_set_weight, ++ }, ++ {} /* terminate */ ++}; ++ ++#else /* CONFIG_BFQ_GROUP_IOSCHED */ ++ ++static inline void bfqg_stats_update_io_add(struct bfq_group *bfqg, ++ struct bfq_queue *bfqq, int op, int op_flags) { } ++static inline void ++bfqg_stats_update_io_remove(struct bfq_group *bfqg, int op, int op_flags) { } ++static inline void ++bfqg_stats_update_io_merged(struct bfq_group *bfqg, int op, int op_flags) { } ++static inline void bfqg_stats_update_completion(struct bfq_group *bfqg, ++ uint64_t start_time, uint64_t io_start_time, int op, ++ int op_flags) { } ++static inline void ++bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg, ++ struct bfq_group *curr_bfqg) { } ++static inline void bfqg_stats_end_empty_time(struct bfqg_stats *stats) { } ++static inline void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { } ++static inline void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { } ++static inline void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { } ++static inline void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { } ++static inline void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { } ++ ++static void bfq_init_entity(struct bfq_entity *entity, ++ struct bfq_group *bfqg) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ entity->weight = entity->new_weight; ++ entity->orig_weight = entity->new_weight; ++ if (bfqq) { ++ bfqq->ioprio = bfqq->new_ioprio; ++ bfqq->ioprio_class = bfqq->new_ioprio_class; ++ } ++ entity->sched_data = &bfqg->sched_data; ++} ++ ++static struct bfq_group * ++bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) ++{ ++ struct bfq_data *bfqd = bic_to_bfqd(bic); ++ ++ return bfqd->root_group; ++} ++ ++static void bfq_end_wr_async(struct bfq_data *bfqd) ++{ ++ bfq_end_wr_async_queues(bfqd, bfqd->root_group); ++} ++ ++static struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd, ++ struct blkcg *blkcg) ++{ ++ return bfqd->root_group; ++} ++ ++static struct bfq_group *bfqq_group(struct bfq_queue *bfqq) ++{ ++ return bfqq->bfqd->root_group; ++} ++ ++static struct bfq_group * ++bfq_create_group_hierarchy(struct bfq_data *bfqd, int node) ++{ ++ struct bfq_group *bfqg; ++ int i; ++ ++ bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node); ++ if (!bfqg) ++ return NULL; ++ ++ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) ++ bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT; ++ ++ return bfqg; ++} ++#endif +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq.h linux-4.8-bfq/block/bfq.h +--- linux-4.8/block/bfq.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq.h 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,886 @@ ++/* ++ * BFQ-v8r4 for 4.8.0: data structures and common functions prototypes. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2015 Paolo Valente ++ * ++ * Copyright (C) 2016 Paolo Valente ++ */ ++ ++#ifndef _BFQ_H ++#define _BFQ_H ++ ++#include ++#include ++#include ++#include ++#include ++ ++#define BFQ_IOPRIO_CLASSES 3 ++#define BFQ_CL_IDLE_TIMEOUT (HZ/5) ++ ++#define BFQ_MIN_WEIGHT 1 ++#define BFQ_MAX_WEIGHT 1000 ++#define BFQ_WEIGHT_CONVERSION_COEFF 10 ++ ++#define BFQ_DEFAULT_QUEUE_IOPRIO 4 ++ ++#define BFQ_WEIGHT_LEGACY_DFL 100 ++#define BFQ_DEFAULT_GRP_IOPRIO 0 ++#define BFQ_DEFAULT_GRP_CLASS IOPRIO_CLASS_BE ++ ++/* ++ * Soft real-time applications are extremely more latency sensitive ++ * than interactive ones. Over-raise the weight of the former to ++ * privilege them against the latter. ++ */ ++#define BFQ_SOFTRT_WEIGHT_FACTOR 100 ++ ++struct bfq_entity; ++ ++/** ++ * struct bfq_service_tree - per ioprio_class service tree. ++ * ++ * Each service tree represents a B-WF2Q+ scheduler on its own. Each ++ * ioprio_class has its own independent scheduler, and so its own ++ * bfq_service_tree. All the fields are protected by the queue lock ++ * of the containing bfqd. ++ */ ++struct bfq_service_tree { ++ /* tree for active entities (i.e., those backlogged) */ ++ struct rb_root active; ++ /* tree for idle entities (i.e., not backlogged, with V <= F_i)*/ ++ struct rb_root idle; ++ ++ struct bfq_entity *first_idle; /* idle entity with minimum F_i */ ++ struct bfq_entity *last_idle; /* idle entity with maximum F_i */ ++ ++ u64 vtime; /* scheduler virtual time */ ++ /* scheduler weight sum; active and idle entities contribute to it */ ++ unsigned long wsum; ++}; ++ ++/** ++ * struct bfq_sched_data - multi-class scheduler. ++ * ++ * bfq_sched_data is the basic scheduler queue. It supports three ++ * ioprio_classes, and can be used either as a toplevel queue or as an ++ * intermediate queue on a hierarchical setup. @next_in_service ++ * points to the active entity of the sched_data service trees that ++ * will be scheduled next. It is used to reduce the number of steps ++ * needed for each hierarchical-schedule update. ++ * ++ * The supported ioprio_classes are the same as in CFQ, in descending ++ * priority order, IOPRIO_CLASS_RT, IOPRIO_CLASS_BE, IOPRIO_CLASS_IDLE. ++ * Requests from higher priority queues are served before all the ++ * requests from lower priority queues; among requests of the same ++ * queue requests are served according to B-WF2Q+. ++ * All the fields are protected by the queue lock of the containing bfqd. ++ */ ++struct bfq_sched_data { ++ struct bfq_entity *in_service_entity; /* entity in service */ ++ /* head-of-the-line entity in the scheduler (see comments above) */ ++ struct bfq_entity *next_in_service; ++ /* array of service trees, one per ioprio_class */ ++ struct bfq_service_tree service_tree[BFQ_IOPRIO_CLASSES]; ++}; ++ ++/** ++ * struct bfq_weight_counter - counter of the number of all active entities ++ * with a given weight. ++ */ ++struct bfq_weight_counter { ++ unsigned int weight; /* weight of the entities this counter refers to */ ++ unsigned int num_active; /* nr of active entities with this weight */ ++ /* ++ * Weights tree member (see bfq_data's @queue_weights_tree and ++ * @group_weights_tree) ++ */ ++ struct rb_node weights_node; ++}; ++ ++/** ++ * struct bfq_entity - schedulable entity. ++ * ++ * A bfq_entity is used to represent either a bfq_queue (leaf node in the ++ * cgroup hierarchy) or a bfq_group into the upper level scheduler. Each ++ * entity belongs to the sched_data of the parent group in the cgroup ++ * hierarchy. Non-leaf entities have also their own sched_data, stored ++ * in @my_sched_data. ++ * ++ * Each entity stores independently its priority values; this would ++ * allow different weights on different devices, but this ++ * functionality is not exported to userspace by now. Priorities and ++ * weights are updated lazily, first storing the new values into the ++ * new_* fields, then setting the @prio_changed flag. As soon as ++ * there is a transition in the entity state that allows the priority ++ * update to take place the effective and the requested priority ++ * values are synchronized. ++ * ++ * Unless cgroups are used, the weight value is calculated from the ++ * ioprio to export the same interface as CFQ. When dealing with ++ * ``well-behaved'' queues (i.e., queues that do not spend too much ++ * time to consume their budget and have true sequential behavior, and ++ * when there are no external factors breaking anticipation) the ++ * relative weights at each level of the cgroups hierarchy should be ++ * guaranteed. All the fields are protected by the queue lock of the ++ * containing bfqd. ++ */ ++struct bfq_entity { ++ struct rb_node rb_node; /* service_tree member */ ++ /* pointer to the weight counter associated with this entity */ ++ struct bfq_weight_counter *weight_counter; ++ ++ /* ++ * flag, true if the entity is on a tree (either the active or ++ * the idle one of its service_tree). ++ */ ++ int on_st; ++ ++ u64 finish; /* B-WF2Q+ finish timestamp (aka F_i) */ ++ u64 start; /* B-WF2Q+ start timestamp (aka S_i) */ ++ ++ /* tree the entity is enqueued into; %NULL if not on a tree */ ++ struct rb_root *tree; ++ ++ /* ++ * minimum start time of the (active) subtree rooted at this ++ * entity; used for O(log N) lookups into active trees ++ */ ++ u64 min_start; ++ ++ /* amount of service received during the last service slot */ ++ int service; ++ ++ /* budget, used also to calculate F_i: F_i = S_i + @budget / @weight */ ++ int budget; ++ ++ unsigned int weight; /* weight of the queue */ ++ unsigned int new_weight; /* next weight if a change is in progress */ ++ ++ /* original weight, used to implement weight boosting */ ++ unsigned int orig_weight; ++ ++ /* parent entity, for hierarchical scheduling */ ++ struct bfq_entity *parent; ++ ++ /* ++ * For non-leaf nodes in the hierarchy, the associated ++ * scheduler queue, %NULL on leaf nodes. ++ */ ++ struct bfq_sched_data *my_sched_data; ++ /* the scheduler queue this entity belongs to */ ++ struct bfq_sched_data *sched_data; ++ ++ /* flag, set to request a weight, ioprio or ioprio_class change */ ++ int prio_changed; ++}; ++ ++struct bfq_group; ++ ++/** ++ * struct bfq_queue - leaf schedulable entity. ++ * ++ * A bfq_queue is a leaf request queue; it can be associated with an ++ * io_context or more, if it is async or shared between cooperating ++ * processes. @cgroup holds a reference to the cgroup, to be sure that it ++ * does not disappear while a bfqq still references it (mostly to avoid ++ * races between request issuing and task migration followed by cgroup ++ * destruction). ++ * All the fields are protected by the queue lock of the containing bfqd. ++ */ ++struct bfq_queue { ++ /* reference counter */ ++ int ref; ++ /* parent bfq_data */ ++ struct bfq_data *bfqd; ++ ++ /* current ioprio and ioprio class */ ++ unsigned short ioprio, ioprio_class; ++ /* next ioprio and ioprio class if a change is in progress */ ++ unsigned short new_ioprio, new_ioprio_class; ++ ++ /* ++ * Shared bfq_queue if queue is cooperating with one or more ++ * other queues. ++ */ ++ struct bfq_queue *new_bfqq; ++ /* request-position tree member (see bfq_group's @rq_pos_tree) */ ++ struct rb_node pos_node; ++ /* request-position tree root (see bfq_group's @rq_pos_tree) */ ++ struct rb_root *pos_root; ++ ++ /* sorted list of pending requests */ ++ struct rb_root sort_list; ++ /* if fifo isn't expired, next request to serve */ ++ struct request *next_rq; ++ /* number of sync and async requests queued */ ++ int queued[2]; ++ /* number of sync and async requests currently allocated */ ++ int allocated[2]; ++ /* number of pending metadata requests */ ++ int meta_pending; ++ /* fifo list of requests in sort_list */ ++ struct list_head fifo; ++ ++ /* entity representing this queue in the scheduler */ ++ struct bfq_entity entity; ++ ++ /* maximum budget allowed from the feedback mechanism */ ++ int max_budget; ++ /* budget expiration (in jiffies) */ ++ unsigned long budget_timeout; ++ ++ /* number of requests on the dispatch list or inside driver */ ++ int dispatched; ++ ++ unsigned int flags; /* status flags.*/ ++ ++ /* node for active/idle bfqq list inside parent bfqd */ ++ struct list_head bfqq_list; ++ ++ /* bit vector: a 1 for each seeky requests in history */ ++ u32 seek_history; ++ ++ /* node for the device's burst list */ ++ struct hlist_node burst_list_node; ++ ++ /* position of the last request enqueued */ ++ sector_t last_request_pos; ++ ++ /* Number of consecutive pairs of request completion and ++ * arrival, such that the queue becomes idle after the ++ * completion, but the next request arrives within an idle ++ * time slice; used only if the queue's IO_bound flag has been ++ * cleared. ++ */ ++ unsigned int requests_within_timer; ++ ++ /* pid of the process owning the queue, used for logging purposes */ ++ pid_t pid; ++ ++ /* ++ * Pointer to the bfq_io_cq owning the bfq_queue, set to %NULL ++ * if the queue is shared. ++ */ ++ struct bfq_io_cq *bic; ++ ++ /* current maximum weight-raising time for this queue */ ++ unsigned long wr_cur_max_time; ++ /* ++ * Minimum time instant such that, only if a new request is ++ * enqueued after this time instant in an idle @bfq_queue with ++ * no outstanding requests, then the task associated with the ++ * queue it is deemed as soft real-time (see the comments on ++ * the function bfq_bfqq_softrt_next_start()) ++ */ ++ unsigned long soft_rt_next_start; ++ /* ++ * Start time of the current weight-raising period if ++ * the @bfq-queue is being weight-raised, otherwise ++ * finish time of the last weight-raising period. ++ */ ++ unsigned long last_wr_start_finish; ++ /* factor by which the weight of this queue is multiplied */ ++ unsigned int wr_coeff; ++ /* ++ * Time of the last transition of the @bfq_queue from idle to ++ * backlogged. ++ */ ++ unsigned long last_idle_bklogged; ++ /* ++ * Cumulative service received from the @bfq_queue since the ++ * last transition from idle to backlogged. ++ */ ++ unsigned long service_from_backlogged; ++ /* ++ * Value of wr start time when switching to soft rt ++ */ ++ unsigned long wr_start_at_switch_to_srt; ++ ++ unsigned long split_time; /* time of last split */ ++}; ++ ++/** ++ * struct bfq_ttime - per process thinktime stats. ++ */ ++struct bfq_ttime { ++ u64 last_end_request; /* completion time of last request */ ++ ++ u64 ttime_total; /* total process thinktime */ ++ unsigned long ttime_samples; /* number of thinktime samples */ ++ u64 ttime_mean; /* average process thinktime */ ++ ++}; ++ ++/** ++ * struct bfq_io_cq - per (request_queue, io_context) structure. ++ */ ++struct bfq_io_cq { ++ /* associated io_cq structure */ ++ struct io_cq icq; /* must be the first member */ ++ /* array of two process queues, the sync and the async */ ++ struct bfq_queue *bfqq[2]; ++ /* associated @bfq_ttime struct */ ++ struct bfq_ttime ttime; ++ /* per (request_queue, blkcg) ioprio */ ++ int ioprio; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ uint64_t blkcg_serial_nr; /* the current blkcg serial */ ++#endif ++ ++ /* ++ * Snapshot of the idle window before merging; taken to ++ * remember this value while the queue is merged, so as to be ++ * able to restore it in case of split. ++ */ ++ bool saved_idle_window; ++ /* ++ * Same purpose as the previous two fields for the I/O bound ++ * classification of a queue. ++ */ ++ bool saved_IO_bound; ++ ++ /* ++ * Same purpose as the previous fields for the value of the ++ * field keeping the queue's belonging to a large burst ++ */ ++ bool saved_in_large_burst; ++ /* ++ * True if the queue belonged to a burst list before its merge ++ * with another cooperating queue. ++ */ ++ bool was_in_burst_list; ++ ++ /* ++ * Similar to previous fields: save wr information. ++ */ ++ unsigned long saved_wr_coeff; ++ unsigned long saved_last_wr_start_finish; ++ unsigned long saved_wr_start_at_switch_to_srt; ++}; ++ ++enum bfq_device_speed { ++ BFQ_BFQD_FAST, ++ BFQ_BFQD_SLOW, ++}; ++ ++/** ++ * struct bfq_data - per-device data structure. ++ * ++ * All the fields are protected by the @queue lock. ++ */ ++struct bfq_data { ++ /* request queue for the device */ ++ struct request_queue *queue; ++ ++ /* root bfq_group for the device */ ++ struct bfq_group *root_group; ++ ++ /* ++ * rbtree of weight counters of @bfq_queues, sorted by ++ * weight. Used to keep track of whether all @bfq_queues have ++ * the same weight. The tree contains one counter for each ++ * distinct weight associated to some active and not ++ * weight-raised @bfq_queue (see the comments to the functions ++ * bfq_weights_tree_[add|remove] for further details). ++ */ ++ struct rb_root queue_weights_tree; ++ /* ++ * rbtree of non-queue @bfq_entity weight counters, sorted by ++ * weight. Used to keep track of whether all @bfq_groups have ++ * the same weight. The tree contains one counter for each ++ * distinct weight associated to some active @bfq_group (see ++ * the comments to the functions bfq_weights_tree_[add|remove] ++ * for further details). ++ */ ++ struct rb_root group_weights_tree; ++ ++ /* ++ * Number of bfq_queues containing requests (including the ++ * queue in service, even if it is idling). ++ */ ++ int busy_queues; ++ /* number of weight-raised busy @bfq_queues */ ++ int wr_busy_queues; ++ /* number of queued requests */ ++ int queued; ++ /* number of requests dispatched and waiting for completion */ ++ int rq_in_driver; ++ ++ /* ++ * Maximum number of requests in driver in the last ++ * @hw_tag_samples completed requests. ++ */ ++ int max_rq_in_driver; ++ /* number of samples used to calculate hw_tag */ ++ int hw_tag_samples; ++ /* flag set to one if the driver is showing a queueing behavior */ ++ int hw_tag; ++ ++ /* number of budgets assigned */ ++ int budgets_assigned; ++ ++ /* ++ * Timer set when idling (waiting) for the next request from ++ * the queue in service. ++ */ ++ struct hrtimer idle_slice_timer; ++ /* delayed work to restart dispatching on the request queue */ ++ struct work_struct unplug_work; ++ ++ /* bfq_queue in service */ ++ struct bfq_queue *in_service_queue; ++ /* bfq_io_cq (bic) associated with the @in_service_queue */ ++ struct bfq_io_cq *in_service_bic; ++ ++ /* on-disk position of the last served request */ ++ sector_t last_position; ++ ++ /* time of last request completion (ns) */ ++ u64 last_completion; ++ ++ /* time of first rq dispatch in current observation interval (ns) */ ++ u64 first_dispatch; ++ /* time of last rq dispatch in current observation interval (ns) */ ++ u64 last_dispatch; ++ ++ /* beginning of the last budget */ ++ ktime_t last_budget_start; ++ /* beginning of the last idle slice */ ++ ktime_t last_idling_start; ++ ++ /* number of samples in current observation interval */ ++ int peak_rate_samples; ++ /* num of samples of seq dispatches in current observation interval */ ++ u32 sequential_samples; ++ /* total num of sectors transferred in current observation interval */ ++ u64 tot_sectors_dispatched; ++ /* max rq size seen during current observation interval (sectors) */ ++ u32 last_rq_max_size; ++ /* time elapsed from first dispatch in current observ. interval (us) */ ++ u64 delta_from_first; ++ /* current estimate of device peak rate */ ++ u32 peak_rate; ++ ++ /* maximum budget allotted to a bfq_queue before rescheduling */ ++ int bfq_max_budget; ++ ++ /* list of all the bfq_queues active on the device */ ++ struct list_head active_list; ++ /* list of all the bfq_queues idle on the device */ ++ struct list_head idle_list; ++ ++ /* ++ * Timeout for async/sync requests; when it fires, requests ++ * are served in fifo order. ++ */ ++ u64 bfq_fifo_expire[2]; ++ /* weight of backward seeks wrt forward ones */ ++ unsigned int bfq_back_penalty; ++ /* maximum allowed backward seek */ ++ unsigned int bfq_back_max; ++ /* maximum idling time */ ++ u32 bfq_slice_idle; ++ /* last time CLASS_IDLE was served */ ++ u64 bfq_class_idle_last_service; ++ ++ /* user-configured max budget value (0 for auto-tuning) */ ++ int bfq_user_max_budget; ++ /* ++ * Timeout for bfq_queues to consume their budget; used to ++ * prevent seeky queues from imposing long latencies to ++ * sequential or quasi-sequential ones (this also implies that ++ * seeky queues cannot receive guarantees in the service ++ * domain; after a timeout they are charged for the time they ++ * have been in service, to preserve fairness among them, but ++ * without service-domain guarantees). ++ */ ++ unsigned int bfq_timeout; ++ ++ /* ++ * Number of consecutive requests that must be issued within ++ * the idle time slice to set again idling to a queue which ++ * was marked as non-I/O-bound (see the definition of the ++ * IO_bound flag for further details). ++ */ ++ unsigned int bfq_requests_within_timer; ++ ++ /* ++ * Force device idling whenever needed to provide accurate ++ * service guarantees, without caring about throughput ++ * issues. CAVEAT: this may even increase latencies, in case ++ * of useless idling for processes that did stop doing I/O. ++ */ ++ bool strict_guarantees; ++ ++ /* ++ * Last time at which a queue entered the current burst of ++ * queues being activated shortly after each other; for more ++ * details about this and the following parameters related to ++ * a burst of activations, see the comments on the function ++ * bfq_handle_burst. ++ */ ++ unsigned long last_ins_in_burst; ++ /* ++ * Reference time interval used to decide whether a queue has ++ * been activated shortly after @last_ins_in_burst. ++ */ ++ unsigned long bfq_burst_interval; ++ /* number of queues in the current burst of queue activations */ ++ int burst_size; ++ ++ /* common parent entity for the queues in the burst */ ++ struct bfq_entity *burst_parent_entity; ++ /* Maximum burst size above which the current queue-activation ++ * burst is deemed as 'large'. ++ */ ++ unsigned long bfq_large_burst_thresh; ++ /* true if a large queue-activation burst is in progress */ ++ bool large_burst; ++ /* ++ * Head of the burst list (as for the above fields, more ++ * details in the comments on the function bfq_handle_burst). ++ */ ++ struct hlist_head burst_list; ++ ++ /* if set to true, low-latency heuristics are enabled */ ++ bool low_latency; ++ /* ++ * Maximum factor by which the weight of a weight-raised queue ++ * is multiplied. ++ */ ++ unsigned int bfq_wr_coeff; ++ /* maximum duration of a weight-raising period (jiffies) */ ++ unsigned int bfq_wr_max_time; ++ ++ /* Maximum weight-raising duration for soft real-time processes */ ++ unsigned int bfq_wr_rt_max_time; ++ /* ++ * Minimum idle period after which weight-raising may be ++ * reactivated for a queue (in jiffies). ++ */ ++ unsigned int bfq_wr_min_idle_time; ++ /* ++ * Minimum period between request arrivals after which ++ * weight-raising may be reactivated for an already busy async ++ * queue (in jiffies). ++ */ ++ unsigned long bfq_wr_min_inter_arr_async; ++ ++ /* Max service-rate for a soft real-time queue, in sectors/sec */ ++ unsigned int bfq_wr_max_softrt_rate; ++ /* ++ * Cached value of the product R*T, used for computing the ++ * maximum duration of weight raising automatically. ++ */ ++ u64 RT_prod; ++ /* device-speed class for the low-latency heuristic */ ++ enum bfq_device_speed device_speed; ++ ++ /* fallback dummy bfqq for extreme OOM conditions */ ++ struct bfq_queue oom_bfqq; ++}; ++ ++enum bfqq_state_flags { ++ BFQ_BFQQ_FLAG_just_created = 0, /* queue just allocated */ ++ BFQ_BFQQ_FLAG_busy, /* has requests or is in service */ ++ BFQ_BFQQ_FLAG_wait_request, /* waiting for a request */ ++ BFQ_BFQQ_FLAG_non_blocking_wait_rq, /* ++ * waiting for a request ++ * without idling the device ++ */ ++ BFQ_BFQQ_FLAG_must_alloc, /* must be allowed rq alloc */ ++ BFQ_BFQQ_FLAG_fifo_expire, /* FIFO checked in this slice */ ++ BFQ_BFQQ_FLAG_idle_window, /* slice idling enabled */ ++ BFQ_BFQQ_FLAG_sync, /* synchronous queue */ ++ BFQ_BFQQ_FLAG_IO_bound, /* ++ * bfqq has timed-out at least once ++ * having consumed at most 2/10 of ++ * its budget ++ */ ++ BFQ_BFQQ_FLAG_in_large_burst, /* ++ * bfqq activated in a large burst, ++ * see comments to bfq_handle_burst. ++ */ ++ BFQ_BFQQ_FLAG_softrt_update, /* ++ * may need softrt-next-start ++ * update ++ */ ++ BFQ_BFQQ_FLAG_coop, /* bfqq is shared */ ++ BFQ_BFQQ_FLAG_split_coop /* shared bfqq will be split */ ++}; ++ ++#define BFQ_BFQQ_FNS(name) \ ++static void bfq_mark_bfqq_##name(struct bfq_queue *bfqq) \ ++{ \ ++ (bfqq)->flags |= (1 << BFQ_BFQQ_FLAG_##name); \ ++} \ ++static void bfq_clear_bfqq_##name(struct bfq_queue *bfqq) \ ++{ \ ++ (bfqq)->flags &= ~(1 << BFQ_BFQQ_FLAG_##name); \ ++} \ ++static int bfq_bfqq_##name(const struct bfq_queue *bfqq) \ ++{ \ ++ return ((bfqq)->flags & (1 << BFQ_BFQQ_FLAG_##name)) != 0; \ ++} ++ ++BFQ_BFQQ_FNS(just_created); ++BFQ_BFQQ_FNS(busy); ++BFQ_BFQQ_FNS(wait_request); ++BFQ_BFQQ_FNS(non_blocking_wait_rq); ++BFQ_BFQQ_FNS(must_alloc); ++BFQ_BFQQ_FNS(fifo_expire); ++BFQ_BFQQ_FNS(idle_window); ++BFQ_BFQQ_FNS(sync); ++BFQ_BFQQ_FNS(IO_bound); ++BFQ_BFQQ_FNS(in_large_burst); ++BFQ_BFQQ_FNS(coop); ++BFQ_BFQQ_FNS(split_coop); ++BFQ_BFQQ_FNS(softrt_update); ++#undef BFQ_BFQQ_FNS ++ ++/* Logging facilities. */ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static struct bfq_group *bfqq_group(struct bfq_queue *bfqq); ++static struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg); ++ ++#define bfq_log_bfqq(bfqd, bfqq, fmt, args...) do { \ ++ char __pbuf[128]; \ ++ \ ++ assert_spin_locked((bfqd)->queue->queue_lock); \ ++ blkg_path(bfqg_to_blkg(bfqq_group(bfqq)), __pbuf, sizeof(__pbuf)); \ ++ blk_add_trace_msg((bfqd)->queue, "bfq%d%c %s " fmt, \ ++ (bfqq)->pid, \ ++ bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ ++ __pbuf, ##args); \ ++} while (0) ++ ++#define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do { \ ++ char __pbuf[128]; \ ++ \ ++ blkg_path(bfqg_to_blkg(bfqg), __pbuf, sizeof(__pbuf)); \ ++ blk_add_trace_msg((bfqd)->queue, "%s " fmt, __pbuf, ##args); \ ++} while (0) ++ ++#else /* CONFIG_BFQ_GROUP_IOSCHED */ ++ ++#define bfq_log_bfqq(bfqd, bfqq, fmt, args...) \ ++ blk_add_trace_msg((bfqd)->queue, "bfq%d%c " fmt, (bfqq)->pid, \ ++ bfq_bfqq_sync((bfqq)) ? 'S' : 'A', \ ++ ##args) ++#define bfq_log_bfqg(bfqd, bfqg, fmt, args...) do {} while (0) ++ ++#endif /* CONFIG_BFQ_GROUP_IOSCHED */ ++ ++#define bfq_log(bfqd, fmt, args...) \ ++ blk_add_trace_msg((bfqd)->queue, "bfq " fmt, ##args) ++ ++/* Expiration reasons. */ ++enum bfqq_expiration { ++ BFQ_BFQQ_TOO_IDLE = 0, /* ++ * queue has been idling for ++ * too long ++ */ ++ BFQ_BFQQ_BUDGET_TIMEOUT, /* budget took too long to be used */ ++ BFQ_BFQQ_BUDGET_EXHAUSTED, /* budget consumed */ ++ BFQ_BFQQ_NO_MORE_REQUESTS, /* the queue has no more requests */ ++ BFQ_BFQQ_PREEMPTED /* preemption in progress */ ++}; ++ ++ ++struct bfqg_stats { ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ /* number of ios merged */ ++ struct blkg_rwstat merged; ++ /* total time spent on device in ns, may not be accurate w/ queueing */ ++ struct blkg_rwstat service_time; ++ /* total time spent waiting in scheduler queue in ns */ ++ struct blkg_rwstat wait_time; ++ /* number of IOs queued up */ ++ struct blkg_rwstat queued; ++ /* total disk time and nr sectors dispatched by this group */ ++ struct blkg_stat time; ++ /* sum of number of ios queued across all samples */ ++ struct blkg_stat avg_queue_size_sum; ++ /* count of samples taken for average */ ++ struct blkg_stat avg_queue_size_samples; ++ /* how many times this group has been removed from service tree */ ++ struct blkg_stat dequeue; ++ /* total time spent waiting for it to be assigned a timeslice. */ ++ struct blkg_stat group_wait_time; ++ /* time spent idling for this blkcg_gq */ ++ struct blkg_stat idle_time; ++ /* total time with empty current active q with other requests queued */ ++ struct blkg_stat empty_time; ++ /* fields after this shouldn't be cleared on stat reset */ ++ uint64_t start_group_wait_time; ++ uint64_t start_idle_time; ++ uint64_t start_empty_time; ++ uint16_t flags; ++#endif ++}; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++/* ++ * struct bfq_group_data - per-blkcg storage for the blkio subsystem. ++ * ++ * @ps: @blkcg_policy_storage that this structure inherits ++ * @weight: weight of the bfq_group ++ */ ++struct bfq_group_data { ++ /* must be the first member */ ++ struct blkcg_policy_data pd; ++ ++ unsigned int weight; ++}; ++ ++/** ++ * struct bfq_group - per (device, cgroup) data structure. ++ * @entity: schedulable entity to insert into the parent group sched_data. ++ * @sched_data: own sched_data, to contain child entities (they may be ++ * both bfq_queues and bfq_groups). ++ * @bfqd: the bfq_data for the device this group acts upon. ++ * @async_bfqq: array of async queues for all the tasks belonging to ++ * the group, one queue per ioprio value per ioprio_class, ++ * except for the idle class that has only one queue. ++ * @async_idle_bfqq: async queue for the idle class (ioprio is ignored). ++ * @my_entity: pointer to @entity, %NULL for the toplevel group; used ++ * to avoid too many special cases during group creation/ ++ * migration. ++ * @active_entities: number of active entities belonging to the group; ++ * unused for the root group. Used to know whether there ++ * are groups with more than one active @bfq_entity ++ * (see the comments to the function ++ * bfq_bfqq_may_idle()). ++ * @rq_pos_tree: rbtree sorted by next_request position, used when ++ * determining if two or more queues have interleaving ++ * requests (see bfq_find_close_cooperator()). ++ * ++ * Each (device, cgroup) pair has its own bfq_group, i.e., for each cgroup ++ * there is a set of bfq_groups, each one collecting the lower-level ++ * entities belonging to the group that are acting on the same device. ++ * ++ * Locking works as follows: ++ * o @bfqd is protected by the queue lock, RCU is used to access it ++ * from the readers. ++ * o All the other fields are protected by the @bfqd queue lock. ++ */ ++struct bfq_group { ++ /* must be the first member */ ++ struct blkg_policy_data pd; ++ ++ struct bfq_entity entity; ++ struct bfq_sched_data sched_data; ++ ++ void *bfqd; ++ ++ struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR]; ++ struct bfq_queue *async_idle_bfqq; ++ ++ struct bfq_entity *my_entity; ++ ++ int active_entities; ++ ++ struct rb_root rq_pos_tree; ++ ++ struct bfqg_stats stats; ++}; ++ ++#else ++struct bfq_group { ++ struct bfq_sched_data sched_data; ++ ++ struct bfq_queue *async_bfqq[2][IOPRIO_BE_NR]; ++ struct bfq_queue *async_idle_bfqq; ++ ++ struct rb_root rq_pos_tree; ++}; ++#endif ++ ++static struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity); ++ ++static struct bfq_service_tree * ++bfq_entity_service_tree(struct bfq_entity *entity) ++{ ++ struct bfq_sched_data *sched_data = entity->sched_data; ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ unsigned int idx = bfqq ? bfqq->ioprio_class - 1 : ++ BFQ_DEFAULT_GRP_CLASS - 1; ++ ++ BUG_ON(idx >= BFQ_IOPRIO_CLASSES); ++ BUG_ON(sched_data == NULL); ++ ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "entity_service_tree %p %d", ++ sched_data->service_tree + idx, idx); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "entity_service_tree %p %d", ++ sched_data->service_tree + idx, idx); ++ } ++#endif ++ return sched_data->service_tree + idx; ++} ++ ++static struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync) ++{ ++ return bic->bfqq[is_sync]; ++} ++ ++static void bic_set_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq, ++ bool is_sync) ++{ ++ bic->bfqq[is_sync] = bfqq; ++} ++ ++static struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic) ++{ ++ return bic->icq.q->elevator->elevator_data; ++} ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ ++static struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *group_entity = bfqq->entity.parent; ++ ++ if (!group_entity) ++ group_entity = &bfqq->bfqd->root_group->entity; ++ ++ return container_of(group_entity, struct bfq_group, entity); ++} ++ ++#else ++ ++static struct bfq_group *bfq_bfqq_to_bfqg(struct bfq_queue *bfqq) ++{ ++ return bfqq->bfqd->root_group; ++} ++ ++#endif ++ ++static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio); ++static void bfq_put_queue(struct bfq_queue *bfqq); ++static void bfq_dispatch_insert(struct request_queue *q, struct request *rq); ++static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, ++ struct bio *bio, bool is_sync, ++ struct bfq_io_cq *bic); ++static void bfq_end_wr_async_queues(struct bfq_data *bfqd, ++ struct bfq_group *bfqg); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg); ++#endif ++static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq); ++ ++#endif /* _BFQ_H */ +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq-ioc.c linux-4.8-bfq/block/bfq-ioc.c +--- linux-4.8/block/bfq-ioc.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq-ioc.c 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,36 @@ ++/* ++ * BFQ: I/O context handling. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2010 Paolo Valente ++ */ ++ ++/** ++ * icq_to_bic - convert iocontext queue structure to bfq_io_cq. ++ * @icq: the iocontext queue. ++ */ ++static struct bfq_io_cq *icq_to_bic(struct io_cq *icq) ++{ ++ /* bic->icq is the first member, %NULL will convert to %NULL */ ++ return container_of(icq, struct bfq_io_cq, icq); ++} ++ ++/** ++ * bfq_bic_lookup - search into @ioc a bic associated to @bfqd. ++ * @bfqd: the lookup key. ++ * @ioc: the io_context of the process doing I/O. ++ * ++ * Queue lock must be held. ++ */ ++static struct bfq_io_cq *bfq_bic_lookup(struct bfq_data *bfqd, ++ struct io_context *ioc) ++{ ++ if (ioc) ++ return icq_to_bic(ioc_lookup_icq(ioc, bfqd->queue)); ++ return NULL; ++} +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq-iosched.c linux-4.8-bfq/block/bfq-iosched.c +--- linux-4.8/block/bfq-iosched.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq-iosched.c 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,5288 @@ ++/* ++ * Budget Fair Queueing (BFQ) disk scheduler. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2015 Paolo Valente ++ * ++ * Copyright (C) 2016 Paolo Valente ++ * ++ * Licensed under the GPL-2 as detailed in the accompanying COPYING.BFQ ++ * file. ++ * ++ * BFQ is a proportional-share storage-I/O scheduling algorithm based ++ * on the slice-by-slice service scheme of CFQ. But BFQ assigns ++ * budgets, measured in number of sectors, to processes instead of ++ * time slices. The device is not granted to the in-service process ++ * for a given time slice, but until it has exhausted its assigned ++ * budget. This change from the time to the service domain enables BFQ ++ * to distribute the device throughput among processes as desired, ++ * without any distortion due to throughput fluctuations, or to device ++ * internal queueing. BFQ uses an ad hoc internal scheduler, called ++ * B-WF2Q+, to schedule processes according to their budgets. More ++ * precisely, BFQ schedules queues associated with processes. Thanks to ++ * the accurate policy of B-WF2Q+, BFQ can afford to assign high ++ * budgets to I/O-bound processes issuing sequential requests (to ++ * boost the throughput), and yet guarantee a low latency to ++ * interactive and soft real-time applications. ++ * ++ * BFQ is described in [1], where also a reference to the initial, more ++ * theoretical paper on BFQ can be found. The interested reader can find ++ * in the latter paper full details on the main algorithm, as well as ++ * formulas of the guarantees and formal proofs of all the properties. ++ * With respect to the version of BFQ presented in these papers, this ++ * implementation adds a few more heuristics, such as the one that ++ * guarantees a low latency to soft real-time applications, and a ++ * hierarchical extension based on H-WF2Q+. ++ * ++ * B-WF2Q+ is based on WF2Q+, that is described in [2], together with ++ * H-WF2Q+, while the augmented tree used to implement B-WF2Q+ with O(log N) ++ * complexity derives from the one introduced with EEVDF in [3]. ++ * ++ * [1] P. Valente and M. Andreolini, ``Improving Application Responsiveness ++ * with the BFQ Disk I/O Scheduler'', ++ * Proceedings of the 5th Annual International Systems and Storage ++ * Conference (SYSTOR '12), June 2012. ++ * ++ * http://algogroup.unimo.it/people/paolo/disk_sched/bf1-v1-suite-results.pdf ++ * ++ * [2] Jon C.R. Bennett and H. Zhang, ``Hierarchical Packet Fair Queueing ++ * Algorithms,'' IEEE/ACM Transactions on Networking, 5(5):675-689, ++ * Oct 1997. ++ * ++ * http://www.cs.cmu.edu/~hzhang/papers/TON-97-Oct.ps.gz ++ * ++ * [3] I. Stoica and H. Abdel-Wahab, ``Earliest Eligible Virtual Deadline ++ * First: A Flexible and Accurate Mechanism for Proportional Share ++ * Resource Allocation,'' technical report. ++ * ++ * http://www.cs.berkeley.edu/~istoica/papers/eevdf-tr-95.pdf ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "bfq.h" ++#include "blk.h" ++ ++/* Expiration time of sync (0) and async (1) requests, in ns. */ ++static const u64 bfq_fifo_expire[2] = { NSEC_PER_SEC / 4, NSEC_PER_SEC / 8 }; ++ ++/* Maximum backwards seek, in KiB. */ ++static const int bfq_back_max = 16 * 1024; ++ ++/* Penalty of a backwards seek, in number of sectors. */ ++static const int bfq_back_penalty = 2; ++ ++/* Idling period duration, in ns. */ ++static u32 bfq_slice_idle = NSEC_PER_SEC / 125; ++ ++/* Minimum number of assigned budgets for which stats are safe to compute. */ ++static const int bfq_stats_min_budgets = 194; ++ ++/* Default maximum budget values, in sectors and number of requests. */ ++static const int bfq_default_max_budget = 16 * 1024; ++ ++/* ++ * Async to sync throughput distribution is controlled as follows: ++ * when an async request is served, the entity is charged the number ++ * of sectors of the request, multiplied by the factor below ++ */ ++static const int bfq_async_charge_factor = 10; ++ ++/* Default timeout values, in jiffies, approximating CFQ defaults. */ ++static const int bfq_timeout = HZ / 8; ++ ++struct kmem_cache *bfq_pool; ++ ++/* Below this threshold (in ns), we consider thinktime immediate. */ ++#define BFQ_MIN_TT (2 * NSEC_PER_MSEC) ++ ++/* hw_tag detection: parallel requests threshold and min samples needed. */ ++#define BFQ_HW_QUEUE_THRESHOLD 4 ++#define BFQ_HW_QUEUE_SAMPLES 32 ++ ++#define BFQQ_SEEK_THR (sector_t)(8 * 100) ++#define BFQQ_CLOSE_THR (sector_t)(8 * 1024) ++#define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 32/8) ++ ++/* Min number of samples required to perform peak-rate update */ ++#define BFQ_RATE_MIN_SAMPLES 32 ++/* Min observation time interval required to perform a peak-rate update (ns) */ ++#define BFQ_RATE_MIN_INTERVAL 300*NSEC_PER_MSEC ++/* Target observation time interval for a peak-rate update (ns) */ ++#define BFQ_RATE_REF_INTERVAL NSEC_PER_SEC ++ ++/* Shift used for peak rate fixed precision calculations. */ ++#define BFQ_RATE_SHIFT 16 ++ ++/* ++ * By default, BFQ computes the duration of the weight raising for ++ * interactive applications automatically, using the following formula: ++ * duration = (R / r) * T, where r is the peak rate of the device, and ++ * R and T are two reference parameters. ++ * In particular, R is the peak rate of the reference device (see below), ++ * and T is a reference time: given the systems that are likely to be ++ * installed on the reference device according to its speed class, T is ++ * about the maximum time needed, under BFQ and while reading two files in ++ * parallel, to load typical large applications on these systems. ++ * In practice, the slower/faster the device at hand is, the more/less it ++ * takes to load applications with respect to the reference device. ++ * Accordingly, the longer/shorter BFQ grants weight raising to interactive ++ * applications. ++ * ++ * BFQ uses four different reference pairs (R, T), depending on: ++ * . whether the device is rotational or non-rotational; ++ * . whether the device is slow, such as old or portable HDDs, as well as ++ * SD cards, or fast, such as newer HDDs and SSDs. ++ * ++ * The device's speed class is dynamically (re)detected in ++ * bfq_update_peak_rate() every time the estimated peak rate is updated. ++ * ++ * In the following definitions, R_slow[0]/R_fast[0] and ++ * T_slow[0]/T_fast[0] are the reference values for a slow/fast ++ * rotational device, whereas R_slow[1]/R_fast[1] and ++ * T_slow[1]/T_fast[1] are the reference values for a slow/fast ++ * non-rotational device. Finally, device_speed_thresh are the ++ * thresholds used to switch between speed classes. The reference ++ * rates are not the actual peak rates of the devices used as a ++ * reference, but slightly lower values. The reason for using these ++ * slightly lower values is that the peak-rate estimator tends to ++ * yield slightly lower values than the actual peak rate (it can yield ++ * the actual peak rate only if there is only one process doing I/O, ++ * and the process does sequential I/O). ++ * ++ * Both the reference peak rates and the thresholds are measured in ++ * sectors/usec, left-shifted by BFQ_RATE_SHIFT. ++ */ ++static int R_slow[2] = {1000, 10700}; ++static int R_fast[2] = {14000, 33000}; ++/* ++ * To improve readability, a conversion function is used to initialize the ++ * following arrays, which entails that they can be initialized only in a ++ * function. ++ */ ++static int T_slow[2]; ++static int T_fast[2]; ++static int device_speed_thresh[2]; ++ ++#define BFQ_SERVICE_TREE_INIT ((struct bfq_service_tree) \ ++ { RB_ROOT, RB_ROOT, NULL, NULL, 0, 0 }) ++ ++#define RQ_BIC(rq) ((struct bfq_io_cq *) (rq)->elv.priv[0]) ++#define RQ_BFQQ(rq) ((rq)->elv.priv[1]) ++ ++static void bfq_schedule_dispatch(struct bfq_data *bfqd); ++ ++#include "bfq-ioc.c" ++#include "bfq-sched.c" ++#include "bfq-cgroup.c" ++ ++#define bfq_class_idle(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_IDLE) ++#define bfq_class_rt(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_RT) ++ ++#define bfq_sample_valid(samples) ((samples) > 80) ++ ++/* ++ * We regard a request as SYNC, if either it's a read or has the SYNC bit ++ * set (in which case it could also be a direct WRITE). ++ */ ++static int bfq_bio_sync(struct bio *bio) ++{ ++ return bio_data_dir(bio) == READ || (bio->bi_opf & REQ_SYNC); ++} ++ ++/* ++ * Scheduler run of queue, if there are requests pending and no one in the ++ * driver that will restart queueing. ++ */ ++static void bfq_schedule_dispatch(struct bfq_data *bfqd) ++{ ++ if (bfqd->queued != 0) { ++ bfq_log(bfqd, "schedule dispatch"); ++ kblockd_schedule_work(&bfqd->unplug_work); ++ } ++} ++ ++/* ++ * Lifted from AS - choose which of rq1 and rq2 that is best served now. ++ * We choose the request that is closesr to the head right now. Distance ++ * behind the head is penalized and only allowed to a certain extent. ++ */ ++static struct request *bfq_choose_req(struct bfq_data *bfqd, ++ struct request *rq1, ++ struct request *rq2, ++ sector_t last) ++{ ++ sector_t s1, s2, d1 = 0, d2 = 0; ++ unsigned long back_max; ++#define BFQ_RQ1_WRAP 0x01 /* request 1 wraps */ ++#define BFQ_RQ2_WRAP 0x02 /* request 2 wraps */ ++ unsigned int wrap = 0; /* bit mask: requests behind the disk head? */ ++ ++ if (!rq1 || rq1 == rq2) ++ return rq2; ++ if (!rq2) ++ return rq1; ++ ++ if (rq_is_sync(rq1) && !rq_is_sync(rq2)) ++ return rq1; ++ else if (rq_is_sync(rq2) && !rq_is_sync(rq1)) ++ return rq2; ++ if ((rq1->cmd_flags & REQ_META) && !(rq2->cmd_flags & REQ_META)) ++ return rq1; ++ else if ((rq2->cmd_flags & REQ_META) && !(rq1->cmd_flags & REQ_META)) ++ return rq2; ++ ++ s1 = blk_rq_pos(rq1); ++ s2 = blk_rq_pos(rq2); ++ ++ /* ++ * By definition, 1KiB is 2 sectors. ++ */ ++ back_max = bfqd->bfq_back_max * 2; ++ ++ /* ++ * Strict one way elevator _except_ in the case where we allow ++ * short backward seeks which are biased as twice the cost of a ++ * similar forward seek. ++ */ ++ if (s1 >= last) ++ d1 = s1 - last; ++ else if (s1 + back_max >= last) ++ d1 = (last - s1) * bfqd->bfq_back_penalty; ++ else ++ wrap |= BFQ_RQ1_WRAP; ++ ++ if (s2 >= last) ++ d2 = s2 - last; ++ else if (s2 + back_max >= last) ++ d2 = (last - s2) * bfqd->bfq_back_penalty; ++ else ++ wrap |= BFQ_RQ2_WRAP; ++ ++ /* Found required data */ ++ ++ /* ++ * By doing switch() on the bit mask "wrap" we avoid having to ++ * check two variables for all permutations: --> faster! ++ */ ++ switch (wrap) { ++ case 0: /* common case for CFQ: rq1 and rq2 not wrapped */ ++ if (d1 < d2) ++ return rq1; ++ else if (d2 < d1) ++ return rq2; ++ ++ if (s1 >= s2) ++ return rq1; ++ else ++ return rq2; ++ ++ case BFQ_RQ2_WRAP: ++ return rq1; ++ case BFQ_RQ1_WRAP: ++ return rq2; ++ case (BFQ_RQ1_WRAP|BFQ_RQ2_WRAP): /* both rqs wrapped */ ++ default: ++ /* ++ * Since both rqs are wrapped, ++ * start with the one that's further behind head ++ * (--> only *one* back seek required), ++ * since back seek takes more time than forward. ++ */ ++ if (s1 <= s2) ++ return rq1; ++ else ++ return rq2; ++ } ++} ++ ++static struct bfq_queue * ++bfq_rq_pos_tree_lookup(struct bfq_data *bfqd, struct rb_root *root, ++ sector_t sector, struct rb_node **ret_parent, ++ struct rb_node ***rb_link) ++{ ++ struct rb_node **p, *parent; ++ struct bfq_queue *bfqq = NULL; ++ ++ parent = NULL; ++ p = &root->rb_node; ++ while (*p) { ++ struct rb_node **n; ++ ++ parent = *p; ++ bfqq = rb_entry(parent, struct bfq_queue, pos_node); ++ ++ /* ++ * Sort strictly based on sector. Smallest to the left, ++ * largest to the right. ++ */ ++ if (sector > blk_rq_pos(bfqq->next_rq)) ++ n = &(*p)->rb_right; ++ else if (sector < blk_rq_pos(bfqq->next_rq)) ++ n = &(*p)->rb_left; ++ else ++ break; ++ p = n; ++ bfqq = NULL; ++ } ++ ++ *ret_parent = parent; ++ if (rb_link) ++ *rb_link = p; ++ ++ bfq_log(bfqd, "rq_pos_tree_lookup %llu: returning %d", ++ (unsigned long long) sector, ++ bfqq ? bfqq->pid : 0); ++ ++ return bfqq; ++} ++ ++static void bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ struct rb_node **p, *parent; ++ struct bfq_queue *__bfqq; ++ ++ if (bfqq->pos_root) { ++ rb_erase(&bfqq->pos_node, bfqq->pos_root); ++ bfqq->pos_root = NULL; ++ } ++ ++ if (bfq_class_idle(bfqq)) ++ return; ++ if (!bfqq->next_rq) ++ return; ++ ++ bfqq->pos_root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; ++ __bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root, ++ blk_rq_pos(bfqq->next_rq), &parent, &p); ++ if (!__bfqq) { ++ rb_link_node(&bfqq->pos_node, parent, p); ++ rb_insert_color(&bfqq->pos_node, bfqq->pos_root); ++ } else ++ bfqq->pos_root = NULL; ++} ++ ++/* ++ * Tell whether there are active queues or groups with differentiated weights. ++ */ ++static bool bfq_differentiated_weights(struct bfq_data *bfqd) ++{ ++ /* ++ * For weights to differ, at least one of the trees must contain ++ * at least two nodes. ++ */ ++ return (!RB_EMPTY_ROOT(&bfqd->queue_weights_tree) && ++ (bfqd->queue_weights_tree.rb_node->rb_left || ++ bfqd->queue_weights_tree.rb_node->rb_right) ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ ) || ++ (!RB_EMPTY_ROOT(&bfqd->group_weights_tree) && ++ (bfqd->group_weights_tree.rb_node->rb_left || ++ bfqd->group_weights_tree.rb_node->rb_right) ++#endif ++ ); ++} ++ ++/* ++ * The following function returns true if every queue must receive the ++ * same share of the throughput (this condition is used when deciding ++ * whether idling may be disabled, see the comments in the function ++ * bfq_bfqq_may_idle()). ++ * ++ * Such a scenario occurs when: ++ * 1) all active queues have the same weight, ++ * 2) all active groups at the same level in the groups tree have the same ++ * weight, ++ * 3) all active groups at the same level in the groups tree have the same ++ * number of children. ++ * ++ * Unfortunately, keeping the necessary state for evaluating exactly the ++ * above symmetry conditions would be quite complex and time-consuming. ++ * Therefore this function evaluates, instead, the following stronger ++ * sub-conditions, for which it is much easier to maintain the needed ++ * state: ++ * 1) all active queues have the same weight, ++ * 2) all active groups have the same weight, ++ * 3) all active groups have at most one active child each. ++ * In particular, the last two conditions are always true if hierarchical ++ * support and the cgroups interface are not enabled, thus no state needs ++ * to be maintained in this case. ++ */ ++static bool bfq_symmetric_scenario(struct bfq_data *bfqd) ++{ ++ return !bfq_differentiated_weights(bfqd); ++} ++ ++/* ++ * If the weight-counter tree passed as input contains no counter for ++ * the weight of the input entity, then add that counter; otherwise just ++ * increment the existing counter. ++ * ++ * Note that weight-counter trees contain few nodes in mostly symmetric ++ * scenarios. For example, if all queues have the same weight, then the ++ * weight-counter tree for the queues may contain at most one node. ++ * This holds even if low_latency is on, because weight-raised queues ++ * are not inserted in the tree. ++ * In most scenarios, the rate at which nodes are created/destroyed ++ * should be low too. ++ */ ++static void bfq_weights_tree_add(struct bfq_data *bfqd, ++ struct bfq_entity *entity, ++ struct rb_root *root) ++{ ++ struct rb_node **new = &(root->rb_node), *parent = NULL; ++ ++ /* ++ * Do not insert if the entity is already associated with a ++ * counter, which happens if: ++ * 1) the entity is associated with a queue, ++ * 2) a request arrival has caused the queue to become both ++ * non-weight-raised, and hence change its weight, and ++ * backlogged; in this respect, each of the two events ++ * causes an invocation of this function, ++ * 3) this is the invocation of this function caused by the ++ * second event. This second invocation is actually useless, ++ * and we handle this fact by exiting immediately. More ++ * efficient or clearer solutions might possibly be adopted. ++ */ ++ if (entity->weight_counter) ++ return; ++ ++ while (*new) { ++ struct bfq_weight_counter *__counter = container_of(*new, ++ struct bfq_weight_counter, ++ weights_node); ++ parent = *new; ++ ++ if (entity->weight == __counter->weight) { ++ entity->weight_counter = __counter; ++ goto inc_counter; ++ } ++ if (entity->weight < __counter->weight) ++ new = &((*new)->rb_left); ++ else ++ new = &((*new)->rb_right); ++ } ++ ++ entity->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), ++ GFP_ATOMIC); ++ entity->weight_counter->weight = entity->weight; ++ rb_link_node(&entity->weight_counter->weights_node, parent, new); ++ rb_insert_color(&entity->weight_counter->weights_node, root); ++ ++inc_counter: ++ entity->weight_counter->num_active++; ++} ++ ++/* ++ * Decrement the weight counter associated with the entity, and, if the ++ * counter reaches 0, remove the counter from the tree. ++ * See the comments to the function bfq_weights_tree_add() for considerations ++ * about overhead. ++ */ ++static void bfq_weights_tree_remove(struct bfq_data *bfqd, ++ struct bfq_entity *entity, ++ struct rb_root *root) ++{ ++ if (!entity->weight_counter) ++ return; ++ ++ BUG_ON(RB_EMPTY_ROOT(root)); ++ BUG_ON(entity->weight_counter->weight != entity->weight); ++ ++ BUG_ON(!entity->weight_counter->num_active); ++ entity->weight_counter->num_active--; ++ if (entity->weight_counter->num_active > 0) ++ goto reset_entity_pointer; ++ ++ rb_erase(&entity->weight_counter->weights_node, root); ++ kfree(entity->weight_counter); ++ ++reset_entity_pointer: ++ entity->weight_counter = NULL; ++} ++ ++static struct request *bfq_find_next_rq(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ struct request *last) ++{ ++ struct rb_node *rbnext = rb_next(&last->rb_node); ++ struct rb_node *rbprev = rb_prev(&last->rb_node); ++ struct request *next = NULL, *prev = NULL; ++ ++ BUG_ON(RB_EMPTY_NODE(&last->rb_node)); ++ ++ if (rbprev) ++ prev = rb_entry_rq(rbprev); ++ ++ if (rbnext) ++ next = rb_entry_rq(rbnext); ++ else { ++ rbnext = rb_first(&bfqq->sort_list); ++ if (rbnext && rbnext != &last->rb_node) ++ next = rb_entry_rq(rbnext); ++ } ++ ++ return bfq_choose_req(bfqd, next, prev, blk_rq_pos(last)); ++} ++ ++/* see the definition of bfq_async_charge_factor for details */ ++static unsigned long bfq_serv_to_charge(struct request *rq, ++ struct bfq_queue *bfqq) ++{ ++ if (bfq_bfqq_sync(bfqq) || bfqq->wr_coeff > 1) ++ return blk_rq_sectors(rq); ++ ++ /* ++ * If there are no weight-raised queues, then amplify service ++ * by just the async charge factor; otherwise amplify service ++ * by twice the async charge factor, to further reduce latency ++ * for weight-raised queues. ++ */ ++ if (bfqq->bfqd->wr_busy_queues == 0) ++ return blk_rq_sectors(rq) * bfq_async_charge_factor; ++ ++ return blk_rq_sectors(rq) * 2 * bfq_async_charge_factor; ++} ++ ++/** ++ * bfq_updated_next_req - update the queue after a new next_rq selection. ++ * @bfqd: the device data the queue belongs to. ++ * @bfqq: the queue to update. ++ * ++ * If the first request of a queue changes we make sure that the queue ++ * has enough budget to serve at least its first request (if the ++ * request has grown). We do this because if the queue has not enough ++ * budget for its first request, it has to go through two dispatch ++ * rounds to actually get it dispatched. ++ */ ++static void bfq_updated_next_req(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ struct bfq_service_tree *st = bfq_entity_service_tree(entity); ++ struct request *next_rq = bfqq->next_rq; ++ unsigned long new_budget; ++ ++ if (!next_rq) ++ return; ++ ++ if (bfqq == bfqd->in_service_queue) ++ /* ++ * In order not to break guarantees, budgets cannot be ++ * changed after an entity has been selected. ++ */ ++ return; ++ ++ BUG_ON(entity->tree != &st->active); ++ BUG_ON(entity == entity->sched_data->in_service_entity); ++ ++ new_budget = max_t(unsigned long, bfqq->max_budget, ++ bfq_serv_to_charge(next_rq, bfqq)); ++ if (entity->budget != new_budget) { ++ entity->budget = new_budget; ++ bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", ++ new_budget); ++ bfq_activate_bfqq(bfqd, bfqq); ++ } ++} ++ ++static unsigned int bfq_wr_duration(struct bfq_data *bfqd) ++{ ++ u64 dur; ++ ++ if (bfqd->bfq_wr_max_time > 0) ++ return bfqd->bfq_wr_max_time; ++ ++ dur = bfqd->RT_prod; ++ do_div(dur, bfqd->peak_rate); ++ ++ /* ++ * Limit duration between 3 and 13 seconds. Tests show that ++ * higher values than 13 seconds often yield the opposite of ++ * the desired result, i.e., worsen responsiveness by letting ++ * non-interactive and non-soft-real-time applications ++ * preserve weight raising for a too long time interval. ++ * ++ * On the other end, lower values than 3 seconds make it ++ * difficult for most interactive tasks to complete their jobs ++ * before weight-raising finishes. ++ */ ++ if (dur > msecs_to_jiffies(13000)) ++ dur = msecs_to_jiffies(13000); ++ else if (dur < msecs_to_jiffies(3000)) ++ dur = msecs_to_jiffies(3000); ++ ++ return dur; ++} ++ ++static void ++bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_io_cq *bic) ++{ ++ if (bic->saved_idle_window) ++ bfq_mark_bfqq_idle_window(bfqq); ++ else ++ bfq_clear_bfqq_idle_window(bfqq); ++ ++ if (bic->saved_IO_bound) ++ bfq_mark_bfqq_IO_bound(bfqq); ++ else ++ bfq_clear_bfqq_IO_bound(bfqq); ++ ++ bfqq->wr_coeff = bic->saved_wr_coeff; ++ bfqq->wr_start_at_switch_to_srt = bic->saved_wr_start_at_switch_to_srt; ++ BUG_ON(time_is_after_jiffies(bfqq->wr_start_at_switch_to_srt)); ++ bfqq->last_wr_start_finish = bic->saved_last_wr_start_finish; ++ BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish)); ++ ++ if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) || ++ time_is_before_jiffies(bfqq->last_wr_start_finish + ++ bfqq->wr_cur_max_time))) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "resume state: switching off wr"); ++ ++ bfqq->wr_coeff = 1; ++ } ++ /* make sure weight will be updated, however we got here */ ++ bfqq->entity.prio_changed = 1; ++} ++ ++static int bfqq_process_refs(struct bfq_queue *bfqq) ++{ ++ int process_refs, io_refs; ++ ++ lockdep_assert_held(bfqq->bfqd->queue->queue_lock); ++ ++ io_refs = bfqq->allocated[READ] + bfqq->allocated[WRITE]; ++ process_refs = bfqq->ref - io_refs - bfqq->entity.on_st; ++ BUG_ON(process_refs < 0); ++ return process_refs; ++} ++ ++/* Empty burst list and add just bfqq (see comments to bfq_handle_burst) */ ++static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ struct bfq_queue *item; ++ struct hlist_node *n; ++ ++ hlist_for_each_entry_safe(item, n, &bfqd->burst_list, burst_list_node) ++ hlist_del_init(&item->burst_list_node); ++ hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); ++ bfqd->burst_size = 1; ++ bfqd->burst_parent_entity = bfqq->entity.parent; ++} ++ ++/* Add bfqq to the list of queues in current burst (see bfq_handle_burst) */ ++static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ /* Increment burst size to take into account also bfqq */ ++ bfqd->burst_size++; ++ ++ bfq_log_bfqq(bfqd, bfqq, "add_to_burst %d", bfqd->burst_size); ++ ++ BUG_ON(bfqd->burst_size > bfqd->bfq_large_burst_thresh); ++ ++ if (bfqd->burst_size == bfqd->bfq_large_burst_thresh) { ++ struct bfq_queue *pos, *bfqq_item; ++ struct hlist_node *n; ++ ++ /* ++ * Enough queues have been activated shortly after each ++ * other to consider this burst as large. ++ */ ++ bfqd->large_burst = true; ++ bfq_log_bfqq(bfqd, bfqq, "add_to_burst: large burst started"); ++ ++ /* ++ * We can now mark all queues in the burst list as ++ * belonging to a large burst. ++ */ ++ hlist_for_each_entry(bfqq_item, &bfqd->burst_list, ++ burst_list_node) { ++ bfq_mark_bfqq_in_large_burst(bfqq_item); ++ bfq_log_bfqq(bfqd, bfqq_item, "marked in large burst"); ++ } ++ bfq_mark_bfqq_in_large_burst(bfqq); ++ bfq_log_bfqq(bfqd, bfqq, "marked in large burst"); ++ ++ /* ++ * From now on, and until the current burst finishes, any ++ * new queue being activated shortly after the last queue ++ * was inserted in the burst can be immediately marked as ++ * belonging to a large burst. So the burst list is not ++ * needed any more. Remove it. ++ */ ++ hlist_for_each_entry_safe(pos, n, &bfqd->burst_list, ++ burst_list_node) ++ hlist_del_init(&pos->burst_list_node); ++ } else /* ++ * Burst not yet large: add bfqq to the burst list. Do ++ * not increment the ref counter for bfqq, because bfqq ++ * is removed from the burst list before freeing bfqq ++ * in put_queue. ++ */ ++ hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); ++} ++ ++/* ++ * If many queues belonging to the same group happen to be created ++ * shortly after each other, then the processes associated with these ++ * queues have typically a common goal. In particular, bursts of queue ++ * creations are usually caused by services or applications that spawn ++ * many parallel threads/processes. Examples are systemd during boot, ++ * or git grep. To help these processes get their job done as soon as ++ * possible, it is usually better to not grant either weight-raising ++ * or device idling to their queues. ++ * ++ * In this comment we describe, firstly, the reasons why this fact ++ * holds, and, secondly, the next function, which implements the main ++ * steps needed to properly mark these queues so that they can then be ++ * treated in a different way. ++ * ++ * The above services or applications benefit mostly from a high ++ * throughput: the quicker the requests of the activated queues are ++ * cumulatively served, the sooner the target job of these queues gets ++ * completed. As a consequence, weight-raising any of these queues, ++ * which also implies idling the device for it, is almost always ++ * counterproductive. In most cases it just lowers throughput. ++ * ++ * On the other hand, a burst of queue creations may be caused also by ++ * the start of an application that does not consist of a lot of ++ * parallel I/O-bound threads. In fact, with a complex application, ++ * several short processes may need to be executed to start-up the ++ * application. In this respect, to start an application as quickly as ++ * possible, the best thing to do is in any case to privilege the I/O ++ * related to the application with respect to all other ++ * I/O. Therefore, the best strategy to start as quickly as possible ++ * an application that causes a burst of queue creations is to ++ * weight-raise all the queues created during the burst. This is the ++ * exact opposite of the best strategy for the other type of bursts. ++ * ++ * In the end, to take the best action for each of the two cases, the ++ * two types of bursts need to be distinguished. Fortunately, this ++ * seems relatively easy, by looking at the sizes of the bursts. In ++ * particular, we found a threshold such that only bursts with a ++ * larger size than that threshold are apparently caused by ++ * services or commands such as systemd or git grep. For brevity, ++ * hereafter we call just 'large' these bursts. BFQ *does not* ++ * weight-raise queues whose creation occurs in a large burst. In ++ * addition, for each of these queues BFQ performs or does not perform ++ * idling depending on which choice boosts the throughput more. The ++ * exact choice depends on the device and request pattern at ++ * hand. ++ * ++ * Unfortunately, false positives may occur while an interactive task ++ * is starting (e.g., an application is being started). The ++ * consequence is that the queues associated with the task do not ++ * enjoy weight raising as expected. Fortunately these false positives ++ * are very rare. They typically occur if some service happens to ++ * start doing I/O exactly when the interactive task starts. ++ * ++ * Turning back to the next function, it implements all the steps ++ * needed to detect the occurrence of a large burst and to properly ++ * mark all the queues belonging to it (so that they can then be ++ * treated in a different way). This goal is achieved by maintaining a ++ * "burst list" that holds, temporarily, the queues that belong to the ++ * burst in progress. The list is then used to mark these queues as ++ * belonging to a large burst if the burst does become large. The main ++ * steps are the following. ++ * ++ * . when the very first queue is created, the queue is inserted into the ++ * list (as it could be the first queue in a possible burst) ++ * ++ * . if the current burst has not yet become large, and a queue Q that does ++ * not yet belong to the burst is activated shortly after the last time ++ * at which a new queue entered the burst list, then the function appends ++ * Q to the burst list ++ * ++ * . if, as a consequence of the previous step, the burst size reaches ++ * the large-burst threshold, then ++ * ++ * . all the queues in the burst list are marked as belonging to a ++ * large burst ++ * ++ * . the burst list is deleted; in fact, the burst list already served ++ * its purpose (keeping temporarily track of the queues in a burst, ++ * so as to be able to mark them as belonging to a large burst in the ++ * previous sub-step), and now is not needed any more ++ * ++ * . the device enters a large-burst mode ++ * ++ * . if a queue Q that does not belong to the burst is created while ++ * the device is in large-burst mode and shortly after the last time ++ * at which a queue either entered the burst list or was marked as ++ * belonging to the current large burst, then Q is immediately marked ++ * as belonging to a large burst. ++ * ++ * . if a queue Q that does not belong to the burst is created a while ++ * later, i.e., not shortly after, than the last time at which a queue ++ * either entered the burst list or was marked as belonging to the ++ * current large burst, then the current burst is deemed as finished and: ++ * ++ * . the large-burst mode is reset if set ++ * ++ * . the burst list is emptied ++ * ++ * . Q is inserted in the burst list, as Q may be the first queue ++ * in a possible new burst (then the burst list contains just Q ++ * after this step). ++ */ ++static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ /* ++ * If bfqq is already in the burst list or is part of a large ++ * burst, or finally has just been split, then there is ++ * nothing else to do. ++ */ ++ if (!hlist_unhashed(&bfqq->burst_list_node) || ++ bfq_bfqq_in_large_burst(bfqq) || ++ time_is_after_eq_jiffies(bfqq->split_time + ++ msecs_to_jiffies(10))) ++ return; ++ ++ /* ++ * If bfqq's creation happens late enough, or bfqq belongs to ++ * a different group than the burst group, then the current ++ * burst is finished, and related data structures must be ++ * reset. ++ * ++ * In this respect, consider the special case where bfqq is ++ * the very first queue created after BFQ is selected for this ++ * device. In this case, last_ins_in_burst and ++ * burst_parent_entity are not yet significant when we get ++ * here. But it is easy to verify that, whether or not the ++ * following condition is true, bfqq will end up being ++ * inserted into the burst list. In particular the list will ++ * happen to contain only bfqq. And this is exactly what has ++ * to happen, as bfqq may be the first queue of the first ++ * burst. ++ */ ++ if (time_is_before_jiffies(bfqd->last_ins_in_burst + ++ bfqd->bfq_burst_interval) || ++ bfqq->entity.parent != bfqd->burst_parent_entity) { ++ bfqd->large_burst = false; ++ bfq_reset_burst_list(bfqd, bfqq); ++ bfq_log_bfqq(bfqd, bfqq, ++ "handle_burst: late activation or different group"); ++ goto end; ++ } ++ ++ /* ++ * If we get here, then bfqq is being activated shortly after the ++ * last queue. So, if the current burst is also large, we can mark ++ * bfqq as belonging to this large burst immediately. ++ */ ++ if (bfqd->large_burst) { ++ bfq_log_bfqq(bfqd, bfqq, "handle_burst: marked in burst"); ++ bfq_mark_bfqq_in_large_burst(bfqq); ++ goto end; ++ } ++ ++ /* ++ * If we get here, then a large-burst state has not yet been ++ * reached, but bfqq is being activated shortly after the last ++ * queue. Then we add bfqq to the burst. ++ */ ++ bfq_add_to_burst(bfqd, bfqq); ++end: ++ /* ++ * At this point, bfqq either has been added to the current ++ * burst or has caused the current burst to terminate and a ++ * possible new burst to start. In particular, in the second ++ * case, bfqq has become the first queue in the possible new ++ * burst. In both cases last_ins_in_burst needs to be moved ++ * forward. ++ */ ++ bfqd->last_ins_in_burst = jiffies; ++ ++} ++ ++static int bfq_bfqq_budget_left(struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ return entity->budget - entity->service; ++} ++ ++/* ++ * If enough samples have been computed, return the current max budget ++ * stored in bfqd, which is dynamically updated according to the ++ * estimated disk peak rate; otherwise return the default max budget ++ */ ++static int bfq_max_budget(struct bfq_data *bfqd) ++{ ++ if (bfqd->budgets_assigned < bfq_stats_min_budgets) ++ return bfq_default_max_budget; ++ else ++ return bfqd->bfq_max_budget; ++} ++ ++/* ++ * Return min budget, which is a fraction of the current or default ++ * max budget (trying with 1/32) ++ */ ++static int bfq_min_budget(struct bfq_data *bfqd) ++{ ++ if (bfqd->budgets_assigned < bfq_stats_min_budgets) ++ return bfq_default_max_budget / 32; ++ else ++ return bfqd->bfq_max_budget / 32; ++} ++ ++static void bfq_bfqq_expire(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ bool compensate, ++ enum bfqq_expiration reason); ++ ++/* ++ * The next function, invoked after the input queue bfqq switches from ++ * idle to busy, updates the budget of bfqq. The function also tells ++ * whether the in-service queue should be expired, by returning ++ * true. The purpose of expiring the in-service queue is to give bfqq ++ * the chance to possibly preempt the in-service queue, and the reason ++ * for preempting the in-service queue is to achieve one of the two ++ * goals below. ++ * ++ * 1. Guarantee to bfqq its reserved bandwidth even if bfqq has ++ * expired because it has remained idle. In particular, bfqq may have ++ * expired for one of the following two reasons: ++ * ++ * - BFQ_BFQQ_NO_MORE_REQUEST bfqq did not enjoy any device idling and ++ * did not make it to issue a new request before its last request ++ * was served; ++ * ++ * - BFQ_BFQQ_TOO_IDLE bfqq did enjoy device idling, but did not issue ++ * a new request before the expiration of the idling-time. ++ * ++ * Even if bfqq has expired for one of the above reasons, the process ++ * associated with the queue may be however issuing requests greedily, ++ * and thus be sensitive to the bandwidth it receives (bfqq may have ++ * remained idle for other reasons: CPU high load, bfqq not enjoying ++ * idling, I/O throttling somewhere in the path from the process to ++ * the I/O scheduler, ...). But if, after every expiration for one of ++ * the above two reasons, bfqq has to wait for the service of at least ++ * one full budget of another queue before being served again, then ++ * bfqq is likely to get a much lower bandwidth or resource time than ++ * its reserved ones. To address this issue, two countermeasures need ++ * to be taken. ++ * ++ * First, the budget and the timestamps of bfqq need to be updated in ++ * a special way on bfqq reactivation: they need to be updated as if ++ * bfqq did not remain idle and did not expire. In fact, if they are ++ * computed as if bfqq expired and remained idle until reactivation, ++ * then the process associated with bfqq is treated as if, instead of ++ * being greedy, it stopped issuing requests when bfqq remained idle, ++ * and restarts issuing requests only on this reactivation. In other ++ * words, the scheduler does not help the process recover the "service ++ * hole" between bfqq expiration and reactivation. As a consequence, ++ * the process receives a lower bandwidth than its reserved one. In ++ * contrast, to recover this hole, the budget must be updated as if ++ * bfqq was not expired at all before this reactivation, i.e., it must ++ * be set to the value of the remaining budget when bfqq was ++ * expired. Along the same line, timestamps need to be assigned the ++ * value they had the last time bfqq was selected for service, i.e., ++ * before last expiration. Thus timestamps need to be back-shifted ++ * with respect to their normal computation (see [1] for more details ++ * on this tricky aspect). ++ * ++ * Secondly, to allow the process to recover the hole, the in-service ++ * queue must be expired too, to give bfqq the chance to preempt it ++ * immediately. In fact, if bfqq has to wait for a full budget of the ++ * in-service queue to be completed, then it may become impossible to ++ * let the process recover the hole, even if the back-shifted ++ * timestamps of bfqq are lower than those of the in-service queue. If ++ * this happens for most or all of the holes, then the process may not ++ * receive its reserved bandwidth. In this respect, it is worth noting ++ * that, being the service of outstanding requests unpreemptible, a ++ * little fraction of the holes may however be unrecoverable, thereby ++ * causing a little loss of bandwidth. ++ * ++ * The last important point is detecting whether bfqq does need this ++ * bandwidth recovery. In this respect, the next function deems the ++ * process associated with bfqq greedy, and thus allows it to recover ++ * the hole, if: 1) the process is waiting for the arrival of a new ++ * request (which implies that bfqq expired for one of the above two ++ * reasons), and 2) such a request has arrived soon. The first ++ * condition is controlled through the flag non_blocking_wait_rq, ++ * while the second through the flag arrived_in_time. If both ++ * conditions hold, then the function computes the budget in the ++ * above-described special way, and signals that the in-service queue ++ * should be expired. Timestamp back-shifting is done later in ++ * __bfq_activate_entity. ++ * ++ * 2. Reduce latency. Even if timestamps are not backshifted to let ++ * the process associated with bfqq recover a service hole, bfqq may ++ * however happen to have, after being (re)activated, a lower finish ++ * timestamp than the in-service queue. That is, the next budget of ++ * bfqq may have to be completed before the one of the in-service ++ * queue. If this is the case, then preempting the in-service queue ++ * allows this goal to be achieved, apart from the unpreemptible, ++ * outstanding requests mentioned above. ++ * ++ * Unfortunately, regardless of which of the above two goals one wants ++ * to achieve, service trees need first to be updated to know whether ++ * the in-service queue must be preempted. To have service trees ++ * correctly updated, the in-service queue must be expired and ++ * rescheduled, and bfqq must be scheduled too. This is one of the ++ * most costly operations (in future versions, the scheduling ++ * mechanism may be re-designed in such a way to make it possible to ++ * know whether preemption is needed without needing to update service ++ * trees). In addition, queue preemptions almost always cause random ++ * I/O, and thus loss of throughput. Because of these facts, the next ++ * function adopts the following simple scheme to avoid both costly ++ * operations and too frequent preemptions: it requests the expiration ++ * of the in-service queue (unconditionally) only for queues that need ++ * to recover a hole, or that either are weight-raised or deserve to ++ * be weight-raised. ++ */ ++static bool bfq_bfqq_update_budg_for_activation(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ bool arrived_in_time, ++ bool wr_or_deserves_wr) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ if (bfq_bfqq_non_blocking_wait_rq(bfqq) && arrived_in_time) { ++ /* ++ * We do not clear the flag non_blocking_wait_rq here, as ++ * the latter is used in bfq_activate_bfqq to signal ++ * that timestamps need to be back-shifted (and is ++ * cleared right after). ++ */ ++ ++ /* ++ * In next assignment we rely on that either ++ * entity->service or entity->budget are not updated ++ * on expiration if bfqq is empty (see ++ * __bfq_bfqq_recalc_budget). Thus both quantities ++ * remain unchanged after such an expiration, and the ++ * following statement therefore assigns to ++ * entity->budget the remaining budget on such an ++ * expiration. For clarity, entity->service is not ++ * updated on expiration in any case, and, in normal ++ * operation, is reset only when bfqq is selected for ++ * service (see bfq_get_next_queue). ++ */ ++ BUG_ON(bfqq->max_budget < 0); ++ entity->budget = min_t(unsigned long, ++ bfq_bfqq_budget_left(bfqq), ++ bfqq->max_budget); ++ ++ BUG_ON(entity->budget < 0); ++ return true; ++ } ++ ++ BUG_ON(bfqq->max_budget < 0); ++ entity->budget = max_t(unsigned long, bfqq->max_budget, ++ bfq_serv_to_charge(bfqq->next_rq, bfqq)); ++ BUG_ON(entity->budget < 0); ++ ++ bfq_clear_bfqq_non_blocking_wait_rq(bfqq); ++ return wr_or_deserves_wr; ++} ++ ++static void bfq_update_bfqq_wr_on_rq_arrival(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ unsigned int old_wr_coeff, ++ bool wr_or_deserves_wr, ++ bool interactive, ++ bool in_burst, ++ bool soft_rt) ++{ ++ if (old_wr_coeff == 1 && wr_or_deserves_wr) { ++ /* start a weight-raising period */ ++ if (interactive) { ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff; ++ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); ++ } else { ++ bfqq->wr_start_at_switch_to_srt = jiffies; ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff * ++ BFQ_SOFTRT_WEIGHT_FACTOR; ++ bfqq->wr_cur_max_time = ++ bfqd->bfq_wr_rt_max_time; ++ } ++ /* ++ * If needed, further reduce budget to make sure it is ++ * close to bfqq's backlog, so as to reduce the ++ * scheduling-error component due to a too large ++ * budget. Do not care about throughput consequences, ++ * but only about latency. Finally, do not assign a ++ * too small budget either, to avoid increasing ++ * latency by causing too frequent expirations. ++ */ ++ bfqq->entity.budget = min_t(unsigned long, ++ bfqq->entity.budget, ++ 2 * bfq_min_budget(bfqd)); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "wrais starting at %lu, rais_max_time %u", ++ jiffies, ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ } else if (old_wr_coeff > 1) { ++ if (interactive) { /* update wr coeff and duration */ ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff; ++ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); ++ } else if (in_burst) { ++ bfqq->wr_coeff = 1; ++ bfq_log_bfqq(bfqd, bfqq, ++ "wrais ending at %lu, rais_max_time %u", ++ jiffies, ++ jiffies_to_msecs(bfqq-> ++ wr_cur_max_time)); ++ } else if (soft_rt) { ++ /* ++ * The application is now or still meeting the ++ * requirements for being deemed soft rt. We ++ * can then correctly and safely (re)charge ++ * the weight-raising duration for the ++ * application with the weight-raising ++ * duration for soft rt applications. ++ * ++ * In particular, doing this recharge now, i.e., ++ * before the weight-raising period for the ++ * application finishes, reduces the probability ++ * of the following negative scenario: ++ * 1) the weight of a soft rt application is ++ * raised at startup (as for any newly ++ * created application), ++ * 2) since the application is not interactive, ++ * at a certain time weight-raising is ++ * stopped for the application, ++ * 3) at that time the application happens to ++ * still have pending requests, and hence ++ * is destined to not have a chance to be ++ * deemed soft rt before these requests are ++ * completed (see the comments to the ++ * function bfq_bfqq_softrt_next_start() ++ * for details on soft rt detection), ++ * 4) these pending requests experience a high ++ * latency because the application is not ++ * weight-raised while they are pending. ++ */ ++ if (bfqq->wr_cur_max_time != ++ bfqd->bfq_wr_rt_max_time) { ++ bfqq->wr_start_at_switch_to_srt = ++ bfqq->last_wr_start_finish; ++ BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish)); ++ ++ bfqq->wr_cur_max_time = ++ bfqd->bfq_wr_rt_max_time; ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff * ++ BFQ_SOFTRT_WEIGHT_FACTOR; ++ bfq_log_bfqq(bfqd, bfqq, ++ "switching to soft_rt wr"); ++ } else ++ bfq_log_bfqq(bfqd, bfqq, ++ "moving forward soft_rt wr duration"); ++ bfqq->last_wr_start_finish = jiffies; ++ } ++ } ++} ++ ++static bool bfq_bfqq_idle_for_long_time(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ return bfqq->dispatched == 0 && ++ time_is_before_jiffies( ++ bfqq->budget_timeout + ++ bfqd->bfq_wr_min_idle_time); ++} ++ ++static void bfq_bfqq_handle_idle_busy_switch(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ int old_wr_coeff, ++ struct request *rq, ++ bool *interactive) ++{ ++ bool soft_rt, in_burst, wr_or_deserves_wr, ++ bfqq_wants_to_preempt, ++ idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq), ++ /* ++ * See the comments on ++ * bfq_bfqq_update_budg_for_activation for ++ * details on the usage of the next variable. ++ */ ++ arrived_in_time = ktime_get_ns() <= ++ RQ_BIC(rq)->ttime.last_end_request + ++ bfqd->bfq_slice_idle * 3; ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "bfq_add_request non-busy: " ++ "jiffies %lu, in_time %d, idle_long %d busyw %d " ++ "wr_coeff %u", ++ jiffies, arrived_in_time, ++ idle_for_long_time, ++ bfq_bfqq_non_blocking_wait_rq(bfqq), ++ old_wr_coeff); ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ bfqg_stats_update_io_add(bfqq_group(RQ_BFQQ(rq)), bfqq, ++ req_op(rq), rq->cmd_flags); ++ ++ /* ++ * bfqq deserves to be weight-raised if: ++ * - it is sync, ++ * - it does not belong to a large burst, ++ * - it has been idle for enough time or is soft real-time, ++ * - is linked to a bfq_io_cq (it is not shared in any sense) ++ */ ++ in_burst = bfq_bfqq_in_large_burst(bfqq); ++ soft_rt = bfqd->bfq_wr_max_softrt_rate > 0 && ++ !in_burst && ++ time_is_before_jiffies(bfqq->soft_rt_next_start); ++ *interactive = ++ !in_burst && ++ idle_for_long_time; ++ wr_or_deserves_wr = bfqd->low_latency && ++ (bfqq->wr_coeff > 1 || ++ (bfq_bfqq_sync(bfqq) && ++ bfqq->bic && (*interactive || soft_rt))); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "bfq_add_request: " ++ "in_burst %d, " ++ "soft_rt %d (next %lu), inter %d, bic %p", ++ bfq_bfqq_in_large_burst(bfqq), soft_rt, ++ bfqq->soft_rt_next_start, ++ *interactive, ++ bfqq->bic); ++ ++ /* ++ * Using the last flag, update budget and check whether bfqq ++ * may want to preempt the in-service queue. ++ */ ++ bfqq_wants_to_preempt = ++ bfq_bfqq_update_budg_for_activation(bfqd, bfqq, ++ arrived_in_time, ++ wr_or_deserves_wr); ++ ++ /* ++ * If bfqq happened to be activated in a burst, but has been ++ * idle for much more than an interactive queue, then we ++ * assume that, in the overall I/O initiated in the burst, the ++ * I/O associated with bfqq is finished. So bfqq does not need ++ * to be treated as a queue belonging to a burst ++ * anymore. Accordingly, we reset bfqq's in_large_burst flag ++ * if set, and remove bfqq from the burst list if it's ++ * there. We do not decrement burst_size, because the fact ++ * that bfqq does not need to belong to the burst list any ++ * more does not invalidate the fact that bfqq was created in ++ * a burst. ++ */ ++ if (likely(!bfq_bfqq_just_created(bfqq)) && ++ idle_for_long_time && ++ time_is_before_jiffies( ++ bfqq->budget_timeout + ++ msecs_to_jiffies(10000))) { ++ hlist_del_init(&bfqq->burst_list_node); ++ bfq_clear_bfqq_in_large_burst(bfqq); ++ } ++ ++ bfq_clear_bfqq_just_created(bfqq); ++ ++ if (!bfq_bfqq_IO_bound(bfqq)) { ++ if (arrived_in_time) { ++ bfqq->requests_within_timer++; ++ if (bfqq->requests_within_timer >= ++ bfqd->bfq_requests_within_timer) ++ bfq_mark_bfqq_IO_bound(bfqq); ++ } else ++ bfqq->requests_within_timer = 0; ++ bfq_log_bfqq(bfqd, bfqq, "requests in time %d", ++ bfqq->requests_within_timer); ++ } ++ ++ if (bfqd->low_latency) { ++ if (unlikely(time_is_after_jiffies(bfqq->split_time))) ++ /* wraparound */ ++ bfqq->split_time = ++ jiffies - bfqd->bfq_wr_min_idle_time - 1; ++ ++ if (time_is_before_jiffies(bfqq->split_time + ++ bfqd->bfq_wr_min_idle_time)) { ++ bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq, ++ old_wr_coeff, ++ wr_or_deserves_wr, ++ *interactive, ++ in_burst, ++ soft_rt); ++ ++ if (old_wr_coeff != bfqq->wr_coeff) ++ bfqq->entity.prio_changed = 1; ++ } ++ } ++ ++ bfqq->last_idle_bklogged = jiffies; ++ bfqq->service_from_backlogged = 0; ++ bfq_clear_bfqq_softrt_update(bfqq); ++ ++ bfq_add_bfqq_busy(bfqd, bfqq); ++ ++ /* ++ * Expire in-service queue only if preemption may be needed ++ * for guarantees. In this respect, the function ++ * next_queue_may_preempt just checks a simple, necessary ++ * condition, and not a sufficient condition based on ++ * timestamps. In fact, for the latter condition to be ++ * evaluated, timestamps would need first to be updated, and ++ * this operation is quite costly (see the comments on the ++ * function bfq_bfqq_update_budg_for_activation). ++ */ ++ if (bfqd->in_service_queue && bfqq_wants_to_preempt && ++ bfqd->in_service_queue->wr_coeff < bfqq->wr_coeff && ++ next_queue_may_preempt(bfqd)) { ++ struct bfq_queue *in_serv = ++ bfqd->in_service_queue; ++ BUG_ON(in_serv == bfqq); ++ ++ bfq_bfqq_expire(bfqd, bfqd->in_service_queue, ++ false, BFQ_BFQQ_PREEMPTED); ++ BUG_ON(in_serv->entity.budget < 0); ++ } ++} ++ ++static void bfq_add_request(struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ struct bfq_data *bfqd = bfqq->bfqd; ++ struct request *next_rq, *prev; ++ unsigned int old_wr_coeff = bfqq->wr_coeff; ++ bool interactive = false; ++ ++ bfq_log_bfqq(bfqd, bfqq, "add_request: size %u %s", ++ blk_rq_sectors(rq), rq_is_sync(rq) ? "S" : "A"); ++ ++ if (bfqq->wr_coeff > 1) /* queue is being weight-raised */ ++ bfq_log_bfqq(bfqd, bfqq, ++ "raising period dur %u/%u msec, old coeff %u, w %d(%d)", ++ jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), ++ jiffies_to_msecs(bfqq->wr_cur_max_time), ++ bfqq->wr_coeff, ++ bfqq->entity.weight, bfqq->entity.orig_weight); ++ ++ bfqq->queued[rq_is_sync(rq)]++; ++ bfqd->queued++; ++ ++ elv_rb_add(&bfqq->sort_list, rq); ++ ++ /* ++ * Check if this request is a better next-to-serve candidate. ++ */ ++ prev = bfqq->next_rq; ++ next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position); ++ BUG_ON(!next_rq); ++ bfqq->next_rq = next_rq; ++ ++ /* ++ * Adjust priority tree position, if next_rq changes. ++ */ ++ if (prev != bfqq->next_rq) ++ bfq_pos_tree_add_move(bfqd, bfqq); ++ ++ if (!bfq_bfqq_busy(bfqq)) /* switching to busy ... */ ++ bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff, ++ rq, &interactive); ++ else { ++ if (bfqd->low_latency && old_wr_coeff == 1 && !rq_is_sync(rq) && ++ time_is_before_jiffies( ++ bfqq->last_wr_start_finish + ++ bfqd->bfq_wr_min_inter_arr_async)) { ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff; ++ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); ++ ++ bfqd->wr_busy_queues++; ++ bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqd, bfqq, ++ "non-idle wrais starting, " ++ "wr_max_time %u wr_busy %d", ++ jiffies_to_msecs(bfqq->wr_cur_max_time), ++ bfqd->wr_busy_queues); ++ } ++ if (prev != bfqq->next_rq) ++ bfq_updated_next_req(bfqd, bfqq); ++ } ++ ++ /* ++ * Assign jiffies to last_wr_start_finish in the following ++ * cases: ++ * ++ * . if bfqq is not going to be weight-raised, because, for ++ * non weight-raised queues, last_wr_start_finish stores the ++ * arrival time of the last request; as of now, this piece ++ * of information is used only for deciding whether to ++ * weight-raise async queues ++ * ++ * . if bfqq is not weight-raised, because, if bfqq is now ++ * switching to weight-raised, then last_wr_start_finish ++ * stores the time when weight-raising starts ++ * ++ * . if bfqq is interactive, because, regardless of whether ++ * bfqq is currently weight-raised, the weight-raising ++ * period must start or restart (this case is considered ++ * separately because it is not detected by the above ++ * conditions, if bfqq is already weight-raised) ++ * ++ * last_wr_start_finish has to be updated also if bfqq is soft ++ * real-time, because the weight-raising period is constantly ++ * restarted on idle-to-busy transitions for these queues, but ++ * this is already done in bfq_bfqq_handle_idle_busy_switch if ++ * needed. ++ */ ++ if (bfqd->low_latency && ++ (old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive)) ++ bfqq->last_wr_start_finish = jiffies; ++} ++ ++static struct request *bfq_find_rq_fmerge(struct bfq_data *bfqd, ++ struct bio *bio) ++{ ++ struct task_struct *tsk = current; ++ struct bfq_io_cq *bic; ++ struct bfq_queue *bfqq; ++ ++ bic = bfq_bic_lookup(bfqd, tsk->io_context); ++ if (!bic) ++ return NULL; ++ ++ bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio)); ++ if (bfqq) ++ return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio)); ++ ++ return NULL; ++} ++ ++static sector_t get_sdist(sector_t last_pos, struct request *rq) ++{ ++ sector_t sdist = 0; ++ ++ if (last_pos) { ++ if (last_pos < blk_rq_pos(rq)) ++ sdist = blk_rq_pos(rq) - last_pos; ++ else ++ sdist = last_pos - blk_rq_pos(rq); ++ } ++ ++ return sdist; ++} ++ ++static void bfq_activate_request(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ bfqd->rq_in_driver++; ++} ++ ++static void bfq_deactivate_request(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ ++ BUG_ON(bfqd->rq_in_driver == 0); ++ bfqd->rq_in_driver--; ++} ++ ++static void bfq_remove_request(struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ struct bfq_data *bfqd = bfqq->bfqd; ++ const int sync = rq_is_sync(rq); ++ ++ BUG_ON(bfqq->entity.service > bfqq->entity.budget && ++ bfqq == bfqd->in_service_queue); ++ ++ if (bfqq->next_rq == rq) { ++ bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); ++ bfq_updated_next_req(bfqd, bfqq); ++ } ++ ++ if (rq->queuelist.prev != &rq->queuelist) ++ list_del_init(&rq->queuelist); ++ BUG_ON(bfqq->queued[sync] == 0); ++ bfqq->queued[sync]--; ++ bfqd->queued--; ++ elv_rb_del(&bfqq->sort_list, rq); ++ ++ if (RB_EMPTY_ROOT(&bfqq->sort_list)) { ++ BUG_ON(bfqq->entity.budget < 0); ++ ++ if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { ++ bfq_del_bfqq_busy(bfqd, bfqq, 1); ++ ++ /* bfqq emptied. In normal operation, when ++ * bfqq is empty, bfqq->entity.service and ++ * bfqq->entity.budget must contain, ++ * respectively, the service received and the ++ * budget used last time bfqq emptied. These ++ * facts do not hold in this case, as at least ++ * this last removal occurred while bfqq is ++ * not in service. To avoid inconsistencies, ++ * reset both bfqq->entity.service and ++ * bfqq->entity.budget. ++ */ ++ bfqq->entity.budget = bfqq->entity.service = 0; ++ } ++ ++ /* ++ * Remove queue from request-position tree as it is empty. ++ */ ++ if (bfqq->pos_root) { ++ rb_erase(&bfqq->pos_node, bfqq->pos_root); ++ bfqq->pos_root = NULL; ++ } ++ } ++ ++ if (rq->cmd_flags & REQ_META) { ++ BUG_ON(bfqq->meta_pending == 0); ++ bfqq->meta_pending--; ++ } ++ bfqg_stats_update_io_remove(bfqq_group(bfqq), req_op(rq), ++ rq->cmd_flags); ++} ++ ++static int bfq_merge(struct request_queue *q, struct request **req, ++ struct bio *bio) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct request *__rq; ++ ++ __rq = bfq_find_rq_fmerge(bfqd, bio); ++ if (__rq && elv_bio_merge_ok(__rq, bio)) { ++ *req = __rq; ++ return ELEVATOR_FRONT_MERGE; ++ } ++ ++ return ELEVATOR_NO_MERGE; ++} ++ ++static void bfq_merged_request(struct request_queue *q, struct request *req, ++ int type) ++{ ++ if (type == ELEVATOR_FRONT_MERGE && ++ rb_prev(&req->rb_node) && ++ blk_rq_pos(req) < ++ blk_rq_pos(container_of(rb_prev(&req->rb_node), ++ struct request, rb_node))) { ++ struct bfq_queue *bfqq = RQ_BFQQ(req); ++ struct bfq_data *bfqd = bfqq->bfqd; ++ struct request *prev, *next_rq; ++ ++ /* Reposition request in its sort_list */ ++ elv_rb_del(&bfqq->sort_list, req); ++ elv_rb_add(&bfqq->sort_list, req); ++ /* Choose next request to be served for bfqq */ ++ prev = bfqq->next_rq; ++ next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req, ++ bfqd->last_position); ++ BUG_ON(!next_rq); ++ bfqq->next_rq = next_rq; ++ /* ++ * If next_rq changes, update both the queue's budget to ++ * fit the new request and the queue's position in its ++ * rq_pos_tree. ++ */ ++ if (prev != bfqq->next_rq) { ++ bfq_updated_next_req(bfqd, bfqq); ++ bfq_pos_tree_add_move(bfqd, bfqq); ++ } ++ } ++} ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static void bfq_bio_merged(struct request_queue *q, struct request *req, ++ struct bio *bio) ++{ ++ bfqg_stats_update_io_merged(bfqq_group(RQ_BFQQ(req)), bio_op(bio), ++ bio->bi_opf); ++} ++#endif ++ ++static void bfq_merged_requests(struct request_queue *q, struct request *rq, ++ struct request *next) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq), *next_bfqq = RQ_BFQQ(next); ++ ++ /* ++ * If next and rq belong to the same bfq_queue and next is older ++ * than rq, then reposition rq in the fifo (by substituting next ++ * with rq). Otherwise, if next and rq belong to different ++ * bfq_queues, never reposition rq: in fact, we would have to ++ * reposition it with respect to next's position in its own fifo, ++ * which would most certainly be too expensive with respect to ++ * the benefits. ++ */ ++ if (bfqq == next_bfqq && ++ !list_empty(&rq->queuelist) && !list_empty(&next->queuelist) && ++ next->fifo_time < rq->fifo_time) { ++ list_del_init(&rq->queuelist); ++ list_replace_init(&next->queuelist, &rq->queuelist); ++ rq->fifo_time = next->fifo_time; ++ } ++ ++ if (bfqq->next_rq == next) ++ bfqq->next_rq = rq; ++ ++ bfq_remove_request(next); ++ bfqg_stats_update_io_merged(bfqq_group(bfqq), req_op(next), ++ next->cmd_flags); ++} ++ ++/* Must be called with bfqq != NULL */ ++static void bfq_bfqq_end_wr(struct bfq_queue *bfqq) ++{ ++ BUG_ON(!bfqq); ++ ++ if (bfq_bfqq_busy(bfqq)) ++ bfqq->bfqd->wr_busy_queues--; ++ bfqq->wr_coeff = 1; ++ bfqq->wr_cur_max_time = 0; ++ bfqq->last_wr_start_finish = jiffies; ++ /* ++ * Trigger a weight change on the next invocation of ++ * __bfq_entity_update_weight_prio. ++ */ ++ bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "end_wr: wrais ending at %lu, rais_max_time %u", ++ bfqq->last_wr_start_finish, ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "end_wr: wr_busy %d", ++ bfqq->bfqd->wr_busy_queues); ++} ++ ++static void bfq_end_wr_async_queues(struct bfq_data *bfqd, ++ struct bfq_group *bfqg) ++{ ++ int i, j; ++ ++ for (i = 0; i < 2; i++) ++ for (j = 0; j < IOPRIO_BE_NR; j++) ++ if (bfqg->async_bfqq[i][j]) ++ bfq_bfqq_end_wr(bfqg->async_bfqq[i][j]); ++ if (bfqg->async_idle_bfqq) ++ bfq_bfqq_end_wr(bfqg->async_idle_bfqq); ++} ++ ++static void bfq_end_wr(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq; ++ ++ spin_lock_irq(bfqd->queue->queue_lock); ++ ++ list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) ++ bfq_bfqq_end_wr(bfqq); ++ list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) ++ bfq_bfqq_end_wr(bfqq); ++ bfq_end_wr_async(bfqd); ++ ++ spin_unlock_irq(bfqd->queue->queue_lock); ++} ++ ++static sector_t bfq_io_struct_pos(void *io_struct, bool request) ++{ ++ if (request) ++ return blk_rq_pos(io_struct); ++ else ++ return ((struct bio *)io_struct)->bi_iter.bi_sector; ++} ++ ++static int bfq_rq_close_to_sector(void *io_struct, bool request, ++ sector_t sector) ++{ ++ return abs(bfq_io_struct_pos(io_struct, request) - sector) <= ++ BFQQ_CLOSE_THR; ++} ++ ++static struct bfq_queue *bfqq_find_close(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ sector_t sector) ++{ ++ struct rb_root *root = &bfq_bfqq_to_bfqg(bfqq)->rq_pos_tree; ++ struct rb_node *parent, *node; ++ struct bfq_queue *__bfqq; ++ ++ if (RB_EMPTY_ROOT(root)) ++ return NULL; ++ ++ /* ++ * First, if we find a request starting at the end of the last ++ * request, choose it. ++ */ ++ __bfqq = bfq_rq_pos_tree_lookup(bfqd, root, sector, &parent, NULL); ++ if (__bfqq) ++ return __bfqq; ++ ++ /* ++ * If the exact sector wasn't found, the parent of the NULL leaf ++ * will contain the closest sector (rq_pos_tree sorted by ++ * next_request position). ++ */ ++ __bfqq = rb_entry(parent, struct bfq_queue, pos_node); ++ if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) ++ return __bfqq; ++ ++ if (blk_rq_pos(__bfqq->next_rq) < sector) ++ node = rb_next(&__bfqq->pos_node); ++ else ++ node = rb_prev(&__bfqq->pos_node); ++ if (!node) ++ return NULL; ++ ++ __bfqq = rb_entry(node, struct bfq_queue, pos_node); ++ if (bfq_rq_close_to_sector(__bfqq->next_rq, true, sector)) ++ return __bfqq; ++ ++ return NULL; ++} ++ ++static struct bfq_queue *bfq_find_close_cooperator(struct bfq_data *bfqd, ++ struct bfq_queue *cur_bfqq, ++ sector_t sector) ++{ ++ struct bfq_queue *bfqq; ++ ++ /* ++ * We shall notice if some of the queues are cooperating, ++ * e.g., working closely on the same area of the device. In ++ * that case, we can group them together and: 1) don't waste ++ * time idling, and 2) serve the union of their requests in ++ * the best possible order for throughput. ++ */ ++ bfqq = bfqq_find_close(bfqd, cur_bfqq, sector); ++ if (!bfqq || bfqq == cur_bfqq) ++ return NULL; ++ ++ return bfqq; ++} ++ ++static struct bfq_queue * ++bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) ++{ ++ int process_refs, new_process_refs; ++ struct bfq_queue *__bfqq; ++ ++ /* ++ * If there are no process references on the new_bfqq, then it is ++ * unsafe to follow the ->new_bfqq chain as other bfqq's in the chain ++ * may have dropped their last reference (not just their last process ++ * reference). ++ */ ++ if (!bfqq_process_refs(new_bfqq)) ++ return NULL; ++ ++ /* Avoid a circular list and skip interim queue merges. */ ++ while ((__bfqq = new_bfqq->new_bfqq)) { ++ if (__bfqq == bfqq) ++ return NULL; ++ new_bfqq = __bfqq; ++ } ++ ++ process_refs = bfqq_process_refs(bfqq); ++ new_process_refs = bfqq_process_refs(new_bfqq); ++ /* ++ * If the process for the bfqq has gone away, there is no ++ * sense in merging the queues. ++ */ ++ if (process_refs == 0 || new_process_refs == 0) ++ return NULL; ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", ++ new_bfqq->pid); ++ ++ /* ++ * Merging is just a redirection: the requests of the process ++ * owning one of the two queues are redirected to the other queue. ++ * The latter queue, in its turn, is set as shared if this is the ++ * first time that the requests of some process are redirected to ++ * it. ++ * ++ * We redirect bfqq to new_bfqq and not the opposite, because we ++ * are in the context of the process owning bfqq, hence we have ++ * the io_cq of this process. So we can immediately configure this ++ * io_cq to redirect the requests of the process to new_bfqq. ++ * ++ * NOTE, even if new_bfqq coincides with the in-service queue, the ++ * io_cq of new_bfqq is not available, because, if the in-service ++ * queue is shared, bfqd->in_service_bic may not point to the ++ * io_cq of the in-service queue. ++ * Redirecting the requests of the process owning bfqq to the ++ * currently in-service queue is in any case the best option, as ++ * we feed the in-service queue with new requests close to the ++ * last request served and, by doing so, hopefully increase the ++ * throughput. ++ */ ++ bfqq->new_bfqq = new_bfqq; ++ new_bfqq->ref += process_refs; ++ return new_bfqq; ++} ++ ++static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, ++ struct bfq_queue *new_bfqq) ++{ ++ if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) || ++ (bfqq->ioprio_class != new_bfqq->ioprio_class)) ++ return false; ++ ++ /* ++ * If either of the queues has already been detected as seeky, ++ * then merging it with the other queue is unlikely to lead to ++ * sequential I/O. ++ */ ++ if (BFQQ_SEEKY(bfqq) || BFQQ_SEEKY(new_bfqq)) ++ return false; ++ ++ /* ++ * Interleaved I/O is known to be done by (some) applications ++ * only for reads, so it does not make sense to merge async ++ * queues. ++ */ ++ if (!bfq_bfqq_sync(bfqq) || !bfq_bfqq_sync(new_bfqq)) ++ return false; ++ ++ return true; ++} ++ ++/* ++ * If this function returns true, then bfqq cannot be merged. The idea ++ * is that true cooperation happens very early after processes start ++ * to do I/O. Usually, late cooperations are just accidental false ++ * positives. In case bfqq is weight-raised, such false positives ++ * would evidently degrade latency guarantees for bfqq. ++ */ ++bool wr_from_too_long(struct bfq_queue *bfqq) ++{ ++ return bfqq->wr_coeff > 1 && ++ time_is_before_jiffies(bfqq->last_wr_start_finish + ++ msecs_to_jiffies(100)); ++} ++ ++/* ++ * Attempt to schedule a merge of bfqq with the currently in-service ++ * queue or with a close queue among the scheduled queues. Return ++ * NULL if no merge was scheduled, a pointer to the shared bfq_queue ++ * structure otherwise. ++ * ++ * The OOM queue is not allowed to participate to cooperation: in fact, since ++ * the requests temporarily redirected to the OOM queue could be redirected ++ * again to dedicated queues at any time, the state needed to correctly ++ * handle merging with the OOM queue would be quite complex and expensive ++ * to maintain. Besides, in such a critical condition as an out of memory, ++ * the benefits of queue merging may be little relevant, or even negligible. ++ * ++ * Weight-raised queues can be merged only if their weight-raising ++ * period has just started. In fact cooperating processes are usually ++ * started together. Thus, with this filter we avoid false positives ++ * that would jeopardize low-latency guarantees. ++ * ++ * WARNING: queue merging may impair fairness among non-weight raised ++ * queues, for at least two reasons: 1) the original weight of a ++ * merged queue may change during the merged state, 2) even being the ++ * weight the same, a merged queue may be bloated with many more ++ * requests than the ones produced by its originally-associated ++ * process. ++ */ ++static struct bfq_queue * ++bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ void *io_struct, bool request) ++{ ++ struct bfq_queue *in_service_bfqq, *new_bfqq; ++ ++ if (bfqq->new_bfqq) ++ return bfqq->new_bfqq; ++ ++ if (io_struct && wr_from_too_long(bfqq) && ++ likely(bfqq != &bfqd->oom_bfqq)) ++ bfq_log_bfqq(bfqd, bfqq, ++ "would have looked for coop, but bfq%d wr", ++ bfqq->pid); ++ ++ if (!io_struct || ++ wr_from_too_long(bfqq) || ++ unlikely(bfqq == &bfqd->oom_bfqq)) ++ return NULL; ++ ++ /* If there is only one backlogged queue, don't search. */ ++ if (bfqd->busy_queues == 1) ++ return NULL; ++ ++ in_service_bfqq = bfqd->in_service_queue; ++ ++ if (in_service_bfqq && in_service_bfqq != bfqq && ++ bfqd->in_service_bic && wr_from_too_long(in_service_bfqq) ++ && likely(in_service_bfqq == &bfqd->oom_bfqq)) ++ bfq_log_bfqq(bfqd, bfqq, ++ "would have tried merge with in-service-queue, but wr"); ++ ++ if (!in_service_bfqq || in_service_bfqq == bfqq || ++ !bfqd->in_service_bic || wr_from_too_long(in_service_bfqq) || ++ unlikely(in_service_bfqq == &bfqd->oom_bfqq)) ++ goto check_scheduled; ++ ++ if (bfq_rq_close_to_sector(io_struct, request, bfqd->last_position) && ++ bfqq->entity.parent == in_service_bfqq->entity.parent && ++ bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) { ++ new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq); ++ if (new_bfqq) ++ return new_bfqq; ++ } ++ /* ++ * Check whether there is a cooperator among currently scheduled ++ * queues. The only thing we need is that the bio/request is not ++ * NULL, as we need it to establish whether a cooperator exists. ++ */ ++check_scheduled: ++ new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, ++ bfq_io_struct_pos(io_struct, request)); ++ ++ BUG_ON(new_bfqq && bfqq->entity.parent != new_bfqq->entity.parent); ++ ++ if (new_bfqq && wr_from_too_long(new_bfqq) && ++ likely(new_bfqq != &bfqd->oom_bfqq) && ++ bfq_may_be_close_cooperator(bfqq, new_bfqq)) ++ bfq_log_bfqq(bfqd, bfqq, ++ "would have merged with bfq%d, but wr", ++ new_bfqq->pid); ++ ++ if (new_bfqq && !wr_from_too_long(new_bfqq) && ++ likely(new_bfqq != &bfqd->oom_bfqq) && ++ bfq_may_be_close_cooperator(bfqq, new_bfqq)) ++ return bfq_setup_merge(bfqq, new_bfqq); ++ ++ return NULL; ++} ++ ++static void bfq_bfqq_save_state(struct bfq_queue *bfqq) ++{ ++ struct bfq_io_cq *bic = bfqq->bic; ++ ++ /* ++ * If !bfqq->bic, the queue is already shared or its requests ++ * have already been redirected to a shared queue; both idle window ++ * and weight raising state have already been saved. Do nothing. ++ */ ++ if (!bic) ++ return; ++ ++ bic->saved_idle_window = bfq_bfqq_idle_window(bfqq); ++ bic->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); ++ bic->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); ++ bic->was_in_burst_list = !hlist_unhashed(&bfqq->burst_list_node); ++ bic->saved_wr_coeff = bfqq->wr_coeff; ++ bic->saved_wr_start_at_switch_to_srt = bfqq->wr_start_at_switch_to_srt; ++ bic->saved_last_wr_start_finish = bfqq->last_wr_start_finish; ++ BUG_ON(time_is_after_jiffies(bfqq->last_wr_start_finish)); ++} ++ ++static void bfq_get_bic_reference(struct bfq_queue *bfqq) ++{ ++ /* ++ * If bfqq->bic has a non-NULL value, the bic to which it belongs ++ * is about to begin using a shared bfq_queue. ++ */ ++ if (bfqq->bic) ++ atomic_long_inc(&bfqq->bic->icq.ioc->refcount); ++} ++ ++static void ++bfq_merge_bfqqs(struct bfq_data *bfqd, struct bfq_io_cq *bic, ++ struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) ++{ ++ bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", ++ (unsigned long) new_bfqq->pid); ++ /* Save weight raising and idle window of the merged queues */ ++ bfq_bfqq_save_state(bfqq); ++ bfq_bfqq_save_state(new_bfqq); ++ if (bfq_bfqq_IO_bound(bfqq)) ++ bfq_mark_bfqq_IO_bound(new_bfqq); ++ bfq_clear_bfqq_IO_bound(bfqq); ++ ++ /* ++ * If bfqq is weight-raised, then let new_bfqq inherit ++ * weight-raising. To reduce false positives, neglect the case ++ * where bfqq has just been created, but has not yet made it ++ * to be weight-raised (which may happen because EQM may merge ++ * bfqq even before bfq_add_request is executed for the first ++ * time for bfqq). Handling this case would however be very ++ * easy, thanks to the flag just_created. ++ */ ++ if (new_bfqq->wr_coeff == 1 && bfqq->wr_coeff > 1) { ++ new_bfqq->wr_coeff = bfqq->wr_coeff; ++ new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time; ++ new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish; ++ new_bfqq->wr_start_at_switch_to_srt = bfqq->wr_start_at_switch_to_srt; ++ if (bfq_bfqq_busy(new_bfqq)) ++ bfqd->wr_busy_queues++; ++ new_bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqd, new_bfqq, ++ "wr start after merge with %d, rais_max_time %u", ++ bfqq->pid, ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ } ++ ++ if (bfqq->wr_coeff > 1) { /* bfqq has given its wr to new_bfqq */ ++ bfqq->wr_coeff = 1; ++ bfqq->entity.prio_changed = 1; ++ if (bfq_bfqq_busy(bfqq)) ++ bfqd->wr_busy_queues--; ++ } ++ ++ bfq_log_bfqq(bfqd, new_bfqq, "merge_bfqqs: wr_busy %d", ++ bfqd->wr_busy_queues); ++ ++ /* ++ * Grab a reference to the bic, to prevent it from being destroyed ++ * before being possibly touched by a bfq_split_bfqq(). ++ */ ++ bfq_get_bic_reference(bfqq); ++ bfq_get_bic_reference(new_bfqq); ++ /* ++ * Merge queues (that is, let bic redirect its requests to new_bfqq) ++ */ ++ bic_set_bfqq(bic, new_bfqq, 1); ++ bfq_mark_bfqq_coop(new_bfqq); ++ /* ++ * new_bfqq now belongs to at least two bics (it is a shared queue): ++ * set new_bfqq->bic to NULL. bfqq either: ++ * - does not belong to any bic any more, and hence bfqq->bic must ++ * be set to NULL, or ++ * - is a queue whose owning bics have already been redirected to a ++ * different queue, hence the queue is destined to not belong to ++ * any bic soon and bfqq->bic is already NULL (therefore the next ++ * assignment causes no harm). ++ */ ++ new_bfqq->bic = NULL; ++ bfqq->bic = NULL; ++ bfq_put_queue(bfqq); ++} ++ ++static int bfq_allow_bio_merge(struct request_queue *q, struct request *rq, ++ struct bio *bio) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct bfq_io_cq *bic; ++ struct bfq_queue *bfqq, *new_bfqq; ++ ++ /* ++ * Disallow merge of a sync bio into an async request. ++ */ ++ if (bfq_bio_sync(bio) && !rq_is_sync(rq)) ++ return false; ++ ++ /* ++ * Lookup the bfqq that this bio will be queued with. Allow ++ * merge only if rq is queued there. ++ * Queue lock is held here. ++ */ ++ bic = bfq_bic_lookup(bfqd, current->io_context); ++ if (!bic) ++ return false; ++ ++ bfqq = bic_to_bfqq(bic, bfq_bio_sync(bio)); ++ /* ++ * We take advantage of this function to perform an early merge ++ * of the queues of possible cooperating processes. ++ */ ++ if (bfqq) { ++ new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false); ++ if (new_bfqq) { ++ bfq_merge_bfqqs(bfqd, bic, bfqq, new_bfqq); ++ /* ++ * If we get here, the bio will be queued in the ++ * shared queue, i.e., new_bfqq, so use new_bfqq ++ * to decide whether bio and rq can be merged. ++ */ ++ bfqq = new_bfqq; ++ } ++ } ++ ++ return bfqq == RQ_BFQQ(rq); ++} ++ ++static int bfq_allow_rq_merge(struct request_queue *q, struct request *rq, ++ struct request *next) ++{ ++ return RQ_BFQQ(rq) == RQ_BFQQ(next); ++} ++ ++/* ++ * Set the maximum time for the in-service queue to consume its ++ * budget. This prevents seeky processes from lowering the throughput. ++ * In practice, a time-slice service scheme is used with seeky ++ * processes. ++ */ ++static void bfq_set_budget_timeout(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ unsigned int timeout_coeff; ++ ++ if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) ++ timeout_coeff = 1; ++ else ++ timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight; ++ ++ bfqd->last_budget_start = ktime_get(); ++ ++ bfqq->budget_timeout = jiffies + ++ bfqd->bfq_timeout * timeout_coeff; ++ ++ bfq_log_bfqq(bfqd, bfqq, "set budget_timeout %u", ++ jiffies_to_msecs(bfqd->bfq_timeout * timeout_coeff)); ++} ++ ++static void __bfq_set_in_service_queue(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ if (bfqq) { ++ bfqg_stats_update_avg_queue_size(bfqq_group(bfqq)); ++ bfq_mark_bfqq_must_alloc(bfqq); ++ bfq_clear_bfqq_fifo_expire(bfqq); ++ ++ bfqd->budgets_assigned = (bfqd->budgets_assigned*7 + 256) / 8; ++ ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list)); ++ ++ if (time_is_before_jiffies(bfqq->last_wr_start_finish) && ++ bfqq->wr_coeff > 1 && ++ bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && ++ time_is_before_jiffies(bfqq->budget_timeout)) { ++ /* ++ * For soft real-time queues, move the start ++ * of the weight-raising period forward by the ++ * time the queue has not received any ++ * service. Otherwise, a relatively long ++ * service delay is likely to cause the ++ * weight-raising period of the queue to end, ++ * because of the short duration of the ++ * weight-raising period of a soft real-time ++ * queue. It is worth noting that this move ++ * is not so dangerous for the other queues, ++ * because soft real-time queues are not ++ * greedy. ++ * ++ * To not add a further variable, we use the ++ * overloaded field budget_timeout to ++ * determine for how long the queue has not ++ * received service, i.e., how much time has ++ * elapsed since the queue expired. However, ++ * this is a little imprecise, because ++ * budget_timeout is set to jiffies if bfqq ++ * not only expires, but also remains with no ++ * request. ++ */ ++ bfqq->last_wr_start_finish += jiffies - ++ max_t(unsigned long, bfqq->last_wr_start_finish, ++ bfqq->budget_timeout); ++ if (time_is_after_jiffies(bfqq->last_wr_start_finish)) { ++ pr_crit( ++ "BFQ WARNING:last %lu budget %lu jiffies %lu", ++ bfqq->last_wr_start_finish, ++ bfqq->budget_timeout, ++ jiffies); ++ pr_crit("diff %lu", jiffies - ++ max_t(unsigned long, ++ bfqq->last_wr_start_finish, ++ bfqq->budget_timeout)); ++ bfqq->last_wr_start_finish = jiffies; ++ } ++ } ++ ++ bfq_set_budget_timeout(bfqd, bfqq); ++ bfq_log_bfqq(bfqd, bfqq, ++ "set_in_service_queue, cur-budget = %d", ++ bfqq->entity.budget); ++ } else ++ bfq_log(bfqd, "set_in_service_queue: NULL"); ++ ++ bfqd->in_service_queue = bfqq; ++} ++ ++/* ++ * Get and set a new queue for service. ++ */ ++static struct bfq_queue *bfq_set_in_service_queue(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); ++ ++ __bfq_set_in_service_queue(bfqd, bfqq); ++ return bfqq; ++} ++ ++static void bfq_arm_slice_timer(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq = bfqd->in_service_queue; ++ struct bfq_io_cq *bic; ++ u32 sl; ++ ++ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list)); ++ ++ /* Processes have exited, don't wait. */ ++ bic = bfqd->in_service_bic; ++ if (!bic || atomic_read(&bic->icq.ioc->active_ref) == 0) ++ return; ++ ++ bfq_mark_bfqq_wait_request(bfqq); ++ ++ /* ++ * We don't want to idle for seeks, but we do want to allow ++ * fair distribution of slice time for a process doing back-to-back ++ * seeks. So allow a little bit of time for him to submit a new rq. ++ * ++ * To prevent processes with (partly) seeky workloads from ++ * being too ill-treated, grant them a small fraction of the ++ * assigned budget before reducing the waiting time to ++ * BFQ_MIN_TT. This happened to help reduce latency. ++ */ ++ sl = bfqd->bfq_slice_idle; ++ /* ++ * Unless the queue is being weight-raised or the scenario is ++ * asymmetric, grant only minimum idle time if the queue ++ * is seeky. A long idling is preserved for a weight-raised ++ * queue, or, more in general, in an asymemtric scenario, ++ * because a long idling is needed for guaranteeing to a queue ++ * its reserved share of the throughput (in particular, it is ++ * needed if the queue has a higher weight than some other ++ * queue). ++ */ ++ if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && ++ bfq_symmetric_scenario(bfqd)) ++ sl = min_t(u32, sl, BFQ_MIN_TT); ++ ++ bfqd->last_idling_start = ktime_get(); ++ hrtimer_start(&bfqd->idle_slice_timer, ns_to_ktime(sl), ++ HRTIMER_MODE_REL); ++ bfqg_stats_set_start_idle_time(bfqq_group(bfqq)); ++ bfq_log(bfqd, "arm idle: %ld/%ld ms", ++ sl / NSEC_PER_MSEC, bfqd->bfq_slice_idle / NSEC_PER_MSEC); ++} ++ ++/* ++ * In autotuning mode, max_budget is dynamically recomputed as the ++ * amount of sectors transferred in timeout at the estimated peak ++ * rate. This enables BFQ to utilize a full timeslice with a full ++ * budget, even if the in-service queue is served at peak rate. And ++ * this maximises throughput with sequential workloads. ++ */ ++static unsigned long bfq_calc_max_budget(struct bfq_data *bfqd) ++{ ++ return (u64)bfqd->peak_rate * USEC_PER_MSEC * ++ jiffies_to_msecs(bfqd->bfq_timeout)>>BFQ_RATE_SHIFT; ++} ++ ++/* ++ * Update parameters related to throughput and responsiveness, as a ++ * function of the estimated peak rate. See comments on ++ * bfq_calc_max_budget(), and on T_slow and T_fast arrays. ++ */ ++void update_thr_responsiveness_params(struct bfq_data *bfqd) ++{ ++ int dev_type = blk_queue_nonrot(bfqd->queue); ++ ++ if (bfqd->bfq_user_max_budget == 0) { ++ bfqd->bfq_max_budget = ++ bfq_calc_max_budget(bfqd); ++ BUG_ON(bfqd->bfq_max_budget < 0); ++ bfq_log(bfqd, "new max_budget = %d", ++ bfqd->bfq_max_budget); ++ } ++ ++ if (bfqd->device_speed == BFQ_BFQD_FAST && ++ bfqd->peak_rate < device_speed_thresh[dev_type]) { ++ bfqd->device_speed = BFQ_BFQD_SLOW; ++ bfqd->RT_prod = R_slow[dev_type] * ++ T_slow[dev_type]; ++ } else if (bfqd->device_speed == BFQ_BFQD_SLOW && ++ bfqd->peak_rate > device_speed_thresh[dev_type]) { ++ bfqd->device_speed = BFQ_BFQD_FAST; ++ bfqd->RT_prod = R_fast[dev_type] * ++ T_fast[dev_type]; ++ } ++ ++ bfq_log(bfqd, ++"dev_type %s dev_speed_class = %s (%llu sects/sec), thresh %llu setcs/sec", ++ dev_type == 0 ? "ROT" : "NONROT", ++ bfqd->device_speed == BFQ_BFQD_FAST ? "FAST" : "SLOW", ++ bfqd->device_speed == BFQ_BFQD_FAST ? ++ (USEC_PER_SEC*(u64)R_fast[dev_type])>>BFQ_RATE_SHIFT : ++ (USEC_PER_SEC*(u64)R_slow[dev_type])>>BFQ_RATE_SHIFT, ++ (USEC_PER_SEC*(u64)device_speed_thresh[dev_type])>> ++ BFQ_RATE_SHIFT); ++} ++ ++void bfq_reset_rate_computation(struct bfq_data *bfqd, struct request *rq) ++{ ++ if (rq != NULL) { /* new rq dispatch now, reset accordingly */ ++ bfqd->last_dispatch = bfqd->first_dispatch = ktime_get_ns() ; ++ bfqd->peak_rate_samples = 1; ++ bfqd->sequential_samples = 0; ++ bfqd->tot_sectors_dispatched = bfqd->last_rq_max_size = ++ blk_rq_sectors(rq); ++ } else /* no new rq dispatched, just reset the number of samples */ ++ bfqd->peak_rate_samples = 0; /* full re-init on next disp. */ ++ ++ bfq_log(bfqd, ++ "reset_rate_computation at end, sample %u/%u tot_sects %llu", ++ bfqd->peak_rate_samples, bfqd->sequential_samples, ++ bfqd->tot_sectors_dispatched); ++} ++ ++void bfq_update_rate_reset(struct bfq_data *bfqd, struct request *rq) ++{ ++ u32 rate, weight, divisor; ++ ++ /* ++ * For the convergence property to hold (see comments on ++ * bfq_update_peak_rate()) and for the assessment to be ++ * reliable, a minimum number of samples must be present, and ++ * a minimum amount of time must have elapsed. If not so, do ++ * not compute new rate. Just reset parameters, to get ready ++ * for a new evaluation attempt. ++ */ ++ if (bfqd->peak_rate_samples < BFQ_RATE_MIN_SAMPLES || ++ bfqd->delta_from_first < BFQ_RATE_MIN_INTERVAL) { ++ bfq_log(bfqd, ++ "update_rate_reset: only resetting, delta_first %lluus samples %d", ++ bfqd->delta_from_first>>10, bfqd->peak_rate_samples); ++ goto reset_computation; ++ } ++ ++ /* ++ * If a new request completion has occurred after last ++ * dispatch, then, to approximate the rate at which requests ++ * have been served by the device, it is more precise to ++ * extend the observation interval to the last completion. ++ */ ++ bfqd->delta_from_first = ++ max_t(u64, bfqd->delta_from_first, ++ bfqd->last_completion - bfqd->first_dispatch); ++ ++ BUG_ON(bfqd->delta_from_first == 0); ++ /* ++ * Rate computed in sects/usec, and not sects/nsec, for ++ * precision issues. ++ */ ++ rate = div64_ul(bfqd->tot_sectors_dispatched<delta_from_first, NSEC_PER_USEC)); ++ ++ bfq_log(bfqd, ++"update_rate_reset: tot_sects %llu delta_first %lluus rate %llu sects/s (%d)", ++ bfqd->tot_sectors_dispatched, bfqd->delta_from_first>>10, ++ ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), ++ rate > 20< 20M sectors/sec) ++ */ ++ if ((bfqd->peak_rate_samples > (3 * bfqd->sequential_samples)>>2 && ++ rate <= bfqd->peak_rate) || ++ rate > 20<peak_rate_samples, bfqd->sequential_samples, ++ ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), ++ ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); ++ goto reset_computation; ++ } else { ++ bfq_log(bfqd, ++ "update_rate_reset: do update, samples %u/%u rate/peak %llu/%llu", ++ bfqd->peak_rate_samples, bfqd->sequential_samples, ++ ((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT), ++ ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); ++ } ++ ++ /* ++ * We have to update the peak rate, at last! To this purpose, ++ * we use a low-pass filter. We compute the smoothing constant ++ * of the filter as a function of the 'weight' of the new ++ * measured rate. ++ * ++ * As can be seen in next formulas, we define this weight as a ++ * quantity proportional to how sequential the workload is, ++ * and to how long the observation time interval is. ++ * ++ * The weight runs from 0 to 8. The maximum value of the ++ * weight, 8, yields the minimum value for the smoothing ++ * constant. At this minimum value for the smoothing constant, ++ * the measured rate contributes for half of the next value of ++ * the estimated peak rate. ++ * ++ * So, the first step is to compute the weight as a function ++ * of how sequential the workload is. Note that the weight ++ * cannot reach 9, because bfqd->sequential_samples cannot ++ * become equal to bfqd->peak_rate_samples, which, in its ++ * turn, holds true because bfqd->sequential_samples is not ++ * incremented for the first sample. ++ */ ++ weight = (9 * bfqd->sequential_samples) / bfqd->peak_rate_samples; ++ ++ /* ++ * Second step: further refine the weight as a function of the ++ * duration of the observation interval. ++ */ ++ weight = min_t(u32, 8, ++ div_u64(weight * bfqd->delta_from_first, ++ BFQ_RATE_REF_INTERVAL)); ++ ++ /* ++ * Divisor ranging from 10, for minimum weight, to 2, for ++ * maximum weight. ++ */ ++ divisor = 10 - weight; ++ BUG_ON(divisor == 0); ++ ++ /* ++ * Finally, update peak rate: ++ * ++ * peak_rate = peak_rate * (divisor-1) / divisor + rate / divisor ++ */ ++ bfqd->peak_rate *= divisor-1; ++ bfqd->peak_rate /= divisor; ++ rate /= divisor; /* smoothing constant alpha = 1/divisor */ ++ ++ bfq_log(bfqd, ++ "update_rate_reset: divisor %d tmp_peak_rate %llu tmp_rate %u", ++ divisor, ++ ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT), ++ (u32)((USEC_PER_SEC*(u64)rate)>>BFQ_RATE_SHIFT)); ++ ++ BUG_ON(bfqd->peak_rate == 0); ++ BUG_ON(bfqd->peak_rate > 20<peak_rate += rate; ++ update_thr_responsiveness_params(bfqd); ++ BUG_ON(bfqd->peak_rate > 20<peak_rate_samples == 0) { /* first dispatch */ ++ bfq_log(bfqd, ++ "update_peak_rate: goto reset, samples %d", ++ bfqd->peak_rate_samples) ; ++ bfq_reset_rate_computation(bfqd, rq); ++ goto update_last_values; /* will add one sample */ ++ } ++ ++ /* ++ * Device idle for very long: the observation interval lasting ++ * up to this dispatch cannot be a valid observation interval ++ * for computing a new peak rate (similarly to the late- ++ * completion event in bfq_completed_request()). Go to ++ * update_rate_and_reset to have the following three steps ++ * taken: ++ * - close the observation interval at the last (previous) ++ * request dispatch or completion ++ * - compute rate, if possible, for that observation interval ++ * - start a new observation interval with this dispatch ++ */ ++ if (now_ns - bfqd->last_dispatch > 100*NSEC_PER_MSEC && ++ bfqd->rq_in_driver == 0) { ++ bfq_log(bfqd, ++"update_peak_rate: jumping to updating&resetting delta_last %lluus samples %d", ++ (now_ns - bfqd->last_dispatch)>>10, ++ bfqd->peak_rate_samples) ; ++ goto update_rate_and_reset; ++ } ++ ++ /* Update sampling information */ ++ bfqd->peak_rate_samples++; ++ ++ if ((bfqd->rq_in_driver > 0 || ++ now_ns - bfqd->last_completion < BFQ_MIN_TT) ++ && get_sdist(bfqd->last_position, rq) < BFQQ_SEEK_THR) ++ bfqd->sequential_samples++; ++ ++ bfqd->tot_sectors_dispatched += blk_rq_sectors(rq); ++ ++ /* Reset max observed rq size every 32 dispatches */ ++ if (likely(bfqd->peak_rate_samples % 32)) ++ bfqd->last_rq_max_size = ++ max_t(u32, blk_rq_sectors(rq), bfqd->last_rq_max_size); ++ else ++ bfqd->last_rq_max_size = blk_rq_sectors(rq); ++ ++ bfqd->delta_from_first = now_ns - bfqd->first_dispatch; ++ ++ bfq_log(bfqd, ++ "update_peak_rate: added samples %u/%u tot_sects %llu delta_first %lluus", ++ bfqd->peak_rate_samples, bfqd->sequential_samples, ++ bfqd->tot_sectors_dispatched, ++ bfqd->delta_from_first>>10); ++ ++ /* Target observation interval not yet reached, go on sampling */ ++ if (bfqd->delta_from_first < BFQ_RATE_REF_INTERVAL) ++ goto update_last_values; ++ ++update_rate_and_reset: ++ bfq_update_rate_reset(bfqd, rq); ++update_last_values: ++ bfqd->last_position = blk_rq_pos(rq) + blk_rq_sectors(rq); ++ bfqd->last_dispatch = now_ns; ++ ++ bfq_log(bfqd, ++ "update_peak_rate: delta_first %lluus last_pos %llu peak_rate %llu", ++ (now_ns - bfqd->first_dispatch)>>10, ++ (unsigned long long) bfqd->last_position, ++ ((USEC_PER_SEC*(u64)bfqd->peak_rate)>>BFQ_RATE_SHIFT)); ++ bfq_log(bfqd, ++ "update_peak_rate: samples at end %d", bfqd->peak_rate_samples); ++} ++ ++/* ++ * Move request from internal lists to the dispatch list of the request queue ++ */ ++static void bfq_dispatch_insert(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ ++ /* ++ * For consistency, the next instruction should have been executed ++ * after removing the request from the queue and dispatching it. ++ * We execute instead this instruction before bfq_remove_request() ++ * (and hence introduce a temporary inconsistency), for efficiency. ++ * In fact, in a forced_dispatch, this prevents two counters related ++ * to bfqq->dispatched to risk to be uselessly decremented if bfqq ++ * is not in service, and then to be incremented again after ++ * incrementing bfqq->dispatched. ++ */ ++ bfqq->dispatched++; ++ bfq_update_peak_rate(q->elevator->elevator_data, rq); ++ ++ bfq_remove_request(rq); ++ elv_dispatch_sort(q, rq); ++} ++ ++/* ++ * Return expired entry, or NULL to just start from scratch in rbtree. ++ */ ++static struct request *bfq_check_fifo(struct bfq_queue *bfqq) ++{ ++ struct request *rq = NULL; ++ ++ if (bfq_bfqq_fifo_expire(bfqq)) ++ return NULL; ++ ++ bfq_mark_bfqq_fifo_expire(bfqq); ++ ++ if (list_empty(&bfqq->fifo)) ++ return NULL; ++ ++ rq = rq_entry_fifo(bfqq->fifo.next); ++ ++ if (ktime_get_ns() < rq->fifo_time) ++ return NULL; ++ ++ return rq; ++} ++ ++static void __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ BUG_ON(bfqq != bfqd->in_service_queue); ++ ++ __bfq_bfqd_reset_in_service(bfqd); ++ ++ /* ++ * If this bfqq is shared between multiple processes, check ++ * to make sure that those processes are still issuing I/Os ++ * within the mean seek distance. If not, it may be time to ++ * break the queues apart again. ++ */ ++ if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq)) ++ bfq_mark_bfqq_split_coop(bfqq); ++ ++ if (RB_EMPTY_ROOT(&bfqq->sort_list)) { ++ if (bfqq->dispatched == 0) ++ /* ++ * Overloading budget_timeout field to store ++ * the time at which the queue remains with no ++ * backlog and no outstanding request; used by ++ * the weight-raising mechanism. ++ */ ++ bfqq->budget_timeout = jiffies; ++ ++ bfq_del_bfqq_busy(bfqd, bfqq, 1); ++ } else { ++ bfq_activate_bfqq(bfqd, bfqq); ++ /* ++ * Resort priority tree of potential close cooperators. ++ */ ++ bfq_pos_tree_add_move(bfqd, bfqq); ++ } ++} ++ ++/** ++ * __bfq_bfqq_recalc_budget - try to adapt the budget to the @bfqq behavior. ++ * @bfqd: device data. ++ * @bfqq: queue to update. ++ * @reason: reason for expiration. ++ * ++ * Handle the feedback on @bfqq budget at queue expiration. ++ * See the body for detailed comments. ++ */ ++static void __bfq_bfqq_recalc_budget(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ enum bfqq_expiration reason) ++{ ++ struct request *next_rq; ++ int budget, min_budget; ++ ++ BUG_ON(bfqq != bfqd->in_service_queue); ++ ++ min_budget = bfq_min_budget(bfqd); ++ ++ if (bfqq->wr_coeff == 1) ++ budget = bfqq->max_budget; ++ else /* ++ * Use a constant, low budget for weight-raised queues, ++ * to help achieve a low latency. Keep it slightly higher ++ * than the minimum possible budget, to cause a little ++ * bit fewer expirations. ++ */ ++ budget = 2 * min_budget; ++ ++ bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d", ++ bfqq->entity.budget, bfq_bfqq_budget_left(bfqq)); ++ bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d", ++ budget, bfq_min_budget(bfqd)); ++ bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", ++ bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); ++ ++ if (bfq_bfqq_sync(bfqq) && bfqq->wr_coeff == 1) { ++ switch (reason) { ++ /* ++ * Caveat: in all the following cases we trade latency ++ * for throughput. ++ */ ++ case BFQ_BFQQ_TOO_IDLE: ++ /* ++ * This is the only case where we may reduce ++ * the budget: if there is no request of the ++ * process still waiting for completion, then ++ * we assume (tentatively) that the timer has ++ * expired because the batch of requests of ++ * the process could have been served with a ++ * smaller budget. Hence, betting that ++ * process will behave in the same way when it ++ * becomes backlogged again, we reduce its ++ * next budget. As long as we guess right, ++ * this budget cut reduces the latency ++ * experienced by the process. ++ * ++ * However, if there are still outstanding ++ * requests, then the process may have not yet ++ * issued its next request just because it is ++ * still waiting for the completion of some of ++ * the still outstanding ones. So in this ++ * subcase we do not reduce its budget, on the ++ * contrary we increase it to possibly boost ++ * the throughput, as discussed in the ++ * comments to the BUDGET_TIMEOUT case. ++ */ ++ if (bfqq->dispatched > 0) /* still outstanding reqs */ ++ budget = min(budget * 2, bfqd->bfq_max_budget); ++ else { ++ if (budget > 5 * min_budget) ++ budget -= 4 * min_budget; ++ else ++ budget = min_budget; ++ } ++ break; ++ case BFQ_BFQQ_BUDGET_TIMEOUT: ++ /* ++ * We double the budget here because it gives ++ * the chance to boost the throughput if this ++ * is not a seeky process (and has bumped into ++ * this timeout because of, e.g., ZBR). ++ */ ++ budget = min(budget * 2, bfqd->bfq_max_budget); ++ break; ++ case BFQ_BFQQ_BUDGET_EXHAUSTED: ++ /* ++ * The process still has backlog, and did not ++ * let either the budget timeout or the disk ++ * idling timeout expire. Hence it is not ++ * seeky, has a short thinktime and may be ++ * happy with a higher budget too. So ++ * definitely increase the budget of this good ++ * candidate to boost the disk throughput. ++ */ ++ budget = min(budget * 4, bfqd->bfq_max_budget); ++ break; ++ case BFQ_BFQQ_NO_MORE_REQUESTS: ++ /* ++ * For queues that expire for this reason, it ++ * is particularly important to keep the ++ * budget close to the actual service they ++ * need. Doing so reduces the timestamp ++ * misalignment problem described in the ++ * comments in the body of ++ * __bfq_activate_entity. In fact, suppose ++ * that a queue systematically expires for ++ * BFQ_BFQQ_NO_MORE_REQUESTS and presents a ++ * new request in time to enjoy timestamp ++ * back-shifting. The larger the budget of the ++ * queue is with respect to the service the ++ * queue actually requests in each service ++ * slot, the more times the queue can be ++ * reactivated with the same virtual finish ++ * time. It follows that, even if this finish ++ * time is pushed to the system virtual time ++ * to reduce the consequent timestamp ++ * misalignment, the queue unjustly enjoys for ++ * many re-activations a lower finish time ++ * than all newly activated queues. ++ * ++ * The service needed by bfqq is measured ++ * quite precisely by bfqq->entity.service. ++ * Since bfqq does not enjoy device idling, ++ * bfqq->entity.service is equal to the number ++ * of sectors that the process associated with ++ * bfqq requested to read/write before waiting ++ * for request completions, or blocking for ++ * other reasons. ++ */ ++ budget = max_t(int, bfqq->entity.service, min_budget); ++ break; ++ default: ++ return; ++ } ++ } else if (!bfq_bfqq_sync(bfqq)) ++ /* ++ * Async queues get always the maximum possible ++ * budget, as for them we do not care about latency ++ * (in addition, their ability to dispatch is limited ++ * by the charging factor). ++ */ ++ budget = bfqd->bfq_max_budget; ++ ++ bfqq->max_budget = budget; ++ ++ if (bfqd->budgets_assigned >= bfq_stats_min_budgets && ++ !bfqd->bfq_user_max_budget) ++ bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget); ++ ++ /* ++ * If there is still backlog, then assign a new budget, making ++ * sure that it is large enough for the next request. Since ++ * the finish time of bfqq must be kept in sync with the ++ * budget, be sure to call __bfq_bfqq_expire() *after* this ++ * update. ++ * ++ * If there is no backlog, then no need to update the budget; ++ * it will be updated on the arrival of a new request. ++ */ ++ next_rq = bfqq->next_rq; ++ if (next_rq) { ++ BUG_ON(reason == BFQ_BFQQ_TOO_IDLE || ++ reason == BFQ_BFQQ_NO_MORE_REQUESTS); ++ bfqq->entity.budget = max_t(unsigned long, bfqq->max_budget, ++ bfq_serv_to_charge(next_rq, bfqq)); ++ BUG_ON(!bfq_bfqq_busy(bfqq)); ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list)); ++ } ++ ++ bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %d", ++ next_rq ? blk_rq_sectors(next_rq) : 0, ++ bfqq->entity.budget); ++} ++ ++/* ++ * Return true if the process associated with bfqq is "slow". The slow ++ * flag is used, in addition to the budget timeout, to reduce the ++ * amount of service provided to seeky processes, and thus reduce ++ * their chances to lower the throughput. More details in the comments ++ * on the function bfq_bfqq_expire(). ++ * ++ * An important observation is in order: as discussed in the comments ++ * on the function bfq_update_peak_rate(), with devices with internal ++ * queues, it is hard if ever possible to know when and for how long ++ * an I/O request is processed by the device (apart from the trivial ++ * I/O pattern where a new request is dispatched only after the ++ * previous one has been completed). This makes it hard to evaluate ++ * the real rate at which the I/O requests of each bfq_queue are ++ * served. In fact, for an I/O scheduler like BFQ, serving a ++ * bfq_queue means just dispatching its requests during its service ++ * slot (i.e., until the budget of the queue is exhausted, or the ++ * queue remains idle, or, finally, a timeout fires). But, during the ++ * service slot of a bfq_queue, around 100 ms at most, the device may ++ * be even still processing requests of bfq_queues served in previous ++ * service slots. On the opposite end, the requests of the in-service ++ * bfq_queue may be completed after the service slot of the queue ++ * finishes. ++ * ++ * Anyway, unless more sophisticated solutions are used ++ * (where possible), the sum of the sizes of the requests dispatched ++ * during the service slot of a bfq_queue is probably the only ++ * approximation available for the service received by the bfq_queue ++ * during its service slot. And this sum is the quantity used in this ++ * function to evaluate the I/O speed of a process. ++ */ ++static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ bool compensate, enum bfqq_expiration reason, ++ unsigned long *delta_ms) ++{ ++ ktime_t delta_ktime; ++ u32 delta_usecs; ++ bool slow = BFQQ_SEEKY(bfqq); /* if delta too short, use seekyness */ ++ ++ if (!bfq_bfqq_sync(bfqq)) ++ return false; ++ ++ if (compensate) ++ delta_ktime = bfqd->last_idling_start; ++ else ++ delta_ktime = ktime_get(); ++ delta_ktime = ktime_sub(delta_ktime, bfqd->last_budget_start); ++ delta_usecs = ktime_to_us(delta_ktime); ++ ++ /* don't trust short/unrealistic values. */ ++ if (delta_usecs < 1000 || delta_usecs >= LONG_MAX) { ++ if (blk_queue_nonrot(bfqd->queue)) ++ /* ++ * give same worst-case guarantees as idling ++ * for seeky ++ */ ++ *delta_ms = BFQ_MIN_TT / NSEC_PER_MSEC; ++ else /* charge at least one seek */ ++ *delta_ms = bfq_slice_idle / NSEC_PER_MSEC; ++ ++ bfq_log(bfqd, "bfq_bfqq_is_slow: unrealistic %u", delta_usecs); ++ ++ return slow; ++ } ++ ++ *delta_ms = delta_usecs / USEC_PER_MSEC; ++ ++ /* ++ * Use only long (> 20ms) intervals to filter out excessive ++ * spikes in service rate estimation. ++ */ ++ if (delta_usecs > 20000) { ++ /* ++ * Caveat for rotational devices: processes doing I/O ++ * in the slower disk zones tend to be slow(er) even ++ * if not seeky. In this respect, the estimated peak ++ * rate is likely to be an average over the disk ++ * surface. Accordingly, to not be too harsh with ++ * unlucky processes, a process is deemed slow only if ++ * its rate has been lower than half of the estimated ++ * peak rate. ++ */ ++ slow = bfqq->entity.service < bfqd->bfq_max_budget / 2; ++ bfq_log(bfqd, "bfq_bfqq_is_slow: relative rate %d/%d", ++ bfqq->entity.service, bfqd->bfq_max_budget); ++ } ++ ++ bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow); ++ ++ return slow; ++} ++ ++/* ++ * To be deemed as soft real-time, an application must meet two ++ * requirements. First, the application must not require an average ++ * bandwidth higher than the approximate bandwidth required to playback or ++ * record a compressed high-definition video. ++ * The next function is invoked on the completion of the last request of a ++ * batch, to compute the next-start time instant, soft_rt_next_start, such ++ * that, if the next request of the application does not arrive before ++ * soft_rt_next_start, then the above requirement on the bandwidth is met. ++ * ++ * The second requirement is that the request pattern of the application is ++ * isochronous, i.e., that, after issuing a request or a batch of requests, ++ * the application stops issuing new requests until all its pending requests ++ * have been completed. After that, the application may issue a new batch, ++ * and so on. ++ * For this reason the next function is invoked to compute ++ * soft_rt_next_start only for applications that meet this requirement, ++ * whereas soft_rt_next_start is set to infinity for applications that do ++ * not. ++ * ++ * Unfortunately, even a greedy application may happen to behave in an ++ * isochronous way if the CPU load is high. In fact, the application may ++ * stop issuing requests while the CPUs are busy serving other processes, ++ * then restart, then stop again for a while, and so on. In addition, if ++ * the disk achieves a low enough throughput with the request pattern ++ * issued by the application (e.g., because the request pattern is random ++ * and/or the device is slow), then the application may meet the above ++ * bandwidth requirement too. To prevent such a greedy application to be ++ * deemed as soft real-time, a further rule is used in the computation of ++ * soft_rt_next_start: soft_rt_next_start must be higher than the current ++ * time plus the maximum time for which the arrival of a request is waited ++ * for when a sync queue becomes idle, namely bfqd->bfq_slice_idle. ++ * This filters out greedy applications, as the latter issue instead their ++ * next request as soon as possible after the last one has been completed ++ * (in contrast, when a batch of requests is completed, a soft real-time ++ * application spends some time processing data). ++ * ++ * Unfortunately, the last filter may easily generate false positives if ++ * only bfqd->bfq_slice_idle is used as a reference time interval and one ++ * or both the following cases occur: ++ * 1) HZ is so low that the duration of a jiffy is comparable to or higher ++ * than bfqd->bfq_slice_idle. This happens, e.g., on slow devices with ++ * HZ=100. ++ * 2) jiffies, instead of increasing at a constant rate, may stop increasing ++ * for a while, then suddenly 'jump' by several units to recover the lost ++ * increments. This seems to happen, e.g., inside virtual machines. ++ * To address this issue, we do not use as a reference time interval just ++ * bfqd->bfq_slice_idle, but bfqd->bfq_slice_idle plus a few jiffies. In ++ * particular we add the minimum number of jiffies for which the filter ++ * seems to be quite precise also in embedded systems and KVM/QEMU virtual ++ * machines. ++ */ ++static unsigned long bfq_bfqq_softrt_next_start(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ bfq_log_bfqq(bfqd, bfqq, ++"softrt_next_start: service_blkg %lu soft_rate %u sects/sec interval %u", ++ bfqq->service_from_backlogged, ++ bfqd->bfq_wr_max_softrt_rate, ++ jiffies_to_msecs(HZ * bfqq->service_from_backlogged / ++ bfqd->bfq_wr_max_softrt_rate)); ++ ++ return max(bfqq->last_idle_bklogged + ++ HZ * bfqq->service_from_backlogged / ++ bfqd->bfq_wr_max_softrt_rate, ++ jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); ++} ++ ++/* ++ * Return the farthest future time instant according to jiffies ++ * macros. ++ */ ++static unsigned long bfq_greatest_from_now(void) ++{ ++ return jiffies + MAX_JIFFY_OFFSET; ++} ++ ++/* ++ * Return the farthest past time instant according to jiffies ++ * macros. ++ */ ++static unsigned long bfq_smallest_from_now(void) ++{ ++ return jiffies - MAX_JIFFY_OFFSET; ++} ++ ++/** ++ * bfq_bfqq_expire - expire a queue. ++ * @bfqd: device owning the queue. ++ * @bfqq: the queue to expire. ++ * @compensate: if true, compensate for the time spent idling. ++ * @reason: the reason causing the expiration. ++ * ++ * If the process associated with bfqq does slow I/O (e.g., because it ++ * issues random requests), we charge bfqq with the time it has been ++ * in service instead of the service it has received (see ++ * bfq_bfqq_charge_time for details on how this goal is achieved). As ++ * a consequence, bfqq will typically get higher timestamps upon ++ * reactivation, and hence it will be rescheduled as if it had ++ * received more service than what it has actually received. In the ++ * end, bfqq receives less service in proportion to how slowly its ++ * associated process consumes its budgets (and hence how seriously it ++ * tends to lower the throughput). In addition, this time-charging ++ * strategy guarantees time fairness among slow processes. In ++ * contrast, if the process associated with bfqq is not slow, we ++ * charge bfqq exactly with the service it has received. ++ * ++ * Charging time to the first type of queues and the exact service to ++ * the other has the effect of using the WF2Q+ policy to schedule the ++ * former on a timeslice basis, without violating service domain ++ * guarantees among the latter. ++ */ ++static void bfq_bfqq_expire(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ bool compensate, ++ enum bfqq_expiration reason) ++{ ++ bool slow; ++ unsigned long delta = 0; ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ BUG_ON(bfqq != bfqd->in_service_queue); ++ ++ /* ++ * Check whether the process is slow (see bfq_bfqq_is_slow). ++ */ ++ slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, reason, &delta); ++ ++ /* ++ * Increase service_from_backlogged before next statement, ++ * because the possible next invocation of ++ * bfq_bfqq_charge_time would likely inflate ++ * entity->service. In contrast, service_from_backlogged must ++ * contain real service, to enable the soft real-time ++ * heuristic to correctly compute the bandwidth consumed by ++ * bfqq. ++ */ ++ bfqq->service_from_backlogged += entity->service; ++ ++ /* ++ * As above explained, charge slow (typically seeky) and ++ * timed-out queues with the time and not the service ++ * received, to favor sequential workloads. ++ * ++ * Processes doing I/O in the slower disk zones will tend to ++ * be slow(er) even if not seeky. Therefore, since the ++ * estimated peak rate is actually an average over the disk ++ * surface, these processes may timeout just for bad luck. To ++ * avoid punishing them, do not charge time to processes that ++ * succeeded in consuming at least 2/3 of their budget. This ++ * allows BFQ to preserve enough elasticity to still perform ++ * bandwidth, and not time, distribution with little unlucky ++ * or quasi-sequential processes. ++ */ ++ if (bfqq->wr_coeff == 1 && ++ (slow || ++ (reason == BFQ_BFQQ_BUDGET_TIMEOUT && ++ bfq_bfqq_budget_left(bfqq) >= entity->budget / 3))) ++ bfq_bfqq_charge_time(bfqd, bfqq, delta); ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ ++ if (reason == BFQ_BFQQ_TOO_IDLE && ++ entity->service <= 2 * entity->budget / 10) ++ bfq_clear_bfqq_IO_bound(bfqq); ++ ++ if (bfqd->low_latency && bfqq->wr_coeff == 1) ++ bfqq->last_wr_start_finish = jiffies; ++ ++ if (bfqd->low_latency && bfqd->bfq_wr_max_softrt_rate > 0 && ++ RB_EMPTY_ROOT(&bfqq->sort_list)) { ++ /* ++ * If we get here, and there are no outstanding ++ * requests, then the request pattern is isochronous ++ * (see the comments on the function ++ * bfq_bfqq_softrt_next_start()). Thus we can compute ++ * soft_rt_next_start. If, instead, the queue still ++ * has outstanding requests, then we have to wait for ++ * the completion of all the outstanding requests to ++ * discover whether the request pattern is actually ++ * isochronous. ++ */ ++ BUG_ON(bfqd->busy_queues < 1); ++ if (bfqq->dispatched == 0) { ++ bfqq->soft_rt_next_start = ++ bfq_bfqq_softrt_next_start(bfqd, bfqq); ++ bfq_log_bfqq(bfqd, bfqq, "new soft_rt_next %lu", ++ bfqq->soft_rt_next_start); ++ } else { ++ /* ++ * The application is still waiting for the ++ * completion of one or more requests: ++ * prevent it from possibly being incorrectly ++ * deemed as soft real-time by setting its ++ * soft_rt_next_start to infinity. In fact, ++ * without this assignment, the application ++ * would be incorrectly deemed as soft ++ * real-time if: ++ * 1) it issued a new request before the ++ * completion of all its in-flight ++ * requests, and ++ * 2) at that time, its soft_rt_next_start ++ * happened to be in the past. ++ */ ++ bfqq->soft_rt_next_start = ++ bfq_greatest_from_now(); ++ /* ++ * Schedule an update of soft_rt_next_start to when ++ * the task may be discovered to be isochronous. ++ */ ++ bfq_mark_bfqq_softrt_update(bfqq); ++ } ++ } ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "expire (%d, slow %d, num_disp %d, idle_win %d, weight %d)", ++ reason, slow, bfqq->dispatched, ++ bfq_bfqq_idle_window(bfqq), entity->weight); ++ ++ /* ++ * Increase, decrease or leave budget unchanged according to ++ * reason. ++ */ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ __bfq_bfqq_recalc_budget(bfqd, bfqq, reason); ++ BUG_ON(bfqq->next_rq == NULL && ++ bfqq->entity.budget < bfqq->entity.service); ++ __bfq_bfqq_expire(bfqd, bfqq); ++ ++ BUG_ON(!bfq_bfqq_busy(bfqq) && reason == BFQ_BFQQ_BUDGET_EXHAUSTED && ++ !bfq_class_idle(bfqq)); ++ ++ if (!bfq_bfqq_busy(bfqq) && ++ reason != BFQ_BFQQ_BUDGET_TIMEOUT && ++ reason != BFQ_BFQQ_BUDGET_EXHAUSTED) ++ bfq_mark_bfqq_non_blocking_wait_rq(bfqq); ++} ++ ++/* ++ * Budget timeout is not implemented through a dedicated timer, but ++ * just checked on request arrivals and completions, as well as on ++ * idle timer expirations. ++ */ ++static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq) ++{ ++ return time_is_before_eq_jiffies(bfqq->budget_timeout); ++} ++ ++/* ++ * If we expire a queue that is actively waiting (i.e., with the ++ * device idled) for the arrival of a new request, then we may incur ++ * the timestamp misalignment problem described in the body of the ++ * function __bfq_activate_entity. Hence we return true only if this ++ * condition does not hold, or if the queue is slow enough to deserve ++ * only to be kicked off for preserving a high throughput. ++ */ ++static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) ++{ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "may_budget_timeout: wait_request %d left %d timeout %d", ++ bfq_bfqq_wait_request(bfqq), ++ bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3, ++ bfq_bfqq_budget_timeout(bfqq)); ++ ++ return (!bfq_bfqq_wait_request(bfqq) || ++ bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3) ++ && ++ bfq_bfqq_budget_timeout(bfqq); ++} ++ ++/* ++ * For a queue that becomes empty, device idling is allowed only if ++ * this function returns true for that queue. As a consequence, since ++ * device idling plays a critical role for both throughput boosting ++ * and service guarantees, the return value of this function plays a ++ * critical role as well. ++ * ++ * In a nutshell, this function returns true only if idling is ++ * beneficial for throughput or, even if detrimental for throughput, ++ * idling is however necessary to preserve service guarantees (low ++ * latency, desired throughput distribution, ...). In particular, on ++ * NCQ-capable devices, this function tries to return false, so as to ++ * help keep the drives' internal queues full, whenever this helps the ++ * device boost the throughput without causing any service-guarantee ++ * issue. ++ * ++ * In more detail, the return value of this function is obtained by, ++ * first, computing a number of boolean variables that take into ++ * account throughput and service-guarantee issues, and, then, ++ * combining these variables in a logical expression. Most of the ++ * issues taken into account are not trivial. We discuss these issues ++ * while introducing the variables. ++ */ ++static bool bfq_bfqq_may_idle(struct bfq_queue *bfqq) ++{ ++ struct bfq_data *bfqd = bfqq->bfqd; ++ bool idling_boosts_thr, idling_boosts_thr_without_issues, ++ idling_needed_for_service_guarantees, ++ asymmetric_scenario; ++ ++ if (bfqd->strict_guarantees) ++ return true; ++ ++ /* ++ * The next variable takes into account the cases where idling ++ * boosts the throughput. ++ * ++ * The value of the variable is computed considering, first, that ++ * idling is virtually always beneficial for the throughput if: ++ * (a) the device is not NCQ-capable, or ++ * (b) regardless of the presence of NCQ, the device is rotational ++ * and the request pattern for bfqq is I/O-bound and sequential. ++ * ++ * Secondly, and in contrast to the above item (b), idling an ++ * NCQ-capable flash-based device would not boost the ++ * throughput even with sequential I/O; rather it would lower ++ * the throughput in proportion to how fast the device ++ * is. Accordingly, the next variable is true if any of the ++ * above conditions (a) and (b) is true, and, in particular, ++ * happens to be false if bfqd is an NCQ-capable flash-based ++ * device. ++ */ ++ idling_boosts_thr = !bfqd->hw_tag || ++ (!blk_queue_nonrot(bfqd->queue) && bfq_bfqq_IO_bound(bfqq) && ++ bfq_bfqq_idle_window(bfqq)); ++ ++ /* ++ * The value of the next variable, ++ * idling_boosts_thr_without_issues, is equal to that of ++ * idling_boosts_thr, unless a special case holds. In this ++ * special case, described below, idling may cause problems to ++ * weight-raised queues. ++ * ++ * When the request pool is saturated (e.g., in the presence ++ * of write hogs), if the processes associated with ++ * non-weight-raised queues ask for requests at a lower rate, ++ * then processes associated with weight-raised queues have a ++ * higher probability to get a request from the pool ++ * immediately (or at least soon) when they need one. Thus ++ * they have a higher probability to actually get a fraction ++ * of the device throughput proportional to their high ++ * weight. This is especially true with NCQ-capable drives, ++ * which enqueue several requests in advance, and further ++ * reorder internally-queued requests. ++ * ++ * For this reason, we force to false the value of ++ * idling_boosts_thr_without_issues if there are weight-raised ++ * busy queues. In this case, and if bfqq is not weight-raised, ++ * this guarantees that the device is not idled for bfqq (if, ++ * instead, bfqq is weight-raised, then idling will be ++ * guaranteed by another variable, see below). Combined with ++ * the timestamping rules of BFQ (see [1] for details), this ++ * behavior causes bfqq, and hence any sync non-weight-raised ++ * queue, to get a lower number of requests served, and thus ++ * to ask for a lower number of requests from the request ++ * pool, before the busy weight-raised queues get served ++ * again. This often mitigates starvation problems in the ++ * presence of heavy write workloads and NCQ, thereby ++ * guaranteeing a higher application and system responsiveness ++ * in these hostile scenarios. ++ */ ++ idling_boosts_thr_without_issues = idling_boosts_thr && ++ bfqd->wr_busy_queues == 0; ++ ++ /* ++ * There is then a case where idling must be performed not ++ * for throughput concerns, but to preserve service ++ * guarantees. ++ * ++ * To introduce this case, we can note that allowing the drive ++ * to enqueue more than one request at a time, and hence ++ * delegating de facto final scheduling decisions to the ++ * drive's internal scheduler, entails loss of control on the ++ * actual request service order. In particular, the critical ++ * situation is when requests from different processes happen ++ * to be present, at the same time, in the internal queue(s) ++ * of the drive. In such a situation, the drive, by deciding ++ * the service order of the internally-queued requests, does ++ * determine also the actual throughput distribution among ++ * these processes. But the drive typically has no notion or ++ * concern about per-process throughput distribution, and ++ * makes its decisions only on a per-request basis. Therefore, ++ * the service distribution enforced by the drive's internal ++ * scheduler is likely to coincide with the desired ++ * device-throughput distribution only in a completely ++ * symmetric scenario where: ++ * (i) each of these processes must get the same throughput as ++ * the others; ++ * (ii) all these processes have the same I/O pattern ++ * (either sequential or random). ++ * In fact, in such a scenario, the drive will tend to treat ++ * the requests of each of these processes in about the same ++ * way as the requests of the others, and thus to provide ++ * each of these processes with about the same throughput ++ * (which is exactly the desired throughput distribution). In ++ * contrast, in any asymmetric scenario, device idling is ++ * certainly needed to guarantee that bfqq receives its ++ * assigned fraction of the device throughput (see [1] for ++ * details). ++ * ++ * We address this issue by controlling, actually, only the ++ * symmetry sub-condition (i), i.e., provided that ++ * sub-condition (i) holds, idling is not performed, ++ * regardless of whether sub-condition (ii) holds. In other ++ * words, only if sub-condition (i) holds, then idling is ++ * allowed, and the device tends to be prevented from queueing ++ * many requests, possibly of several processes. The reason ++ * for not controlling also sub-condition (ii) is that we ++ * exploit preemption to preserve guarantees in case of ++ * symmetric scenarios, even if (ii) does not hold, as ++ * explained in the next two paragraphs. ++ * ++ * Even if a queue, say Q, is expired when it remains idle, Q ++ * can still preempt the new in-service queue if the next ++ * request of Q arrives soon (see the comments on ++ * bfq_bfqq_update_budg_for_activation). If all queues and ++ * groups have the same weight, this form of preemption, ++ * combined with the hole-recovery heuristic described in the ++ * comments on function bfq_bfqq_update_budg_for_activation, ++ * are enough to preserve a correct bandwidth distribution in ++ * the mid term, even without idling. In fact, even if not ++ * idling allows the internal queues of the device to contain ++ * many requests, and thus to reorder requests, we can rather ++ * safely assume that the internal scheduler still preserves a ++ * minimum of mid-term fairness. The motivation for using ++ * preemption instead of idling is that, by not idling, ++ * service guarantees are preserved without minimally ++ * sacrificing throughput. In other words, both a high ++ * throughput and its desired distribution are obtained. ++ * ++ * More precisely, this preemption-based, idleless approach ++ * provides fairness in terms of IOPS, and not sectors per ++ * second. This can be seen with a simple example. Suppose ++ * that there are two queues with the same weight, but that ++ * the first queue receives requests of 8 sectors, while the ++ * second queue receives requests of 1024 sectors. In ++ * addition, suppose that each of the two queues contains at ++ * most one request at a time, which implies that each queue ++ * always remains idle after it is served. Finally, after ++ * remaining idle, each queue receives very quickly a new ++ * request. It follows that the two queues are served ++ * alternatively, preempting each other if needed. This ++ * implies that, although both queues have the same weight, ++ * the queue with large requests receives a service that is ++ * 1024/8 times as high as the service received by the other ++ * queue. ++ * ++ * On the other hand, device idling is performed, and thus ++ * pure sector-domain guarantees are provided, for the ++ * following queues, which are likely to need stronger ++ * throughput guarantees: weight-raised queues, and queues ++ * with a higher weight than other queues. When such queues ++ * are active, sub-condition (i) is false, which triggers ++ * device idling. ++ * ++ * According to the above considerations, the next variable is ++ * true (only) if sub-condition (i) holds. To compute the ++ * value of this variable, we not only use the return value of ++ * the function bfq_symmetric_scenario(), but also check ++ * whether bfqq is being weight-raised, because ++ * bfq_symmetric_scenario() does not take into account also ++ * weight-raised queues (see comments on ++ * bfq_weights_tree_add()). ++ * ++ * As a side note, it is worth considering that the above ++ * device-idling countermeasures may however fail in the ++ * following unlucky scenario: if idling is (correctly) ++ * disabled in a time period during which all symmetry ++ * sub-conditions hold, and hence the device is allowed to ++ * enqueue many requests, but at some later point in time some ++ * sub-condition stops to hold, then it may become impossible ++ * to let requests be served in the desired order until all ++ * the requests already queued in the device have been served. ++ */ ++ asymmetric_scenario = bfqq->wr_coeff > 1 || ++ !bfq_symmetric_scenario(bfqd); ++ ++ /* ++ * Finally, there is a case where maximizing throughput is the ++ * best choice even if it may cause unfairness toward ++ * bfqq. Such a case is when bfqq became active in a burst of ++ * queue activations. Queues that became active during a large ++ * burst benefit only from throughput, as discussed in the ++ * comments on bfq_handle_burst. Thus, if bfqq became active ++ * in a burst and not idling the device maximizes throughput, ++ * then the device must no be idled, because not idling the ++ * device provides bfqq and all other queues in the burst with ++ * maximum benefit. Combining this and the above case, we can ++ * now establish when idling is actually needed to preserve ++ * service guarantees. ++ */ ++ idling_needed_for_service_guarantees = ++ asymmetric_scenario && !bfq_bfqq_in_large_burst(bfqq); ++ ++ /* ++ * We have now all the components we need to compute the return ++ * value of the function, which is true only if both the following ++ * conditions hold: ++ * 1) bfqq is sync, because idling make sense only for sync queues; ++ * 2) idling either boosts the throughput (without issues), or ++ * is necessary to preserve service guarantees. ++ */ ++ bfq_log_bfqq(bfqd, bfqq, "may_idle: sync %d idling_boosts_thr %d", ++ bfq_bfqq_sync(bfqq), idling_boosts_thr); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "may_idle: wr_busy %d boosts %d IO-bound %d guar %d", ++ bfqd->wr_busy_queues, ++ idling_boosts_thr_without_issues, ++ bfq_bfqq_IO_bound(bfqq), ++ idling_needed_for_service_guarantees); ++ ++ return bfq_bfqq_sync(bfqq) && ++ (idling_boosts_thr_without_issues || ++ idling_needed_for_service_guarantees); ++} ++ ++/* ++ * If the in-service queue is empty but the function bfq_bfqq_may_idle ++ * returns true, then: ++ * 1) the queue must remain in service and cannot be expired, and ++ * 2) the device must be idled to wait for the possible arrival of a new ++ * request for the queue. ++ * See the comments on the function bfq_bfqq_may_idle for the reasons ++ * why performing device idling is the best choice to boost the throughput ++ * and preserve service guarantees when bfq_bfqq_may_idle itself ++ * returns true. ++ */ ++static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) ++{ ++ struct bfq_data *bfqd = bfqq->bfqd; ++ ++ return RB_EMPTY_ROOT(&bfqq->sort_list) && bfqd->bfq_slice_idle != 0 && ++ bfq_bfqq_may_idle(bfqq); ++} ++ ++/* ++ * Select a queue for service. If we have a current queue in service, ++ * check whether to continue servicing it, or retrieve and set a new one. ++ */ ++static struct bfq_queue *bfq_select_queue(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq; ++ struct request *next_rq; ++ enum bfqq_expiration reason = BFQ_BFQQ_BUDGET_TIMEOUT; ++ ++ bfqq = bfqd->in_service_queue; ++ if (!bfqq) ++ goto new_queue; ++ ++ bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); ++ ++ if (bfq_may_expire_for_budg_timeout(bfqq) && ++ !hrtimer_active(&bfqd->idle_slice_timer) && ++ !bfq_bfqq_must_idle(bfqq)) ++ goto expire; ++ ++ next_rq = bfqq->next_rq; ++ /* ++ * If bfqq has requests queued and it has enough budget left to ++ * serve them, keep the queue, otherwise expire it. ++ */ ++ if (next_rq) { ++ if (bfq_serv_to_charge(next_rq, bfqq) > ++ bfq_bfqq_budget_left(bfqq)) { ++ reason = BFQ_BFQQ_BUDGET_EXHAUSTED; ++ goto expire; ++ } else { ++ /* ++ * The idle timer may be pending because we may ++ * not disable disk idling even when a new request ++ * arrives. ++ */ ++ if (bfq_bfqq_wait_request(bfqq)) { ++ BUG_ON(!hrtimer_active(&bfqd->idle_slice_timer)); ++ /* ++ * If we get here: 1) at least a new request ++ * has arrived but we have not disabled the ++ * timer because the request was too small, ++ * 2) then the block layer has unplugged ++ * the device, causing the dispatch to be ++ * invoked. ++ * ++ * Since the device is unplugged, now the ++ * requests are probably large enough to ++ * provide a reasonable throughput. ++ * So we disable idling. ++ */ ++ bfq_clear_bfqq_wait_request(bfqq); ++ hrtimer_try_to_cancel(&bfqd->idle_slice_timer); ++ bfqg_stats_update_idle_time(bfqq_group(bfqq)); ++ } ++ goto keep_queue; ++ } ++ } ++ ++ /* ++ * No requests pending. However, if the in-service queue is idling ++ * for a new request, or has requests waiting for a completion and ++ * may idle after their completion, then keep it anyway. ++ */ ++ if (hrtimer_active(&bfqd->idle_slice_timer) || ++ (bfqq->dispatched != 0 && bfq_bfqq_may_idle(bfqq))) { ++ bfqq = NULL; ++ goto keep_queue; ++ } ++ ++ reason = BFQ_BFQQ_NO_MORE_REQUESTS; ++expire: ++ bfq_bfqq_expire(bfqd, bfqq, false, reason); ++new_queue: ++ bfqq = bfq_set_in_service_queue(bfqd); ++ bfq_log(bfqd, "select_queue: new queue %d returned", ++ bfqq ? bfqq->pid : 0); ++keep_queue: ++ return bfqq; ++} ++ ++static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */ ++ BUG_ON(bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && ++ time_is_after_jiffies(bfqq->last_wr_start_finish)); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "raising period dur %u/%u msec, old coeff %u, w %d(%d)", ++ jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), ++ jiffies_to_msecs(bfqq->wr_cur_max_time), ++ bfqq->wr_coeff, ++ bfqq->entity.weight, bfqq->entity.orig_weight); ++ ++ BUG_ON(bfqq != bfqd->in_service_queue && entity->weight != ++ entity->orig_weight * bfqq->wr_coeff); ++ if (entity->prio_changed) ++ bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change"); ++ ++ /* ++ * If the queue was activated in a burst, or too much ++ * time has elapsed from the beginning of this ++ * weight-raising period, then end weight raising. ++ */ ++ if (bfq_bfqq_in_large_burst(bfqq)) ++ bfq_bfqq_end_wr(bfqq); ++ else if (time_is_before_jiffies(bfqq->last_wr_start_finish + ++ bfqq->wr_cur_max_time)) { ++ if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time || ++ time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt + ++ bfq_wr_duration(bfqd))) ++ bfq_bfqq_end_wr(bfqq); ++ else { ++ /* switch back to interactive wr */ ++ bfqq->wr_coeff = bfqd->bfq_wr_coeff; ++ bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); ++ bfqq->last_wr_start_finish = ++ bfqq->wr_start_at_switch_to_srt; ++ BUG_ON(time_is_after_jiffies( ++ bfqq->last_wr_start_finish)); ++ bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqd, bfqq, ++ "back to interactive wr"); ++ } ++ } ++ } ++ /* Update weight both if it must be raised and if it must be lowered */ ++ if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1)) ++ __bfq_entity_update_weight_prio( ++ bfq_entity_service_tree(entity), ++ entity); ++} ++ ++/* ++ * Dispatch one request from bfqq, moving it to the request queue ++ * dispatch list. ++ */ ++static int bfq_dispatch_request(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq) ++{ ++ int dispatched = 0; ++ struct request *rq; ++ unsigned long service_to_charge; ++ ++ BUG_ON(RB_EMPTY_ROOT(&bfqq->sort_list)); ++ ++ /* Follow expired path, else get first next available. */ ++ rq = bfq_check_fifo(bfqq); ++ if (!rq) ++ rq = bfqq->next_rq; ++ service_to_charge = bfq_serv_to_charge(rq, bfqq); ++ ++ if (service_to_charge > bfq_bfqq_budget_left(bfqq)) { ++ /* ++ * This may happen if the next rq is chosen in fifo order ++ * instead of sector order. The budget is properly ++ * dimensioned to be always sufficient to serve the next ++ * request only if it is chosen in sector order. The reason ++ * is that it would be quite inefficient and little useful ++ * to always make sure that the budget is large enough to ++ * serve even the possible next rq in fifo order. ++ * In fact, requests are seldom served in fifo order. ++ * ++ * Expire the queue for budget exhaustion, and make sure ++ * that the next act_budget is enough to serve the next ++ * request, even if it comes from the fifo expired path. ++ */ ++ bfqq->next_rq = rq; ++ /* ++ * Since this dispatch is failed, make sure that ++ * a new one will be performed ++ */ ++ if (!bfqd->rq_in_driver) ++ bfq_schedule_dispatch(bfqd); ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ goto expire; ++ } ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ /* Finally, insert request into driver dispatch list. */ ++ bfq_bfqq_served(bfqq, service_to_charge); ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ ++ bfq_dispatch_insert(bfqd->queue, rq); ++ ++ /* ++ * If weight raising has to terminate for bfqq, then next ++ * function causes an immediate update of bfqq's weight, ++ * without waiting for next activation. As a consequence, on ++ * expiration, bfqq will be timestamped as if has never been ++ * weight-raised during this service slot, even if it has ++ * received part or even most of the service as a ++ * weight-raised queue. This inflates bfqq's timestamps, which ++ * is beneficial, as bfqq is then more willing to leave the ++ * device immediately to possible other weight-raised queues. ++ */ ++ bfq_update_wr_data(bfqd, bfqq); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "dispatched %u sec req (%llu), budg left %d", ++ blk_rq_sectors(rq), ++ (unsigned long long) blk_rq_pos(rq), ++ bfq_bfqq_budget_left(bfqq)); ++ ++ dispatched++; ++ ++ if (!bfqd->in_service_bic) { ++ atomic_long_inc(&RQ_BIC(rq)->icq.ioc->refcount); ++ bfqd->in_service_bic = RQ_BIC(rq); ++ } ++ ++ if (bfqd->busy_queues > 1 && bfq_class_idle(bfqq)) ++ goto expire; ++ ++ return dispatched; ++ ++expire: ++ bfq_bfqq_expire(bfqd, bfqq, false, BFQ_BFQQ_BUDGET_EXHAUSTED); ++ return dispatched; ++} ++ ++static int __bfq_forced_dispatch_bfqq(struct bfq_queue *bfqq) ++{ ++ int dispatched = 0; ++ ++ while (bfqq->next_rq) { ++ bfq_dispatch_insert(bfqq->bfqd->queue, bfqq->next_rq); ++ dispatched++; ++ } ++ ++ BUG_ON(!list_empty(&bfqq->fifo)); ++ return dispatched; ++} ++ ++/* ++ * Drain our current requests. ++ * Used for barriers and when switching io schedulers on-the-fly. ++ */ ++static int bfq_forced_dispatch(struct bfq_data *bfqd) ++{ ++ struct bfq_queue *bfqq, *n; ++ struct bfq_service_tree *st; ++ int dispatched = 0; ++ ++ bfqq = bfqd->in_service_queue; ++ if (bfqq) ++ __bfq_bfqq_expire(bfqd, bfqq); ++ ++ /* ++ * Loop through classes, and be careful to leave the scheduler ++ * in a consistent state, as feedback mechanisms and vtime ++ * updates cannot be disabled during the process. ++ */ ++ list_for_each_entry_safe(bfqq, n, &bfqd->active_list, bfqq_list) { ++ st = bfq_entity_service_tree(&bfqq->entity); ++ ++ dispatched += __bfq_forced_dispatch_bfqq(bfqq); ++ ++ bfqq->max_budget = bfq_max_budget(bfqd); ++ bfq_forget_idle(st); ++ } ++ ++ BUG_ON(bfqd->busy_queues != 0); ++ ++ return dispatched; ++} ++ ++static int bfq_dispatch_requests(struct request_queue *q, int force) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct bfq_queue *bfqq; ++ ++ bfq_log(bfqd, "dispatch requests: %d busy queues", bfqd->busy_queues); ++ ++ if (bfqd->busy_queues == 0) ++ return 0; ++ ++ if (unlikely(force)) ++ return bfq_forced_dispatch(bfqd); ++ ++ /* ++ * Force device to serve one request at a time if ++ * strict_guarantees is true. Forcing this service scheme is ++ * currently the ONLY way to guarantee that the request ++ * service order enforced by the scheduler is respected by a ++ * queueing device. Otherwise the device is free even to make ++ * some unlucky request wait for as long as the device ++ * wishes. ++ * ++ * Of course, serving one request at at time may cause loss of ++ * throughput. ++ */ ++ if (bfqd->strict_guarantees && bfqd->rq_in_driver > 0) ++ return 0; ++ ++ bfqq = bfq_select_queue(bfqd); ++ if (!bfqq) ++ return 0; ++ ++ BUG_ON(bfqq->entity.budget < bfqq->entity.service); ++ ++ BUG_ON(bfq_bfqq_wait_request(bfqq)); ++ ++ if (!bfq_dispatch_request(bfqd, bfqq)) ++ return 0; ++ ++ bfq_log_bfqq(bfqd, bfqq, "dispatched %s request", ++ bfq_bfqq_sync(bfqq) ? "sync" : "async"); ++ ++ BUG_ON(bfqq->next_rq == NULL && ++ bfqq->entity.budget < bfqq->entity.service); ++ return 1; ++} ++ ++/* ++ * Task holds one reference to the queue, dropped when task exits. Each rq ++ * in-flight on this queue also holds a reference, dropped when rq is freed. ++ * ++ * Queue lock must be held here. ++ */ ++static void bfq_put_queue(struct bfq_queue *bfqq) ++{ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ struct bfq_group *bfqg = bfqq_group(bfqq); ++#endif ++ ++ BUG_ON(bfqq->ref <= 0); ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); ++ bfqq->ref--; ++ if (bfqq->ref) ++ return; ++ ++ BUG_ON(rb_first(&bfqq->sort_list)); ++ BUG_ON(bfqq->allocated[READ] + bfqq->allocated[WRITE] != 0); ++ BUG_ON(bfqq->entity.tree); ++ BUG_ON(bfq_bfqq_busy(bfqq)); ++ BUG_ON(bfqq->bfqd->in_service_queue == bfqq); ++ ++ if (bfq_bfqq_sync(bfqq)) ++ /* ++ * The fact that this queue is being destroyed does not ++ * invalidate the fact that this queue may have been ++ * activated during the current burst. As a consequence, ++ * although the queue does not exist anymore, and hence ++ * needs to be removed from the burst list if there, ++ * the burst size has not to be decremented. ++ */ ++ hlist_del_init(&bfqq->burst_list_node); ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p freed", bfqq); ++ ++ kmem_cache_free(bfq_pool, bfqq); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ bfqg_put(bfqg); ++#endif ++} ++ ++static void bfq_put_cooperator(struct bfq_queue *bfqq) ++{ ++ struct bfq_queue *__bfqq, *next; ++ ++ /* ++ * If this queue was scheduled to merge with another queue, be ++ * sure to drop the reference taken on that queue (and others in ++ * the merge chain). See bfq_setup_merge and bfq_merge_bfqqs. ++ */ ++ __bfqq = bfqq->new_bfqq; ++ while (__bfqq) { ++ if (__bfqq == bfqq) ++ break; ++ next = __bfqq->new_bfqq; ++ bfq_put_queue(__bfqq); ++ __bfqq = next; ++ } ++} ++ ++static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ if (bfqq == bfqd->in_service_queue) { ++ __bfq_bfqq_expire(bfqd, bfqq); ++ bfq_schedule_dispatch(bfqd); ++ } ++ ++ bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref); ++ ++ bfq_put_cooperator(bfqq); ++ ++ bfq_put_queue(bfqq); ++} ++ ++static void bfq_init_icq(struct io_cq *icq) ++{ ++ icq_to_bic(icq)->ttime.last_end_request = ktime_get_ns() - (1ULL<<32); ++} ++ ++static void bfq_exit_icq(struct io_cq *icq) ++{ ++ struct bfq_io_cq *bic = icq_to_bic(icq); ++ struct bfq_data *bfqd = bic_to_bfqd(bic); ++ ++ if (bic_to_bfqq(bic, false)) { ++ bfq_exit_bfqq(bfqd, bic_to_bfqq(bic, false)); ++ bic_set_bfqq(bic, NULL, false); ++ } ++ ++ if (bic_to_bfqq(bic, true)) { ++ /* ++ * If the bic is using a shared queue, put the reference ++ * taken on the io_context when the bic started using a ++ * shared bfq_queue. ++ */ ++ if (bfq_bfqq_coop(bic_to_bfqq(bic, true))) ++ put_io_context(icq->ioc); ++ bfq_exit_bfqq(bfqd, bic_to_bfqq(bic, true)); ++ bic_set_bfqq(bic, NULL, true); ++ } ++} ++ ++/* ++ * Update the entity prio values; note that the new values will not ++ * be used until the next (re)activation. ++ */ ++static void bfq_set_next_ioprio_data(struct bfq_queue *bfqq, ++ struct bfq_io_cq *bic) ++{ ++ struct task_struct *tsk = current; ++ int ioprio_class; ++ ++ ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio); ++ switch (ioprio_class) { ++ default: ++ dev_err(bfqq->bfqd->queue->backing_dev_info.dev, ++ "bfq: bad prio class %d\n", ioprio_class); ++ case IOPRIO_CLASS_NONE: ++ /* ++ * No prio set, inherit CPU scheduling settings. ++ */ ++ bfqq->new_ioprio = task_nice_ioprio(tsk); ++ bfqq->new_ioprio_class = task_nice_ioclass(tsk); ++ break; ++ case IOPRIO_CLASS_RT: ++ bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); ++ bfqq->new_ioprio_class = IOPRIO_CLASS_RT; ++ break; ++ case IOPRIO_CLASS_BE: ++ bfqq->new_ioprio = IOPRIO_PRIO_DATA(bic->ioprio); ++ bfqq->new_ioprio_class = IOPRIO_CLASS_BE; ++ break; ++ case IOPRIO_CLASS_IDLE: ++ bfqq->new_ioprio_class = IOPRIO_CLASS_IDLE; ++ bfqq->new_ioprio = 7; ++ bfq_clear_bfqq_idle_window(bfqq); ++ break; ++ } ++ ++ if (bfqq->new_ioprio >= IOPRIO_BE_NR) { ++ pr_crit("bfq_set_next_ioprio_data: new_ioprio %d\n", ++ bfqq->new_ioprio); ++ BUG(); ++ } ++ ++ bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio); ++ bfqq->entity.prio_changed = 1; ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "set_next_ioprio_data: bic_class %d prio %d class %d", ++ ioprio_class, bfqq->new_ioprio, bfqq->new_ioprio_class); ++} ++ ++static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio) ++{ ++ struct bfq_data *bfqd = bic_to_bfqd(bic); ++ struct bfq_queue *bfqq; ++ unsigned long uninitialized_var(flags); ++ int ioprio = bic->icq.ioc->ioprio; ++ ++ /* ++ * This condition may trigger on a newly created bic, be sure to ++ * drop the lock before returning. ++ */ ++ if (unlikely(!bfqd) || likely(bic->ioprio == ioprio)) ++ return; ++ ++ bic->ioprio = ioprio; ++ ++ bfqq = bic_to_bfqq(bic, false); ++ if (bfqq) { ++ bfq_put_queue(bfqq); ++ bfqq = bfq_get_queue(bfqd, bio, BLK_RW_ASYNC, bic); ++ bic_set_bfqq(bic, bfqq, false); ++ bfq_log_bfqq(bfqd, bfqq, ++ "check_ioprio_change: bfqq %p %d", ++ bfqq, bfqq->ref); ++ } ++ ++ bfqq = bic_to_bfqq(bic, true); ++ if (bfqq) ++ bfq_set_next_ioprio_data(bfqq, bic); ++} ++ ++static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ struct bfq_io_cq *bic, pid_t pid, int is_sync) ++{ ++ RB_CLEAR_NODE(&bfqq->entity.rb_node); ++ INIT_LIST_HEAD(&bfqq->fifo); ++ INIT_HLIST_NODE(&bfqq->burst_list_node); ++ BUG_ON(!hlist_unhashed(&bfqq->burst_list_node)); ++ ++ bfqq->ref = 0; ++ bfqq->bfqd = bfqd; ++ ++ if (bic) ++ bfq_set_next_ioprio_data(bfqq, bic); ++ ++ if (is_sync) { ++ if (!bfq_class_idle(bfqq)) ++ bfq_mark_bfqq_idle_window(bfqq); ++ bfq_mark_bfqq_sync(bfqq); ++ bfq_mark_bfqq_just_created(bfqq); ++ } else ++ bfq_clear_bfqq_sync(bfqq); ++ bfq_mark_bfqq_IO_bound(bfqq); ++ ++ /* Tentative initial value to trade off between thr and lat */ ++ bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3; ++ bfqq->pid = pid; ++ ++ bfqq->wr_coeff = 1; ++ bfqq->last_wr_start_finish = jiffies; ++ bfqq->wr_start_at_switch_to_srt = bfq_smallest_from_now(); ++ bfqq->budget_timeout = bfq_smallest_from_now(); ++ bfqq->split_time = bfq_smallest_from_now(); ++ ++ /* ++ * Set to the value for which bfqq will not be deemed as ++ * soft rt when it becomes backlogged. ++ */ ++ bfqq->soft_rt_next_start = bfq_greatest_from_now(); ++ ++ /* first request is almost certainly seeky */ ++ bfqq->seek_history = 1; ++} ++ ++static struct bfq_queue **bfq_async_queue_prio(struct bfq_data *bfqd, ++ struct bfq_group *bfqg, ++ int ioprio_class, int ioprio) ++{ ++ switch (ioprio_class) { ++ case IOPRIO_CLASS_RT: ++ return &bfqg->async_bfqq[0][ioprio]; ++ case IOPRIO_CLASS_NONE: ++ ioprio = IOPRIO_NORM; ++ /* fall through */ ++ case IOPRIO_CLASS_BE: ++ return &bfqg->async_bfqq[1][ioprio]; ++ case IOPRIO_CLASS_IDLE: ++ return &bfqg->async_idle_bfqq; ++ default: ++ BUG(); ++ } ++} ++ ++static struct bfq_queue *bfq_get_queue(struct bfq_data *bfqd, ++ struct bio *bio, bool is_sync, ++ struct bfq_io_cq *bic) ++{ ++ const int ioprio = IOPRIO_PRIO_DATA(bic->ioprio); ++ const int ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio); ++ struct bfq_queue **async_bfqq = NULL; ++ struct bfq_queue *bfqq; ++ struct bfq_group *bfqg; ++ ++ rcu_read_lock(); ++ ++ bfqg = bfq_find_set_group(bfqd, bio_blkcg(bio)); ++ if (!bfqg) { ++ bfqq = &bfqd->oom_bfqq; ++ goto out; ++ } ++ ++ if (!is_sync) { ++ async_bfqq = bfq_async_queue_prio(bfqd, bfqg, ioprio_class, ++ ioprio); ++ bfqq = *async_bfqq; ++ if (bfqq) ++ goto out; ++ } ++ ++ bfqq = kmem_cache_alloc_node(bfq_pool, GFP_NOWAIT | __GFP_ZERO, ++ bfqd->queue->node); ++ ++ if (bfqq) { ++ bfq_init_bfqq(bfqd, bfqq, bic, current->pid, ++ is_sync); ++ bfq_init_entity(&bfqq->entity, bfqg); ++ bfq_log_bfqq(bfqd, bfqq, "allocated"); ++ } else { ++ bfqq = &bfqd->oom_bfqq; ++ bfq_log_bfqq(bfqd, bfqq, "using oom bfqq"); ++ goto out; ++ } ++ ++ /* ++ * Pin the queue now that it's allocated, scheduler exit will ++ * prune it. ++ */ ++ if (async_bfqq) { ++ bfqq->ref++; ++ bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", ++ bfqq, bfqq->ref); ++ *async_bfqq = bfqq; ++ } ++ ++out: ++ bfqq->ref++; ++ bfq_log_bfqq(bfqd, bfqq, "get_queue, at end: %p, %d", bfqq, bfqq->ref); ++ rcu_read_unlock(); ++ return bfqq; ++} ++ ++static void bfq_update_io_thinktime(struct bfq_data *bfqd, ++ struct bfq_io_cq *bic) ++{ ++ struct bfq_ttime *ttime = &bic->ttime; ++ u64 elapsed = ktime_get_ns() - bic->ttime.last_end_request; ++ ++ elapsed = min_t(u64, elapsed, 2 * bfqd->bfq_slice_idle); ++ ++ ttime->ttime_samples = (7*bic->ttime.ttime_samples + 256) / 8; ++ ttime->ttime_total = div_u64(7*ttime->ttime_total + 256*elapsed, 8); ++ ttime->ttime_mean = div64_ul(ttime->ttime_total + 128, ++ ttime->ttime_samples); ++} ++ ++static void ++bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ struct request *rq) ++{ ++ bfqq->seek_history <<= 1; ++ bfqq->seek_history |= ++ get_sdist(bfqq->last_request_pos, rq) > BFQQ_SEEK_THR; ++} ++ ++/* ++ * Disable idle window if the process thinks too long or seeks so much that ++ * it doesn't matter. ++ */ ++static void bfq_update_idle_window(struct bfq_data *bfqd, ++ struct bfq_queue *bfqq, ++ struct bfq_io_cq *bic) ++{ ++ int enable_idle; ++ ++ /* Don't idle for async or idle io prio class. */ ++ if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq)) ++ return; ++ ++ /* Idle window just restored, statistics are meaningless. */ ++ if (time_is_after_eq_jiffies(bfqq->split_time + ++ bfqd->bfq_wr_min_idle_time)) ++ return; ++ ++ enable_idle = bfq_bfqq_idle_window(bfqq); ++ ++ if (atomic_read(&bic->icq.ioc->active_ref) == 0 || ++ bfqd->bfq_slice_idle == 0 || ++ (bfqd->hw_tag && BFQQ_SEEKY(bfqq) && ++ bfqq->wr_coeff == 1)) ++ enable_idle = 0; ++ else if (bfq_sample_valid(bic->ttime.ttime_samples)) { ++ if (bic->ttime.ttime_mean > bfqd->bfq_slice_idle && ++ bfqq->wr_coeff == 1) ++ enable_idle = 0; ++ else ++ enable_idle = 1; ++ } ++ bfq_log_bfqq(bfqd, bfqq, "update_idle_window: enable_idle %d", ++ enable_idle); ++ ++ if (enable_idle) ++ bfq_mark_bfqq_idle_window(bfqq); ++ else ++ bfq_clear_bfqq_idle_window(bfqq); ++} ++ ++/* ++ * Called when a new fs request (rq) is added to bfqq. Check if there's ++ * something we should do about it. ++ */ ++static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ struct request *rq) ++{ ++ struct bfq_io_cq *bic = RQ_BIC(rq); ++ ++ if (rq->cmd_flags & REQ_META) ++ bfqq->meta_pending++; ++ ++ bfq_update_io_thinktime(bfqd, bic); ++ bfq_update_io_seektime(bfqd, bfqq, rq); ++ if (bfqq->entity.service > bfq_max_budget(bfqd) / 8 || ++ !BFQQ_SEEKY(bfqq)) ++ bfq_update_idle_window(bfqd, bfqq, bic); ++ ++ bfq_log_bfqq(bfqd, bfqq, ++ "rq_enqueued: idle_window=%d (seeky %d)", ++ bfq_bfqq_idle_window(bfqq), BFQQ_SEEKY(bfqq)); ++ ++ bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); ++ ++ if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) { ++ bool small_req = bfqq->queued[rq_is_sync(rq)] == 1 && ++ blk_rq_sectors(rq) < 32; ++ bool budget_timeout = bfq_bfqq_budget_timeout(bfqq); ++ ++ /* ++ * There is just this request queued: if the request ++ * is small and the queue is not to be expired, then ++ * just exit. ++ * ++ * In this way, if the device is being idled to wait ++ * for a new request from the in-service queue, we ++ * avoid unplugging the device and committing the ++ * device to serve just a small request. On the ++ * contrary, we wait for the block layer to decide ++ * when to unplug the device: hopefully, new requests ++ * will be merged to this one quickly, then the device ++ * will be unplugged and larger requests will be ++ * dispatched. ++ */ ++ if (small_req && !budget_timeout) ++ return; ++ ++ /* ++ * A large enough request arrived, or the queue is to ++ * be expired: in both cases disk idling is to be ++ * stopped, so clear wait_request flag and reset ++ * timer. ++ */ ++ bfq_clear_bfqq_wait_request(bfqq); ++ hrtimer_try_to_cancel(&bfqd->idle_slice_timer); ++ bfqg_stats_update_idle_time(bfqq_group(bfqq)); ++ ++ /* ++ * The queue is not empty, because a new request just ++ * arrived. Hence we can safely expire the queue, in ++ * case of budget timeout, without risking that the ++ * timestamps of the queue are not updated correctly. ++ * See [1] for more details. ++ */ ++ if (budget_timeout) ++ bfq_bfqq_expire(bfqd, bfqq, false, ++ BFQ_BFQQ_BUDGET_TIMEOUT); ++ ++ /* ++ * Let the request rip immediately, or let a new queue be ++ * selected if bfqq has just been expired. ++ */ ++ __blk_run_queue(bfqd->queue); ++ } ++} ++ ++static void bfq_insert_request(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct bfq_queue *bfqq = RQ_BFQQ(rq), *new_bfqq; ++ ++ assert_spin_locked(bfqd->queue->queue_lock); ++ ++ /* ++ * An unplug may trigger a requeue of a request from the device ++ * driver: make sure we are in process context while trying to ++ * merge two bfq_queues. ++ */ ++ if (!in_interrupt()) { ++ new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true); ++ if (new_bfqq) { ++ if (bic_to_bfqq(RQ_BIC(rq), 1) != bfqq) ++ new_bfqq = bic_to_bfqq(RQ_BIC(rq), 1); ++ /* ++ * Release the request's reference to the old bfqq ++ * and make sure one is taken to the shared queue. ++ */ ++ new_bfqq->allocated[rq_data_dir(rq)]++; ++ bfqq->allocated[rq_data_dir(rq)]--; ++ new_bfqq->ref++; ++ bfq_clear_bfqq_just_created(bfqq); ++ bfq_put_queue(bfqq); ++ if (bic_to_bfqq(RQ_BIC(rq), 1) == bfqq) ++ bfq_merge_bfqqs(bfqd, RQ_BIC(rq), ++ bfqq, new_bfqq); ++ rq->elv.priv[1] = new_bfqq; ++ bfqq = new_bfqq; ++ } ++ } ++ ++ bfq_add_request(rq); ++ ++ rq->fifo_time = ktime_get_ns() + ++ jiffies_to_nsecs(bfqd->bfq_fifo_expire[rq_is_sync(rq)]); ++ list_add_tail(&rq->queuelist, &bfqq->fifo); ++ ++ bfq_rq_enqueued(bfqd, bfqq, rq); ++} ++ ++static void bfq_update_hw_tag(struct bfq_data *bfqd) ++{ ++ bfqd->max_rq_in_driver = max_t(int, bfqd->max_rq_in_driver, ++ bfqd->rq_in_driver); ++ ++ if (bfqd->hw_tag == 1) ++ return; ++ ++ /* ++ * This sample is valid if the number of outstanding requests ++ * is large enough to allow a queueing behavior. Note that the ++ * sum is not exact, as it's not taking into account deactivated ++ * requests. ++ */ ++ if (bfqd->rq_in_driver + bfqd->queued < BFQ_HW_QUEUE_THRESHOLD) ++ return; ++ ++ if (bfqd->hw_tag_samples++ < BFQ_HW_QUEUE_SAMPLES) ++ return; ++ ++ bfqd->hw_tag = bfqd->max_rq_in_driver > BFQ_HW_QUEUE_THRESHOLD; ++ bfqd->max_rq_in_driver = 0; ++ bfqd->hw_tag_samples = 0; ++} ++ ++static void bfq_completed_request(struct request_queue *q, struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ struct bfq_data *bfqd = bfqq->bfqd; ++ u64 now_ns; ++ u32 delta_us; ++ ++ bfq_log_bfqq(bfqd, bfqq, "completed one req with %u sects left", ++ blk_rq_sectors(rq)); ++ ++ assert_spin_locked(bfqd->queue->queue_lock); ++ bfq_update_hw_tag(bfqd); ++ ++ BUG_ON(!bfqd->rq_in_driver); ++ BUG_ON(!bfqq->dispatched); ++ bfqd->rq_in_driver--; ++ bfqq->dispatched--; ++ bfqg_stats_update_completion(bfqq_group(bfqq), ++ rq_start_time_ns(rq), ++ rq_io_start_time_ns(rq), req_op(rq), ++ rq->cmd_flags); ++ ++ if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) { ++ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list)); ++ /* ++ * Set budget_timeout (which we overload to store the ++ * time at which the queue remains with no backlog and ++ * no outstanding request; used by the weight-raising ++ * mechanism). ++ */ ++ bfqq->budget_timeout = jiffies; ++ ++ bfq_weights_tree_remove(bfqd, &bfqq->entity, ++ &bfqd->queue_weights_tree); ++ } ++ ++ now_ns = ktime_get_ns(); ++ ++ RQ_BIC(rq)->ttime.last_end_request = now_ns; ++ ++ /* ++ * Using us instead of ns, to get a reasonable precision in ++ * computing rate in next check. ++ */ ++ delta_us = div_u64(now_ns - bfqd->last_completion, NSEC_PER_USEC); ++ ++ bfq_log(bfqd, "rq_completed: delta %uus/%luus max_size %u rate %llu/%llu", ++ delta_us, BFQ_MIN_TT/NSEC_PER_USEC, bfqd->last_rq_max_size, ++ (USEC_PER_SEC* ++ (u64)((bfqd->last_rq_max_size<>BFQ_RATE_SHIFT, ++ (USEC_PER_SEC*(u64)(1UL<<(BFQ_RATE_SHIFT-10)))>>BFQ_RATE_SHIFT); ++ ++ /* ++ * If the request took rather long to complete, and, according ++ * to the maximum request size recorded, this completion latency ++ * implies that the request was certainly served at a very low ++ * rate (less than 1M sectors/sec), then the whole observation ++ * interval that lasts up to this time instant cannot be a ++ * valid time interval for computing a new peak rate. Invoke ++ * bfq_update_rate_reset to have the following three steps ++ * taken: ++ * - close the observation interval at the last (previous) ++ * request dispatch or completion ++ * - compute rate, if possible, for that observation interval ++ * - reset to zero samples, which will trigger a proper ++ * re-initialization of the observation interval on next ++ * dispatch ++ */ ++ if (delta_us > BFQ_MIN_TT/NSEC_PER_USEC && ++ (bfqd->last_rq_max_size<last_completion = now_ns; ++ ++ /* ++ * If we are waiting to discover whether the request pattern ++ * of the task associated with the queue is actually ++ * isochronous, and both requisites for this condition to hold ++ * are now satisfied, then compute soft_rt_next_start (see the ++ * comments on the function bfq_bfqq_softrt_next_start()). We ++ * schedule this delayed check when bfqq expires, if it still ++ * has in-flight requests. ++ */ ++ if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 && ++ RB_EMPTY_ROOT(&bfqq->sort_list)) ++ bfqq->soft_rt_next_start = ++ bfq_bfqq_softrt_next_start(bfqd, bfqq); ++ ++ /* ++ * If this is the in-service queue, check if it needs to be expired, ++ * or if we want to idle in case it has no pending requests. ++ */ ++ if (bfqd->in_service_queue == bfqq) { ++ if (bfqq->dispatched == 0 && bfq_bfqq_must_idle(bfqq)) { ++ bfq_arm_slice_timer(bfqd); ++ goto out; ++ } else if (bfq_may_expire_for_budg_timeout(bfqq)) ++ bfq_bfqq_expire(bfqd, bfqq, false, ++ BFQ_BFQQ_BUDGET_TIMEOUT); ++ else if (RB_EMPTY_ROOT(&bfqq->sort_list) && ++ (bfqq->dispatched == 0 || ++ !bfq_bfqq_may_idle(bfqq))) ++ bfq_bfqq_expire(bfqd, bfqq, false, ++ BFQ_BFQQ_NO_MORE_REQUESTS); ++ } ++ ++ if (!bfqd->rq_in_driver) ++ bfq_schedule_dispatch(bfqd); ++ ++out: ++ return; ++} ++ ++static int __bfq_may_queue(struct bfq_queue *bfqq) ++{ ++ if (bfq_bfqq_wait_request(bfqq) && bfq_bfqq_must_alloc(bfqq)) { ++ bfq_clear_bfqq_must_alloc(bfqq); ++ return ELV_MQUEUE_MUST; ++ } ++ ++ return ELV_MQUEUE_MAY; ++} ++ ++static int bfq_may_queue(struct request_queue *q, int op, int op_flags) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct task_struct *tsk = current; ++ struct bfq_io_cq *bic; ++ struct bfq_queue *bfqq; ++ ++ /* ++ * Don't force setup of a queue from here, as a call to may_queue ++ * does not necessarily imply that a request actually will be ++ * queued. So just lookup a possibly existing queue, or return ++ * 'may queue' if that fails. ++ */ ++ bic = bfq_bic_lookup(bfqd, tsk->io_context); ++ if (!bic) ++ return ELV_MQUEUE_MAY; ++ ++ bfqq = bic_to_bfqq(bic, rw_is_sync(op, op_flags)); ++ if (bfqq) ++ return __bfq_may_queue(bfqq); ++ ++ return ELV_MQUEUE_MAY; ++} ++ ++/* ++ * Queue lock held here. ++ */ ++static void bfq_put_request(struct request *rq) ++{ ++ struct bfq_queue *bfqq = RQ_BFQQ(rq); ++ ++ if (bfqq) { ++ const int rw = rq_data_dir(rq); ++ ++ BUG_ON(!bfqq->allocated[rw]); ++ bfqq->allocated[rw]--; ++ ++ rq->elv.priv[0] = NULL; ++ rq->elv.priv[1] = NULL; ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "put_request %p, %d", ++ bfqq, bfqq->ref); ++ bfq_put_queue(bfqq); ++ } ++} ++ ++/* ++ * Returns NULL if a new bfqq should be allocated, or the old bfqq if this ++ * was the last process referring to that bfqq. ++ */ ++static struct bfq_queue * ++bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq) ++{ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); ++ ++ put_io_context(bic->icq.ioc); ++ ++ if (bfqq_process_refs(bfqq) == 1) { ++ bfqq->pid = current->pid; ++ bfq_clear_bfqq_coop(bfqq); ++ bfq_clear_bfqq_split_coop(bfqq); ++ return bfqq; ++ } ++ ++ bic_set_bfqq(bic, NULL, 1); ++ ++ bfq_put_cooperator(bfqq); ++ ++ bfq_put_queue(bfqq); ++ return NULL; ++} ++ ++/* ++ * Allocate bfq data structures associated with this request. ++ */ ++static int bfq_set_request(struct request_queue *q, struct request *rq, ++ struct bio *bio, gfp_t gfp_mask) ++{ ++ struct bfq_data *bfqd = q->elevator->elevator_data; ++ struct bfq_io_cq *bic = icq_to_bic(rq->elv.icq); ++ const int rw = rq_data_dir(rq); ++ const int is_sync = rq_is_sync(rq); ++ struct bfq_queue *bfqq; ++ unsigned long flags; ++ bool split = false; ++ ++ spin_lock_irqsave(q->queue_lock, flags); ++ bfq_check_ioprio_change(bic, bio); ++ ++ if (!bic) ++ goto queue_fail; ++ ++ bfq_bic_update_cgroup(bic, bio); ++ ++new_queue: ++ bfqq = bic_to_bfqq(bic, is_sync); ++ if (!bfqq || bfqq == &bfqd->oom_bfqq) { ++ if (bfqq) ++ bfq_put_queue(bfqq); ++ bfqq = bfq_get_queue(bfqd, bio, is_sync, bic); ++ BUG_ON(!hlist_unhashed(&bfqq->burst_list_node)); ++ ++ bic_set_bfqq(bic, bfqq, is_sync); ++ if (split && is_sync) { ++ bfq_log_bfqq(bfqd, bfqq, ++ "set_request: was_in_list %d " ++ "was_in_large_burst %d " ++ "large burst in progress %d", ++ bic->was_in_burst_list, ++ bic->saved_in_large_burst, ++ bfqd->large_burst); ++ ++ if ((bic->was_in_burst_list && bfqd->large_burst) || ++ bic->saved_in_large_burst) { ++ bfq_log_bfqq(bfqd, bfqq, ++ "set_request: marking in " ++ "large burst"); ++ bfq_mark_bfqq_in_large_burst(bfqq); ++ } else { ++ bfq_log_bfqq(bfqd, bfqq, ++ "set_request: clearing in " ++ "large burst"); ++ bfq_clear_bfqq_in_large_burst(bfqq); ++ if (bic->was_in_burst_list) ++ hlist_add_head(&bfqq->burst_list_node, ++ &bfqd->burst_list); ++ } ++ bfqq->split_time = jiffies; ++ } ++ } else { ++ /* If the queue was seeky for too long, break it apart. */ ++ if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq)) { ++ bfq_log_bfqq(bfqd, bfqq, "breaking apart bfqq"); ++ ++ /* Update bic before losing reference to bfqq */ ++ if (bfq_bfqq_in_large_burst(bfqq)) ++ bic->saved_in_large_burst = true; ++ ++ bfqq = bfq_split_bfqq(bic, bfqq); ++ split = true; ++ if (!bfqq) ++ goto new_queue; ++ } ++ } ++ ++ bfqq->allocated[rw]++; ++ bfqq->ref++; ++ bfq_log_bfqq(bfqd, bfqq, "set_request: bfqq %p, %d", bfqq, bfqq->ref); ++ ++ rq->elv.priv[0] = bic; ++ rq->elv.priv[1] = bfqq; ++ ++ /* ++ * If a bfq_queue has only one process reference, it is owned ++ * by only one bfq_io_cq: we can set the bic field of the ++ * bfq_queue to the address of that structure. Also, if the ++ * queue has just been split, mark a flag so that the ++ * information is available to the other scheduler hooks. ++ */ ++ if (likely(bfqq != &bfqd->oom_bfqq) && bfqq_process_refs(bfqq) == 1) { ++ bfqq->bic = bic; ++ if (split) { ++ /* ++ * If the queue has just been split from a shared ++ * queue, restore the idle window and the possible ++ * weight raising period. ++ */ ++ bfq_bfqq_resume_state(bfqq, bic); ++ } ++ } ++ ++ if (unlikely(bfq_bfqq_just_created(bfqq))) ++ bfq_handle_burst(bfqd, bfqq); ++ ++ spin_unlock_irqrestore(q->queue_lock, flags); ++ ++ return 0; ++ ++queue_fail: ++ bfq_schedule_dispatch(bfqd); ++ spin_unlock_irqrestore(q->queue_lock, flags); ++ ++ return 1; ++} ++ ++static void bfq_kick_queue(struct work_struct *work) ++{ ++ struct bfq_data *bfqd = ++ container_of(work, struct bfq_data, unplug_work); ++ struct request_queue *q = bfqd->queue; ++ ++ spin_lock_irq(q->queue_lock); ++ __blk_run_queue(q); ++ spin_unlock_irq(q->queue_lock); ++} ++ ++/* ++ * Handler of the expiration of the timer running if the in-service queue ++ * is idling inside its time slice. ++ */ ++static enum hrtimer_restart bfq_idle_slice_timer(struct hrtimer *timer) ++{ ++ struct bfq_data *bfqd = container_of(timer, struct bfq_data, ++ idle_slice_timer); ++ struct bfq_queue *bfqq; ++ unsigned long flags; ++ enum bfqq_expiration reason; ++ ++ spin_lock_irqsave(bfqd->queue->queue_lock, flags); ++ ++ bfqq = bfqd->in_service_queue; ++ /* ++ * Theoretical race here: the in-service queue can be NULL or ++ * different from the queue that was idling if the timer handler ++ * spins on the queue_lock and a new request arrives for the ++ * current queue and there is a full dispatch cycle that changes ++ * the in-service queue. This can hardly happen, but in the worst ++ * case we just expire a queue too early. ++ */ ++ if (bfqq) { ++ bfq_log_bfqq(bfqd, bfqq, "slice_timer expired"); ++ bfq_clear_bfqq_wait_request(bfqq); ++ ++ if (bfq_bfqq_budget_timeout(bfqq)) ++ /* ++ * Also here the queue can be safely expired ++ * for budget timeout without wasting ++ * guarantees ++ */ ++ reason = BFQ_BFQQ_BUDGET_TIMEOUT; ++ else if (bfqq->queued[0] == 0 && bfqq->queued[1] == 0) ++ /* ++ * The queue may not be empty upon timer expiration, ++ * because we may not disable the timer when the ++ * first request of the in-service queue arrives ++ * during disk idling. ++ */ ++ reason = BFQ_BFQQ_TOO_IDLE; ++ else ++ goto schedule_dispatch; ++ ++ bfq_bfqq_expire(bfqd, bfqq, true, reason); ++ } ++ ++schedule_dispatch: ++ bfq_schedule_dispatch(bfqd); ++ ++ spin_unlock_irqrestore(bfqd->queue->queue_lock, flags); ++ return HRTIMER_NORESTART; ++} ++ ++static void bfq_shutdown_timer_wq(struct bfq_data *bfqd) ++{ ++ hrtimer_cancel(&bfqd->idle_slice_timer); ++ cancel_work_sync(&bfqd->unplug_work); ++} ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static void __bfq_put_async_bfqq(struct bfq_data *bfqd, ++ struct bfq_queue **bfqq_ptr) ++{ ++ struct bfq_group *root_group = bfqd->root_group; ++ struct bfq_queue *bfqq = *bfqq_ptr; ++ ++ bfq_log(bfqd, "put_async_bfqq: %p", bfqq); ++ if (bfqq) { ++ bfq_bfqq_move(bfqd, bfqq, root_group); ++ bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", ++ bfqq, bfqq->ref); ++ bfq_put_queue(bfqq); ++ *bfqq_ptr = NULL; ++ } ++} ++ ++/* ++ * Release all the bfqg references to its async queues. If we are ++ * deallocating the group these queues may still contain requests, so ++ * we reparent them to the root cgroup (i.e., the only one that will ++ * exist for sure until all the requests on a device are gone). ++ */ ++static void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg) ++{ ++ int i, j; ++ ++ for (i = 0; i < 2; i++) ++ for (j = 0; j < IOPRIO_BE_NR; j++) ++ __bfq_put_async_bfqq(bfqd, &bfqg->async_bfqq[i][j]); ++ ++ __bfq_put_async_bfqq(bfqd, &bfqg->async_idle_bfqq); ++} ++#endif ++ ++static void bfq_exit_queue(struct elevator_queue *e) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ struct request_queue *q = bfqd->queue; ++ struct bfq_queue *bfqq, *n; ++ ++ bfq_shutdown_timer_wq(bfqd); ++ ++ spin_lock_irq(q->queue_lock); ++ ++ BUG_ON(bfqd->in_service_queue); ++ list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list) ++ bfq_deactivate_bfqq(bfqd, bfqq, 0); ++ ++ spin_unlock_irq(q->queue_lock); ++ ++ bfq_shutdown_timer_wq(bfqd); ++ ++ BUG_ON(hrtimer_active(&bfqd->idle_slice_timer)); ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ blkcg_deactivate_policy(q, &blkcg_policy_bfq); ++#else ++ kfree(bfqd->root_group); ++#endif ++ ++ kfree(bfqd); ++} ++ ++static void bfq_init_root_group(struct bfq_group *root_group, ++ struct bfq_data *bfqd) ++{ ++ int i; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ root_group->entity.parent = NULL; ++ root_group->my_entity = NULL; ++ root_group->bfqd = bfqd; ++#endif ++ root_group->rq_pos_tree = RB_ROOT; ++ for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) ++ root_group->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT; ++} ++ ++static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) ++{ ++ struct bfq_data *bfqd; ++ struct elevator_queue *eq; ++ ++ eq = elevator_alloc(q, e); ++ if (!eq) ++ return -ENOMEM; ++ ++ bfqd = kzalloc_node(sizeof(*bfqd), GFP_KERNEL, q->node); ++ if (!bfqd) { ++ kobject_put(&eq->kobj); ++ return -ENOMEM; ++ } ++ eq->elevator_data = bfqd; ++ ++ /* ++ * Our fallback bfqq if bfq_find_alloc_queue() runs into OOM issues. ++ * Grab a permanent reference to it, so that the normal code flow ++ * will not attempt to free it. ++ */ ++ bfq_init_bfqq(bfqd, &bfqd->oom_bfqq, NULL, 1, 0); ++ bfqd->oom_bfqq.ref++; ++ bfqd->oom_bfqq.new_ioprio = BFQ_DEFAULT_QUEUE_IOPRIO; ++ bfqd->oom_bfqq.new_ioprio_class = IOPRIO_CLASS_BE; ++ bfqd->oom_bfqq.entity.new_weight = ++ bfq_ioprio_to_weight(bfqd->oom_bfqq.new_ioprio); ++ ++ /* oom_bfqq does not participate to bursts */ ++ bfq_clear_bfqq_just_created(&bfqd->oom_bfqq); ++ /* ++ * Trigger weight initialization, according to ioprio, at the ++ * oom_bfqq's first activation. The oom_bfqq's ioprio and ioprio ++ * class won't be changed any more. ++ */ ++ bfqd->oom_bfqq.entity.prio_changed = 1; ++ ++ bfqd->queue = q; ++ ++ spin_lock_irq(q->queue_lock); ++ q->elevator = eq; ++ spin_unlock_irq(q->queue_lock); ++ ++ bfqd->root_group = bfq_create_group_hierarchy(bfqd, q->node); ++ if (!bfqd->root_group) ++ goto out_free; ++ bfq_init_root_group(bfqd->root_group, bfqd); ++ bfq_init_entity(&bfqd->oom_bfqq.entity, bfqd->root_group); ++ ++ hrtimer_init(&bfqd->idle_slice_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_REL); ++ bfqd->idle_slice_timer.function = bfq_idle_slice_timer; ++ ++ bfqd->queue_weights_tree = RB_ROOT; ++ bfqd->group_weights_tree = RB_ROOT; ++ ++ INIT_WORK(&bfqd->unplug_work, bfq_kick_queue); ++ ++ INIT_LIST_HEAD(&bfqd->active_list); ++ INIT_LIST_HEAD(&bfqd->idle_list); ++ INIT_HLIST_HEAD(&bfqd->burst_list); ++ ++ bfqd->hw_tag = -1; ++ ++ bfqd->bfq_max_budget = bfq_default_max_budget; ++ ++ bfqd->bfq_fifo_expire[0] = bfq_fifo_expire[0]; ++ bfqd->bfq_fifo_expire[1] = bfq_fifo_expire[1]; ++ bfqd->bfq_back_max = bfq_back_max; ++ bfqd->bfq_back_penalty = bfq_back_penalty; ++ bfqd->bfq_slice_idle = bfq_slice_idle; ++ bfqd->bfq_class_idle_last_service = 0; ++ bfqd->bfq_timeout = bfq_timeout; ++ ++ bfqd->bfq_requests_within_timer = 120; ++ ++ bfqd->bfq_large_burst_thresh = 8; ++ bfqd->bfq_burst_interval = msecs_to_jiffies(180); ++ ++ bfqd->low_latency = true; ++ ++ /* ++ * Trade-off between responsiveness and fairness. ++ */ ++ bfqd->bfq_wr_coeff = 30; ++ bfqd->bfq_wr_rt_max_time = msecs_to_jiffies(300); ++ bfqd->bfq_wr_max_time = 0; ++ bfqd->bfq_wr_min_idle_time = msecs_to_jiffies(2000); ++ bfqd->bfq_wr_min_inter_arr_async = msecs_to_jiffies(500); ++ bfqd->bfq_wr_max_softrt_rate = 7000; /* ++ * Approximate rate required ++ * to playback or record a ++ * high-definition compressed ++ * video. ++ */ ++ bfqd->wr_busy_queues = 0; ++ ++ /* ++ * Begin by assuming, optimistically, that the device is a ++ * high-speed one, and that its peak rate is equal to 2/3 of ++ * the highest reference rate. ++ */ ++ bfqd->RT_prod = R_fast[blk_queue_nonrot(bfqd->queue)] * ++ T_fast[blk_queue_nonrot(bfqd->queue)]; ++ bfqd->peak_rate = R_fast[blk_queue_nonrot(bfqd->queue)] * 2 / 3; ++ bfqd->device_speed = BFQ_BFQD_FAST; ++ ++ return 0; ++ ++out_free: ++ kfree(bfqd); ++ kobject_put(&eq->kobj); ++ return -ENOMEM; ++} ++ ++static void bfq_slab_kill(void) ++{ ++ kmem_cache_destroy(bfq_pool); ++} ++ ++static int __init bfq_slab_setup(void) ++{ ++ bfq_pool = KMEM_CACHE(bfq_queue, 0); ++ if (!bfq_pool) ++ return -ENOMEM; ++ return 0; ++} ++ ++static ssize_t bfq_var_show(unsigned int var, char *page) ++{ ++ return sprintf(page, "%u\n", var); ++} ++ ++static ssize_t bfq_var_store(unsigned long *var, const char *page, ++ size_t count) ++{ ++ unsigned long new_val; ++ int ret = kstrtoul(page, 10, &new_val); ++ ++ if (ret == 0) ++ *var = new_val; ++ ++ return count; ++} ++ ++static ssize_t bfq_wr_max_time_show(struct elevator_queue *e, char *page) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ ++ return sprintf(page, "%d\n", bfqd->bfq_wr_max_time > 0 ? ++ jiffies_to_msecs(bfqd->bfq_wr_max_time) : ++ jiffies_to_msecs(bfq_wr_duration(bfqd))); ++} ++ ++static ssize_t bfq_weights_show(struct elevator_queue *e, char *page) ++{ ++ struct bfq_queue *bfqq; ++ struct bfq_data *bfqd = e->elevator_data; ++ ssize_t num_char = 0; ++ ++ num_char += sprintf(page + num_char, "Tot reqs queued %d\n\n", ++ bfqd->queued); ++ ++ spin_lock_irq(bfqd->queue->queue_lock); ++ ++ num_char += sprintf(page + num_char, "Active:\n"); ++ list_for_each_entry(bfqq, &bfqd->active_list, bfqq_list) { ++ num_char += sprintf(page + num_char, ++ "pid%d: weight %hu, nr_queued %d %d, ", ++ bfqq->pid, ++ bfqq->entity.weight, ++ bfqq->queued[0], ++ bfqq->queued[1]); ++ num_char += sprintf(page + num_char, ++ "dur %d/%u\n", ++ jiffies_to_msecs( ++ jiffies - ++ bfqq->last_wr_start_finish), ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ } ++ ++ num_char += sprintf(page + num_char, "Idle:\n"); ++ list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) { ++ num_char += sprintf(page + num_char, ++ "pid%d: weight %hu, dur %d/%u\n", ++ bfqq->pid, ++ bfqq->entity.weight, ++ jiffies_to_msecs(jiffies - ++ bfqq->last_wr_start_finish), ++ jiffies_to_msecs(bfqq->wr_cur_max_time)); ++ } ++ ++ spin_unlock_irq(bfqd->queue->queue_lock); ++ ++ return num_char; ++} ++ ++#define SHOW_FUNCTION(__FUNC, __VAR, __CONV) \ ++static ssize_t __FUNC(struct elevator_queue *e, char *page) \ ++{ \ ++ struct bfq_data *bfqd = e->elevator_data; \ ++ u64 __data = __VAR; \ ++ if (__CONV == 1) \ ++ __data = jiffies_to_msecs(__data); \ ++ else if (__CONV == 2) \ ++ __data = div_u64(__data, NSEC_PER_MSEC); \ ++ return bfq_var_show(__data, (page)); \ ++} ++SHOW_FUNCTION(bfq_fifo_expire_sync_show, bfqd->bfq_fifo_expire[1], 2); ++SHOW_FUNCTION(bfq_fifo_expire_async_show, bfqd->bfq_fifo_expire[0], 2); ++SHOW_FUNCTION(bfq_back_seek_max_show, bfqd->bfq_back_max, 0); ++SHOW_FUNCTION(bfq_back_seek_penalty_show, bfqd->bfq_back_penalty, 0); ++SHOW_FUNCTION(bfq_slice_idle_show, bfqd->bfq_slice_idle, 2); ++SHOW_FUNCTION(bfq_max_budget_show, bfqd->bfq_user_max_budget, 0); ++SHOW_FUNCTION(bfq_timeout_sync_show, bfqd->bfq_timeout, 1); ++SHOW_FUNCTION(bfq_strict_guarantees_show, bfqd->strict_guarantees, 0); ++SHOW_FUNCTION(bfq_low_latency_show, bfqd->low_latency, 0); ++SHOW_FUNCTION(bfq_wr_coeff_show, bfqd->bfq_wr_coeff, 0); ++SHOW_FUNCTION(bfq_wr_rt_max_time_show, bfqd->bfq_wr_rt_max_time, 1); ++SHOW_FUNCTION(bfq_wr_min_idle_time_show, bfqd->bfq_wr_min_idle_time, 1); ++SHOW_FUNCTION(bfq_wr_min_inter_arr_async_show, bfqd->bfq_wr_min_inter_arr_async, ++ 1); ++SHOW_FUNCTION(bfq_wr_max_softrt_rate_show, bfqd->bfq_wr_max_softrt_rate, 0); ++#undef SHOW_FUNCTION ++ ++#define USEC_SHOW_FUNCTION(__FUNC, __VAR) \ ++static ssize_t __FUNC(struct elevator_queue *e, char *page) \ ++{ \ ++ struct bfq_data *bfqd = e->elevator_data; \ ++ u64 __data = __VAR; \ ++ __data = div_u64(__data, NSEC_PER_USEC); \ ++ return bfq_var_show(__data, (page)); \ ++} ++USEC_SHOW_FUNCTION(bfq_slice_idle_us_show, bfqd->bfq_slice_idle); ++#undef USEC_SHOW_FUNCTION ++ ++#define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV) \ ++static ssize_t \ ++__FUNC(struct elevator_queue *e, const char *page, size_t count) \ ++{ \ ++ struct bfq_data *bfqd = e->elevator_data; \ ++ unsigned long uninitialized_var(__data); \ ++ int ret = bfq_var_store(&__data, (page), count); \ ++ if (__data < (MIN)) \ ++ __data = (MIN); \ ++ else if (__data > (MAX)) \ ++ __data = (MAX); \ ++ if (__CONV == 1) \ ++ *(__PTR) = msecs_to_jiffies(__data); \ ++ else if (__CONV == 2) \ ++ *(__PTR) = (u64)__data * NSEC_PER_MSEC; \ ++ else \ ++ *(__PTR) = __data; \ ++ return ret; \ ++} ++STORE_FUNCTION(bfq_fifo_expire_sync_store, &bfqd->bfq_fifo_expire[1], 1, ++ INT_MAX, 2); ++STORE_FUNCTION(bfq_fifo_expire_async_store, &bfqd->bfq_fifo_expire[0], 1, ++ INT_MAX, 2); ++STORE_FUNCTION(bfq_back_seek_max_store, &bfqd->bfq_back_max, 0, INT_MAX, 0); ++STORE_FUNCTION(bfq_back_seek_penalty_store, &bfqd->bfq_back_penalty, 1, ++ INT_MAX, 0); ++STORE_FUNCTION(bfq_slice_idle_store, &bfqd->bfq_slice_idle, 0, INT_MAX, 2); ++STORE_FUNCTION(bfq_wr_coeff_store, &bfqd->bfq_wr_coeff, 1, INT_MAX, 0); ++STORE_FUNCTION(bfq_wr_max_time_store, &bfqd->bfq_wr_max_time, 0, INT_MAX, 1); ++STORE_FUNCTION(bfq_wr_rt_max_time_store, &bfqd->bfq_wr_rt_max_time, 0, INT_MAX, ++ 1); ++STORE_FUNCTION(bfq_wr_min_idle_time_store, &bfqd->bfq_wr_min_idle_time, 0, ++ INT_MAX, 1); ++STORE_FUNCTION(bfq_wr_min_inter_arr_async_store, ++ &bfqd->bfq_wr_min_inter_arr_async, 0, INT_MAX, 1); ++STORE_FUNCTION(bfq_wr_max_softrt_rate_store, &bfqd->bfq_wr_max_softrt_rate, 0, ++ INT_MAX, 0); ++#undef STORE_FUNCTION ++ ++#define USEC_STORE_FUNCTION(__FUNC, __PTR, MIN, MAX) \ ++static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count)\ ++{ \ ++ struct bfq_data *bfqd = e->elevator_data; \ ++ unsigned long __data; \ ++ int ret = bfq_var_store(&__data, (page), count); \ ++ if (__data < (MIN)) \ ++ __data = (MIN); \ ++ else if (__data > (MAX)) \ ++ __data = (MAX); \ ++ *(__PTR) = (u64)__data * NSEC_PER_USEC; \ ++ return ret; \ ++} ++USEC_STORE_FUNCTION(bfq_slice_idle_us_store, &bfqd->bfq_slice_idle, 0, ++ UINT_MAX); ++#undef USEC_STORE_FUNCTION ++ ++/* do nothing for the moment */ ++static ssize_t bfq_weights_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ return count; ++} ++ ++static ssize_t bfq_max_budget_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ unsigned long uninitialized_var(__data); ++ int ret = bfq_var_store(&__data, (page), count); ++ ++ if (__data == 0) ++ bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); ++ else { ++ if (__data > INT_MAX) ++ __data = INT_MAX; ++ bfqd->bfq_max_budget = __data; ++ } ++ ++ bfqd->bfq_user_max_budget = __data; ++ ++ return ret; ++} ++ ++/* ++ * Leaving this name to preserve name compatibility with cfq ++ * parameters, but this timeout is used for both sync and async. ++ */ ++static ssize_t bfq_timeout_sync_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ unsigned long uninitialized_var(__data); ++ int ret = bfq_var_store(&__data, (page), count); ++ ++ if (__data < 1) ++ __data = 1; ++ else if (__data > INT_MAX) ++ __data = INT_MAX; ++ ++ bfqd->bfq_timeout = msecs_to_jiffies(__data); ++ if (bfqd->bfq_user_max_budget == 0) ++ bfqd->bfq_max_budget = bfq_calc_max_budget(bfqd); ++ ++ return ret; ++} ++ ++static ssize_t bfq_strict_guarantees_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ unsigned long uninitialized_var(__data); ++ int ret = bfq_var_store(&__data, (page), count); ++ ++ if (__data > 1) ++ __data = 1; ++ if (!bfqd->strict_guarantees && __data == 1 ++ && bfqd->bfq_slice_idle < 8 * NSEC_PER_MSEC) ++ bfqd->bfq_slice_idle = 8 * NSEC_PER_MSEC; ++ ++ bfqd->strict_guarantees = __data; ++ ++ return ret; ++} ++ ++static ssize_t bfq_low_latency_store(struct elevator_queue *e, ++ const char *page, size_t count) ++{ ++ struct bfq_data *bfqd = e->elevator_data; ++ unsigned long uninitialized_var(__data); ++ int ret = bfq_var_store(&__data, (page), count); ++ ++ if (__data > 1) ++ __data = 1; ++ if (__data == 0 && bfqd->low_latency != 0) ++ bfq_end_wr(bfqd); ++ bfqd->low_latency = __data; ++ ++ return ret; ++} ++ ++#define BFQ_ATTR(name) \ ++ __ATTR(name, S_IRUGO|S_IWUSR, bfq_##name##_show, bfq_##name##_store) ++ ++static struct elv_fs_entry bfq_attrs[] = { ++ BFQ_ATTR(fifo_expire_sync), ++ BFQ_ATTR(fifo_expire_async), ++ BFQ_ATTR(back_seek_max), ++ BFQ_ATTR(back_seek_penalty), ++ BFQ_ATTR(slice_idle), ++ BFQ_ATTR(slice_idle_us), ++ BFQ_ATTR(max_budget), ++ BFQ_ATTR(timeout_sync), ++ BFQ_ATTR(strict_guarantees), ++ BFQ_ATTR(low_latency), ++ BFQ_ATTR(wr_coeff), ++ BFQ_ATTR(wr_max_time), ++ BFQ_ATTR(wr_rt_max_time), ++ BFQ_ATTR(wr_min_idle_time), ++ BFQ_ATTR(wr_min_inter_arr_async), ++ BFQ_ATTR(wr_max_softrt_rate), ++ BFQ_ATTR(weights), ++ __ATTR_NULL ++}; ++ ++static struct elevator_type iosched_bfq = { ++ .ops = { ++ .elevator_merge_fn = bfq_merge, ++ .elevator_merged_fn = bfq_merged_request, ++ .elevator_merge_req_fn = bfq_merged_requests, ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ .elevator_bio_merged_fn = bfq_bio_merged, ++#endif ++ .elevator_allow_bio_merge_fn = bfq_allow_bio_merge, ++ .elevator_allow_rq_merge_fn = bfq_allow_rq_merge, ++ .elevator_dispatch_fn = bfq_dispatch_requests, ++ .elevator_add_req_fn = bfq_insert_request, ++ .elevator_activate_req_fn = bfq_activate_request, ++ .elevator_deactivate_req_fn = bfq_deactivate_request, ++ .elevator_completed_req_fn = bfq_completed_request, ++ .elevator_former_req_fn = elv_rb_former_request, ++ .elevator_latter_req_fn = elv_rb_latter_request, ++ .elevator_init_icq_fn = bfq_init_icq, ++ .elevator_exit_icq_fn = bfq_exit_icq, ++ .elevator_set_req_fn = bfq_set_request, ++ .elevator_put_req_fn = bfq_put_request, ++ .elevator_may_queue_fn = bfq_may_queue, ++ .elevator_init_fn = bfq_init_queue, ++ .elevator_exit_fn = bfq_exit_queue, ++ }, ++ .icq_size = sizeof(struct bfq_io_cq), ++ .icq_align = __alignof__(struct bfq_io_cq), ++ .elevator_attrs = bfq_attrs, ++ .elevator_name = "bfq", ++ .elevator_owner = THIS_MODULE, ++}; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static struct blkcg_policy blkcg_policy_bfq = { ++ .dfl_cftypes = bfq_blkg_files, ++ .legacy_cftypes = bfq_blkcg_legacy_files, ++ ++ .cpd_alloc_fn = bfq_cpd_alloc, ++ .cpd_init_fn = bfq_cpd_init, ++ .cpd_bind_fn = bfq_cpd_init, ++ .cpd_free_fn = bfq_cpd_free, ++ ++ .pd_alloc_fn = bfq_pd_alloc, ++ .pd_init_fn = bfq_pd_init, ++ .pd_offline_fn = bfq_pd_offline, ++ .pd_free_fn = bfq_pd_free, ++ .pd_reset_stats_fn = bfq_pd_reset_stats, ++}; ++#endif ++ ++static int __init bfq_init(void) ++{ ++ int ret; ++ char msg[50] = "BFQ I/O-scheduler: v8r4"; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ ret = blkcg_policy_register(&blkcg_policy_bfq); ++ if (ret) ++ return ret; ++#endif ++ ++ ret = -ENOMEM; ++ if (bfq_slab_setup()) ++ goto err_pol_unreg; ++ ++ /* ++ * Times to load large popular applications for the typical ++ * systems installed on the reference devices (see the ++ * comments before the definitions of the next two ++ * arrays). Actually, we use slightly slower values, as the ++ * estimated peak rate tends to be smaller than the actual ++ * peak rate. The reason for this last fact is that estimates ++ * are computed over much shorter time intervals than the long ++ * intervals typically used for benchmarking. Why? First, to ++ * adapt more quickly to variations. Second, because an I/O ++ * scheduler cannot rely on a peak-rate-evaluation workload to ++ * be run for a long time. ++ */ ++ T_slow[0] = msecs_to_jiffies(3500); /* actually 4 sec */ ++ T_slow[1] = msecs_to_jiffies(1000); /* actually 1.5 sec */ ++ T_fast[0] = msecs_to_jiffies(7000); /* actually 8 sec */ ++ T_fast[1] = msecs_to_jiffies(2500); /* actually 3 sec */ ++ ++ /* ++ * Thresholds that determine the switch between speed classes ++ * (see the comments before the definition of the array ++ * device_speed_thresh). These thresholds are biased towards ++ * transitions to the fast class. This is safer than the ++ * opposite bias. In fact, a wrong transition to the slow ++ * class results in short weight-raising periods, because the ++ * speed of the device then tends to be higher that the ++ * reference peak rate. On the opposite end, a wrong ++ * transition to the fast class tends to increase ++ * weight-raising periods, because of the opposite reason. ++ */ ++ device_speed_thresh[0] = (4 * R_slow[0]) / 3; ++ device_speed_thresh[1] = (4 * R_slow[1]) / 3; ++ ++ ret = elv_register(&iosched_bfq); ++ if (ret) ++ goto err_pol_unreg; ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ strcat(msg, " (with cgroups support)"); ++#endif ++ pr_info("%s", msg); ++ ++ return 0; ++ ++err_pol_unreg: ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ blkcg_policy_unregister(&blkcg_policy_bfq); ++#endif ++ return ret; ++} ++ ++static void __exit bfq_exit(void) ++{ ++ elv_unregister(&iosched_bfq); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ blkcg_policy_unregister(&blkcg_policy_bfq); ++#endif ++ bfq_slab_kill(); ++} ++ ++module_init(bfq_init); ++module_exit(bfq_exit); ++ ++MODULE_AUTHOR("Arianna Avanzini, Fabio Checconi, Paolo Valente"); ++MODULE_LICENSE("GPL"); +diff -w -uNr '--exclude=.git' linux-4.8/block/bfq-sched.c linux-4.8-bfq/block/bfq-sched.c +--- linux-4.8/block/bfq-sched.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.8-bfq/block/bfq-sched.c 2016-10-18 12:57:57.691164881 +0200 +@@ -0,0 +1,1501 @@ ++/* ++ * BFQ: Hierarchical B-WF2Q+ scheduler. ++ * ++ * Based on ideas and code from CFQ: ++ * Copyright (C) 2003 Jens Axboe ++ * ++ * Copyright (C) 2008 Fabio Checconi ++ * Paolo Valente ++ * ++ * Copyright (C) 2015 Paolo Valente ++ * ++ * Copyright (C) 2016 Paolo Valente ++ */ ++ ++static struct bfq_group *bfqq_group(struct bfq_queue *bfqq); ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++#define for_each_entity(entity) \ ++ for (; entity ; entity = entity->parent) ++ ++#define for_each_entity_safe(entity, parent) \ ++ for (; entity && ({ parent = entity->parent; 1; }); entity = parent) ++ ++ ++static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, ++ int extract, ++ struct bfq_data *bfqd); ++ ++static void bfq_update_budget(struct bfq_entity *next_in_service) ++{ ++ struct bfq_entity *bfqg_entity; ++ struct bfq_group *bfqg; ++ struct bfq_sched_data *group_sd; ++ ++ BUG_ON(!next_in_service); ++ ++ group_sd = next_in_service->sched_data; ++ ++ bfqg = container_of(group_sd, struct bfq_group, sched_data); ++ /* ++ * bfq_group's my_entity field is not NULL only if the group ++ * is not the root group. We must not touch the root entity ++ * as it must never become an in-service entity. ++ */ ++ bfqg_entity = bfqg->my_entity; ++ if (bfqg_entity) ++ bfqg_entity->budget = next_in_service->budget; ++} ++ ++static int bfq_update_next_in_service(struct bfq_sched_data *sd) ++{ ++ struct bfq_entity *next_in_service; ++ struct bfq_queue *bfqq; ++ ++ if (sd->in_service_entity) ++ /* will update/requeue at the end of service */ ++ return 0; ++ ++ /* ++ * NOTE: this can be improved in many ways, such as returning ++ * 1 (and thus propagating upwards the update) only when the ++ * budget changes, or caching the bfqq that will be scheduled ++ * next from this subtree. By now we worry more about ++ * correctness than about performance... ++ */ ++ next_in_service = bfq_lookup_next_entity(sd, 0, NULL); ++ sd->next_in_service = next_in_service; ++ ++ if (next_in_service) ++ bfq_update_budget(next_in_service); ++ else ++ goto exit; ++ ++ bfqq = bfq_entity_to_bfqq(next_in_service); ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "update_next_in_service: chosen this queue"); ++ else { ++ struct bfq_group *bfqg = ++ container_of(next_in_service, ++ struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "update_next_in_service: chosen this entity"); ++ } ++exit: ++ return 1; ++} ++ ++static void bfq_check_next_in_service(struct bfq_sched_data *sd, ++ struct bfq_entity *entity) ++{ ++ WARN_ON(sd->next_in_service != entity); ++} ++#else ++#define for_each_entity(entity) \ ++ for (; entity ; entity = NULL) ++ ++#define for_each_entity_safe(entity, parent) \ ++ for (parent = NULL; entity ; entity = parent) ++ ++static int bfq_update_next_in_service(struct bfq_sched_data *sd) ++{ ++ return 0; ++} ++ ++static void bfq_check_next_in_service(struct bfq_sched_data *sd, ++ struct bfq_entity *entity) ++{ ++} ++ ++static void bfq_update_budget(struct bfq_entity *next_in_service) ++{ ++} ++#endif ++ ++/* ++ * Shift for timestamp calculations. This actually limits the maximum ++ * service allowed in one timestamp delta (small shift values increase it), ++ * the maximum total weight that can be used for the queues in the system ++ * (big shift values increase it), and the period of virtual time ++ * wraparounds. ++ */ ++#define WFQ_SERVICE_SHIFT 22 ++ ++/** ++ * bfq_gt - compare two timestamps. ++ * @a: first ts. ++ * @b: second ts. ++ * ++ * Return @a > @b, dealing with wrapping correctly. ++ */ ++static int bfq_gt(u64 a, u64 b) ++{ ++ return (s64)(a - b) > 0; ++} ++ ++static struct bfq_queue *bfq_entity_to_bfqq(struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = NULL; ++ ++ BUG_ON(!entity); ++ ++ if (!entity->my_sched_data) ++ bfqq = container_of(entity, struct bfq_queue, entity); ++ ++ return bfqq; ++} ++ ++ ++/** ++ * bfq_delta - map service into the virtual time domain. ++ * @service: amount of service. ++ * @weight: scale factor (weight of an entity or weight sum). ++ */ ++static u64 bfq_delta(unsigned long service, unsigned long weight) ++{ ++ u64 d = (u64)service << WFQ_SERVICE_SHIFT; ++ ++ do_div(d, weight); ++ return d; ++} ++ ++/** ++ * bfq_calc_finish - assign the finish time to an entity. ++ * @entity: the entity to act upon. ++ * @service: the service to be charged to the entity. ++ */ ++static void bfq_calc_finish(struct bfq_entity *entity, unsigned long service) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ unsigned long long start, finish, delta; ++ ++ BUG_ON(entity->weight == 0); ++ ++ entity->finish = entity->start + ++ bfq_delta(service, entity->weight); ++ ++ start = ((entity->start>>10)*1000)>>12; ++ finish = ((entity->finish>>10)*1000)>>12; ++ delta = ((bfq_delta(service, entity->weight)>>10)*1000)>>12; ++ ++ if (bfqq) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "calc_finish: serv %lu, w %d", ++ service, entity->weight); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "calc_finish: start %llu, finish %llu, delta %llu", ++ start, finish, delta); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ } else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "calc_finish group: serv %lu, w %d", ++ service, entity->weight); ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "calc_finish group: start %llu, finish %llu, delta %llu", ++ start, finish, delta); ++#endif ++ } ++} ++ ++/** ++ * bfq_entity_of - get an entity from a node. ++ * @node: the node field of the entity. ++ * ++ * Convert a node pointer to the relative entity. This is used only ++ * to simplify the logic of some functions and not as the generic ++ * conversion mechanism because, e.g., in the tree walking functions, ++ * the check for a %NULL value would be redundant. ++ */ ++static struct bfq_entity *bfq_entity_of(struct rb_node *node) ++{ ++ struct bfq_entity *entity = NULL; ++ ++ if (node) ++ entity = rb_entry(node, struct bfq_entity, rb_node); ++ ++ return entity; ++} ++ ++/** ++ * bfq_extract - remove an entity from a tree. ++ * @root: the tree root. ++ * @entity: the entity to remove. ++ */ ++static void bfq_extract(struct rb_root *root, struct bfq_entity *entity) ++{ ++ BUG_ON(entity->tree != root); ++ ++ entity->tree = NULL; ++ rb_erase(&entity->rb_node, root); ++} ++ ++/** ++ * bfq_idle_extract - extract an entity from the idle tree. ++ * @st: the service tree of the owning @entity. ++ * @entity: the entity being removed. ++ */ ++static void bfq_idle_extract(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct rb_node *next; ++ ++ BUG_ON(entity->tree != &st->idle); ++ ++ if (entity == st->first_idle) { ++ next = rb_next(&entity->rb_node); ++ st->first_idle = bfq_entity_of(next); ++ } ++ ++ if (entity == st->last_idle) { ++ next = rb_prev(&entity->rb_node); ++ st->last_idle = bfq_entity_of(next); ++ } ++ ++ bfq_extract(&st->idle, entity); ++ ++ if (bfqq) ++ list_del(&bfqq->bfqq_list); ++} ++ ++/** ++ * bfq_insert - generic tree insertion. ++ * @root: tree root. ++ * @entity: entity to insert. ++ * ++ * This is used for the idle and the active tree, since they are both ++ * ordered by finish time. ++ */ ++static void bfq_insert(struct rb_root *root, struct bfq_entity *entity) ++{ ++ struct bfq_entity *entry; ++ struct rb_node **node = &root->rb_node; ++ struct rb_node *parent = NULL; ++ ++ BUG_ON(entity->tree); ++ ++ while (*node) { ++ parent = *node; ++ entry = rb_entry(parent, struct bfq_entity, rb_node); ++ ++ if (bfq_gt(entry->finish, entity->finish)) ++ node = &parent->rb_left; ++ else ++ node = &parent->rb_right; ++ } ++ ++ rb_link_node(&entity->rb_node, parent, node); ++ rb_insert_color(&entity->rb_node, root); ++ ++ entity->tree = root; ++} ++ ++/** ++ * bfq_update_min - update the min_start field of a entity. ++ * @entity: the entity to update. ++ * @node: one of its children. ++ * ++ * This function is called when @entity may store an invalid value for ++ * min_start due to updates to the active tree. The function assumes ++ * that the subtree rooted at @node (which may be its left or its right ++ * child) has a valid min_start value. ++ */ ++static void bfq_update_min(struct bfq_entity *entity, struct rb_node *node) ++{ ++ struct bfq_entity *child; ++ ++ if (node) { ++ child = rb_entry(node, struct bfq_entity, rb_node); ++ if (bfq_gt(entity->min_start, child->min_start)) ++ entity->min_start = child->min_start; ++ } ++} ++ ++/** ++ * bfq_update_active_node - recalculate min_start. ++ * @node: the node to update. ++ * ++ * @node may have changed position or one of its children may have moved, ++ * this function updates its min_start value. The left and right subtrees ++ * are assumed to hold a correct min_start value. ++ */ ++static void bfq_update_active_node(struct rb_node *node) ++{ ++ struct bfq_entity *entity = rb_entry(node, struct bfq_entity, rb_node); ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ entity->min_start = entity->start; ++ bfq_update_min(entity, node->rb_right); ++ bfq_update_min(entity, node->rb_left); ++ ++ if (bfqq) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "update_active_node: new min_start %llu", ++ ((entity->min_start>>10)*1000)>>12); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ } else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "update_active_node: new min_start %llu", ++ ((entity->min_start>>10)*1000)>>12); ++#endif ++ } ++} ++ ++/** ++ * bfq_update_active_tree - update min_start for the whole active tree. ++ * @node: the starting node. ++ * ++ * @node must be the deepest modified node after an update. This function ++ * updates its min_start using the values held by its children, assuming ++ * that they did not change, and then updates all the nodes that may have ++ * changed in the path to the root. The only nodes that may have changed ++ * are the ones in the path or their siblings. ++ */ ++static void bfq_update_active_tree(struct rb_node *node) ++{ ++ struct rb_node *parent; ++ ++up: ++ bfq_update_active_node(node); ++ ++ parent = rb_parent(node); ++ if (!parent) ++ return; ++ ++ if (node == parent->rb_left && parent->rb_right) ++ bfq_update_active_node(parent->rb_right); ++ else if (parent->rb_left) ++ bfq_update_active_node(parent->rb_left); ++ ++ node = parent; ++ goto up; ++} ++ ++static void bfq_weights_tree_add(struct bfq_data *bfqd, ++ struct bfq_entity *entity, ++ struct rb_root *root); ++ ++static void bfq_weights_tree_remove(struct bfq_data *bfqd, ++ struct bfq_entity *entity, ++ struct rb_root *root); ++ ++ ++/** ++ * bfq_active_insert - insert an entity in the active tree of its ++ * group/device. ++ * @st: the service tree of the entity. ++ * @entity: the entity being inserted. ++ * ++ * The active tree is ordered by finish time, but an extra key is kept ++ * per each node, containing the minimum value for the start times of ++ * its children (and the node itself), so it's possible to search for ++ * the eligible node with the lowest finish time in logarithmic time. ++ */ ++static void bfq_active_insert(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct rb_node *node = &entity->rb_node; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ struct bfq_sched_data *sd = NULL; ++ struct bfq_group *bfqg = NULL; ++ struct bfq_data *bfqd = NULL; ++#endif ++ ++ bfq_insert(&st->active, entity); ++ ++ if (node->rb_left) ++ node = node->rb_left; ++ else if (node->rb_right) ++ node = node->rb_right; ++ ++ bfq_update_active_tree(node); ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ sd = entity->sched_data; ++ bfqg = container_of(sd, struct bfq_group, sched_data); ++ BUG_ON(!bfqg); ++ bfqd = (struct bfq_data *)bfqg->bfqd; ++#endif ++ if (bfqq) ++ list_add(&bfqq->bfqq_list, &bfqq->bfqd->active_list); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { /* bfq_group */ ++ BUG_ON(!bfqd); ++ bfq_weights_tree_add(bfqd, entity, &bfqd->group_weights_tree); ++ } ++ if (bfqg != bfqd->root_group) { ++ BUG_ON(!bfqg); ++ BUG_ON(!bfqd); ++ bfqg->active_entities++; ++ } ++#endif ++} ++ ++/** ++ * bfq_ioprio_to_weight - calc a weight from an ioprio. ++ * @ioprio: the ioprio value to convert. ++ */ ++static unsigned short bfq_ioprio_to_weight(int ioprio) ++{ ++ BUG_ON(ioprio < 0 || ioprio >= IOPRIO_BE_NR); ++ return (IOPRIO_BE_NR - ioprio) * BFQ_WEIGHT_CONVERSION_COEFF; ++} ++ ++/** ++ * bfq_weight_to_ioprio - calc an ioprio from a weight. ++ * @weight: the weight value to convert. ++ * ++ * To preserve as much as possible the old only-ioprio user interface, ++ * 0 is used as an escape ioprio value for weights (numerically) equal or ++ * larger than IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF. ++ */ ++static unsigned short bfq_weight_to_ioprio(int weight) ++{ ++ BUG_ON(weight < BFQ_MIN_WEIGHT || weight > BFQ_MAX_WEIGHT); ++ return IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - weight < 0 ? ++ 0 : IOPRIO_BE_NR * BFQ_WEIGHT_CONVERSION_COEFF - weight; ++} ++ ++static void bfq_get_entity(struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ if (bfqq) { ++ bfqq->ref++; ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "get_entity: %p %d", ++ bfqq, bfqq->ref); ++ } ++} ++ ++/** ++ * bfq_find_deepest - find the deepest node that an extraction can modify. ++ * @node: the node being removed. ++ * ++ * Do the first step of an extraction in an rb tree, looking for the ++ * node that will replace @node, and returning the deepest node that ++ * the following modifications to the tree can touch. If @node is the ++ * last node in the tree return %NULL. ++ */ ++static struct rb_node *bfq_find_deepest(struct rb_node *node) ++{ ++ struct rb_node *deepest; ++ ++ if (!node->rb_right && !node->rb_left) ++ deepest = rb_parent(node); ++ else if (!node->rb_right) ++ deepest = node->rb_left; ++ else if (!node->rb_left) ++ deepest = node->rb_right; ++ else { ++ deepest = rb_next(node); ++ if (deepest->rb_right) ++ deepest = deepest->rb_right; ++ else if (rb_parent(deepest) != node) ++ deepest = rb_parent(deepest); ++ } ++ ++ return deepest; ++} ++ ++/** ++ * bfq_active_extract - remove an entity from the active tree. ++ * @st: the service_tree containing the tree. ++ * @entity: the entity being removed. ++ */ ++static void bfq_active_extract(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct rb_node *node; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ struct bfq_sched_data *sd = NULL; ++ struct bfq_group *bfqg = NULL; ++ struct bfq_data *bfqd = NULL; ++#endif ++ ++ node = bfq_find_deepest(&entity->rb_node); ++ bfq_extract(&st->active, entity); ++ ++ if (node) ++ bfq_update_active_tree(node); ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ sd = entity->sched_data; ++ bfqg = container_of(sd, struct bfq_group, sched_data); ++ BUG_ON(!bfqg); ++ bfqd = (struct bfq_data *)bfqg->bfqd; ++#endif ++ if (bfqq) ++ list_del(&bfqq->bfqq_list); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { /* bfq_group */ ++ BUG_ON(!bfqd); ++ bfq_weights_tree_remove(bfqd, entity, ++ &bfqd->group_weights_tree); ++ } ++ if (bfqg != bfqd->root_group) { ++ BUG_ON(!bfqg); ++ BUG_ON(!bfqd); ++ BUG_ON(!bfqg->active_entities); ++ bfqg->active_entities--; ++ } ++#endif ++} ++ ++/** ++ * bfq_idle_insert - insert an entity into the idle tree. ++ * @st: the service tree containing the tree. ++ * @entity: the entity to insert. ++ */ ++static void bfq_idle_insert(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct bfq_entity *first_idle = st->first_idle; ++ struct bfq_entity *last_idle = st->last_idle; ++ ++ if (!first_idle || bfq_gt(first_idle->finish, entity->finish)) ++ st->first_idle = entity; ++ if (!last_idle || bfq_gt(entity->finish, last_idle->finish)) ++ st->last_idle = entity; ++ ++ bfq_insert(&st->idle, entity); ++ ++ if (bfqq) ++ list_add(&bfqq->bfqq_list, &bfqq->bfqd->idle_list); ++} ++ ++/** ++ * bfq_forget_entity - remove an entity from the wfq trees. ++ * @st: the service tree. ++ * @entity: the entity being removed. ++ * ++ * Update the device status and forget everything about @entity, putting ++ * the device reference to it, if it is a queue. Entities belonging to ++ * groups are not refcounted. ++ */ ++static void bfq_forget_entity(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ struct bfq_sched_data *sd; ++ ++ BUG_ON(!entity->on_st); ++ ++ entity->on_st = 0; ++ st->wsum -= entity->weight; ++ if (bfqq) { ++ sd = entity->sched_data; ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "forget_entity: %p %d", ++ bfqq, bfqq->ref); ++ bfq_put_queue(bfqq); ++ } ++} ++ ++/** ++ * bfq_put_idle_entity - release the idle tree ref of an entity. ++ * @st: service tree for the entity. ++ * @entity: the entity being released. ++ */ ++static void bfq_put_idle_entity(struct bfq_service_tree *st, ++ struct bfq_entity *entity) ++{ ++ bfq_idle_extract(st, entity); ++ bfq_forget_entity(st, entity); ++} ++ ++/** ++ * bfq_forget_idle - update the idle tree if necessary. ++ * @st: the service tree to act upon. ++ * ++ * To preserve the global O(log N) complexity we only remove one entry here; ++ * as the idle tree will not grow indefinitely this can be done safely. ++ */ ++static void bfq_forget_idle(struct bfq_service_tree *st) ++{ ++ struct bfq_entity *first_idle = st->first_idle; ++ struct bfq_entity *last_idle = st->last_idle; ++ ++ if (RB_EMPTY_ROOT(&st->active) && last_idle && ++ !bfq_gt(last_idle->finish, st->vtime)) { ++ /* ++ * Forget the whole idle tree, increasing the vtime past ++ * the last finish time of idle entities. ++ */ ++ st->vtime = last_idle->finish; ++ } ++ ++ if (first_idle && !bfq_gt(first_idle->finish, st->vtime)) ++ bfq_put_idle_entity(st, first_idle); ++} ++ ++static struct bfq_service_tree * ++__bfq_entity_update_weight_prio(struct bfq_service_tree *old_st, ++ struct bfq_entity *entity) ++{ ++ struct bfq_service_tree *new_st = old_st; ++ ++ if (entity->prio_changed) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ unsigned int prev_weight, new_weight; ++ struct bfq_data *bfqd = NULL; ++ struct rb_root *root; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ struct bfq_sched_data *sd; ++ struct bfq_group *bfqg; ++#endif ++ ++ if (bfqq) ++ bfqd = bfqq->bfqd; ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ sd = entity->my_sched_data; ++ bfqg = container_of(sd, struct bfq_group, sched_data); ++ BUG_ON(!bfqg); ++ bfqd = (struct bfq_data *)bfqg->bfqd; ++ BUG_ON(!bfqd); ++ } ++#endif ++ ++ BUG_ON(old_st->wsum < entity->weight); ++ old_st->wsum -= entity->weight; ++ ++ if (entity->new_weight != entity->orig_weight) { ++ if (entity->new_weight < BFQ_MIN_WEIGHT || ++ entity->new_weight > BFQ_MAX_WEIGHT) { ++ pr_crit("update_weight_prio: new_weight %d\n", ++ entity->new_weight); ++ if (entity->new_weight < BFQ_MIN_WEIGHT) ++ entity->new_weight = BFQ_MIN_WEIGHT; ++ else ++ entity->new_weight = BFQ_MAX_WEIGHT; ++ } ++ entity->orig_weight = entity->new_weight; ++ if (bfqq) ++ bfqq->ioprio = ++ bfq_weight_to_ioprio(entity->orig_weight); ++ } ++ ++ if (bfqq) ++ bfqq->ioprio_class = bfqq->new_ioprio_class; ++ entity->prio_changed = 0; ++ ++ /* ++ * NOTE: here we may be changing the weight too early, ++ * this will cause unfairness. The correct approach ++ * would have required additional complexity to defer ++ * weight changes to the proper time instants (i.e., ++ * when entity->finish <= old_st->vtime). ++ */ ++ new_st = bfq_entity_service_tree(entity); ++ ++ prev_weight = entity->weight; ++ new_weight = entity->orig_weight * ++ (bfqq ? bfqq->wr_coeff : 1); ++ /* ++ * If the weight of the entity changes, remove the entity ++ * from its old weight counter (if there is a counter ++ * associated with the entity), and add it to the counter ++ * associated with its new weight. ++ */ ++ if (prev_weight != new_weight) { ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "weight changed %d %d(%d %d)", ++ prev_weight, new_weight, ++ entity->orig_weight, ++ bfqq->wr_coeff); ++ ++ root = bfqq ? &bfqd->queue_weights_tree : ++ &bfqd->group_weights_tree; ++ bfq_weights_tree_remove(bfqd, entity, root); ++ } ++ entity->weight = new_weight; ++ /* ++ * Add the entity to its weights tree only if it is ++ * not associated with a weight-raised queue. ++ */ ++ if (prev_weight != new_weight && ++ (bfqq ? bfqq->wr_coeff == 1 : 1)) ++ /* If we get here, root has been initialized. */ ++ bfq_weights_tree_add(bfqd, entity, root); ++ ++ new_st->wsum += entity->weight; ++ ++ if (new_st != old_st) ++ entity->start = new_st->vtime; ++ } ++ ++ return new_st; ++} ++ ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++static void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg); ++#endif ++ ++/** ++ * bfq_bfqq_served - update the scheduler status after selection for ++ * service. ++ * @bfqq: the queue being served. ++ * @served: bytes to transfer. ++ * ++ * NOTE: this can be optimized, as the timestamps of upper level entities ++ * are synchronized every time a new bfqq is selected for service. By now, ++ * we keep it to better check consistency. ++ */ ++static void bfq_bfqq_served(struct bfq_queue *bfqq, int served) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ struct bfq_service_tree *st; ++ ++ for_each_entity(entity) { ++ st = bfq_entity_service_tree(entity); ++ ++ entity->service += served; ++ ++ BUG_ON(st->wsum == 0); ++ ++ st->vtime += bfq_delta(served, st->wsum); ++ bfq_forget_idle(st); ++ } ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ bfqg_stats_set_start_empty_time(bfqq_group(bfqq)); ++#endif ++ st = bfq_entity_service_tree(&bfqq->entity); ++ bfq_log_bfqq(bfqq->bfqd, bfqq, "bfqq_served %d secs, vtime %llu on %p", ++ served, ((st->vtime>>10)*1000)>>12, st); ++} ++ ++/** ++ * bfq_bfqq_charge_time - charge an amount of service equivalent to the length ++ * of the time interval during which bfqq has been in ++ * service. ++ * @bfqd: the device ++ * @bfqq: the queue that needs a service update. ++ * @time_ms: the amount of time during which the queue has received service ++ * ++ * If a queue does not consume its budget fast enough, then providing ++ * the queue with service fairness may impair throughput, more or less ++ * severely. For this reason, queues that consume their budget slowly ++ * are provided with time fairness instead of service fairness. This ++ * goal is achieved through the BFQ scheduling engine, even if such an ++ * engine works in the service, and not in the time domain. The trick ++ * is charging these queues with an inflated amount of service, equal ++ * to the amount of service that they would have received during their ++ * service slot if they had been fast, i.e., if their requests had ++ * been dispatched at a rate equal to the estimated peak rate. ++ * ++ * It is worth noting that time fairness can cause important ++ * distortions in terms of bandwidth distribution, on devices with ++ * internal queueing. The reason is that I/O requests dispatched ++ * during the service slot of a queue may be served after that service ++ * slot is finished, and may have a total processing time loosely ++ * correlated with the duration of the service slot. This is ++ * especially true for short service slots. ++ */ ++static void bfq_bfqq_charge_time(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ unsigned long time_ms) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ int tot_serv_to_charge = entity->service; ++ unsigned int timeout_ms = jiffies_to_msecs(bfq_timeout); ++ ++ if (time_ms > 0 && time_ms < timeout_ms) ++ tot_serv_to_charge = ++ (bfqd->bfq_max_budget * time_ms) / timeout_ms; ++ ++ if (tot_serv_to_charge < entity->service) ++ tot_serv_to_charge = entity->service; ++ ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "charge_time: %lu/%u ms, %d/%d/%d sectors", ++ time_ms, timeout_ms, entity->service, ++ tot_serv_to_charge, entity->budget); ++ ++ /* Increase budget to avoid inconsistencies */ ++ if (tot_serv_to_charge > entity->budget) ++ entity->budget = tot_serv_to_charge; ++ ++ bfq_bfqq_served(bfqq, ++ max_t(int, 0, tot_serv_to_charge - entity->service)); ++} ++ ++/** ++ * __bfq_activate_entity - activate an entity. ++ * @entity: the entity being activated. ++ * @non_blocking_wait_rq: true if this entity was waiting for a request ++ * ++ * Called whenever an entity is activated, i.e., it is not active and one ++ * of its children receives a new request, or has to be reactivated due to ++ * budget exhaustion. It uses the current budget of the entity (and the ++ * service received if @entity is active) of the queue to calculate its ++ * timestamps. ++ */ ++static void __bfq_activate_entity(struct bfq_entity *entity, ++ bool non_blocking_wait_rq) ++{ ++ struct bfq_sched_data *sd = entity->sched_data; ++ struct bfq_service_tree *st = bfq_entity_service_tree(entity); ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ bool backshifted = false; ++ ++ BUG_ON(!sd); ++ BUG_ON(!st); ++ if (entity == sd->in_service_entity) { ++ BUG_ON(entity->tree); ++ /* ++ * If we are requeueing the current entity we have ++ * to take care of not charging to it service it has ++ * not received. ++ */ ++ bfq_calc_finish(entity, entity->service); ++ entity->start = entity->finish; ++ sd->in_service_entity = NULL; ++ } else if (entity->tree == &st->active) { ++ /* ++ * Requeueing an entity due to a change of some ++ * next_in_service entity below it. We reuse the ++ * old start time. ++ */ ++ bfq_active_extract(st, entity); ++ } else { ++ unsigned long long min_vstart; ++ ++ /* See comments on bfq_fqq_update_budg_for_activation */ ++ if (non_blocking_wait_rq && bfq_gt(st->vtime, entity->finish)) { ++ backshifted = true; ++ min_vstart = entity->finish; ++ } else ++ min_vstart = st->vtime; ++ ++ if (entity->tree == &st->idle) { ++ /* ++ * Must be on the idle tree, bfq_idle_extract() will ++ * check for that. ++ */ ++ bfq_idle_extract(st, entity); ++ entity->start = bfq_gt(min_vstart, entity->finish) ? ++ min_vstart : entity->finish; ++ } else { ++ /* ++ * The finish time of the entity may be invalid, and ++ * it is in the past for sure, otherwise the queue ++ * would have been on the idle tree. ++ */ ++ entity->start = min_vstart; ++ st->wsum += entity->weight; ++ bfq_get_entity(entity); ++ ++ BUG_ON(entity->on_st); ++ entity->on_st = 1; ++ } ++ } ++ ++ st = __bfq_entity_update_weight_prio(st, entity); ++ bfq_calc_finish(entity, entity->budget); ++ ++ /* ++ * If some queues enjoy backshifting for a while, then their ++ * (virtual) finish timestamps may happen to become lower and ++ * lower than the system virtual time. In particular, if ++ * these queues often happen to be idle for short time ++ * periods, and during such time periods other queues with ++ * higher timestamps happen to be busy, then the backshifted ++ * timestamps of the former queues can become much lower than ++ * the system virtual time. In fact, to serve the queues with ++ * higher timestamps while the ones with lower timestamps are ++ * idle, the system virtual time may be pushed-up to much ++ * higher values than the finish timestamps of the idle ++ * queues. As a consequence, the finish timestamps of all new ++ * or newly activated queues may end up being much larger than ++ * those of lucky queues with backshifted timestamps. The ++ * latter queues may then monopolize the device for a lot of ++ * time. This would simply break service guarantees. ++ * ++ * To reduce this problem, push up a little bit the ++ * backshifted timestamps of the queue associated with this ++ * entity (only a queue can happen to have the backshifted ++ * flag set): just enough to let the finish timestamp of the ++ * queue be equal to the current value of the system virtual ++ * time. This may introduce a little unfairness among queues ++ * with backshifted timestamps, but it does not break ++ * worst-case fairness guarantees. ++ * ++ * As a special case, if bfqq is weight-raised, push up ++ * timestamps much less, to keep very low the probability that ++ * this push up causes the backshifted finish timestamps of ++ * weight-raised queues to become higher than the backshifted ++ * finish timestamps of non weight-raised queues. ++ */ ++ if (backshifted && bfq_gt(st->vtime, entity->finish)) { ++ unsigned long delta = st->vtime - entity->finish; ++ ++ if (bfqq) ++ delta /= bfqq->wr_coeff; ++ ++ entity->start += delta; ++ entity->finish += delta; ++ ++ if (bfqq) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "__activate_entity: new queue finish %llu", ++ ((entity->finish>>10)*1000)>>12); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ } else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "__activate_entity: new group finish %llu", ++ ((entity->finish>>10)*1000)>>12); ++#endif ++ } ++ } ++ ++ bfq_active_insert(st, entity); ++ ++ if (bfqq) { ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "__activate_entity: queue %seligible in st %p", ++ entity->start <= st->vtime ? "" : "non ", st); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ } else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "__activate_entity: group %seligible in st %p", ++ entity->start <= st->vtime ? "" : "non ", st); ++#endif ++ } ++} ++ ++/** ++ * bfq_activate_entity - activate an entity and its ancestors if necessary. ++ * @entity: the entity to activate. ++ * @non_blocking_wait_rq: true if this entity was waiting for a request ++ * ++ * Activate @entity and all the entities on the path from it to the root. ++ */ ++static void bfq_activate_entity(struct bfq_entity *entity, ++ bool non_blocking_wait_rq) ++{ ++ struct bfq_sched_data *sd; ++ ++ for_each_entity(entity) { ++ BUG_ON(!entity); ++ __bfq_activate_entity(entity, non_blocking_wait_rq); ++ ++ sd = entity->sched_data; ++ if (!bfq_update_next_in_service(sd)) ++ /* ++ * No need to propagate the activation to the ++ * upper entities, as they will be updated when ++ * the in-service entity is rescheduled. ++ */ ++ break; ++ } ++} ++ ++/** ++ * __bfq_deactivate_entity - deactivate an entity from its service tree. ++ * @entity: the entity to deactivate. ++ * @requeue: if false, the entity will not be put into the idle tree. ++ * ++ * Deactivate an entity, independently from its previous state. If the ++ * entity was not on a service tree just return, otherwise if it is on ++ * any scheduler tree, extract it from that tree, and if necessary ++ * and if the caller did not specify @requeue, put it on the idle tree. ++ * ++ * Return %1 if the caller should update the entity hierarchy, i.e., ++ * if the entity was in service or if it was the next_in_service for ++ * its sched_data; return %0 otherwise. ++ */ ++static int __bfq_deactivate_entity(struct bfq_entity *entity, int requeue) ++{ ++ struct bfq_sched_data *sd = entity->sched_data; ++ struct bfq_service_tree *st; ++ int was_in_service; ++ int ret = 0; ++ ++ if (sd == NULL || !entity->on_st) /* never activated, or inactive */ ++ return 0; ++ ++ st = bfq_entity_service_tree(entity); ++ was_in_service = entity == sd->in_service_entity; ++ ++ BUG_ON(was_in_service && entity->tree); ++ ++ if (was_in_service) { ++ bfq_calc_finish(entity, entity->service); ++ sd->in_service_entity = NULL; ++ } else if (entity->tree == &st->active) ++ bfq_active_extract(st, entity); ++ else if (entity->tree == &st->idle) ++ bfq_idle_extract(st, entity); ++ else if (entity->tree) ++ BUG(); ++ ++ if (was_in_service || sd->next_in_service == entity) ++ ret = bfq_update_next_in_service(sd); ++ ++ if (!requeue || !bfq_gt(entity->finish, st->vtime)) ++ bfq_forget_entity(st, entity); ++ else ++ bfq_idle_insert(st, entity); ++ ++ BUG_ON(sd->in_service_entity == entity); ++ BUG_ON(sd->next_in_service == entity); ++ ++ return ret; ++} ++ ++/** ++ * bfq_deactivate_entity - deactivate an entity. ++ * @entity: the entity to deactivate. ++ * @requeue: true if the entity can be put on the idle tree ++ */ ++static void bfq_deactivate_entity(struct bfq_entity *entity, int requeue) ++{ ++ struct bfq_sched_data *sd; ++ struct bfq_entity *parent; ++ ++ for_each_entity_safe(entity, parent) { ++ sd = entity->sched_data; ++ ++ if (!__bfq_deactivate_entity(entity, requeue)) ++ /* ++ * next_in_service has not been changed, so ++ * no upwards update is needed ++ */ ++ break; ++ ++ if (sd->next_in_service) ++ /* ++ * The parent entity is still backlogged, ++ * because next_in_service is not NULL, and ++ * next_in_service has been updated (see ++ * comment on the body of the above if): ++ * upwards update of the schedule is needed. ++ */ ++ goto update; ++ ++ /* ++ * If we get here, then the parent is no more backlogged and ++ * we want to propagate the deactivation upwards. ++ */ ++ requeue = 1; ++ } ++ ++ return; ++ ++update: ++ entity = parent; ++ for_each_entity(entity) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ __bfq_activate_entity(entity, false); ++ ++ sd = entity->sched_data; ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "invoking udpdate_next for this queue"); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, ++ struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "invoking udpdate_next for this entity"); ++ } ++#endif ++ if (!bfq_update_next_in_service(sd)) ++ break; ++ } ++} ++ ++/** ++ * bfq_update_vtime - update vtime if necessary. ++ * @st: the service tree to act upon. ++ * ++ * If necessary update the service tree vtime to have at least one ++ * eligible entity, skipping to its start time. Assumes that the ++ * active tree of the device is not empty. ++ * ++ * NOTE: this hierarchical implementation updates vtimes quite often, ++ * we may end up with reactivated processes getting timestamps after a ++ * vtime skip done because we needed a ->first_active entity on some ++ * intermediate node. ++ */ ++static void bfq_update_vtime(struct bfq_service_tree *st) ++{ ++ struct bfq_entity *entry; ++ struct rb_node *node = st->active.rb_node; ++ ++ entry = rb_entry(node, struct bfq_entity, rb_node); ++ if (bfq_gt(entry->min_start, st->vtime)) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entry); ++ st->vtime = entry->min_start; ++ ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "update_vtime: new vtime %llu %p", ++ ((st->vtime>>10)*1000)>>12, st); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entry, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "update_vtime: new vtime %llu %p", ++ ((st->vtime>>10)*1000)>>12, st); ++ } ++#endif ++ bfq_forget_idle(st); ++ } ++} ++ ++/** ++ * bfq_first_active_entity - find the eligible entity with ++ * the smallest finish time ++ * @st: the service tree to select from. ++ * ++ * This function searches the first schedulable entity, starting from the ++ * root of the tree and going on the left every time on this side there is ++ * a subtree with at least one eligible (start >= vtime) entity. The path on ++ * the right is followed only if a) the left subtree contains no eligible ++ * entities and b) no eligible entity has been found yet. ++ */ ++static struct bfq_entity *bfq_first_active_entity(struct bfq_service_tree *st) ++{ ++ struct bfq_entity *entry, *first = NULL; ++ struct rb_node *node = st->active.rb_node; ++ ++ while (node) { ++ entry = rb_entry(node, struct bfq_entity, rb_node); ++left: ++ if (!bfq_gt(entry->start, st->vtime)) ++ first = entry; ++ ++ BUG_ON(bfq_gt(entry->min_start, st->vtime)); ++ ++ if (node->rb_left) { ++ entry = rb_entry(node->rb_left, ++ struct bfq_entity, rb_node); ++ if (!bfq_gt(entry->min_start, st->vtime)) { ++ node = node->rb_left; ++ goto left; ++ } ++ } ++ if (first) ++ break; ++ node = node->rb_right; ++ } ++ ++ BUG_ON(!first && !RB_EMPTY_ROOT(&st->active)); ++ return first; ++} ++ ++/** ++ * __bfq_lookup_next_entity - return the first eligible entity in @st. ++ * @st: the service tree. ++ * ++ * Update the virtual time in @st and return the first eligible entity ++ * it contains. ++ */ ++static struct bfq_entity * ++__bfq_lookup_next_entity(struct bfq_service_tree *st, bool force) ++{ ++ struct bfq_entity *entity, *new_next_in_service = NULL; ++ struct bfq_queue *bfqq; ++ ++ if (RB_EMPTY_ROOT(&st->active)) ++ return NULL; ++ ++ bfq_update_vtime(st); ++ entity = bfq_first_active_entity(st); ++ BUG_ON(bfq_gt(entity->start, st->vtime)); ++ ++ bfqq = bfq_entity_to_bfqq(entity); ++ if (bfqq) ++ bfq_log_bfqq(bfqq->bfqd, bfqq, ++ "__lookup_next: start %llu vtime %llu st %p", ++ ((entity->start>>10)*1000)>>12, ++ ((st->vtime>>10)*1000)>>12, st); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg((struct bfq_data *)bfqg->bfqd, bfqg, ++ "__lookup_next: start %llu vtime %llu st %p", ++ ((entity->start>>10)*1000)>>12, ++ ((st->vtime>>10)*1000)>>12, st); ++ } ++#endif ++ ++ /* ++ * If the chosen entity does not match with the sched_data's ++ * next_in_service and we are forcedly serving the IDLE priority ++ * class tree, bubble up budget update. ++ */ ++ if (unlikely(force && entity != entity->sched_data->next_in_service)) { ++ new_next_in_service = entity; ++ for_each_entity(new_next_in_service) ++ bfq_update_budget(new_next_in_service); ++ } ++ ++ return entity; ++} ++ ++/** ++ * bfq_lookup_next_entity - return the first eligible entity in @sd. ++ * @sd: the sched_data. ++ * @extract: if true the returned entity will be also extracted from @sd. ++ * ++ * NOTE: since we cache the next_in_service entity at each level of the ++ * hierarchy, the complexity of the lookup can be decreased with ++ * absolutely no effort just returning the cached next_in_service value; ++ * we prefer to do full lookups to test the consistency of * the data ++ * structures. ++ */ ++static struct bfq_entity *bfq_lookup_next_entity(struct bfq_sched_data *sd, ++ int extract, ++ struct bfq_data *bfqd) ++{ ++ struct bfq_service_tree *st = sd->service_tree; ++ struct bfq_entity *entity; ++ int i = 0; ++ ++ BUG_ON(sd->in_service_entity); ++ ++ /* ++ * Choose from idle class, if needed to guarantee a minimum ++ * bandwidth to this class. This should also mitigate ++ * priority-inversion problems in case a low priority task is ++ * holding file system resources. ++ */ ++ if (bfqd && ++ jiffies - bfqd->bfq_class_idle_last_service > ++ BFQ_CL_IDLE_TIMEOUT) { ++ entity = __bfq_lookup_next_entity(st + BFQ_IOPRIO_CLASSES - 1, ++ true); ++ if (entity) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ if (bfqq) ++ bfq_log_bfqq(bfqd, bfqq, ++ "idle chosen from st %p %d", ++ st + BFQ_IOPRIO_CLASSES - 1, ++ BFQ_IOPRIO_CLASSES - 1); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg(bfqd, bfqg, ++ "idle chosen from st %p %d", ++ st + BFQ_IOPRIO_CLASSES - 1, ++ BFQ_IOPRIO_CLASSES - 1); ++ } ++#endif ++ i = BFQ_IOPRIO_CLASSES - 1; ++ bfqd->bfq_class_idle_last_service = jiffies; ++ sd->next_in_service = entity; ++ } ++ } ++ for (; i < BFQ_IOPRIO_CLASSES; i++) { ++ entity = __bfq_lookup_next_entity(st + i, false); ++ if (entity) { ++ if (bfqd != NULL) { ++ struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity); ++ ++ if (bfqq) ++ bfq_log_bfqq(bfqd, bfqq, ++ "chosen from st %p %d", ++ st + i, i); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg(bfqd, bfqg, ++ "chosen from st %p %d", ++ st + i, i); ++ } ++#endif ++ } ++ ++ if (extract) { ++ bfq_check_next_in_service(sd, entity); ++ bfq_active_extract(st + i, entity); ++ sd->in_service_entity = entity; ++ sd->next_in_service = NULL; ++ } ++ break; ++ } ++ } ++ ++ return entity; ++} ++ ++static bool next_queue_may_preempt(struct bfq_data *bfqd) ++{ ++ struct bfq_sched_data *sd = &bfqd->root_group->sched_data; ++ ++ return sd->next_in_service != sd->in_service_entity; ++} ++ ++/* ++ * Get next queue for service. ++ */ ++static struct bfq_queue *bfq_get_next_queue(struct bfq_data *bfqd) ++{ ++ struct bfq_entity *entity = NULL; ++ struct bfq_sched_data *sd; ++ struct bfq_queue *bfqq; ++ ++ BUG_ON(bfqd->in_service_queue); ++ ++ if (bfqd->busy_queues == 0) ++ return NULL; ++ ++ sd = &bfqd->root_group->sched_data; ++ for (; sd ; sd = entity->my_sched_data) { ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ if (entity) { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg(bfqd, bfqg, ++ "get_next_queue: lookup in this group"); ++ } else ++ bfq_log_bfqg(bfqd, bfqd->root_group, ++ "get_next_queue: lookup in root group"); ++#endif ++ ++ entity = bfq_lookup_next_entity(sd, 1, bfqd); ++ ++ bfqq = bfq_entity_to_bfqq(entity); ++ if (bfqq) ++ bfq_log_bfqq(bfqd, bfqq, ++ "get_next_queue: this queue, finish %llu", ++ (((entity->finish>>10)*1000)>>10)>>2); ++#ifdef CONFIG_BFQ_GROUP_IOSCHED ++ else { ++ struct bfq_group *bfqg = ++ container_of(entity, struct bfq_group, entity); ++ ++ bfq_log_bfqg(bfqd, bfqg, ++ "get_next_queue: this entity, finish %llu", ++ (((entity->finish>>10)*1000)>>10)>>2); ++ } ++#endif ++ ++ BUG_ON(!entity); ++ entity->service = 0; ++ } ++ ++ bfqq = bfq_entity_to_bfqq(entity); ++ BUG_ON(!bfqq); ++ ++ return bfqq; ++} ++ ++static void __bfq_bfqd_reset_in_service(struct bfq_data *bfqd) ++{ ++ if (bfqd->in_service_bic) { ++ put_io_context(bfqd->in_service_bic->icq.ioc); ++ bfqd->in_service_bic = NULL; ++ } ++ ++ bfq_clear_bfqq_wait_request(bfqd->in_service_queue); ++ hrtimer_try_to_cancel(&bfqd->idle_slice_timer); ++ bfqd->in_service_queue = NULL; ++} ++ ++static void bfq_deactivate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ int requeue) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ bfq_deactivate_entity(entity, requeue); ++} ++ ++static void bfq_activate_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ struct bfq_entity *entity = &bfqq->entity; ++ ++ bfq_activate_entity(entity, bfq_bfqq_non_blocking_wait_rq(bfqq)); ++ bfq_clear_bfqq_non_blocking_wait_rq(bfqq); ++} ++ ++static void bfqg_stats_update_dequeue(struct bfq_group *bfqg); ++ ++/* ++ * Called when the bfqq no longer has requests pending, remove it from ++ * the service tree. ++ */ ++static void bfq_del_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq, ++ int requeue) ++{ ++ BUG_ON(!bfq_bfqq_busy(bfqq)); ++ BUG_ON(!RB_EMPTY_ROOT(&bfqq->sort_list)); ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ ++ bfq_log_bfqq(bfqd, bfqq, "del from busy"); ++ ++ bfq_clear_bfqq_busy(bfqq); ++ ++ BUG_ON(bfqd->busy_queues == 0); ++ bfqd->busy_queues--; ++ ++ if (!bfqq->dispatched) ++ bfq_weights_tree_remove(bfqd, &bfqq->entity, ++ &bfqd->queue_weights_tree); ++ ++ if (bfqq->wr_coeff > 1) ++ bfqd->wr_busy_queues--; ++ ++ bfqg_stats_update_dequeue(bfqq_group(bfqq)); ++ ++ BUG_ON(bfqq->entity.budget < 0); ++ ++ bfq_deactivate_bfqq(bfqd, bfqq, requeue); ++ ++ BUG_ON(bfqq->entity.budget < 0); ++} ++ ++/* ++ * Called when an inactive queue receives a new request. ++ */ ++static void bfq_add_bfqq_busy(struct bfq_data *bfqd, struct bfq_queue *bfqq) ++{ ++ BUG_ON(bfq_bfqq_busy(bfqq)); ++ BUG_ON(bfqq == bfqd->in_service_queue); ++ ++ bfq_log_bfqq(bfqd, bfqq, "add to busy"); ++ ++ bfq_activate_bfqq(bfqd, bfqq); ++ ++ bfq_mark_bfqq_busy(bfqq); ++ bfqd->busy_queues++; ++ ++ if (!bfqq->dispatched) ++ if (bfqq->wr_coeff == 1) ++ bfq_weights_tree_add(bfqd, &bfqq->entity, ++ &bfqd->queue_weights_tree); ++ ++ if (bfqq->wr_coeff > 1) ++ bfqd->wr_busy_queues++; ++} +diff -w -uNr '--exclude=.git' linux-4.8/block/Kconfig.iosched linux-4.8-bfq/block/Kconfig.iosched +--- linux-4.8/block/Kconfig.iosched 2016-10-03 01:24:33.000000000 +0200 ++++ linux-4.8-bfq/block/Kconfig.iosched 2016-10-18 12:57:57.691164881 +0200 +@@ -39,6 +39,25 @@ + ---help--- + Enable group IO scheduling in CFQ. + ++config IOSCHED_BFQ ++ tristate "BFQ I/O scheduler" ++ default n ++ ---help--- ++ The BFQ I/O scheduler tries to distribute bandwidth among ++ all processes according to their weights. ++ It aims at distributing the bandwidth as desired, independently of ++ the disk parameters and with any workload. It also tries to ++ guarantee low latency to interactive and soft real-time ++ applications. If compiled built-in (saying Y here), BFQ can ++ be configured to support hierarchical scheduling. ++ ++config BFQ_GROUP_IOSCHED ++ bool "BFQ hierarchical scheduling support" ++ depends on IOSCHED_BFQ && BLK_CGROUP ++ default n ++ ---help--- ++ Enable hierarchical scheduling in BFQ, using the blkio controller. ++ + choice + prompt "Default I/O scheduler" + default DEFAULT_CFQ +@@ -52,6 +71,16 @@ + config DEFAULT_CFQ + bool "CFQ" if IOSCHED_CFQ=y + ++ config DEFAULT_BFQ ++ bool "BFQ" if IOSCHED_BFQ=y ++ help ++ Selects BFQ as the default I/O scheduler which will be ++ used by default for all block devices. ++ The BFQ I/O scheduler aims at distributing the bandwidth ++ as desired, independently of the disk parameters and with ++ any workload. It also tries to guarantee low latency to ++ interactive and soft real-time applications. ++ + config DEFAULT_NOOP + bool "No-op" + +@@ -61,6 +90,7 @@ + string + default "deadline" if DEFAULT_DEADLINE + default "cfq" if DEFAULT_CFQ ++ default "bfq" if DEFAULT_BFQ + default "noop" if DEFAULT_NOOP + + endmenu +diff -w -uNr '--exclude=.git' linux-4.8/block/Makefile linux-4.8-bfq/block/Makefile +--- linux-4.8/block/Makefile 2016-10-03 01:24:33.000000000 +0200 ++++ linux-4.8-bfq/block/Makefile 2016-10-18 12:57:57.691164881 +0200 +@@ -18,6 +18,7 @@ + obj-$(CONFIG_IOSCHED_NOOP) += noop-iosched.o + obj-$(CONFIG_IOSCHED_DEADLINE) += deadline-iosched.o + obj-$(CONFIG_IOSCHED_CFQ) += cfq-iosched.o ++obj-$(CONFIG_IOSCHED_BFQ) += bfq-iosched.o + + obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o + obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o +diff -w -uNr '--exclude=.git' linux-4.8/include/linux/blkdev.h linux-4.8-bfq/include/linux/blkdev.h +--- linux-4.8/include/linux/blkdev.h 2016-10-03 01:24:33.000000000 +0200 ++++ linux-4.8-bfq/include/linux/blkdev.h 2016-10-18 12:57:59.951179412 +0200 +@@ -45,7 +45,7 @@ + * Maximum number of blkcg policies allowed to be registered concurrently. + * Defined here to simplify include dependency. + */ +-#define BLKCG_MAX_POLS 2 ++#define BLKCG_MAX_POLS 3 + + typedef void (rq_end_io_fn)(struct request *, int); + diff --git a/kernel/tools/cpupowertools/files/patches/mageia/net-add-recursion-limit-to-GRO.patch b/kernel/tools/cpupowertools/files/patches/mageia/net-add-recursion-limit-to-GRO.patch new file mode 100644 index 00000000..a3e3aa52 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/net-add-recursion-limit-to-GRO.patch @@ -0,0 +1,242 @@ +From fcd91dd449867c6bfe56a81cabba76b829fd05cd Mon Sep 17 00:00:00 2001 +From: Sabrina Dubroca +Date: Thu, 20 Oct 2016 15:58:02 +0200 +Subject: [PATCH] net: add recursion limit to GRO +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Currently, GRO can do unlimited recursion through the gro_receive +handlers. This was fixed for tunneling protocols by limiting tunnel GRO +to one level with encap_mark, but both VLAN and TEB still have this +problem. Thus, the kernel is vulnerable to a stack overflow, if we +receive a packet composed entirely of VLAN headers. + +This patch adds a recursion counter to the GRO layer to prevent stack +overflow. When a gro_receive function hits the recursion limit, GRO is +aborted for this skb and it is processed normally. This recursion +counter is put in the GRO CB, but could be turned into a percpu counter +if we run out of space in the CB. + +Thanks to Vladimír Beneš for the initial bug report. + +Fixes: CVE-2016-7039 +Fixes: 9b174d88c257 ("net: Add Transparent Ethernet Bridging GRO support.") +Fixes: 66e5133f19e9 ("vlan: Add GRO support for non hardware accelerated vlan") +Signed-off-by: Sabrina Dubroca +Reviewed-by: Jiri Benc +Acked-by: Hannes Frederic Sowa +Acked-by: Tom Herbert +Signed-off-by: David S. Miller +--- + drivers/net/geneve.c | 2 +- + drivers/net/vxlan.c | 2 +- + include/linux/netdevice.h | 39 ++++++++++++++++++++++++++++++++++++++- + net/8021q/vlan.c | 2 +- + net/core/dev.c | 1 + + net/ethernet/eth.c | 2 +- + net/ipv4/af_inet.c | 2 +- + net/ipv4/fou.c | 4 ++-- + net/ipv4/gre_offload.c | 2 +- + net/ipv4/udp_offload.c | 2 +- + net/ipv6/ip6_offload.c | 2 +- + 11 files changed, 49 insertions(+), 11 deletions(-) + +diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c +index 3c20e87..16af1ce 100644 +--- a/drivers/net/geneve.c ++++ b/drivers/net/geneve.c +@@ -453,7 +453,7 @@ static struct sk_buff **geneve_gro_receive(struct sock *sk, + + skb_gro_pull(skb, gh_len); + skb_gro_postpull_rcsum(skb, gh, gh_len); +- pp = ptype->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + flush = 0; + + out_unlock: +diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c +index e7d1668..c1639a3 100644 +--- a/drivers/net/vxlan.c ++++ b/drivers/net/vxlan.c +@@ -583,7 +583,7 @@ static struct sk_buff **vxlan_gro_receive(struct sock *sk, + } + } + +- pp = eth_gro_receive(head, skb); ++ pp = call_gro_receive(eth_gro_receive, head, skb); + flush = 0; + + out: +diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h +index 465e128..91ee364 100644 +--- a/include/linux/netdevice.h ++++ b/include/linux/netdevice.h +@@ -2169,7 +2169,10 @@ struct napi_gro_cb { + /* Used to determine if flush_id can be ignored */ + u8 is_atomic:1; + +- /* 5 bit hole */ ++ /* Number of gro_receive callbacks this packet already went through */ ++ u8 recursion_counter:4; ++ ++ /* 1 bit hole */ + + /* used to support CHECKSUM_COMPLETE for tunneling protocols */ + __wsum csum; +@@ -2180,6 +2183,40 @@ struct napi_gro_cb { + + #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) + ++#define GRO_RECURSION_LIMIT 15 ++static inline int gro_recursion_inc_test(struct sk_buff *skb) ++{ ++ return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; ++} ++ ++typedef struct sk_buff **(*gro_receive_t)(struct sk_buff **, struct sk_buff *); ++static inline struct sk_buff **call_gro_receive(gro_receive_t cb, ++ struct sk_buff **head, ++ struct sk_buff *skb) ++{ ++ if (unlikely(gro_recursion_inc_test(skb))) { ++ NAPI_GRO_CB(skb)->flush |= 1; ++ return NULL; ++ } ++ ++ return cb(head, skb); ++} ++ ++typedef struct sk_buff **(*gro_receive_sk_t)(struct sock *, struct sk_buff **, ++ struct sk_buff *); ++static inline struct sk_buff **call_gro_receive_sk(gro_receive_sk_t cb, ++ struct sock *sk, ++ struct sk_buff **head, ++ struct sk_buff *skb) ++{ ++ if (unlikely(gro_recursion_inc_test(skb))) { ++ NAPI_GRO_CB(skb)->flush |= 1; ++ return NULL; ++ } ++ ++ return cb(sk, head, skb); ++} ++ + struct packet_type { + __be16 type; /* This is really htons(ether_type). */ + struct net_device *dev; /* NULL is wildcarded here */ +diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c +index 8de138d..f2531ad 100644 +--- a/net/8021q/vlan.c ++++ b/net/8021q/vlan.c +@@ -664,7 +664,7 @@ static struct sk_buff **vlan_gro_receive(struct sk_buff **head, + + skb_gro_pull(skb, sizeof(*vhdr)); + skb_gro_postpull_rcsum(skb, vhdr, sizeof(*vhdr)); +- pp = ptype->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +diff --git a/net/core/dev.c b/net/core/dev.c +index b09ac57..dbc8713 100644 +--- a/net/core/dev.c ++++ b/net/core/dev.c +@@ -4511,6 +4511,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff + NAPI_GRO_CB(skb)->flush = 0; + NAPI_GRO_CB(skb)->free = 0; + NAPI_GRO_CB(skb)->encap_mark = 0; ++ NAPI_GRO_CB(skb)->recursion_counter = 0; + NAPI_GRO_CB(skb)->is_fou = 0; + NAPI_GRO_CB(skb)->is_atomic = 1; + NAPI_GRO_CB(skb)->gro_remcsum_start = 0; +diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c +index 66dff5e..02acfff 100644 +--- a/net/ethernet/eth.c ++++ b/net/ethernet/eth.c +@@ -439,7 +439,7 @@ struct sk_buff **eth_gro_receive(struct sk_buff **head, + + skb_gro_pull(skb, sizeof(*eh)); + skb_gro_postpull_rcsum(skb, eh, sizeof(*eh)); +- pp = ptype->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c +index 1effc98..9648c97 100644 +--- a/net/ipv4/af_inet.c ++++ b/net/ipv4/af_inet.c +@@ -1391,7 +1391,7 @@ struct sk_buff **inet_gro_receive(struct sk_buff **head, struct sk_buff *skb) + skb_gro_pull(skb, sizeof(*iph)); + skb_set_transport_header(skb, skb_gro_offset(skb)); + +- pp = ops->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c +index cf50f7e..030d153 100644 +--- a/net/ipv4/fou.c ++++ b/net/ipv4/fou.c +@@ -249,7 +249,7 @@ static struct sk_buff **fou_gro_receive(struct sock *sk, + if (!ops || !ops->callbacks.gro_receive) + goto out_unlock; + +- pp = ops->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +@@ -441,7 +441,7 @@ static struct sk_buff **gue_gro_receive(struct sock *sk, + if (WARN_ON_ONCE(!ops || !ops->callbacks.gro_receive)) + goto out_unlock; + +- pp = ops->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); + flush = 0; + + out_unlock: +diff --git a/net/ipv4/gre_offload.c b/net/ipv4/gre_offload.c +index 96e0efe..d5cac99 100644 +--- a/net/ipv4/gre_offload.c ++++ b/net/ipv4/gre_offload.c +@@ -229,7 +229,7 @@ static struct sk_buff **gre_gro_receive(struct sk_buff **head, + /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/ + skb_gro_postpull_rcsum(skb, greh, grehlen); + +- pp = ptype->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); + flush = 0; + + out_unlock: +diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c +index f9333c9..b2be1d9 100644 +--- a/net/ipv4/udp_offload.c ++++ b/net/ipv4/udp_offload.c +@@ -295,7 +295,7 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, + + skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ + skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); +- pp = udp_sk(sk)->gro_receive(sk, head, skb); ++ pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb); + + out_unlock: + rcu_read_unlock(); +diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c +index e7bfd55..1fcf61f 100644 +--- a/net/ipv6/ip6_offload.c ++++ b/net/ipv6/ip6_offload.c +@@ -246,7 +246,7 @@ static struct sk_buff **ipv6_gro_receive(struct sk_buff **head, + + skb_gro_postpull_rcsum(skb, iph, nlen); + +- pp = ops->callbacks.gro_receive(head, skb); ++ pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); + + out_unlock: + rcu_read_unlock(); +-- +2.10.1 + diff --git a/kernel/tools/cpupowertools/files/patches/mageia/net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch b/kernel/tools/cpupowertools/files/patches/mageia/net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch new file mode 100644 index 00000000..c251e081 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch @@ -0,0 +1,429 @@ +From cf4747d7535a936105f0abe8d8109d3fe339162b Mon Sep 17 00:00:00 2001 +From: Larry Finger +Date: Wed, 12 Oct 2016 13:54:04 -0500 +Subject: [PATCH] rtlwifi: Fix regression caused by commit d86e64768859 + +In commit d86e64768859 ("rtlwifi: rtl818x: constify local structures"), +the configuration struct for most of the drivers was changed to be +constant. The problem is that five of the modified drivers need to be +able to update the firmware name based on the exact model of the card. +As the file names were stored in one of the members of that struct, +these drivers would fail with a kernel BUG splat when they tried to +update the firmware name. + +Rather than reverting the previous commit, I used a suggestion by +Johannes Berg and made the firmware file name pointers be local to +the routines that update the software variables. + +The configuration struct of rtl8192cu, which was not touched in the +previous patch, is now constantfied. + +Fixes: d86e64768859 ("rtlwifi: rtl818x: constify local structures") +Suggested-by: Johannes Berg +Signed-off-by: Larry Finger +Cc: Stable # 4.8 +Cc: Julia Lawall +Signed-off-by: Kalle Valo +--- + drivers/net/wireless/realtek/rtlwifi/core.c | 2 +- + drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c | 8 ++++---- + drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c | 13 +++++-------- + drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c | 12 ++++++------ + drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c | 6 +++--- + drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c | 8 ++++---- + drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c | 9 +++++---- + drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c | 12 +++++------- + drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c | 6 +++--- + drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c | 18 +++++++++--------- + drivers/net/wireless/realtek/rtlwifi/wifi.h | 2 -- + 11 files changed, 45 insertions(+), 51 deletions(-) + +diff --git a/drivers/net/wireless/realtek/rtlwifi/core.c b/drivers/net/wireless/realtek/rtlwifi/core.c +index f95760c..8e7f23c 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/core.c ++++ b/drivers/net/wireless/realtek/rtlwifi/core.c +@@ -111,7 +111,7 @@ static void rtl_fw_do_work(const struct firmware *firmware, void *context, + if (!err) + goto found_alt; + } +- pr_err("Firmware %s not available\n", rtlpriv->cfg->fw_name); ++ pr_err("Selected firmware is not available\n"); + rtlpriv->max_fw_size = 0; + return; + } +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c +index e7b11b4..f361808 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c +@@ -86,6 +86,7 @@ int rtl88e_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + u8 tid; ++ char *fw_name; + + rtl8188ee_bt_reg_init(hw); + rtlpriv->dm.dm_initialgain_enable = 1; +@@ -169,10 +170,10 @@ int rtl88e_init_sw_vars(struct ieee80211_hw *hw) + return 1; + } + +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8188efw.bin"; ++ fw_name = "rtlwifi/rtl8188efw.bin"; + rtlpriv->max_fw_size = 0x8000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -284,7 +285,6 @@ static const struct rtl_hal_cfg rtl88ee_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl88e_pci", +- .fw_name = "rtlwifi/rtl8188efw.bin", + .ops = &rtl8188ee_hal_ops, + .mod_params = &rtl88ee_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c +index 87aa209..8b6e37c 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c +@@ -96,6 +96,7 @@ int rtl92c_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); ++ char *fw_name = "rtlwifi/rtl8192cfwU.bin"; + + rtl8192ce_bt_reg_init(hw); + +@@ -167,15 +168,12 @@ int rtl92c_init_sw_vars(struct ieee80211_hw *hw) + } + + /* request fw */ +- if (IS_VENDOR_UMC_A_CUT(rtlhal->version) && +- !IS_92C_SERIAL(rtlhal->version)) +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cfwU.bin"; +- else if (IS_81XXC_VENDOR_UMC_B_CUT(rtlhal->version)) +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cfwU_B.bin"; ++ if (IS_81XXC_VENDOR_UMC_B_CUT(rtlhal->version)) ++ fw_name = "rtlwifi/rtl8192cfwU_B.bin"; + + rtlpriv->max_fw_size = 0x4000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -262,7 +260,6 @@ static const struct rtl_hal_cfg rtl92ce_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl92c_pci", +- .fw_name = "rtlwifi/rtl8192cfw.bin", + .ops = &rtl8192ce_hal_ops, + .mod_params = &rtl92ce_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c +index 7c6f7f0..f953320 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c +@@ -59,6 +59,7 @@ static int rtl92cu_init_sw_vars(struct ieee80211_hw *hw) + { + struct rtl_priv *rtlpriv = rtl_priv(hw); + int err; ++ char *fw_name; + + rtlpriv->dm.dm_initialgain_enable = true; + rtlpriv->dm.dm_flag = 0; +@@ -77,18 +78,18 @@ static int rtl92cu_init_sw_vars(struct ieee80211_hw *hw) + } + if (IS_VENDOR_UMC_A_CUT(rtlpriv->rtlhal.version) && + !IS_92C_SERIAL(rtlpriv->rtlhal.version)) { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cufw_A.bin"; ++ fw_name = "rtlwifi/rtl8192cufw_A.bin"; + } else if (IS_81XXC_VENDOR_UMC_B_CUT(rtlpriv->rtlhal.version)) { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cufw_B.bin"; ++ fw_name = "rtlwifi/rtl8192cufw_B.bin"; + } else { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cufw_TMSC.bin"; ++ fw_name = "rtlwifi/rtl8192cufw_TMSC.bin"; + } + /* provide name of alternative file */ + rtlpriv->cfg->alt_fw_name = "rtlwifi/rtl8192cufw.bin"; +- pr_info("Loading firmware %s\n", rtlpriv->cfg->fw_name); ++ pr_info("Loading firmware %s\n", fw_name); + rtlpriv->max_fw_size = 0x4000; + err = request_firmware_nowait(THIS_MODULE, 1, +- rtlpriv->cfg->fw_name, rtlpriv->io.dev, ++ fw_name, rtlpriv->io.dev, + GFP_KERNEL, hw, rtl_fw_cb); + return err; + } +@@ -187,7 +188,6 @@ static struct rtl_hal_usbint_cfg rtl92cu_interface_cfg = { + + static struct rtl_hal_cfg rtl92cu_hal_cfg = { + .name = "rtl92c_usb", +- .fw_name = "rtlwifi/rtl8192cufw.bin", + .ops = &rtl8192cu_hal_ops, + .mod_params = &rtl92cu_mod_params, + .usb_interface_cfg = &rtl92cu_interface_cfg, +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c +index 0538a4d..1ebfee1 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c +@@ -92,6 +92,7 @@ static int rtl92d_init_sw_vars(struct ieee80211_hw *hw) + u8 tid; + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); ++ char *fw_name = "rtlwifi/rtl8192defw.bin"; + + rtlpriv->dm.dm_initialgain_enable = true; + rtlpriv->dm.dm_flag = 0; +@@ -181,10 +182,10 @@ static int rtl92d_init_sw_vars(struct ieee80211_hw *hw) + + rtlpriv->max_fw_size = 0x8000; + pr_info("Driver for Realtek RTL8192DE WLAN interface\n"); +- pr_info("Loading firmware file %s\n", rtlpriv->cfg->fw_name); ++ pr_info("Loading firmware file %s\n", fw_name); + + /* request fw */ +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -266,7 +267,6 @@ static const struct rtl_hal_cfg rtl92de_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl8192de", +- .fw_name = "rtlwifi/rtl8192defw.bin", + .ops = &rtl8192de_hal_ops, + .mod_params = &rtl92de_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c +index ac299cb..46b605de3 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c +@@ -91,6 +91,7 @@ int rtl92ee_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + int err = 0; ++ char *fw_name; + + rtl92ee_bt_reg_init(hw); + rtlpci->msi_support = rtlpriv->cfg->mod_params->msi_support; +@@ -170,11 +171,11 @@ int rtl92ee_init_sw_vars(struct ieee80211_hw *hw) + } + + /* request fw */ +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8192eefw.bin"; ++ fw_name = "rtlwifi/rtl8192eefw.bin"; + + rtlpriv->max_fw_size = 0x8000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -266,7 +267,6 @@ static const struct rtl_hal_cfg rtl92ee_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl92ee_pci", +- .fw_name = "rtlwifi/rtl8192eefw.bin", + .ops = &rtl8192ee_hal_ops, + .mod_params = &rtl92ee_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c +index 5e8e02d..3e1eaea 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c +@@ -89,12 +89,13 @@ static void rtl92se_fw_cb(const struct firmware *firmware, void *context) + struct ieee80211_hw *hw = context; + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rt_firmware *pfirmware = NULL; ++ char *fw_name = "rtlwifi/rtl8192sefw.bin"; + + RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD, + "Firmware callback routine entered!\n"); + complete(&rtlpriv->firmware_loading_complete); + if (!firmware) { +- pr_err("Firmware %s not available\n", rtlpriv->cfg->fw_name); ++ pr_err("Firmware %s not available\n", fw_name); + rtlpriv->max_fw_size = 0; + return; + } +@@ -117,6 +118,7 @@ static int rtl92s_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + int err = 0; + u16 earlyrxthreshold = 7; ++ char *fw_name = "rtlwifi/rtl8192sefw.bin"; + + rtlpriv->dm.dm_initialgain_enable = true; + rtlpriv->dm.dm_flag = 0; +@@ -214,9 +216,9 @@ static int rtl92s_init_sw_vars(struct ieee80211_hw *hw) + rtlpriv->max_fw_size = RTL8190_MAX_FIRMWARE_CODE_SIZE*2 + + sizeof(struct fw_hdr); + pr_info("Driver for Realtek RTL8192SE/RTL8191SE\n" +- "Loading firmware %s\n", rtlpriv->cfg->fw_name); ++ "Loading firmware %s\n", fw_name); + /* request fw */ +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl92se_fw_cb); + if (err) { +@@ -310,7 +312,6 @@ static const struct rtl_hal_cfg rtl92se_hal_cfg = { + .bar_id = 1, + .write_readback = false, + .name = "rtl92s_pci", +- .fw_name = "rtlwifi/rtl8192sefw.bin", + .ops = &rtl8192se_hal_ops, + .mod_params = &rtl92se_mod_params, + +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c +index 89c828a..c51a9e8 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c +@@ -94,6 +94,7 @@ int rtl8723e_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); + int err = 0; ++ char *fw_name = "rtlwifi/rtl8723fw.bin"; + + rtl8723e_bt_reg_init(hw); + +@@ -176,14 +177,12 @@ int rtl8723e_init_sw_vars(struct ieee80211_hw *hw) + return 1; + } + +- if (IS_VENDOR_8723_A_CUT(rtlhal->version)) +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8723fw.bin"; +- else if (IS_81xxC_VENDOR_UMC_B_CUT(rtlhal->version)) +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8723fw_B.bin"; ++ if (IS_81xxC_VENDOR_UMC_B_CUT(rtlhal->version)) ++ fw_name = "rtlwifi/rtl8723fw_B.bin"; + + rtlpriv->max_fw_size = 0x6000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -280,7 +279,6 @@ static const struct rtl_hal_cfg rtl8723e_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl8723e_pci", +- .fw_name = "rtlwifi/rtl8723efw.bin", + .ops = &rtl8723e_hal_ops, + .mod_params = &rtl8723e_mod_params, + .maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL, +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c +index 20b53f0..847644d 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c +@@ -91,6 +91,7 @@ int rtl8723be_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_priv *rtlpriv = rtl_priv(hw); + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + struct rtl_mac *mac = rtl_mac(rtl_priv(hw)); ++ char *fw_name = "rtlwifi/rtl8723befw.bin"; + + rtl8723be_bt_reg_init(hw); + rtlpriv->btcoexist.btc_ops = rtl_btc_get_ops_pointer(); +@@ -184,8 +185,8 @@ int rtl8723be_init_sw_vars(struct ieee80211_hw *hw) + } + + rtlpriv->max_fw_size = 0x8000; +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -280,7 +281,6 @@ static const struct rtl_hal_cfg rtl8723be_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl8723be_pci", +- .fw_name = "rtlwifi/rtl8723befw.bin", + .ops = &rtl8723be_hal_ops, + .mod_params = &rtl8723be_mod_params, + .maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL, +diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c +index 22f687b1..297938e 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c ++++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c +@@ -93,6 +93,7 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw) + struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + struct rtl_mac *mac = rtl_mac(rtl_priv(hw)); + struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); ++ char *fw_name, *wowlan_fw_name; + + rtl8821ae_bt_reg_init(hw); + rtlpriv->btcoexist.btc_ops = rtl_btc_get_ops_pointer(); +@@ -203,17 +204,17 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw) + } + + if (rtlhal->hw_type == HARDWARE_TYPE_RTL8812AE) { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8812aefw.bin"; +- rtlpriv->cfg->wowlan_fw_name = "rtlwifi/rtl8812aefw_wowlan.bin"; ++ fw_name = "rtlwifi/rtl8812aefw.bin"; ++ wowlan_fw_name = "rtlwifi/rtl8812aefw_wowlan.bin"; + } else { +- rtlpriv->cfg->fw_name = "rtlwifi/rtl8821aefw.bin"; +- rtlpriv->cfg->wowlan_fw_name = "rtlwifi/rtl8821aefw_wowlan.bin"; ++ fw_name = "rtlwifi/rtl8821aefw.bin"; ++ wowlan_fw_name = "rtlwifi/rtl8821aefw_wowlan.bin"; + } + + rtlpriv->max_fw_size = 0x8000; + /*load normal firmware*/ +- pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); +- err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, ++ pr_info("Using firmware %s\n", fw_name); ++ err = request_firmware_nowait(THIS_MODULE, 1, fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_fw_cb); + if (err) { +@@ -222,9 +223,9 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw) + return 1; + } + /*load wowlan firmware*/ +- pr_info("Using firmware %s\n", rtlpriv->cfg->wowlan_fw_name); ++ pr_info("Using firmware %s\n", wowlan_fw_name); + err = request_firmware_nowait(THIS_MODULE, 1, +- rtlpriv->cfg->wowlan_fw_name, ++ wowlan_fw_name, + rtlpriv->io.dev, GFP_KERNEL, hw, + rtl_wowlan_fw_cb); + if (err) { +@@ -320,7 +321,6 @@ static const struct rtl_hal_cfg rtl8821ae_hal_cfg = { + .bar_id = 2, + .write_readback = true, + .name = "rtl8821ae_pci", +- .fw_name = "rtlwifi/rtl8821aefw.bin", + .ops = &rtl8821ae_hal_ops, + .mod_params = &rtl8821ae_mod_params, + .maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL, +diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h +index 595f7d5..dafe486 100644 +--- a/drivers/net/wireless/realtek/rtlwifi/wifi.h ++++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h +@@ -2278,9 +2278,7 @@ struct rtl_hal_cfg { + u8 bar_id; + bool write_readback; + char *name; +- char *fw_name; + char *alt_fw_name; +- char *wowlan_fw_name; + struct rtl_hal_ops *ops; + struct rtl_mod_params *mod_params; + struct rtl_hal_usbint_cfg *usb_interface_cfg; +-- +2.10.1 + diff --git a/kernel/tools/cpupowertools/files/patches/mageia/nvmepatch1-V4.patch b/kernel/tools/cpupowertools/files/patches/mageia/nvmepatch1-V4.patch new file mode 100644 index 00000000..9bcefef0 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/nvmepatch1-V4.patch @@ -0,0 +1,111 @@ +diff --git a/drivers/nvme/host/scsi.c b/drivers/nvme/host/scsi.c +index e947e298a737..44009105f8c8 100644 +--- a/drivers/nvme/host/scsi.c ++++ b/drivers/nvme/host/scsi.c +@@ -72,15 +72,6 @@ static int sg_version_num = 30534; /* 2 digits for each component */ + #define ALL_LUNS_RETURNED 0x02 + #define ALL_WELL_KNOWN_LUNS_RETURNED 0x01 + #define RESTRICTED_LUNS_RETURNED 0x00 +-#define NVME_POWER_STATE_START_VALID 0x00 +-#define NVME_POWER_STATE_ACTIVE 0x01 +-#define NVME_POWER_STATE_IDLE 0x02 +-#define NVME_POWER_STATE_STANDBY 0x03 +-#define NVME_POWER_STATE_LU_CONTROL 0x07 +-#define POWER_STATE_0 0 +-#define POWER_STATE_1 1 +-#define POWER_STATE_2 2 +-#define POWER_STATE_3 3 + #define DOWNLOAD_SAVE_ACTIVATE 0x05 + #define DOWNLOAD_SAVE_DEFER_ACTIVATE 0x0E + #define ACTIVATE_DEFERRED_MICROCODE 0x0F +@@ -1229,64 +1220,6 @@ static void nvme_trans_fill_read_cap(u8 *response, struct nvme_id_ns *id_ns, + + /* Start Stop Unit Helper Functions */ + +-static int nvme_trans_power_state(struct nvme_ns *ns, struct sg_io_hdr *hdr, +- u8 pc, u8 pcmod, u8 start) +-{ +- int res; +- int nvme_sc; +- struct nvme_id_ctrl *id_ctrl; +- int lowest_pow_st; /* max npss = lowest power consumption */ +- unsigned ps_desired = 0; +- +- nvme_sc = nvme_identify_ctrl(ns->ctrl, &id_ctrl); +- res = nvme_trans_status_code(hdr, nvme_sc); +- if (res) +- return res; +- +- lowest_pow_st = max(POWER_STATE_0, (int)(id_ctrl->npss - 1)); +- kfree(id_ctrl); +- +- switch (pc) { +- case NVME_POWER_STATE_START_VALID: +- /* Action unspecified if POWER CONDITION MODIFIER != 0 */ +- if (pcmod == 0 && start == 0x1) +- ps_desired = POWER_STATE_0; +- if (pcmod == 0 && start == 0x0) +- ps_desired = lowest_pow_st; +- break; +- case NVME_POWER_STATE_ACTIVE: +- /* Action unspecified if POWER CONDITION MODIFIER != 0 */ +- if (pcmod == 0) +- ps_desired = POWER_STATE_0; +- break; +- case NVME_POWER_STATE_IDLE: +- /* Action unspecified if POWER CONDITION MODIFIER != [0,1,2] */ +- if (pcmod == 0x0) +- ps_desired = POWER_STATE_1; +- else if (pcmod == 0x1) +- ps_desired = POWER_STATE_2; +- else if (pcmod == 0x2) +- ps_desired = POWER_STATE_3; +- break; +- case NVME_POWER_STATE_STANDBY: +- /* Action unspecified if POWER CONDITION MODIFIER != [0,1] */ +- if (pcmod == 0x0) +- ps_desired = max(POWER_STATE_0, (lowest_pow_st - 2)); +- else if (pcmod == 0x1) +- ps_desired = max(POWER_STATE_0, (lowest_pow_st - 1)); +- break; +- case NVME_POWER_STATE_LU_CONTROL: +- default: +- res = nvme_trans_completion(hdr, SAM_STAT_CHECK_CONDITION, +- ILLEGAL_REQUEST, SCSI_ASC_INVALID_CDB, +- SCSI_ASCQ_CAUSE_NOT_REPORTABLE); +- break; +- } +- nvme_sc = nvme_set_features(ns->ctrl, NVME_FEAT_POWER_MGMT, ps_desired, 0, +- NULL); +- return nvme_trans_status_code(hdr, nvme_sc); +-} +- + static int nvme_trans_send_activate_fw_cmd(struct nvme_ns *ns, struct sg_io_hdr *hdr, + u8 buffer_id) + { +@@ -2235,11 +2168,10 @@ static int nvme_trans_synchronize_cache(struct nvme_ns *ns, + static int nvme_trans_start_stop(struct nvme_ns *ns, struct sg_io_hdr *hdr, + u8 *cmd) + { +- u8 immed, pcmod, pc, no_flush, start; ++ u8 immed, pcmod, no_flush, start; + + immed = cmd[1] & 0x01; + pcmod = cmd[3] & 0x0f; +- pc = (cmd[4] & 0xf0) >> 4; + no_flush = cmd[4] & 0x04; + start = cmd[4] & 0x01; + +@@ -2254,8 +2186,8 @@ static int nvme_trans_start_stop(struct nvme_ns *ns, struct sg_io_hdr *hdr, + if (res) + return res; + } +- /* Setup the expected power state transition */ +- return nvme_trans_power_state(ns, hdr, pc, pcmod, start); ++ ++ return 0; + } + } + +-- +2.7.4 diff --git a/kernel/tools/cpupowertools/files/patches/mageia/nvmepatch2-V4.patch b/kernel/tools/cpupowertools/files/patches/mageia/nvmepatch2-V4.patch new file mode 100644 index 00000000..454cb4a5 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/nvmepatch2-V4.patch @@ -0,0 +1,107 @@ +diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c +index 2feacc70bf61..b92a7f767bfc 100644 +--- a/drivers/nvme/host/core.c ++++ b/drivers/nvme/host/core.c +@@ -597,7 +597,7 @@ int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid, + } + + int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, +- dma_addr_t dma_addr, u32 *result) ++ void *buffer, size_t buflen, u32 *result) + { + struct nvme_command c; + struct nvme_completion cqe; +@@ -606,10 +606,9 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, + memset(&c, 0, sizeof(c)); + c.features.opcode = nvme_admin_get_features; + c.features.nsid = cpu_to_le32(nsid); +- c.features.dptr.prp1 = cpu_to_le64(dma_addr); + c.features.fid = cpu_to_le32(fid); + +- ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, NULL, 0, 0, ++ ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, buffer, buflen, 0, + NVME_QID_ANY, 0, 0); + if (ret >= 0 && result) + *result = le32_to_cpu(cqe.result); +@@ -617,7 +616,7 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, + } + + int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, +- dma_addr_t dma_addr, u32 *result) ++ void *buffer, size_t buflen, u32 *result) + { + struct nvme_command c; + struct nvme_completion cqe; +@@ -625,12 +624,11 @@ int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, + + memset(&c, 0, sizeof(c)); + c.features.opcode = nvme_admin_set_features; +- c.features.dptr.prp1 = cpu_to_le64(dma_addr); + c.features.fid = cpu_to_le32(fid); + c.features.dword11 = cpu_to_le32(dword11); + +- ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, NULL, 0, 0, +- NVME_QID_ANY, 0, 0); ++ ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &cqe, ++ buffer, buflen, 0, NVME_QID_ANY, 0, 0); + if (ret >= 0 && result) + *result = le32_to_cpu(cqe.result); + return ret; +@@ -664,7 +662,7 @@ int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count) + u32 result; + int status, nr_io_queues; + +- status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, 0, ++ status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0, + &result); + if (status < 0) + return status; +diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h +index ab18b78102bf..c2151761ec5f 100644 +--- a/drivers/nvme/host/nvme.h ++++ b/drivers/nvme/host/nvme.h +@@ -292,9 +292,9 @@ int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid, + struct nvme_id_ns **id); + int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log); + int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, +- dma_addr_t dma_addr, u32 *result); ++ void *buffer, size_t buflen, u32 *result); + int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, +- dma_addr_t dma_addr, u32 *result); ++ void *buffer, size_t buflen, u32 *result); + int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count); + void nvme_start_keep_alive(struct nvme_ctrl *ctrl); + void nvme_stop_keep_alive(struct nvme_ctrl *ctrl); +diff --git a/drivers/nvme/host/scsi.c b/drivers/nvme/host/scsi.c +index 44009105f8c8..c2a0a1c7d05d 100644 +--- a/drivers/nvme/host/scsi.c ++++ b/drivers/nvme/host/scsi.c +@@ -906,7 +906,7 @@ static int nvme_trans_log_temperature(struct nvme_ns *ns, struct sg_io_hdr *hdr, + kfree(smart_log); + + /* Get Features for Temp Threshold */ +- res = nvme_get_features(ns->ctrl, NVME_FEAT_TEMP_THRESH, 0, 0, ++ res = nvme_get_features(ns->ctrl, NVME_FEAT_TEMP_THRESH, 0, NULL, 0, + &feature_resp); + if (res != NVME_SC_SUCCESS) + temp_c_thresh = LOG_TEMP_UNKNOWN; +@@ -1039,7 +1039,7 @@ static int nvme_trans_fill_caching_page(struct nvme_ns *ns, + if (len < MODE_PAGE_CACHING_LEN) + return -EINVAL; + +- nvme_sc = nvme_get_features(ns->ctrl, NVME_FEAT_VOLATILE_WC, 0, 0, ++ nvme_sc = nvme_get_features(ns->ctrl, NVME_FEAT_VOLATILE_WC, 0, NULL, 0, + &feature_resp); + res = nvme_trans_status_code(hdr, nvme_sc); + if (res) +@@ -1328,7 +1328,7 @@ static int nvme_trans_modesel_get_mp(struct nvme_ns *ns, struct sg_io_hdr *hdr, + case MODE_PAGE_CACHING: + dword11 = ((mode_page[2] & CACHING_MODE_PAGE_WCE_MASK) ? 1 : 0); + nvme_sc = nvme_set_features(ns->ctrl, NVME_FEAT_VOLATILE_WC, +- dword11, 0, NULL); ++ dword11, NULL, 0, NULL); + res = nvme_trans_status_code(hdr, nvme_sc); + break; + case MODE_PAGE_CONTROL: +-- +2.7.4 diff --git a/kernel/tools/cpupowertools/files/patches/mageia/nvmepatch3-V4.patch b/kernel/tools/cpupowertools/files/patches/mageia/nvmepatch3-V4.patch new file mode 100644 index 00000000..797c4681 --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/nvmepatch3-V4.patch @@ -0,0 +1,264 @@ +diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c +index b92a7f767bfc..a143c72c3d0d 100644 +--- a/drivers/nvme/host/core.c ++++ b/drivers/nvme/host/core.c +@@ -56,6 +56,11 @@ EXPORT_SYMBOL_GPL(nvme_max_retries); + static int nvme_char_major; + module_param(nvme_char_major, int, 0); + ++static unsigned long default_ps_max_latency_us = 25000; ++module_param(default_ps_max_latency_us, ulong, 0644); ++MODULE_PARM_DESC(ps_max_latency_us, ++ "default max power saving latency; overridden per device in sysfs"); ++ + static LIST_HEAD(nvme_ctrl_list); + static DEFINE_SPINLOCK(dev_list_lock); + +@@ -1205,6 +1210,98 @@ static void nvme_set_queue_limits(struct nvme_ctrl *ctrl, + blk_queue_write_cache(q, vwc, vwc); + } + ++static void nvme_configure_apst(struct nvme_ctrl *ctrl) ++{ ++ /* ++ * APST (Autonomous Power State Transition) lets us program a ++ * table of power state transitions that the controller will ++ * perform automatically. We configure it with a simple ++ * heuristic: we are willing to spend at most 2% of the time ++ * transitioning between power states. Therefore, when running ++ * in any given state, we will enter the next lower-power ++ * non-operational state after waiting 100 * (enlat + exlat) ++ * microseconds, as long as that state's total latency is under ++ * the requested maximum latency. ++ * ++ * We will not autonomously enter any non-operational state for ++ * which the total latency exceeds ps_max_latency_us. Users ++ * can set ps_max_latency_us to zero to turn off APST. ++ */ ++ ++ unsigned apste; ++ struct nvme_feat_auto_pst *table; ++ int ret; ++ ++ if (!ctrl->apsta) ++ return; /* APST isn't supported. */ ++ ++ if (ctrl->npss > 31) { ++ dev_warn(ctrl->device, "NPSS is invalid; not using APST\n"); ++ return; ++ } ++ ++ table = kzalloc(sizeof(*table), GFP_KERNEL); ++ if (!table) ++ return; ++ ++ if (ctrl->ps_max_latency_us == 0) { ++ /* Turn off APST. */ ++ apste = 0; ++ } else { ++ __le64 target = cpu_to_le64(0); ++ int state; ++ ++ /* ++ * Walk through all states from lowest- to highest-power. ++ * According to the spec, lower-numbered states use more ++ * power. NPSS, despite the name, is the index of the ++ * lowest-power state, not the number of states. ++ */ ++ for (state = (int)ctrl->npss; state >= 0; state--) { ++ u64 total_latency_us, transition_ms; ++ ++ if (target) ++ table->entries[state] = target; ++ ++ /* ++ * Is this state a useful non-operational state for ++ * higher-power states to autonomously transition to? ++ */ ++ if (!(ctrl->psd[state].flags & 2)) ++ continue; /* It's an operational state. */ ++ ++ total_latency_us = ++ (u64)cpu_to_le32(ctrl->psd[state].entry_lat) + ++ + cpu_to_le32(ctrl->psd[state].exit_lat); ++ if (total_latency_us > ctrl->ps_max_latency_us) ++ continue; ++ ++ /* ++ * This state is good. Use it as the APST idle ++ * target for higher power states. ++ */ ++ transition_ms = total_latency_us + 19; ++ do_div(transition_ms, 20); ++ if (transition_ms >= (1 << 24)) ++ transition_ms = (1 << 24); ++ ++ target = cpu_to_le64((state << 3) | ++ (transition_ms << 8)); ++ } ++ ++ apste = 1; ++ } ++ ++ ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste, ++ table, sizeof(*table), NULL); ++ if (ret) ++ dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret); ++ ++ kfree(table); ++} ++ ++static struct attribute_group nvme_dev_dynamic_attrs_group; ++ + /* + * Initialize the cached copies of the Identify data and various controller + * register in our nvme_ctrl structure. This should be called as soon as +@@ -1271,6 +1368,10 @@ int nvme_init_identify(struct nvme_ctrl *ctrl) + ctrl->sgls = le32_to_cpu(id->sgls); + ctrl->kas = le16_to_cpu(id->kas); + ++ ctrl->npss = id->npss; ++ ctrl->apsta = id->apsta; ++ memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd)); ++ + if (ctrl->ops->is_fabrics) { + ctrl->icdoff = le16_to_cpu(id->icdoff); + ctrl->ioccsz = le32_to_cpu(id->ioccsz); +@@ -1294,6 +1395,10 @@ int nvme_init_identify(struct nvme_ctrl *ctrl) + } + + kfree(id); ++ ++ nvme_configure_apst(ctrl); ++ ++ sysfs_update_group(&ctrl->device->kobj, &nvme_dev_dynamic_attrs_group); + return ret; + } + EXPORT_SYMBOL_GPL(nvme_init_identify); +@@ -1564,6 +1669,41 @@ static ssize_t nvme_sysfs_show_address(struct device *dev, + } + static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL); + ++static ssize_t nvme_sysfs_show_ps_max_latency_us( ++ struct device *dev, ++ struct device_attribute *attr, ++ char *buf) ++{ ++ struct nvme_ctrl *ctrl = dev_get_drvdata(dev); ++ ++ return snprintf(buf, PAGE_SIZE, "%llu\n", ctrl->ps_max_latency_us); ++} ++ ++static ssize_t nvme_sysfs_store_ps_max_latency_us( ++ struct device *dev, ++ struct device_attribute *attr, ++ const char *buf, size_t size) ++{ ++ struct nvme_ctrl *ctrl = dev_get_drvdata(dev); ++ int ret; ++ u64 val; ++ ++ ret = kstrtoull(buf, 10, &val); ++ if (ret) ++ return ret; ++ ++ if (ctrl->ps_max_latency_us != val) { ++ ctrl->ps_max_latency_us = val; ++ nvme_configure_apst(ctrl); ++ } ++ ++ return size; ++} ++ ++static DEVICE_ATTR(ps_max_latency_us, 0644, ++ nvme_sysfs_show_ps_max_latency_us, ++ nvme_sysfs_store_ps_max_latency_us); ++ + static struct attribute *nvme_dev_attrs[] = { + &dev_attr_reset_controller.attr, + &dev_attr_rescan_controller.attr, +@@ -1605,8 +1745,33 @@ static struct attribute_group nvme_dev_attrs_group = { + .is_visible = nvme_dev_attrs_are_visible, + }; + ++static struct attribute *nvme_dev_dynamic_attrs[] = { ++ &dev_attr_ps_max_latency_us.attr, ++ NULL ++}; ++ ++static umode_t nvme_dev_dynamic_attrs_are_visible(struct kobject *kobj, ++ struct attribute *a, int n) ++{ ++ struct device *dev = container_of(kobj, struct device, kobj); ++ struct nvme_ctrl *ctrl = dev_get_drvdata(dev); ++ ++ if (a == &dev_attr_ps_max_latency_us.attr) { ++ if (!ctrl->apsta) ++ return 0; ++ } ++ ++ return a->mode; ++} ++ ++static struct attribute_group nvme_dev_dynamic_attrs_group = { ++ .attrs = nvme_dev_dynamic_attrs, ++ .is_visible = nvme_dev_dynamic_attrs_are_visible, ++}; ++ + static const struct attribute_group *nvme_dev_attr_groups[] = { + &nvme_dev_attrs_group, ++ &nvme_dev_dynamic_attrs_group, + NULL, + }; + +@@ -1991,6 +2156,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, + ctrl->quirks = quirks; + INIT_WORK(&ctrl->scan_work, nvme_scan_work); + INIT_WORK(&ctrl->async_event_work, nvme_async_event_work); ++ ctrl->ps_max_latency_us = default_ps_max_latency_us; + + ret = nvme_set_instance(ctrl); + if (ret) +diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h +index c2151761ec5f..16d51006afc0 100644 +--- a/drivers/nvme/host/nvme.h ++++ b/drivers/nvme/host/nvme.h +@@ -129,13 +129,19 @@ struct nvme_ctrl { + u32 vs; + u32 sgls; + u16 kas; ++ u8 npss; ++ u8 apsta; + unsigned int kato; + bool subsystem; + unsigned long quirks; ++ struct nvme_id_power_state psd[32]; + struct work_struct scan_work; + struct work_struct async_event_work; + struct delayed_work ka_work; + ++ /* Power saving configuration */ ++ u64 ps_max_latency_us; ++ + /* Fabrics only */ + u16 sqsize; + u32 ioccsz; +diff --git a/include/linux/nvme.h b/include/linux/nvme.h +index d8b37bab2887..a76237dac4b0 100644 +--- a/include/linux/nvme.h ++++ b/include/linux/nvme.h +@@ -543,6 +543,12 @@ struct nvme_dsm_range { + __le64 slba; + }; + ++/* Features */ ++ ++struct nvme_feat_auto_pst { ++ __le64 entries[32]; ++}; ++ + /* Admin commands */ + + enum nvme_admin_opcode { +-- +2.7.4 diff --git a/kernel/tools/cpupowertools/files/patches/mageia/series b/kernel/tools/cpupowertools/files/patches/mageia/series index 6452397c..e7a005b7 100644 --- a/kernel/tools/cpupowertools/files/patches/mageia/series +++ b/kernel/tools/cpupowertools/files/patches/mageia/series @@ -41,11 +41,19 @@ Revert-cpufreq-pcc-Enable-autoload-of-pcc-cpufreq-fo.patch # breaks nvidia304 Revert-x86-mm-mtrr-Remove-kernel-internal-MTRR-inter.patch +# fix non-booting systems +x86-smpboot-Init-apic-mapping-before-usage.patch + ### ### Core ### base-cacheinfo-silence-DT-warnings.patch +# Patch for having the cpupower setting the governor +# (from https://bugzilla.kernel.org/show_bug.cgi?id=135391, same as +# https://bugzilla.redhat.com/show_bug.cgi?id=1374212) +kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch + ### ### i2c ### @@ -119,8 +127,20 @@ ata-prefer-ata-drivers-over-ide-drivers-when-both-are-built.patch # Nice SSD speedup block-Make-CFQ-default-to-IOPS-mode-on-SSDs.patch -# ahci ids -ahci-add-new-Intel-device-IDs.patch +# Include new BFQ I/O scheduler +# (See http://algo.ing.unimo.it/people/paolo/disk_sched) +# +# Release of BFQ I/O scheduler v8r4 (rediffed from git https://github.com/linusw/linux-bfq) +linux-BFQ-v8r4.patch + +# Let NVME drives enter in power state +# (see https://github.com/damige/linux-nvme/tree/master/src) +nvmepatch1-V4.patch +nvmepatch2-V4.patch +nvmepatch3-V4.patch + +# fix feature check, reported on dev@ ml +dm-raid-fix-compat_features-validation.patch ### ### Char @@ -134,9 +154,8 @@ Revert-ipmi-Start-the-timer-and-thread-on-internal-m.patch ### # aufs from: http://aufs.sourceforge.net/ (mga#8314) -fs-aufs-4.7.patch -fs-aufs-4.7-modular.patch -fs-aufs-4.7-ver-fix.patch +fs-aufs-4.8.patch +fs-aufs-4.8-modular.patch ### ### FireWire @@ -171,6 +190,14 @@ gpu-drm-mach64-linux-3.14-buildfix.patch gpu-drm-mach64-3.17-buildfix.patch gpu-drm-mach64-3.18-buildfix.patch +# Attempt to fix FIFO underrun problems: +# (from https://patchwork.freedesktop.org/patch/113642/ and +# also from https://bugs.freedesktop.org/attachment.cgi?id=127264, first reference from +# https://bugs.freedesktop.org/show_bug.cgi?id=97450). Does not completely resolve the +# issues but reduce. +# now in drm-next: https://cgit.freedesktop.org/~airlied/linux/commit?h=drm-next&id=5a920b85f2c6e3fd7d9dd9bb3f3345e9085e2360 +gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch + ### ### Hardware Monitoring ### @@ -222,6 +249,12 @@ net-netfilter-psd.patch net-netfilter-psd-mdv.patch net-netfilter-psd-2.6.35-buildfix.patch +# rtlwifi regression +net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch + +# CVE-2016-7039 +net-add-recursion-limit-to-GRO.patch + ### ### Platform drivers ### @@ -232,10 +265,6 @@ platform-x86-add-shuttle-wmi-driver.patch platform-x86-shuttle-wmi-drop-devinit-exit.patch platform-x86-shuttle-wmi-4.2-buildfix.patch -# -intel_th-Fix-a-deadlock-in-modprobing.patch -intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch - ### ### RTC ### @@ -331,6 +360,7 @@ video-mageia-logo.patch # add rtl8723bs support (mga#15874) 3rd-rtl8723bs.patch 3rd-rtl8723bs-4.7-buildfix.patch +3rd-rtl8723bs-4.8-buildfix.patch ### ### Security @@ -348,10 +378,6 @@ video-mageia-logo.patch ### ARM ### -# RPi2 support -# Read MAC address from DT -arm-0001-ARM-bcm2835-dt-Add-the-ethernet-to-the-device-trees.patch - ### ### IA64 ### @@ -360,6 +386,11 @@ arm-0001-ARM-bcm2835-dt-Add-the-ethernet-to-the-device-trees.patch ### Tools ### +# wipe powerpc refrence so we can nuke dangling symlinks (mga#17676) +tools-testing-selftest-Makefile-remove-powerpc-reference.patch + ### ### UAPI ### +# breaks linux-arm build +Revert-include-uapi-linux-atm_zatm.h-include-linux-t.patch diff --git a/kernel/tools/cpupowertools/files/patches/mageia/tools-testing-selftest-Makefile-remove-powerpc-reference.patch b/kernel/tools/cpupowertools/files/patches/mageia/tools-testing-selftest-Makefile-remove-powerpc-reference.patch new file mode 100644 index 00000000..677e97df --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/tools-testing-selftest-Makefile-remove-powerpc-reference.patch @@ -0,0 +1,10 @@ +--- linux/tools/testing/selftests/Makefile.orig ++++ linux/tools/testing/selftests/Makefile +@@ -15,7 +15,6 @@ TARGETS += memory-hotplug + TARGETS += mount + TARGETS += mqueue + TARGETS += net +-TARGETS += powerpc + TARGETS += pstore + TARGETS += ptrace + TARGETS += seccomp diff --git a/kernel/tools/cpupowertools/files/patches/mageia/x86-smpboot-Init-apic-mapping-before-usage.patch b/kernel/tools/cpupowertools/files/patches/mageia/x86-smpboot-Init-apic-mapping-before-usage.patch new file mode 100644 index 00000000..6c2e766a --- /dev/null +++ b/kernel/tools/cpupowertools/files/patches/mageia/x86-smpboot-Init-apic-mapping-before-usage.patch @@ -0,0 +1,61 @@ +From 1e90a13d0c3dc94512af1ccb2b6563e8297838fa Mon Sep 17 00:00:00 2001 +From: Thomas Gleixner +Date: Sat, 29 Oct 2016 13:42:42 +0200 +Subject: [PATCH] x86/smpboot: Init apic mapping before usage + +The recent changes, which forced the registration of the boot cpu on UP +systems, which do not have ACPI tables, have been fixed for systems w/o +local APIC, but left a wreckage for systems which have neither ACPI nor +mptables, but the CPU has an APIC, e.g. virtualbox. + +The boot process crashes in prefill_possible_map() as it wants to register +the boot cpu, which needs to access the local apic, but the local APIC is +not yet mapped. + +There is no reason why init_apic_mapping() can't be invoked before +prefill_possible_map(). So instead of playing another silly early mapping +game, as the ACPI/mptables code does, we just move init_apic_mapping() +before the call to prefill_possible_map(). + +In hindsight, I should have noticed that combination earlier. + +Sorry for the churn (also in stable)! + +Fixes: ff8560512b8d ("x86/boot/smp: Don't try to poke disabled/non-existent APIC") +Reported-and-debugged-by: Michal Necasek +Reported-and-tested-by: Wolfgang Bauer +Cc: prarit@redhat.com +Cc: ville.syrjala@linux.intel.com +Cc: michael.thayer@oracle.com +Cc: knut.osmundsen@oracle.com +Cc: frank.mehnert@oracle.com +Cc: Borislav Petkov +Cc: stable@vger.kernel.org +Link: http://lkml.kernel.org/r/alpine.DEB.2.20.1610282114380.5053@nanos +Signed-off-by: Thomas Gleixner +--- + arch/x86/kernel/setup.c | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c +index bbfbca5..9c337b0 100644 +--- a/arch/x86/kernel/setup.c ++++ b/arch/x86/kernel/setup.c +@@ -1222,11 +1222,16 @@ void __init setup_arch(char **cmdline_p) + if (smp_found_config) + get_smp_config(); + ++ /* ++ * Systems w/o ACPI and mptables might not have it mapped the local ++ * APIC yet, but prefill_possible_map() might need to access it. ++ */ ++ init_apic_mappings(); ++ + prefill_possible_map(); + + init_cpu_to_node(); + +- init_apic_mappings(); + io_apic_init_mappings(); + + kvm_guest_init(); diff --git a/kernel/tools/cpupowertools/pspec.xml b/kernel/tools/cpupowertools/pspec.xml index 3115b317..dc701efa 100644 --- a/kernel/tools/cpupowertools/pspec.xml +++ b/kernel/tools/cpupowertools/pspec.xml @@ -12,14 +12,14 @@ app:console CPU power management tools cpupowertools contains utilities to manage power management and frequency scaling policies of modern CPUs. - https://www.kernel.org/pub/linux/kernel/v4.x/linux-4.7.tar.gz + https://www.kernel.org/pub/linux/kernel/v4.x/linux-4.8.tar.gz pciutils-devel - - - patches/linux/patch-4.7.2.xz - + + + patches/linux/patch-4.8.6.xz + patches/mageia/Revert-ipmi-Start-the-timer-and-thread-on-internal-m.patch @@ -27,9 +27,11 @@ patches/mageia/x86-boot-video-80x25-if-break.patch patches/mageia/x86-default_poweroff_up_machines.patch patches/mageia/x86-increase-default-minimum-vmalloc-area-by-64MB-to-192MB.patch - patches/mageia/Revert-cpufreq-pcc-Enable-autoload-of-pcc-cpufreq-fo.patch + patches/mageia/Revert-cpufreq-pcc-Enable-autoload-of-pcc-cpufreq-fo.patch + patches/mageia/x86-smpboot-Init-apic-mapping-before-usage.patch patches/mageia/Revert-x86-mm-mtrr-Remove-kernel-internal-MTRR-inter.patch patches/mageia/base-cacheinfo-silence-DT-warnings.patch + patches/mageia/kernel-governor-cpupower-Correct-return-type-of-cpu_power_is_cpu_onl.patch patches/mageia/pci-add-ALI-M5229-ide-compatibility-mode-quirk.patch patches/mageia/pci-quirks-drop-devinit-exit.patch patches/mageia/acpi-CLEVO-M360S-disable_acpi_irq.patch @@ -48,9 +50,13 @@ patches/mageia/block-floppy-disable-pnp-modalias.patch patches/mageia/ata-prefer-ata-drivers-over-ide-drivers-when-both-are-built.patch patches/mageia/block-Make-CFQ-default-to-IOPS-mode-on-SSDs.patch - patches/mageia/fs-aufs-4.7.patch - patches/mageia/fs-aufs-4.7-modular.patch - patches/mageia/fs-aufs-4.7-ver-fix.patch + patches/mageia/linux-BFQ-v8r4.patch + patches/mageia/nvmepatch1-V4.patch + patches/mageia/nvmepatch2-V4.patch + patches/mageia/nvmepatch3-V4.patch + patches/mageia/dm-raid-fix-compat_features-validation.patch + patches/mageia/fs-aufs-4.8.patch + patches/mageia/fs-aufs-4.8-modular.patch patches/mageia/firewire-ieee1394-module-aliases.patch patches/mageia/char-agp-intel-new-Q57-id.patch @@ -71,6 +77,7 @@ patches/mageia/gpu-drm-mach64-linux-3.14-buildfix.patch patches/mageia/gpu-drm-mach64-3.17-buildfix.patch patches/mageia/gpu-drm-mach64-3.18-buildfix.patch + patches/mageia/gpu-drm-i915-gen9-fix-DDB-partitioning-for-multi-screen-cases.patch patches/mageia/input-i8042-quirks-for-Fujitsu-Lifebook-A544-and-Lif.patch patches/mageia/net-sis190-fix-list-usage.patch patches/mageia/net-netfilter-IFWLOG.patch @@ -83,11 +90,11 @@ patches/mageia/net-netfilter-psd.patch patches/mageia/net-netfilter-psd-mdv.patch patches/mageia/net-netfilter-psd-2.6.35-buildfix.patch + patches/mageia/net-wireless-rtlwifi-Fix-regression-caused-by-commit-d86e64768859.patch + patches/mageia/net-add-recursion-limit-to-GRO.patch patches/mageia/platform-x86-add-shuttle-wmi-driver.patch patches/mageia/platform-x86-shuttle-wmi-drop-devinit-exit.patch patches/mageia/platform-x86-shuttle-wmi-4.2-buildfix.patch - patches/mageia/intel_th-Fix-a-deadlock-in-modprobing.patch - patches/mageia/intel_th-pci-Add-Kaby-Lake-PCH-H-support.patch patches/mageia/include-kbuild-export-pci_ids.patch patches/mageia/hid-usbhid-IBM-BladeCenterHS20-quirk.patch patches/mageia/usb-storage-unusual_devs-add-id.patch @@ -120,9 +127,10 @@ patches/mageia/3rd-viahss-3.0-buildfix.patch patches/mageia/3rd-rtl8723bs.patch patches/mageia/3rd-rtl8723bs-4.7-buildfix.patch - patches/mageia/ahci-add-new-Intel-device-IDs.patch - patches/mageia/arm-0001-ARM-bcm2835-dt-Add-the-ethernet-to-the-device-trees.patch - + patches/mageia/3rd-rtl8723bs-4.8-buildfix.patch + patches/mageia/tools-testing-selftest-Makefile-remove-powerpc-reference.patch + patches/mageia/Revert-include-uapi-linux-atm_zatm.h-include-linux-t.patch +