diff options
Diffstat (limited to 'fs/aufs')
46 files changed, 452 insertions, 260 deletions
diff --git a/fs/aufs/branch.c b/fs/aufs/branch.c index 07c842d14..80b018f57 100644 --- a/fs/aufs/branch.c +++ b/fs/aufs/branch.c @@ -38,7 +38,7 @@ static void au_br_do_free(struct au_branch *br) if (br->br_fhsm) { au_br_fhsm_fin(br->br_fhsm); - kfree(br->br_fhsm); + au_delayed_kfree(br->br_fhsm); } key = br->br_dykey; @@ -52,8 +52,9 @@ static void au_br_do_free(struct au_branch *br) lockdep_off(); path_put(&br->br_path); lockdep_on(); - kfree(wbr); - kfree(br); + if (wbr) + au_delayed_kfree(wbr); + au_delayed_kfree(br); } /* @@ -151,11 +152,12 @@ static struct au_branch *au_br_alloc(struct super_block *sb, int new_nbranch, return add_branch; /* success */ out_wbr: - kfree(add_branch->br_wbr); + if (add_branch->br_wbr) + au_delayed_kfree(add_branch->br_wbr); out_hnotify: au_hnotify_fin_br(add_branch); out_br: - kfree(add_branch); + au_delayed_kfree(add_branch); out: return ERR_PTR(err); } @@ -321,7 +323,7 @@ static int au_br_init_wh(struct super_block *sb, struct au_branch *br, br->br_perm = old_perm; if (!err && wbr && !au_br_writable(new_perm)) { - kfree(wbr); + au_delayed_kfree(wbr); br->br_wbr = NULL; } @@ -481,6 +483,7 @@ int au_br_add(struct super_block *sb, struct au_opt_add *add, int remount) root = sb->s_root; root_inode = d_inode(root); IMustLock(root_inode); + IiMustWriteLock(root_inode); err = test_add(sb, add, remount); if (unlikely(err < 0)) goto out; @@ -1343,7 +1346,7 @@ int au_br_mod(struct super_block *sb, struct au_opt_mod *mod, int remount, if (br->br_wbr) { err = au_wbr_init(br, sb, mod->perm); if (unlikely(err)) { - kfree(br->br_wbr); + au_delayed_kfree(br->br_wbr); br->br_wbr = NULL; } } @@ -1355,7 +1358,7 @@ int au_br_mod(struct super_block *sb, struct au_opt_mod *mod, int remount, if (!au_br_fhsm(mod->perm)) { /* fhsm --> non-fhsm */ au_br_fhsm_fin(br->br_fhsm); - kfree(br->br_fhsm); + au_delayed_kfree(br->br_fhsm); br->br_fhsm = NULL; } } else if (au_br_fhsm(mod->perm)) @@ -1367,7 +1370,8 @@ int au_br_mod(struct super_block *sb, struct au_opt_mod *mod, int remount, goto out; /* success */ out_bf: - kfree(bf); + if (bf) + au_delayed_kfree(bf); out: AuTraceErr(err); return err; diff --git a/fs/aufs/cpup.c b/fs/aufs/cpup.c index 52f87de63..484cdc3ee 100644 --- a/fs/aufs/cpup.c +++ b/fs/aufs/cpup.c @@ -344,9 +344,9 @@ int au_copy_file(struct file *dst, struct file *src, loff_t len) dst->f_pos = 0; err = au_do_copy_file(dst, src, len, buf, blksize); if (do_kfree) - kfree(buf); + au_delayed_kfree(buf); else - free_page((unsigned long)buf); + au_delayed_free_page((unsigned long)buf); out: return err; @@ -506,7 +506,7 @@ static int au_do_cpup_symlink(struct path *h_path, struct dentry *h_src, sym.k[symlen] = 0; err = vfsub_symlink(h_dir, h_path, sym.k); } - free_page((unsigned long)sym.k); + au_delayed_free_page((unsigned long)sym.k); out: return err; @@ -877,7 +877,7 @@ out_rev: } out_parent: dput(dst_parent); - kfree(a); + au_delayed_kfree(a); out: return err; } diff --git a/fs/aufs/dbgaufs.c b/fs/aufs/dbgaufs.c index 21ac77398..f85a813a4 100644 --- a/fs/aufs/dbgaufs.c +++ b/fs/aufs/dbgaufs.c @@ -28,7 +28,7 @@ struct dbgaufs_arg { static int dbgaufs_xi_release(struct inode *inode __maybe_unused, struct file *file) { - kfree(file->private_data); + au_delayed_kfree(file->private_data); return 0; } @@ -90,7 +90,7 @@ struct dbgaufs_plink_arg { static int dbgaufs_plink_release(struct inode *inode __maybe_unused, struct file *file) { - free_page((unsigned long)file->private_data); + au_delayed_free_page((unsigned long)file->private_data); return 0; } @@ -154,7 +154,7 @@ static int dbgaufs_plink_open(struct inode *inode, struct file *file) goto out; /* success */ out_free: - free_page((unsigned long)p); + au_delayed_free_page((unsigned long)p); out: return err; } @@ -281,8 +281,11 @@ void dbgaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex) br = au_sbr(sb, bindex); xi = &br->br_xino; AuDebugOn(xi->xi_dbgaufs); + /* debugfs acquires the parent i_mutex */ + lockdep_off(); xi->xi_dbgaufs = debugfs_create_file(name, dbgaufs_mode, parent, sbinfo, &dbgaufs_xino_fop); + lockdep_on(); /* ignore an error */ if (unlikely(!xi->xi_dbgaufs)) AuWarn1("failed %s under debugfs\n", name); diff --git a/fs/aufs/dcsub.c b/fs/aufs/dcsub.c index e72accebb..09bc36af7 100644 --- a/fs/aufs/dcsub.c +++ b/fs/aufs/dcsub.c @@ -16,7 +16,7 @@ static void au_dpage_free(struct au_dpage *dpage) p = dpage->dentries; for (i = 0; i < dpage->ndentry; i++) dput(*p++); - free_page((unsigned long)dpage->dentries); + au_delayed_free_page((unsigned long)dpage->dentries); } int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp) @@ -39,7 +39,7 @@ int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp) return 0; /* success */ out_dpages: - kfree(dpages->dpages); + au_delayed_kfree(dpages->dpages); out: return err; } @@ -52,7 +52,7 @@ void au_dpages_free(struct au_dcsub_pages *dpages) p = dpages->dpages; for (i = 0; i < dpages->ndpage; i++) au_dpage_free(p++); - kfree(dpages->dpages); + au_delayed_kfree(dpages->dpages); } static int au_dpages_append(struct au_dcsub_pages *dpages, diff --git a/fs/aufs/debug.c b/fs/aufs/debug.c index 44bdb5ff8..443fee39c 100644 --- a/fs/aufs/debug.c +++ b/fs/aufs/debug.c @@ -323,7 +323,7 @@ void au_dpri_sb(struct super_block *sb) au_br_count_init(&a->fake); err = do_pri_br(-1, &a->fake); au_br_count_fin(&a->fake); - kfree(a); + au_delayed_kfree(a); dpri("dev 0x%x\n", sb->s_dev); if (err || !au_test_aufs(sb)) return; @@ -331,9 +331,8 @@ void au_dpri_sb(struct super_block *sb) sbinfo = au_sbi(sb); if (!sbinfo) return; - dpri("nw %lld, gen %u, kobj %d\n", - percpu_counter_sum(&sbinfo->si_nowait.nw_len), - sbinfo->si_generation, + dpri("nw %d, gen %u, kobj %d\n", + atomic_read(&sbinfo->si_nowait.nw_len), sbinfo->si_generation, atomic_read(&sbinfo->si_kobj.kref.refcount)); for (bindex = 0; bindex <= sbinfo->si_bbot; bindex++) do_pri_br(bindex, sbinfo->si_branch[0 + bindex]); diff --git a/fs/aufs/dentry.c b/fs/aufs/dentry.c index 4bb153086..51af2cc72 100644 --- a/fs/aufs/dentry.c +++ b/fs/aufs/dentry.c @@ -201,7 +201,7 @@ int au_lkup_dentry(struct dentry *dentry, aufs_bindex_t btop, out_parent: dput(parent); - kfree(whname.name); + au_delayed_kfree(whname.name); out: return err; } diff --git a/fs/aufs/dentry.h b/fs/aufs/dentry.h index bdc1af622..dc027191c 100644 --- a/fs/aufs/dentry.h +++ b/fs/aufs/dentry.h @@ -25,7 +25,10 @@ struct au_dinfo { struct au_rwsem di_rwsem; aufs_bindex_t di_btop, di_bbot, di_bwh, di_bdiropq; unsigned char di_tmpfile; /* to allow the different name */ - struct au_hdentry *di_hdentry; + union { + struct au_hdentry *di_hdentry; + struct llist_node di_lnode; /* delayed free */ + }; } ____cacheline_aligned_in_smp; /* ---------------------------------------------------------------------- */ diff --git a/fs/aufs/dinfo.c b/fs/aufs/dinfo.c index 43b550973..361e86142 100644 --- a/fs/aufs/dinfo.c +++ b/fs/aufs/dinfo.c @@ -40,7 +40,7 @@ struct au_dinfo *au_di_alloc(struct super_block *sb, unsigned int lsc) goto out; } - au_cache_free_dinfo(dinfo); + au_cache_dfree_dinfo(dinfo); dinfo = NULL; out: @@ -60,8 +60,8 @@ void au_di_free(struct au_dinfo *dinfo) while (bindex++ <= bbot) au_hdput(p++); } - kfree(dinfo->di_hdentry); - au_cache_free_dinfo(dinfo); + au_delayed_kfree(dinfo->di_hdentry); + au_cache_dfree_dinfo(dinfo); } void au_di_swap(struct au_dinfo *a, struct au_dinfo *b) diff --git a/fs/aufs/dir.c b/fs/aufs/dir.c index 716158c3c..395563e4b 100644 --- a/fs/aufs/dir.c +++ b/fs/aufs/dir.c @@ -143,7 +143,7 @@ out_unlock: out: dput(a->dentry); au_nwt_done(&au_sbi(sb)->si_nowait); - kfree(arg); + au_delayed_kfree(arg); } void au_dir_ts(struct inode *dir, aufs_bindex_t bindex) @@ -179,7 +179,7 @@ void au_dir_ts(struct inode *dir, aufs_bindex_t bindex) if (unlikely(wkq_err)) { pr_err("wkq %d\n", wkq_err); dput(dentry); - kfree(arg); + au_delayed_kfree(arg); } out: @@ -298,7 +298,7 @@ static int aufs_open_dir(struct inode *inode __maybe_unused, }; err = au_do_open(file, &args); if (unlikely(err)) - kfree(fidir); + au_delayed_kfree(fidir); } si_read_unlock(sb); return err; @@ -310,8 +310,11 @@ static int aufs_release_dir(struct inode *inode __maybe_unused, struct au_vdir *vdir_cache; struct au_finfo *finfo; struct au_fidir *fidir; + struct au_hfile *hf; aufs_bindex_t bindex, bbot; + int execed, delayed; + delayed = (current->flags & PF_KTHREAD) || in_interrupt(); finfo = au_fi(file); fidir = finfo->fi_hdir; if (fidir) { @@ -319,22 +322,25 @@ static int aufs_release_dir(struct inode *inode __maybe_unused, &au_sbi(file->f_path.dentry->d_sb)->si_files); vdir_cache = fidir->fd_vdir_cache; /* lock-free */ if (vdir_cache) - au_vdir_free(vdir_cache); + au_vdir_free(vdir_cache, delayed); bindex = finfo->fi_btop; if (bindex >= 0) { + execed = vfsub_file_execed(file); + hf = fidir->fd_hfile + bindex; /* * calls fput() instead of filp_close(), * since no dnotify or lock for the lower file. */ bbot = fidir->fd_bbot; - for (; bindex <= bbot; bindex++) - au_set_h_fptr(file, bindex, NULL); + for (; bindex <= bbot; bindex++, hf++) + if (hf->hf_file) + au_hfput(hf, execed); } - kfree(fidir); + au_delayed_kfree(fidir); finfo->fi_hdir = NULL; } - au_finfo_fin(file); + au_finfo_fin(file, delayed); return 0; } @@ -453,7 +459,7 @@ static int aufs_fsync_dir(struct file *file, loff_t start, loff_t end, /* ---------------------------------------------------------------------- */ -static int aufs_iterate(struct file *file, struct dir_context *ctx) +static int aufs_iterate_shared(struct file *file, struct dir_context *ctx) { int err; struct dentry *dentry; @@ -731,7 +737,7 @@ const struct file_operations aufs_dir_fop = { .owner = THIS_MODULE, .llseek = default_llseek, .read = generic_read_dir, - .iterate = aufs_iterate, + .iterate_shared = aufs_iterate_shared, .unlocked_ioctl = aufs_ioctl_dir, #ifdef CONFIG_COMPAT .compat_ioctl = aufs_compat_ioctl_dir, diff --git a/fs/aufs/dir.h b/fs/aufs/dir.h index b0a79d722..4f3945ab8 100644 --- a/fs/aufs/dir.h +++ b/fs/aufs/dir.h @@ -29,7 +29,10 @@ struct au_vdir_destr { struct au_vdir_dehstr { struct hlist_node hash; - struct au_vdir_destr *str; + union { + struct au_vdir_destr *str; + struct llist_node lnode; /* delayed free */ + }; } ____cacheline_aligned_in_smp; struct au_vdir_de { @@ -67,7 +70,10 @@ struct au_vdir { unsigned long vd_version; unsigned int vd_deblk_sz; - unsigned long vd_jiffy; + union { + unsigned long vd_jiffy; + struct llist_node vd_lnode; /* delayed free */ + }; } ____cacheline_aligned_in_smp; /* ---------------------------------------------------------------------- */ @@ -91,7 +97,7 @@ int au_nhash_test_known_wh(struct au_nhash *whlist, char *name, int nlen); int au_nhash_append_wh(struct au_nhash *whlist, char *name, int nlen, ino_t ino, unsigned int d_type, aufs_bindex_t bindex, unsigned char shwh); -void au_vdir_free(struct au_vdir *vdir); +void au_vdir_free(struct au_vdir *vdir, int atonce); int au_vdir_init(struct file *file); int au_vdir_fill_de(struct file *file, struct dir_context *ctx); diff --git a/fs/aufs/dynop.c b/fs/aufs/dynop.c index dfb3a718d..1deb54077 100644 --- a/fs/aufs/dynop.c +++ b/fs/aufs/dynop.c @@ -14,17 +14,17 @@ * How large will these lists be? * Usually just a few elements, 20-30 at most for each, I guess. */ -static struct au_splhead dynop[AuDyLast]; +static struct au_sphlhead dynop[AuDyLast]; -static struct au_dykey *dy_gfind_get(struct au_splhead *spl, const void *h_op) +static struct au_dykey *dy_gfind_get(struct au_sphlhead *sphl, const void *h_op) { struct au_dykey *key, *tmp; - struct list_head *head; + struct hlist_head *head; key = NULL; - head = &spl->head; + head = &sphl->head; rcu_read_lock(); - list_for_each_entry_rcu(tmp, head, dk_list) + hlist_for_each_entry_rcu(tmp, head, dk_hnode) if (tmp->dk_op.dy_hop == h_op) { key = tmp; kref_get(&key->dk_kref); @@ -71,24 +71,24 @@ static struct au_dykey *dy_bradd(struct au_branch *br, struct au_dykey *key) } /* kref_get() if @key is already added */ -static struct au_dykey *dy_gadd(struct au_splhead *spl, struct au_dykey *key) +static struct au_dykey *dy_gadd(struct au_sphlhead *sphl, struct au_dykey *key) { struct au_dykey *tmp, *found; - struct list_head *head; + struct hlist_head *head; const void *h_op = key->dk_op.dy_hop; found = NULL; - head = &spl->head; - spin_lock(&spl->spin); - list_for_each_entry(tmp, head, dk_list) + head = &sphl->head; + spin_lock(&sphl->spin); + hlist_for_each_entry(tmp, head, dk_hnode) if (tmp->dk_op.dy_hop == h_op) { kref_get(&tmp->dk_kref); found = tmp; break; } if (!found) - list_add_rcu(&key->dk_list, head); - spin_unlock(&spl->spin); + hlist_add_head_rcu(&key->dk_hnode, head); + spin_unlock(&sphl->spin); if (!found) DyPrSym(key); @@ -101,17 +101,17 @@ static void dy_free_rcu(struct rcu_head *rcu) key = container_of(rcu, struct au_dykey, dk_rcu); DyPrSym(key); - kfree(key); + kfree(key); /* not delayed */ } static void dy_free(struct kref *kref) { struct au_dykey *key; - struct au_splhead *spl; + struct au_sphlhead *sphl; key = container_of(kref, struct au_dykey, dk_kref); - spl = dynop + key->dk_op.dy_type; - au_spl_del_rcu(&key->dk_list, spl); + sphl = dynop + key->dk_op.dy_type; + au_sphl_del_rcu(&key->dk_hnode, sphl); call_rcu(&key->dk_rcu, dy_free_rcu); } @@ -196,7 +196,7 @@ static void dy_bug(struct kref *kref) static struct au_dykey *dy_get(struct au_dynop *op, struct au_branch *br) { struct au_dykey *key, *old; - struct au_splhead *spl; + struct au_sphlhead *sphl; struct op { unsigned int sz; void (*set)(struct au_dykey *key, const void *h_op, @@ -210,8 +210,8 @@ static struct au_dykey *dy_get(struct au_dynop *op, struct au_branch *br) }; const struct op *p; - spl = dynop + op->dy_type; - key = dy_gfind_get(spl, op->dy_hop); + sphl = dynop + op->dy_type; + key = dy_gfind_get(sphl, op->dy_hop); if (key) goto out_add; /* success */ @@ -225,9 +225,9 @@ static struct au_dykey *dy_get(struct au_dynop *op, struct au_branch *br) key->dk_op.dy_hop = op->dy_hop; kref_init(&key->dk_kref); p->set(key, op->dy_hop, au_br_sb(br)); - old = dy_gadd(spl, key); + old = dy_gadd(sphl, key); if (old) { - kfree(key); + au_delayed_kfree(key); key = old; } @@ -322,16 +322,16 @@ int au_dy_irefresh(struct inode *inode) void au_dy_arefresh(int do_dx) { - struct au_splhead *spl; - struct list_head *head; + struct au_sphlhead *sphl; + struct hlist_head *head; struct au_dykey *key; - spl = dynop + AuDy_AOP; - head = &spl->head; - spin_lock(&spl->spin); - list_for_each_entry(key, head, dk_list) + sphl = dynop + AuDy_AOP; + head = &sphl->head; + spin_lock(&sphl->spin); + hlist_for_each_entry(key, head, dk_hnode) dy_adx((void *)key, do_dx); - spin_unlock(&spl->spin); + spin_unlock(&sphl->spin); } /* ---------------------------------------------------------------------- */ @@ -344,7 +344,7 @@ void __init au_dy_init(void) BUILD_BUG_ON(offsetof(struct au_dyaop, da_key)); for (i = 0; i < AuDyLast; i++) - au_spl_init(dynop + i); + au_sphl_init(dynop + i); } void au_dy_fin(void) @@ -352,5 +352,5 @@ void au_dy_fin(void) int i; for (i = 0; i < AuDyLast; i++) - WARN_ON(!list_empty(&dynop[i].head)); + WARN_ON(!hlist_empty(&dynop[i].head)); } diff --git a/fs/aufs/dynop.h b/fs/aufs/dynop.h index 8680bfc53..5db2da2a3 100644 --- a/fs/aufs/dynop.h +++ b/fs/aufs/dynop.h @@ -26,7 +26,7 @@ struct au_dynop { struct au_dykey { union { - struct list_head dk_list; + struct hlist_node dk_hnode; struct rcu_head dk_rcu; }; struct au_dynop dk_op; diff --git a/fs/aufs/export.c b/fs/aufs/export.c index 98a7036c2..6b5a7be87 100644 --- a/fs/aufs/export.c +++ b/fs/aufs/export.c @@ -404,7 +404,7 @@ static struct dentry *au_lkup_by_ino(struct path *path, ino_t ino, } out_name: - free_page((unsigned long)arg.name); + au_delayed_free_page((unsigned long)arg.name); out_file: fput(file); out: @@ -558,7 +558,7 @@ out_relock: dentry = ERR_PTR(-ESTALE); } out_pathname: - free_page((unsigned long)pathname); + au_delayed_free_page((unsigned long)pathname); out_h_parent: dput(h_parent); out: diff --git a/fs/aufs/f_op.c b/fs/aufs/f_op.c index 504b76751..00475fb72 100644 --- a/fs/aufs/f_op.c +++ b/fs/aufs/f_op.c @@ -86,6 +86,7 @@ int aufs_release_nondir(struct inode *inode __maybe_unused, struct file *file) { struct au_finfo *finfo; aufs_bindex_t bindex; + int delayed; finfo = au_fi(file); au_sphl_del(&finfo->fi_hlist, @@ -94,7 +95,8 @@ int aufs_release_nondir(struct inode *inode __maybe_unused, struct file *file) if (bindex >= 0) au_set_h_fptr(file, bindex, NULL); - au_finfo_fin(file); + delayed = (current->flags & PF_KTHREAD) || in_interrupt(); + au_finfo_fin(file, delayed); return 0; } diff --git a/fs/aufs/file.c b/fs/aufs/file.c index 0113f3ef4..429b56840 100644 --- a/fs/aufs/file.c +++ b/fs/aufs/file.c @@ -260,7 +260,7 @@ int au_do_open(struct file *file, struct au_do_open_args *args) } if (unlikely(err)) { finfo->fi_hdir = NULL; - au_finfo_fin(file); + au_finfo_fin(file, /*atonce*/0); } out: @@ -580,6 +580,7 @@ out: static void au_do_refresh_dir(struct file *file) { + int execed; aufs_bindex_t bindex, bbot, new_bindex, brid; struct au_hfile *p, tmp, *q; struct au_finfo *finfo; @@ -618,6 +619,7 @@ static void au_do_refresh_dir(struct file *file) } } + execed = vfsub_file_execed(file); p = fidir->fd_hfile; if (!au_test_mmapped(file) && !d_unlinked(file->f_path.dentry)) { bbot = au_sbbot(sb); @@ -626,14 +628,14 @@ static void au_do_refresh_dir(struct file *file) if (p->hf_file) { if (file_inode(p->hf_file)) break; - au_hfput(p, file); + au_hfput(p, execed); } } else { bbot = au_br_index(sb, brid); for (finfo->fi_btop = 0; finfo->fi_btop < bbot; finfo->fi_btop++, p++) if (p->hf_file) - au_hfput(p, file); + au_hfput(p, execed); bbot = au_sbbot(sb); } @@ -643,7 +645,7 @@ static void au_do_refresh_dir(struct file *file) if (p->hf_file) { if (file_inode(p->hf_file)) break; - au_hfput(p, file); + au_hfput(p, execed); } AuDebugOn(fidir->fd_bbot < finfo->fi_btop); } @@ -758,8 +760,7 @@ static int aufs_readpage(struct file *file __maybe_unused, struct page *page) } /* it will never be called, but necessary to support O_DIRECT */ -static ssize_t aufs_direct_IO(struct kiocb *iocb, struct iov_iter *iter, - loff_t offset) +static ssize_t aufs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) { BUG(); return 0; } /* they will never be called. */ diff --git a/fs/aufs/file.h b/fs/aufs/file.h index 96ab0a088..6ecd0df33 100644 --- a/fs/aufs/file.h +++ b/fs/aufs/file.h @@ -50,7 +50,10 @@ struct au_finfo { struct au_fidir *fi_hdir; /* for dir only */ struct hlist_node fi_hlist; - struct file *fi_file; /* very ugly */ + union { + struct file *fi_file; /* very ugly */ + struct llist_node fi_lnode; /* delayed free */ + }; } ____cacheline_aligned_in_smp; /* ---------------------------------------------------------------------- */ @@ -101,7 +104,7 @@ int aufs_release_nondir(struct inode *inode __maybe_unused, struct file *file); struct file *au_read_pre(struct file *file, int keep_fi); /* finfo.c */ -void au_hfput(struct au_hfile *hf, struct file *file); +void au_hfput(struct au_hfile *hf, int execed); void au_set_h_fptr(struct file *file, aufs_bindex_t bindex, struct file *h_file); @@ -110,7 +113,7 @@ struct au_fidir *au_fidir_alloc(struct super_block *sb); int au_fidir_realloc(struct au_finfo *finfo, int nbr); void au_fi_init_once(void *_fi); -void au_finfo_fin(struct file *file); +void au_finfo_fin(struct file *file, int atonce); int au_finfo_init(struct file *file, struct au_fidir *fidir); /* ioctl.c */ diff --git a/fs/aufs/finfo.c b/fs/aufs/finfo.c index 07f6eb32d..e0c16b2f1 100644 --- a/fs/aufs/finfo.c +++ b/fs/aufs/finfo.c @@ -8,10 +8,9 @@ #include "aufs.h" -void au_hfput(struct au_hfile *hf, struct file *file) +void au_hfput(struct au_hfile *hf, int execed) { - /* todo: direct access f_flags */ - if (vfsub_file_flags(file) & __FMODE_EXEC) + if (execed) allow_write_access(hf->hf_file); fput(hf->hf_file); hf->hf_file = NULL; @@ -33,7 +32,7 @@ void au_set_h_fptr(struct file *file, aufs_bindex_t bindex, struct file *val) hf = fidir->fd_hfile + bindex; if (hf && hf->hf_file) - au_hfput(hf, file); + au_hfput(hf, vfsub_file_execed(file)); if (val) { FiMustWriteLock(file); AuDebugOn(IS_ERR_OR_NULL(file->f_path.dentry)); @@ -90,7 +89,7 @@ int au_fidir_realloc(struct au_finfo *finfo, int nbr) /* ---------------------------------------------------------------------- */ -void au_finfo_fin(struct file *file) +void au_finfo_fin(struct file *file, int atonce) { struct au_finfo *finfo; @@ -99,7 +98,10 @@ void au_finfo_fin(struct file *file) finfo = au_fi(file); AuDebugOn(finfo->fi_hdir); AuRwDestroy(&finfo->fi_rwsem); - au_cache_free_finfo(finfo); + if (!atonce) + au_cache_dfree_finfo(finfo); + else + au_cache_free_finfo(finfo); } void au_fi_init_once(void *_finfo) diff --git a/fs/aufs/fstype.h b/fs/aufs/fstype.h index 725b2ffff..95b78b96c 100644 --- a/fs/aufs/fstype.h +++ b/fs/aufs/fstype.h @@ -28,7 +28,7 @@ static inline const char *au_sbtype(struct super_block *sb) static inline int au_test_iso9660(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_ISO9660_FS) || defined(CONFIG_ISO9660_FS_MODULE) +#if IS_ENABLED(CONFIG_ISO9660_FS) return sb->s_magic == ISOFS_SUPER_MAGIC; #else return 0; @@ -37,7 +37,7 @@ static inline int au_test_iso9660(struct super_block *sb __maybe_unused) static inline int au_test_romfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_ROMFS_FS) || defined(CONFIG_ROMFS_FS_MODULE) +#if IS_ENABLED(CONFIG_ROMFS_FS) return sb->s_magic == ROMFS_MAGIC; #else return 0; @@ -46,7 +46,7 @@ static inline int au_test_romfs(struct super_block *sb __maybe_unused) static inline int au_test_cramfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_CRAMFS) || defined(CONFIG_CRAMFS_MODULE) +#if IS_ENABLED(CONFIG_CRAMFS) return sb->s_magic == CRAMFS_MAGIC; #endif return 0; @@ -54,7 +54,7 @@ static inline int au_test_cramfs(struct super_block *sb __maybe_unused) static inline int au_test_nfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_NFS_FS) || defined(CONFIG_NFS_FS_MODULE) +#if IS_ENABLED(CONFIG_NFS_FS) return sb->s_magic == NFS_SUPER_MAGIC; #else return 0; @@ -63,7 +63,7 @@ static inline int au_test_nfs(struct super_block *sb __maybe_unused) static inline int au_test_fuse(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_FUSE_FS) || defined(CONFIG_FUSE_FS_MODULE) +#if IS_ENABLED(CONFIG_FUSE_FS) return sb->s_magic == FUSE_SUPER_MAGIC; #else return 0; @@ -72,7 +72,7 @@ static inline int au_test_fuse(struct super_block *sb __maybe_unused) static inline int au_test_xfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_XFS_FS) || defined(CONFIG_XFS_FS_MODULE) +#if IS_ENABLED(CONFIG_XFS_FS) return sb->s_magic == XFS_SB_MAGIC; #else return 0; @@ -90,7 +90,7 @@ static inline int au_test_tmpfs(struct super_block *sb __maybe_unused) static inline int au_test_ecryptfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_ECRYPT_FS) || defined(CONFIG_ECRYPT_FS_MODULE) +#if IS_ENABLED(CONFIG_ECRYPT_FS) return !strcmp(au_sbtype(sb), "ecryptfs"); #else return 0; @@ -104,7 +104,7 @@ static inline int au_test_ramfs(struct super_block *sb) static inline int au_test_ubifs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_UBIFS_FS) || defined(CONFIG_UBIFS_FS_MODULE) +#if IS_ENABLED(CONFIG_UBIFS_FS) return sb->s_magic == UBIFS_SUPER_MAGIC; #else return 0; @@ -131,7 +131,7 @@ static inline int au_test_sysfs(struct super_block *sb __maybe_unused) static inline int au_test_configfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_CONFIGFS_FS) || defined(CONFIG_CONFIGFS_FS_MODULE) +#if IS_ENABLED(CONFIG_CONFIGFS_FS) return sb->s_magic == CONFIGFS_MAGIC; #else return 0; @@ -140,7 +140,7 @@ static inline int au_test_configfs(struct super_block *sb __maybe_unused) static inline int au_test_minix(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_MINIX_FS) || defined(CONFIG_MINIX_FS_MODULE) +#if IS_ENABLED(CONFIG_MINIX_FS) return sb->s_magic == MINIX3_SUPER_MAGIC || sb->s_magic == MINIX2_SUPER_MAGIC || sb->s_magic == MINIX2_SUPER_MAGIC2 @@ -153,7 +153,7 @@ static inline int au_test_minix(struct super_block *sb __maybe_unused) static inline int au_test_fat(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_FAT_FS) || defined(CONFIG_FAT_FS_MODULE) +#if IS_ENABLED(CONFIG_FAT_FS) return sb->s_magic == MSDOS_SUPER_MAGIC; #else return 0; @@ -181,7 +181,7 @@ static inline int au_test_securityfs(struct super_block *sb __maybe_unused) static inline int au_test_squashfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_SQUASHFS) || defined(CONFIG_SQUASHFS_MODULE) +#if IS_ENABLED(CONFIG_SQUASHFS) return sb->s_magic == SQUASHFS_MAGIC; #else return 0; @@ -190,7 +190,7 @@ static inline int au_test_squashfs(struct super_block *sb __maybe_unused) static inline int au_test_btrfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_BTRFS_FS) || defined(CONFIG_BTRFS_FS_MODULE) +#if IS_ENABLED(CONFIG_BTRFS_FS) return sb->s_magic == BTRFS_SUPER_MAGIC; #else return 0; @@ -199,7 +199,7 @@ static inline int au_test_btrfs(struct super_block *sb __maybe_unused) static inline int au_test_xenfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_XENFS) || defined(CONFIG_XENFS_MODULE) +#if IS_ENABLED(CONFIG_XENFS) return sb->s_magic == XENFS_SUPER_MAGIC; #else return 0; @@ -217,7 +217,7 @@ static inline int au_test_debugfs(struct super_block *sb __maybe_unused) static inline int au_test_nilfs(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_NILFS) || defined(CONFIG_NILFS_MODULE) +#if IS_ENABLED(CONFIG_NILFS) return sb->s_magic == NILFS_SUPER_MAGIC; #else return 0; @@ -226,7 +226,7 @@ static inline int au_test_nilfs(struct super_block *sb __maybe_unused) static inline int au_test_hfsplus(struct super_block *sb __maybe_unused) { -#if defined(CONFIG_HFSPLUS_FS) || defined(CONFIG_HFSPLUS_FS_MODULE) +#if IS_ENABLED(CONFIG_HFSPLUS_FS) return sb->s_magic == HFSPLUS_SUPER_MAGIC; #else return 0; diff --git a/fs/aufs/hfsnotify.c b/fs/aufs/hfsnotify.c index 6afef3f07..110f8f53c 100644 --- a/fs/aufs/hfsnotify.c +++ b/fs/aufs/hfsnotify.c @@ -19,7 +19,7 @@ static void au_hfsn_free_mark(struct fsnotify_mark *mark) struct au_hnotify *hn = container_of(mark, struct au_hnotify, hn_mark); /* AuDbg("here\n"); */ - au_cache_free_hnotify(hn); + au_cache_dfree_hnotify(hn); smp_mb__before_atomic(); if (atomic64_dec_and_test(&au_hfsn_ifree)) wake_up(&au_hfsn_wq); @@ -143,7 +143,7 @@ static void au_hfsn_free_group(struct fsnotify_group *group) struct au_br_hfsnotify *hfsn = group->private; /* AuDbg("here\n"); */ - kfree(hfsn); + au_delayed_kfree(hfsn); } static int au_hfsn_handle_event(struct fsnotify_group *group, @@ -237,7 +237,7 @@ static int au_hfsn_init_br(struct au_branch *br, int perm) goto out; /* success */ out_hfsn: - kfree(hfsn); + au_delayed_kfree(hfsn); out: return err; } diff --git a/fs/aufs/hnotify.c b/fs/aufs/hnotify.c index 3016c5e73..a05aed870 100644 --- a/fs/aufs/hnotify.c +++ b/fs/aufs/hnotify.c @@ -22,7 +22,7 @@ int au_hn_alloc(struct au_hinode *hinode, struct inode *inode) AuTraceErr(err); if (unlikely(err)) { hinode->hi_notify = NULL; - au_cache_free_hnotify(hn); + au_cache_dfree_hnotify(hn); /* * The upper dir was removed by udba, but the same named * dir left. In this case, aufs assignes a new inode @@ -46,7 +46,7 @@ void au_hn_free(struct au_hinode *hinode) if (hn) { hinode->hi_notify = NULL; if (au_hnotify_op.free(hinode, hn)) - au_cache_free_hnotify(hn); + au_cache_dfree_hnotify(hn); } } @@ -480,7 +480,7 @@ static void au_hn_bh(void *_args) || au_ftest_hnjob(a->flags[AuHn_CHILD], GEN))) { inode = lookup_wlock_by_ino(sb, bfound, h_ino); try_iput = 1; - } + } args.flags = a->flags[AuHn_CHILD]; args.dentry = dentry; @@ -519,7 +519,7 @@ out: iput(a->dir); si_write_unlock(sb); au_nwt_done(&sbinfo->si_nowait); - kfree(a); + au_delayed_kfree(a); } /* ---------------------------------------------------------------------- */ @@ -625,7 +625,7 @@ int au_hnotify(struct inode *h_dir, struct au_hnotify *hnotify, u32 mask, iput(args->h_child_inode); iput(args->h_dir); iput(args->dir); - kfree(args); + au_delayed_kfree(args); } out: @@ -666,17 +666,26 @@ void au_hnotify_fin_br(struct au_branch *br) static void au_hn_destroy_cache(void) { - kmem_cache_destroy(au_cachep[AuCache_HNOTIFY]); - au_cachep[AuCache_HNOTIFY] = NULL; + struct au_cache *cp; + + flush_delayed_work(&au_dfree.dwork); + cp = au_dfree.cache + AuCache_HNOTIFY; + AuDebugOn(!llist_empty(&cp->llist)); + kmem_cache_destroy(cp->cache); + cp->cache = NULL; } +AU_CACHE_DFREE_FUNC(hnotify, HNOTIFY, hn_lnode); + int __init au_hnotify_init(void) { int err; + struct au_cache *cp; err = -ENOMEM; - au_cachep[AuCache_HNOTIFY] = AuCache(au_hnotify); - if (au_cachep[AuCache_HNOTIFY]) { + cp = au_dfree.cache + AuCache_HNOTIFY; + cp->cache = AuCache(au_hnotify); + if (cp->cache) { err = 0; if (au_hnotify_op.init) err = au_hnotify_op.init(); @@ -689,9 +698,13 @@ int __init au_hnotify_init(void) void au_hnotify_fin(void) { + struct au_cache *cp; + if (au_hnotify_op.fin) au_hnotify_op.fin(); + /* cf. au_cache_fin() */ - if (au_cachep[AuCache_HNOTIFY]) + cp = au_dfree.cache + AuCache_HNOTIFY; + if (cp->cache) au_hn_destroy_cache(); } diff --git a/fs/aufs/i_op.c b/fs/aufs/i_op.c index c274958dd..f5447fd34 100644 --- a/fs/aufs/i_op.c +++ b/fs/aufs/i_op.c @@ -268,8 +268,7 @@ static int aufs_atomic_open(struct inode *dir, struct dentry *dentry, { int err, h_opened = *opened; unsigned int lkup_flags; - struct dentry *parent; - struct dentry *d; + struct dentry *parent, *d; struct au_sphlhead *aopen; struct vfsub_aopen_args args = { .open_flag = open_flag, @@ -364,10 +363,10 @@ out_unlock: di_write_unlock(parent); aufs_read_unlock(dentry, AuLock_DW); AuDbgDentry(dentry); - if (unlikely(err)) + if (unlikely(err < 0)) goto out; out_no_open: - if (!err && !(*opened & FILE_CREATED)) { + if (err >= 0 && !(*opened & FILE_CREATED)) { AuLabel(out_no_open); dget(dentry); err = finish_no_open(file, dentry); @@ -567,17 +566,17 @@ out: return err; } -void au_pin_hdir_set_owner(struct au_pin *p, struct task_struct *task) +static void au_pin_hdir_set_owner(struct au_pin *p, struct task_struct *task) { -#if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_SMP) - p->hdir->hi_inode->i_mutex.owner = task; +#if !defined(CONFIG_RWSEM_GENERIC_SPINLOCK) && defined(CONFIG_RWSEM_SPIN_ON_OWNER) + p->hdir->hi_inode->i_rwsem.owner = task; #endif } void au_pin_hdir_acquire_nest(struct au_pin *p) { if (p->hdir) { - mutex_acquire_nest(&p->hdir->hi_inode->i_mutex.dep_map, + rwsem_acquire_nest(&p->hdir->hi_inode->i_rwsem.dep_map, p->lsc_hi, 0, NULL, _RET_IP_); au_pin_hdir_set_owner(p, current); } @@ -587,7 +586,7 @@ void au_pin_hdir_release(struct au_pin *p) { if (p->hdir) { au_pin_hdir_set_owner(p, p->task); - mutex_release(&p->hdir->hi_inode->i_mutex.dep_map, 1, _RET_IP_); + rwsem_release(&p->hdir->hi_inode->i_rwsem.dep_map, 1, _RET_IP_); } } @@ -981,7 +980,7 @@ out_dentry: out_si: si_read_unlock(sb); out_kfree: - kfree(a); + au_delayed_kfree(a); out: AuTraceErr(err); return err; @@ -1016,15 +1015,15 @@ out: return err; } -ssize_t au_srxattr(struct dentry *dentry, struct au_srxattr *arg) +ssize_t au_srxattr(struct dentry *dentry, struct inode *inode, + struct au_srxattr *arg) { int err; struct path h_path; struct super_block *sb; struct au_icpup_args *a; - struct inode *inode, *h_inode; + struct inode *h_inode; - inode = d_inode(dentry); IMustLock(inode); err = -ENOMEM; @@ -1046,6 +1045,7 @@ ssize_t au_srxattr(struct dentry *dentry, struct au_srxattr *arg) inode_unlock(a->h_inode); switch (arg->type) { case AU_XATTR_SET: + AuDebugOn(d_is_negative(h_path.dentry)); err = vfsub_setxattr(h_path.dentry, arg->u.set.name, arg->u.set.value, arg->u.set.size, arg->u.set.flags); @@ -1073,7 +1073,7 @@ out_di: di_write_unlock(dentry); si_read_unlock(sb); out_kfree: - kfree(a); + au_delayed_kfree(a); out: AuTraceErr(err); return err; diff --git a/fs/aufs/i_op_add.c b/fs/aufs/i_op_add.c index 8e3fb61e8..2a9aa0993 100644 --- a/fs/aufs/i_op_add.c +++ b/fs/aufs/i_op_add.c @@ -332,7 +332,7 @@ out_unlock: if (!try_aopen) aufs_read_unlock(dentry, AuLock_DW); out_free: - kfree(a); + au_delayed_kfree(a); out: return err; } @@ -796,7 +796,7 @@ out_unlock: } aufs_read_and_write_unlock2(dentry, src_dentry); out_kfree: - kfree(a); + au_delayed_kfree(a); out: AuTraceErr(err); return err; @@ -905,7 +905,7 @@ out_unlock: } aufs_read_unlock(dentry, AuLock_DW); out_free: - kfree(a); + au_delayed_kfree(a); out: return err; } diff --git a/fs/aufs/i_op_del.c b/fs/aufs/i_op_del.c index 35b923ccc..2fe8f07ef 100644 --- a/fs/aufs/i_op_del.c +++ b/fs/aufs/i_op_del.c @@ -381,7 +381,7 @@ out_parent: out_unlock: aufs_read_unlock(dentry, AuLock_DW); out_free: - kfree(a); + au_delayed_kfree(a); out: return err; } @@ -491,7 +491,7 @@ out_parent: out_unlock: aufs_read_unlock(dentry, AuLock_DW); out_free: - kfree(a); + au_delayed_kfree(a); out: AuTraceErr(err); return err; diff --git a/fs/aufs/i_op_ren.c b/fs/aufs/i_op_ren.c index df697de83..43a09090c 100644 --- a/fs/aufs/i_op_ren.c +++ b/fs/aufs/i_op_ren.c @@ -995,7 +995,7 @@ out_free: iput(a->dst_inode); if (a->thargs) au_whtmp_rmdir_free(a->thargs); - kfree(a); + au_delayed_kfree(a); out: AuTraceErr(err); return err; diff --git a/fs/aufs/iinfo.c b/fs/aufs/iinfo.c index fae4c2795..c83db695d 100644 --- a/fs/aufs/iinfo.c +++ b/fs/aufs/iinfo.c @@ -254,7 +254,7 @@ void au_iinfo_fin(struct inode *inode) iinfo = au_ii(inode); if (iinfo->ii_vdir) - au_vdir_free(iinfo->ii_vdir); + au_vdir_free(iinfo->ii_vdir, /*atonce*/0); bindex = iinfo->ii_btop; if (bindex >= 0) { @@ -266,6 +266,6 @@ void au_iinfo_fin(struct inode *inode) hi++; } } - kfree(iinfo->ii_hinode); + au_delayed_kfree(iinfo->ii_hinode); AuRwDestroy(&iinfo->ii_rwsem); } diff --git a/fs/aufs/inode.h b/fs/aufs/inode.h index 14f788268..60995c054 100644 --- a/fs/aufs/inode.h +++ b/fs/aufs/inode.h @@ -22,7 +22,10 @@ struct au_hnotify { /* never use fsnotify_add_vfsmount_mark() */ struct fsnotify_mark hn_mark; #endif - struct inode *hn_aufs_inode; /* no get/put */ + union { + struct inode *hn_aufs_inode; /* no get/put */ + struct llist_node hn_lnode; /* delayed free */ + }; #endif } ____cacheline_aligned_in_smp; @@ -65,7 +68,10 @@ struct au_iinfo { struct au_icntnr { struct au_iinfo iinfo; struct inode vfs_inode; - struct hlist_node plink; + union { + struct hlist_node plink; + struct llist_node lnode; /* delayed free */ + }; } ____cacheline_aligned_in_smp; /* au_pin flags */ @@ -98,7 +104,6 @@ struct au_pin { void au_pin_hdir_unlock(struct au_pin *p); int au_pin_hdir_lock(struct au_pin *p); int au_pin_hdir_relock(struct au_pin *p); -void au_pin_hdir_set_owner(struct au_pin *p, struct task_struct *task); void au_pin_hdir_acquire_nest(struct au_pin *p); void au_pin_hdir_release(struct au_pin *p); @@ -293,10 +298,10 @@ AuStubVoid(au_plink_half_refresh, struct super_block *sb, aufs_bindex_t br_id); int au_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, int ignore_flags, unsigned int verbose); ssize_t aufs_listxattr(struct dentry *dentry, char *list, size_t size); -ssize_t aufs_getxattr(struct dentry *dentry, const char *name, void *value, - size_t size); -int aufs_setxattr(struct dentry *dentry, const char *name, const void *value, - size_t size, int flags); +ssize_t aufs_getxattr(struct dentry *dentry, struct inode *inode, + const char *name, void *value, size_t size); +int aufs_setxattr(struct dentry *dentry, struct inode *inode, const char *name, + const void *value, size_t size, int flags); int aufs_removexattr(struct dentry *dentry, const char *name); /* void au_xattr_init(struct super_block *sb); */ @@ -336,7 +341,8 @@ struct au_srxattr { } acl_set; } u; }; -ssize_t au_srxattr(struct dentry *dentry, struct au_srxattr *arg); +ssize_t au_srxattr(struct dentry *dentry, struct inode *inode, + struct au_srxattr *arg); #endif /* ---------------------------------------------------------------------- */ diff --git a/fs/aufs/loop.c b/fs/aufs/loop.c index 5711e7a2f..e92a34599 100644 --- a/fs/aufs/loop.c +++ b/fs/aufs/loop.c @@ -129,5 +129,5 @@ void au_loopback_fin(void) { if (backing_file_func) symbol_put(loop_backing_file); - kfree(au_warn_loopback_array); + au_delayed_kfree(au_warn_loopback_array); } diff --git a/fs/aufs/module.c b/fs/aufs/module.c index 88f8f4123..ede9a232a 100644 --- a/fs/aufs/module.c +++ b/fs/aufs/module.c @@ -22,17 +22,64 @@ void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp) } /* ---------------------------------------------------------------------- */ - /* * aufs caches */ -struct kmem_cache *au_cachep[AuCache_Last] = { - [0] = NULL -}; + +struct au_dfree au_dfree; + +/* delayed free */ +static void au_do_dfree(struct work_struct *work __maybe_unused) +{ + struct llist_head *head; + struct llist_node *node, *next; + +#define AU_CACHE_DFREE_DO_BODY(name, idx, lnode) do { \ + head = &au_dfree.cache[AuCache_##idx].llist; \ + node = llist_del_all(head); \ + for (; node; node = next) { \ + struct au_##name *p = \ + p = llist_entry(node, struct au_##name, \ + lnode); \ + next = llist_next(node); \ + au_cache_free_##name(p); \ + } \ + } while (0) + + AU_CACHE_DFREE_DO_BODY(dinfo, DINFO, di_lnode); + AU_CACHE_DFREE_DO_BODY(icntnr, ICNTNR, lnode); + AU_CACHE_DFREE_DO_BODY(finfo, FINFO, fi_lnode); + AU_CACHE_DFREE_DO_BODY(vdir, VDIR, vd_lnode); + AU_CACHE_DFREE_DO_BODY(vdir_dehstr, DEHSTR, lnode); +#ifdef CONFIG_AUFS_HNOTIFY + AU_CACHE_DFREE_DO_BODY(hnotify, HNOTIFY, hn_lnode); +#endif + +#define AU_DFREE_DO_BODY(llist, func) do { \ + node = llist_del_all(llist); \ + for (; node; node = next) { \ + next = llist_next(node); \ + func(node); \ + } \ + } while (0) + + AU_DFREE_DO_BODY(au_dfree.llist + AU_DFREE_KFREE, kfree); + AU_DFREE_DO_BODY(au_dfree.llist + AU_DFREE_FREE_PAGE, au_free_page); + +#undef AU_CACHE_DFREE_DO_BODY +#undef AU_DFREE_DO_BODY +} + +AU_CACHE_DFREE_FUNC(dinfo, DINFO, di_lnode); +AU_CACHE_DFREE_FUNC(icntnr, ICNTNR, lnode); +AU_CACHE_DFREE_FUNC(finfo, FINFO, fi_lnode); +AU_CACHE_DFREE_FUNC(vdir, VDIR, vd_lnode); +AU_CACHE_DFREE_FUNC(vdir_dehstr, DEHSTR, lnode); static void au_cache_fin(void) { int i; + struct au_cache *cp; /* * Make sure all delayed rcu free inodes are flushed before we @@ -42,27 +89,33 @@ static void au_cache_fin(void) /* excluding AuCache_HNOTIFY */ BUILD_BUG_ON(AuCache_HNOTIFY + 1 != AuCache_Last); + flush_delayed_work(&au_dfree.dwork); for (i = 0; i < AuCache_HNOTIFY; i++) { - kmem_cache_destroy(au_cachep[i]); - au_cachep[i] = NULL; + cp = au_dfree.cache + i; + AuDebugOn(!llist_empty(&cp->llist)); + kmem_cache_destroy(cp->cache); + cp->cache = NULL; } } static int __init au_cache_init(void) { - au_cachep[AuCache_DINFO] = AuCacheCtor(au_dinfo, au_di_init_once); - if (au_cachep[AuCache_DINFO]) + struct au_cache *cp; + + cp = au_dfree.cache; + cp[AuCache_DINFO].cache = AuCacheCtor(au_dinfo, au_di_init_once); + if (cp[AuCache_DINFO].cache) /* SLAB_DESTROY_BY_RCU */ - au_cachep[AuCache_ICNTNR] = AuCacheCtor(au_icntnr, - au_icntnr_init_once); - if (au_cachep[AuCache_ICNTNR]) - au_cachep[AuCache_FINFO] = AuCacheCtor(au_finfo, - au_fi_init_once); - if (au_cachep[AuCache_FINFO]) - au_cachep[AuCache_VDIR] = AuCache(au_vdir); - if (au_cachep[AuCache_VDIR]) - au_cachep[AuCache_DEHSTR] = AuCache(au_vdir_dehstr); - if (au_cachep[AuCache_DEHSTR]) + cp[AuCache_ICNTNR].cache = AuCacheCtor(au_icntnr, + au_icntnr_init_once); + if (cp[AuCache_ICNTNR].cache) + cp[AuCache_FINFO].cache = AuCacheCtor(au_finfo, + au_fi_init_once); + if (cp[AuCache_FINFO].cache) + cp[AuCache_VDIR].cache = AuCache(au_vdir); + if (cp[AuCache_VDIR].cache) + cp[AuCache_DEHSTR].cache = AuCache(au_vdir_dehstr); + if (cp[AuCache_DEHSTR].cache) return 0; au_cache_fin(); @@ -124,6 +177,7 @@ static int __init aufs_init(void) { int err, i; char *p; + struct au_cache *cp; p = au_esc_chars; for (i = 1; i <= ' '; i++) @@ -138,6 +192,16 @@ static int __init aufs_init(void) for (i = 0; i < AuIop_Last; i++) aufs_iop_nogetattr[i].getattr = NULL; + /* First, initialize au_dfree */ + for (i = 0; i < AuCache_Last; i++) { /* including hnotify */ + cp = au_dfree.cache + i; + cp->cache = NULL; + init_llist_head(&cp->llist); + } + for (i = 0; i < AU_DFREE_Last; i++) + init_llist_head(au_dfree.llist + i); + INIT_DELAYED_WORK(&au_dfree.dwork, au_do_dfree); + au_sbilist_init(); sysaufs_brs_init(); au_debug_init(); @@ -188,6 +252,7 @@ out_procfs: out_sysaufs: sysaufs_fin(); au_dy_fin(); + flush_delayed_work(&au_dfree.dwork); out: return err; } @@ -203,6 +268,7 @@ static void __exit aufs_exit(void) au_procfs_fin(); sysaufs_fin(); au_dy_fin(); + flush_delayed_work(&au_dfree.dwork); } module_init(aufs_init); diff --git a/fs/aufs/module.h b/fs/aufs/module.h index 1383e3da9..681dc75f5 100644 --- a/fs/aufs/module.h +++ b/fs/aufs/module.h @@ -12,6 +12,7 @@ #ifdef __KERNEL__ #include <linux/slab.h> +#include "debug.h" struct path; struct seq_file; @@ -38,7 +39,7 @@ AuStubVoid(au_procfs_fin, void); /* ---------------------------------------------------------------------- */ -/* kmem cache */ +/* kmem cache and delayed free */ enum { AuCache_DINFO, AuCache_ICNTNR, @@ -49,19 +50,54 @@ enum { AuCache_Last }; +enum { + AU_DFREE_KFREE, + AU_DFREE_FREE_PAGE, + AU_DFREE_Last +}; + +struct au_cache { + struct kmem_cache *cache; + struct llist_head llist; /* delayed free */ +}; + +/* + * in order to reduce the cost of the internal timer, consolidate all the + * delayed free works into a single delayed_work. + */ +struct au_dfree { + struct au_cache cache[AuCache_Last]; + struct llist_head llist[AU_DFREE_Last]; + struct delayed_work dwork; +}; + +extern struct au_dfree au_dfree; + #define AuCacheFlags (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD) #define AuCache(type) KMEM_CACHE(type, AuCacheFlags) #define AuCacheCtor(type, ctor) \ kmem_cache_create(#type, sizeof(struct type), \ __alignof__(struct type), AuCacheFlags, ctor) -extern struct kmem_cache *au_cachep[]; +#define AU_DFREE_DELAY msecs_to_jiffies(10) +#define AU_DFREE_BODY(lnode, llist) do { \ + if (llist_add(lnode, llist)) \ + schedule_delayed_work(&au_dfree.dwork, \ + AU_DFREE_DELAY); \ + } while (0) +#define AU_CACHE_DFREE_FUNC(name, idx, lnode) \ + void au_cache_dfree_##name(struct au_##name *p) \ + { \ + struct au_cache *cp = au_dfree.cache + AuCache_##idx; \ + AU_DFREE_BODY(&p->lnode, &cp->llist); \ + } #define AuCacheFuncs(name, index) \ static inline struct au_##name *au_cache_alloc_##name(void) \ -{ return kmem_cache_alloc(au_cachep[AuCache_##index], GFP_NOFS); } \ +{ return kmem_cache_alloc(au_dfree.cache[AuCache_##index].cache, GFP_NOFS); } \ static inline void au_cache_free_##name(struct au_##name *p) \ -{ kmem_cache_free(au_cachep[AuCache_##index], p); } +{ kmem_cache_free(au_dfree.cache[AuCache_##index].cache, p); } \ +void au_cache_dfree_##name(struct au_##name *p) AuCacheFuncs(dinfo, DINFO); AuCacheFuncs(icntnr, ICNTNR); @@ -72,5 +108,24 @@ AuCacheFuncs(vdir_dehstr, DEHSTR); AuCacheFuncs(hnotify, HNOTIFY); #endif +static inline void au_delayed_kfree(const void *p) +{ + AuDebugOn(!p); + AuDebugOn(ksize(p) < sizeof(struct llist_node)); + + AU_DFREE_BODY((void *)p, au_dfree.llist + AU_DFREE_KFREE); +} + +/* cast only */ +static inline void au_free_page(void *p) +{ + free_page((unsigned long)p); +} + +static inline void au_delayed_free_page(unsigned long addr) +{ + AU_DFREE_BODY((void *)addr, au_dfree.llist + AU_DFREE_FREE_PAGE); +} + #endif /* __KERNEL__ */ #endif /* __AUFS_MODULE_H__ */ diff --git a/fs/aufs/mvdown.c b/fs/aufs/mvdown.c index a48aa1c50..b0e5b8483 100644 --- a/fs/aufs/mvdown.c +++ b/fs/aufs/mvdown.c @@ -684,7 +684,7 @@ out_free: e = copy_to_user(uarg, &args->mvdown, sizeof(args->mvdown)); if (unlikely(e)) err = -EFAULT; - kfree(args); + au_delayed_kfree(args); out: AuTraceErr(err); return err; diff --git a/fs/aufs/opts.c b/fs/aufs/opts.c index 79c7d7156..1d21f0de1 100644 --- a/fs/aufs/opts.c +++ b/fs/aufs/opts.c @@ -1243,7 +1243,7 @@ int au_opts_parse(struct super_block *sb, char *str, struct au_opts *opts) } } - kfree(a); + au_delayed_kfree(a); dump_opts(opts); if (unlikely(err)) au_opts_free(opts); @@ -1665,7 +1665,8 @@ int au_opts_verify(struct super_block *sb, unsigned long sb_flags, au_hn_inode_unlock(hdir); if (!err && do_free) { - kfree(wbr); + if (wbr) + au_delayed_kfree(wbr); br->br_wbr = NULL; } } diff --git a/fs/aufs/plink.c b/fs/aufs/plink.c index c42734dd4..d8d75e85d 100644 --- a/fs/aufs/plink.c +++ b/fs/aufs/plink.c @@ -32,6 +32,7 @@ int au_plink_maint(struct super_block *sb, int flags) { int err; pid_t pid, ppid; + struct task_struct *parent, *prev; struct au_sbinfo *sbi; SiMustAnyLock(sb); @@ -46,11 +47,22 @@ int au_plink_maint(struct super_block *sb, int flags) goto out; /* todo: it highly depends upon /sbin/mount.aufs */ + prev = NULL; + parent = current; + ppid = 0; rcu_read_lock(); - ppid = task_pid_vnr(rcu_dereference(current->real_parent)); + while (1) { + parent = rcu_dereference(parent->real_parent); + if (parent == prev) + break; + ppid = task_pid_vnr(parent); + if (pid == ppid) { + rcu_read_unlock(); + goto out; + } + prev = parent; + } rcu_read_unlock(); - if (pid == ppid) - goto out; if (au_ftest_lock(flags, NOPLMW)) { /* if there is no i_mutex lock in VFS, we don't need to wait */ diff --git a/fs/aufs/posix_acl.c b/fs/aufs/posix_acl.c index 0ddd46997..1ee17eeef 100644 --- a/fs/aufs/posix_acl.c +++ b/fs/aufs/posix_acl.c @@ -58,7 +58,8 @@ int aufs_set_acl(struct inode *inode, struct posix_acl *acl, int type) }, }; - inode_lock(inode); + IMustLock(inode); + if (inode->i_ino == AUFS_ROOT_INO) dentry = dget(inode->i_sb->s_root); else { @@ -73,13 +74,12 @@ int aufs_set_acl(struct inode *inode, struct posix_acl *acl, int type) } } - ssz = au_srxattr(dentry, &arg); + ssz = au_srxattr(dentry, inode, &arg); dput(dentry); err = ssz; if (ssz >= 0) err = 0; out: - inode_unlock(inode); return err; } diff --git a/fs/aufs/rdu.c b/fs/aufs/rdu.c index 7180f18e6..54abc14d3 100644 --- a/fs/aufs/rdu.c +++ b/fs/aufs/rdu.c @@ -135,7 +135,7 @@ static int au_rdu(struct file *file, struct aufs_rdu *rdu) arg.end += rdu->sz; err = -ENOTDIR; - if (unlikely(!file->f_op->iterate)) + if (unlikely(!file->f_op->iterate && !file->f_op->iterate_shared)) goto out; err = security_file_permission(file, MAY_READ); @@ -145,15 +145,7 @@ static int au_rdu(struct file *file, struct aufs_rdu *rdu) dentry = file->f_path.dentry; inode = d_inode(dentry); -#if 1 - inode_lock(inode); -#else - /* todo: create a new inline func inode_lock_killable() */ - err = mutex_lock_killable(&inode->i_mutex); - AuTraceErr(err); - if (unlikely(err)) - goto out; -#endif + inode_lock_shared(inode); arg.sb = inode->i_sb; err = si_read_lock(arg.sb, AuLock_FLUSH | AuLock_NOPLM); @@ -210,7 +202,7 @@ out_unlock: out_si: si_read_unlock(arg.sb); out_mtx: - inode_unlock(inode); + inode_unlock_shared(inode); out: AuTraceErr(err); return err; diff --git a/fs/aufs/sbinfo.c b/fs/aufs/sbinfo.c index aa970593a..25f60549c 100644 --- a/fs/aufs/sbinfo.c +++ b/fs/aufs/sbinfo.c @@ -20,7 +20,7 @@ void au_si_free(struct kobject *kobj) sbinfo = container_of(kobj, struct au_sbinfo, si_kobj); for (i = 0; i < AuPlink_NHASH; i++) AuDebugOn(!hlist_empty(&sbinfo->si_plink[i].head)); - au_nwt_fin(&sbinfo->si_nowait); + AuDebugOn(atomic_read(&sbinfo->si_nowait.nw_len)); AuDebugOn(percpu_counter_sum(&sbinfo->si_ninodes)); percpu_counter_destroy(&sbinfo->si_ninodes); @@ -31,14 +31,15 @@ void au_si_free(struct kobject *kobj) au_br_free(sbinfo); au_rw_write_unlock(&sbinfo->si_rwsem); - kfree(sbinfo->si_branch); + au_delayed_kfree(sbinfo->si_branch); for (i = 0; i < AU_NPIDMAP; i++) - kfree(sbinfo->au_si_pid.pid_bitmap[i]); + if (sbinfo->au_si_pid.pid_bitmap[i]) + au_delayed_kfree(sbinfo->au_si_pid.pid_bitmap[i]); mutex_destroy(&sbinfo->au_si_pid.pid_mtx); mutex_destroy(&sbinfo->si_xib_mtx); AuRwDestroy(&sbinfo->si_rwsem); - kfree(sbinfo); + au_delayed_kfree(sbinfo); } int au_si_alloc(struct super_block *sb) @@ -110,9 +111,9 @@ int au_si_alloc(struct super_block *sb) return 0; /* success */ out_br: - kfree(sbinfo->si_branch); + au_delayed_kfree(sbinfo->si_branch); out_sbinfo: - kfree(sbinfo); + au_delayed_kfree(sbinfo); out: return err; } diff --git a/fs/aufs/spl.h b/fs/aufs/spl.h index f9b528826..8f519db37 100644 --- a/fs/aufs/spl.h +++ b/fs/aufs/spl.h @@ -11,6 +11,7 @@ #ifdef __KERNEL__ +#if 0 struct au_splhead { spinlock_t spin; struct list_head head; @@ -43,6 +44,7 @@ static inline void au_spl_del_rcu(struct list_head *list, list_del_rcu(list); spin_unlock(&spl->spin); } +#endif /* ---------------------------------------------------------------------- */ diff --git a/fs/aufs/super.c b/fs/aufs/super.c index 8bd2d9ca4..58a773ccc 100644 --- a/fs/aufs/super.c +++ b/fs/aufs/super.c @@ -33,8 +33,7 @@ static void aufs_destroy_inode_cb(struct rcu_head *head) { struct inode *inode = container_of(head, struct inode, i_rcu); - INIT_HLIST_HEAD(&inode->i_dentry); - au_cache_free_icntnr(container_of(inode, struct au_icntnr, vfs_inode)); + au_cache_dfree_icntnr(container_of(inode, struct au_icntnr, vfs_inode)); } static void aufs_destroy_inode(struct inode *inode) @@ -816,7 +815,7 @@ static int aufs_remount_fs(struct super_block *sb, int *flags, char *data) out_mtx: inode_unlock(inode); out_opts: - free_page((unsigned long)opts.opt); + au_delayed_free_page((unsigned long)opts.opt); out: err = cvt_err(err); AuTraceErr(err); @@ -957,7 +956,7 @@ out_info: kobject_put(&sbinfo->si_kobj); sb->s_fs_info = NULL; out_opts: - free_page((unsigned long)opts.opt); + au_delayed_free_page((unsigned long)opts.opt); out: AuTraceErr(err); err = cvt_err(err); diff --git a/fs/aufs/sysfs.c b/fs/aufs/sysfs.c index 0efb77a4e..f4c31705b 100644 --- a/fs/aufs/sysfs.c +++ b/fs/aufs/sysfs.c @@ -173,7 +173,7 @@ out_seq: if (unlikely(err == PAGE_SIZE)) err = -EFBIG; } - kfree(seq); + au_delayed_kfree(seq); out_unlock: si_read_unlock(sb); out: @@ -244,9 +244,9 @@ static int au_brinfo(struct super_block *sb, union aufs_brinfo __user *arg) err = -EFAULT; out_seq: - kfree(seq); + au_delayed_kfree(seq); out_buf: - free_page((unsigned long)buf); + au_delayed_free_page((unsigned long)buf); out: si_read_unlock(sb); return err; diff --git a/fs/aufs/vdir.c b/fs/aufs/vdir.c index feddcc204..e6e3a1eaa 100644 --- a/fs/aufs/vdir.c +++ b/fs/aufs/vdir.c @@ -97,7 +97,7 @@ static void au_nhash_wh_do_free(struct hlist_head *head) struct hlist_node *node; hlist_for_each_entry_safe(pos, node, head, wh_hash) - kfree(pos); + au_delayed_kfree(pos); } static void au_nhash_de_do_free(struct hlist_head *head) @@ -106,7 +106,7 @@ static void au_nhash_de_do_free(struct hlist_head *head) struct hlist_node *node; hlist_for_each_entry_safe(pos, node, head, hash) - au_cache_free_vdir_dehstr(pos); + au_cache_dfree_vdir_dehstr(pos); } static void au_nhash_do_free(struct au_nhash *nhash, @@ -124,7 +124,7 @@ static void au_nhash_do_free(struct au_nhash *nhash, nhash_count(head); free(head++); } - kfree(nhash->nh_head); + au_delayed_kfree(nhash->nh_head); } void au_nhash_wh_free(struct au_nhash *whlist) @@ -167,6 +167,8 @@ static struct hlist_head *au_name_hash(struct au_nhash *nhash, AuDebugOn(!nhash->nh_num || !nhash->nh_head); v = 0; + if (len > 8) + len = 8; while (len--) v += *name++; /* v = hash_long(v, magic_bit); */ @@ -335,15 +337,23 @@ out: /* ---------------------------------------------------------------------- */ -void au_vdir_free(struct au_vdir *vdir) +void au_vdir_free(struct au_vdir *vdir, int atonce) { unsigned char **deblk; deblk = vdir->vd_deblk; - while (vdir->vd_nblk--) - kfree(*deblk++); - kfree(vdir->vd_deblk); - au_cache_free_vdir(vdir); + if (!atonce) { + while (vdir->vd_nblk--) + au_delayed_kfree(*deblk++); + au_delayed_kfree(vdir->vd_deblk); + au_cache_dfree_vdir(vdir); + } else { + /* not delayed */ + while (vdir->vd_nblk--) + kfree(*deblk++); + kfree(vdir->vd_deblk); + au_cache_free_vdir(vdir); + } } static struct au_vdir *alloc_vdir(struct file *file) @@ -377,10 +387,10 @@ static struct au_vdir *alloc_vdir(struct file *file) if (!err) return vdir; /* success */ - kfree(vdir->vd_deblk); + au_delayed_kfree(vdir->vd_deblk); out_free: - au_cache_free_vdir(vdir); + au_cache_dfree_vdir(vdir); out: vdir = ERR_PTR(err); return vdir; @@ -392,7 +402,7 @@ static int reinit_vdir(struct au_vdir *vdir) union au_vdir_deblk_p p, deblk_end; while (vdir->vd_nblk > 1) { - kfree(vdir->vd_deblk[vdir->vd_nblk - 1]); + au_delayed_kfree(vdir->vd_deblk[vdir->vd_nblk - 1]); /* vdir->vd_deblk[vdir->vd_nblk - 1] = NULL; */ vdir->vd_nblk--; } @@ -523,7 +533,7 @@ static int au_handle_shwh(struct super_block *sb, struct au_vdir *vdir, } } - free_page((unsigned long)o); + au_delayed_free_page((unsigned long)o); out: AuTraceErr(err); @@ -624,6 +634,7 @@ static int read_vdir(struct file *file, int may_read) err = 0; inode = file_inode(file); IMustLock(inode); + IiMustWriteLock(inode); SiMustAnyLock(inode->i_sb); allocated = NULL; @@ -661,7 +672,7 @@ static int read_vdir(struct file *file, int may_read) if (allocated) au_set_ivdir(inode, allocated); } else if (allocated) - au_vdir_free(allocated); + au_vdir_free(allocated, /*atonce*/0); out: return err; @@ -755,7 +766,7 @@ int au_vdir_init(struct file *file) if (allocated) au_set_fvdir_cache(file, allocated); } else if (allocated) - au_vdir_free(allocated); + au_vdir_free(allocated, /*atonce*/0); out: return err; diff --git a/fs/aufs/vfsub.h b/fs/aufs/vfsub.h index 291bfae7a..64fff0f0d 100644 --- a/fs/aufs/vfsub.h +++ b/fs/aufs/vfsub.h @@ -183,6 +183,12 @@ static inline unsigned int vfsub_file_flags(struct file *file) return flags; } +static inline int vfsub_file_execed(struct file *file) +{ + /* todo: direct access f_flags */ + return !!(vfsub_file_flags(file) & __FMODE_EXEC); +} + #if 0 /* reserved */ static inline void vfsub_file_accessed(struct file *h_file) { diff --git a/fs/aufs/wbr_policy.c b/fs/aufs/wbr_policy.c index 91a010a11..79eb9b49f 100644 --- a/fs/aufs/wbr_policy.c +++ b/fs/aufs/wbr_policy.c @@ -448,7 +448,7 @@ static void au_mfs(struct dentry *dentry, struct dentry *parent) mfs->mfsrr_bytes = bavail; AuDbg("b%d\n", mfs->mfs_bindex); - kfree(st); + au_delayed_kfree(st); } static int au_wbr_create_mfs(struct dentry *dentry, unsigned int flags) diff --git a/fs/aufs/whout.c b/fs/aufs/whout.c index 16a96f88f..54feb0837 100644 --- a/fs/aufs/whout.c +++ b/fs/aufs/whout.c @@ -149,7 +149,7 @@ struct dentry *au_whtmp_lkup(struct dentry *h_parent, struct au_branch *br, out_name: if (name != defname) - kfree(name); + au_delayed_kfree(name); out: AuTraceErrPtr(dentry); return dentry; @@ -588,7 +588,7 @@ out: au_br_put(a->br); si_write_unlock(a->sb); au_nwt_done(&au_sbi(a->sb)->si_nowait); - kfree(arg); + au_delayed_kfree(arg); if (unlikely(err)) AuIOErr("err %d\n", err); } @@ -616,7 +616,7 @@ static void kick_reinit_br_wh(struct super_block *sb, struct au_branch *br) if (unlikely(wkq_err)) { atomic_dec(&br->br_wbr->wbr_wh_running); au_br_put(br); - kfree(arg); + au_delayed_kfree(arg); } do_dec = 0; } @@ -775,7 +775,7 @@ struct dentry *au_wh_lkup(struct dentry *h_parent, struct qstr *base_name, wh_dentry = ERR_PTR(err); if (!err) { wh_dentry = vfsub_lkup_one(&wh_name, h_parent); - kfree(wh_name.name); + au_delayed_kfree(wh_name.name); } return wh_dentry; } @@ -851,7 +851,7 @@ static int del_wh_children(struct dentry *h_dentry, struct au_nhash *whlist, break; } } - free_page((unsigned long)wh_name.name); + au_delayed_free_page((unsigned long)wh_name.name); out: return err; @@ -893,7 +893,7 @@ struct au_whtmp_rmdir *au_whtmp_rmdir_alloc(struct super_block *sb, gfp_t gfp) rdhash = AUFS_RDHASH_DEF; err = au_nhash_alloc(&whtmp->whlist, rdhash, gfp); if (unlikely(err)) { - kfree(whtmp); + au_delayed_kfree(whtmp); whtmp = ERR_PTR(err); } @@ -908,7 +908,7 @@ void au_whtmp_rmdir_free(struct au_whtmp_rmdir *whtmp) dput(whtmp->wh_dentry); iput(whtmp->dir); au_nhash_wh_free(&whtmp->whlist); - kfree(whtmp); + au_delayed_kfree(whtmp); } /* diff --git a/fs/aufs/wkq.c b/fs/aufs/wkq.c index 65c0137a5..216d08f7c 100644 --- a/fs/aufs/wkq.c +++ b/fs/aufs/wkq.c @@ -41,7 +41,7 @@ static void wkq_func(struct work_struct *wk) else { kobject_put(wkinfo->kobj); module_put(THIS_MODULE); /* todo: ?? */ - kfree(wkinfo); + au_delayed_kfree(wkinfo); } } @@ -64,7 +64,7 @@ static int au_wkq_comp_alloc(struct au_wkinfo *wkinfo, struct completion **comp) static void au_wkq_comp_free(struct completion *comp) { - kfree(comp); + au_delayed_kfree(comp); } #else @@ -145,7 +145,7 @@ int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb, int err; struct au_wkinfo *wkinfo; - percpu_counter_inc(&au_sbi(sb)->si_nowait.nw_len); + atomic_inc(&au_sbi(sb)->si_nowait.nw_len); /* * wkq_func() must free this wkinfo. @@ -175,16 +175,11 @@ int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb, void au_nwt_init(struct au_nowait_tasks *nwt) { - percpu_counter_init(&nwt->nw_len, 0, GFP_NOFS); + atomic_set(&nwt->nw_len, 0); + /* smp_mb(); */ /* atomic_set */ init_waitqueue_head(&nwt->nw_wq); } -void au_nwt_fin(struct au_nowait_tasks *nwt) -{ - AuDebugOn(percpu_counter_sum(&nwt->nw_len)); - percpu_counter_destroy(&nwt->nw_len); -} - void au_wkq_fin(void) { destroy_workqueue(au_wkq); diff --git a/fs/aufs/wkq.h b/fs/aufs/wkq.h index 752b9c52d..6d63f7464 100644 --- a/fs/aufs/wkq.h +++ b/fs/aufs/wkq.h @@ -12,6 +12,8 @@ #ifdef __KERNEL__ +#include <linux/percpu_counter.h> + struct super_block; /* ---------------------------------------------------------------------- */ @@ -20,7 +22,7 @@ struct super_block; * in the next operation, wait for the 'nowait' tasks in system-wide workqueue */ struct au_nowait_tasks { - struct percpu_counter nw_len; + atomic_t nw_len; wait_queue_head_t nw_wq; }; @@ -47,7 +49,6 @@ int au_wkq_do_wait(unsigned int flags, au_wkq_func_t func, void *args); int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb, unsigned int flags); void au_nwt_init(struct au_nowait_tasks *nwt); -void au_nwt_fin(struct au_nowait_tasks *nwt); int __init au_wkq_init(void); void au_wkq_fin(void); @@ -65,14 +66,13 @@ static inline int au_wkq_wait(au_wkq_func_t func, void *args) static inline void au_nwt_done(struct au_nowait_tasks *nwt) { - percpu_counter_dec(&nwt->nw_len); - if (!percpu_counter_sum(&nwt->nw_len)) + if (atomic_dec_and_test(&nwt->nw_len)) wake_up_all(&nwt->nw_wq); } static inline int au_nwt_flush(struct au_nowait_tasks *nwt) { - wait_event(nwt->nw_wq, !percpu_counter_sum(&nwt->nw_len)); + wait_event(nwt->nw_wq, !atomic_read(&nwt->nw_len)); return 0; } diff --git a/fs/aufs/xattr.c b/fs/aufs/xattr.c index 890c10795..2a148c993 100644 --- a/fs/aufs/xattr.c +++ b/fs/aufs/xattr.c @@ -163,10 +163,12 @@ int au_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, int ignore_flags, AuTraceErr(err); } - kfree(value); + if (value) + au_delayed_kfree(value); out_free: - kfree(o); + if (o) + au_delayed_kfree(o); out: if (!unlocked) inode_unlock(h_isrc); @@ -220,6 +222,7 @@ static ssize_t au_lgxattr(struct dentry *dentry, struct au_lgxattr *arg) arg->u.list.list, arg->u.list.size); break; case AU_XATTR_GET: + AuDebugOn(d_is_negative(h_path.dentry)); err = vfs_getxattr(h_path.dentry, arg->u.get.name, arg->u.get.value, arg->u.get.size); @@ -248,8 +251,8 @@ ssize_t aufs_listxattr(struct dentry *dentry, char *list, size_t size) return au_lgxattr(dentry, &arg); } -ssize_t aufs_getxattr(struct dentry *dentry, const char *name, void *value, - size_t size) +ssize_t aufs_getxattr(struct dentry *dentry, struct inode *inode __maybe_unused, + const char *name, void *value, size_t size) { struct au_lgxattr arg = { .type = AU_XATTR_GET, @@ -263,8 +266,8 @@ ssize_t aufs_getxattr(struct dentry *dentry, const char *name, void *value, return au_lgxattr(dentry, &arg); } -int aufs_setxattr(struct dentry *dentry, const char *name, const void *value, - size_t size, int flags) +int aufs_setxattr(struct dentry *dentry, struct inode *inode, const char *name, + const void *value, size_t size, int flags) { struct au_srxattr arg = { .type = AU_XATTR_SET, @@ -276,7 +279,7 @@ int aufs_setxattr(struct dentry *dentry, const char *name, const void *value, }, }; - return au_srxattr(dentry, &arg); + return au_srxattr(dentry, inode, &arg); } int aufs_removexattr(struct dentry *dentry, const char *name) @@ -288,7 +291,7 @@ int aufs_removexattr(struct dentry *dentry, const char *name) }, }; - return au_srxattr(dentry, &arg); + return au_srxattr(dentry, d_inode(dentry), &arg); } /* ---------------------------------------------------------------------- */ diff --git a/fs/aufs/xino.c b/fs/aufs/xino.c index 4d1d5d439..2a411f9c7 100644 --- a/fs/aufs/xino.c +++ b/fs/aufs/xino.c @@ -325,7 +325,7 @@ int au_xino_trunc(struct super_block *sb, aufs_bindex_t bindex) AuErr1("statfs err %d, ignored\n", err); out_st: - kfree(st); + au_delayed_kfree(st); out: return err; } @@ -360,7 +360,7 @@ static void xino_do_trunc(void *_args) au_br_put(br); si_write_unlock(sb); au_nwt_done(&au_sbi(sb)->si_nowait); - kfree(args); + au_delayed_kfree(args); } static int xino_trunc_test(struct super_block *sb, struct au_branch *br) @@ -402,7 +402,7 @@ static void xino_try_trunc(struct super_block *sb, struct au_branch *br) args = kmalloc(sizeof(*args), GFP_NOFS); if (unlikely(!args)) { AuErr1("no memory\n"); - goto out_args; + goto out; } au_br_get(br); @@ -414,9 +414,8 @@ static void xino_try_trunc(struct super_block *sb, struct au_branch *br) pr_err("wkq %d\n", wkq_err); au_br_put(br); + au_delayed_kfree(args); -out_args: - kfree(args); out: atomic_dec(&br->br_xino_running); } @@ -939,7 +938,7 @@ static int xib_restore(struct super_block *sb) (sb, au_sbr(sb, bindex)->br_xino.xi_file, page); else AuDbg("b%d\n", bindex); - free_page((unsigned long)page); + au_delayed_free_page((unsigned long)page); out: return err; @@ -1016,7 +1015,8 @@ static void xino_clear_xib(struct super_block *sb) if (sbinfo->si_xib) fput(sbinfo->si_xib); sbinfo->si_xib = NULL; - free_page((unsigned long)sbinfo->si_xib_buf); + if (sbinfo->si_xib_buf) + au_delayed_free_page((unsigned long)sbinfo->si_xib_buf); sbinfo->si_xib_buf = NULL; } @@ -1059,7 +1059,8 @@ static int au_xino_set_xib(struct super_block *sb, struct file *base) goto out; /* success */ out_free: - free_page((unsigned long)sbinfo->si_xib_buf); + if (sbinfo->si_xib_buf) + au_delayed_free_page((unsigned long)sbinfo->si_xib_buf); sbinfo->si_xib_buf = NULL; if (err >= 0) err = -EIO; @@ -1152,7 +1153,7 @@ out_pair: fput(p->new); else break; - kfree(fpair); + au_delayed_kfree(fpair); out: return err; } @@ -1263,7 +1264,7 @@ struct file *au_xino_def(struct super_block *sb) if (!IS_ERR(file)) au_xino_brid_set(sb, br->br_id); } - free_page((unsigned long)page); + au_delayed_free_page((unsigned long)page); } else { file = au_xino_create(sb, AUFS_XINO_DEFPATH, /*silent*/0); if (IS_ERR(file)) |