diff options
Diffstat (limited to 'include/linux/memcontrol.h')
| -rw-r--r-- | include/linux/memcontrol.h | 609 |
1 files changed, 609 insertions, 0 deletions
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h new file mode 100644 index 000000000..6c8918114 --- /dev/null +++ b/include/linux/memcontrol.h @@ -0,0 +1,609 @@ +/* memcontrol.h - Memory Controller + * + * Copyright IBM Corporation, 2007 + * Author Balbir Singh <balbir@linux.vnet.ibm.com> + * + * Copyright 2007 OpenVZ SWsoft Inc + * Author: Pavel Emelianov <xemul@openvz.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _LINUX_MEMCONTROL_H +#define _LINUX_MEMCONTROL_H +#include <linux/cgroup.h> +#include <linux/vm_event_item.h> +#include <linux/hardirq.h> +#include <linux/jump_label.h> + +struct mem_cgroup; +struct page; +struct mm_struct; +struct kmem_cache; + +/* + * The corresponding mem_cgroup_stat_names is defined in mm/memcontrol.c, + * These two lists should keep in accord with each other. + */ +enum mem_cgroup_stat_index { + /* + * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. + */ + MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ + MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */ + MEM_CGROUP_STAT_RSS_HUGE, /* # of pages charged as anon huge */ + MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */ + MEM_CGROUP_STAT_WRITEBACK, /* # of pages under writeback */ + MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */ + MEM_CGROUP_STAT_NSTATS, +}; + +struct mem_cgroup_reclaim_cookie { + struct zone *zone; + int priority; + unsigned int generation; +}; + +enum mem_cgroup_events_index { + MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */ + MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */ + MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */ + MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */ + MEM_CGROUP_EVENTS_NSTATS, + /* default hierarchy events */ + MEMCG_LOW = MEM_CGROUP_EVENTS_NSTATS, + MEMCG_HIGH, + MEMCG_MAX, + MEMCG_OOM, + MEMCG_NR_EVENTS, +}; + +#ifdef CONFIG_MEMCG +void mem_cgroup_events(struct mem_cgroup *memcg, + enum mem_cgroup_events_index idx, + unsigned int nr); + +bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg); + +int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, + gfp_t gfp_mask, struct mem_cgroup **memcgp); +void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, + bool lrucare); +void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg); +void mem_cgroup_uncharge(struct page *page); +void mem_cgroup_uncharge_list(struct list_head *page_list); + +void mem_cgroup_migrate(struct page *oldpage, struct page *newpage, + bool lrucare); + +struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *); +struct lruvec *mem_cgroup_page_lruvec(struct page *, struct zone *); + +bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, + struct mem_cgroup *root); +bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg); + +extern struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page); +extern struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p); + +extern struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg); +extern struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css); + +static inline bool mm_match_cgroup(struct mm_struct *mm, + struct mem_cgroup *memcg) +{ + struct mem_cgroup *task_memcg; + bool match = false; + + rcu_read_lock(); + task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); + if (task_memcg) + match = mem_cgroup_is_descendant(task_memcg, memcg); + rcu_read_unlock(); + return match; +} + +extern struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg); + +struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *, + struct mem_cgroup *, + struct mem_cgroup_reclaim_cookie *); +void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); + +/* + * For memory reclaim. + */ +int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec); +bool mem_cgroup_lruvec_online(struct lruvec *lruvec); +int mem_cgroup_select_victim_node(struct mem_cgroup *memcg); +unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list); +void mem_cgroup_update_lru_size(struct lruvec *, enum lru_list, int); +extern void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, + struct task_struct *p); + +static inline void mem_cgroup_oom_enable(void) +{ + WARN_ON(current->memcg_oom.may_oom); + current->memcg_oom.may_oom = 1; +} + +static inline void mem_cgroup_oom_disable(void) +{ + WARN_ON(!current->memcg_oom.may_oom); + current->memcg_oom.may_oom = 0; +} + +static inline bool task_in_memcg_oom(struct task_struct *p) +{ + return p->memcg_oom.memcg; +} + +bool mem_cgroup_oom_synchronize(bool wait); + +#ifdef CONFIG_MEMCG_SWAP +extern int do_swap_account; +#endif + +static inline bool mem_cgroup_disabled(void) +{ + if (memory_cgrp_subsys.disabled) + return true; + return false; +} + +struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page); +void mem_cgroup_update_page_stat(struct mem_cgroup *memcg, + enum mem_cgroup_stat_index idx, int val); +void mem_cgroup_end_page_stat(struct mem_cgroup *memcg); + +static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg, + enum mem_cgroup_stat_index idx) +{ + mem_cgroup_update_page_stat(memcg, idx, 1); +} + +static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg, + enum mem_cgroup_stat_index idx) +{ + mem_cgroup_update_page_stat(memcg, idx, -1); +} + +unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, + gfp_t gfp_mask, + unsigned long *total_scanned); + +void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx); +static inline void mem_cgroup_count_vm_event(struct mm_struct *mm, + enum vm_event_item idx) +{ + if (mem_cgroup_disabled()) + return; + __mem_cgroup_count_vm_event(mm, idx); +} +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +void mem_cgroup_split_huge_fixup(struct page *head); +#endif + +#else /* CONFIG_MEMCG */ +struct mem_cgroup; + +static inline void mem_cgroup_events(struct mem_cgroup *memcg, + enum mem_cgroup_events_index idx, + unsigned int nr) +{ +} + +static inline bool mem_cgroup_low(struct mem_cgroup *root, + struct mem_cgroup *memcg) +{ + return false; +} + +static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm, + gfp_t gfp_mask, + struct mem_cgroup **memcgp) +{ + *memcgp = NULL; + return 0; +} + +static inline void mem_cgroup_commit_charge(struct page *page, + struct mem_cgroup *memcg, + bool lrucare) +{ +} + +static inline void mem_cgroup_cancel_charge(struct page *page, + struct mem_cgroup *memcg) +{ +} + +static inline void mem_cgroup_uncharge(struct page *page) +{ +} + +static inline void mem_cgroup_uncharge_list(struct list_head *page_list) +{ +} + +static inline void mem_cgroup_migrate(struct page *oldpage, + struct page *newpage, + bool lrucare) +{ +} + +static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone, + struct mem_cgroup *memcg) +{ + return &zone->lruvec; +} + +static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page, + struct zone *zone) +{ + return &zone->lruvec; +} + +static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) +{ + return NULL; +} + +static inline bool mm_match_cgroup(struct mm_struct *mm, + struct mem_cgroup *memcg) +{ + return true; +} + +static inline bool task_in_mem_cgroup(struct task_struct *task, + const struct mem_cgroup *memcg) +{ + return true; +} + +static inline struct cgroup_subsys_state + *mem_cgroup_css(struct mem_cgroup *memcg) +{ + return NULL; +} + +static inline struct mem_cgroup * +mem_cgroup_iter(struct mem_cgroup *root, + struct mem_cgroup *prev, + struct mem_cgroup_reclaim_cookie *reclaim) +{ + return NULL; +} + +static inline void mem_cgroup_iter_break(struct mem_cgroup *root, + struct mem_cgroup *prev) +{ +} + +static inline bool mem_cgroup_disabled(void) +{ + return true; +} + +static inline int +mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec) +{ + return 1; +} + +static inline bool mem_cgroup_lruvec_online(struct lruvec *lruvec) +{ + return true; +} + +static inline unsigned long +mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) +{ + return 0; +} + +static inline void +mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, + int increment) +{ +} + +static inline void +mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) +{ +} + +static inline struct mem_cgroup *mem_cgroup_begin_page_stat(struct page *page) +{ + return NULL; +} + +static inline void mem_cgroup_end_page_stat(struct mem_cgroup *memcg) +{ +} + +static inline void mem_cgroup_oom_enable(void) +{ +} + +static inline void mem_cgroup_oom_disable(void) +{ +} + +static inline bool task_in_memcg_oom(struct task_struct *p) +{ + return false; +} + +static inline bool mem_cgroup_oom_synchronize(bool wait) +{ + return false; +} + +static inline void mem_cgroup_inc_page_stat(struct mem_cgroup *memcg, + enum mem_cgroup_stat_index idx) +{ +} + +static inline void mem_cgroup_dec_page_stat(struct mem_cgroup *memcg, + enum mem_cgroup_stat_index idx) +{ +} + +static inline +unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, + gfp_t gfp_mask, + unsigned long *total_scanned) +{ + return 0; +} + +static inline void mem_cgroup_split_huge_fixup(struct page *head) +{ +} + +static inline +void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx) +{ +} +#endif /* CONFIG_MEMCG */ + +enum { + UNDER_LIMIT, + SOFT_LIMIT, + OVER_LIMIT, +}; + +struct sock; +#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM) +void sock_update_memcg(struct sock *sk); +void sock_release_memcg(struct sock *sk); +#else +static inline void sock_update_memcg(struct sock *sk) +{ +} +static inline void sock_release_memcg(struct sock *sk) +{ +} +#endif /* CONFIG_INET && CONFIG_MEMCG_KMEM */ + +#ifdef CONFIG_MEMCG_KMEM +extern struct static_key memcg_kmem_enabled_key; + +extern int memcg_nr_cache_ids; +extern void memcg_get_cache_ids(void); +extern void memcg_put_cache_ids(void); + +/* + * Helper macro to loop through all memcg-specific caches. Callers must still + * check if the cache is valid (it is either valid or NULL). + * the slab_mutex must be held when looping through those caches + */ +#define for_each_memcg_cache_index(_idx) \ + for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++) + +static inline bool memcg_kmem_enabled(void) +{ + return static_key_false(&memcg_kmem_enabled_key); +} + +bool memcg_kmem_is_active(struct mem_cgroup *memcg); + +/* + * In general, we'll do everything in our power to not incur in any overhead + * for non-memcg users for the kmem functions. Not even a function call, if we + * can avoid it. + * + * Therefore, we'll inline all those functions so that in the best case, we'll + * see that kmemcg is off for everybody and proceed quickly. If it is on, + * we'll still do most of the flag checking inline. We check a lot of + * conditions, but because they are pretty simple, they are expected to be + * fast. + */ +bool __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, + int order); +void __memcg_kmem_commit_charge(struct page *page, + struct mem_cgroup *memcg, int order); +void __memcg_kmem_uncharge_pages(struct page *page, int order); + +int memcg_cache_id(struct mem_cgroup *memcg); + +struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep); +void __memcg_kmem_put_cache(struct kmem_cache *cachep); + +struct mem_cgroup *__mem_cgroup_from_kmem(void *ptr); + +int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, + unsigned long nr_pages); +void memcg_uncharge_kmem(struct mem_cgroup *memcg, unsigned long nr_pages); + +/** + * memcg_kmem_newpage_charge: verify if a new kmem allocation is allowed. + * @gfp: the gfp allocation flags. + * @memcg: a pointer to the memcg this was charged against. + * @order: allocation order. + * + * returns true if the memcg where the current task belongs can hold this + * allocation. + * + * We return true automatically if this allocation is not to be accounted to + * any memcg. + */ +static inline bool +memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, int order) +{ + if (!memcg_kmem_enabled()) + return true; + + if (gfp & __GFP_NOACCOUNT) + return true; + /* + * __GFP_NOFAIL allocations will move on even if charging is not + * possible. Therefore we don't even try, and have this allocation + * unaccounted. We could in theory charge it forcibly, but we hope + * those allocations are rare, and won't be worth the trouble. + */ + if (gfp & __GFP_NOFAIL) + return true; + if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD)) + return true; + + /* If the test is dying, just let it go. */ + if (unlikely(fatal_signal_pending(current))) + return true; + + return __memcg_kmem_newpage_charge(gfp, memcg, order); +} + +/** + * memcg_kmem_uncharge_pages: uncharge pages from memcg + * @page: pointer to struct page being freed + * @order: allocation order. + */ +static inline void +memcg_kmem_uncharge_pages(struct page *page, int order) +{ + if (memcg_kmem_enabled()) + __memcg_kmem_uncharge_pages(page, order); +} + +/** + * memcg_kmem_commit_charge: embeds correct memcg in a page + * @page: pointer to struct page recently allocated + * @memcg: the memcg structure we charged against + * @order: allocation order. + * + * Needs to be called after memcg_kmem_newpage_charge, regardless of success or + * failure of the allocation. if @page is NULL, this function will revert the + * charges. Otherwise, it will commit @page to @memcg. + */ +static inline void +memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, int order) +{ + if (memcg_kmem_enabled() && memcg) + __memcg_kmem_commit_charge(page, memcg, order); +} + +/** + * memcg_kmem_get_cache: selects the correct per-memcg cache for allocation + * @cachep: the original global kmem cache + * @gfp: allocation flags. + * + * All memory allocated from a per-memcg cache is charged to the owner memcg. + */ +static __always_inline struct kmem_cache * +memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) +{ + if (!memcg_kmem_enabled()) + return cachep; + if (gfp & __GFP_NOACCOUNT) + return cachep; + if (gfp & __GFP_NOFAIL) + return cachep; + if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD)) + return cachep; + if (unlikely(fatal_signal_pending(current))) + return cachep; + + return __memcg_kmem_get_cache(cachep); +} + +static __always_inline void memcg_kmem_put_cache(struct kmem_cache *cachep) +{ + if (memcg_kmem_enabled()) + __memcg_kmem_put_cache(cachep); +} + +static __always_inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr) +{ + if (!memcg_kmem_enabled()) + return NULL; + return __mem_cgroup_from_kmem(ptr); +} +#else +#define for_each_memcg_cache_index(_idx) \ + for (; NULL; ) + +static inline bool memcg_kmem_enabled(void) +{ + return false; +} + +static inline bool memcg_kmem_is_active(struct mem_cgroup *memcg) +{ + return false; +} + +static inline bool +memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, int order) +{ + return true; +} + +static inline void memcg_kmem_uncharge_pages(struct page *page, int order) +{ +} + +static inline void +memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, int order) +{ +} + +static inline int memcg_cache_id(struct mem_cgroup *memcg) +{ + return -1; +} + +static inline void memcg_get_cache_ids(void) +{ +} + +static inline void memcg_put_cache_ids(void) +{ +} + +static inline struct kmem_cache * +memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) +{ + return cachep; +} + +static inline void memcg_kmem_put_cache(struct kmem_cache *cachep) +{ +} + +static inline struct mem_cgroup *mem_cgroup_from_kmem(void *ptr) +{ + return NULL; +} +#endif /* CONFIG_MEMCG_KMEM */ +#endif /* _LINUX_MEMCONTROL_H */ + |