summaryrefslogtreecommitdiff
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/kasan/kasan.c4
-rw-r--r--mm/kasan/quarantine.c9
-rw-r--r--mm/memcontrol.c31
-rw-r--r--mm/page_io.c3
-rw-r--r--mm/swapfile.c1
-rw-r--r--mm/vmscan.c19
6 files changed, 34 insertions, 33 deletions
diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c
index 6845f9294..50e92c585 100644
--- a/mm/kasan/kasan.c
+++ b/mm/kasan/kasan.c
@@ -562,7 +562,7 @@ void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size,
unsigned long redzone_start;
unsigned long redzone_end;
- if (flags & __GFP_RECLAIM)
+ if (gfpflags_allow_blocking(flags))
quarantine_reduce();
if (unlikely(object == NULL))
@@ -595,7 +595,7 @@ void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
unsigned long redzone_start;
unsigned long redzone_end;
- if (flags & __GFP_RECLAIM)
+ if (gfpflags_allow_blocking(flags))
quarantine_reduce();
if (unlikely(ptr == NULL))
diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c
index 65793f150..a04887c62 100644
--- a/mm/kasan/quarantine.c
+++ b/mm/kasan/quarantine.c
@@ -196,7 +196,7 @@ void quarantine_put(struct kasan_free_meta *info, struct kmem_cache *cache)
void quarantine_reduce(void)
{
- size_t new_quarantine_size;
+ size_t new_quarantine_size, percpu_quarantines;
unsigned long flags;
struct qlist_head to_free = QLIST_INIT;
size_t size_to_free = 0;
@@ -214,7 +214,12 @@ void quarantine_reduce(void)
*/
new_quarantine_size = (READ_ONCE(totalram_pages) << PAGE_SHIFT) /
QUARANTINE_FRACTION;
- new_quarantine_size -= QUARANTINE_PERCPU_SIZE * num_online_cpus();
+ percpu_quarantines = QUARANTINE_PERCPU_SIZE * num_online_cpus();
+ if (WARN_ONCE(new_quarantine_size < percpu_quarantines,
+ "Too little memory, disabling global KASAN quarantine.\n"))
+ new_quarantine_size = 0;
+ else
+ new_quarantine_size -= percpu_quarantines;
WRITE_ONCE(quarantine_size, new_quarantine_size);
last = global_quarantine.head;
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index ca847d96a..fcb5b8cb4 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1797,17 +1797,22 @@ static DEFINE_MUTEX(percpu_charge_mutex);
static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
{
struct memcg_stock_pcp *stock;
+ unsigned long flags;
bool ret = false;
if (nr_pages > CHARGE_BATCH)
return ret;
- stock = &get_cpu_var(memcg_stock);
+ local_irq_save(flags);
+
+ stock = this_cpu_ptr(&memcg_stock);
if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
stock->nr_pages -= nr_pages;
ret = true;
}
- put_cpu_var(memcg_stock);
+
+ local_irq_restore(flags);
+
return ret;
}
@@ -1828,15 +1833,18 @@ static void drain_stock(struct memcg_stock_pcp *stock)
stock->cached = NULL;
}
-/*
- * This must be called under preempt disabled or must be called by
- * a thread which is pinned to local cpu.
- */
static void drain_local_stock(struct work_struct *dummy)
{
- struct memcg_stock_pcp *stock = this_cpu_ptr(&memcg_stock);
+ struct memcg_stock_pcp *stock;
+ unsigned long flags;
+
+ local_irq_save(flags);
+
+ stock = this_cpu_ptr(&memcg_stock);
drain_stock(stock);
clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
+
+ local_irq_restore(flags);
}
/*
@@ -1845,14 +1853,19 @@ static void drain_local_stock(struct work_struct *dummy)
*/
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
{
- struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
+ struct memcg_stock_pcp *stock;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ stock = this_cpu_ptr(&memcg_stock);
if (stock->cached != memcg) { /* reset if necessary */
drain_stock(stock);
stock->cached = memcg;
}
stock->nr_pages += nr_pages;
- put_cpu_var(memcg_stock);
+
+ local_irq_restore(flags);
}
/*
diff --git a/mm/page_io.c b/mm/page_io.c
index 242dba075..dc1af1e63 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -262,6 +262,7 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc,
int ret, rw = WRITE;
struct swap_info_struct *sis = page_swap_info(page);
+ BUG_ON(!PageSwapCache(page));
if (sis->flags & SWP_FILE) {
struct kiocb kiocb;
struct file *swap_file = sis->swap_file;
@@ -333,6 +334,7 @@ int swap_readpage(struct page *page)
int ret = 0;
struct swap_info_struct *sis = page_swap_info(page);
+ BUG_ON(!PageSwapCache(page));
VM_BUG_ON_PAGE(!PageLocked(page), page);
VM_BUG_ON_PAGE(PageUptodate(page), page);
if (frontswap_load(page) == 0) {
@@ -381,6 +383,7 @@ int swap_set_page_dirty(struct page *page)
if (sis->flags & SWP_FILE) {
struct address_space *mapping = sis->swap_file->f_mapping;
+ BUG_ON(!PageSwapCache(page));
return mapping->a_ops->set_page_dirty(page);
} else {
return __set_page_dirty_no_writeback(page);
diff --git a/mm/swapfile.c b/mm/swapfile.c
index dddefaeaa..6cff7ba52 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -2778,7 +2778,6 @@ int swapcache_prepare(swp_entry_t entry)
struct swap_info_struct *page_swap_info(struct page *page)
{
swp_entry_t swap = { .val = page_private(page) };
- BUG_ON(!PageSwapCache(page));
return swap_info[swp_type(swap)];
}
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 00d20d0f0..d12a0a63d 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2150,23 +2150,6 @@ out:
}
}
-#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
-static void init_tlb_ubc(void)
-{
- /*
- * This deliberately does not clear the cpumask as it's expensive
- * and unnecessary. If there happens to be data in there then the
- * first SWAP_CLUSTER_MAX pages will send an unnecessary IPI and
- * then will be cleared.
- */
- current->tlb_ubc.flush_required = false;
-}
-#else
-static inline void init_tlb_ubc(void)
-{
-}
-#endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
-
/*
* This is a basic per-zone page freer. Used by both kswapd and direct reclaim.
*/
@@ -2202,8 +2185,6 @@ static void shrink_zone_memcg(struct zone *zone, struct mem_cgroup *memcg,
scan_adjusted = (global_reclaim(sc) && !current_is_kswapd() &&
sc->priority == DEF_PRIORITY);
- init_tlb_ubc();
-
blk_start_plug(&plug);
while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
nr[LRU_INACTIVE_FILE]) {