diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/Kconfig | 67 | ||||
| -rw-r--r-- | mm/Makefile | 37 | ||||
| -rw-r--r-- | mm/ashmem.c | 749 | ||||
| -rw-r--r-- | mm/backing-dev.c | 143 | ||||
| -rw-r--r-- | mm/bootmem.c | 2 | ||||
| -rw-r--r-- | mm/bounce.c | 13 | ||||
| -rw-r--r-- | mm/cma-best-fit.c | 408 | ||||
| -rw-r--r-- | mm/cma.c | 1431 | ||||
| -rw-r--r-- | mm/compaction-cma.c | 860 | ||||
| -rw-r--r-- | mm/compaction.c | 6 | ||||
| -rw-r--r-- | mm/debug-pagealloc.c | 56 | ||||
| -rw-r--r-- | mm/dmapool.c | 5 | ||||
| -rw-r--r-- | mm/failslab.c | 39 | ||||
| -rw-r--r-- | mm/filemap.c | 150 | ||||
| -rw-r--r-- | mm/filemap_xip.c | 2 | ||||
| -rw-r--r-- | mm/fremap.c | 1 | ||||
| -rw-r--r-- | mm/highmem.c | 21 | ||||
| -rw-r--r-- | mm/huge_memory.c | 87 | ||||
| -rw-r--r-- | mm/hugetlb.c | 358 | ||||
| -rw-r--r-- | mm/init-mm.c | 2 | ||||
| -rw-r--r-- | mm/internal.h | 37 | ||||
| -rw-r--r-- | mm/kmemleak.c | 22 | ||||
| -rw-r--r-- | mm/ksm.c | 5 | ||||
| -rw-r--r-- | mm/maccess.c | 2 | ||||
| -rw-r--r-- | mm/madvise.c | 8 | ||||
| -rw-r--r-- | mm/memblock.c | 19 | ||||
| -rw-r--r-- | mm/memcontrol.c | 1391 | ||||
| -rw-r--r-- | mm/memory-failure.c | 149 | ||||
| -rw-r--r-- | mm/memory.c | 270 | ||||
| -rw-r--r-- | mm/memory_hotplug.c | 89 | ||||
| -rw-r--r-- | mm/mempolicy.c | 116 | ||||
| -rw-r--r-- | mm/mempool.c | 2 | ||||
| -rw-r--r-- | mm/migrate-cma.c | 1627 | ||||
| -rw-r--r-- | mm/migrate.c | 9 | ||||
| -rw-r--r-- | mm/mincore.c | 11 | ||||
| -rw-r--r-- | mm/mlock.c | 17 | ||||
| -rw-r--r-- | mm/mm_init.c | 2 | ||||
| -rw-r--r-- | mm/mmap.c | 84 | ||||
| -rw-r--r-- | mm/mmu_context.c | 2 | ||||
| -rw-r--r-- | mm/mmu_notifier.c | 6 | ||||
| -rw-r--r-- | mm/mmzone.c | 1 | ||||
| -rw-r--r-- | mm/mremap.c | 42 | ||||
| -rw-r--r-- | mm/nobootmem.c | 2 | ||||
| -rw-r--r-- | mm/nommu.c | 43 | ||||
| -rw-r--r-- | mm/oom_kill.c | 59 | ||||
| -rw-r--r-- | mm/page-writeback.c | 1005 | ||||
| -rw-r--r-- | mm/page_alloc-cma.c | 6442 | ||||
| -rw-r--r-- | mm/page_alloc.c | 307 | ||||
| -rw-r--r-- | mm/page_cgroup.c | 23 | ||||
| -rw-r--r-- | mm/page_isolation-cma.c | 147 | ||||
| -rw-r--r-- | mm/pagewalk.c | 76 | ||||
| -rw-r--r-- | mm/percpu-vm.c | 27 | ||||
| -rw-r--r-- | mm/percpu.c | 30 | ||||
| -rw-r--r-- | mm/quicklist.c | 1 | ||||
| -rw-r--r-- | mm/readahead.c | 2 | ||||
| -rw-r--r-- | mm/rmap.c | 93 | ||||
| -rw-r--r-- | mm/shmem.c | 1996 | ||||
| -rw-r--r-- | mm/slab.c | 144 | ||||
| -rw-r--r-- | mm/slob.c | 10 | ||||
| -rw-r--r-- | mm/slub.c | 1313 | ||||
| -rw-r--r-- | mm/sparse-vmemmap.c | 1 | ||||
| -rw-r--r-- | mm/sparse.c | 7 | ||||
| -rw-r--r-- | mm/swap.c | 101 | ||||
| -rw-r--r-- | mm/swap_state.c | 1 | ||||
| -rw-r--r-- | mm/swapfile.c | 210 | ||||
| -rw-r--r-- | mm/thrash.c | 19 | ||||
| -rw-r--r-- | mm/truncate.c | 228 | ||||
| -rw-r--r-- | mm/util.c | 2 | ||||
| -rw-r--r-- | mm/vmalloc.c | 94 | ||||
| -rw-r--r-- | mm/vmscan.c | 500 | ||||
| -rw-r--r-- | mm/vmstat.c | 15 |
71 files changed, 16008 insertions, 5238 deletions
@@ -131,9 +131,6 @@ config SPARSEMEM_VMEMMAP config HAVE_MEMBLOCK boolean -config NO_BOOTMEM - boolean - # eventually, we can have this option just 'select SPARSEMEM' config MEMORY_HOTPLUG bool "Allow for memory hot-add" @@ -192,7 +189,7 @@ config COMPACTION config MIGRATION bool "Page migration" def_bool y - depends on NUMA || ARCH_ENABLE_MEMORY_HOTREMOVE || COMPACTION + depends on NUMA || ARCH_ENABLE_MEMORY_HOTREMOVE || COMPACTION || DMA_CMA help Allows the migration of the physical location of pages of processes while the virtual addresses are not changed. This is useful in @@ -359,7 +356,7 @@ config CLEANCACHE for clean pages that the kernel's pageframe replacement algorithm (PFRA) would like to keep around, but can't since there isn't enough memory. So when the PFRA "evicts" a page, it first attempts to use - cleancache code to put the data contained in that page into + cleancacne code to put the data contained in that page into "transcendent memory", memory that is not directly accessible or addressable by the kernel and is of unknown and possibly time-varying size. And when a cleancache-enabled @@ -373,3 +370,63 @@ config CLEANCACHE in a negligible performance hit. If unsure, say Y to enable cleancache + +config CMA + bool "Contiguous Memory Allocator framework" + # Currently there is only one allocator so force it on + select CMA_BEST_FIT + help + This enables the Contiguous Memory Allocator framework which + allows drivers to allocate big physically-contiguous blocks of + memory for use with hardware components that do not support I/O + map nor scatter-gather. + + If you select this option you will also have to select at least + one allocator algorithm below. + + To make use of CMA you need to specify the regions and + driver->region mapping on command line when booting the kernel. + +config CMA_DEVELOPEMENT + bool "Include CMA developement features" + depends on CMA + help + This lets you enable some developement features of the CMA + freamework. + +config CMA_DEBUG + bool "CMA debug messages" + depends on CMA_DEVELOPEMENT + help + Enable debug messages in CMA code. + +config CMA_SYSFS + bool "CMA SysFS interface support" + depends on CMA_DEVELOPEMENT + help + Enable support for SysFS interface. + +config CMA_CMDLINE + bool "CMA command line parameters support" + depends on CMA_DEVELOPEMENT + help + Enable support for cma, cma.map and cma.asterisk command line + parameters. + +config CMA_BEST_FIT + bool "CMA best-fit allocator" + depends on CMA + help + This is a best-fit algorithm running in O(n log n) time where + n is the number of existing holes (which is never greater then + the number of allocated regions and usually much smaller). It + allocates area from the smallest hole that is big enough for + allocation in question. + +config DEBUG_VMALLOC + bool "Enable VMALLOC debugging support" + help + This enables the inclusion of a few debugging parameters, which + are the PID and the name of the task requesting vmalloc. + + Disabling these can result in savings in code size. diff --git a/mm/Makefile b/mm/Makefile index 50ec00e..9b994ce 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -5,15 +5,25 @@ mmu-y := nommu.o mmu-$(CONFIG_MMU) := fremap.o highmem.o madvise.o memory.o mincore.o \ mlock.o mmap.o mprotect.o mremap.o msync.o rmap.o \ - vmalloc.o pagewalk.o pgtable-generic.o \ - process_vm_access.o + vmalloc.o pagewalk.o pgtable-generic.o obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ - maccess.o page_alloc.o page-writeback.o \ + maccess.o page-writeback.o \ readahead.o swap.o truncate.o vmscan.o shmem.o \ prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \ - page_isolation.o mm_init.o mmu_context.o percpu.o \ + mm_init.o mmu_context.o percpu.o \ $(mmu-y) + +ifdef CONFIG_SLP + obj-y += page_alloc-slp.o page_isolation-slp.o +else +ifdef CONFIG_DMA_CMA + obj-y += page_alloc-cma.o page_isolation-cma.o +else + obj-y += page_alloc.o page_isolation.o +endif +endif + obj-y += init-mm.o ifdef CONFIG_NO_BOOTMEM @@ -31,8 +41,17 @@ obj-$(CONFIG_HUGETLBFS) += hugetlb.o obj-$(CONFIG_NUMA) += mempolicy.o obj-$(CONFIG_SPARSEMEM) += sparse.o obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o +obj-$(CONFIG_ASHMEM) += ashmem.o obj-$(CONFIG_SLOB) += slob.o +ifdef CONFIG_SLP +obj-$(CONFIG_COMPACTION) += compaction-slp.o +else +ifdef CONFIG_DMA_CMA +obj-$(CONFIG_COMPACTION) += compaction-cma.o +else obj-$(CONFIG_COMPACTION) += compaction.o +endif +endif obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o obj-$(CONFIG_KSM) += ksm.o obj-$(CONFIG_PAGE_POISONING) += debug-pagealloc.o @@ -40,9 +59,17 @@ obj-$(CONFIG_SLAB) += slab.o obj-$(CONFIG_SLUB) += slub.o obj-$(CONFIG_KMEMCHECK) += kmemcheck.o obj-$(CONFIG_FAILSLAB) += failslab.o +ifdef CONFIG_SLP +obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug-slp.o +else obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o +endif obj-$(CONFIG_FS_XIP) += filemap_xip.o +ifdef CONFIG_DMA_CMA +obj-$(CONFIG_MIGRATION) += migrate-cma.o +else obj-$(CONFIG_MIGRATION) += migrate.o +endif obj-$(CONFIG_QUICKLIST) += quicklist.o obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o obj-$(CONFIG_CGROUP_MEM_RES_CTLR) += memcontrol.o page_cgroup.o @@ -51,3 +78,5 @@ obj-$(CONFIG_HWPOISON_INJECT) += hwpoison-inject.o obj-$(CONFIG_DEBUG_KMEMLEAK) += kmemleak.o obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak-test.o obj-$(CONFIG_CLEANCACHE) += cleancache.o +obj-$(CONFIG_CMA) += cma.o +obj-$(CONFIG_CMA_BEST_FIT) += cma-best-fit.o diff --git a/mm/ashmem.c b/mm/ashmem.c new file mode 100644 index 0000000..e98f358 --- /dev/null +++ b/mm/ashmem.c @@ -0,0 +1,749 @@ +/* mm/ashmem.c +** +** Anonymous Shared Memory Subsystem, ashmem +** +** Copyright (C) 2008 Google, Inc. +** +** Robert Love <rlove@google.com> +** +** This software is licensed under the terms of the GNU General Public +** License version 2, as published by the Free Software Foundation, and +** may be copied, distributed, and modified under those terms. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +*/ + +#include <linux/module.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/miscdevice.h> +#include <linux/security.h> +#include <linux/mm.h> +#include <linux/mman.h> +#include <linux/uaccess.h> +#include <linux/personality.h> +#include <linux/bitops.h> +#include <linux/mutex.h> +#include <linux/shmem_fs.h> +#include <linux/ashmem.h> + +#define ASHMEM_NAME_PREFIX "dev/ashmem/" +#define ASHMEM_NAME_PREFIX_LEN (sizeof(ASHMEM_NAME_PREFIX) - 1) +#define ASHMEM_FULL_NAME_LEN (ASHMEM_NAME_LEN + ASHMEM_NAME_PREFIX_LEN) + +/* + * ashmem_area - anonymous shared memory area + * Lifecycle: From our parent file's open() until its release() + * Locking: Protected by `ashmem_mutex' + * Big Note: Mappings do NOT pin this structure; it dies on close() + */ +struct ashmem_area { + char name[ASHMEM_FULL_NAME_LEN];/* optional name for /proc/pid/maps */ + struct list_head unpinned_list; /* list of all ashmem areas */ + struct file *file; /* the shmem-based backing file */ + size_t size; /* size of the mapping, in bytes */ + unsigned long prot_mask; /* allowed prot bits, as vm_flags */ +}; + +/* + * ashmem_range - represents an interval of unpinned (evictable) pages + * Lifecycle: From unpin to pin + * Locking: Protected by `ashmem_mutex' + */ +struct ashmem_range { + struct list_head lru; /* entry in LRU list */ + struct list_head unpinned; /* entry in its area's unpinned list */ + struct ashmem_area *asma; /* associated area */ + size_t pgstart; /* starting page, inclusive */ + size_t pgend; /* ending page, inclusive */ + unsigned int purged; /* ASHMEM_NOT or ASHMEM_WAS_PURGED */ +}; + +/* LRU list of unpinned pages, protected by ashmem_mutex */ +static LIST_HEAD(ashmem_lru_list); + +/* Count of pages on our LRU list, protected by ashmem_mutex */ +static unsigned long lru_count; + +/* + * ashmem_mutex - protects the list of and each individual ashmem_area + * + * Lock Ordering: ashmex_mutex -> i_mutex -> i_alloc_sem + */ +static DEFINE_MUTEX(ashmem_mutex); + +static struct kmem_cache *ashmem_area_cachep __read_mostly; +static struct kmem_cache *ashmem_range_cachep __read_mostly; + +#define range_size(range) \ + ((range)->pgend - (range)->pgstart + 1) + +#define range_on_lru(range) \ + ((range)->purged == ASHMEM_NOT_PURGED) + +#define page_range_subsumes_range(range, start, end) \ + (((range)->pgstart >= (start)) && ((range)->pgend <= (end))) + +#define page_range_subsumed_by_range(range, start, end) \ + (((range)->pgstart <= (start)) && ((range)->pgend >= (end))) + +#define page_in_range(range, page) \ + (((range)->pgstart <= (page)) && ((range)->pgend >= (page))) + +#define page_range_in_range(range, start, end) \ + (page_in_range(range, start) || page_in_range(range, end) || \ + page_range_subsumes_range(range, start, end)) + +#define range_before_page(range, page) \ + ((range)->pgend < (page)) + +#define PROT_MASK (PROT_EXEC | PROT_READ | PROT_WRITE) + +static inline void lru_add(struct ashmem_range *range) +{ + list_add_tail(&range->lru, &ashmem_lru_list); + lru_count += range_size(range); +} + +static inline void lru_del(struct ashmem_range *range) +{ + list_del(&range->lru); + lru_count -= range_size(range); +} + +/* + * range_alloc - allocate and initialize a new ashmem_range structure + * + * 'asma' - associated ashmem_area + * 'prev_range' - the previous ashmem_range in the sorted asma->unpinned list + * 'purged' - initial purge value (ASMEM_NOT_PURGED or ASHMEM_WAS_PURGED) + * 'start' - starting page, inclusive + * 'end' - ending page, inclusive + * + * Caller must hold ashmem_mutex. + */ +static int range_alloc(struct ashmem_area *asma, + struct ashmem_range *prev_range, unsigned int purged, + size_t start, size_t end) +{ + struct ashmem_range *range; + + range = kmem_cache_zalloc(ashmem_range_cachep, GFP_KERNEL); + if (unlikely(!range)) + return -ENOMEM; + + range->asma = asma; + range->pgstart = start; + range->pgend = end; + range->purged = purged; + + list_add_tail(&range->unpinned, &prev_range->unpinned); + + if (range_on_lru(range)) + lru_add(range); + + return 0; +} + +static void range_del(struct ashmem_range *range) +{ + list_del(&range->unpinned); + if (range_on_lru(range)) + lru_del(range); + kmem_cache_free(ashmem_range_cachep, range); +} + +/* + * range_shrink - shrinks a range + * + * Caller must hold ashmem_mutex. + */ +static inline void range_shrink(struct ashmem_range *range, + size_t start, size_t end) +{ + size_t pre = range_size(range); + + range->pgstart = start; + range->pgend = end; + + if (range_on_lru(range)) + lru_count -= pre - range_size(range); +} + +static int ashmem_open(struct inode *inode, struct file *file) +{ + struct ashmem_area *asma; + int ret; + + ret = generic_file_open(inode, file); + if (unlikely(ret)) + return ret; + + asma = kmem_cache_zalloc(ashmem_area_cachep, GFP_KERNEL); + if (unlikely(!asma)) + return -ENOMEM; + + INIT_LIST_HEAD(&asma->unpinned_list); + memcpy(asma->name, ASHMEM_NAME_PREFIX, ASHMEM_NAME_PREFIX_LEN); + asma->prot_mask = PROT_MASK; + file->private_data = asma; + + return 0; +} + +static int ashmem_release(struct inode *ignored, struct file *file) +{ + struct ashmem_area *asma = file->private_data; + struct ashmem_range *range, *next; + + mutex_lock(&ashmem_mutex); + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) + range_del(range); + mutex_unlock(&ashmem_mutex); + + if (asma->file) + fput(asma->file); + kmem_cache_free(ashmem_area_cachep, asma); + + return 0; +} + +static ssize_t ashmem_read(struct file *file, char __user *buf, + size_t len, loff_t *pos) +{ + struct ashmem_area *asma = file->private_data; + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* If size is not set, or set to 0, always return EOF. */ + if (asma->size == 0) { + goto out; + } + + if (!asma->file) { + ret = -EBADF; + goto out; + } + + ret = asma->file->f_op->read(asma->file, buf, len, pos); + if (ret < 0) { + goto out; + } + + /** Update backing file pos, since f_ops->read() doesn't */ + asma->file->f_pos = *pos; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +static loff_t ashmem_llseek(struct file *file, loff_t offset, int origin) +{ + struct ashmem_area *asma = file->private_data; + int ret; + + mutex_lock(&ashmem_mutex); + + if (asma->size == 0) { + ret = -EINVAL; + goto out; + } + + if (!asma->file) { + ret = -EBADF; + goto out; + } + + ret = asma->file->f_op->llseek(asma->file, offset, origin); + if (ret < 0) { + goto out; + } + + /** Copy f_pos from backing file, since f_ops->llseek() sets it */ + file->f_pos = asma->file->f_pos; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +static inline unsigned long +calc_vm_may_flags(unsigned long prot) +{ + return _calc_vm_trans(prot, PROT_READ, VM_MAYREAD ) | + _calc_vm_trans(prot, PROT_WRITE, VM_MAYWRITE) | + _calc_vm_trans(prot, PROT_EXEC, VM_MAYEXEC); +} + +static int ashmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct ashmem_area *asma = file->private_data; + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* user needs to SET_SIZE before mapping */ + if (unlikely(!asma->size)) { + ret = -EINVAL; + goto out; + } + + /* requested protection bits must match our allowed protection mask */ + if (unlikely((vma->vm_flags & ~calc_vm_prot_bits(asma->prot_mask)) & + calc_vm_prot_bits(PROT_MASK))) { + ret = -EPERM; + goto out; + } + vma->vm_flags &= ~calc_vm_may_flags(~asma->prot_mask); + + if (!asma->file) { + char *name = ASHMEM_NAME_DEF; + struct file *vmfile; + + if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0') + name = asma->name; + + /* ... and allocate the backing shmem file */ + vmfile = shmem_file_setup(name, asma->size, vma->vm_flags); + if (unlikely(IS_ERR(vmfile))) { + ret = PTR_ERR(vmfile); + goto out; + } + asma->file = vmfile; + } + get_file(asma->file); + + if (vma->vm_flags & VM_SHARED) + shmem_set_file(vma, asma->file); + else { + if (vma->vm_file) + fput(vma->vm_file); + vma->vm_file = asma->file; + } + vma->vm_flags |= VM_CAN_NONLINEAR; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +/* + * ashmem_shrink - our cache shrinker, called from mm/vmscan.c :: shrink_slab + * + * 'nr_to_scan' is the number of objects (pages) to prune, or 0 to query how + * many objects (pages) we have in total. + * + * 'gfp_mask' is the mask of the allocation that got us into this mess. + * + * Return value is the number of objects (pages) remaining, or -1 if we cannot + * proceed without risk of deadlock (due to gfp_mask). + * + * We approximate LRU via least-recently-unpinned, jettisoning unpinned partial + * chunks of ashmem regions LRU-wise one-at-a-time until we hit 'nr_to_scan' + * pages freed. + */ +static int ashmem_shrink(struct shrinker *s, struct shrink_control *sc) +{ + struct ashmem_range *range, *next; + + /* We might recurse into filesystem code, so bail out if necessary */ + if (sc->nr_to_scan && !(sc->gfp_mask & __GFP_FS)) + return -1; + if (!sc->nr_to_scan) + return lru_count; + + if (!mutex_trylock(&ashmem_mutex)) + return lru_count; + list_for_each_entry_safe(range, next, &ashmem_lru_list, lru) { + struct inode *inode = range->asma->file->f_dentry->d_inode; + loff_t start = range->pgstart * PAGE_SIZE; + loff_t end = (range->pgend + 1) * PAGE_SIZE - 1; + + vmtruncate_range(inode, start, end); + range->purged = ASHMEM_WAS_PURGED; + lru_del(range); + + sc->nr_to_scan -= range_size(range); + if (sc->nr_to_scan <= 0) + break; + } + mutex_unlock(&ashmem_mutex); + + return lru_count; +} + +static struct shrinker ashmem_shrinker = { + .shrink = ashmem_shrink, + .seeks = DEFAULT_SEEKS * 4, +}; + +static int set_prot_mask(struct ashmem_area *asma, unsigned long prot) +{ + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* the user can only remove, not add, protection bits */ + if (unlikely((asma->prot_mask & prot) != prot)) { + ret = -EINVAL; + goto out; + } + + /* does the application expect PROT_READ to imply PROT_EXEC? */ + if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) + prot |= PROT_EXEC; + + asma->prot_mask = prot; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +static int set_name(struct ashmem_area *asma, void __user *name) +{ + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* cannot change an existing mapping's name */ + if (unlikely(asma->file)) { + ret = -EINVAL; + goto out; + } + + if (unlikely(copy_from_user(asma->name + ASHMEM_NAME_PREFIX_LEN, + name, ASHMEM_NAME_LEN))) + ret = -EFAULT; + asma->name[ASHMEM_FULL_NAME_LEN-1] = '\0'; + +out: + mutex_unlock(&ashmem_mutex); + + return ret; +} + +static int get_name(struct ashmem_area *asma, void __user *name) +{ + int ret = 0; + + mutex_lock(&ashmem_mutex); + if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0') { + size_t len; + + /* + * Copying only `len', instead of ASHMEM_NAME_LEN, bytes + * prevents us from revealing one user's stack to another. + */ + len = strlen(asma->name + ASHMEM_NAME_PREFIX_LEN) + 1; + if (unlikely(copy_to_user(name, + asma->name + ASHMEM_NAME_PREFIX_LEN, len))) + ret = -EFAULT; + } else { + if (unlikely(copy_to_user(name, ASHMEM_NAME_DEF, + sizeof(ASHMEM_NAME_DEF)))) + ret = -EFAULT; + } + mutex_unlock(&ashmem_mutex); + + return ret; +} + +/* + * ashmem_pin - pin the given ashmem region, returning whether it was + * previously purged (ASHMEM_WAS_PURGED) or not (ASHMEM_NOT_PURGED). + * + * Caller must hold ashmem_mutex. + */ +static int ashmem_pin(struct ashmem_area *asma, size_t pgstart, size_t pgend) +{ + struct ashmem_range *range, *next; + int ret = ASHMEM_NOT_PURGED; + + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) { + /* moved past last applicable page; we can short circuit */ + if (range_before_page(range, pgstart)) + break; + + /* + * The user can ask us to pin pages that span multiple ranges, + * or to pin pages that aren't even unpinned, so this is messy. + * + * Four cases: + * 1. The requested range subsumes an existing range, so we + * just remove the entire matching range. + * 2. The requested range overlaps the start of an existing + * range, so we just update that range. + * 3. The requested range overlaps the end of an existing + * range, so we just update that range. + * 4. The requested range punches a hole in an existing range, + * so we have to update one side of the range and then + * create a new range for the other side. + */ + if (page_range_in_range(range, pgstart, pgend)) { + ret |= range->purged; + + /* Case #1: Easy. Just nuke the whole thing. */ + if (page_range_subsumes_range(range, pgstart, pgend)) { + range_del(range); + continue; + } + + /* Case #2: We overlap from the start, so adjust it */ + if (range->pgstart >= pgstart) { + range_shrink(range, pgend + 1, range->pgend); + continue; + } + + /* Case #3: We overlap from the rear, so adjust it */ + if (range->pgend <= pgend) { + range_shrink(range, range->pgstart, pgstart-1); + continue; + } + + /* + * Case #4: We eat a chunk out of the middle. A bit + * more complicated, we allocate a new range for the + * second half and adjust the first chunk's endpoint. + */ + range_alloc(asma, range, range->purged, + pgend + 1, range->pgend); + range_shrink(range, range->pgstart, pgstart - 1); + break; + } + } + + return ret; +} + +/* + * ashmem_unpin - unpin the given range of pages. Returns zero on success. + * + * Caller must hold ashmem_mutex. + */ +static int ashmem_unpin(struct ashmem_area *asma, size_t pgstart, size_t pgend) +{ + struct ashmem_range *range, *next; + unsigned int purged = ASHMEM_NOT_PURGED; + +restart: + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) { + /* short circuit: this is our insertion point */ + if (range_before_page(range, pgstart)) + break; + + /* + * The user can ask us to unpin pages that are already entirely + * or partially pinned. We handle those two cases here. + */ + if (page_range_subsumed_by_range(range, pgstart, pgend)) + return 0; + if (page_range_in_range(range, pgstart, pgend)) { + pgstart = min_t(size_t, range->pgstart, pgstart), + pgend = max_t(size_t, range->pgend, pgend); + purged |= range->purged; + range_del(range); + goto restart; + } + } + + return range_alloc(asma, range, purged, pgstart, pgend); +} + +/* + * ashmem_get_pin_status - Returns ASHMEM_IS_UNPINNED if _any_ pages in the + * given interval are unpinned and ASHMEM_IS_PINNED otherwise. + * + * Caller must hold ashmem_mutex. + */ +static int ashmem_get_pin_status(struct ashmem_area *asma, size_t pgstart, + size_t pgend) +{ + struct ashmem_range *range; + int ret = ASHMEM_IS_PINNED; + + list_for_each_entry(range, &asma->unpinned_list, unpinned) { + if (range_before_page(range, pgstart)) + break; + if (page_range_in_range(range, pgstart, pgend)) { + ret = ASHMEM_IS_UNPINNED; + break; + } + } + + return ret; +} + +static int ashmem_pin_unpin(struct ashmem_area *asma, unsigned long cmd, + void __user *p) +{ + struct ashmem_pin pin; + size_t pgstart, pgend; + int ret = -EINVAL; + + if (unlikely(!asma->file)) + return -EINVAL; + + if (unlikely(copy_from_user(&pin, p, sizeof(pin)))) + return -EFAULT; + + /* per custom, you can pass zero for len to mean "everything onward" */ + if (!pin.len) + pin.len = PAGE_ALIGN(asma->size) - pin.offset; + + if (unlikely((pin.offset | pin.len) & ~PAGE_MASK)) + return -EINVAL; + + if (unlikely(((__u32) -1) - pin.offset < pin.len)) + return -EINVAL; + + if (unlikely(PAGE_ALIGN(asma->size) < pin.offset + pin.len)) + return -EINVAL; + + pgstart = pin.offset / PAGE_SIZE; + pgend = pgstart + (pin.len / PAGE_SIZE) - 1; + + mutex_lock(&ashmem_mutex); + + switch (cmd) { + case ASHMEM_PIN: + ret = ashmem_pin(asma, pgstart, pgend); + break; + case ASHMEM_UNPIN: + ret = ashmem_unpin(asma, pgstart, pgend); + break; + case ASHMEM_GET_PIN_STATUS: + ret = ashmem_get_pin_status(asma, pgstart, pgend); + break; + } + + mutex_unlock(&ashmem_mutex); + + return ret; +} + +static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct ashmem_area *asma = file->private_data; + long ret = -ENOTTY; + + switch (cmd) { + case ASHMEM_SET_NAME: + ret = set_name(asma, (void __user *) arg); + break; + case ASHMEM_GET_NAME: + ret = get_name(asma, (void __user *) arg); + break; + case ASHMEM_SET_SIZE: + ret = -EINVAL; + if (!asma->file) { + ret = 0; + asma->size = (size_t) arg; + } + break; + case ASHMEM_GET_SIZE: + ret = asma->size; + break; + case ASHMEM_SET_PROT_MASK: + ret = set_prot_mask(asma, arg); + break; + case ASHMEM_GET_PROT_MASK: + ret = asma->prot_mask; + break; + case ASHMEM_PIN: + case ASHMEM_UNPIN: + case ASHMEM_GET_PIN_STATUS: + ret = ashmem_pin_unpin(asma, cmd, (void __user *) arg); + break; + case ASHMEM_PURGE_ALL_CACHES: + ret = -EPERM; + if (capable(CAP_SYS_ADMIN)) { + struct shrink_control sc = { + .gfp_mask = GFP_KERNEL, + .nr_to_scan = 0, + }; + ret = ashmem_shrink(&ashmem_shrinker, &sc); + sc.nr_to_scan = ret; + ashmem_shrink(&ashmem_shrinker, &sc); + } + break; + } + + return ret; +} + +static struct file_operations ashmem_fops = { + .owner = THIS_MODULE, + .open = ashmem_open, + .release = ashmem_release, + .read = ashmem_read, + .llseek = ashmem_llseek, + .mmap = ashmem_mmap, + .unlocked_ioctl = ashmem_ioctl, + .compat_ioctl = ashmem_ioctl, +}; + +static struct miscdevice ashmem_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "ashmem", + .fops = &ashmem_fops, +}; + +static int __init ashmem_init(void) +{ + int ret; + + ashmem_area_cachep = kmem_cache_create("ashmem_area_cache", + sizeof(struct ashmem_area), + 0, 0, NULL); + if (unlikely(!ashmem_area_cachep)) { + printk(KERN_ERR "ashmem: failed to create slab cache\n"); + return -ENOMEM; + } + + ashmem_range_cachep = kmem_cache_create("ashmem_range_cache", + sizeof(struct ashmem_range), + 0, 0, NULL); + if (unlikely(!ashmem_range_cachep)) { + printk(KERN_ERR "ashmem: failed to create slab cache\n"); + return -ENOMEM; + } + + ret = misc_register(&ashmem_misc); + if (unlikely(ret)) { + printk(KERN_ERR "ashmem: failed to register misc device!\n"); + return ret; + } + + register_shrinker(&ashmem_shrinker); + + printk(KERN_INFO "ashmem: initialized\n"); + + return 0; +} + +static void __exit ashmem_exit(void) +{ + int ret; + + unregister_shrinker(&ashmem_shrinker); + + ret = misc_deregister(&ashmem_misc); + if (unlikely(ret)) + printk(KERN_ERR "ashmem: failed to unregister misc device!\n"); + + kmem_cache_destroy(ashmem_range_cachep); + kmem_cache_destroy(ashmem_area_cachep); + + printk(KERN_INFO "ashmem: unloaded\n"); +} + +module_init(ashmem_init); +module_exit(ashmem_exit); + +MODULE_LICENSE("GPL"); diff --git a/mm/backing-dev.c b/mm/backing-dev.c index 2b49dd2..b3b122f 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -45,17 +45,6 @@ static struct timer_list sync_supers_timer; static int bdi_sync_supers(void *); static void sync_supers_timer_fn(unsigned long); -void bdi_lock_two(struct bdi_writeback *wb1, struct bdi_writeback *wb2) -{ - if (wb1 < wb2) { - spin_lock(&wb1->list_lock); - spin_lock_nested(&wb2->list_lock, 1); - } else { - spin_lock(&wb2->list_lock); - spin_lock_nested(&wb1->list_lock, 1); - } -} - #ifdef CONFIG_DEBUG_FS #include <linux/debugfs.h> #include <linux/seq_file.h> @@ -78,44 +67,34 @@ static int bdi_debug_stats_show(struct seq_file *m, void *v) struct inode *inode; nr_dirty = nr_io = nr_more_io = 0; - spin_lock(&wb->list_lock); + spin_lock(&inode_wb_list_lock); list_for_each_entry(inode, &wb->b_dirty, i_wb_list) nr_dirty++; list_for_each_entry(inode, &wb->b_io, i_wb_list) nr_io++; list_for_each_entry(inode, &wb->b_more_io, i_wb_list) nr_more_io++; - spin_unlock(&wb->list_lock); + spin_unlock(&inode_wb_list_lock); global_dirty_limits(&background_thresh, &dirty_thresh); bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh); #define K(x) ((x) << (PAGE_SHIFT - 10)) seq_printf(m, - "BdiWriteback: %10lu kB\n" - "BdiReclaimable: %10lu kB\n" - "BdiDirtyThresh: %10lu kB\n" - "DirtyThresh: %10lu kB\n" - "BackgroundThresh: %10lu kB\n" - "BdiDirtied: %10lu kB\n" - "BdiWritten: %10lu kB\n" - "BdiWriteBandwidth: %10lu kBps\n" - "b_dirty: %10lu\n" - "b_io: %10lu\n" - "b_more_io: %10lu\n" - "bdi_list: %10u\n" - "state: %10lx\n", + "BdiWriteback: %8lu kB\n" + "BdiReclaimable: %8lu kB\n" + "BdiDirtyThresh: %8lu kB\n" + "DirtyThresh: %8lu kB\n" + "BackgroundThresh: %8lu kB\n" + "b_dirty: %8lu\n" + "b_io: %8lu\n" + "b_more_io: %8lu\n" + "bdi_list: %8u\n" + "state: %8lx\n", (unsigned long) K(bdi_stat(bdi, BDI_WRITEBACK)), (unsigned long) K(bdi_stat(bdi, BDI_RECLAIMABLE)), - K(bdi_thresh), - K(dirty_thresh), - K(background_thresh), - (unsigned long) K(bdi_stat(bdi, BDI_DIRTIED)), - (unsigned long) K(bdi_stat(bdi, BDI_WRITTEN)), - (unsigned long) K(bdi->write_bandwidth), - nr_dirty, - nr_io, - nr_more_io, + K(bdi_thresh), K(dirty_thresh), + K(background_thresh), nr_dirty, nr_io, nr_more_io, !list_empty(&bdi->bdi_list), bdi->state); #undef K @@ -270,6 +249,18 @@ int bdi_has_dirty_io(struct backing_dev_info *bdi) return wb_has_dirty_io(&bdi->wb); } +static void bdi_flush_io(struct backing_dev_info *bdi) +{ + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + .older_than_this = NULL, + .range_cyclic = 1, + .nr_to_write = 1024, + }; + + writeback_inodes_wb(&bdi->wb, &wbc); +} + /* * kupdated() used to do this. We cannot do it from the bdi_forker_thread() * or we risk deadlocking on ->s_umount. The longer term solution would be @@ -318,7 +309,7 @@ static void wakeup_timer_fn(unsigned long data) if (bdi->wb.task) { trace_writeback_wake_thread(bdi); wake_up_process(bdi->wb.task); - } else if (bdi->dev) { + } else { /* * When bdi tasks are inactive for long time, they are killed. * In this case we have to wake-up the forker thread which @@ -361,17 +352,6 @@ static unsigned long bdi_longest_inactive(void) return max(5UL * 60 * HZ, interval); } -/* - * Clear pending bit and wakeup anybody waiting for flusher thread creation or - * shutdown - */ -static void bdi_clear_pending(struct backing_dev_info *bdi) -{ - clear_bit(BDI_pending, &bdi->state); - smp_mb__after_clear_bit(); - wake_up_bit(&bdi->state, BDI_pending); -} - static int bdi_forker_thread(void *ptr) { struct bdi_writeback *me = ptr; @@ -403,12 +383,6 @@ static int bdi_forker_thread(void *ptr) } spin_lock_bh(&bdi_lock); - /* - * In the following loop we are going to check whether we have - * some work to do without any synchronization with tasks - * waking us up to do work for them. Set the task state here - * so that we don't miss wakeups after verifying conditions. - */ set_current_state(TASK_INTERRUPTIBLE); list_for_each_entry(bdi, &bdi_list, bdi_list) { @@ -472,11 +446,9 @@ static int bdi_forker_thread(void *ptr) if (IS_ERR(task)) { /* * If thread creation fails, force writeout of - * the bdi from the thread. Hopefully 1024 is - * large enough for efficient IO. + * the bdi from the thread. */ - writeback_inodes_wb(&bdi->wb, 1024, - WB_REASON_FORKER_THREAD); + bdi_flush_io(bdi); } else { /* * The spinlock makes sure we do not lose @@ -489,13 +461,11 @@ static int bdi_forker_thread(void *ptr) spin_unlock_bh(&bdi->wb_lock); wake_up_process(task); } - bdi_clear_pending(bdi); break; case KILL_THREAD: __set_current_state(TASK_RUNNING); kthread_stop(task); - bdi_clear_pending(bdi); break; case NO_ACTION: @@ -511,8 +481,16 @@ static int bdi_forker_thread(void *ptr) else schedule_timeout(msecs_to_jiffies(dirty_writeback_interval * 10)); try_to_freeze(); - break; + /* Back to the main loop */ + continue; } + + /* + * Clear pending bit and wakeup anybody waiting to tear us down. + */ + clear_bit(BDI_pending, &bdi->state); + smp_mb__after_clear_bit(); + wake_up_bit(&bdi->state, BDI_pending); } return 0; @@ -527,7 +505,7 @@ static void bdi_remove_from_list(struct backing_dev_info *bdi) list_del_rcu(&bdi->bdi_list); spin_unlock_bh(&bdi_lock); - synchronize_rcu_expedited(); + synchronize_rcu(); } int bdi_register(struct backing_dev_info *bdi, struct device *parent, @@ -584,8 +562,6 @@ EXPORT_SYMBOL(bdi_register_dev); */ static void bdi_wb_shutdown(struct backing_dev_info *bdi) { - struct task_struct *task; - if (!bdi_cap_writeback_dirty(bdi)) return; @@ -606,14 +582,9 @@ static void bdi_wb_shutdown(struct backing_dev_info *bdi) * unfreeze of the thread before calling kthread_stop(), otherwise * it would never exet if it is currently stuck in the refrigerator. */ - spin_lock_bh(&bdi->wb_lock); - task = bdi->wb.task; - bdi->wb.task = NULL; - spin_unlock_bh(&bdi->wb_lock); - - if (task) { - thaw_process(task); - kthread_stop(task); + if (bdi->wb.task) { + thaw_process(bdi->wb.task); + kthread_stop(bdi->wb.task); } } @@ -634,9 +605,7 @@ static void bdi_prune_sb(struct backing_dev_info *bdi) void bdi_unregister(struct backing_dev_info *bdi) { - struct device *dev = bdi->dev; - - if (dev) { + if (bdi->dev) { bdi_set_min_ratio(bdi, 0); trace_writeback_bdi_unregister(bdi); bdi_prune_sb(bdi); @@ -645,12 +614,8 @@ void bdi_unregister(struct backing_dev_info *bdi) if (!bdi_cap_flush_forker(bdi)) bdi_wb_shutdown(bdi); bdi_debug_unregister(bdi); - - spin_lock_bh(&bdi->wb_lock); + device_unregister(bdi->dev); bdi->dev = NULL; - spin_unlock_bh(&bdi->wb_lock); - - device_unregister(dev); } } EXPORT_SYMBOL(bdi_unregister); @@ -664,15 +629,9 @@ static void bdi_wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi) INIT_LIST_HEAD(&wb->b_dirty); INIT_LIST_HEAD(&wb->b_io); INIT_LIST_HEAD(&wb->b_more_io); - spin_lock_init(&wb->list_lock); setup_timer(&wb->wakeup_timer, wakeup_timer_fn, (unsigned long)bdi); } -/* - * Initial write bandwidth: 100 MB/s - */ -#define INIT_BW (100 << (20 - PAGE_SHIFT)) - int bdi_init(struct backing_dev_info *bdi) { int i, err; @@ -695,15 +654,6 @@ int bdi_init(struct backing_dev_info *bdi) } bdi->dirty_exceeded = 0; - - bdi->bw_time_stamp = jiffies; - bdi->written_stamp = 0; - - bdi->balanced_dirty_ratelimit = INIT_BW; - bdi->dirty_ratelimit = INIT_BW; - bdi->write_bandwidth = INIT_BW; - bdi->avg_write_bandwidth = INIT_BW; - err = prop_local_init_percpu(&bdi->completions); if (err) { @@ -727,12 +677,11 @@ void bdi_destroy(struct backing_dev_info *bdi) if (bdi_has_dirty_io(bdi)) { struct bdi_writeback *dst = &default_backing_dev_info.wb; - bdi_lock_two(&bdi->wb, dst); + spin_lock(&inode_wb_list_lock); list_splice(&bdi->wb.b_dirty, &dst->b_dirty); list_splice(&bdi->wb.b_io, &dst->b_io); list_splice(&bdi->wb.b_more_io, &dst->b_more_io); - spin_unlock(&bdi->wb.list_lock); - spin_unlock(&dst->list_lock); + spin_unlock(&inode_wb_list_lock); } bdi_unregister(bdi); diff --git a/mm/bootmem.c b/mm/bootmem.c index b863822..9686c4e 100644 --- a/mm/bootmem.c +++ b/mm/bootmem.c @@ -12,7 +12,7 @@ #include <linux/pfn.h> #include <linux/slab.h> #include <linux/bootmem.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/kmemleak.h> #include <linux/range.h> #include <linux/memblock.h> diff --git a/mm/bounce.c b/mm/bounce.c index f71a3b3..1481de6 100644 --- a/mm/bounce.c +++ b/mm/bounce.c @@ -4,7 +4,7 @@ */ #include <linux/mm.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/swap.h> #include <linux/gfp.h> #include <linux/bio.h> @@ -14,7 +14,6 @@ #include <linux/init.h> #include <linux/hash.h> #include <linux/highmem.h> -#include <linux/bootmem.h> #include <asm/tlbflush.h> #include <trace/events/block.h> @@ -27,10 +26,12 @@ static mempool_t *page_pool, *isa_page_pool; #ifdef CONFIG_HIGHMEM static __init int init_emergency_pool(void) { -#ifndef CONFIG_MEMORY_HOTPLUG - if (max_pfn <= max_low_pfn) + struct sysinfo i; + si_meminfo(&i); + si_swapinfo(&i); + + if (!i.totalhigh) return 0; -#endif page_pool = mempool_create_page_pool(POOL_SIZE, 0); BUG_ON(!page_pool); @@ -132,7 +133,7 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool, int err) /* * free up bounce indirect pages used */ - bio_for_each_segment_all(bvec, bio, i) { + __bio_for_each_segment(bvec, bio, i, 0) { org_vec = bio_orig->bi_io_vec + i; if (bvec->bv_page == org_vec->bv_page) continue; diff --git a/mm/cma-best-fit.c b/mm/cma-best-fit.c new file mode 100644 index 0000000..24c27c8 --- /dev/null +++ b/mm/cma-best-fit.c @@ -0,0 +1,408 @@ +/* + * Contiguous Memory Allocator framework: Best Fit allocator + * Copyright (c) 2010 by Samsung Electronics. + * Written by Michal Nazarewicz (m.nazarewicz@samsung.com) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License or (at your optional) any later version of the license. + */ + +#define pr_fmt(fmt) "cma: bf: " fmt + +#ifdef CONFIG_CMA_DEBUG +# define DEBUG +#endif + +#include <linux/errno.h> /* Error numbers */ +#include <linux/slab.h> /* kmalloc() */ + +#include <linux/cma.h> /* CMA structures */ + + +/************************* Data Types *************************/ + +struct cma_bf_item { + struct cma_chunk ch; + struct rb_node by_size; +}; + +struct cma_bf_private { + struct rb_root by_start_root; + struct rb_root by_size_root; +}; + + +/************************* Prototypes *************************/ + +/* + * Those are only for holes. They must be called whenever hole's + * properties change but also whenever chunk becomes a hole or hole + * becames a chunk. + */ +static void __cma_bf_hole_insert_by_size(struct cma_bf_item *item); +static void __cma_bf_hole_erase_by_size(struct cma_bf_item *item); +static int __must_check +__cma_bf_hole_insert_by_start(struct cma_bf_item *item); +static void __cma_bf_hole_erase_by_start(struct cma_bf_item *item); + +/** + * __cma_bf_hole_take - takes a chunk of memory out of a hole. + * @hole: hole to take chunk from + * @size: chunk's size + * @alignment: chunk's starting address alignment (must be power of two) + * + * Takes a @size bytes large chunk from hole @hole which must be able + * to hold the chunk. The "must be able" includes also alignment + * constraint. + * + * Returns allocated item or NULL on error (if kmalloc() failed). + */ +static struct cma_bf_item *__must_check +__cma_bf_hole_take(struct cma_bf_item *hole, size_t size, dma_addr_t alignment); + +/** + * __cma_bf_hole_merge_maybe - tries to merge hole with neighbours. + * @item: hole to try and merge + * + * Which items are preserved is undefined so you may not rely on it. + */ +static void __cma_bf_hole_merge_maybe(struct cma_bf_item *item); + + +/************************* Device API *************************/ + +int cma_bf_init(struct cma_region *reg) +{ + struct cma_bf_private *prv; + struct cma_bf_item *item; + + prv = kzalloc(sizeof *prv, GFP_KERNEL); + if (unlikely(!prv)) + return -ENOMEM; + + item = kzalloc(sizeof *item, GFP_KERNEL); + if (unlikely(!item)) { + kfree(prv); + return -ENOMEM; + } + + item->ch.start = reg->start; + item->ch.size = reg->size; + item->ch.reg = reg; + + rb_root_init(&prv->by_start_root, &item->ch.by_start); + rb_root_init(&prv->by_size_root, &item->by_size); + + reg->private_data = prv; + return 0; +} + +void cma_bf_cleanup(struct cma_region *reg) +{ + struct cma_bf_private *prv = reg->private_data; + struct cma_bf_item *item = + rb_entry(prv->by_size_root.rb_node, + struct cma_bf_item, by_size); + + /* We can assume there is only a single hole in the tree. */ + WARN_ON(item->by_size.rb_left || item->by_size.rb_right || + item->ch.by_start.rb_left || item->ch.by_start.rb_right); + + kfree(item); + kfree(prv); +} + +struct cma_chunk *cma_bf_alloc(struct cma_region *reg, + size_t size, dma_addr_t alignment) +{ + struct cma_bf_private *prv = reg->private_data; + struct rb_node *node = prv->by_size_root.rb_node; + struct cma_bf_item *item = NULL; + + /* First find hole that is large enough */ + while (node) { + struct cma_bf_item *i = + rb_entry(node, struct cma_bf_item, by_size); + + if (i->ch.size < size) { + node = node->rb_right; + } else if (i->ch.size >= size) { + node = node->rb_left; + item = i; + } + } + if (!item) + return NULL; + + /* Now look for items which can satisfy alignment requirements */ + node = &item->by_size; + for (;;) { + dma_addr_t start = ALIGN(item->ch.start, alignment); + dma_addr_t end = item->ch.start + item->ch.size; + if (start < end && end - start >= size) { + item = __cma_bf_hole_take(item, size, alignment); + return likely(item) ? &item->ch : NULL; + } + + node = rb_next(node); + if (!node) + return NULL; + + item = rb_entry(node, struct cma_bf_item, by_size); + } +} + +void cma_bf_free(struct cma_chunk *chunk) +{ + struct cma_bf_item *item = container_of(chunk, struct cma_bf_item, ch); + + /* Add new hole */ + if (unlikely(__cma_bf_hole_insert_by_start(item))) { + /* + * We're screwed... Just free the item and forget + * about it. Things are broken beyond repair so no + * sense in trying to recover. + */ + kfree(item); + } else { + __cma_bf_hole_insert_by_size(item); + + /* Merge with prev and next sibling */ + __cma_bf_hole_merge_maybe(item); + } +} + + +/************************* Basic Tree Manipulation *************************/ + +static void __cma_bf_hole_insert_by_size(struct cma_bf_item *item) +{ + struct cma_bf_private *prv = item->ch.reg->private_data; + struct rb_node **link = &prv->by_size_root.rb_node, *parent = NULL; + const typeof(item->ch.size) value = item->ch.size; + + while (*link) { + struct cma_bf_item *i; + parent = *link; + i = rb_entry(parent, struct cma_bf_item, by_size); + link = value <= i->ch.size + ? &parent->rb_left + : &parent->rb_right; + } + + rb_link_node(&item->by_size, parent, link); + rb_insert_color(&item->by_size, &prv->by_size_root); +} + +static void __cma_bf_hole_erase_by_size(struct cma_bf_item *item) +{ + struct cma_bf_private *prv = item->ch.reg->private_data; + rb_erase(&item->by_size, &prv->by_size_root); +} + +static int __must_check +__cma_bf_hole_insert_by_start(struct cma_bf_item *item) +{ + struct cma_bf_private *prv = item->ch.reg->private_data; + struct rb_node **link = &prv->by_start_root.rb_node, *parent = NULL; + const typeof(item->ch.start) value = item->ch.start; + + while (*link) { + struct cma_bf_item *i; + parent = *link; + i = rb_entry(parent, struct cma_bf_item, ch.by_start); + + if (WARN_ON(value == i->ch.start)) + /* + * This should *never* happen. And I mean + * *never*. We could even BUG on it but + * hopefully things are only a bit broken, + * ie. system can still run. We produce + * a warning and return an error. + */ + return -EBUSY; + + link = value <= i->ch.start + ? &parent->rb_left + : &parent->rb_right; + } + + rb_link_node(&item->ch.by_start, parent, link); + rb_insert_color(&item->ch.by_start, &prv->by_start_root); + return 0; +} + +static void __cma_bf_hole_erase_by_start(struct cma_bf_item *item) +{ + struct cma_bf_private *prv = item->ch.reg->private_data; + rb_erase(&item->ch.by_start, &prv->by_start_root); +} + + +/************************* More Tree Manipulation *************************/ + +static struct cma_bf_item *__must_check +__cma_bf_hole_take(struct cma_bf_item *hole, size_t size, size_t alignment) +{ + struct cma_bf_item *item; + + /* + * There are three cases: + * 1. the chunk takes the whole hole, + * 2. the chunk is at the beginning or at the end of the hole, or + * 3. the chunk is in the middle of the hole. + */ + + + /* Case 1, the whole hole */ + if (size == hole->ch.size) { + __cma_bf_hole_erase_by_size(hole); + __cma_bf_hole_erase_by_start(hole); + return hole; + } + + + /* Allocate */ + item = kmalloc(sizeof *item, GFP_KERNEL); + if (unlikely(!item)) + return NULL; + + item->ch.start = ALIGN(hole->ch.start, alignment); + item->ch.size = size; + + /* Case 3, in the middle */ + if (item->ch.start != hole->ch.start + && item->ch.start + item->ch.size != + hole->ch.start + hole->ch.size) { + struct cma_bf_item *tail; + + /* + * Space between the end of the chunk and the end of + * the region, ie. space left after the end of the + * chunk. If this is dividable by alignment we can + * move the chunk to the end of the hole. + */ + size_t left = + hole->ch.start + hole->ch.size - + (item->ch.start + item->ch.size); + if (left % alignment == 0) { + item->ch.start += left; + goto case_2; + } + + /* + * We are going to add a hole at the end. This way, + * we will reduce the problem to case 2 -- the chunk + * will be at the end of the hole. + */ + tail = kmalloc(sizeof *tail, GFP_KERNEL); + if (unlikely(!tail)) { + kfree(item); + return NULL; + } + + tail->ch.start = item->ch.start + item->ch.size; + tail->ch.size = + hole->ch.start + hole->ch.size - tail->ch.start; + tail->ch.reg = hole->ch.reg; + + if (unlikely(__cma_bf_hole_insert_by_start(tail))) { + /* + * Things are broken beyond repair... Abort + * inserting the hole but still continue with + * allocation (seems like the best we can do). + */ + + hole->ch.size = tail->ch.start - hole->ch.start; + kfree(tail); + } else { + __cma_bf_hole_insert_by_size(tail); + /* + * It's important that we first insert the new + * hole in the tree sorted by size and later + * reduce the size of the old hole. We will + * update the position of the old hole in the + * rb tree in code that handles case 2. + */ + hole->ch.size = tail->ch.start - hole->ch.start; + } + + /* Go to case 2 */ + } + + + /* Case 2, at the beginning or at the end */ +case_2: + /* No need to update the tree; order preserved. */ + if (item->ch.start == hole->ch.start) + hole->ch.start += item->ch.size; + + /* Alter hole's size */ + hole->ch.size -= size; + __cma_bf_hole_erase_by_size(hole); + __cma_bf_hole_insert_by_size(hole); + + return item; +} + + +static void __cma_bf_hole_merge_maybe(struct cma_bf_item *item) +{ + struct cma_bf_item *prev; + struct rb_node *node; + int twice = 2; + + node = rb_prev(&item->ch.by_start); + if (unlikely(!node)) + goto next; + prev = rb_entry(node, struct cma_bf_item, ch.by_start); + + for (;;) { + if (prev->ch.start + prev->ch.size == item->ch.start) { + /* Remove previous hole from trees */ + __cma_bf_hole_erase_by_size(prev); + __cma_bf_hole_erase_by_start(prev); + + /* Alter this hole */ + item->ch.size += prev->ch.size; + item->ch.start = prev->ch.start; + __cma_bf_hole_erase_by_size(item); + __cma_bf_hole_insert_by_size(item); + /* + * No need to update by start trees as we do + * not break sequence order + */ + + /* Free prev hole */ + kfree(prev); + } + +next: + if (!--twice) + break; + + node = rb_next(&item->ch.by_start); + if (unlikely(!node)) + break; + prev = item; + item = rb_entry(node, struct cma_bf_item, ch.by_start); + } +} + + + +/************************* Register *************************/ +static int cma_bf_module_init(void) +{ + static struct cma_allocator alloc = { + .name = "bf", + .init = cma_bf_init, + .cleanup = cma_bf_cleanup, + .alloc = cma_bf_alloc, + .free = cma_bf_free, + }; + return cma_allocator_register(&alloc); +} +module_init(cma_bf_module_init); diff --git a/mm/cma.c b/mm/cma.c new file mode 100644 index 0000000..2e5805b --- /dev/null +++ b/mm/cma.c @@ -0,0 +1,1431 @@ +/* + * Contiguous Memory Allocator framework + * Copyright (c) 2010 by Samsung Electronics. + * Written by Michal Nazarewicz (m.nazarewicz@samsung.com) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License or (at your optional) any later version of the license. + */ + +/* + * See Documentation/contiguous-memory.txt for details. + */ + +#define pr_fmt(fmt) "cma: " fmt + +#ifdef CONFIG_CMA_DEBUG +# define DEBUG +#endif + +#ifndef CONFIG_NO_BOOTMEM +# include <linux/bootmem.h> /* alloc_bootmem_pages_nopanic() */ +#endif +#ifdef CONFIG_HAVE_MEMBLOCK +# include <linux/memblock.h> /* memblock*() */ +#endif +#include <linux/device.h> /* struct device, dev_name() */ +#include <linux/errno.h> /* Error numbers */ +#include <linux/err.h> /* IS_ERR, PTR_ERR, etc. */ +#include <linux/mm.h> /* PAGE_ALIGN() */ +#include <linux/module.h> /* EXPORT_SYMBOL_GPL() */ +#include <linux/mutex.h> /* mutex */ +#include <linux/slab.h> /* kmalloc() */ +#include <linux/string.h> /* str*() */ + +#include <linux/cma.h> +#include <linux/vmalloc.h> + +/* + * Protects cma_regions, cma_allocators, cma_map, cma_map_length, + * cma_kobj, cma_sysfs_regions and cma_chunks_by_start. + */ +static DEFINE_MUTEX(cma_mutex); + + + +/************************* Map attribute *************************/ + +static const char *cma_map; +static size_t cma_map_length; + +/* + * map-attr ::= [ rules [ ';' ] ] + * rules ::= rule [ ';' rules ] + * rule ::= patterns '=' regions + * patterns ::= pattern [ ',' patterns ] + * regions ::= REG-NAME [ ',' regions ] + * pattern ::= dev-pattern [ '/' TYPE-NAME ] | '/' TYPE-NAME + * + * See Documentation/contiguous-memory.txt for details. + */ +static ssize_t cma_map_validate(const char *param) +{ + const char *ch = param; + + if (*ch == '\0' || *ch == '\n') + return 0; + + for (;;) { + const char *start = ch; + + while (*ch && *ch != '\n' && *ch != ';' && *ch != '=') + ++ch; + + if (*ch != '=' || start == ch) { + pr_err("map: expecting \"<patterns>=<regions>\" near %s\n", + start); + return -EINVAL; + } + + while (*++ch != ';') + if (*ch == '\0' || *ch == '\n') + return ch - param; + if (ch[1] == '\0' || ch[1] == '\n') + return ch - param; + ++ch; + } +} + +static int __init cma_map_param(char *param) +{ + ssize_t len; + + pr_debug("param: map: %s\n", param); + + len = cma_map_validate(param); + if (len < 0) + return len; + + cma_map = param; + cma_map_length = len; + return 0; +} + +#if defined CONFIG_CMA_CMDLINE + +early_param("cma.map", cma_map_param); + +#endif + + + +/************************* Early regions *************************/ + +struct list_head cma_early_regions __initdata = + LIST_HEAD_INIT(cma_early_regions); + +#ifdef CONFIG_CMA_CMDLINE + +/* + * regions-attr ::= [ regions [ ';' ] ] + * regions ::= region [ ';' regions ] + * + * region ::= [ '-' ] reg-name + * '=' size + * [ '@' start ] + * [ '/' alignment ] + * [ ':' alloc-name ] + * + * See Documentation/contiguous-memory.txt for details. + * + * Example: + * cma=reg1=64M:bf;reg2=32M@0x100000:bf;reg3=64M/1M:bf + * + * If allocator is ommited the first available allocater will be used. + */ + +#define NUMPARSE(cond_ch, type, cond) ({ \ + unsigned long long v = 0; \ + if (*param == (cond_ch)) { \ + const char *const msg = param + 1; \ + v = memparse(msg, ¶m); \ + if (!v || v > ~(type)0 || !(cond)) { \ + pr_err("param: invalid value near %s\n", msg); \ + ret = -EINVAL; \ + break; \ + } \ + } \ + v; \ + }) + +static int __init cma_param_parse(char *param) +{ + static struct cma_region regions[16]; + + size_t left = ARRAY_SIZE(regions); + struct cma_region *reg = regions; + int ret = 0; + + pr_debug("param: %s\n", param); + + for (; *param; ++reg) { + dma_addr_t start, alignment; + size_t size; + + if (unlikely(!--left)) { + pr_err("param: too many early regions\n"); + return -ENOSPC; + } + + /* Parse name */ + reg->name = param; + param = strchr(param, '='); + if (!param || param == reg->name) { + pr_err("param: expected \"<name>=\" near %s\n", + reg->name); + ret = -EINVAL; + break; + } + *param = '\0'; + + /* Parse numbers */ + size = NUMPARSE('\0', size_t, true); + start = NUMPARSE('@', dma_addr_t, true); + alignment = NUMPARSE('/', dma_addr_t, (v & (v - 1)) == 0); + + alignment = max(alignment, (dma_addr_t)PAGE_SIZE); + start = ALIGN(start, alignment); + size = PAGE_ALIGN(size); + if (start + size < start) { + pr_err("param: invalid start, size combination\n"); + ret = -EINVAL; + break; + } + + /* Parse allocator */ + if (*param == ':') { + reg->alloc_name = ++param; + while (*param && *param != ';') + ++param; + if (param == reg->alloc_name) + reg->alloc_name = NULL; + } + + /* Go to next */ + if (*param == ';') { + *param = '\0'; + ++param; + } else if (*param) { + pr_err("param: expecting ';' or end of parameter near %s\n", + param); + ret = -EINVAL; + break; + } + + /* Add */ + reg->size = size; + reg->start = start; + reg->alignment = alignment; + reg->copy_name = 1; + + list_add_tail(®->list, &cma_early_regions); + + pr_debug("param: registering early region %s (%p@%p/%p)\n", + reg->name, (void *)reg->size, (void *)reg->start, + (void *)reg->alignment); + } + + return ret; +} +early_param("cma", cma_param_parse); + +#undef NUMPARSE + +#endif + + +int __init __must_check cma_early_region_register(struct cma_region *reg) +{ + dma_addr_t start, alignment; + size_t size; + + if (reg->alignment & (reg->alignment - 1)) + return -EINVAL; + + alignment = max(reg->alignment, (dma_addr_t)PAGE_SIZE); + start = ALIGN(reg->start, alignment); + size = PAGE_ALIGN(reg->size); + + if (start + size < start) + return -EINVAL; + + reg->size = size; + reg->start = start; + reg->alignment = alignment; + + list_add_tail(®->list, &cma_early_regions); + + pr_debug("param: registering early region %s (%p@%p/%p)\n", + reg->name, (void *)reg->size, (void *)reg->start, + (void *)reg->alignment); + + return 0; +} + + + +/************************* Regions & Allocators *************************/ + +static void __cma_sysfs_region_add(struct cma_region *reg); + +static int __cma_region_attach_alloc(struct cma_region *reg); +static void __maybe_unused __cma_region_detach_alloc(struct cma_region *reg); + + +/* List of all regions. Named regions are kept before unnamed. */ +static LIST_HEAD(cma_regions); + +#define cma_foreach_region(reg) \ + list_for_each_entry(reg, &cma_regions, list) + +bool cma_is_registered_region(phys_addr_t start, size_t size) +{ + struct cma_region *reg; + + if (start + size <= start) + return false; + + cma_foreach_region(reg) { + if ((start >= reg->start) && + ((start + size) <= (reg->start + reg->size)) && + (size <= reg->size) && + (start < (reg->start + reg->size))) + + return true; + } + return false; +} + +int __must_check cma_region_register(struct cma_region *reg) +{ + const char *name, *alloc_name; + struct cma_region *r; + char *ch = NULL; + int ret = 0; + + if (!reg->size || reg->start + reg->size < reg->start) + return -EINVAL; + + reg->users = 0; + reg->used = 0; + reg->private_data = NULL; + reg->registered = 0; + reg->free_space = reg->size; + + /* Copy name and alloc_name */ + name = reg->name; + alloc_name = reg->alloc_name; + if (reg->copy_name && (reg->name || reg->alloc_name)) { + size_t name_size, alloc_size; + + name_size = reg->name ? strlen(reg->name) + 1 : 0; + alloc_size = reg->alloc_name ? strlen(reg->alloc_name) + 1 : 0; + + ch = kmalloc(name_size + alloc_size, GFP_KERNEL); + if (!ch) { + pr_err("%s: not enough memory to allocate name\n", + reg->name ?: "(private)"); + return -ENOMEM; + } + + if (name_size) { + memcpy(ch, reg->name, name_size); + name = ch; + ch += name_size; + } + + if (alloc_size) { + memcpy(ch, reg->alloc_name, alloc_size); + alloc_name = ch; + } + } + + mutex_lock(&cma_mutex); + + /* Don't let regions overlap */ + cma_foreach_region(r) + if (r->start + r->size > reg->start && + r->start < reg->start + reg->size) { + ret = -EADDRINUSE; + goto done; + } + + if (reg->alloc) { + ret = __cma_region_attach_alloc(reg); + if (unlikely(ret < 0)) + goto done; + } + + reg->name = name; + reg->alloc_name = alloc_name; + reg->registered = 1; + ch = NULL; + + /* + * Keep named at the beginning and unnamed (private) at the + * end. This helps in traversal when named region is looked + * for. + */ + if (name) + list_add(®->list, &cma_regions); + else + list_add_tail(®->list, &cma_regions); + + __cma_sysfs_region_add(reg); + +done: + mutex_unlock(&cma_mutex); + + pr_debug("%s: region %sregistered\n", + reg->name ?: "(private)", ret ? "not " : ""); + kfree(ch); + + return ret; +} +EXPORT_SYMBOL_GPL(cma_region_register); + +static struct cma_region *__must_check +__cma_region_find(const char **namep) +{ + struct cma_region *reg; + const char *ch, *name; + size_t n; + + ch = *namep; + while (*ch && *ch != ',' && *ch != ';') + ++ch; + name = *namep; + *namep = *ch == ',' ? ch + 1 : ch; + n = ch - name; + + /* + * Named regions are kept in front of unnamed so if we + * encounter unnamed region we can stop. + */ + cma_foreach_region(reg) + if (!reg->name) + break; + else if (!strncmp(name, reg->name, n) && !reg->name[n]) + return reg; + + return NULL; +} + + +/* List of all allocators. */ +static LIST_HEAD(cma_allocators); + +#define cma_foreach_allocator(alloc) \ + list_for_each_entry(alloc, &cma_allocators, list) + +int cma_allocator_register(struct cma_allocator *alloc) +{ + struct cma_region *reg; + int first; + + if (!alloc->alloc || !alloc->free) + return -EINVAL; + + mutex_lock(&cma_mutex); + + first = list_empty(&cma_allocators); + + list_add_tail(&alloc->list, &cma_allocators); + + /* + * Attach this allocator to all allocator-less regions that + * request this particular allocator (reg->alloc_name equals + * alloc->name) or if region wants the first available + * allocator and we are the first. + */ + cma_foreach_region(reg) { + if (reg->alloc) + continue; + if (reg->alloc_name + ? alloc->name && !strcmp(alloc->name, reg->alloc_name) + : (!reg->used && first)) + continue; + + reg->alloc = alloc; + __cma_region_attach_alloc(reg); + } + + mutex_unlock(&cma_mutex); + + pr_debug("%s: allocator registered\n", alloc->name ?: "(unnamed)"); + + return 0; +} +EXPORT_SYMBOL_GPL(cma_allocator_register); + +static struct cma_allocator *__must_check +__cma_allocator_find(const char *name) +{ + struct cma_allocator *alloc; + + if (!name) + return list_empty(&cma_allocators) + ? NULL + : list_entry(cma_allocators.next, + struct cma_allocator, list); + + cma_foreach_allocator(alloc) + if (alloc->name && !strcmp(name, alloc->name)) + return alloc; + + return NULL; +} + + + +/************************* Initialise CMA *************************/ + +int __init cma_set_defaults(struct cma_region *regions, const char *map) +{ + if (map) { + int ret = cma_map_param((char *)map); + if (unlikely(ret < 0)) + return ret; + } + + if (!regions) + return 0; + + for (; regions->size; ++regions) { + int ret = cma_early_region_register(regions); + if (unlikely(ret < 0)) + return ret; + } + + return 0; +} + + +int __init cma_early_region_reserve(struct cma_region *reg) +{ + int tried = 0; + + if (!reg->size || (reg->alignment & (reg->alignment - 1)) || + reg->reserved) + return -EINVAL; + +#ifndef CONFIG_NO_BOOTMEM + + tried = 1; + + { + void *ptr = __alloc_bootmem_nopanic(reg->size, reg->alignment, + reg->start); + if (ptr) { + reg->start = virt_to_phys(ptr); + reg->reserved = 1; + return 0; + } + } + +#endif + +#ifdef CONFIG_HAVE_MEMBLOCK + + tried = 1; + + if (reg->start) { + if (!memblock_is_region_reserved(reg->start, reg->size) && + memblock_reserve(reg->start, reg->size) >= 0) { + reg->reserved = 1; + return 0; + } + } else { + /* + * Use __memblock_alloc_base() since + * memblock_alloc_base() panic()s. + */ + u64 ret = __memblock_alloc_base(reg->size, reg->alignment, 0); + if (ret && + ret < ~(dma_addr_t)0 && + ret + reg->size < ~(dma_addr_t)0 && + ret + reg->size > ret) { + reg->start = ret; + reg->reserved = 1; + return 0; + } + + if (ret) + memblock_free(ret, reg->size); + } + +#endif + + return tried ? -ENOMEM : -EOPNOTSUPP; +} + +void __init cma_early_regions_reserve(int (*reserve)(struct cma_region *reg)) +{ + struct cma_region *reg; + + pr_debug("init: reserving early regions\n"); + + if (!reserve) + reserve = cma_early_region_reserve; + + list_for_each_entry(reg, &cma_early_regions, list) { + if (reg->reserved) { + /* nothing */ + } else if (reserve(reg) >= 0) { + pr_debug("init: %s: reserved %p@%p\n", + reg->name ?: "(private)", + (void *)reg->size, (void *)reg->start); + reg->reserved = 1; + } else { + pr_warn("init: %s: unable to reserve %p@%p/%p\n", + reg->name ?: "(private)", + (void *)reg->size, (void *)reg->start, + (void *)reg->alignment); + } + } +} + + +static int __init cma_init(void) +{ + struct cma_region *reg, *n; + + pr_debug("init: initialising\n"); + + if (cma_map) { + char *val = kmemdup(cma_map, cma_map_length + 1, GFP_KERNEL); + cma_map = val; + if (!val) + return -ENOMEM; + val[cma_map_length] = '\0'; + } + + list_for_each_entry_safe(reg, n, &cma_early_regions, list) { + INIT_LIST_HEAD(®->list); + /* + * We don't care if there was an error. It's a pity + * but there's not much we can do about it any way. + * If the error is on a region that was parsed from + * command line then it will stay and waste a bit of + * space; if it was registered using + * cma_early_region_register() it's caller's + * responsibility to do something about it. + */ + if (reg->reserved && cma_region_register(reg) < 0) + /* ignore error */; + } + + INIT_LIST_HEAD(&cma_early_regions); + + return 0; +} +/* + * We want to be initialised earlier than module_init/__initcall so + * that drivers that want to grab memory at boot time will get CMA + * ready. subsys_initcall() seems early enough and not too early at + * the same time. + */ +subsys_initcall(cma_init); + + + +/************************* SysFS *************************/ + +#if defined CONFIG_CMA_SYSFS + +static struct kobject cma_sysfs_regions; +static int cma_sysfs_regions_ready; + + +#define CMA_ATTR_INLINE(_type, _name) \ + (&((struct cma_ ## _type ## _attribute){ \ + .attr = { \ + .name = __stringify(_name), \ + .mode = 0644, \ + }, \ + .show = cma_sysfs_ ## _type ## _ ## _name ## _show, \ + .store = cma_sysfs_ ## _type ## _ ## _name ## _store, \ + }).attr) + +#define CMA_ATTR_RO_INLINE(_type, _name) \ + (&((struct cma_ ## _type ## _attribute){ \ + .attr = { \ + .name = __stringify(_name), \ + .mode = 0444, \ + }, \ + .show = cma_sysfs_ ## _type ## _ ## _name ## _show, \ + }).attr) + + +struct cma_root_attribute { + struct attribute attr; + ssize_t (*show)(char *buf); + int (*store)(const char *buf); +}; + +static ssize_t cma_sysfs_root_map_show(char *page) +{ + ssize_t len; + + len = cma_map_length; + if (!len) { + *page = 0; + len = 0; + } else { + if (len > (size_t)PAGE_SIZE - 1) + len = (size_t)PAGE_SIZE - 1; + memcpy(page, cma_map, len); + page[len++] = '\n'; + } + + return len; +} + +static int cma_sysfs_root_map_store(const char *page) +{ + ssize_t len = cma_map_validate(page); + char *val = NULL; + + if (len < 0) + return len; + + if (len) { + val = kmemdup(page, len + 1, GFP_KERNEL); + if (!val) + return -ENOMEM; + val[len] = '\0'; + } + + kfree(cma_map); + cma_map = val; + cma_map_length = len; + + return 0; +} + +static ssize_t cma_sysfs_root_allocators_show(char *page) +{ + struct cma_allocator *alloc; + size_t left = PAGE_SIZE; + char *ch = page; + + cma_foreach_allocator(alloc) { + ssize_t l = snprintf(ch, left, "%s ", alloc->name ?: "-"); + ch += l; + left -= l; + } + + if (ch != page) + ch[-1] = '\n'; + return ch - page; +} + +static ssize_t +cma_sysfs_root_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct cma_root_attribute *rattr = + container_of(attr, struct cma_root_attribute, attr); + ssize_t ret; + + mutex_lock(&cma_mutex); + ret = rattr->show(buf); + mutex_unlock(&cma_mutex); + + return ret; +} + +static ssize_t +cma_sysfs_root_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct cma_root_attribute *rattr = + container_of(attr, struct cma_root_attribute, attr); + int ret; + + mutex_lock(&cma_mutex); + ret = rattr->store(buf); + mutex_unlock(&cma_mutex); + + return ret < 0 ? ret : count; +} + +static struct kobj_type cma_sysfs_root_type = { + .sysfs_ops = &(const struct sysfs_ops){ + .show = cma_sysfs_root_show, + .store = cma_sysfs_root_store, + }, + .default_attrs = (struct attribute * []) { + CMA_ATTR_INLINE(root, map), + CMA_ATTR_RO_INLINE(root, allocators), + NULL + }, +}; + +static int __init cma_sysfs_init(void) +{ + static struct kobject root; + static struct kobj_type fake_type; + + struct cma_region *reg; + int ret; + + /* Root */ + ret = kobject_init_and_add(&root, &cma_sysfs_root_type, + mm_kobj, "contiguous"); + if (unlikely(ret < 0)) { + pr_err("init: unable to add root kobject: %d\n", ret); + return ret; + } + + /* Regions */ + ret = kobject_init_and_add(&cma_sysfs_regions, &fake_type, + &root, "regions"); + if (unlikely(ret < 0)) { + pr_err("init: unable to add regions kobject: %d\n", ret); + return ret; + } + + mutex_lock(&cma_mutex); + cma_sysfs_regions_ready = 1; + cma_foreach_region(reg) + __cma_sysfs_region_add(reg); + mutex_unlock(&cma_mutex); + + return 0; +} +device_initcall(cma_sysfs_init); + + + +struct cma_region_attribute { + struct attribute attr; + ssize_t (*show)(struct cma_region *reg, char *buf); + int (*store)(struct cma_region *reg, const char *buf); +}; + + +static ssize_t cma_sysfs_region_name_show(struct cma_region *reg, char *page) +{ + return reg->name ? snprintf(page, PAGE_SIZE, "%s\n", reg->name) : 0; +} + +static ssize_t cma_sysfs_region_start_show(struct cma_region *reg, char *page) +{ + return snprintf(page, PAGE_SIZE, "%p\n", (void *)reg->start); +} + +static ssize_t cma_sysfs_region_size_show(struct cma_region *reg, char *page) +{ + return snprintf(page, PAGE_SIZE, "%zu\n", reg->size); +} + +static ssize_t cma_sysfs_region_free_show(struct cma_region *reg, char *page) +{ + return snprintf(page, PAGE_SIZE, "%zu\n", reg->free_space); +} + +static ssize_t cma_sysfs_region_users_show(struct cma_region *reg, char *page) +{ + return snprintf(page, PAGE_SIZE, "%u\n", reg->users); +} + +static ssize_t cma_sysfs_region_alloc_show(struct cma_region *reg, char *page) +{ + if (reg->alloc) + return snprintf(page, PAGE_SIZE, "%s\n", + reg->alloc->name ?: "-"); + else if (reg->alloc_name) + return snprintf(page, PAGE_SIZE, "[%s]\n", reg->alloc_name); + else + return 0; +} + +static int +cma_sysfs_region_alloc_store(struct cma_region *reg, const char *page) +{ + char *s; + + if (reg->alloc && reg->users) + return -EBUSY; + + if (!*page || *page == '\n') { + s = NULL; + } else { + size_t len; + + for (s = (char *)page; *++s && *s != '\n'; ) + /* nop */; + + len = s - page; + s = kmemdup(page, len + 1, GFP_KERNEL); + if (!s) + return -ENOMEM; + s[len] = '\0'; + } + + if (reg->alloc) + __cma_region_detach_alloc(reg); + + if (reg->free_alloc_name) + kfree(reg->alloc_name); + + reg->alloc_name = s; + reg->free_alloc_name = !!s; + + return 0; +} + + +static ssize_t +cma_sysfs_region_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct cma_region *reg = container_of(kobj, struct cma_region, kobj); + struct cma_region_attribute *rattr = + container_of(attr, struct cma_region_attribute, attr); + ssize_t ret; + + mutex_lock(&cma_mutex); + ret = rattr->show(reg, buf); + mutex_unlock(&cma_mutex); + + return ret; +} + +static int +cma_sysfs_region_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct cma_region *reg = container_of(kobj, struct cma_region, kobj); + struct cma_region_attribute *rattr = + container_of(attr, struct cma_region_attribute, attr); + int ret; + + mutex_lock(&cma_mutex); + ret = rattr->store(reg, buf); + mutex_unlock(&cma_mutex); + + return ret < 0 ? ret : count; +} + +static struct kobj_type cma_sysfs_region_type = { + .sysfs_ops = &(const struct sysfs_ops){ + .show = cma_sysfs_region_show, + .store = cma_sysfs_region_store, + }, + .default_attrs = (struct attribute * []) { + CMA_ATTR_RO_INLINE(region, name), + CMA_ATTR_RO_INLINE(region, start), + CMA_ATTR_RO_INLINE(region, size), + CMA_ATTR_RO_INLINE(region, free), + CMA_ATTR_RO_INLINE(region, users), + CMA_ATTR_INLINE(region, alloc), + NULL + }, +}; + +static void __cma_sysfs_region_add(struct cma_region *reg) +{ + int ret; + + if (!cma_sysfs_regions_ready) + return; + + memset(®->kobj, 0, sizeof reg->kobj); + + ret = kobject_init_and_add(®->kobj, &cma_sysfs_region_type, + &cma_sysfs_regions, + "%p", (void *)reg->start); + + if (reg->name && + sysfs_create_link(&cma_sysfs_regions, ®->kobj, reg->name) < 0) + /* Ignore any errors. */; +} + +#else + +static void __cma_sysfs_region_add(struct cma_region *reg) +{ + /* nop */ +} + +#endif + + +/************************* Chunks *************************/ + +/* All chunks sorted by start address. */ +static struct rb_root cma_chunks_by_start; + +static struct cma_chunk *__must_check __cma_chunk_find(dma_addr_t addr) +{ + struct cma_chunk *chunk; + struct rb_node *n; + + for (n = cma_chunks_by_start.rb_node; n; ) { + chunk = rb_entry(n, struct cma_chunk, by_start); + if (addr < chunk->start) + n = n->rb_left; + else if (addr > chunk->start) + n = n->rb_right; + else + return chunk; + } + WARN(1, KERN_WARNING "no chunk starting at %p\n", (void *)addr); + return NULL; +} + +static int __must_check __cma_chunk_insert(struct cma_chunk *chunk) +{ + struct rb_node **new, *parent = NULL; + typeof(chunk->start) addr = chunk->start; + + for (new = &cma_chunks_by_start.rb_node; *new; ) { + struct cma_chunk *c = + container_of(*new, struct cma_chunk, by_start); + + parent = *new; + if (addr < c->start) { + new = &(*new)->rb_left; + } else if (addr > c->start) { + new = &(*new)->rb_right; + } else { + /* + * We should never be here. If we are it + * means allocator gave us an invalid chunk + * (one that has already been allocated) so we + * refuse to accept it. Our caller will + * recover by freeing the chunk. + */ + WARN_ON(1); + return -EADDRINUSE; + } + } + + rb_link_node(&chunk->by_start, parent, new); + rb_insert_color(&chunk->by_start, &cma_chunks_by_start); + + return 0; +} + +static void __cma_chunk_free(struct cma_chunk *chunk) +{ + rb_erase(&chunk->by_start, &cma_chunks_by_start); + + chunk->reg->free_space += chunk->size; + --chunk->reg->users; + + chunk->reg->alloc->free(chunk); +} + + +/************************* The Device API *************************/ + +static const char *__must_check +__cma_where_from(const struct device *dev, const char *type); + + +/* Allocate. */ + +static dma_addr_t __must_check +__cma_alloc_from_region(struct cma_region *reg, + size_t size, dma_addr_t alignment) +{ + struct cma_chunk *chunk; + + pr_debug("allocate %p/%p from %s\n", + (void *)size, (void *)alignment, + reg ? reg->name ?: "(private)" : "(null)"); + + if (!reg || reg->free_space < size) + return -ENOMEM; + + if (!reg->alloc) { + if (!reg->used) + __cma_region_attach_alloc(reg); + if (!reg->alloc) + return -ENOMEM; + } + + chunk = reg->alloc->alloc(reg, size, alignment); + if (!chunk) + return -ENOMEM; + + if (unlikely(__cma_chunk_insert(chunk) < 0)) { + /* We should *never* be here. */ + chunk->reg->alloc->free(chunk); + kfree(chunk); + return -EADDRINUSE; + } + + chunk->reg = reg; + ++reg->users; + reg->free_space -= chunk->size; + pr_debug("allocated at %p\n", (void *)chunk->start); + return chunk->start; +} + +dma_addr_t __must_check +cma_alloc_from_region(struct cma_region *reg, + size_t size, dma_addr_t alignment) +{ + dma_addr_t addr; + + pr_debug("allocate %p/%p from %s\n", + (void *)size, (void *)alignment, + reg ? reg->name ?: "(private)" : "(null)"); + + if (!size || alignment & (alignment - 1) || !reg) + return -EINVAL; + + mutex_lock(&cma_mutex); + + addr = reg->registered ? + __cma_alloc_from_region(reg, PAGE_ALIGN(size), + max(alignment, (dma_addr_t)PAGE_SIZE)) : + -EINVAL; + + mutex_unlock(&cma_mutex); + + return addr; +} +EXPORT_SYMBOL_GPL(cma_alloc_from_region); + +dma_addr_t __must_check +__cma_alloc(const struct device *dev, const char *type, + dma_addr_t size, dma_addr_t alignment) +{ + struct cma_region *reg; + const char *from; + dma_addr_t addr; + + if (dev) + pr_debug("allocate %p/%p for %s/%s\n", + (void *)size, (void *)alignment, + dev_name(dev), type ?: ""); + + if (!size || (alignment & ~alignment)) + return -EINVAL; + + if (alignment < PAGE_SIZE) + alignment = PAGE_SIZE; + + if (!IS_ALIGNED(size, alignment)) + size = ALIGN(size, alignment); + + mutex_lock(&cma_mutex); + + from = __cma_where_from(dev, type); + if (unlikely(IS_ERR(from))) { + addr = PTR_ERR(from); + goto done; + } + + pr_debug("allocate %p/%p from one of %s\n", + (void *)size, (void *)alignment, from); + + while (*from && *from != ';') { + reg = __cma_region_find(&from); + addr = __cma_alloc_from_region(reg, size, alignment); + if (!IS_ERR_VALUE(addr)) + goto done; + } + + pr_debug("not enough memory\n"); + addr = -ENOMEM; + +done: + mutex_unlock(&cma_mutex); + + return addr; +} +EXPORT_SYMBOL_GPL(__cma_alloc); + + +void *cma_get_virt(dma_addr_t phys, dma_addr_t size, int noncached) +{ + unsigned long num_pages, i; + struct page **pages; + void *virt; + + if (noncached) { + num_pages = size >> PAGE_SHIFT; + pages = kmalloc(num_pages * sizeof(struct page *), GFP_KERNEL); + + if (!pages) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < num_pages; i++) + pages[i] = pfn_to_page((phys >> PAGE_SHIFT) + i); + + virt = vmap(pages, num_pages, VM_MAP, + pgprot_writecombine(PAGE_KERNEL)); + + if (!virt) { + kfree(pages); + return ERR_PTR(-ENOMEM); + } + + kfree(pages); + } else { + virt = phys_to_virt((unsigned long)phys); + } + + return virt; +} +EXPORT_SYMBOL_GPL(cma_get_virt); + +/* Query information about regions. */ +static void __cma_info_add(struct cma_info *infop, struct cma_region *reg) +{ + infop->total_size += reg->size; + infop->free_size += reg->free_space; + if (infop->lower_bound > reg->start) + infop->lower_bound = reg->start; + if (infop->upper_bound < reg->start + reg->size) + infop->upper_bound = reg->start + reg->size; + ++infop->count; +} + +int +__cma_info(struct cma_info *infop, const struct device *dev, const char *type) +{ + struct cma_info info = { ~(dma_addr_t)0, 0, 0, 0, 0 }; + struct cma_region *reg; + const char *from; + int ret; + + if (unlikely(!infop)) + return -EINVAL; + + mutex_lock(&cma_mutex); + + from = __cma_where_from(dev, type); + if (IS_ERR(from)) { + ret = PTR_ERR(from); + info.lower_bound = 0; + goto done; + } + + while (*from && *from != ';') { + reg = __cma_region_find(&from); + if (reg) + __cma_info_add(&info, reg); + } + + ret = 0; +done: + mutex_unlock(&cma_mutex); + + memcpy(infop, &info, sizeof info); + return ret; +} +EXPORT_SYMBOL_GPL(__cma_info); + + +/* Freeing. */ +int cma_free(dma_addr_t addr) +{ + struct cma_chunk *c; + int ret; + + mutex_lock(&cma_mutex); + + c = __cma_chunk_find(addr); + + if (c) { + __cma_chunk_free(c); + ret = 0; + } else { + ret = -ENOENT; + } + + mutex_unlock(&cma_mutex); + + if (c) + pr_debug("free(%p): freed\n", (void *)addr); + else + pr_err("free(%p): not found\n", (void *)addr); + return ret; +} +EXPORT_SYMBOL_GPL(cma_free); + + +/************************* Miscellaneous *************************/ + +static int __cma_region_attach_alloc(struct cma_region *reg) +{ + struct cma_allocator *alloc; + int ret; + + /* + * If reg->alloc is set then caller wants us to use this + * allocator. Otherwise we need to find one by name. + */ + if (reg->alloc) { + alloc = reg->alloc; + } else { + alloc = __cma_allocator_find(reg->alloc_name); + if (!alloc) { + pr_warn("init: %s: %s: no such allocator\n", + reg->name ?: "(private)", + reg->alloc_name ?: "(default)"); + reg->used = 1; + return -ENOENT; + } + } + + /* Try to initialise the allocator. */ + reg->private_data = NULL; + ret = alloc->init ? alloc->init(reg) : 0; + if (unlikely(ret < 0)) { + pr_err("init: %s: %s: unable to initialise allocator\n", + reg->name ?: "(private)", alloc->name ?: "(unnamed)"); + reg->alloc = NULL; + reg->used = 1; + } else { + reg->alloc = alloc; + pr_debug("init: %s: %s: initialised allocator\n", + reg->name ?: "(private)", alloc->name ?: "(unnamed)"); + } + return ret; +} + +static void __cma_region_detach_alloc(struct cma_region *reg) +{ + if (!reg->alloc) + return; + + if (reg->alloc->cleanup) + reg->alloc->cleanup(reg); + + reg->alloc = NULL; + reg->used = 1; +} + + +/* + * s ::= rules + * rules ::= rule [ ';' rules ] + * rule ::= patterns '=' regions + * patterns ::= pattern [ ',' patterns ] + * regions ::= REG-NAME [ ',' regions ] + * pattern ::= dev-pattern [ '/' TYPE-NAME ] | '/' TYPE-NAME + */ +static const char *__must_check +__cma_where_from(const struct device *dev, const char *type) +{ + /* + * This function matches the pattern from the map attribute + * agains given device name and type. Type may be of course + * NULL or an emtpy string. + */ + + const char *s, *name; + int name_matched = 0; + + /* + * If dev is NULL we were called in alternative form where + * type is the from string. All we have to do is return it. + */ + if (!dev) + return type ?: ERR_PTR(-EINVAL); + + if (!cma_map) + return ERR_PTR(-ENOENT); + + name = dev_name(dev); + if (WARN_ON(!name || !*name)) + return ERR_PTR(-EINVAL); + + if (!type) + type = "common"; + + /* + * Now we go throught the cma_map attribute. + */ + for (s = cma_map; *s; ++s) { + const char *c; + + /* + * If the pattern starts with a slash, the device part of the + * pattern matches if it matched previously. + */ + if (*s == '/') { + if (!name_matched) + goto look_for_next; + goto match_type; + } + + /* + * We are now trying to match the device name. This also + * updates the name_matched variable. If, while reading the + * spec, we ecnounter comma it means that the pattern does not + * match and we need to start over with another pattern (the + * one afther the comma). If we encounter equal sign we need + * to start over with another rule. If there is a character + * that does not match, we neet to look for a comma (to get + * another pattern) or semicolon (to get another rule) and try + * again if there is one somewhere. + */ + + name_matched = 0; + + for (c = name; *s != '*' && *c; ++c, ++s) + if (*s == '=') + goto next_rule; + else if (*s == ',') + goto next_pattern; + else if (*s != '?' && *c != *s) + goto look_for_next; + if (*s == '*') + ++s; + + name_matched = 1; + + /* + * Now we need to match the type part of the pattern. If the + * pattern is missing it we match only if type points to an + * empty string. Otherwise wy try to match it just like name. + */ + if (*s == '/') { +match_type: /* s points to '/' */ + ++s; + + for (c = type; *s && *c; ++c, ++s) + if (*s == '=') + goto next_rule; + else if (*s == ',') + goto next_pattern; + else if (*c != *s) + goto look_for_next; + } + + /* Return the string behind the '=' sign of the rule. */ + if (*s == '=') + return s + 1; + else if (*s == ',') + return strchr(s, '=') + 1; + + /* Pattern did not match */ + +look_for_next: + do { + ++s; + } while (*s != ',' && *s != '='); + if (*s == ',') + continue; + +next_rule: /* s points to '=' */ + s = strchr(s, ';'); + if (!s) + break; + +next_pattern: + continue; + } + + return ERR_PTR(-ENOENT); +} diff --git a/mm/compaction-cma.c b/mm/compaction-cma.c new file mode 100644 index 0000000..f3ce63a --- /dev/null +++ b/mm/compaction-cma.c @@ -0,0 +1,860 @@ +/* + * linux/mm/compaction.c + * + * Memory compaction for the reduction of external fragmentation. Note that + * this heavily depends upon page migration to do all the real heavy + * lifting + * + * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie> + */ +#include <linux/swap.h> +#include <linux/migrate.h> +#include <linux/compaction.h> +#include <linux/mm_inline.h> +#include <linux/backing-dev.h> +#include <linux/sysctl.h> +#include <linux/sysfs.h> +#include "internal.h" + +#if defined CONFIG_COMPACTION || defined CONFIG_DMA_CMA + +#define CREATE_TRACE_POINTS +#include <trace/events/compaction.h> + +static unsigned long release_freepages(struct list_head *freelist) +{ + struct page *page, *next; + unsigned long count = 0; + + list_for_each_entry_safe(page, next, freelist, lru) { + list_del(&page->lru); + __free_page(page); + count++; + } + + return count; +} + +static void map_pages(struct list_head *list) +{ + struct page *page; + + list_for_each_entry(page, list, lru) { + arch_alloc_page(page, 0); + kernel_map_pages(page, 1, 1); + } +} + +static inline bool migrate_async_suitable(int migratetype) +{ + return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE; +} + +/* + * Isolate free pages onto a private freelist. Caller must hold zone->lock. + * If @strict is true, will abort returning 0 on any invalid PFNs or non-free + * pages inside of the pageblock (even though it may still end up isolating + * some pages). + */ +static unsigned long isolate_freepages_block(unsigned long blockpfn, + unsigned long end_pfn, + struct list_head *freelist, + bool strict, bool for_cma) +{ + int nr_scanned = 0, total_isolated = 0; + struct page *cursor; + + cursor = pfn_to_page(blockpfn); + + /* Isolate free pages. This assumes the block is valid */ + for (; blockpfn < end_pfn; blockpfn++, cursor++) { + int isolated, i; + struct page *page = cursor; + + if (!pfn_valid_within(blockpfn)) { + if (strict) + return 0; + continue; + } + nr_scanned++; + + if (!PageBuddy(page)) { + if (strict) + return 0; + continue; + } + + /* Found a free page, break it into order-0 pages */ + isolated = split_free_page(page, for_cma); + if (!isolated && strict) + return 0; + total_isolated += isolated; + for (i = 0; i < isolated; i++) { + list_add(&page->lru, freelist); + page++; + } + + /* If a page was split, advance to the end of it */ + if (isolated) { + blockpfn += isolated - 1; + cursor += isolated - 1; + } + } + + trace_mm_compaction_isolate_freepages(nr_scanned, total_isolated); + return total_isolated; +} + +/** + * isolate_freepages_range() - isolate free pages. + * @start_pfn: The first PFN to start isolating. + * @end_pfn: The one-past-last PFN. + * + * Non-free pages, invalid PFNs, or zone boundaries within the + * [start_pfn, end_pfn) range are considered errors, cause function to + * undo its actions and return zero. + * + * Otherwise, function returns one-past-the-last PFN of isolated page + * (which may be greater then end_pfn if end fell in a middle of + * a free page). + */ +unsigned long +isolate_freepages_range(unsigned long start_pfn, unsigned long end_pfn, + bool for_cma) +{ + unsigned long isolated, pfn, block_end_pfn, flags; + struct zone *zone = NULL; + LIST_HEAD(freelist); + + if (pfn_valid(start_pfn)) + zone = page_zone(pfn_to_page(start_pfn)); + + for (pfn = start_pfn; pfn < end_pfn; pfn += isolated) { + if (!pfn_valid(pfn) || zone != page_zone(pfn_to_page(pfn))) + break; + + /* + * On subsequent iterations ALIGN() is actually not needed, + * but we keep it that we not to complicate the code. + */ + block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); + block_end_pfn = min(block_end_pfn, end_pfn); + + spin_lock_irqsave(&zone->lock, flags); + isolated = isolate_freepages_block(pfn, block_end_pfn, + &freelist, true, for_cma); + spin_unlock_irqrestore(&zone->lock, flags); + + /* + * In strict mode, isolate_freepages_block() returns 0 if + * there are any holes in the block (ie. invalid PFNs or + * non-free pages). + */ + if (!isolated) + break; + + /* + * If we managed to isolate pages, it is always (1 << n) * + * pageblock_nr_pages for some non-negative n. (Max order + * page may span two pageblocks). + */ + } + + /* split_free_page does not map the pages */ + map_pages(&freelist); + + if (pfn < end_pfn) { + /* Loop terminated early, cleanup. */ + release_freepages(&freelist); + return 0; + } + + /* We don't use freelists for anything. */ + return pfn; +} + +/* Update the number of anon and file isolated pages in the zone */ +static void acct_isolated(struct zone *zone, struct compact_control *cc) +{ + struct page *page; + unsigned int count[2] = { 0, }; + + list_for_each_entry(page, &cc->migratepages, lru) + count[!!page_is_file_cache(page)]++; + + __mod_zone_page_state(zone, NR_ISOLATED_ANON, count[0]); + __mod_zone_page_state(zone, NR_ISOLATED_FILE, count[1]); +} + +/* Similar to reclaim, but different enough that they don't share logic */ +static bool too_many_isolated(struct zone *zone) +{ + unsigned long active, inactive, isolated; + + inactive = zone_page_state(zone, NR_INACTIVE_FILE) + + zone_page_state(zone, NR_INACTIVE_ANON); + active = zone_page_state(zone, NR_ACTIVE_FILE) + + zone_page_state(zone, NR_ACTIVE_ANON); + isolated = zone_page_state(zone, NR_ISOLATED_FILE) + + zone_page_state(zone, NR_ISOLATED_ANON); + + return isolated > (inactive + active) / 2; +} + +/** + * isolate_migratepages_range() - isolate all migrate-able pages in range. + * @zone: Zone pages are in. + * @cc: Compaction control structure. + * @low_pfn: The first PFN of the range. + * @end_pfn: The one-past-the-last PFN of the range. + * + * Isolate all pages that can be migrated from the range specified by + * [low_pfn, end_pfn). Returns zero if there is a fatal signal + * pending), otherwise PFN of the first page that was not scanned + * (which may be both less, equal to or more then end_pfn). + * + * Assumes that cc->migratepages is empty and cc->nr_migratepages is + * zero. + * + * Apart from cc->migratepages and cc->nr_migratetypes this function + * does not modify any cc's fields, in particular it does not modify + * (or read for that matter) cc->migrate_pfn. + */ +unsigned long +isolate_migratepages_range(struct zone *zone, struct compact_control *cc, + unsigned long low_pfn, unsigned long end_pfn) +{ + unsigned long last_pageblock_nr = 0, pageblock_nr; + unsigned long nr_scanned = 0, nr_isolated = 0; + struct list_head *migratelist = &cc->migratepages; + + /* + * Ensure that there are not too many pages isolated from the LRU + * list by either parallel reclaimers or compaction. If there are, + * delay for some time until fewer pages are isolated + */ + while (unlikely(too_many_isolated(zone))) { + /* async migration should just abort */ + if (!cc->sync) + return 0; + + congestion_wait(BLK_RW_ASYNC, HZ/10); + + if (fatal_signal_pending(current)) + return 0; + } + + /* Time to isolate some pages for migration */ + cond_resched(); + spin_lock_irq(&zone->lru_lock); + for (; low_pfn < end_pfn; low_pfn++) { + struct page *page; + bool locked = true; + + /* give a chance to irqs before checking need_resched() */ + if (!((low_pfn+1) % SWAP_CLUSTER_MAX)) { + spin_unlock_irq(&zone->lru_lock); + locked = false; + } + if (need_resched() || spin_is_contended(&zone->lru_lock)) { + if (locked) + spin_unlock_irq(&zone->lru_lock); + cond_resched(); + spin_lock_irq(&zone->lru_lock); + if (fatal_signal_pending(current)) + break; + } else if (!locked) + spin_lock_irq(&zone->lru_lock); + + if (!pfn_valid_within(low_pfn)) + continue; + nr_scanned++; + + /* Get the page and skip if free */ + page = pfn_to_page(low_pfn); + if (PageBuddy(page)) + continue; + + /* + * For async migration, also only scan in MOVABLE blocks. Async + * migration is optimistic to see if the minimum amount of work + * satisfies the allocation + */ + pageblock_nr = low_pfn >> pageblock_order; + if (!cc->sync && last_pageblock_nr != pageblock_nr && + !migrate_async_suitable(get_pageblock_migratetype(page))) { + low_pfn += pageblock_nr_pages; + low_pfn = ALIGN(low_pfn, pageblock_nr_pages) - 1; + last_pageblock_nr = pageblock_nr; + continue; + } + + if (!PageLRU(page)) + continue; + + /* + * PageLRU is set, and lru_lock excludes isolation, + * splitting and collapsing (collapsing has already + * happened if PageLRU is set). + */ + if (PageTransHuge(page)) { + low_pfn += (1 << compound_order(page)) - 1; + continue; + } + + /* Try isolate the page */ + if (__isolate_lru_page(page, + ISOLATE_ACTIVE|ISOLATE_INACTIVE, 0) != 0) + continue; + + VM_BUG_ON(PageTransCompound(page)); + + /* Successfully isolated */ + del_page_from_lru_list(zone, page, page_lru(page)); + list_add(&page->lru, migratelist); + cc->nr_migratepages++; + nr_isolated++; + + /* Avoid isolating too much */ + if (cc->nr_migratepages == COMPACT_CLUSTER_MAX) { + ++low_pfn; + break; + } + } + + acct_isolated(zone, cc); + + spin_unlock_irq(&zone->lru_lock); + + trace_mm_compaction_isolate_migratepages(nr_scanned, nr_isolated); + + return low_pfn; +} + +#endif /* CONFIG_COMPACTION || CONFIG_DMA_CMA */ +#ifdef CONFIG_COMPACTION + +/* Returns true if the page is within a block suitable for migration to */ +static bool suitable_migration_target(struct page *page) +{ + + int migratetype = get_pageblock_migratetype(page); + + /* Don't interfere with memory hot-remove */ + /* or the min_free_kbytes blocks */ + if (migratetype == MIGRATE_ISOLATE || migratetype == MIGRATE_RESERVE) + return false; + + /* If the page is a large free page, then allow migration */ + if (PageBuddy(page) && page_order(page) >= pageblock_order) + return true; + + /* If the block is MIGRATE_MOVABLE or MIGRATE_CMA, allow migration */ + if (migrate_async_suitable(migratetype)) + return true; + + /* Otherwise skip the block */ + return false; +} + +/* + * Based on information in the current compact_control, find blocks + * suitable for isolating free pages from and then isolate them. + */ +static void isolate_freepages(struct zone *zone, + struct compact_control *cc) +{ + struct page *page; + unsigned long high_pfn, low_pfn, pfn, zone_end_pfn, end_pfn; + unsigned long flags; + int nr_freepages = cc->nr_freepages; + struct list_head *freelist = &cc->freepages; + + /* + * Initialise the free scanner. The starting point is where we last + * scanned from (or the end of the zone if starting). The low point + * is the end of the pageblock the migration scanner is using. + */ + pfn = cc->free_pfn; + low_pfn = cc->migrate_pfn + pageblock_nr_pages; + + /* + * Take care that if the migration scanner is at the end of the zone + * that the free scanner does not accidentally move to the next zone + * in the next isolation cycle. + */ + high_pfn = min(low_pfn, pfn); + + zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages; + + /* + * Isolate free pages until enough are available to migrate the + * pages on cc->migratepages. We stop searching if the migrate + * and free page scanners meet or enough free pages are isolated. + */ + for (; pfn > low_pfn && cc->nr_migratepages > nr_freepages; + pfn -= pageblock_nr_pages) { + unsigned long isolated; + + if (!pfn_valid(pfn)) + continue; + + /* + * Check for overlapping nodes/zones. It's possible on some + * configurations to have a setup like + * node0 node1 node0 + * i.e. it's possible that all pages within a zones range of + * pages do not belong to a single zone. + */ + page = pfn_to_page(pfn); + if (page_zone(page) != zone) + continue; + + /* Check the block is suitable for migration */ + if (!suitable_migration_target(page)) + continue; + + /* + * Found a block suitable for isolating free pages from. Now + * we disabled interrupts, double check things are ok and + * isolate the pages. This is to minimise the time IRQs + * are disabled + */ + isolated = 0; + spin_lock_irqsave(&zone->lock, flags); + if (suitable_migration_target(page)) { + end_pfn = min(pfn + pageblock_nr_pages, zone_end_pfn); + isolated = isolate_freepages_block(pfn, end_pfn, + freelist, false, false); + nr_freepages += isolated; + } + spin_unlock_irqrestore(&zone->lock, flags); + + /* + * Record the highest PFN we isolated pages from. When next + * looking for free pages, the search will restart here as + * page migration may have returned some pages to the allocator + */ + if (isolated) + high_pfn = max(high_pfn, pfn); + } + + /* split_free_page does not map the pages */ + map_pages(freelist); + + cc->free_pfn = high_pfn; + cc->nr_freepages = nr_freepages; +} + +/* + * This is a migrate-callback that "allocates" freepages by taking pages + * from the isolated freelists in the block we are migrating to. + */ +static struct page *compaction_alloc(struct page *migratepage, + unsigned long data, + int **result) +{ + struct compact_control *cc = (struct compact_control *)data; + struct page *freepage; + + /* Isolate free pages if necessary */ + if (list_empty(&cc->freepages)) { + isolate_freepages(cc->zone, cc); + + if (list_empty(&cc->freepages)) + return NULL; + } + + freepage = list_entry(cc->freepages.next, struct page, lru); + list_del(&freepage->lru); + cc->nr_freepages--; + + return freepage; +} + +/* + * We cannot control nr_migratepages and nr_freepages fully when migration is + * running as migrate_pages() has no knowledge of compact_control. When + * migration is complete, we count the number of pages on the lists by hand. + */ +static void update_nr_listpages(struct compact_control *cc) +{ + int nr_migratepages = 0; + int nr_freepages = 0; + struct page *page; + + list_for_each_entry(page, &cc->migratepages, lru) + nr_migratepages++; + list_for_each_entry(page, &cc->freepages, lru) + nr_freepages++; + + cc->nr_migratepages = nr_migratepages; + cc->nr_freepages = nr_freepages; +} + +/* possible outcome of isolate_migratepages */ +typedef enum { + ISOLATE_ABORT, /* Abort compaction now */ + ISOLATE_NONE, /* No pages isolated, continue scanning */ + ISOLATE_SUCCESS, /* Pages isolated, migrate */ +} isolate_migrate_t; + +/* + * Isolate all pages that can be migrated from the block pointed to by + * the migrate scanner within compact_control. + */ +static isolate_migrate_t isolate_migratepages(struct zone *zone, + struct compact_control *cc) +{ + unsigned long low_pfn, end_pfn; + + /* Do not scan outside zone boundaries */ + low_pfn = max(cc->migrate_pfn, zone->zone_start_pfn); + + /* Only scan within a pageblock boundary */ + end_pfn = ALIGN(low_pfn + pageblock_nr_pages, pageblock_nr_pages); + + /* Do not cross the free scanner or scan within a memory hole */ + if (end_pfn > cc->free_pfn || !pfn_valid(low_pfn)) { + cc->migrate_pfn = end_pfn; + return ISOLATE_NONE; + } + + /* Perform the isolation */ + low_pfn = isolate_migratepages_range(zone, cc, low_pfn, end_pfn); + if (!low_pfn) + return ISOLATE_ABORT; + + cc->migrate_pfn = low_pfn; + + return ISOLATE_SUCCESS; +} + +static int compact_finished(struct zone *zone, + struct compact_control *cc) +{ + unsigned int order; + unsigned long watermark; + + if (fatal_signal_pending(current)) + return COMPACT_PARTIAL; + + /* Compaction run completes if the migrate and free scanner meet */ + if (cc->free_pfn <= cc->migrate_pfn) + return COMPACT_COMPLETE; + + /* + * order == -1 is expected when compacting via + * /proc/sys/vm/compact_memory + */ + if (cc->order == -1) + return COMPACT_CONTINUE; + + /* Compaction run is not finished if the watermark is not met */ + watermark = low_wmark_pages(zone); + watermark += (1 << cc->order); + + if (!zone_watermark_ok(zone, cc->order, watermark, 0, 0)) + return COMPACT_CONTINUE; + + /* Direct compactor: Is a suitable page free? */ + for (order = cc->order; order < MAX_ORDER; order++) { + /* Job done if page is free of the right migratetype */ + if (!list_empty(&zone->free_area[order].free_list[cc->migratetype])) + return COMPACT_PARTIAL; + + /* Job done if allocation would set block type */ + if (order >= pageblock_order && zone->free_area[order].nr_free) + return COMPACT_PARTIAL; + } + + return COMPACT_CONTINUE; +} + +/* + * compaction_suitable: Is this suitable to run compaction on this zone now? + * Returns + * COMPACT_SKIPPED - If there are too few free pages for compaction + * COMPACT_PARTIAL - If the allocation would succeed without compaction + * COMPACT_CONTINUE - If compaction should run now + */ +unsigned long compaction_suitable(struct zone *zone, int order) +{ + int fragindex; + unsigned long watermark; + + /* + * order == -1 is expected when compacting via + * /proc/sys/vm/compact_memory + */ + if (order == -1) + return COMPACT_CONTINUE; + + /* + * Watermarks for order-0 must be met for compaction. Note the 2UL. + * This is because during migration, copies of pages need to be + * allocated and for a short time, the footprint is higher + */ + watermark = low_wmark_pages(zone) + (2UL << order); + if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) + return COMPACT_SKIPPED; + + /* + * fragmentation index determines if allocation failures are due to + * low memory or external fragmentation + * + * index of -1000 implies allocations might succeed depending on + * watermarks + * index towards 0 implies failure is due to lack of memory + * index towards 1000 implies failure is due to fragmentation + * + * Only compact if a failure would be due to fragmentation. + */ + fragindex = fragmentation_index(zone, order); + if (fragindex >= 0 && fragindex <= sysctl_extfrag_threshold) + return COMPACT_SKIPPED; + + if (fragindex == -1000 && zone_watermark_ok(zone, order, watermark, + 0, 0)) + return COMPACT_PARTIAL; + + return COMPACT_CONTINUE; +} + +static int compact_zone(struct zone *zone, struct compact_control *cc) +{ + int ret; + + ret = compaction_suitable(zone, cc->order); + switch (ret) { + case COMPACT_PARTIAL: + case COMPACT_SKIPPED: + /* Compaction is likely to fail */ + return ret; + case COMPACT_CONTINUE: + /* Fall through to compaction */ + ; + } + + /* Setup to move all movable pages to the end of the zone */ + cc->migrate_pfn = zone->zone_start_pfn; + cc->free_pfn = cc->migrate_pfn + zone->spanned_pages; + cc->free_pfn &= ~(pageblock_nr_pages-1); + + migrate_prep_local(); + + while ((ret = compact_finished(zone, cc)) == COMPACT_CONTINUE) { + unsigned long nr_migrate, nr_remaining; + int err; + + switch (isolate_migratepages(zone, cc)) { + case ISOLATE_ABORT: + ret = COMPACT_PARTIAL; + goto out; + case ISOLATE_NONE: + continue; + case ISOLATE_SUCCESS: + ; + } + + nr_migrate = cc->nr_migratepages; + err = migrate_pages(&cc->migratepages, compaction_alloc, + (unsigned long)cc, false, + cc->sync ? MIGRATE_SYNC_LIGHT : MIGRATE_ASYNC + , 0); + update_nr_listpages(cc); + nr_remaining = cc->nr_migratepages; + + count_vm_event(COMPACTBLOCKS); + count_vm_events(COMPACTPAGES, nr_migrate - nr_remaining); + if (nr_remaining) + count_vm_events(COMPACTPAGEFAILED, nr_remaining); + trace_mm_compaction_migratepages(nr_migrate - nr_remaining, + nr_remaining); + + /* Release LRU pages not migrated */ + if (err) { + putback_lru_pages(&cc->migratepages); + cc->nr_migratepages = 0; + if (err == -ENOMEM) { + ret = COMPACT_PARTIAL; + goto out; + } + } + + } + +out: + /* Release free pages and check accounting */ + cc->nr_freepages -= release_freepages(&cc->freepages); + VM_BUG_ON(cc->nr_freepages != 0); + + return ret; +} + +static unsigned long compact_zone_order(struct zone *zone, + int order, gfp_t gfp_mask, + bool sync) +{ + struct compact_control cc = { + .nr_freepages = 0, + .nr_migratepages = 0, + .order = order, + .migratetype = allocflags_to_migratetype(gfp_mask), + .zone = zone, + .sync = sync, + }; + INIT_LIST_HEAD(&cc.freepages); + INIT_LIST_HEAD(&cc.migratepages); + + return compact_zone(zone, &cc); +} + +int sysctl_extfrag_threshold = 500; + +/** + * try_to_compact_pages - Direct compact to satisfy a high-order allocation + * @zonelist: The zonelist used for the current allocation + * @order: The order of the current allocation + * @gfp_mask: The GFP mask of the current allocation + * @nodemask: The allowed nodes to allocate from + * @sync: Whether migration is synchronous or not + * + * This is the main entry point for direct page compaction. + */ +unsigned long try_to_compact_pages(struct zonelist *zonelist, + int order, gfp_t gfp_mask, nodemask_t *nodemask, + bool sync) +{ + enum zone_type high_zoneidx = gfp_zone(gfp_mask); + int may_enter_fs = gfp_mask & __GFP_FS; + int may_perform_io = gfp_mask & __GFP_IO; + struct zoneref *z; + struct zone *zone; + int rc = COMPACT_SKIPPED; + + /* + * Check whether it is worth even starting compaction + * The order check is made because an assumption is made + * that the page allocator can satisfy the "cheaper" orders + * without taking special steps + */ + if (!order || !may_enter_fs || !may_perform_io) + return rc; + +#ifdef CONFIG_MACH_Q1_BD + /* Temporary log to get information whether the compaction works well */ + printk(KERN_NOTICE "%s, order=%d, sync=%d\n", __func__, order, sync); +#endif + count_vm_event(COMPACTSTALL); + + /* Compact each zone in the list */ + for_each_zone_zonelist_nodemask(zone, z, zonelist, high_zoneidx, + nodemask) { + int status; + + status = compact_zone_order(zone, order, gfp_mask, sync); + rc = max(status, rc); + + /* If a normal allocation would succeed, stop compacting */ + if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0, 0)) + break; + } + + return rc; +} + + +/* Compact all zones within a node */ +static int compact_node(int nid) +{ + int zoneid; + pg_data_t *pgdat; + struct zone *zone; + + if (nid < 0 || nid >= nr_node_ids || !node_online(nid)) + return -EINVAL; + pgdat = NODE_DATA(nid); + + /* Flush pending updates to the LRU lists */ + lru_add_drain_all(); + + for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { + struct compact_control cc = { + .nr_freepages = 0, + .nr_migratepages = 0, + .order = -1, + }; + + zone = &pgdat->node_zones[zoneid]; + if (!populated_zone(zone)) + continue; + + cc.zone = zone; + INIT_LIST_HEAD(&cc.freepages); + INIT_LIST_HEAD(&cc.migratepages); + + compact_zone(zone, &cc); + + VM_BUG_ON(!list_empty(&cc.freepages)); + VM_BUG_ON(!list_empty(&cc.migratepages)); + } + + return 0; +} + +/* Compact all nodes in the system */ +static int compact_nodes(void) +{ + int nid; + + for_each_online_node(nid) + compact_node(nid); + + return COMPACT_COMPLETE; +} + +/* The written value is actually unused, all memory is compacted */ +int sysctl_compact_memory; + +/* This is the entry point for compacting all nodes via /proc/sys/vm */ +int sysctl_compaction_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *length, loff_t *ppos) +{ + if (write) + return compact_nodes(); + + return 0; +} + +int sysctl_extfrag_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *length, loff_t *ppos) +{ + proc_dointvec_minmax(table, write, buffer, length, ppos); + + return 0; +} + +#if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) +ssize_t sysfs_compact_node(struct sys_device *dev, + struct sysdev_attribute *attr, + const char *buf, size_t count) +{ + compact_node(dev->id); + + return count; +} +static SYSDEV_ATTR(compact, S_IWUSR, NULL, sysfs_compact_node); + +int compaction_register_node(struct node *node) +{ + return sysdev_create_file(&node->sysdev, &attr_compact); +} + +void compaction_unregister_node(struct node *node) +{ + return sysdev_remove_file(&node->sysdev, &attr_compact); +} +#endif /* CONFIG_SYSFS && CONFIG_NUMA */ + +#endif /* CONFIG_COMPACTION */ diff --git a/mm/compaction.c b/mm/compaction.c index 5f8ec82..f550ecf 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -607,7 +607,7 @@ out: return ret; } -static unsigned long compact_zone_order(struct zone *zone, +unsigned long compact_zone_order(struct zone *zone, int order, gfp_t gfp_mask, bool sync) { @@ -656,6 +656,10 @@ unsigned long try_to_compact_pages(struct zonelist *zonelist, if (!order || !may_enter_fs || !may_perform_io) return rc; +#ifdef CONFIG_MACH_Q1_BD + /* Temporary log to get information whether the compaction works well */ + printk(KERN_NOTICE "%s, order=%d, sync=%d\n", __func__, order, sync); +#endif count_vm_event(COMPACTSTALL); /* Compact each zone in the list */ diff --git a/mm/debug-pagealloc.c b/mm/debug-pagealloc.c index 7cea557..a1e3324 100644 --- a/mm/debug-pagealloc.c +++ b/mm/debug-pagealloc.c @@ -1,10 +1,7 @@ #include <linux/kernel.h> -#include <linux/string.h> #include <linux/mm.h> -#include <linux/highmem.h> #include <linux/page-debug-flags.h> #include <linux/poison.h> -#include <linux/ratelimit.h> static inline void set_page_poison(struct page *page) { @@ -21,13 +18,28 @@ static inline bool page_poison(struct page *page) return test_bit(PAGE_DEBUG_FLAG_POISON, &page->debug_flags); } +static void poison_highpage(struct page *page) +{ + /* + * Page poisoning for highmem pages is not implemented. + * + * This can be called from interrupt contexts. + * So we need to create a new kmap_atomic slot for this + * application and it will need interrupt protection. + */ +} + static void poison_page(struct page *page) { - void *addr = kmap_atomic(page); + void *addr; + if (PageHighMem(page)) { + poison_highpage(page); + return; + } set_page_poison(page); + addr = page_address(page); memset(addr, PAGE_POISON, PAGE_SIZE); - kunmap_atomic(addr); } static void poison_pages(struct page *page, int n) @@ -47,12 +59,14 @@ static bool single_bit_flip(unsigned char a, unsigned char b) static void check_poison_mem(unsigned char *mem, size_t bytes) { - static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 10); unsigned char *start; unsigned char *end; - start = memchr_inv(mem, PAGE_POISON, bytes); - if (!start) + for (start = mem; start < mem + bytes; start++) { + if (*start != PAGE_POISON) + break; + } + if (start == mem + bytes) return; for (end = mem + bytes - 1; end > start; end--) { @@ -60,7 +74,7 @@ static void check_poison_mem(unsigned char *mem, size_t bytes) break; } - if (!__ratelimit(&ratelimit)) + if (!printk_ratelimit()) return; else if (start == end && single_bit_flip(*start, PAGE_POISON)) printk(KERN_ERR "pagealloc: single bit error\n"); @@ -72,17 +86,27 @@ static void check_poison_mem(unsigned char *mem, size_t bytes) dump_stack(); } -static void unpoison_page(struct page *page) +static void unpoison_highpage(struct page *page) { - void *addr; + /* + * See comment in poison_highpage(). + * Highmem pages should not be poisoned for now + */ + BUG_ON(page_poison(page)); +} - if (!page_poison(page)) +static void unpoison_page(struct page *page) +{ + if (PageHighMem(page)) { + unpoison_highpage(page); return; + } + if (page_poison(page)) { + void *addr = page_address(page); - addr = kmap_atomic(page); - check_poison_mem(addr, PAGE_SIZE); - clear_page_poison(page); - kunmap_atomic(addr); + check_poison_mem(addr, PAGE_SIZE); + clear_page_poison(page); + } } static void unpoison_pages(struct page *page, int n) diff --git a/mm/dmapool.c b/mm/dmapool.c index da1b0f0..f8e675e 100644 --- a/mm/dmapool.c +++ b/mm/dmapool.c @@ -27,12 +27,11 @@ #include <linux/dmapool.h> #include <linux/kernel.h> #include <linux/list.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/mutex.h> #include <linux/poison.h> #include <linux/sched.h> #include <linux/slab.h> -#include <linux/stat.h> #include <linux/spinlock.h> #include <linux/string.h> #include <linux/types.h> @@ -484,7 +483,7 @@ void dmam_pool_destroy(struct dma_pool *pool) { struct device *dev = pool->dev; - WARN_ON(devres_destroy(dev, dmam_pool_release, dmam_pool_match, pool)); dma_pool_destroy(pool); + WARN_ON(devres_destroy(dev, dmam_pool_release, dmam_pool_match, pool)); } EXPORT_SYMBOL(dmam_pool_destroy); diff --git a/mm/failslab.c b/mm/failslab.c index 0dd7b8f..c5f88f2 100644 --- a/mm/failslab.c +++ b/mm/failslab.c @@ -5,6 +5,10 @@ static struct { struct fault_attr attr; u32 ignore_gfp_wait; int cache_filter; +#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS + struct dentry *ignore_gfp_wait_file; + struct dentry *cache_filter_file; +#endif } failslab = { .attr = FAULT_ATTR_INITIALIZER, .ignore_gfp_wait = 1, @@ -34,25 +38,32 @@ __setup("failslab=", setup_failslab); #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS static int __init failslab_debugfs_init(void) { - struct dentry *dir; mode_t mode = S_IFREG | S_IRUSR | S_IWUSR; + struct dentry *dir; + int err; + + err = init_fault_attr_dentries(&failslab.attr, "failslab"); + if (err) + return err; + dir = failslab.attr.dentries.dir; - dir = fault_create_debugfs_attr("failslab", NULL, &failslab.attr); - if (IS_ERR(dir)) - return PTR_ERR(dir); + failslab.ignore_gfp_wait_file = + debugfs_create_bool("ignore-gfp-wait", mode, dir, + &failslab.ignore_gfp_wait); - if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, - &failslab.ignore_gfp_wait)) - goto fail; - if (!debugfs_create_bool("cache-filter", mode, dir, - &failslab.cache_filter)) - goto fail; + failslab.cache_filter_file = + debugfs_create_bool("cache-filter", mode, dir, + &failslab.cache_filter); - return 0; -fail: - debugfs_remove_recursive(dir); + if (!failslab.ignore_gfp_wait_file || + !failslab.cache_filter_file) { + err = -ENOMEM; + debugfs_remove(failslab.cache_filter_file); + debugfs_remove(failslab.ignore_gfp_wait_file); + cleanup_fault_attr_dentries(&failslab.attr); + } - return -ENOMEM; + return err; } late_initcall(failslab_debugfs_init); diff --git a/mm/filemap.c b/mm/filemap.c index 6c009c2..10481eb 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -9,7 +9,7 @@ * most "normal" filesystems (but you don't /have/ to use this: * the NFS filesystem used to do this differently, for example) */ -#include <linux/export.h> +#include <linux/module.h> #include <linux/compiler.h> #include <linux/fs.h> #include <linux/uaccess.h> @@ -33,6 +33,7 @@ #include <linux/cpuset.h> #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */ #include <linux/memcontrol.h> +#include <linux/mm_inline.h> /* for page_is_file_cache() */ #include <linux/cleancache.h> #include "internal.h" @@ -77,7 +78,10 @@ * ->i_mutex (generic_file_buffered_write) * ->mmap_sem (fault_in_pages_readable->do_page_fault) * - * bdi->wb.list_lock + * ->i_mutex + * ->i_alloc_sem (various) + * + * inode_wb_list_lock * sb_lock (fs/fs-writeback.c) * ->mapping->tree_lock (__sync_single_inode) * @@ -95,9 +99,9 @@ * ->zone.lru_lock (check_pte_range->isolate_lru_page) * ->private_lock (page_remove_rmap->set_page_dirty) * ->tree_lock (page_remove_rmap->set_page_dirty) - * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) + * inode_wb_list_lock (page_remove_rmap->set_page_dirty) * ->inode->i_lock (page_remove_rmap->set_page_dirty) - * bdi.wb->list_lock (zap_pte_range->set_page_dirty) + * inode_wb_list_lock (zap_pte_range->set_page_dirty) * ->inode->i_lock (zap_pte_range->set_page_dirty) * ->private_lock (zap_pte_range->__set_page_dirty_buffers) * @@ -127,7 +131,6 @@ void __delete_from_page_cache(struct page *page) radix_tree_delete(&mapping->page_tree, page->index); page->mapping = NULL; - /* Leave page->index set: truncation lookup relies upon it */ mapping->nrpages--; __dec_zone_page_state(page, NR_FILE_PAGES); if (PageSwapBacked(page)) @@ -447,7 +450,6 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping, int error; VM_BUG_ON(!PageLocked(page)); - VM_BUG_ON(PageSwapBacked(page)); error = mem_cgroup_cache_charge(page, current->mm, gfp_mask & GFP_RECLAIM_MASK); @@ -465,10 +467,11 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping, if (likely(!error)) { mapping->nrpages++; __inc_zone_page_state(page, NR_FILE_PAGES); + if (PageSwapBacked(page)) + __inc_zone_page_state(page, NR_SHMEM); spin_unlock_irq(&mapping->tree_lock); } else { page->mapping = NULL; - /* Leave page->index set: truncation relies upon it */ spin_unlock_irq(&mapping->tree_lock); mem_cgroup_uncharge_cache_page(page); page_cache_release(page); @@ -486,9 +489,22 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping, { int ret; + /* + * Splice_read and readahead add shmem/tmpfs pages into the page cache + * before shmem_readpage has a chance to mark them as SwapBacked: they + * need to go on the anon lru below, and mem_cgroup_cache_charge + * (called in add_to_page_cache) needs to know where they're going too. + */ + if (mapping_cap_swap_backed(mapping)) + SetPageSwapBacked(page); + ret = add_to_page_cache(page, mapping, offset, gfp_mask); - if (ret == 0) - lru_cache_add_file(page); + if (ret == 0) { + if (page_is_file_cache(page)) + lru_cache_add_file(page); + else + lru_cache_add_anon(page); + } return ret; } EXPORT_SYMBOL_GPL(add_to_page_cache_lru); @@ -688,16 +704,9 @@ repeat: page = radix_tree_deref_slot(pagep); if (unlikely(!page)) goto out; - if (radix_tree_exception(page)) { - if (radix_tree_deref_retry(page)) - goto repeat; - /* - * Otherwise, shmem/tmpfs must be storing a swap entry - * here as an exceptional entry: so return it without - * attempting to raise page count. - */ - goto out; - } + if (radix_tree_deref_retry(page)) + goto repeat; + if (!page_cache_get_speculative(page)) goto repeat; @@ -734,7 +743,7 @@ struct page *find_lock_page(struct address_space *mapping, pgoff_t offset) repeat: page = find_get_page(mapping, offset); - if (page && !radix_tree_exception(page)) { + if (page) { lock_page(page); /* Has the page been truncated? */ if (unlikely(page->mapping != mapping)) { @@ -816,14 +825,13 @@ unsigned find_get_pages(struct address_space *mapping, pgoff_t start, { unsigned int i; unsigned int ret; - unsigned int nr_found, nr_skip; + unsigned int nr_found; rcu_read_lock(); restart: nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree, - (void ***)pages, NULL, start, nr_pages); + (void ***)pages, start, nr_pages); ret = 0; - nr_skip = 0; for (i = 0; i < nr_found; i++) { struct page *page; repeat: @@ -831,23 +839,13 @@ repeat: if (unlikely(!page)) continue; - if (radix_tree_exception(page)) { - if (radix_tree_deref_retry(page)) { - /* - * Transient condition which can only trigger - * when entry at index 0 moves out of or back - * to root: none yet gotten, safe to restart. - */ - WARN_ON(start | i); - goto restart; - } - /* - * Otherwise, shmem/tmpfs must be storing a swap entry - * here as an exceptional entry: so skip over it - - * we only reach this from invalidate_mapping_pages(). - */ - nr_skip++; - continue; + /* + * This can only trigger when the entry at index 0 moves out + * of or back to the root: none yet gotten, safe to restart. + */ + if (radix_tree_deref_retry(page)) { + WARN_ON(start | i); + goto restart; } if (!page_cache_get_speculative(page)) @@ -867,7 +865,7 @@ repeat: * If all entries were removed before we could secure them, * try again, because callers stop trying once 0 is returned. */ - if (unlikely(!ret && nr_found > nr_skip)) + if (unlikely(!ret && nr_found)) goto restart; rcu_read_unlock(); return ret; @@ -895,7 +893,7 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, rcu_read_lock(); restart: nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree, - (void ***)pages, NULL, index, nr_pages); + (void ***)pages, index, nr_pages); ret = 0; for (i = 0; i < nr_found; i++) { struct page *page; @@ -904,22 +902,12 @@ repeat: if (unlikely(!page)) continue; - if (radix_tree_exception(page)) { - if (radix_tree_deref_retry(page)) { - /* - * Transient condition which can only trigger - * when entry at index 0 moves out of or back - * to root: none yet gotten, safe to restart. - */ - goto restart; - } - /* - * Otherwise, shmem/tmpfs must be storing a swap entry - * here as an exceptional entry: so stop looking for - * contiguous pages. - */ - break; - } + /* + * This can only trigger when the entry at index 0 moves out + * of or back to the root: none yet gotten, safe to restart. + */ + if (radix_tree_deref_retry(page)) + goto restart; if (!page_cache_get_speculative(page)) goto repeat; @@ -979,21 +967,12 @@ repeat: if (unlikely(!page)) continue; - if (radix_tree_exception(page)) { - if (radix_tree_deref_retry(page)) { - /* - * Transient condition which can only trigger - * when entry at index 0 moves out of or back - * to root: none yet gotten, safe to restart. - */ - goto restart; - } - /* - * This function is never used on a shmem/tmpfs - * mapping, so a swap entry won't be found here. - */ - BUG(); - } + /* + * This can only trigger when the entry at index 0 moves out + * of or back to the root: none yet gotten, safe to restart. + */ + if (radix_tree_deref_retry(page)) + goto restart; if (!page_cache_get_speculative(page)) goto repeat; @@ -1805,7 +1784,7 @@ EXPORT_SYMBOL(generic_file_readonly_mmap); static struct page *__read_cache_page(struct address_space *mapping, pgoff_t index, - int (*filler)(void *, struct page *), + int (*filler)(void *,struct page*), void *data, gfp_t gfp) { @@ -1836,7 +1815,7 @@ repeat: static struct page *do_read_cache_page(struct address_space *mapping, pgoff_t index, - int (*filler)(void *, struct page *), + int (*filler)(void *,struct page*), void *data, gfp_t gfp) @@ -1876,7 +1855,7 @@ out: * @mapping: the page's address_space * @index: the page index * @filler: function to perform the read - * @data: first arg to filler(data, page) function, often left as NULL + * @data: destination for read data * * Same as read_cache_page, but don't wait for page to become unlocked * after submitting it to the filler. @@ -1888,7 +1867,7 @@ out: */ struct page *read_cache_page_async(struct address_space *mapping, pgoff_t index, - int (*filler)(void *, struct page *), + int (*filler)(void *,struct page*), void *data) { return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping)); @@ -1933,7 +1912,7 @@ EXPORT_SYMBOL(read_cache_page_gfp); * @mapping: the page's address_space * @index: the page index * @filler: function to perform the read - * @data: first arg to filler(data, page) function, often left as NULL + * @data: destination for read data * * Read into the page cache. If a page already exists, and PageUptodate() is * not set, try to fill the page then wait for it to become unlocked. @@ -1942,7 +1921,7 @@ EXPORT_SYMBOL(read_cache_page_gfp); */ struct page *read_cache_page(struct address_space *mapping, pgoff_t index, - int (*filler)(void *, struct page *), + int (*filler)(void *,struct page*), void *data) { return wait_on_page_read(read_cache_page_async(mapping, index, filler, data)); @@ -2007,8 +1986,8 @@ int file_remove_suid(struct file *file) error = security_inode_killpriv(dentry); if (!error && killsuid) error = __remove_suid(dentry, killsuid); - if (!error) - inode_has_no_xattr(inode); + if (!error && (inode->i_sb->s_flags & MS_NOSEC)) + inode->i_flags |= S_NOSEC; return error; } @@ -2101,7 +2080,6 @@ void iov_iter_advance(struct iov_iter *i, size_t bytes) } else { const struct iovec *iov = i->iov; size_t base = i->iov_offset; - unsigned long nr_segs = i->nr_segs; /* * The !iov->iov_len check ensures we skip over unlikely @@ -2117,13 +2095,11 @@ void iov_iter_advance(struct iov_iter *i, size_t bytes) base += copy; if (iov->iov_len == base) { iov++; - nr_segs--; base = 0; } } i->iov = iov; i->iov_offset = base; - i->nr_segs = nr_segs; } } EXPORT_SYMBOL(iov_iter_advance); @@ -2393,6 +2369,7 @@ static ssize_t generic_perform_write(struct file *file, iov_iter_count(i)); again: + /* * Bring in the user page that we will copy from _first_. * Otherwise there's a nasty deadlock on copying from the @@ -2448,10 +2425,7 @@ again: written += copied; balance_dirty_pages_ratelimited(mapping); - if (fatal_signal_pending(current)) { - status = -EINTR; - break; - } + } while (iov_iter_count(i)); return written ? written : status; diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c index a4eb311..dee9429 100644 --- a/mm/filemap_xip.c +++ b/mm/filemap_xip.c @@ -10,7 +10,7 @@ #include <linux/fs.h> #include <linux/pagemap.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/uio.h> #include <linux/rmap.h> #include <linux/mmu_notifier.h> diff --git a/mm/fremap.c b/mm/fremap.c index 9ed4fd4..b8e0e2d 100644 --- a/mm/fremap.c +++ b/mm/fremap.c @@ -13,6 +13,7 @@ #include <linux/pagemap.h> #include <linux/swapops.h> #include <linux/rmap.h> +#include <linux/module.h> #include <linux/syscalls.h> #include <linux/mmu_notifier.h> diff --git a/mm/highmem.c b/mm/highmem.c index 09fc744..693394d 100644 --- a/mm/highmem.c +++ b/mm/highmem.c @@ -17,7 +17,7 @@ */ #include <linux/mm.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/swap.h> #include <linux/bio.h> #include <linux/pagemap.h> @@ -94,19 +94,6 @@ static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait); do { spin_unlock(&kmap_lock); (void)(flags); } while (0) #endif -struct page *kmap_to_page(void *vaddr) -{ - unsigned long addr = (unsigned long)vaddr; - - if (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) { - int i = (addr - PKMAP_ADDR(0)) >> PAGE_SHIFT; - return pte_page(pkmap_page_table[i]); - } - - return virt_to_page(addr); -} -EXPORT_SYMBOL(kmap_to_page); - static void flush_all_zero_pkmaps(void) { int i; @@ -263,7 +250,7 @@ void *kmap_high_get(struct page *page) #endif /** - * kunmap_high - unmap a highmem page into memory + * kunmap_high - map a highmem page into memory * @page: &struct page to unmap * * If ARCH_NEEDS_KMAP_HIGH_GET is not defined then this may be called @@ -339,7 +326,7 @@ static struct page_address_slot { spinlock_t lock; /* Protect this bucket's list */ } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER]; -static struct page_address_slot *page_slot(const struct page *page) +static struct page_address_slot *page_slot(struct page *page) { return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)]; } @@ -350,7 +337,7 @@ static struct page_address_slot *page_slot(const struct page *page) * * Returns the page's virtual address. */ -void *page_address(const struct page *page) +void *page_address(struct page *page) { unsigned long flags; void *ret; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 79166c2..78f7186 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -89,8 +89,7 @@ struct khugepaged_scan { struct list_head mm_head; struct mm_slot *mm_slot; unsigned long address; -}; -static struct khugepaged_scan khugepaged_scan = { +} khugepaged_scan = { .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head), }; @@ -682,7 +681,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, if (haddr >= vma->vm_start && haddr + HPAGE_PMD_SIZE <= vma->vm_end) { if (unlikely(anon_vma_prepare(vma))) return VM_FAULT_OOM; - if (unlikely(khugepaged_enter(vma, vma->vm_flags))) + if (unlikely(khugepaged_enter(vma))) return VM_FAULT_OOM; page = alloc_hugepage_vma(transparent_hugepage_defrag(vma), vma, haddr, numa_node_id(), 0); @@ -832,7 +831,7 @@ static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm, for (i = 0; i < HPAGE_PMD_NR; i++) { copy_user_highpage(pages[i], page + i, - haddr + PAGE_SIZE * i, vma); + haddr + PAGE_SHIFT*i, vma); __SetPageUptodate(pages[i]); cond_resched(); } @@ -1058,51 +1057,6 @@ int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, return ret; } -int move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma, - unsigned long old_addr, - unsigned long new_addr, unsigned long old_end, - pmd_t *old_pmd, pmd_t *new_pmd) -{ - int ret = 0; - pmd_t pmd; - - struct mm_struct *mm = vma->vm_mm; - - if ((old_addr & ~HPAGE_PMD_MASK) || - (new_addr & ~HPAGE_PMD_MASK) || - old_end - old_addr < HPAGE_PMD_SIZE || - (new_vma->vm_flags & VM_NOHUGEPAGE)) - goto out; - - /* - * The destination pmd shouldn't be established, free_pgtables() - * should have release it. - */ - if (WARN_ON(!pmd_none(*new_pmd))) { - VM_BUG_ON(pmd_trans_huge(*new_pmd)); - goto out; - } - - spin_lock(&mm->page_table_lock); - if (likely(pmd_trans_huge(*old_pmd))) { - if (pmd_trans_splitting(*old_pmd)) { - spin_unlock(&mm->page_table_lock); - wait_split_huge_page(vma->anon_vma, old_pmd); - ret = -1; - } else { - pmd = pmdp_get_and_clear(mm, old_addr, old_pmd); - VM_BUG_ON(!pmd_none(*new_pmd)); - set_pmd_at(mm, new_addr, new_pmd, pmd); - spin_unlock(&mm->page_table_lock); - ret = 1; - } - } else { - spin_unlock(&mm->page_table_lock); - } -out: - return ret; -} - int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, pgprot_t newprot) { @@ -1493,7 +1447,7 @@ int hugepage_madvise(struct vm_area_struct *vma, * register it here without waiting a page fault that * may not happen any time soon. */ - if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags))) + if (unlikely(khugepaged_enter_vma_merge(vma))) return -ENOMEM; break; case MADV_NOHUGEPAGE: @@ -1625,8 +1579,7 @@ int __khugepaged_enter(struct mm_struct *mm) return 0; } -int khugepaged_enter_vma_merge(struct vm_area_struct *vma, - unsigned long vm_flags) +int khugepaged_enter_vma_merge(struct vm_area_struct *vma) { unsigned long hstart, hend; if (!vma->anon_vma) @@ -1642,11 +1595,11 @@ int khugepaged_enter_vma_merge(struct vm_area_struct *vma, * If is_pfn_mapping() is true is_learn_pfn_mapping() must be * true too, verify it here. */ - VM_BUG_ON(is_linear_pfn_mapping(vma) || vm_flags & VM_NO_THP); + VM_BUG_ON(is_linear_pfn_mapping(vma) || vma->vm_flags & VM_NO_THP); hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; hend = vma->vm_end & HPAGE_PMD_MASK; if (hstart < hend) - return khugepaged_enter(vma, vm_flags); + return khugepaged_enter(vma); return 0; } @@ -1662,13 +1615,14 @@ void __khugepaged_exit(struct mm_struct *mm) list_del(&mm_slot->mm_node); free = 1; } - spin_unlock(&khugepaged_mm_lock); if (free) { + spin_unlock(&khugepaged_mm_lock); clear_bit(MMF_VM_HUGEPAGE, &mm->flags); free_mm_slot(mm_slot); mmdrop(mm); } else if (mm_slot) { + spin_unlock(&khugepaged_mm_lock); /* * This is required to serialize against * khugepaged_test_exit() (which is guaranteed to run @@ -1679,7 +1633,8 @@ void __khugepaged_exit(struct mm_struct *mm) */ down_write(&mm->mmap_sem); up_write(&mm->mmap_sem); - } + } else + spin_unlock(&khugepaged_mm_lock); } static void release_pte_page(struct page *page) @@ -1979,7 +1934,7 @@ static void collapse_huge_page(struct mm_struct *mm, BUG_ON(!pmd_none(*pmd)); page_add_new_anon_rmap(new_page, vma, address); set_pmd_at(mm, address, pmd, _pmd); - update_mmu_cache(vma, address, _pmd); + update_mmu_cache(vma, address, entry); prepare_pmd_huge_pte(pgtable, mm); spin_unlock(&mm->page_table_lock); @@ -2096,8 +2051,6 @@ static void collect_mm_slot(struct mm_slot *mm_slot) static unsigned int khugepaged_scan_mm_slot(unsigned int pages, struct page **hpage) - __releases(&khugepaged_mm_lock) - __acquires(&khugepaged_mm_lock) { struct mm_slot *mm_slot; struct mm_struct *mm; @@ -2270,8 +2223,12 @@ static void khugepaged_do_scan(struct page **hpage) static void khugepaged_alloc_sleep(void) { - wait_event_freezable_timeout(khugepaged_wait, false, - msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); + DEFINE_WAIT(wait); + add_wait_queue(&khugepaged_wait, &wait); + schedule_timeout_interruptible( + msecs_to_jiffies( + khugepaged_alloc_sleep_millisecs)); + remove_wait_queue(&khugepaged_wait, &wait); } #ifndef CONFIG_NUMA @@ -2320,10 +2277,14 @@ static void khugepaged_loop(void) if (unlikely(kthread_should_stop())) break; if (khugepaged_has_work()) { + DEFINE_WAIT(wait); if (!khugepaged_scan_sleep_millisecs) continue; - wait_event_freezable_timeout(khugepaged_wait, false, - msecs_to_jiffies(khugepaged_scan_sleep_millisecs)); + add_wait_queue(&khugepaged_wait, &wait); + schedule_timeout_interruptible( + msecs_to_jiffies( + khugepaged_scan_sleep_millisecs)); + remove_wait_queue(&khugepaged_wait, &wait); } else if (khugepaged_enabled()) wait_event_freezable(khugepaged_wait, khugepaged_wait_event()); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 26922da..6fdad25 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -24,7 +24,7 @@ #include <asm/page.h> #include <asm/pgtable.h> -#include <linux/io.h> +#include <asm/io.h> #include <linux/hugetlb.h> #include <linux/node.h> @@ -53,84 +53,6 @@ static unsigned long __initdata default_hstate_size; */ static DEFINE_SPINLOCK(hugetlb_lock); -static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) -{ - bool free = (spool->count == 0) && (spool->used_hpages == 0); - - spin_unlock(&spool->lock); - - /* If no pages are used, and no other handles to the subpool - * remain, free the subpool the subpool remain */ - if (free) - kfree(spool); -} - -struct hugepage_subpool *hugepage_new_subpool(long nr_blocks) -{ - struct hugepage_subpool *spool; - - spool = kmalloc(sizeof(*spool), GFP_KERNEL); - if (!spool) - return NULL; - - spin_lock_init(&spool->lock); - spool->count = 1; - spool->max_hpages = nr_blocks; - spool->used_hpages = 0; - - return spool; -} - -void hugepage_put_subpool(struct hugepage_subpool *spool) -{ - spin_lock(&spool->lock); - BUG_ON(!spool->count); - spool->count--; - unlock_or_release_subpool(spool); -} - -static int hugepage_subpool_get_pages(struct hugepage_subpool *spool, - long delta) -{ - int ret = 0; - - if (!spool) - return 0; - - spin_lock(&spool->lock); - if ((spool->used_hpages + delta) <= spool->max_hpages) { - spool->used_hpages += delta; - } else { - ret = -ENOMEM; - } - spin_unlock(&spool->lock); - - return ret; -} - -static void hugepage_subpool_put_pages(struct hugepage_subpool *spool, - long delta) -{ - if (!spool) - return; - - spin_lock(&spool->lock); - spool->used_hpages -= delta; - /* If hugetlbfs_put_super couldn't free spool due to - * an outstanding quota reference, free it now. */ - unlock_or_release_subpool(spool); -} - -static inline struct hugepage_subpool *subpool_inode(struct inode *inode) -{ - return HUGETLBFS_SB(inode->i_sb)->spool; -} - -static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) -{ - return subpool_inode(vma->vm_file->f_dentry->d_inode); -} - /* * Region tracking -- allows tracking of reservations and instantiated pages * across the pages in a mapping. @@ -140,10 +62,10 @@ static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) * must either hold the mmap_sem for write, or the mmap_sem for read and * the hugetlb_instantiation mutex: * - * down_write(&mm->mmap_sem); + * down_write(&mm->mmap_sem); * or - * down_read(&mm->mmap_sem); - * mutex_lock(&hugetlb_instantiation_mutex); + * down_read(&mm->mmap_sem); + * mutex_lock(&hugetlb_instantiation_mutex); */ struct file_region { struct list_head link; @@ -588,10 +510,9 @@ static void update_and_free_page(struct hstate *h, struct page *page) h->nr_huge_pages--; h->nr_huge_pages_node[page_to_nid(page)]--; for (i = 0; i < pages_per_huge_page(h); i++) { - page[i].flags &= ~(1 << PG_locked | 1 << PG_error | - 1 << PG_referenced | 1 << PG_dirty | - 1 << PG_active | 1 << PG_reserved | - 1 << PG_private | 1 << PG_writeback); + page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced | + 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved | + 1 << PG_private | 1<< PG_writeback); } set_compound_page_dtor(page, NULL); set_page_refcounted(page); @@ -618,9 +539,9 @@ static void free_huge_page(struct page *page) */ struct hstate *h = page_hstate(page); int nid = page_to_nid(page); - struct hugepage_subpool *spool = - (struct hugepage_subpool *)page_private(page); + struct address_space *mapping; + mapping = (struct address_space *) page_private(page); set_page_private(page, 0); page->mapping = NULL; BUG_ON(page_count(page)); @@ -636,7 +557,8 @@ static void free_huge_page(struct page *page) enqueue_huge_page(h, page); } spin_unlock(&hugetlb_lock); - hugepage_subpool_put_pages(spool, 1); + if (mapping) + hugetlb_put_quota(mapping, 1); } static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) @@ -677,24 +599,8 @@ int PageHuge(struct page *page) return dtor == free_huge_page; } -EXPORT_SYMBOL_GPL(PageHuge); -/* - * PageHeadHuge() only returns true for hugetlbfs head page, but not for - * normal or transparent huge pages. - */ -int PageHeadHuge(struct page *page_head) -{ - compound_page_dtor *dtor; - - if (!PageHead(page_head)) - return 0; - - dtor = get_compound_page_dtor(page_head); - - return dtor == free_huge_page; -} -EXPORT_SYMBOL_GPL(PageHeadHuge); +EXPORT_SYMBOL_GPL(PageHuge); pgoff_t __basepage_index(struct page *page) { @@ -1078,19 +984,17 @@ static void return_unused_surplus_pages(struct hstate *h, while (nr_pages--) { if (!free_pool_huge_page(h, &node_states[N_HIGH_MEMORY], 1)) break; - cond_resched_lock(&hugetlb_lock); } } /* * Determine if the huge page at addr within the vma has an associated * reservation. Where it does not we will need to logically increase - * reservation and actually increase subpool usage before an allocation - * can occur. Where any new reservation would be required the - * reservation change is prepared, but not committed. Once the page - * has been allocated from the subpool and instantiated the change should - * be committed via vma_commit_reservation. No action is required on - * failure. + * reservation and actually increase quota before an allocation can occur. + * Where any new reservation would be required the reservation change is + * prepared, but not committed. Once the page has been quota'd allocated + * an instantiated the change should be committed via vma_commit_reservation. + * No action is required on failure. */ static long vma_needs_reservation(struct hstate *h, struct vm_area_struct *vma, unsigned long addr) @@ -1139,24 +1043,24 @@ static void vma_commit_reservation(struct hstate *h, static struct page *alloc_huge_page(struct vm_area_struct *vma, unsigned long addr, int avoid_reserve) { - struct hugepage_subpool *spool = subpool_vma(vma); struct hstate *h = hstate_vma(vma); struct page *page; + struct address_space *mapping = vma->vm_file->f_mapping; + struct inode *inode = mapping->host; long chg; /* - * Processes that did not create the mapping will have no - * reserves and will not have accounted against subpool - * limit. Check that the subpool limit can be made before - * satisfying the allocation MAP_NORESERVE mappings may also - * need pages and subpool limit allocated allocated if no reserve - * mapping overlaps. + * Processes that did not create the mapping will have no reserves and + * will not have accounted against quota. Check that the quota can be + * made before satisfying the allocation + * MAP_NORESERVE mappings may also need pages and quota allocated + * if no reserve mapping overlaps. */ chg = vma_needs_reservation(h, vma, addr); if (chg < 0) return ERR_PTR(-VM_FAULT_OOM); if (chg) - if (hugepage_subpool_get_pages(spool, chg)) + if (hugetlb_get_quota(inode->i_mapping, chg)) return ERR_PTR(-VM_FAULT_SIGBUS); spin_lock(&hugetlb_lock); @@ -1166,12 +1070,12 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma, if (!page) { page = alloc_buddy_huge_page(h, NUMA_NO_NODE); if (!page) { - hugepage_subpool_put_pages(spool, chg); + hugetlb_put_quota(inode->i_mapping, chg); return ERR_PTR(-VM_FAULT_SIGBUS); } } - set_page_private(page, (unsigned long)spool); + set_page_private(page, (unsigned long) mapping); vma_commit_reservation(h, vma, addr); @@ -1226,16 +1130,8 @@ static void __init gather_bootmem_prealloc(void) struct huge_bootmem_page *m; list_for_each_entry(m, &huge_boot_pages, list) { + struct page *page = virt_to_page(m); struct hstate *h = m->hstate; - struct page *page; - -#ifdef CONFIG_HIGHMEM - page = pfn_to_page(m->phys >> PAGE_SHIFT); - free_bootmem_late((unsigned long)m, - sizeof(struct huge_bootmem_page)); -#else - page = virt_to_page(m); -#endif __ClearPageReserved(page); WARN_ON(page_count(page) != 1); prep_compound_huge_page(page, h->order); @@ -1448,7 +1344,6 @@ static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, while (min_count < persistent_huge_pages(h)) { if (!free_pool_huge_page(h, nodes_allowed, 0)) break; - cond_resched_lock(&hugetlb_lock); } while (count < persistent_huge_pages(h)) { if (!adjust_pool_surplus(h, nodes_allowed, 1)) @@ -2206,7 +2101,6 @@ static void hugetlb_vm_op_close(struct vm_area_struct *vma) { struct hstate *h = hstate_vma(vma); struct resv_map *reservations = vma_resv_map(vma); - struct hugepage_subpool *spool = subpool_vma(vma); unsigned long reserve; unsigned long start; unsigned long end; @@ -2222,7 +2116,7 @@ static void hugetlb_vm_op_close(struct vm_area_struct *vma) if (reserve) { hugetlb_acct_memory(h, -reserve); - hugepage_subpool_put_pages(spool, reserve); + hugetlb_put_quota(vma->vm_file->f_mapping, reserve); } } } @@ -2268,35 +2162,11 @@ static void set_huge_ptep_writable(struct vm_area_struct *vma, pte_t entry; entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep))); - if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) + if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) { update_mmu_cache(vma, address, ptep); + } } -static int is_hugetlb_entry_migration(pte_t pte) -{ - swp_entry_t swp; - - if (huge_pte_none(pte) || pte_present(pte)) - return 0; - swp = pte_to_swp_entry(pte); - if (non_swap_entry(swp) && is_migration_entry(swp)) - return 1; - else - return 0; -} - -static int is_hugetlb_entry_hwpoisoned(pte_t pte) -{ - swp_entry_t swp; - - if (huge_pte_none(pte) || pte_present(pte)) - return 0; - swp = pte_to_swp_entry(pte); - if (non_swap_entry(swp) && is_hwpoison_entry(swp)) - return 1; - else - return 0; -} int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, struct vm_area_struct *vma) @@ -2324,24 +2194,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, spin_lock(&dst->page_table_lock); spin_lock_nested(&src->page_table_lock, SINGLE_DEPTH_NESTING); - entry = huge_ptep_get(src_pte); - if (huge_pte_none(entry)) { /* skip none entry */ - ; - } else if (unlikely(is_hugetlb_entry_migration(entry) || - is_hugetlb_entry_hwpoisoned(entry))) { - swp_entry_t swp_entry = pte_to_swp_entry(entry); - - if (is_write_migration_entry(swp_entry) && cow) { - /* - * COW mappings require pages in both - * parent and child to be set to read. - */ - make_migration_entry_read(&swp_entry); - entry = swp_entry_to_pte(swp_entry); - set_huge_pte_at(src, addr, src_pte, entry); - } - set_huge_pte_at(dst, addr, dst_pte, entry); - } else { + if (!huge_pte_none(huge_ptep_get(src_pte))) { if (cow) huge_ptep_set_wrprotect(src, addr, src_pte); entry = huge_ptep_get(src_pte); @@ -2359,6 +2212,32 @@ nomem: return -ENOMEM; } +static int is_hugetlb_entry_migration(pte_t pte) +{ + swp_entry_t swp; + + if (huge_pte_none(pte) || pte_present(pte)) + return 0; + swp = pte_to_swp_entry(pte); + if (non_swap_entry(swp) && is_migration_entry(swp)) { + return 1; + } else + return 0; +} + +static int is_hugetlb_entry_hwpoisoned(pte_t pte) +{ + swp_entry_t swp; + + if (huge_pte_none(pte) || pte_present(pte)) + return 0; + swp = pte_to_swp_entry(pte); + if (non_swap_entry(swp) && is_hwpoison_entry(swp)) { + return 1; + } else + return 0; +} + void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct page *ref_page) { @@ -2418,10 +2297,9 @@ void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, continue; /* - * Migrating hugepage or HWPoisoned hugepage is already - * unmapped and its refcount is dropped + * HWPoisoned hugepage is already unmapped and dropped reference */ - if (unlikely(!pte_present(pte))) + if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) continue; page = pte_page(pte); @@ -2439,30 +2317,27 @@ void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, } } -void __unmap_hugepage_range_final(struct vm_area_struct *vma, - unsigned long start, unsigned long end, - struct page *ref_page) +void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end, struct page *ref_page) { + mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex); __unmap_hugepage_range(vma, start, end, ref_page); - /* * Clear this flag so that x86's huge_pmd_share page_table_shareable * test will fail on a vma being torn down, and not grab a page table * on its way out. We're lucky that the flag has such an appropriate * name, and can in fact be safely cleared here. We could clear it * before the __unmap_hugepage_range above, but all that's necessary - * is to clear it before releasing the i_mmap_mutex. This works - * because in the context this is called, the VMA is about to be - * destroyed and the i_mmap_mutex is held. + * is to clear it before releasing the i_mmap_mutex below. + * + * This works because in the contexts this is called, the VMA is + * going to be destroyed. It is not vunerable to madvise(DONTNEED) + * because madvise is not supported on hugetlbfs. The same applies + * for direct IO. unmap_hugepage_range() is only being called just + * before free_pgtables() so clearing VM_MAYSHARE will not cause + * surprises later. */ vma->vm_flags &= ~VM_MAYSHARE; -} - -void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end, struct page *ref_page) -{ - mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex); - __unmap_hugepage_range(vma, start, end, ref_page); mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex); } @@ -2486,9 +2361,9 @@ static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, * from page cache lookup which is in HPAGE_SIZE units. */ address = address & huge_page_mask(h); - pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + - vma->vm_pgoff; - mapping = vma->vm_file->f_dentry->d_inode->i_mapping; + pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + + (vma->vm_pgoff >> PAGE_SHIFT); + mapping = (struct address_space *)page_private(page); /* * Take the mapping lock for the duration of the table walk. As @@ -2739,7 +2614,7 @@ retry: * So we need to block hugepage fault by PG_hwpoison bit check. */ if (unlikely(PageHWPoison(page))) { - ret = VM_FAULT_HWPOISON | + ret = VM_FAULT_HWPOISON | VM_FAULT_SET_HINDEX(h - hstates); goto backout_unlocked; } @@ -2799,7 +2674,6 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, struct page *pagecache_page = NULL; static DEFINE_MUTEX(hugetlb_instantiation_mutex); struct hstate *h = hstate_vma(vma); - int need_wait_lock = 0; ptep = huge_pte_offset(mm, address); if (ptep) { @@ -2808,7 +2682,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, migration_entry_wait_huge(mm, ptep); return 0; } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) - return VM_FAULT_HWPOISON_LARGE | + return VM_FAULT_HWPOISON_LARGE | VM_FAULT_SET_HINDEX(h - hstates); } @@ -2831,16 +2705,6 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, ret = 0; /* - * entry could be a migration/hwpoison entry at this point, so this - * check prevents the kernel from going below assuming that we have - * a active hugepage in pagecache. This goto expects the 2nd page fault, - * and is_hugetlb_entry_(migration|hwpoisoned) check will properly - * handle it. - */ - if (!pte_present(entry)) - goto out_mutex; - - /* * If we are going to COW the mapping later, we examine the pending * reservations for this page now. This will ensure that any * allocations necessary to record that reservation occur outside the @@ -2859,30 +2723,29 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, vma, address); } - spin_lock(&mm->page_table_lock); - /* Check for a racing update before calling hugetlb_cow */ - if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) - goto out_page_table_lock; - /* * hugetlb_cow() requires page locks of pte_page(entry) and * pagecache_page, so here we need take the former one * when page != pagecache_page or !pagecache_page. + * Note that locking order is always pagecache_page -> page, + * so no worry about deadlock. */ page = pte_page(entry); + get_page(page); if (page != pagecache_page) - if (!trylock_page(page)) { - need_wait_lock = 1; - goto out_page_table_lock; - } + lock_page(page); + + spin_lock(&mm->page_table_lock); + /* Check for a racing update before calling hugetlb_cow */ + if (unlikely(!pte_same(entry, huge_ptep_get(ptep)))) + goto out_page_table_lock; - get_page(page); if (flags & FAULT_FLAG_WRITE) { if (!pte_write(entry)) { ret = hugetlb_cow(mm, vma, address, ptep, entry, pagecache_page); - goto out_put_page; + goto out_page_table_lock; } entry = pte_mkdirty(entry); } @@ -2890,10 +2753,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, if (huge_ptep_set_access_flags(vma, address, ptep, entry, flags & FAULT_FLAG_WRITE)) update_mmu_cache(vma, address, ptep); -out_put_page: - if (page != pagecache_page) - unlock_page(page); - put_page(page); + out_page_table_lock: spin_unlock(&mm->page_table_lock); @@ -2901,18 +2761,13 @@ out_page_table_lock: unlock_page(pagecache_page); put_page(pagecache_page); } + if (page != pagecache_page) + unlock_page(page); + put_page(page); + out_mutex: mutex_unlock(&hugetlb_instantiation_mutex); - /* - * Generally it's safe to hold refcount during waiting page lock. But - * here we just wait to defer the next page fault to avoid busy loop and - * the page is not used after unlocked before returning from the current - * page fault. So we are safe from accessing freed page, even if we wait - * here without taking refcount. - */ - if (need_wait_lock) - wait_on_page_locked(page); return ret; } @@ -3038,22 +2893,7 @@ void hugetlb_change_protection(struct vm_area_struct *vma, continue; if (huge_pmd_unshare(mm, &address, ptep)) continue; - pte = huge_ptep_get(ptep); - if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) - continue; - if (unlikely(is_hugetlb_entry_migration(pte))) { - swp_entry_t entry = pte_to_swp_entry(pte); - - if (is_write_migration_entry(entry)) { - pte_t newpte; - - make_migration_entry_read(&entry); - newpte = swp_entry_to_pte(entry); - set_huge_pte_at(mm, address, ptep, newpte); - } - continue; - } - if (!huge_pte_none(pte)) { + if (!huge_pte_none(huge_ptep_get(ptep))) { pte = huge_ptep_get_and_clear(mm, address, ptep); pte = pte_mkhuge(pte_modify(pte, newprot)); set_huge_pte_at(mm, address, ptep, pte); @@ -3077,12 +2917,11 @@ int hugetlb_reserve_pages(struct inode *inode, { long ret, chg; struct hstate *h = hstate_inode(inode); - struct hugepage_subpool *spool = subpool_inode(inode); /* * Only apply hugepage reservation if asked. At fault time, an * attempt will be made for VM_NORESERVE to allocate a page - * without using reserves + * and filesystem quota without using reserves */ if (vm_flags & VM_NORESERVE) return 0; @@ -3111,19 +2950,19 @@ int hugetlb_reserve_pages(struct inode *inode, goto out_err; } - /* There must be enough pages in the subpool for the mapping */ - if (hugepage_subpool_get_pages(spool, chg)) { + /* There must be enough filesystem quota for the mapping */ + if (hugetlb_get_quota(inode->i_mapping, chg)) { ret = -ENOSPC; goto out_err; } /* * Check enough hugepages are available for the reservation. - * Hand the pages back to the subpool if there are not + * Hand back the quota if there are not */ ret = hugetlb_acct_memory(h, chg); if (ret < 0) { - hugepage_subpool_put_pages(spool, chg); + hugetlb_put_quota(inode->i_mapping, chg); goto out_err; } @@ -3151,13 +2990,12 @@ void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed) { struct hstate *h = hstate_inode(inode); long chg = region_truncate(&inode->i_mapping->private_list, offset); - struct hugepage_subpool *spool = subpool_inode(inode); spin_lock(&inode->i_lock); inode->i_blocks -= (blocks_per_huge_page(h) * freed); spin_unlock(&inode->i_lock); - hugepage_subpool_put_pages(spool, (chg - freed)); + hugetlb_put_quota(inode->i_mapping, (chg - freed)); hugetlb_acct_memory(h, -(chg - freed)); } diff --git a/mm/init-mm.c b/mm/init-mm.c index a56a851..4019979 100644 --- a/mm/init-mm.c +++ b/mm/init-mm.c @@ -5,7 +5,7 @@ #include <linux/list.h> #include <linux/cpumask.h> -#include <linux/atomic.h> +#include <asm/atomic.h> #include <asm/pgtable.h> #include <asm/mmu.h> diff --git a/mm/internal.h b/mm/internal.h index 0c26b5e..c5952d0 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -100,6 +100,41 @@ extern void prep_compound_page(struct page *page, unsigned long order); extern bool is_free_buddy_page(struct page *page); #endif +#ifdef CONFIG_DMA_CMA + +/* + * in mm/compaction.c + */ +/* + * compact_control is used to track pages being migrated and the free pages + * they are being migrated to during memory compaction. The free_pfn starts + * at the end of a zone and migrate_pfn begins at the start. Movable pages + * are moved to the end of a zone during a compaction run and the run + * completes when free_pfn <= migrate_pfn + */ +struct compact_control { + struct list_head freepages; /* List of free pages to migrate to */ + struct list_head migratepages; /* List of pages being migrated */ + unsigned long nr_freepages; /* Number of isolated free pages */ + unsigned long nr_migratepages; /* Number of pages to migrate */ + unsigned long free_pfn; /* isolate_freepages search base */ + unsigned long migrate_pfn; /* isolate_migratepages search base */ + bool sync; /* Synchronous migration */ + + int order; /* order a direct compactor needs */ + int migratetype; /* MOVABLE, RECLAIMABLE etc */ + struct zone *zone; +}; + +unsigned long +isolate_freepages_range(unsigned long start_pfn, unsigned long end_pfn, + bool for_cma); + +unsigned long +isolate_migratepages_range(struct zone *zone, struct compact_control *cc, + unsigned long low_pfn, unsigned long end_pfn); +#endif + /* * function for dealing with page's order in buddy system. @@ -309,5 +344,3 @@ extern u64 hwpoison_filter_flags_mask; extern u64 hwpoison_filter_flags_value; extern u64 hwpoison_filter_memcg; extern u32 hwpoison_filter_enable; - -extern void set_pageblock_order(void); diff --git a/mm/kmemleak.c b/mm/kmemleak.c index cbae846..aacee45 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -69,7 +69,7 @@ #include <linux/sched.h> #include <linux/jiffies.h> #include <linux/delay.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/kthread.h> #include <linux/prio_tree.h> #include <linux/fs.h> @@ -96,7 +96,7 @@ #include <asm/sections.h> #include <asm/processor.h> -#include <linux/atomic.h> +#include <asm/atomic.h> #include <linux/kmemcheck.h> #include <linux/kmemleak.h> @@ -192,8 +192,6 @@ static struct kmem_cache *scan_area_cache; /* set if tracing memory operations is enabled */ static atomic_t kmemleak_enabled = ATOMIC_INIT(0); -/* same as above but only for the kmemleak_free() callback */ -static int kmemleak_free_enabled; /* set in the late_initcall if there were no errors */ static atomic_t kmemleak_initialized = ATOMIC_INIT(0); /* enables or disables early logging of the memory operations */ @@ -746,9 +744,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) } spin_lock_irqsave(&object->lock, flags); - if (size == SIZE_MAX) { - size = object->pointer + object->size - ptr; - } else if (ptr + size > object->pointer + object->size) { + if (ptr + size > object->pointer + object->size) { kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr); dump_object_info(object); kmem_cache_free(scan_area_cache, area); @@ -887,7 +883,7 @@ void __ref kmemleak_free(const void *ptr) { pr_debug("%s(0x%p)\n", __func__, ptr); - if (kmemleak_free_enabled && ptr && !IS_ERR(ptr)) + if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr)) delete_object_full((unsigned long)ptr); else if (atomic_read(&kmemleak_early_log)) log_early(KMEMLEAK_FREE, ptr, 0, 0); @@ -1616,13 +1612,6 @@ static void kmemleak_do_cleanup(struct work_struct *work) mutex_lock(&scan_mutex); stop_scan_thread(); - /* - * Once the scan thread has stopped, it is safe to no longer track - * object freeing. Ordering of the scan thread stopping and the memory - * accesses below is guaranteed by the kthread_stop() function. - */ - kmemleak_free_enabled = 0; - rcu_read_lock(); list_for_each_entry_rcu(object, &object_list, object_list) delete_object_full(object->pointer); @@ -1649,8 +1638,6 @@ static void kmemleak_disable(void) /* check whether it is too early for a kernel thread */ if (atomic_read(&kmemleak_initialized)) schedule_work(&cleanup_work); - else - kmemleak_free_enabled = 0; pr_info("Kernel memory leak detector disabled\n"); } @@ -1699,7 +1686,6 @@ void __init kmemleak_init(void) if (!atomic_read(&kmemleak_error)) { atomic_set(&kmemleak_enabled, 1); atomic_set(&kmemleak_early_log, 0); - kmemleak_free_enabled = 1; } local_irq_restore(flags); @@ -342,7 +342,7 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr) else ret = VM_FAULT_WRITE; put_page(page); - } while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | VM_FAULT_OOM))); + } while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_OOM))); /* * We must loop because handle_mm_fault() may back out if there's * any difficulty e.g. if pte accessed bit gets updated concurrently. @@ -1905,8 +1905,7 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr, oom_score_adj = test_set_oom_score_adj(OOM_SCORE_ADJ_MAX); err = unmerge_and_remove_all_rmap_items(); - compare_swap_oom_score_adj(OOM_SCORE_ADJ_MAX, - oom_score_adj); + test_set_oom_score_adj(oom_score_adj); if (err) { ksm_run = KSM_RUN_STOP; count = err; diff --git a/mm/maccess.c b/mm/maccess.c index d53adf9..4cee182 100644 --- a/mm/maccess.c +++ b/mm/maccess.c @@ -1,7 +1,7 @@ /* * Access kernel memory without faulting. */ -#include <linux/export.h> +#include <linux/module.h> #include <linux/mm.h> #include <linux/uaccess.h> diff --git a/mm/madvise.c b/mm/madvise.c index 23d3a6b..deabe5f6 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -222,10 +222,10 @@ static long madvise_remove(struct vm_area_struct *vma, + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); /* - * vmtruncate_range may need to take i_mutex. We need to - * explicitly grab a reference because the vma (and hence the - * vma's reference to the file) can go away as soon as we drop - * mmap_sem. + * vmtruncate_range may need to take i_mutex and i_alloc_sem. + * We need to explicitly grab a reference because the vma (and + * hence the vma's reference to the file) can go away as soon as + * we drop mmap_sem. */ get_file(f); up_read(¤t->mm->mmap_sem); diff --git a/mm/memblock.c b/mm/memblock.c index 84bec49..a0562d1 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -58,8 +58,7 @@ static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, p return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); } -static long __init_memblock memblock_overlaps_region(struct memblock_type *type, - phys_addr_t base, phys_addr_t size) +long __init_memblock memblock_overlaps_region(struct memblock_type *type, phys_addr_t base, phys_addr_t size) { unsigned long i; @@ -268,7 +267,7 @@ static int __init_memblock memblock_double_array(struct memblock_type *type) return 0; } -int __init_memblock __weak memblock_memory_can_coalesce(phys_addr_t addr1, phys_addr_t size1, +extern int __init_memblock __weak memblock_memory_can_coalesce(phys_addr_t addr1, phys_addr_t size1, phys_addr_t addr2, phys_addr_t size2) { return 1; @@ -627,12 +626,6 @@ phys_addr_t __init memblock_phys_mem_size(void) return memblock.memory_size; } -/* lowest address */ -phys_addr_t __init_memblock memblock_start_of_DRAM(void) -{ - return memblock.memory.regions[0].base; -} - phys_addr_t __init_memblock memblock_end_of_DRAM(void) { int idx = memblock.memory.cnt - 1; @@ -765,9 +758,9 @@ void __init memblock_analyze(void) /* Check marker in the unused last array entry */ WARN_ON(memblock_memory_init_regions[INIT_MEMBLOCK_REGIONS].base - != MEMBLOCK_INACTIVE); + != (phys_addr_t)RED_INACTIVE); WARN_ON(memblock_reserved_init_regions[INIT_MEMBLOCK_REGIONS].base - != MEMBLOCK_INACTIVE); + != (phys_addr_t)RED_INACTIVE); memblock.memory_size = 0; @@ -793,8 +786,8 @@ void __init memblock_init(void) memblock.reserved.max = INIT_MEMBLOCK_REGIONS; /* Write a marker in the unused last array entry */ - memblock.memory.regions[INIT_MEMBLOCK_REGIONS].base = MEMBLOCK_INACTIVE; - memblock.reserved.regions[INIT_MEMBLOCK_REGIONS].base = MEMBLOCK_INACTIVE; + memblock.memory.regions[INIT_MEMBLOCK_REGIONS].base = (phys_addr_t)RED_INACTIVE; + memblock.reserved.regions[INIT_MEMBLOCK_REGIONS].base = (phys_addr_t)RED_INACTIVE; /* Create a dummy zero size MEMBLOCK which will get coalesced away later. * This simplifies the memblock_add() code below... diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 204de6a..d7b51d5 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -33,9 +33,9 @@ #include <linux/bit_spinlock.h> #include <linux/rcupdate.h> #include <linux/limits.h> -#include <linux/export.h> #include <linux/mutex.h> #include <linux/rbtree.h> +#include <linux/shmem_fs.h> #include <linux/slab.h> #include <linux/swap.h> #include <linux/swapops.h> @@ -202,8 +202,8 @@ struct mem_cgroup_eventfd_list { struct eventfd_ctx *eventfd; }; -static void mem_cgroup_threshold(struct mem_cgroup *memcg); -static void mem_cgroup_oom_notify(struct mem_cgroup *memcg); +static void mem_cgroup_threshold(struct mem_cgroup *mem); +static void mem_cgroup_oom_notify(struct mem_cgroup *mem); /* * The memory controller data structure. The memory controller controls both @@ -246,13 +246,10 @@ struct mem_cgroup { * Should the accounting and control be hierarchical, per subtree? */ bool use_hierarchy; - - bool oom_lock; - atomic_t under_oom; - + atomic_t oom_lock; atomic_t refcnt; - int swappiness; + unsigned int swappiness; /* OOM-Killer disable */ int oom_kill_disable; @@ -363,29 +360,29 @@ enum charge_type { #define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2 #define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT) -static void mem_cgroup_get(struct mem_cgroup *memcg); -static void mem_cgroup_put(struct mem_cgroup *memcg); -static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg); -static void drain_all_stock_async(struct mem_cgroup *memcg); +static void mem_cgroup_get(struct mem_cgroup *mem); +static void mem_cgroup_put(struct mem_cgroup *mem); +static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem); +static void drain_all_stock_async(struct mem_cgroup *mem); static struct mem_cgroup_per_zone * -mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid) +mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid) { - return &memcg->info.nodeinfo[nid]->zoneinfo[zid]; + return &mem->info.nodeinfo[nid]->zoneinfo[zid]; } -struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg) +struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem) { - return &memcg->css; + return &mem->css; } static struct mem_cgroup_per_zone * -page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page) +page_cgroup_zoneinfo(struct mem_cgroup *mem, struct page *page) { int nid = page_to_nid(page); int zid = page_zonenum(page); - return mem_cgroup_zoneinfo(memcg, nid, zid); + return mem_cgroup_zoneinfo(mem, nid, zid); } static struct mem_cgroup_tree_per_zone * @@ -404,7 +401,7 @@ soft_limit_tree_from_page(struct page *page) } static void -__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg, +__mem_cgroup_insert_exceeded(struct mem_cgroup *mem, struct mem_cgroup_per_zone *mz, struct mem_cgroup_tree_per_zone *mctz, unsigned long long new_usage_in_excess) @@ -438,7 +435,7 @@ __mem_cgroup_insert_exceeded(struct mem_cgroup *memcg, } static void -__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg, +__mem_cgroup_remove_exceeded(struct mem_cgroup *mem, struct mem_cgroup_per_zone *mz, struct mem_cgroup_tree_per_zone *mctz) { @@ -449,17 +446,17 @@ __mem_cgroup_remove_exceeded(struct mem_cgroup *memcg, } static void -mem_cgroup_remove_exceeded(struct mem_cgroup *memcg, +mem_cgroup_remove_exceeded(struct mem_cgroup *mem, struct mem_cgroup_per_zone *mz, struct mem_cgroup_tree_per_zone *mctz) { spin_lock(&mctz->lock); - __mem_cgroup_remove_exceeded(memcg, mz, mctz); + __mem_cgroup_remove_exceeded(mem, mz, mctz); spin_unlock(&mctz->lock); } -static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) +static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page) { unsigned long long excess; struct mem_cgroup_per_zone *mz; @@ -472,9 +469,9 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) * Necessary to update all ancestors when hierarchy is used. * because their event counter is not touched. */ - for (; memcg; memcg = parent_mem_cgroup(memcg)) { - mz = mem_cgroup_zoneinfo(memcg, nid, zid); - excess = res_counter_soft_limit_excess(&memcg->res); + for (; mem; mem = parent_mem_cgroup(mem)) { + mz = mem_cgroup_zoneinfo(mem, nid, zid); + excess = res_counter_soft_limit_excess(&mem->res); /* * We have to update the tree if mz is on RB-tree or * mem is over its softlimit. @@ -483,18 +480,18 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) spin_lock(&mctz->lock); /* if on-tree, remove it */ if (mz->on_tree) - __mem_cgroup_remove_exceeded(memcg, mz, mctz); + __mem_cgroup_remove_exceeded(mem, mz, mctz); /* * Insert again. mz->usage_in_excess will be updated. * If excess is 0, no tree ops. */ - __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess); + __mem_cgroup_insert_exceeded(mem, mz, mctz, excess); spin_unlock(&mctz->lock); } } } -static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) +static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem) { int node, zone; struct mem_cgroup_per_zone *mz; @@ -502,9 +499,9 @@ static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg) for_each_node_state(node, N_POSSIBLE) { for (zone = 0; zone < MAX_NR_ZONES; zone++) { - mz = mem_cgroup_zoneinfo(memcg, node, zone); + mz = mem_cgroup_zoneinfo(mem, node, zone); mctz = soft_limit_tree_node_zone(node, zone); - mem_cgroup_remove_exceeded(memcg, mz, mctz); + mem_cgroup_remove_exceeded(mem, mz, mctz); } } } @@ -565,7 +562,7 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) * common workload, threashold and synchonization as vmstat[] should be * implemented. */ -static long mem_cgroup_read_stat(struct mem_cgroup *memcg, +static long mem_cgroup_read_stat(struct mem_cgroup *mem, enum mem_cgroup_stat_index idx) { long val = 0; @@ -573,131 +570,111 @@ static long mem_cgroup_read_stat(struct mem_cgroup *memcg, get_online_cpus(); for_each_online_cpu(cpu) - val += per_cpu(memcg->stat->count[idx], cpu); + val += per_cpu(mem->stat->count[idx], cpu); #ifdef CONFIG_HOTPLUG_CPU - spin_lock(&memcg->pcp_counter_lock); - val += memcg->nocpu_base.count[idx]; - spin_unlock(&memcg->pcp_counter_lock); + spin_lock(&mem->pcp_counter_lock); + val += mem->nocpu_base.count[idx]; + spin_unlock(&mem->pcp_counter_lock); #endif put_online_cpus(); return val; } -static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg, +static void mem_cgroup_swap_statistics(struct mem_cgroup *mem, bool charge) { int val = (charge) ? 1 : -1; - this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAPOUT], val); + this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val); } -void mem_cgroup_pgfault(struct mem_cgroup *memcg, int val) +void mem_cgroup_pgfault(struct mem_cgroup *mem, int val) { - this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT], val); + this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_PGFAULT], val); } -void mem_cgroup_pgmajfault(struct mem_cgroup *memcg, int val) +void mem_cgroup_pgmajfault(struct mem_cgroup *mem, int val) { - this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT], val); + this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT], val); } -static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg, +static unsigned long mem_cgroup_read_events(struct mem_cgroup *mem, enum mem_cgroup_events_index idx) { unsigned long val = 0; int cpu; for_each_online_cpu(cpu) - val += per_cpu(memcg->stat->events[idx], cpu); + val += per_cpu(mem->stat->events[idx], cpu); #ifdef CONFIG_HOTPLUG_CPU - spin_lock(&memcg->pcp_counter_lock); - val += memcg->nocpu_base.events[idx]; - spin_unlock(&memcg->pcp_counter_lock); + spin_lock(&mem->pcp_counter_lock); + val += mem->nocpu_base.events[idx]; + spin_unlock(&mem->pcp_counter_lock); #endif return val; } -static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, +static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, bool file, int nr_pages) { preempt_disable(); if (file) - __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE], - nr_pages); + __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages); else - __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS], - nr_pages); + __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages); /* pagein of a big page is an event. So, ignore page size */ if (nr_pages > 0) - __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]); + __this_cpu_inc(mem->stat->events[MEM_CGROUP_EVENTS_PGPGIN]); else { - __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]); + __this_cpu_inc(mem->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]); nr_pages = -nr_pages; /* for event */ } - __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT], nr_pages); + __this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_COUNT], nr_pages); preempt_enable(); } -unsigned long -mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid, - unsigned int lru_mask) -{ - struct mem_cgroup_per_zone *mz; - enum lru_list l; - unsigned long ret = 0; - - mz = mem_cgroup_zoneinfo(memcg, nid, zid); - - for_each_lru(l) { - if (BIT(l) & lru_mask) - ret += MEM_CGROUP_ZSTAT(mz, l); - } - return ret; -} - static unsigned long -mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, - int nid, unsigned int lru_mask) +mem_cgroup_get_zonestat_node(struct mem_cgroup *mem, int nid, enum lru_list idx) { + struct mem_cgroup_per_zone *mz; u64 total = 0; int zid; - for (zid = 0; zid < MAX_NR_ZONES; zid++) - total += mem_cgroup_zone_nr_lru_pages(memcg, - nid, zid, lru_mask); - + for (zid = 0; zid < MAX_NR_ZONES; zid++) { + mz = mem_cgroup_zoneinfo(mem, nid, zid); + total += MEM_CGROUP_ZSTAT(mz, idx); + } return total; } - -static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, - unsigned int lru_mask) +static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem, + enum lru_list idx) { int nid; u64 total = 0; - for_each_node_state(nid, N_HIGH_MEMORY) - total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask); + for_each_online_node(nid) + total += mem_cgroup_get_zonestat_node(mem, nid, idx); return total; } -static bool __memcg_event_check(struct mem_cgroup *memcg, int target) +static bool __memcg_event_check(struct mem_cgroup *mem, int target) { unsigned long val, next; - val = __this_cpu_read(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT]); - next = __this_cpu_read(memcg->stat->targets[target]); + val = this_cpu_read(mem->stat->events[MEM_CGROUP_EVENTS_COUNT]); + next = this_cpu_read(mem->stat->targets[target]); /* from time_after() in jiffies.h */ return ((long)next - (long)val < 0); } -static void __mem_cgroup_target_update(struct mem_cgroup *memcg, int target) +static void __mem_cgroup_target_update(struct mem_cgroup *mem, int target) { unsigned long val, next; - val = __this_cpu_read(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT]); + val = this_cpu_read(mem->stat->events[MEM_CGROUP_EVENTS_COUNT]); switch (target) { case MEM_CGROUP_TARGET_THRESH: @@ -713,36 +690,34 @@ static void __mem_cgroup_target_update(struct mem_cgroup *memcg, int target) return; } - __this_cpu_write(memcg->stat->targets[target], next); + this_cpu_write(mem->stat->targets[target], next); } /* * Check events in order. * */ -static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) +static void memcg_check_events(struct mem_cgroup *mem, struct page *page) { - preempt_disable(); /* threshold event is triggered in finer grain than soft limit */ - if (unlikely(__memcg_event_check(memcg, MEM_CGROUP_TARGET_THRESH))) { - mem_cgroup_threshold(memcg); - __mem_cgroup_target_update(memcg, MEM_CGROUP_TARGET_THRESH); - if (unlikely(__memcg_event_check(memcg, + if (unlikely(__memcg_event_check(mem, MEM_CGROUP_TARGET_THRESH))) { + mem_cgroup_threshold(mem); + __mem_cgroup_target_update(mem, MEM_CGROUP_TARGET_THRESH); + if (unlikely(__memcg_event_check(mem, MEM_CGROUP_TARGET_SOFTLIMIT))) { - mem_cgroup_update_tree(memcg, page); - __mem_cgroup_target_update(memcg, + mem_cgroup_update_tree(mem, page); + __mem_cgroup_target_update(mem, MEM_CGROUP_TARGET_SOFTLIMIT); } #if MAX_NUMNODES > 1 - if (unlikely(__memcg_event_check(memcg, + if (unlikely(__memcg_event_check(mem, MEM_CGROUP_TARGET_NUMAINFO))) { - atomic_inc(&memcg->numainfo_events); - __mem_cgroup_target_update(memcg, + atomic_inc(&mem->numainfo_events); + __mem_cgroup_target_update(mem, MEM_CGROUP_TARGET_NUMAINFO); } #endif } - preempt_enable(); } static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont) @@ -768,7 +743,7 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm) { - struct mem_cgroup *memcg = NULL; + struct mem_cgroup *mem = NULL; if (!mm) return NULL; @@ -779,25 +754,25 @@ struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm) */ rcu_read_lock(); do { - memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); - if (unlikely(!memcg)) + mem = mem_cgroup_from_task(rcu_dereference(mm->owner)); + if (unlikely(!mem)) break; - } while (!css_tryget(&memcg->css)); + } while (!css_tryget(&mem->css)); rcu_read_unlock(); - return memcg; + return mem; } /* The caller has to guarantee "mem" exists before calling this */ -static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *memcg) +static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem) { struct cgroup_subsys_state *css; int found; - if (!memcg) /* ROOT cgroup has the smallest ID */ + if (!mem) /* ROOT cgroup has the smallest ID */ return root_mem_cgroup; /*css_put/get against root is ignored*/ - if (!memcg->use_hierarchy) { - if (css_tryget(&memcg->css)) - return memcg; + if (!mem->use_hierarchy) { + if (css_tryget(&mem->css)) + return mem; return NULL; } rcu_read_lock(); @@ -805,13 +780,13 @@ static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *memcg) * searching a memory cgroup which has the smallest ID under given * ROOT cgroup. (ID >= 1) */ - css = css_get_next(&mem_cgroup_subsys, 1, &memcg->css, &found); + css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found); if (css && css_tryget(css)) - memcg = container_of(css, struct mem_cgroup, css); + mem = container_of(css, struct mem_cgroup, css); else - memcg = NULL; + mem = NULL; rcu_read_unlock(); - return memcg; + return mem; } static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter, @@ -865,29 +840,29 @@ static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter, for_each_mem_cgroup_tree_cond(iter, NULL, true) -static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) +static inline bool mem_cgroup_is_root(struct mem_cgroup *mem) { - return (memcg == root_mem_cgroup); + return (mem == root_mem_cgroup); } void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx) { - struct mem_cgroup *memcg; + struct mem_cgroup *mem; if (!mm) return; rcu_read_lock(); - memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); - if (unlikely(!memcg)) + mem = mem_cgroup_from_task(rcu_dereference(mm->owner)); + if (unlikely(!mem)) goto out; switch (idx) { case PGMAJFAULT: - mem_cgroup_pgmajfault(memcg, 1); + mem_cgroup_pgmajfault(mem, 1); break; case PGFAULT: - mem_cgroup_pgfault(memcg, 1); + mem_cgroup_pgfault(mem, 1); break; default: BUG(); @@ -996,16 +971,6 @@ void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru) return; pc = lookup_page_cgroup(page); VM_BUG_ON(PageCgroupAcctLRU(pc)); - /* - * putback: charge: - * SetPageLRU SetPageCgroupUsed - * smp_mb smp_mb - * PageCgroupUsed && add to memcg LRU PageLRU && add to memcg LRU - * - * Ensure that one of the two sides adds the page to the memcg - * LRU during a race. - */ - smp_mb(); if (!PageCgroupUsed(pc)) return; /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */ @@ -1057,16 +1022,7 @@ static void mem_cgroup_lru_add_after_commit(struct page *page) unsigned long flags; struct zone *zone = page_zone(page); struct page_cgroup *pc = lookup_page_cgroup(page); - /* - * putback: charge: - * SetPageLRU SetPageCgroupUsed - * smp_mb smp_mb - * PageCgroupUsed && add to memcg LRU PageLRU && add to memcg LRU - * - * Ensure that one of the two sides adds the page to the memcg - * LRU during a race. - */ - smp_mb(); + /* taking care of that the page is added to LRU while we commit it */ if (likely(!PageLRU(page))) return; @@ -1087,22 +1043,7 @@ void mem_cgroup_move_lists(struct page *page, mem_cgroup_add_lru_list(page, to); } -/* - * Checks whether given mem is same or in the root_mem_cgroup's - * hierarchy subtree - */ -static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg, - struct mem_cgroup *memcg) -{ - if (root_memcg != memcg) { - return (root_memcg->use_hierarchy && - css_is_ancestor(&memcg->css, &root_memcg->css)); - } - - return true; -} - -int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg) +int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem) { int ret; struct mem_cgroup *curr = NULL; @@ -1116,29 +1057,28 @@ int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg) if (!curr) return 0; /* - * We should check use_hierarchy of "memcg" not "curr". Because checking + * We should check use_hierarchy of "mem" not "curr". Because checking * use_hierarchy of "curr" here make this function true if hierarchy is - * enabled in "curr" and "curr" is a child of "memcg" in *cgroup* - * hierarchy(even if use_hierarchy is disabled in "memcg"). + * enabled in "curr" and "curr" is a child of "mem" in *cgroup* + * hierarchy(even if use_hierarchy is disabled in "mem"). */ - ret = mem_cgroup_same_or_subtree(memcg, curr); + if (mem->use_hierarchy) + ret = css_is_ancestor(&curr->css, &mem->css); + else + ret = (curr == mem); css_put(&curr->css); return ret; } -int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone) +static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages) { - unsigned long inactive_ratio; - int nid = zone_to_nid(zone); - int zid = zone_idx(zone); - unsigned long inactive; unsigned long active; + unsigned long inactive; unsigned long gb; + unsigned long inactive_ratio; - inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid, - BIT(LRU_INACTIVE_ANON)); - active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid, - BIT(LRU_ACTIVE_ANON)); + inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON); + active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON); gb = (inactive + active) >> (30 - PAGE_SHIFT); if (gb) @@ -1146,24 +1086,140 @@ int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone) else inactive_ratio = 1; - return inactive * inactive_ratio < active; + if (present_pages) { + present_pages[0] = inactive; + present_pages[1] = active; + } + + return inactive_ratio; } -int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg, struct zone *zone) +int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg) { unsigned long active; unsigned long inactive; - int zid = zone_idx(zone); - int nid = zone_to_nid(zone); + unsigned long present_pages[2]; + unsigned long inactive_ratio; + + inactive_ratio = calc_inactive_ratio(memcg, present_pages); + + inactive = present_pages[0]; + active = present_pages[1]; - inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid, - BIT(LRU_INACTIVE_FILE)); - active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid, - BIT(LRU_ACTIVE_FILE)); + if (inactive * inactive_ratio < active) + return 1; + + return 0; +} + +int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg) +{ + unsigned long active; + unsigned long inactive; + + inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE); + active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE); return (active > inactive); } +unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, + struct zone *zone, + enum lru_list lru) +{ + int nid = zone_to_nid(zone); + int zid = zone_idx(zone); + struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); + + return MEM_CGROUP_ZSTAT(mz, lru); +} + +static unsigned long mem_cgroup_node_nr_file_lru_pages(struct mem_cgroup *memcg, + int nid) +{ + unsigned long ret; + + ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_FILE) + + mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_FILE); + + return ret; +} + +static unsigned long mem_cgroup_node_nr_anon_lru_pages(struct mem_cgroup *memcg, + int nid) +{ + unsigned long ret; + + ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_ANON) + + mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_ANON); + return ret; +} + +#if MAX_NUMNODES > 1 +static unsigned long mem_cgroup_nr_file_lru_pages(struct mem_cgroup *memcg) +{ + u64 total = 0; + int nid; + + for_each_node_state(nid, N_HIGH_MEMORY) + total += mem_cgroup_node_nr_file_lru_pages(memcg, nid); + + return total; +} + +static unsigned long mem_cgroup_nr_anon_lru_pages(struct mem_cgroup *memcg) +{ + u64 total = 0; + int nid; + + for_each_node_state(nid, N_HIGH_MEMORY) + total += mem_cgroup_node_nr_anon_lru_pages(memcg, nid); + + return total; +} + +static unsigned long +mem_cgroup_node_nr_unevictable_lru_pages(struct mem_cgroup *memcg, int nid) +{ + return mem_cgroup_get_zonestat_node(memcg, nid, LRU_UNEVICTABLE); +} + +static unsigned long +mem_cgroup_nr_unevictable_lru_pages(struct mem_cgroup *memcg) +{ + u64 total = 0; + int nid; + + for_each_node_state(nid, N_HIGH_MEMORY) + total += mem_cgroup_node_nr_unevictable_lru_pages(memcg, nid); + + return total; +} + +static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg, + int nid) +{ + enum lru_list l; + u64 total = 0; + + for_each_lru(l) + total += mem_cgroup_get_zonestat_node(memcg, nid, l); + + return total; +} + +static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg) +{ + u64 total = 0; + int nid; + + for_each_node_state(nid, N_HIGH_MEMORY) + total += mem_cgroup_node_nr_lru_pages(memcg, nid); + + return total; +} +#endif /* CONFIG_NUMA */ + struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, struct zone *zone) { @@ -1264,17 +1320,17 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, * Returns the maximum amount of memory @mem can be charged with, in * pages. */ -static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg) +static unsigned long mem_cgroup_margin(struct mem_cgroup *mem) { unsigned long long margin; - margin = res_counter_margin(&memcg->res); + margin = res_counter_margin(&mem->res); if (do_swap_account) - margin = min(margin, res_counter_margin(&memcg->memsw)); + margin = min(margin, res_counter_margin(&mem->memsw)); return margin >> PAGE_SHIFT; } -int mem_cgroup_swappiness(struct mem_cgroup *memcg) +static unsigned int get_swappiness(struct mem_cgroup *memcg) { struct cgroup *cgrp = memcg->css.cgroup; @@ -1285,33 +1341,33 @@ int mem_cgroup_swappiness(struct mem_cgroup *memcg) return memcg->swappiness; } -static void mem_cgroup_start_move(struct mem_cgroup *memcg) +static void mem_cgroup_start_move(struct mem_cgroup *mem) { int cpu; get_online_cpus(); - spin_lock(&memcg->pcp_counter_lock); + spin_lock(&mem->pcp_counter_lock); for_each_online_cpu(cpu) - per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1; - memcg->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1; - spin_unlock(&memcg->pcp_counter_lock); + per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1; + mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1; + spin_unlock(&mem->pcp_counter_lock); put_online_cpus(); synchronize_rcu(); } -static void mem_cgroup_end_move(struct mem_cgroup *memcg) +static void mem_cgroup_end_move(struct mem_cgroup *mem) { int cpu; - if (!memcg) + if (!mem) return; get_online_cpus(); - spin_lock(&memcg->pcp_counter_lock); + spin_lock(&mem->pcp_counter_lock); for_each_online_cpu(cpu) - per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1; - memcg->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1; - spin_unlock(&memcg->pcp_counter_lock); + per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1; + mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1; + spin_unlock(&mem->pcp_counter_lock); put_online_cpus(); } /* @@ -1326,13 +1382,13 @@ static void mem_cgroup_end_move(struct mem_cgroup *memcg) * waiting at hith-memory prressure caused by "move". */ -static bool mem_cgroup_stealed(struct mem_cgroup *memcg) +static bool mem_cgroup_stealed(struct mem_cgroup *mem) { VM_BUG_ON(!rcu_read_lock_held()); - return this_cpu_read(memcg->stat->count[MEM_CGROUP_ON_MOVE]) > 0; + return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0; } -static bool mem_cgroup_under_move(struct mem_cgroup *memcg) +static bool mem_cgroup_under_move(struct mem_cgroup *mem) { struct mem_cgroup *from; struct mem_cgroup *to; @@ -1346,18 +1402,19 @@ static bool mem_cgroup_under_move(struct mem_cgroup *memcg) to = mc.to; if (!from) goto unlock; - - ret = mem_cgroup_same_or_subtree(memcg, from) - || mem_cgroup_same_or_subtree(memcg, to); + if (from == mem || to == mem + || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css)) + || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css))) + ret = true; unlock: spin_unlock(&mc.lock); return ret; } -static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg) +static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem) { if (mc.moving_task && current != mc.moving_task) { - if (mem_cgroup_under_move(memcg)) { + if (mem_cgroup_under_move(mem)) { DEFINE_WAIT(wait); prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); /* moving charge context might have finished. */ @@ -1441,12 +1498,12 @@ done: * This function returns the number of memcg under hierarchy tree. Returns * 1(self count) if no children. */ -static int mem_cgroup_count_children(struct mem_cgroup *memcg) +static int mem_cgroup_count_children(struct mem_cgroup *mem) { int num = 0; struct mem_cgroup *iter; - for_each_mem_cgroup_tree(iter, memcg) + for_each_mem_cgroup_tree(iter, mem) num++; return num; } @@ -1457,26 +1514,17 @@ static int mem_cgroup_count_children(struct mem_cgroup *memcg) u64 mem_cgroup_get_limit(struct mem_cgroup *memcg) { u64 limit; + u64 memsw; limit = res_counter_read_u64(&memcg->res, RES_LIMIT); + limit += total_swap_pages << PAGE_SHIFT; + memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT); /* - * Do not consider swap space if we cannot swap due to swappiness + * If memsw is finite and limits the amount of swap space available + * to this memcg, return that limit. */ - if (mem_cgroup_swappiness(memcg)) { - u64 memsw; - - limit += total_swap_pages << PAGE_SHIFT; - memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT); - - /* - * If memsw is finite and limits the amount of swap space - * available to this memcg, return that limit. - */ - limit = min(limit, memsw); - } - - return limit; + return min(limit, memsw); } /* @@ -1485,21 +1533,21 @@ u64 mem_cgroup_get_limit(struct mem_cgroup *memcg) * that to reclaim free pages from. */ static struct mem_cgroup * -mem_cgroup_select_victim(struct mem_cgroup *root_memcg) +mem_cgroup_select_victim(struct mem_cgroup *root_mem) { struct mem_cgroup *ret = NULL; struct cgroup_subsys_state *css; int nextid, found; - if (!root_memcg->use_hierarchy) { - css_get(&root_memcg->css); - ret = root_memcg; + if (!root_mem->use_hierarchy) { + css_get(&root_mem->css); + ret = root_mem; } while (!ret) { rcu_read_lock(); - nextid = root_memcg->last_scanned_child + 1; - css = css_get_next(&mem_cgroup_subsys, nextid, &root_memcg->css, + nextid = root_mem->last_scanned_child + 1; + css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css, &found); if (css && css_tryget(css)) ret = container_of(css, struct mem_cgroup, css); @@ -1508,9 +1556,9 @@ mem_cgroup_select_victim(struct mem_cgroup *root_memcg) /* Updates scanning parameter */ if (!css) { /* this means start scan from ID:1 */ - root_memcg->last_scanned_child = 0; + root_mem->last_scanned_child = 0; } else - root_memcg->last_scanned_child = found; + root_mem->last_scanned_child = found; } return ret; @@ -1526,14 +1574,14 @@ mem_cgroup_select_victim(struct mem_cgroup *root_memcg) * reclaimable pages on a node. Returns true if there are any reclaimable * pages in the node. */ -static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg, +static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *mem, int nid, bool noswap) { - if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE)) + if (mem_cgroup_node_nr_file_lru_pages(mem, nid)) return true; if (noswap || !total_swap_pages) return false; - if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON)) + if (mem_cgroup_node_nr_anon_lru_pages(mem, nid)) return true; return false; @@ -1546,29 +1594,29 @@ static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg, * nodes based on the zonelist. So update the list loosely once per 10 secs. * */ -static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg) +static void mem_cgroup_may_update_nodemask(struct mem_cgroup *mem) { int nid; /* * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET * pagein/pageout changes since the last update. */ - if (!atomic_read(&memcg->numainfo_events)) + if (!atomic_read(&mem->numainfo_events)) return; - if (atomic_inc_return(&memcg->numainfo_updating) > 1) + if (atomic_inc_return(&mem->numainfo_updating) > 1) return; /* make a nodemask where this memcg uses memory from */ - memcg->scan_nodes = node_states[N_HIGH_MEMORY]; + mem->scan_nodes = node_states[N_HIGH_MEMORY]; for_each_node_mask(nid, node_states[N_HIGH_MEMORY]) { - if (!test_mem_cgroup_node_reclaimable(memcg, nid, false)) - node_clear(nid, memcg->scan_nodes); + if (!test_mem_cgroup_node_reclaimable(mem, nid, false)) + node_clear(nid, mem->scan_nodes); } - atomic_set(&memcg->numainfo_events, 0); - atomic_set(&memcg->numainfo_updating, 0); + atomic_set(&mem->numainfo_events, 0); + atomic_set(&mem->numainfo_updating, 0); } /* @@ -1583,16 +1631,16 @@ static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg) * * Now, we use round-robin. Better algorithm is welcomed. */ -int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) +int mem_cgroup_select_victim_node(struct mem_cgroup *mem) { int node; - mem_cgroup_may_update_nodemask(memcg); - node = memcg->last_scanned_node; + mem_cgroup_may_update_nodemask(mem); + node = mem->last_scanned_node; - node = next_node(node, memcg->scan_nodes); + node = next_node(node, mem->scan_nodes); if (node == MAX_NUMNODES) - node = first_node(memcg->scan_nodes); + node = first_node(mem->scan_nodes); /* * We call this when we hit limit, not when pages are added to LRU. * No LRU may hold pages because all pages are UNEVICTABLE or @@ -1602,7 +1650,7 @@ int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) if (unlikely(node == MAX_NUMNODES)) node = numa_node_id(); - memcg->last_scanned_node = node; + mem->last_scanned_node = node; return node; } @@ -1612,7 +1660,7 @@ int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) * unused nodes. But scan_nodes is lazily updated and may not cotain * enough new information. We need to do double check. */ -bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap) +bool mem_cgroup_reclaimable(struct mem_cgroup *mem, bool noswap) { int nid; @@ -1620,12 +1668,12 @@ bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap) * quick check...making use of scan_node. * We can skip unused nodes. */ - if (!nodes_empty(memcg->scan_nodes)) { - for (nid = first_node(memcg->scan_nodes); + if (!nodes_empty(mem->scan_nodes)) { + for (nid = first_node(mem->scan_nodes); nid < MAX_NUMNODES; - nid = next_node(nid, memcg->scan_nodes)) { + nid = next_node(nid, mem->scan_nodes)) { - if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap)) + if (test_mem_cgroup_node_reclaimable(mem, nid, noswap)) return true; } } @@ -1633,23 +1681,23 @@ bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap) * Check rest of nodes. */ for_each_node_state(nid, N_HIGH_MEMORY) { - if (node_isset(nid, memcg->scan_nodes)) + if (node_isset(nid, mem->scan_nodes)) continue; - if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap)) + if (test_mem_cgroup_node_reclaimable(mem, nid, noswap)) return true; } return false; } #else -int mem_cgroup_select_victim_node(struct mem_cgroup *memcg) +int mem_cgroup_select_victim_node(struct mem_cgroup *mem) { return 0; } -bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap) +bool mem_cgroup_reclaimable(struct mem_cgroup *mem, bool noswap) { - return test_mem_cgroup_node_reclaimable(memcg, 0, noswap); + return test_mem_cgroup_node_reclaimable(mem, 0, noswap); } #endif @@ -1658,14 +1706,14 @@ bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap) * we reclaimed from, so that we don't end up penalizing one child extensively * based on its position in the children list. * - * root_memcg is the original ancestor that we've been reclaim from. + * root_mem is the original ancestor that we've been reclaim from. * - * We give up and return to the caller when we visit root_memcg twice. + * We give up and return to the caller when we visit root_mem twice. * (other groups can be removed while we're walking....) * * If shrink==true, for avoiding to free too much, this returns immedieately. */ -static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_memcg, +static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem, struct zone *zone, gfp_t gfp_mask, unsigned long reclaim_options, @@ -1680,15 +1728,15 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_memcg, unsigned long excess; unsigned long nr_scanned; - excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT; + excess = res_counter_soft_limit_excess(&root_mem->res) >> PAGE_SHIFT; /* If memsw_is_minimum==1, swap-out is of-no-use. */ - if (!check_soft && !shrink && root_memcg->memsw_is_minimum) + if (!check_soft && !shrink && root_mem->memsw_is_minimum) noswap = true; while (1) { - victim = mem_cgroup_select_victim(root_memcg); - if (victim == root_memcg) { + victim = mem_cgroup_select_victim(root_mem); + if (victim == root_mem) { loop++; /* * We are not draining per cpu cached charges during @@ -1697,7 +1745,7 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_memcg, * charges will not give any. */ if (!check_soft && loop >= 1) - drain_all_stock_async(root_memcg); + drain_all_stock_async(root_mem); if (loop >= 2) { /* * If we have not been able to reclaim @@ -1729,11 +1777,12 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_memcg, /* we use swappiness of local cgroup */ if (check_soft) { ret = mem_cgroup_shrink_node_zone(victim, gfp_mask, - noswap, zone, &nr_scanned); + noswap, get_swappiness(victim), zone, + &nr_scanned); *total_scanned += nr_scanned; } else ret = try_to_free_mem_cgroup_pages(victim, gfp_mask, - noswap); + noswap, get_swappiness(victim)); css_put(&victim->css); /* * At shrinking usage, we can't check we should stop here or @@ -1744,9 +1793,9 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_memcg, return ret; total += ret; if (check_soft) { - if (!res_counter_soft_limit_excess(&root_memcg->res)) + if (!res_counter_soft_limit_excess(&root_mem->res)) return total; - } else if (mem_cgroup_margin(root_memcg)) + } else if (mem_cgroup_margin(root_mem)) return total; } return total; @@ -1755,64 +1804,23 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_memcg, /* * Check OOM-Killer is already running under our hierarchy. * If someone is running, return false. - * Has to be called with memcg_oom_lock */ -static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg) +static bool mem_cgroup_oom_lock(struct mem_cgroup *mem) { - struct mem_cgroup *iter, *failed = NULL; - bool cond = true; + int x, lock_count = 0; + struct mem_cgroup *iter; - for_each_mem_cgroup_tree_cond(iter, memcg, cond) { - if (iter->oom_lock) { - /* - * this subtree of our hierarchy is already locked - * so we cannot give a lock. - */ - failed = iter; - cond = false; - } else - iter->oom_lock = true; + for_each_mem_cgroup_tree(iter, mem) { + x = atomic_inc_return(&iter->oom_lock); + lock_count = max(x, lock_count); } - if (!failed) + if (lock_count == 1) return true; - - /* - * OK, we failed to lock the whole subtree so we have to clean up - * what we set up to the failing subtree - */ - cond = true; - for_each_mem_cgroup_tree_cond(iter, memcg, cond) { - if (iter == failed) { - cond = false; - continue; - } - iter->oom_lock = false; - } return false; } -/* - * Has to be called with memcg_oom_lock - */ -static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg) -{ - struct mem_cgroup *iter; - - for_each_mem_cgroup_tree(iter, memcg) - iter->oom_lock = false; - return 0; -} - -static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg) -{ - struct mem_cgroup *iter; - - for_each_mem_cgroup_tree(iter, memcg) - atomic_inc(&iter->under_oom); -} - -static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) +static int mem_cgroup_oom_unlock(struct mem_cgroup *mem) { struct mem_cgroup *iter; @@ -1821,11 +1829,13 @@ static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg) * mem_cgroup_oom_lock() may not be called. We have to use * atomic_add_unless() here. */ - for_each_mem_cgroup_tree(iter, memcg) - atomic_add_unless(&iter->under_oom, -1, 0); + for_each_mem_cgroup_tree(iter, mem) + atomic_add_unless(&iter->oom_lock, -1, 0); + return 0; } -static DEFINE_SPINLOCK(memcg_oom_lock); + +static DEFINE_MUTEX(memcg_oom_mutex); static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); struct oom_wait_info { @@ -1836,85 +1846,85 @@ struct oom_wait_info { static int memcg_oom_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *arg) { - struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg, - *oom_wait_memcg; + struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg; struct oom_wait_info *oom_wait_info; oom_wait_info = container_of(wait, struct oom_wait_info, wait); - oom_wait_memcg = oom_wait_info->mem; + if (oom_wait_info->mem == wake_mem) + goto wakeup; + /* if no hierarchy, no match */ + if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy) + return 0; /* * Both of oom_wait_info->mem and wake_mem are stable under us. * Then we can use css_is_ancestor without taking care of RCU. */ - if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg) - && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg)) + if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) && + !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css)) return 0; + +wakeup: return autoremove_wake_function(wait, mode, sync, arg); } -static void memcg_wakeup_oom(struct mem_cgroup *memcg) +static void memcg_wakeup_oom(struct mem_cgroup *mem) { - /* for filtering, pass "memcg" as argument. */ - __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg); + /* for filtering, pass "mem" as argument. */ + __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem); } -static void memcg_oom_recover(struct mem_cgroup *memcg) +static void memcg_oom_recover(struct mem_cgroup *mem) { - if (memcg && atomic_read(&memcg->under_oom)) - memcg_wakeup_oom(memcg); + if (mem && atomic_read(&mem->oom_lock)) + memcg_wakeup_oom(mem); } /* * try to call OOM killer. returns false if we should exit memory-reclaim loop. */ -bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask) +bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask) { struct oom_wait_info owait; bool locked, need_to_kill; - owait.mem = memcg; + owait.mem = mem; owait.wait.flags = 0; owait.wait.func = memcg_oom_wake_function; owait.wait.private = current; INIT_LIST_HEAD(&owait.wait.task_list); need_to_kill = true; - mem_cgroup_mark_under_oom(memcg); - - /* At first, try to OOM lock hierarchy under memcg.*/ - spin_lock(&memcg_oom_lock); - locked = mem_cgroup_oom_lock(memcg); + /* At first, try to OOM lock hierarchy under mem.*/ + mutex_lock(&memcg_oom_mutex); + locked = mem_cgroup_oom_lock(mem); /* * Even if signal_pending(), we can't quit charge() loop without * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL * under OOM is always welcomed, use TASK_KILLABLE here. */ prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); - if (!locked || memcg->oom_kill_disable) + if (!locked || mem->oom_kill_disable) need_to_kill = false; if (locked) - mem_cgroup_oom_notify(memcg); - spin_unlock(&memcg_oom_lock); + mem_cgroup_oom_notify(mem); + mutex_unlock(&memcg_oom_mutex); if (need_to_kill) { finish_wait(&memcg_oom_waitq, &owait.wait); - mem_cgroup_out_of_memory(memcg, mask); + mem_cgroup_out_of_memory(mem, mask); } else { schedule(); finish_wait(&memcg_oom_waitq, &owait.wait); } - spin_lock(&memcg_oom_lock); - if (locked) - mem_cgroup_oom_unlock(memcg); - memcg_wakeup_oom(memcg); - spin_unlock(&memcg_oom_lock); - - mem_cgroup_unmark_under_oom(memcg); + mutex_lock(&memcg_oom_mutex); + mem_cgroup_oom_unlock(mem); + memcg_wakeup_oom(mem); + mutex_unlock(&memcg_oom_mutex); if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current)) return false; /* Give chance to dying process */ - schedule_timeout_uninterruptible(1); + schedule_timeout(1); return true; } @@ -1945,7 +1955,7 @@ bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask) void mem_cgroup_update_page_stat(struct page *page, enum mem_cgroup_page_stat_item idx, int val) { - struct mem_cgroup *memcg; + struct mem_cgroup *mem; struct page_cgroup *pc = lookup_page_cgroup(page); bool need_unlock = false; unsigned long uninitialized_var(flags); @@ -1954,16 +1964,16 @@ void mem_cgroup_update_page_stat(struct page *page, return; rcu_read_lock(); - memcg = pc->mem_cgroup; - if (unlikely(!memcg || !PageCgroupUsed(pc))) + mem = pc->mem_cgroup; + if (unlikely(!mem || !PageCgroupUsed(pc))) goto out; /* pc->mem_cgroup is unstable ? */ - if (unlikely(mem_cgroup_stealed(memcg)) || PageTransHuge(page)) { + if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) { /* take a lock against to access pc->mem_cgroup */ move_lock_page_cgroup(pc, &flags); need_unlock = true; - memcg = pc->mem_cgroup; - if (!memcg || !PageCgroupUsed(pc)) + mem = pc->mem_cgroup; + if (!mem || !PageCgroupUsed(pc)) goto out; } @@ -1979,7 +1989,7 @@ void mem_cgroup_update_page_stat(struct page *page, BUG(); } - this_cpu_add(memcg->stat->count[idx], val); + this_cpu_add(mem->stat->count[idx], val); out: if (unlikely(need_unlock)) @@ -2010,13 +2020,13 @@ static DEFINE_MUTEX(percpu_charge_mutex); * cgroup which is not current target, returns false. This stock will be * refilled. */ -static bool consume_stock(struct mem_cgroup *memcg) +static bool consume_stock(struct mem_cgroup *mem) { struct memcg_stock_pcp *stock; bool ret = true; stock = &get_cpu_var(memcg_stock); - if (memcg == stock->cached && stock->nr_pages) + if (mem == stock->cached && stock->nr_pages) stock->nr_pages--; else /* need to call res_counter_charge */ ret = false; @@ -2057,83 +2067,72 @@ static void drain_local_stock(struct work_struct *dummy) * Cache charges(val) which is from res_counter, to local per_cpu area. * This will be consumed by consume_stock() function, later. */ -static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) +static void refill_stock(struct mem_cgroup *mem, unsigned int nr_pages) { struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); - if (stock->cached != memcg) { /* reset if necessary */ + if (stock->cached != mem) { /* reset if necessary */ drain_stock(stock); - stock->cached = memcg; + stock->cached = mem; } stock->nr_pages += nr_pages; put_cpu_var(memcg_stock); } /* - * Drains all per-CPU charge caches for given root_memcg resp. subtree - * of the hierarchy under it. sync flag says whether we should block - * until the work is done. + * Tries to drain stocked charges in other cpus. This function is asynchronous + * and just put a work per cpu for draining localy on each cpu. Caller can + * expects some charges will be back to res_counter later but cannot wait for + * it. */ -static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync) +static void drain_all_stock_async(struct mem_cgroup *root_mem) { int cpu, curcpu; - + /* + * If someone calls draining, avoid adding more kworker runs. + */ + if (!mutex_trylock(&percpu_charge_mutex)) + return; /* Notify other cpus that system-wide "drain" is running */ get_online_cpus(); - curcpu = get_cpu(); + /* + * Get a hint for avoiding draining charges on the current cpu, + * which must be exhausted by our charging. It is not required that + * this be a precise check, so we use raw_smp_processor_id() instead of + * getcpu()/putcpu(). + */ + curcpu = raw_smp_processor_id(); for_each_online_cpu(cpu) { struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); - struct mem_cgroup *memcg; + struct mem_cgroup *mem; - memcg = stock->cached; - if (!memcg || !stock->nr_pages) + if (cpu == curcpu) continue; - if (!mem_cgroup_same_or_subtree(root_memcg, memcg)) + + mem = stock->cached; + if (!mem) continue; - if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { - if (cpu == curcpu) - drain_local_stock(&stock->work); - else - schedule_work_on(cpu, &stock->work); + if (mem != root_mem) { + if (!root_mem->use_hierarchy) + continue; + /* check whether "mem" is under tree of "root_mem" */ + if (!css_is_ancestor(&mem->css, &root_mem->css)) + continue; } + if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) + schedule_work_on(cpu, &stock->work); } - put_cpu(); - - if (!sync) - goto out; - - for_each_online_cpu(cpu) { - struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); - if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) - flush_work(&stock->work); - } -out: put_online_cpus(); -} - -/* - * Tries to drain stocked charges in other cpus. This function is asynchronous - * and just put a work per cpu for draining localy on each cpu. Caller can - * expects some charges will be back to res_counter later but cannot wait for - * it. - */ -static void drain_all_stock_async(struct mem_cgroup *root_memcg) -{ - /* - * If someone calls draining, avoid adding more kworker runs. - */ - if (!mutex_trylock(&percpu_charge_mutex)) - return; - drain_all_stock(root_memcg, false); mutex_unlock(&percpu_charge_mutex); + /* We don't wait for flush_work */ } /* This is a synchronous drain interface. */ -static void drain_all_stock_sync(struct mem_cgroup *root_memcg) +static void drain_all_stock_sync(void) { /* called when force_empty is called */ mutex_lock(&percpu_charge_mutex); - drain_all_stock(root_memcg, true); + schedule_on_each_cpu(drain_local_stock); mutex_unlock(&percpu_charge_mutex); } @@ -2141,35 +2140,35 @@ static void drain_all_stock_sync(struct mem_cgroup *root_memcg) * This function drains percpu counter value from DEAD cpu and * move it to local cpu. Note that this function can be preempted. */ -static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu) +static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu) { int i; - spin_lock(&memcg->pcp_counter_lock); + spin_lock(&mem->pcp_counter_lock); for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) { - long x = per_cpu(memcg->stat->count[i], cpu); + long x = per_cpu(mem->stat->count[i], cpu); - per_cpu(memcg->stat->count[i], cpu) = 0; - memcg->nocpu_base.count[i] += x; + per_cpu(mem->stat->count[i], cpu) = 0; + mem->nocpu_base.count[i] += x; } for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) { - unsigned long x = per_cpu(memcg->stat->events[i], cpu); + unsigned long x = per_cpu(mem->stat->events[i], cpu); - per_cpu(memcg->stat->events[i], cpu) = 0; - memcg->nocpu_base.events[i] += x; + per_cpu(mem->stat->events[i], cpu) = 0; + mem->nocpu_base.events[i] += x; } /* need to clear ON_MOVE value, works as a kind of lock. */ - per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0; - spin_unlock(&memcg->pcp_counter_lock); + per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0; + spin_unlock(&mem->pcp_counter_lock); } -static void synchronize_mem_cgroup_on_move(struct mem_cgroup *memcg, int cpu) +static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu) { int idx = MEM_CGROUP_ON_MOVE; - spin_lock(&memcg->pcp_counter_lock); - per_cpu(memcg->stat->count[idx], cpu) = memcg->nocpu_base.count[idx]; - spin_unlock(&memcg->pcp_counter_lock); + spin_lock(&mem->pcp_counter_lock); + per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx]; + spin_unlock(&mem->pcp_counter_lock); } static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb, @@ -2207,7 +2206,7 @@ enum { CHARGE_OOM_DIE, /* the current is killed because of OOM */ }; -static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, +static int mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask, unsigned int nr_pages, bool oom_check) { unsigned long csize = nr_pages * PAGE_SIZE; @@ -2216,16 +2215,16 @@ static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, unsigned long flags = 0; int ret; - ret = res_counter_charge(&memcg->res, csize, &fail_res); + ret = res_counter_charge(&mem->res, csize, &fail_res); if (likely(!ret)) { if (!do_swap_account) return CHARGE_OK; - ret = res_counter_charge(&memcg->memsw, csize, &fail_res); + ret = res_counter_charge(&mem->memsw, csize, &fail_res); if (likely(!ret)) return CHARGE_OK; - res_counter_uncharge(&memcg->res, csize); + res_counter_uncharge(&mem->res, csize); mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw); flags |= MEM_CGROUP_RECLAIM_NOSWAP; } else @@ -2283,12 +2282,12 @@ static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, static int __mem_cgroup_try_charge(struct mm_struct *mm, gfp_t gfp_mask, unsigned int nr_pages, - struct mem_cgroup **ptr, + struct mem_cgroup **memcg, bool oom) { unsigned int batch = max(CHARGE_BATCH, nr_pages); int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; - struct mem_cgroup *memcg = NULL; + struct mem_cgroup *mem = NULL; int ret; /* @@ -2306,17 +2305,17 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm, * thread group leader migrates. It's possible that mm is not * set, if so charge the init_mm (happens for pagecache usage). */ - if (!*ptr && !mm) + if (!*memcg && !mm) goto bypass; again: - if (*ptr) { /* css should be a valid one */ - memcg = *ptr; - VM_BUG_ON(css_is_removed(&memcg->css)); - if (mem_cgroup_is_root(memcg)) + if (*memcg) { /* css should be a valid one */ + mem = *memcg; + VM_BUG_ON(css_is_removed(&mem->css)); + if (mem_cgroup_is_root(mem)) goto done; - if (nr_pages == 1 && consume_stock(memcg)) + if (nr_pages == 1 && consume_stock(mem)) goto done; - css_get(&memcg->css); + css_get(&mem->css); } else { struct task_struct *p; @@ -2324,7 +2323,7 @@ again: p = rcu_dereference(mm->owner); /* * Because we don't have task_lock(), "p" can exit. - * In that case, "memcg" can point to root or p can be NULL with + * In that case, "mem" can point to root or p can be NULL with * race with swapoff. Then, we have small risk of mis-accouning. * But such kind of mis-account by race always happens because * we don't have cgroup_mutex(). It's overkill and we allo that @@ -2332,12 +2331,12 @@ again: * (*) swapoff at el will charge against mm-struct not against * task-struct. So, mm->owner can be NULL. */ - memcg = mem_cgroup_from_task(p); - if (!memcg || mem_cgroup_is_root(memcg)) { + mem = mem_cgroup_from_task(p); + if (!mem || mem_cgroup_is_root(mem)) { rcu_read_unlock(); goto done; } - if (nr_pages == 1 && consume_stock(memcg)) { + if (nr_pages == 1 && consume_stock(mem)) { /* * It seems dagerous to access memcg without css_get(). * But considering how consume_stok works, it's not @@ -2350,7 +2349,7 @@ again: goto done; } /* after here, we may be blocked. we need to get refcnt */ - if (!css_tryget(&memcg->css)) { + if (!css_tryget(&mem->css)) { rcu_read_unlock(); goto again; } @@ -2362,7 +2361,7 @@ again: /* If killed, bypass charge */ if (fatal_signal_pending(current)) { - css_put(&memcg->css); + css_put(&mem->css); goto bypass; } @@ -2372,43 +2371,43 @@ again: nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; } - ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, oom_check); + ret = mem_cgroup_do_charge(mem, gfp_mask, batch, oom_check); switch (ret) { case CHARGE_OK: break; case CHARGE_RETRY: /* not in OOM situation but retry */ batch = nr_pages; - css_put(&memcg->css); - memcg = NULL; + css_put(&mem->css); + mem = NULL; goto again; case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */ - css_put(&memcg->css); + css_put(&mem->css); goto nomem; case CHARGE_NOMEM: /* OOM routine works */ if (!oom) { - css_put(&memcg->css); + css_put(&mem->css); goto nomem; } /* If oom, we never return -ENOMEM */ nr_oom_retries--; break; case CHARGE_OOM_DIE: /* Killed by OOM Killer */ - css_put(&memcg->css); + css_put(&mem->css); goto bypass; } } while (ret != CHARGE_OK); if (batch > nr_pages) - refill_stock(memcg, batch - nr_pages); - css_put(&memcg->css); + refill_stock(mem, batch - nr_pages); + css_put(&mem->css); done: - *ptr = memcg; + *memcg = mem; return 0; nomem: - *ptr = NULL; + *memcg = NULL; return -ENOMEM; bypass: - *ptr = NULL; + *memcg = NULL; return 0; } @@ -2417,15 +2416,15 @@ bypass: * This function is for that and do uncharge, put css's refcnt. * gotten by try_charge(). */ -static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg, +static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem, unsigned int nr_pages) { - if (!mem_cgroup_is_root(memcg)) { + if (!mem_cgroup_is_root(mem)) { unsigned long bytes = nr_pages * PAGE_SIZE; - res_counter_uncharge(&memcg->res, bytes); + res_counter_uncharge(&mem->res, bytes); if (do_swap_account) - res_counter_uncharge(&memcg->memsw, bytes); + res_counter_uncharge(&mem->memsw, bytes); } } @@ -2450,7 +2449,7 @@ static struct mem_cgroup *mem_cgroup_lookup(unsigned short id) struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) { - struct mem_cgroup *memcg = NULL; + struct mem_cgroup *mem = NULL; struct page_cgroup *pc; unsigned short id; swp_entry_t ent; @@ -2460,23 +2459,23 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) pc = lookup_page_cgroup(page); lock_page_cgroup(pc); if (PageCgroupUsed(pc)) { - memcg = pc->mem_cgroup; - if (memcg && !css_tryget(&memcg->css)) - memcg = NULL; + mem = pc->mem_cgroup; + if (mem && !css_tryget(&mem->css)) + mem = NULL; } else if (PageSwapCache(page)) { ent.val = page_private(page); id = lookup_swap_cgroup(ent); rcu_read_lock(); - memcg = mem_cgroup_lookup(id); - if (memcg && !css_tryget(&memcg->css)) - memcg = NULL; + mem = mem_cgroup_lookup(id); + if (mem && !css_tryget(&mem->css)) + mem = NULL; rcu_read_unlock(); } unlock_page_cgroup(pc); - return memcg; + return mem; } -static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg, +static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, struct page *page, unsigned int nr_pages, struct page_cgroup *pc, @@ -2485,14 +2484,14 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg, lock_page_cgroup(pc); if (unlikely(PageCgroupUsed(pc))) { unlock_page_cgroup(pc); - __mem_cgroup_cancel_charge(memcg, nr_pages); + __mem_cgroup_cancel_charge(mem, nr_pages); return; } /* * we don't need page_cgroup_lock about tail pages, becase they are not * accessed by any other context at this point. */ - pc->mem_cgroup = memcg; + pc->mem_cgroup = mem; /* * We access a page_cgroup asynchronously without lock_page_cgroup(). * Especially when a page_cgroup is taken from a page, pc->mem_cgroup @@ -2515,14 +2514,14 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg, break; } - mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), nr_pages); + mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages); unlock_page_cgroup(pc); /* * "charge_statistics" updated event counter. Then, check it. * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. * if they exceeds softlimit. */ - memcg_check_events(memcg, page); + memcg_check_events(mem, page); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -2709,7 +2708,7 @@ out: static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, gfp_t gfp_mask, enum charge_type ctype) { - struct mem_cgroup *memcg = NULL; + struct mem_cgroup *mem = NULL; unsigned int nr_pages = 1; struct page_cgroup *pc; bool oom = true; @@ -2728,11 +2727,11 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, pc = lookup_page_cgroup(page); BUG_ON(!pc); /* XXX: remove this and move pc lookup into commit */ - ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom); - if (ret || !memcg) + ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &mem, oom); + if (ret || !mem) return ret; - __mem_cgroup_commit_charge(memcg, page, nr_pages, pc, ctype); + __mem_cgroup_commit_charge(mem, page, nr_pages, pc, ctype); return 0; } @@ -2761,7 +2760,7 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, enum charge_type ctype); static void -__mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg, +__mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *mem, enum charge_type ctype) { struct page_cgroup *pc = lookup_page_cgroup(page); @@ -2771,7 +2770,7 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg, * LRU. Take care of it. */ mem_cgroup_lru_del_before_commit(page); - __mem_cgroup_commit_charge(memcg, page, 1, pc, ctype); + __mem_cgroup_commit_charge(mem, page, 1, pc, ctype); mem_cgroup_lru_add_after_commit(page); return; } @@ -2779,20 +2778,44 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg, int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask) { - struct mem_cgroup *memcg = NULL; + struct mem_cgroup *mem = NULL; int ret; if (mem_cgroup_disabled()) return 0; if (PageCompound(page)) return 0; + /* + * Corner case handling. This is called from add_to_page_cache() + * in usual. But some FS (shmem) precharges this page before calling it + * and call add_to_page_cache() with GFP_NOWAIT. + * + * For GFP_NOWAIT case, the page may be pre-charged before calling + * add_to_page_cache(). (See shmem.c) check it here and avoid to call + * charge twice. (It works but has to pay a bit larger cost.) + * And when the page is SwapCache, it should take swap information + * into account. This is under lock_page() now. + */ + if (!(gfp_mask & __GFP_WAIT)) { + struct page_cgroup *pc; + + pc = lookup_page_cgroup(page); + if (!pc) + return 0; + lock_page_cgroup(pc); + if (PageCgroupUsed(pc)) { + unlock_page_cgroup(pc); + return 0; + } + unlock_page_cgroup(pc); + } if (unlikely(!mm)) mm = &init_mm; if (page_is_file_cache(page)) { - ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, &memcg, true); - if (ret || !memcg) + ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, &mem, true); + if (ret || !mem) return ret; /* @@ -2800,15 +2823,15 @@ int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, * put that would remove them from the LRU list, make * sure that they get relinked properly. */ - __mem_cgroup_commit_charge_lrucare(page, memcg, + __mem_cgroup_commit_charge_lrucare(page, mem, MEM_CGROUP_CHARGE_TYPE_CACHE); return ret; } /* shmem */ if (PageSwapCache(page)) { - ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &memcg); + ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); if (!ret) - __mem_cgroup_commit_charge_swapin(page, memcg, + __mem_cgroup_commit_charge_swapin(page, mem, MEM_CGROUP_CHARGE_TYPE_SHMEM); } else ret = mem_cgroup_charge_common(page, mm, gfp_mask, @@ -2827,7 +2850,7 @@ int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page, gfp_t mask, struct mem_cgroup **ptr) { - struct mem_cgroup *memcg; + struct mem_cgroup *mem; int ret; *ptr = NULL; @@ -2845,12 +2868,12 @@ int mem_cgroup_try_charge_swapin(struct mm_struct *mm, */ if (!PageSwapCache(page)) goto charge_cur_mm; - memcg = try_get_mem_cgroup_from_page(page); - if (!memcg) + mem = try_get_mem_cgroup_from_page(page); + if (!mem) goto charge_cur_mm; - *ptr = memcg; + *ptr = mem; ret = __mem_cgroup_try_charge(NULL, mask, 1, ptr, true); - css_put(&memcg->css); + css_put(&mem->css); return ret; charge_cur_mm: if (unlikely(!mm)) @@ -2910,16 +2933,16 @@ void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr) MEM_CGROUP_CHARGE_TYPE_MAPPED); } -void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg) +void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem) { if (mem_cgroup_disabled()) return; - if (!memcg) + if (!mem) return; - __mem_cgroup_cancel_charge(memcg, 1); + __mem_cgroup_cancel_charge(mem, 1); } -static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg, +static void mem_cgroup_do_uncharge(struct mem_cgroup *mem, unsigned int nr_pages, const enum charge_type ctype) { @@ -2937,7 +2960,7 @@ static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg, * uncharges. Then, it's ok to ignore memcg's refcnt. */ if (!batch->memcg) - batch->memcg = memcg; + batch->memcg = mem; /* * do_batch > 0 when unmapping pages or inode invalidate/truncate. * In those cases, all pages freed continuously can be expected to be in @@ -2957,7 +2980,7 @@ static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg, * merge a series of uncharges to an uncharge of res_counter. * If not, we uncharge res_counter ony by one. */ - if (batch->memcg != memcg) + if (batch->memcg != mem) goto direct_uncharge; /* remember freed charge and uncharge it later */ batch->nr_pages++; @@ -2965,11 +2988,11 @@ static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg, batch->memsw_nr_pages++; return; direct_uncharge: - res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE); + res_counter_uncharge(&mem->res, nr_pages * PAGE_SIZE); if (uncharge_memsw) - res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE); - if (unlikely(batch->memcg != memcg)) - memcg_oom_recover(memcg); + res_counter_uncharge(&mem->memsw, nr_pages * PAGE_SIZE); + if (unlikely(batch->memcg != mem)) + memcg_oom_recover(mem); return; } @@ -2979,7 +3002,7 @@ direct_uncharge: static struct mem_cgroup * __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) { - struct mem_cgroup *memcg = NULL; + struct mem_cgroup *mem = NULL; unsigned int nr_pages = 1; struct page_cgroup *pc; @@ -3002,7 +3025,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) lock_page_cgroup(pc); - memcg = pc->mem_cgroup; + mem = pc->mem_cgroup; if (!PageCgroupUsed(pc)) goto unlock_out; @@ -3025,7 +3048,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) break; } - mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), -nr_pages); + mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -nr_pages); ClearPageCgroupUsed(pc); /* @@ -3037,18 +3060,18 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) unlock_page_cgroup(pc); /* - * even after unlock, we have memcg->res.usage here and this memcg + * even after unlock, we have mem->res.usage here and this memcg * will never be freed. */ - memcg_check_events(memcg, page); + memcg_check_events(mem, page); if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) { - mem_cgroup_swap_statistics(memcg, true); - mem_cgroup_get(memcg); + mem_cgroup_swap_statistics(mem, true); + mem_cgroup_get(mem); } - if (!mem_cgroup_is_root(memcg)) - mem_cgroup_do_uncharge(memcg, nr_pages, ctype); + if (!mem_cgroup_is_root(mem)) + mem_cgroup_do_uncharge(mem, nr_pages, ctype); - return memcg; + return mem; unlock_out: unlock_page_cgroup(pc); @@ -3238,7 +3261,7 @@ static inline int mem_cgroup_move_swap_account(swp_entry_t entry, int mem_cgroup_prepare_migration(struct page *page, struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask) { - struct mem_cgroup *memcg = NULL; + struct mem_cgroup *mem = NULL; struct page_cgroup *pc; enum charge_type ctype; int ret = 0; @@ -3252,8 +3275,8 @@ int mem_cgroup_prepare_migration(struct page *page, pc = lookup_page_cgroup(page); lock_page_cgroup(pc); if (PageCgroupUsed(pc)) { - memcg = pc->mem_cgroup; - css_get(&memcg->css); + mem = pc->mem_cgroup; + css_get(&mem->css); /* * At migrating an anonymous page, its mapcount goes down * to 0 and uncharge() will be called. But, even if it's fully @@ -3291,12 +3314,12 @@ int mem_cgroup_prepare_migration(struct page *page, * If the page is not charged at this point, * we return here. */ - if (!memcg) + if (!mem) return 0; - *ptr = memcg; + *ptr = mem; ret = __mem_cgroup_try_charge(NULL, gfp_mask, 1, ptr, false); - css_put(&memcg->css);/* drop extra refcnt */ + css_put(&mem->css);/* drop extra refcnt */ if (ret || *ptr == NULL) { if (PageAnon(page)) { lock_page_cgroup(pc); @@ -3322,21 +3345,21 @@ int mem_cgroup_prepare_migration(struct page *page, ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; else ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM; - __mem_cgroup_commit_charge(memcg, page, 1, pc, ctype); + __mem_cgroup_commit_charge(mem, page, 1, pc, ctype); return ret; } /* remove redundant charge if migration failed*/ -void mem_cgroup_end_migration(struct mem_cgroup *memcg, +void mem_cgroup_end_migration(struct mem_cgroup *mem, struct page *oldpage, struct page *newpage, bool migration_ok) { struct page *used, *unused; struct page_cgroup *pc; - if (!memcg) + if (!mem) return; /* blocks rmdir() */ - cgroup_exclude_rmdir(&memcg->css); + cgroup_exclude_rmdir(&mem->css); if (!migration_ok) { used = oldpage; unused = newpage; @@ -3372,7 +3395,32 @@ void mem_cgroup_end_migration(struct mem_cgroup *memcg, * So, rmdir()->pre_destroy() can be called while we do this charge. * In that case, we need to call pre_destroy() again. check it here. */ - cgroup_release_and_wakeup_rmdir(&memcg->css); + cgroup_release_and_wakeup_rmdir(&mem->css); +} + +/* + * A call to try to shrink memory usage on charge failure at shmem's swapin. + * Calling hierarchical_reclaim is not enough because we should update + * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM. + * Moreover considering hierarchy, we should reclaim from the mem_over_limit, + * not from the memcg which this page would be charged to. + * try_charge_swapin does all of these works properly. + */ +int mem_cgroup_shmem_charge_fallback(struct page *page, + struct mm_struct *mm, + gfp_t gfp_mask) +{ + struct mem_cgroup *mem; + int ret; + + if (mem_cgroup_disabled()) + return 0; + + ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); + if (!ret) + mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */ + + return ret; } /* @@ -3495,7 +3543,7 @@ static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, /* * Rather than hide all in some function, I do this in * open coded manner. You see what this really does. - * We have to guarantee memcg->res.limit < memcg->memsw.limit. + * We have to guarantee mem->res.limit < mem->memsw.limit. */ mutex_lock(&set_limit_mutex); memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT); @@ -3557,7 +3605,7 @@ static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, /* * Rather than hide all in some function, I do this in * open coded manner. You see what this really does. - * We have to guarantee memcg->res.limit < memcg->memsw.limit. + * We have to guarantee mem->res.limit < mem->memsw.limit. */ mutex_lock(&set_limit_mutex); memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT); @@ -3695,7 +3743,7 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, * This routine traverse page_cgroup in given list and drop them all. * *And* this routine doesn't reclaim page itself, just removes page_cgroup. */ -static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg, +static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, int node, int zid, enum lru_list lru) { struct zone *zone; @@ -3706,7 +3754,7 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg, int ret = 0; zone = &NODE_DATA(node)->node_zones[zid]; - mz = mem_cgroup_zoneinfo(memcg, node, zid); + mz = mem_cgroup_zoneinfo(mem, node, zid); list = &mz->lists[lru]; loop = MEM_CGROUP_ZSTAT(mz, lru); @@ -3733,7 +3781,7 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg, page = lookup_cgroup_page(pc); - ret = mem_cgroup_move_parent(page, pc, memcg, GFP_KERNEL); + ret = mem_cgroup_move_parent(page, pc, mem, GFP_KERNEL); if (ret == -ENOMEM) break; @@ -3754,14 +3802,14 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg, * make mem_cgroup's charge to be 0 if there is no task. * This enables deleting this mem_cgroup. */ -static int mem_cgroup_force_empty(struct mem_cgroup *memcg, bool free_all) +static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all) { int ret; int node, zid, shrink; int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; - struct cgroup *cgrp = memcg->css.cgroup; + struct cgroup *cgrp = mem->css.cgroup; - css_get(&memcg->css); + css_get(&mem->css); shrink = 0; /* should free all ? */ @@ -3777,14 +3825,14 @@ move_account: goto out; /* This is for making all *used* pages to be on LRU. */ lru_add_drain_all(); - drain_all_stock_sync(memcg); + drain_all_stock_sync(); ret = 0; - mem_cgroup_start_move(memcg); + mem_cgroup_start_move(mem); for_each_node_state(node, N_HIGH_MEMORY) { for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) { enum lru_list l; for_each_lru(l) { - ret = mem_cgroup_force_empty_list(memcg, + ret = mem_cgroup_force_empty_list(mem, node, zid, l); if (ret) break; @@ -3793,16 +3841,16 @@ move_account: if (ret) break; } - mem_cgroup_end_move(memcg); - memcg_oom_recover(memcg); + mem_cgroup_end_move(mem); + memcg_oom_recover(mem); /* it seems parent cgroup doesn't have enough mem */ if (ret == -ENOMEM) goto try_to_free; cond_resched(); /* "ret" should also be checked to ensure all lists are empty. */ - } while (memcg->res.usage > 0 || ret); + } while (mem->res.usage > 0 || ret); out: - css_put(&memcg->css); + css_put(&mem->css); return ret; try_to_free: @@ -3815,15 +3863,15 @@ try_to_free: lru_add_drain_all(); /* try to free all pages in this cgroup */ shrink = 1; - while (nr_retries && memcg->res.usage > 0) { + while (nr_retries && mem->res.usage > 0) { int progress; if (signal_pending(current)) { ret = -EINTR; goto out; } - progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL, - false); + progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL, + false, get_swappiness(mem)); if (!progress) { nr_retries--; /* maybe some writeback is necessary */ @@ -3851,12 +3899,12 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft, u64 val) { int retval = 0; - struct mem_cgroup *memcg = mem_cgroup_from_cont(cont); + struct mem_cgroup *mem = mem_cgroup_from_cont(cont); struct cgroup *parent = cont->parent; - struct mem_cgroup *parent_memcg = NULL; + struct mem_cgroup *parent_mem = NULL; if (parent) - parent_memcg = mem_cgroup_from_cont(parent); + parent_mem = mem_cgroup_from_cont(parent); cgroup_lock(); /* @@ -3867,10 +3915,10 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft, * For the root cgroup, parent_mem is NULL, we allow value to be * set if there are no children. */ - if ((!parent_memcg || !parent_memcg->use_hierarchy) && + if ((!parent_mem || !parent_mem->use_hierarchy) && (val == 1 || val == 0)) { if (list_empty(&cont->children)) - memcg->use_hierarchy = val; + mem->use_hierarchy = val; else retval = -EBUSY; } else @@ -3881,14 +3929,14 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft, } -static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg, +static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *mem, enum mem_cgroup_stat_index idx) { struct mem_cgroup *iter; long val = 0; /* Per-cpu values can be negative, use a signed accumulator */ - for_each_mem_cgroup_tree(iter, memcg) + for_each_mem_cgroup_tree(iter, mem) val += mem_cgroup_read_stat(iter, idx); if (val < 0) /* race ? */ @@ -3896,29 +3944,29 @@ static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg, return val; } -static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) +static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap) { u64 val; - if (!mem_cgroup_is_root(memcg)) { + if (!mem_cgroup_is_root(mem)) { if (!swap) - return res_counter_read_u64(&memcg->res, RES_USAGE); + return res_counter_read_u64(&mem->res, RES_USAGE); else - return res_counter_read_u64(&memcg->memsw, RES_USAGE); + return res_counter_read_u64(&mem->memsw, RES_USAGE); } - val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE); - val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS); + val = mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_CACHE); + val += mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_RSS); if (swap) - val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAPOUT); + val += mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_SWAPOUT); return val << PAGE_SHIFT; } static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft) { - struct mem_cgroup *memcg = mem_cgroup_from_cont(cont); + struct mem_cgroup *mem = mem_cgroup_from_cont(cont); u64 val; int type, name; @@ -3927,15 +3975,15 @@ static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft) switch (type) { case _MEM: if (name == RES_USAGE) - val = mem_cgroup_usage(memcg, false); + val = mem_cgroup_usage(mem, false); else - val = res_counter_read_u64(&memcg->res, name); + val = res_counter_read_u64(&mem->res, name); break; case _MEMSWAP: if (name == RES_USAGE) - val = mem_cgroup_usage(memcg, true); + val = mem_cgroup_usage(mem, true); else - val = res_counter_read_u64(&memcg->memsw, name); + val = res_counter_read_u64(&mem->memsw, name); break; default: BUG(); @@ -4023,24 +4071,24 @@ out: static int mem_cgroup_reset(struct cgroup *cont, unsigned int event) { - struct mem_cgroup *memcg; + struct mem_cgroup *mem; int type, name; - memcg = mem_cgroup_from_cont(cont); + mem = mem_cgroup_from_cont(cont); type = MEMFILE_TYPE(event); name = MEMFILE_ATTR(event); switch (name) { case RES_MAX_USAGE: if (type == _MEM) - res_counter_reset_max(&memcg->res); + res_counter_reset_max(&mem->res); else - res_counter_reset_max(&memcg->memsw); + res_counter_reset_max(&mem->memsw); break; case RES_FAILCNT: if (type == _MEM) - res_counter_reset_failcnt(&memcg->res); + res_counter_reset_failcnt(&mem->res); else - res_counter_reset_failcnt(&memcg->memsw); + res_counter_reset_failcnt(&mem->memsw); break; } @@ -4057,7 +4105,7 @@ static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp, static int mem_cgroup_move_charge_write(struct cgroup *cgrp, struct cftype *cft, u64 val) { - struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); + struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); if (val >= (1 << NR_MOVE_TYPE)) return -EINVAL; @@ -4067,7 +4115,7 @@ static int mem_cgroup_move_charge_write(struct cgroup *cgrp, * inconsistent. */ cgroup_lock(); - memcg->move_charge_at_immigrate = val; + mem->move_charge_at_immigrate = val; cgroup_unlock(); return 0; @@ -4124,49 +4172,49 @@ struct { static void -mem_cgroup_get_local_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s) +mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) { s64 val; /* per cpu stat */ - val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_CACHE); + val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); s->stat[MCS_CACHE] += val * PAGE_SIZE; - val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_RSS); + val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); s->stat[MCS_RSS] += val * PAGE_SIZE; - val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED); + val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED); s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE; - val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGPGIN); + val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGPGIN); s->stat[MCS_PGPGIN] += val; - val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGPGOUT); + val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGPGOUT); s->stat[MCS_PGPGOUT] += val; if (do_swap_account) { - val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_SWAPOUT); + val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT); s->stat[MCS_SWAP] += val * PAGE_SIZE; } - val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGFAULT); + val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGFAULT); s->stat[MCS_PGFAULT] += val; - val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGMAJFAULT); + val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGMAJFAULT); s->stat[MCS_PGMAJFAULT] += val; /* per zone stat */ - val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_ANON)); + val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON); s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE; - val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_ANON)); + val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON); s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE; - val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_FILE)); + val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE); s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE; - val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_FILE)); + val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE); s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE; - val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE)); + val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE); s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE; } static void -mem_cgroup_get_total_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s) +mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s) { struct mem_cgroup *iter; - for_each_mem_cgroup_tree(iter, memcg) + for_each_mem_cgroup_tree(iter, mem) mem_cgroup_get_local_stat(iter, s); } @@ -4179,37 +4227,35 @@ static int mem_control_numa_stat_show(struct seq_file *m, void *arg) struct cgroup *cont = m->private; struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont); - total_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL); + total_nr = mem_cgroup_nr_lru_pages(mem_cont); seq_printf(m, "total=%lu", total_nr); for_each_node_state(nid, N_HIGH_MEMORY) { - node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, LRU_ALL); + node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid); seq_printf(m, " N%d=%lu", nid, node_nr); } seq_putc(m, '\n'); - file_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_FILE); + file_nr = mem_cgroup_nr_file_lru_pages(mem_cont); seq_printf(m, "file=%lu", file_nr); for_each_node_state(nid, N_HIGH_MEMORY) { - node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, - LRU_ALL_FILE); + node_nr = mem_cgroup_node_nr_file_lru_pages(mem_cont, nid); seq_printf(m, " N%d=%lu", nid, node_nr); } seq_putc(m, '\n'); - anon_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_ANON); + anon_nr = mem_cgroup_nr_anon_lru_pages(mem_cont); seq_printf(m, "anon=%lu", anon_nr); for_each_node_state(nid, N_HIGH_MEMORY) { - node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, - LRU_ALL_ANON); + node_nr = mem_cgroup_node_nr_anon_lru_pages(mem_cont, nid); seq_printf(m, " N%d=%lu", nid, node_nr); } seq_putc(m, '\n'); - unevictable_nr = mem_cgroup_nr_lru_pages(mem_cont, BIT(LRU_UNEVICTABLE)); + unevictable_nr = mem_cgroup_nr_unevictable_lru_pages(mem_cont); seq_printf(m, "unevictable=%lu", unevictable_nr); for_each_node_state(nid, N_HIGH_MEMORY) { - node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, - BIT(LRU_UNEVICTABLE)); + node_nr = mem_cgroup_node_nr_unevictable_lru_pages(mem_cont, + nid); seq_printf(m, " N%d=%lu", nid, node_nr); } seq_putc(m, '\n'); @@ -4252,6 +4298,8 @@ static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft, } #ifdef CONFIG_DEBUG_VM + cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL)); + { int nid, zid; struct mem_cgroup_per_zone *mz; @@ -4285,7 +4333,7 @@ static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft) { struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); - return mem_cgroup_swappiness(memcg); + return get_swappiness(memcg); } static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft, @@ -4394,20 +4442,20 @@ static int compare_thresholds(const void *a, const void *b) return 0; } -static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg) +static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem) { struct mem_cgroup_eventfd_list *ev; - list_for_each_entry(ev, &memcg->oom_notify, list) + list_for_each_entry(ev, &mem->oom_notify, list) eventfd_signal(ev->eventfd, 1); return 0; } -static void mem_cgroup_oom_notify(struct mem_cgroup *memcg) +static void mem_cgroup_oom_notify(struct mem_cgroup *mem) { struct mem_cgroup *iter; - for_each_mem_cgroup_tree(iter, memcg) + for_each_mem_cgroup_tree(iter, mem) mem_cgroup_oom_notify_cb(iter); } @@ -4590,15 +4638,15 @@ static int mem_cgroup_oom_register_event(struct cgroup *cgrp, if (!event) return -ENOMEM; - spin_lock(&memcg_oom_lock); + mutex_lock(&memcg_oom_mutex); event->eventfd = eventfd; list_add(&event->list, &memcg->oom_notify); /* already in OOM ? */ - if (atomic_read(&memcg->under_oom)) + if (atomic_read(&memcg->oom_lock)) eventfd_signal(eventfd, 1); - spin_unlock(&memcg_oom_lock); + mutex_unlock(&memcg_oom_mutex); return 0; } @@ -4606,32 +4654,32 @@ static int mem_cgroup_oom_register_event(struct cgroup *cgrp, static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp, struct cftype *cft, struct eventfd_ctx *eventfd) { - struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); + struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); struct mem_cgroup_eventfd_list *ev, *tmp; int type = MEMFILE_TYPE(cft->private); BUG_ON(type != _OOM_TYPE); - spin_lock(&memcg_oom_lock); + mutex_lock(&memcg_oom_mutex); - list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) { + list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) { if (ev->eventfd == eventfd) { list_del(&ev->list); kfree(ev); } } - spin_unlock(&memcg_oom_lock); + mutex_unlock(&memcg_oom_mutex); } static int mem_cgroup_oom_control_read(struct cgroup *cgrp, struct cftype *cft, struct cgroup_map_cb *cb) { - struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); + struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); - cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable); + cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable); - if (atomic_read(&memcg->under_oom)) + if (atomic_read(&mem->oom_lock)) cb->fill(cb, "under_oom", 1); else cb->fill(cb, "under_oom", 0); @@ -4641,7 +4689,7 @@ static int mem_cgroup_oom_control_read(struct cgroup *cgrp, static int mem_cgroup_oom_control_write(struct cgroup *cgrp, struct cftype *cft, u64 val) { - struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp); + struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp); struct mem_cgroup *parent; /* cannot set to root cgroup and only 0 and 1 are allowed */ @@ -4653,13 +4701,13 @@ static int mem_cgroup_oom_control_write(struct cgroup *cgrp, cgroup_lock(); /* oom-kill-disable is a flag for subhierarchy. */ if ((parent->use_hierarchy) || - (memcg->use_hierarchy && !list_empty(&cgrp->children))) { + (mem->use_hierarchy && !list_empty(&cgrp->children))) { cgroup_unlock(); return -EINVAL; } - memcg->oom_kill_disable = val; + mem->oom_kill_disable = val; if (!val) - memcg_oom_recover(memcg); + memcg_oom_recover(mem); cgroup_unlock(); return 0; } @@ -4795,7 +4843,7 @@ static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss) } #endif -static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node) +static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) { struct mem_cgroup_per_node *pn; struct mem_cgroup_per_zone *mz; @@ -4815,21 +4863,21 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node) if (!pn) return 1; + mem->info.nodeinfo[node] = pn; for (zone = 0; zone < MAX_NR_ZONES; zone++) { mz = &pn->zoneinfo[zone]; for_each_lru(l) INIT_LIST_HEAD(&mz->lists[l]); mz->usage_in_excess = 0; mz->on_tree = false; - mz->mem = memcg; + mz->mem = mem; } - memcg->info.nodeinfo[node] = pn; return 0; } -static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node) +static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) { - kfree(memcg->info.nodeinfo[node]); + kfree(mem->info.nodeinfo[node]); } static struct mem_cgroup *mem_cgroup_alloc(void) @@ -4871,51 +4919,51 @@ out_free: * Removal of cgroup itself succeeds regardless of refs from swap. */ -static void __mem_cgroup_free(struct mem_cgroup *memcg) +static void __mem_cgroup_free(struct mem_cgroup *mem) { int node; - mem_cgroup_remove_from_trees(memcg); - free_css_id(&mem_cgroup_subsys, &memcg->css); + mem_cgroup_remove_from_trees(mem); + free_css_id(&mem_cgroup_subsys, &mem->css); for_each_node_state(node, N_POSSIBLE) - free_mem_cgroup_per_zone_info(memcg, node); + free_mem_cgroup_per_zone_info(mem, node); - free_percpu(memcg->stat); + free_percpu(mem->stat); if (sizeof(struct mem_cgroup) < PAGE_SIZE) - kfree(memcg); + kfree(mem); else - vfree(memcg); + vfree(mem); } -static void mem_cgroup_get(struct mem_cgroup *memcg) +static void mem_cgroup_get(struct mem_cgroup *mem) { - atomic_inc(&memcg->refcnt); + atomic_inc(&mem->refcnt); } -static void __mem_cgroup_put(struct mem_cgroup *memcg, int count) +static void __mem_cgroup_put(struct mem_cgroup *mem, int count) { - if (atomic_sub_and_test(count, &memcg->refcnt)) { - struct mem_cgroup *parent = parent_mem_cgroup(memcg); - __mem_cgroup_free(memcg); + if (atomic_sub_and_test(count, &mem->refcnt)) { + struct mem_cgroup *parent = parent_mem_cgroup(mem); + __mem_cgroup_free(mem); if (parent) mem_cgroup_put(parent); } } -static void mem_cgroup_put(struct mem_cgroup *memcg) +static void mem_cgroup_put(struct mem_cgroup *mem) { - __mem_cgroup_put(memcg, 1); + __mem_cgroup_put(mem, 1); } /* * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled. */ -static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) +static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem) { - if (!memcg->res.parent) + if (!mem->res.parent) return NULL; - return mem_cgroup_from_res_counter(memcg->res.parent, res); + return mem_cgroup_from_res_counter(mem->res.parent, res); } #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP @@ -4958,16 +5006,16 @@ static int mem_cgroup_soft_limit_tree_init(void) static struct cgroup_subsys_state * __ref mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) { - struct mem_cgroup *memcg, *parent; + struct mem_cgroup *mem, *parent; long error = -ENOMEM; int node; - memcg = mem_cgroup_alloc(); - if (!memcg) + mem = mem_cgroup_alloc(); + if (!mem) return ERR_PTR(error); for_each_node_state(node, N_POSSIBLE) - if (alloc_mem_cgroup_per_zone_info(memcg, node)) + if (alloc_mem_cgroup_per_zone_info(mem, node)) goto free_out; /* root ? */ @@ -4977,7 +5025,7 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) parent = NULL; if (mem_cgroup_soft_limit_tree_init()) goto free_out; - root_mem_cgroup = memcg; + root_mem_cgroup = mem; for_each_possible_cpu(cpu) { struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); @@ -4986,13 +5034,13 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) hotcpu_notifier(memcg_cpu_hotplug_callback, 0); } else { parent = mem_cgroup_from_cont(cont->parent); - memcg->use_hierarchy = parent->use_hierarchy; - memcg->oom_kill_disable = parent->oom_kill_disable; + mem->use_hierarchy = parent->use_hierarchy; + mem->oom_kill_disable = parent->oom_kill_disable; } if (parent && parent->use_hierarchy) { - res_counter_init(&memcg->res, &parent->res); - res_counter_init(&memcg->memsw, &parent->memsw); + res_counter_init(&mem->res, &parent->res); + res_counter_init(&mem->memsw, &parent->memsw); /* * We increment refcnt of the parent to ensure that we can * safely access it on res_counter_charge/uncharge. @@ -5001,38 +5049,38 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) */ mem_cgroup_get(parent); } else { - res_counter_init(&memcg->res, NULL); - res_counter_init(&memcg->memsw, NULL); + res_counter_init(&mem->res, NULL); + res_counter_init(&mem->memsw, NULL); } - memcg->last_scanned_child = 0; - memcg->last_scanned_node = MAX_NUMNODES; - INIT_LIST_HEAD(&memcg->oom_notify); + mem->last_scanned_child = 0; + mem->last_scanned_node = MAX_NUMNODES; + INIT_LIST_HEAD(&mem->oom_notify); if (parent) - memcg->swappiness = mem_cgroup_swappiness(parent); - atomic_set(&memcg->refcnt, 1); - memcg->move_charge_at_immigrate = 0; - mutex_init(&memcg->thresholds_lock); - return &memcg->css; + mem->swappiness = get_swappiness(parent); + atomic_set(&mem->refcnt, 1); + mem->move_charge_at_immigrate = 0; + mutex_init(&mem->thresholds_lock); + return &mem->css; free_out: - __mem_cgroup_free(memcg); + __mem_cgroup_free(mem); return ERR_PTR(error); } static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss, struct cgroup *cont) { - struct mem_cgroup *memcg = mem_cgroup_from_cont(cont); + struct mem_cgroup *mem = mem_cgroup_from_cont(cont); - return mem_cgroup_force_empty(memcg, false); + return mem_cgroup_force_empty(mem, false); } static void mem_cgroup_destroy(struct cgroup_subsys *ss, struct cgroup *cont) { - struct mem_cgroup *memcg = mem_cgroup_from_cont(cont); + struct mem_cgroup *mem = mem_cgroup_from_cont(cont); - mem_cgroup_put(memcg); + mem_cgroup_put(mem); } static int mem_cgroup_populate(struct cgroup_subsys *ss, @@ -5055,9 +5103,9 @@ static int mem_cgroup_do_precharge(unsigned long count) { int ret = 0; int batch_count = PRECHARGE_COUNT_AT_ONCE; - struct mem_cgroup *memcg = mc.to; + struct mem_cgroup *mem = mc.to; - if (mem_cgroup_is_root(memcg)) { + if (mem_cgroup_is_root(mem)) { mc.precharge += count; /* we don't need css_get for root */ return ret; @@ -5066,16 +5114,16 @@ static int mem_cgroup_do_precharge(unsigned long count) if (count > 1) { struct res_counter *dummy; /* - * "memcg" cannot be under rmdir() because we've already checked + * "mem" cannot be under rmdir() because we've already checked * by cgroup_lock_live_cgroup() that it is not removed and we * are still under the same cgroup_mutex. So we can postpone * css_get(). */ - if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy)) + if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy)) goto one_by_one; - if (do_swap_account && res_counter_charge(&memcg->memsw, + if (do_swap_account && res_counter_charge(&mem->memsw, PAGE_SIZE * count, &dummy)) { - res_counter_uncharge(&memcg->res, PAGE_SIZE * count); + res_counter_uncharge(&mem->res, PAGE_SIZE * count); goto one_by_one; } mc.precharge += count; @@ -5092,9 +5140,8 @@ one_by_one: batch_count = PRECHARGE_COUNT_AT_ONCE; cond_resched(); } - ret = __mem_cgroup_try_charge(NULL, - GFP_KERNEL, 1, &memcg, false); - if (ret || !memcg) + ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, 1, &mem, false); + if (ret || !mem) /* mem_cgroup_clear_mc() will do uncharge later */ return -ENOMEM; mc.precharge++; @@ -5193,17 +5240,15 @@ static struct page *mc_handle_file_pte(struct vm_area_struct *vma, pgoff = pte_to_pgoff(ptent); /* page is moved even if it's not RSS of this task(page-faulted). */ - page = find_get_page(mapping, pgoff); - -#ifdef CONFIG_SWAP - /* shmem/tmpfs may report page out on swap: account for that too. */ - if (radix_tree_exceptional_entry(page)) { - swp_entry_t swap = radix_to_swp_entry(page); + if (!mapping_cap_swap_backed(mapping)) { /* normal file */ + page = find_get_page(mapping, pgoff); + } else { /* shmem/tmpfs file. we should take account of swap too. */ + swp_entry_t ent; + mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent); if (do_swap_account) - *entry = swap; - page = find_get_page(&swapper_space, swap.val); + entry->val = ent.val; } -#endif + return page; } @@ -5370,13 +5415,13 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss, struct task_struct *p) { int ret = 0; - struct mem_cgroup *memcg = mem_cgroup_from_cont(cgroup); + struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup); - if (memcg->move_charge_at_immigrate) { + if (mem->move_charge_at_immigrate) { struct mm_struct *mm; struct mem_cgroup *from = mem_cgroup_from_task(p); - VM_BUG_ON(from == memcg); + VM_BUG_ON(from == mem); mm = get_task_mm(p); if (!mm) @@ -5391,7 +5436,7 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss, mem_cgroup_start_move(from); spin_lock(&mc.lock); mc.from = from; - mc.to = memcg; + mc.to = mem; spin_unlock(&mc.lock); /* We set mc.moving_task later */ diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 51901b1..809823d 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -42,7 +42,6 @@ #include <linux/sched.h> #include <linux/ksm.h> #include <linux/rmap.h> -#include <linux/export.h> #include <linux/pagemap.h> #include <linux/swap.h> #include <linux/backing-dev.h> @@ -54,7 +53,6 @@ #include <linux/hugetlb.h> #include <linux/memory_hotplug.h> #include <linux/mm_inline.h> -#include <linux/kfifo.h> #include "internal.h" int sysctl_memory_failure_early_kill __read_mostly = 0; @@ -1033,16 +1031,15 @@ int __memory_failure(unsigned long pfn, int trapno, int flags) return 0; } else if (PageHuge(hpage)) { /* - * Check "filter hit" and "race with other subpage." + * Check "just unpoisoned", "filter hit", and + * "race with other subpage." */ lock_page(hpage); - if (PageHWPoison(hpage)) { - if ((hwpoison_filter(p) && TestClearPageHWPoison(p)) - || (p != hpage && TestSetPageHWPoison(hpage))) { - atomic_long_sub(nr_pages, &mce_bad_pages); - unlock_page(hpage); - return 0; - } + if (!PageHWPoison(hpage) + || (hwpoison_filter(p) && TestClearPageHWPoison(p)) + || (p != hpage && TestSetPageHWPoison(hpage))) { + atomic_long_sub(nr_pages, &mce_bad_pages); + return 0; } set_page_hwpoison_huge_page(hpage); res = dequeue_hwpoisoned_huge_page(hpage); @@ -1094,8 +1091,6 @@ int __memory_failure(unsigned long pfn, int trapno, int flags) */ if (!PageHWPoison(p)) { printk(KERN_ERR "MCE %#lx: just unpoisoned\n", pfn); - atomic_long_sub(nr_pages, &mce_bad_pages); - put_page(hpage); res = 0; goto out; } @@ -1183,97 +1178,6 @@ void memory_failure(unsigned long pfn, int trapno) __memory_failure(pfn, trapno, 0); } -#define MEMORY_FAILURE_FIFO_ORDER 4 -#define MEMORY_FAILURE_FIFO_SIZE (1 << MEMORY_FAILURE_FIFO_ORDER) - -struct memory_failure_entry { - unsigned long pfn; - int trapno; - int flags; -}; - -struct memory_failure_cpu { - DECLARE_KFIFO(fifo, struct memory_failure_entry, - MEMORY_FAILURE_FIFO_SIZE); - spinlock_t lock; - struct work_struct work; -}; - -static DEFINE_PER_CPU(struct memory_failure_cpu, memory_failure_cpu); - -/** - * memory_failure_queue - Schedule handling memory failure of a page. - * @pfn: Page Number of the corrupted page - * @trapno: Trap number reported in the signal to user space. - * @flags: Flags for memory failure handling - * - * This function is called by the low level hardware error handler - * when it detects hardware memory corruption of a page. It schedules - * the recovering of error page, including dropping pages, killing - * processes etc. - * - * The function is primarily of use for corruptions that - * happen outside the current execution context (e.g. when - * detected by a background scrubber) - * - * Can run in IRQ context. - */ -void memory_failure_queue(unsigned long pfn, int trapno, int flags) -{ - struct memory_failure_cpu *mf_cpu; - unsigned long proc_flags; - struct memory_failure_entry entry = { - .pfn = pfn, - .trapno = trapno, - .flags = flags, - }; - - mf_cpu = &get_cpu_var(memory_failure_cpu); - spin_lock_irqsave(&mf_cpu->lock, proc_flags); - if (kfifo_put(&mf_cpu->fifo, &entry)) - schedule_work_on(smp_processor_id(), &mf_cpu->work); - else - pr_err("Memory failure: buffer overflow when queuing memory failure at 0x%#lx\n", - pfn); - spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); - put_cpu_var(memory_failure_cpu); -} -EXPORT_SYMBOL_GPL(memory_failure_queue); - -static void memory_failure_work_func(struct work_struct *work) -{ - struct memory_failure_cpu *mf_cpu; - struct memory_failure_entry entry = { 0, }; - unsigned long proc_flags; - int gotten; - - mf_cpu = &__get_cpu_var(memory_failure_cpu); - for (;;) { - spin_lock_irqsave(&mf_cpu->lock, proc_flags); - gotten = kfifo_get(&mf_cpu->fifo, &entry); - spin_unlock_irqrestore(&mf_cpu->lock, proc_flags); - if (!gotten) - break; - __memory_failure(entry.pfn, entry.trapno, entry.flags); - } -} - -static int __init memory_failure_init(void) -{ - struct memory_failure_cpu *mf_cpu; - int cpu; - - for_each_possible_cpu(cpu) { - mf_cpu = &per_cpu(memory_failure_cpu, cpu); - spin_lock_init(&mf_cpu->lock); - INIT_KFIFO(mf_cpu->fifo); - INIT_WORK(&mf_cpu->work, memory_failure_work_func); - } - - return 0; -} -core_initcall(memory_failure_init); - /** * unpoison_memory - Unpoison a previously poisoned page * @pfn: Page number of the to be unpoisoned page @@ -1314,7 +1218,7 @@ int unpoison_memory(unsigned long pfn) * to the end. */ if (PageHuge(page)) { - pr_info("MCE: Memory failure is now running on free hugepage %#lx\n", pfn); + pr_debug("MCE: Memory failure is now running on free hugepage %#lx\n", pfn); return 0; } if (TestClearPageHWPoison(p)) @@ -1403,7 +1307,11 @@ static int get_any_page(struct page *p, unsigned long pfn, int flags) /* Not a free page */ ret = 1; } +#ifndef CONFIG_DMA_CMA unset_migratetype_isolate(p); +#else + unset_migratetype_isolate(p, MIGRATE_MOVABLE); +#endif unlock_memory_hotplug(); return ret; } @@ -1423,7 +1331,7 @@ static int soft_offline_huge_page(struct page *page, int flags) if (PageHWPoison(hpage)) { put_page(hpage); - pr_info("soft offline: %#lx hugepage already poisoned\n", pfn); + pr_debug("soft offline: %#lx hugepage already poisoned\n", pfn); return -EBUSY; } @@ -1437,25 +1345,17 @@ static int soft_offline_huge_page(struct page *page, int flags) list_for_each_entry_safe(page1, page2, &pagelist, lru) put_page(page1); - pr_info("soft offline: %#lx: migration failed %d, type %lx\n", - pfn, ret, page->flags); + pr_debug("soft offline: %#lx: migration failed %d, type %lx\n", + pfn, ret, page->flags); if (ret > 0) ret = -EIO; return ret; } done: - /* overcommit hugetlb page will be freed to buddy */ - if (PageHuge(hpage)) { - if (!PageHWPoison(hpage)) - atomic_long_add(1 << compound_trans_order(hpage), - &mce_bad_pages); - set_page_hwpoison_huge_page(hpage); - dequeue_hwpoisoned_huge_page(hpage); - } else { - SetPageHWPoison(page); - atomic_long_inc(&mce_bad_pages); - } - + if (!PageHWPoison(hpage)) + atomic_long_add(1 << compound_trans_order(hpage), &mce_bad_pages); + set_page_hwpoison_huge_page(hpage); + dequeue_hwpoisoned_huge_page(hpage); /* keep elevated page count for bad page */ return ret; } @@ -1525,7 +1425,7 @@ int soft_offline_page(struct page *page, int flags) } if (!PageLRU(page)) { pr_info("soft_offline: %#lx: unknown non LRU page type %lx\n", - pfn, page->flags); + pfn, page->flags); return -EIO; } @@ -1575,8 +1475,13 @@ int soft_offline_page(struct page *page, int flags) inc_zone_page_state(page, NR_ISOLATED_ANON + page_is_file_cache(page)); list_add(&page->lru, &pagelist); +#ifndef CONFIG_DMA_CMA ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, - false, MIGRATE_SYNC); + false, MIGRATE_SYNC); +#else + ret = migrate_pages(&pagelist, new_page, MPOL_MF_MOVE_ALL, + false, MIGRATE_SYNC, 0); +#endif if (ret) { putback_lru_pages(&pagelist); pr_info("soft offline: %#lx: migration failed %d, type %lx\n", @@ -1586,7 +1491,7 @@ int soft_offline_page(struct page *page, int flags) } } else { pr_info("soft offline: %#lx: isolation failed: %d, page count %d, type %lx\n", - pfn, ret, page_count(page), page->flags); + pfn, ret, page_count(page), page->flags); } if (ret) return ret; diff --git a/mm/memory.c b/mm/memory.c index 7762b1d..5331e67 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -47,8 +47,9 @@ #include <linux/pagemap.h> #include <linux/ksm.h> #include <linux/rmap.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/delayacct.h> +#include <linux/delay.h> #include <linux/init.h> #include <linux/writeback.h> #include <linux/memcontrol.h> @@ -57,6 +58,7 @@ #include <linux/swapops.h> #include <linux/elf.h> #include <linux/gfp.h> +#include <linux/migrate.h> #include <asm/io.h> #include <asm/pgalloc.h> @@ -177,6 +179,7 @@ unsigned long get_mm_counter(struct mm_struct *mm, int member) return 0; return (unsigned long)val; } +EXPORT_SYMBOL(get_mm_counter); void sync_mm_rss(struct task_struct *task, struct mm_struct *mm) { @@ -870,20 +873,20 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, if (!pte_file(pte)) { swp_entry_t entry = pte_to_swp_entry(pte); - if (likely(!non_swap_entry(entry))) { - if (swap_duplicate(entry) < 0) - return entry.val; - - /* make sure dst_mm is on swapoff's mmlist. */ - if (unlikely(list_empty(&dst_mm->mmlist))) { - spin_lock(&mmlist_lock); - if (list_empty(&dst_mm->mmlist)) - list_add(&dst_mm->mmlist, - &src_mm->mmlist); - spin_unlock(&mmlist_lock); - } + if (swap_duplicate(entry) < 0) + return entry.val; + + /* make sure dst_mm is on swapoff's mmlist. */ + if (unlikely(list_empty(&dst_mm->mmlist))) { + spin_lock(&mmlist_lock); + if (list_empty(&dst_mm->mmlist)) + list_add(&dst_mm->mmlist, + &src_mm->mmlist); + spin_unlock(&mmlist_lock); + } + if (likely(!non_swap_entry(entry))) rss[MM_SWAPENTS]++; - } else if (is_write_migration_entry(entry) && + else if (is_write_migration_entry(entry) && is_cow_mapping(vm_flags)) { /* * COW mappings require pages in both parent @@ -1178,10 +1181,8 @@ again: if (unlikely(page_mapcount(page) < 0)) print_bad_pte(vma, addr, ptent, page); force_flush = !__tlb_remove_page(tlb, page); - if (force_flush) { - addr += PAGE_SIZE; + if (force_flush) break; - } continue; } /* @@ -1305,6 +1306,13 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb, return addr; } +#ifdef CONFIG_PREEMPT +# define ZAP_BLOCK_SIZE (8 * PAGE_SIZE) +#else +/* No preempt: go for improved straight-line efficiency */ +# define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE) +#endif + /** * unmap_vmas - unmap a range of memory covered by a list of vma's * @tlb: address of the caller's struct mmu_gather @@ -1318,6 +1326,10 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb, * * Unmap all pages in the vma list. * + * We aim to not hold locks for too long (for scheduling latency reasons). + * So zap pages in ZAP_BLOCK_SIZE bytecounts. This means we need to + * return the ending mmu_gather to the caller. + * * Only addresses between `start' and `end' will be unmapped. * * The VMA list must be sorted in ascending virtual address order. @@ -1365,11 +1377,8 @@ unsigned long unmap_vmas(struct mmu_gather *tlb, * Since no pte has actually been setup, it is * safe to do nothing in this case. */ - if (vma->vm_file) { - mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex); - __unmap_hugepage_range_final(vma, start, end, NULL); - mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex); - } + if (vma->vm_file) + unmap_hugepage_range(vma, start, end, NULL); start = end; } else @@ -1590,6 +1599,25 @@ static inline int stack_guard_page(struct vm_area_struct *vma, unsigned long add stack_guard_page_end(vma, addr+PAGE_SIZE); } +#ifdef CONFIG_DMA_CMA +static inline int __replace_cma_page(struct page *page, struct page **res) +{ + struct page *newpage; + int ret; + + ret = migrate_replace_cma_page(page, &newpage); + if (ret == 0) { + *res = newpage; + return 0; + } + /* + * Migration errors in case of get_user_pages() might not + * be fatal to CMA itself, so better don't fail here. + */ + return 0; +} +#endif + /** * __get_user_pages() - pin user pages in memory * @tsk: task_struct of target task @@ -1740,6 +1768,11 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, int ret; unsigned int fault_flags = 0; +#ifdef CONFIG_DMA_CMA + if (gup_flags & FOLL_NO_CMA) + fault_flags = FAULT_FLAG_NO_CMA; +#endif + /* For mlock, just skip the stack guard page. */ if (foll_flags & FOLL_MLOCK) { if (stack_guard_page(vma, start)) @@ -1767,7 +1800,7 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, else return -EFAULT; } - if (ret & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) + if (ret & VM_FAULT_SIGBUS) return i ? i : -EFAULT; BUG(); } @@ -1805,6 +1838,16 @@ int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, } if (IS_ERR(page)) return i ? i : PTR_ERR(page); + +#ifdef CONFIG_DMA_CMA + if ((gup_flags & FOLL_NO_CMA) + && is_cma_pageblock(page)) { + int rc = __replace_cma_page(page, &page); + if (rc) + return i ? i : rc; + } +#endif + if (pages) { pages[i] = page; @@ -1854,24 +1897,19 @@ int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, unsigned long address, unsigned int fault_flags) { struct vm_area_struct *vma; - vm_flags_t vm_flags; int ret; vma = find_extend_vma(mm, address); if (!vma || address < vma->vm_start) return -EFAULT; - vm_flags = (fault_flags & FAULT_FLAG_WRITE) ? VM_WRITE : VM_READ; - if (!(vm_flags & vma->vm_flags)) - return -EFAULT; - ret = handle_mm_fault(mm, vma, address, fault_flags); if (ret & VM_FAULT_ERROR) { if (ret & VM_FAULT_OOM) return -ENOMEM; if (ret & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) return -EHWPOISON; - if (ret & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV)) + if (ret & VM_FAULT_SIGBUS) return -EFAULT; BUG(); } @@ -1953,6 +1991,26 @@ int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, } EXPORT_SYMBOL(get_user_pages); +#ifdef CONFIG_DMA_CMA +int get_user_pages_nocma(struct task_struct *tsk, struct mm_struct *mm, + unsigned long start, int nr_pages, int write, int force, + struct page **pages, struct vm_area_struct **vmas) +{ + int flags = FOLL_TOUCH | FOLL_NO_CMA; + + if (pages) + flags |= FOLL_GET; + if (write) + flags |= FOLL_WRITE; + if (force) + flags |= FOLL_FORCE; + + return __get_user_pages(tsk, mm, start, nr_pages, flags, pages, vmas, + NULL); +} +EXPORT_SYMBOL(get_user_pages_nocma); +#endif + /** * get_dump_page() - pin user page in memory while writing it to core dump * @addr: user address @@ -2535,7 +2593,11 @@ static inline void cow_user_page(struct page *dst, struct page *src, unsigned lo */ static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pte_t *page_table, pmd_t *pmd, +#ifdef CONFIG_DMA_CMA + spinlock_t *ptl, pte_t orig_pte, unsigned int flags) +#else spinlock_t *ptl, pte_t orig_pte) +#endif __releases(ptl) { struct page *old_page, *new_page; @@ -2661,24 +2723,17 @@ reuse: if (!dirty_page) return ret; + /* + * Yes, Virginia, this is actually required to prevent a race + * with clear_page_dirty_for_io() from clearing the page dirty + * bit after it clear all dirty ptes, but before a racing + * do_wp_page installs a dirty pte. + * + * __do_fault is protected similarly. + */ if (!page_mkwrite) { - struct address_space *mapping; - int dirtied; - - lock_page(dirty_page); - dirtied = set_page_dirty(dirty_page); - VM_BUG_ON(PageAnon(dirty_page)); - mapping = dirty_page->mapping; - unlock_page(dirty_page); - - if (dirtied && mapping) { - /* - * Some device drivers do not set page.mapping - * but still dirty their pages - */ - balance_dirty_pages_ratelimited(mapping); - } - + wait_on_page_locked(dirty_page); + set_page_dirty_balance(dirty_page, page_mkwrite); } put_page(dirty_page); if (page_mkwrite) { @@ -2714,11 +2769,25 @@ gotten: goto oom; if (is_zero_pfn(pte_pfn(orig_pte))) { - new_page = alloc_zeroed_user_highpage_movable(vma, address); +#ifdef CONFIG_DMA_CMA + if (flags & FAULT_FLAG_NO_CMA) + new_page = alloc_zeroed_user_highpage(vma, address); + else +#endif + new_page = + alloc_zeroed_user_highpage_movable(vma, address); + if (!new_page) goto oom; } else { - new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); +#ifdef CONFIG_DMA_CMA + if (flags & FAULT_FLAG_NO_CMA) + new_page = alloc_page_vma(GFP_HIGHUSER, vma, address); + else +#endif + new_page = + alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); + if (!new_page) goto oom; cow_user_page(new_page, old_page, address, vma); @@ -2945,6 +3014,16 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, entry = pte_to_swp_entry(orig_pte); if (unlikely(non_swap_entry(entry))) { if (is_migration_entry(entry)) { +#ifdef CONFIG_DMA_CMA + /* + * FIXME: mszyprow: cruel, brute-force method for + * letting cma/migration to finish it's job without + * stealing the lock migration_entry_wait() and creating + * a live-lock on the faulted page + * (page->_count == 2 migration failure issue) + */ + mdelay(10); +#endif migration_entry_wait(mm, pmd, address); } else if (is_hwpoison_entry(entry)) { ret = VM_FAULT_HWPOISON; @@ -3078,7 +3157,12 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, } if (flags & FAULT_FLAG_WRITE) { +#ifdef CONFIG_DMA_CMA + ret |= do_wp_page(mm, vma, address, page_table, + pmd, ptl, pte, flags); +#else ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte); +#endif if (ret & VM_FAULT_ERROR) ret &= VM_FAULT_ERROR; goto out; @@ -3124,7 +3208,7 @@ static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned lo if (prev && prev->vm_end == address) return prev->vm_flags & VM_GROWSDOWN ? 0 : -ENOMEM; - return expand_downwards(vma, address - PAGE_SIZE); + expand_downwards(vma, address - PAGE_SIZE); } if ((vma->vm_flags & VM_GROWSUP) && address + PAGE_SIZE == vma->vm_end) { struct vm_area_struct *next = vma->vm_next; @@ -3133,7 +3217,7 @@ static inline int check_stack_guard_page(struct vm_area_struct *vma, unsigned lo if (next && next->vm_start == address + PAGE_SIZE) return next->vm_flags & VM_GROWSUP ? 0 : -ENOMEM; - return expand_upwards(vma, address + PAGE_SIZE); + expand_upwards(vma, address + PAGE_SIZE); } return 0; } @@ -3153,13 +3237,9 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, pte_unmap(page_table); - /* File mapping without ->vm_ops ? */ - if (vma->vm_flags & VM_SHARED) - return VM_FAULT_SIGBUS; - /* Check if we need to add a guard page to the stack */ if (check_stack_guard_page(vma, address) < 0) - return VM_FAULT_SIGSEGV; + return VM_FAULT_SIGBUS; /* Use the zero-page for reads */ if (!(flags & FAULT_FLAG_WRITE)) { @@ -3230,34 +3310,14 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, pte_t *page_table; spinlock_t *ptl; struct page *page; - struct page *cow_page; pte_t entry; int anon = 0; + int charged = 0; struct page *dirty_page = NULL; struct vm_fault vmf; int ret; int page_mkwrite = 0; - /* - * If we do COW later, allocate page befor taking lock_page() - * on the file cache page. This will reduce lock holding time. - */ - if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) { - - if (unlikely(anon_vma_prepare(vma))) - return VM_FAULT_OOM; - - cow_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address); - if (!cow_page) - return VM_FAULT_OOM; - - if (mem_cgroup_newpage_charge(cow_page, mm, GFP_KERNEL)) { - page_cache_release(cow_page); - return VM_FAULT_OOM; - } - } else - cow_page = NULL; - vmf.virtual_address = (void __user *)(address & PAGE_MASK); vmf.pgoff = pgoff; vmf.flags = flags; @@ -3266,13 +3326,12 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, ret = vma->vm_ops->fault(vma, &vmf); if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) - goto uncharge_out; + return ret; if (unlikely(PageHWPoison(vmf.page))) { if (ret & VM_FAULT_LOCKED) unlock_page(vmf.page); - ret = VM_FAULT_HWPOISON; - goto uncharge_out; + return VM_FAULT_HWPOISON; } /* @@ -3290,8 +3349,31 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, page = vmf.page; if (flags & FAULT_FLAG_WRITE) { if (!(vma->vm_flags & VM_SHARED)) { - page = cow_page; anon = 1; + if (unlikely(anon_vma_prepare(vma))) { + ret = VM_FAULT_OOM; + goto out; + } + +#ifdef CONFIG_DMA_CMA + if (flags & FAULT_FLAG_NO_CMA) + page = alloc_page_vma(GFP_HIGHUSER, + vma, address); + else +#endif + page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, + vma, address); + + if (!page) { + ret = VM_FAULT_OOM; + goto out; + } + if (mem_cgroup_newpage_charge(page, mm, GFP_KERNEL)) { + ret = VM_FAULT_OOM; + page_cache_release(page); + goto out; + } + charged = 1; copy_user_highpage(page, vmf.page, address, vma); __SetPageUptodate(page); } else { @@ -3360,8 +3442,8 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, /* no need to invalidate: a not-present page won't be cached */ update_mmu_cache(vma, address, page_table); } else { - if (cow_page) - mem_cgroup_uncharge_page(cow_page); + if (charged) + mem_cgroup_uncharge_page(page); if (anon) page_cache_release(page); else @@ -3370,6 +3452,7 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, pte_unmap_unlock(page_table, ptl); +out: if (dirty_page) { struct address_space *mapping = page->mapping; @@ -3399,13 +3482,6 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma, unwritable_page: page_cache_release(page); return ret; -uncharge_out: - /* fs's fault handler get error */ - if (cow_page) { - mem_cgroup_uncharge_page(cow_page); - page_cache_release(cow_page); - } - return ret; } static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, @@ -3416,9 +3492,6 @@ static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; pte_unmap(page_table); - /* The VMA was not fully populated on mmap() or missing VM_DONTEXPAND */ - if (!vma->vm_ops->fault) - return VM_FAULT_SIGBUS; return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte); } @@ -3477,9 +3550,11 @@ int handle_pte_fault(struct mm_struct *mm, entry = *pte; if (!pte_present(entry)) { if (pte_none(entry)) { - if (vma->vm_ops) - return do_linear_fault(mm, vma, address, + if (vma->vm_ops) { + if (likely(vma->vm_ops->fault)) + return do_linear_fault(mm, vma, address, pte, pmd, flags, entry); + } return do_anonymous_page(mm, vma, address, pte, pmd, flags); } @@ -3496,8 +3571,13 @@ int handle_pte_fault(struct mm_struct *mm, goto unlock; if (flags & FAULT_FLAG_WRITE) { if (!pte_write(entry)) +#ifdef CONFIG_DMA_CMA + return do_wp_page(mm, vma, address, + pte, pmd, ptl, entry, flags); +#else return do_wp_page(mm, vma, address, pte, pmd, ptl, entry); +#endif entry = pte_mkdirty(entry); } entry = pte_mkyoung(entry); @@ -3836,7 +3916,7 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, if (follow_phys(vma, addr, write, &prot, &phys_addr)) return -EINVAL; - maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot); + maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot); if (write) memcpy_toio(maddr + offset, buf, len); else diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 09d87b7..94463bc 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -11,7 +11,7 @@ #include <linux/pagemap.h> #include <linux/bootmem.h> #include <linux/compiler.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/pagevec.h> #include <linux/writeback.h> #include <linux/slab.h> @@ -34,17 +34,6 @@ #include "internal.h" -/* - * online_page_callback contains pointer to current page onlining function. - * Initially it is generic_online_page(). If it is required it could be - * changed by calling set_online_page_callback() for callback registration - * and restore_online_page_callback() for generic callback restore. - */ - -static void generic_online_page(struct page *page); - -static online_page_callback_t online_page_callback = generic_online_page; - DEFINE_MUTEX(mem_hotplug_mutex); void lock_memory_hotplug(void) @@ -376,74 +365,23 @@ int __remove_pages(struct zone *zone, unsigned long phys_start_pfn, } EXPORT_SYMBOL_GPL(__remove_pages); -int set_online_page_callback(online_page_callback_t callback) -{ - int rc = -EINVAL; - - lock_memory_hotplug(); - - if (online_page_callback == generic_online_page) { - online_page_callback = callback; - rc = 0; - } - - unlock_memory_hotplug(); - - return rc; -} -EXPORT_SYMBOL_GPL(set_online_page_callback); - -int restore_online_page_callback(online_page_callback_t callback) -{ - int rc = -EINVAL; - - lock_memory_hotplug(); - - if (online_page_callback == callback) { - online_page_callback = generic_online_page; - rc = 0; - } - - unlock_memory_hotplug(); - - return rc; -} -EXPORT_SYMBOL_GPL(restore_online_page_callback); - -void __online_page_set_limits(struct page *page) +void online_page(struct page *page) { unsigned long pfn = page_to_pfn(page); + totalram_pages++; if (pfn >= num_physpages) num_physpages = pfn + 1; -} -EXPORT_SYMBOL_GPL(__online_page_set_limits); - -void __online_page_increment_counters(struct page *page) -{ - totalram_pages++; #ifdef CONFIG_HIGHMEM if (PageHighMem(page)) totalhigh_pages++; #endif -} -EXPORT_SYMBOL_GPL(__online_page_increment_counters); -void __online_page_free(struct page *page) -{ ClearPageReserved(page); init_page_count(page); __free_page(page); } -EXPORT_SYMBOL_GPL(__online_page_free); - -static void generic_online_page(struct page *page) -{ - __online_page_set_limits(page); - __online_page_increment_counters(page); - __online_page_free(page); -} static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, void *arg) @@ -454,7 +392,7 @@ static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, if (PageReserved(pfn_to_page(start_pfn))) for (i = 0; i < nr_pages; i++) { page = pfn_to_page(start_pfn + i); - (*online_page_callback)(page); + online_page(page); onlined_pages++; } *(unsigned long *)arg = onlined_pages; @@ -813,8 +751,13 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) goto out; } /* this function returns # of failed pages */ +#ifndef CONFIG_DMA_CMA + ret = migrate_pages(&source, hotremove_migrate_alloc, 0, + true, MIGRATE_SYNC); +#else ret = migrate_pages(&source, hotremove_migrate_alloc, 0, - true, MIGRATE_SYNC); + true, MIGRATE_SYNC, 0); +#endif if (ret) putback_lru_pages(&source); } @@ -896,7 +839,11 @@ static int __ref offline_pages(unsigned long start_pfn, nr_pages = end_pfn - start_pfn; /* set above range as isolated */ +#ifndef CONFIG_DMA_CMA ret = start_isolate_page_range(start_pfn, end_pfn); +#else + ret = start_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); +#endif if (ret) goto out; @@ -961,7 +908,11 @@ repeat: We cannot do rollback at this point. */ offline_isolated_pages(start_pfn, end_pfn); /* reset pagetype flags and makes migrate type to be MOVABLE */ +#ifndef CONFIG_DMA_CMA undo_isolate_page_range(start_pfn, end_pfn); +#else + undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); +#endif /* removal success */ zone->present_pages -= offlined_pages; zone->zone_pgdat->node_present_pages -= offlined_pages; @@ -986,7 +937,11 @@ failed_removal: start_pfn, end_pfn); memory_notify(MEM_CANCEL_OFFLINE, &arg); /* pushback to free area */ +#ifndef CONFIG_DMA_CMA undo_isolate_page_range(start_pfn, end_pfn); +#else + undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); +#endif out: unlock_memory_hotplug(); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index a72fa33..7065fb3 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -75,7 +75,7 @@ #include <linux/cpuset.h> #include <linux/slab.h> #include <linux/string.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/nsproxy.h> #include <linux/interrupt.h> #include <linux/init.h> @@ -93,7 +93,6 @@ #include <asm/tlbflush.h> #include <asm/uaccess.h> -#include <linux/random.h> #include "internal.h" @@ -111,7 +110,7 @@ enum zone_type policy_zone = 0; /* * run-time system-wide default policy => local allocation */ -static struct mempolicy default_policy = { +struct mempolicy default_policy = { .refcnt = ATOMIC_INIT(1), /* never free it */ .mode = MPOL_PREFERRED, .flags = MPOL_F_LOCAL, @@ -566,23 +565,24 @@ static inline int check_pgd_range(struct vm_area_struct *vma, * If pagelist != NULL then isolate pages from the LRU and * put them on the pagelist. */ -static int +static struct vm_area_struct * check_range(struct mm_struct *mm, unsigned long start, unsigned long end, const nodemask_t *nodes, unsigned long flags, void *private) { - int err = 0; - struct vm_area_struct *vma, *prev; + int err; + struct vm_area_struct *first, *vma, *prev; - vma = find_vma(mm, start); - if (!vma) - return -EFAULT; + + first = find_vma(mm, start); + if (!first) + return ERR_PTR(-EFAULT); prev = NULL; - for (; vma && vma->vm_start < end; vma = vma->vm_next) { + for (vma = first; vma && vma->vm_start < end; vma = vma->vm_next) { if (!(flags & MPOL_MF_DISCONTIG_OK)) { if (!vma->vm_next && vma->vm_end < end) - return -EFAULT; + return ERR_PTR(-EFAULT); if (prev && prev->vm_end < vma->vm_start) - return -EFAULT; + return ERR_PTR(-EFAULT); } if (!is_vm_hugetlb_page(vma) && ((flags & MPOL_MF_STRICT) || @@ -596,12 +596,14 @@ check_range(struct mm_struct *mm, unsigned long start, unsigned long end, start = vma->vm_start; err = check_pgd_range(vma, start, endvma, nodes, flags, private); - if (err) + if (err) { + first = ERR_PTR(err); break; + } } prev = vma; } - return err; + return first; } /* @@ -656,22 +658,14 @@ static int mbind_range(struct mm_struct *mm, unsigned long start, if (!vma || vma->vm_start > start) return -EFAULT; - if (start > vma->vm_start) - prev = vma; - for (; vma && vma->vm_start < end; prev = vma, vma = next) { next = vma->vm_next; vmstart = max(start, vma->vm_start); vmend = min(end, vma->vm_end); - if (mpol_equal(vma_policy(vma), new_pol)) - continue; - - pgoff = vma->vm_pgoff + - ((vmstart - vma->vm_start) >> PAGE_SHIFT); + pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags, - vma->anon_vma, vma->vm_file, pgoff, - new_pol); + vma->anon_vma, vma->vm_file, pgoff, new_pol); if (prev) { vma = prev; next = vma->vm_next; @@ -942,22 +936,24 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest, nodemask_t nmask; LIST_HEAD(pagelist); int err = 0; + struct vm_area_struct *vma; nodes_clear(nmask); node_set(source, nmask); - /* - * This does not "check" the range but isolates all pages that - * need migration. Between passing in the full user address - * space range and MPOL_MF_DISCONTIG_OK, this call can not fail. - */ - VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))); - check_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, + vma = check_range(mm, mm->mmap->vm_start, mm->task_size, &nmask, flags | MPOL_MF_DISCONTIG_OK, &pagelist); + if (IS_ERR(vma)) + return PTR_ERR(vma); if (!list_empty(&pagelist)) { +#ifndef CONFIG_DMA_CMA + err = migrate_pages(&pagelist, new_node_page, dest, + false, MIGRATE_SYNC); +#else err = migrate_pages(&pagelist, new_node_page, dest, - false, MIGRATE_SYNC); + false, MIGRATE_SYNC, 0); +#endif if (err) putback_lru_pages(&pagelist); } @@ -1057,17 +1053,16 @@ out: /* * Allocate a new page for page migration based on vma policy. - * Start by assuming the page is mapped by the same vma as contains @start. + * Start assuming that page is mapped by vma pointed to by @private. * Search forward from there, if not. N.B., this assumes that the * list of pages handed to migrate_pages()--which is how we get here-- * is in virtual address order. */ -static struct page *new_page(struct page *page, unsigned long start, int **x) +static struct page *new_vma_page(struct page *page, unsigned long private, int **x) { - struct vm_area_struct *vma; + struct vm_area_struct *vma = (struct vm_area_struct *)private; unsigned long uninitialized_var(address); - vma = find_vma(current->mm, start); while (vma) { address = page_address_in_vma(page, vma); if (address != -EFAULT) @@ -1093,7 +1088,7 @@ int do_migrate_pages(struct mm_struct *mm, return -ENOSYS; } -static struct page *new_page(struct page *page, unsigned long start, int **x) +static struct page *new_vma_page(struct page *page, unsigned long private, int **x) { return NULL; } @@ -1103,6 +1098,7 @@ static long do_mbind(unsigned long start, unsigned long len, unsigned short mode, unsigned short mode_flags, nodemask_t *nmask, unsigned long flags) { + struct vm_area_struct *vma; struct mm_struct *mm = current->mm; struct mempolicy *new; unsigned long end; @@ -1166,16 +1162,25 @@ static long do_mbind(unsigned long start, unsigned long len, if (err) goto mpol_out; - err = check_range(mm, start, end, nmask, + vma = check_range(mm, start, end, nmask, flags | MPOL_MF_INVERT, &pagelist); - if (!err) { + + err = PTR_ERR(vma); + if (!IS_ERR(vma)) { int nr_failed = 0; err = mbind_range(mm, start, end, new); if (!list_empty(&pagelist)) { - nr_failed = migrate_pages(&pagelist, new_page, - start, false, true); +#ifndef CONFIG_DMA_CMA + nr_failed = migrate_pages(&pagelist, new_vma_page, + (unsigned long)vma, + false, true); +#else + nr_failed = migrate_pages(&pagelist, new_vma_page, + (unsigned long)vma, + false, true, 0); +#endif if (nr_failed) putback_lru_pages(&pagelist); } @@ -1432,9 +1437,7 @@ asmlinkage long compat_sys_get_mempolicy(int __user *policy, err = sys_get_mempolicy(policy, nm, nr_bits+1, addr, flags); if (!err && nmask) { - unsigned long copy_size; - copy_size = min_t(unsigned long, sizeof(bm), alloc_size); - err = copy_from_user(bm, nm, copy_size); + err = copy_from_user(bm, nm, alloc_size); /* ensure entire bitmap is zeroed */ err |= clear_user(nmask, ALIGN(maxnode-1, 8) / 8); err |= compat_put_bitmap(nmask, bm, nr_bits); @@ -1601,14 +1604,8 @@ static unsigned interleave_nodes(struct mempolicy *policy) * task can change it's policy. The system default policy requires no * such protection. */ -unsigned slab_node(void) +unsigned slab_node(struct mempolicy *policy) { - struct mempolicy *policy; - - if (in_interrupt()) - return numa_node_id(); - - policy = current->mempolicy; if (!policy || policy->flags & MPOL_F_LOCAL) return numa_node_id(); @@ -1684,21 +1681,6 @@ static inline unsigned interleave_nid(struct mempolicy *pol, return interleave_nodes(pol); } -/* - * Return the bit number of a random bit set in the nodemask. - * (returns -1 if nodemask is empty) - */ -int node_random(const nodemask_t *maskp) -{ - int w, bit = -1; - - w = nodes_weight(*maskp); - if (w) - bit = bitmap_ord_to_pos(maskp->bits, - get_random_int() % w, MAX_NUMNODES); - return bit; -} - #ifdef CONFIG_HUGETLBFS /* * huge_zonelist(@vma, @addr, @gfp_flags, @mpol) @@ -1989,6 +1971,7 @@ struct mempolicy *__mpol_dup(struct mempolicy *old) } else *new = *old; + rcu_read_lock(); if (current_cpuset_is_being_rebound()) { nodemask_t mems = cpuset_mems_allowed(current); if (new->flags & MPOL_F_REBINDING) @@ -1996,6 +1979,7 @@ struct mempolicy *__mpol_dup(struct mempolicy *old) else mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE); } + rcu_read_unlock(); atomic_set(&new->refcnt, 1); return new; } diff --git a/mm/mempool.c b/mm/mempool.c index e73641b..1a3bc3d 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -10,7 +10,7 @@ #include <linux/mm.h> #include <linux/slab.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/mempool.h> #include <linux/blkdev.h> #include <linux/writeback.h> diff --git a/mm/migrate-cma.c b/mm/migrate-cma.c new file mode 100644 index 0000000..449efb2 --- /dev/null +++ b/mm/migrate-cma.c @@ -0,0 +1,1627 @@ +/* + * Memory Migration functionality - linux/mm/migration.c + * + * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter + * + * Page migration was first developed in the context of the memory hotplug + * project. The main authors of the migration code are: + * + * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> + * Hirokazu Takahashi <taka@valinux.co.jp> + * Dave Hansen <haveblue@us.ibm.com> + * Christoph Lameter + */ + +#include <linux/migrate.h> +#include <linux/module.h> +#include <linux/swap.h> +#include <linux/swapops.h> +#include <linux/pagemap.h> +#include <linux/buffer_head.h> +#include <linux/mm_inline.h> +#include <linux/nsproxy.h> +#include <linux/pagevec.h> +#include <linux/ksm.h> +#include <linux/rmap.h> +#include <linux/topology.h> +#include <linux/cpu.h> +#include <linux/cpuset.h> +#include <linux/writeback.h> +#include <linux/mempolicy.h> +#include <linux/vmalloc.h> +#include <linux/security.h> +#include <linux/memcontrol.h> +#include <linux/syscalls.h> +#include <linux/hugetlb.h> +#include <linux/gfp.h> + +#include <asm/tlbflush.h> + +#include "internal.h" + +#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) + +/* + * migrate_prep() needs to be called before we start compiling a list of pages + * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is + * undesirable, use migrate_prep_local() + */ +int migrate_prep(void) +{ + /* + * Clear the LRU lists so pages can be isolated. + * Note that pages may be moved off the LRU after we have + * drained them. Those pages will fail to migrate like other + * pages that may be busy. + */ + lru_add_drain_all(); + + return 0; +} + +/* Do the necessary work of migrate_prep but not if it involves other CPUs */ +int migrate_prep_local(void) +{ + lru_add_drain(); + + return 0; +} + +/* + * Add isolated pages on the list back to the LRU under page lock + * to avoid leaking evictable pages back onto unevictable list. + */ +void putback_lru_pages(struct list_head *l) +{ + struct page *page; + struct page *page2; + + list_for_each_entry_safe(page, page2, l, lru) { + list_del(&page->lru); + dec_zone_page_state(page, NR_ISOLATED_ANON + + page_is_file_cache(page)); + putback_lru_page(page); + } +} + +/* + * Restore a potential migration pte to a working pte entry + */ +static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, + unsigned long addr, void *old) +{ + struct mm_struct *mm = vma->vm_mm; + swp_entry_t entry; + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *ptep, pte; + spinlock_t *ptl; + + if (unlikely(PageHuge(new))) { + ptep = huge_pte_offset(mm, addr); + if (!ptep) + goto out; + ptl = &mm->page_table_lock; + } else { + pgd = pgd_offset(mm, addr); + if (!pgd_present(*pgd)) + goto out; + + pud = pud_offset(pgd, addr); + if (!pud_present(*pud)) + goto out; + + pmd = pmd_offset(pud, addr); + if (pmd_trans_huge(*pmd)) + goto out; + if (!pmd_present(*pmd)) + goto out; + + ptep = pte_offset_map(pmd, addr); + + /* + * Peek to check is_swap_pte() before taking ptlock? No, we + * can race mremap's move_ptes(), which skips anon_vma lock. + */ + + ptl = pte_lockptr(mm, pmd); + } + + spin_lock(ptl); + pte = *ptep; + if (!is_swap_pte(pte)) + goto unlock; + + entry = pte_to_swp_entry(pte); + + if (!is_migration_entry(entry) || + migration_entry_to_page(entry) != old) + goto unlock; + + get_page(new); + pte = pte_mkold(mk_pte(new, vma->vm_page_prot)); + if (is_write_migration_entry(entry)) + pte = pte_mkwrite(pte); +#ifdef CONFIG_HUGETLB_PAGE + if (PageHuge(new)) + pte = pte_mkhuge(pte); +#endif + flush_cache_page(vma, addr, pte_pfn(pte)); + set_pte_at(mm, addr, ptep, pte); + + if (PageHuge(new)) { + if (PageAnon(new)) + hugepage_add_anon_rmap(new, vma, addr); + else + page_dup_rmap(new); + } else if (PageAnon(new)) + page_add_anon_rmap(new, vma, addr); + else + page_add_file_rmap(new); + + /* No need to invalidate - it was non-present before */ + update_mmu_cache(vma, addr, ptep); +unlock: + pte_unmap_unlock(ptep, ptl); +out: + return SWAP_AGAIN; +} + + +/* + * Get rid of all migration entries and replace them by + * references to the indicated page. + */ +static void remove_migration_ptes(struct page *old, struct page *new) +{ + rmap_walk(new, remove_migration_pte, old); +} + +/* + * Something used the pte of a page under migration. We need to + * get to the page and wait until migration is finished. + * When we return from this function the fault will be retried. + * + * This function is called from do_swap_page(). + */ +void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, + unsigned long address) +{ + pte_t *ptep, pte; + spinlock_t *ptl; + swp_entry_t entry; + struct page *page; + + ptep = pte_offset_map_lock(mm, pmd, address, &ptl); + pte = *ptep; + if (!is_swap_pte(pte)) + goto out; + + entry = pte_to_swp_entry(pte); + if (!is_migration_entry(entry)) + goto out; + + page = migration_entry_to_page(entry); + + /* + * Once radix-tree replacement of page migration started, page_count + * *must* be zero. And, we don't want to call wait_on_page_locked() + * against a page without get_page(). + * So, we use get_page_unless_zero(), here. Even failed, page fault + * will occur again. + */ + if (!get_page_unless_zero(page)) + goto out; + pte_unmap_unlock(ptep, ptl); + wait_on_page_locked(page); + put_page(page); + return; +out: + pte_unmap_unlock(ptep, ptl); +} + +extern struct page *failed_pages[5][10]; + +static int is_failed_page(struct page *page, int pass, int tries) +{ + if (tries == 4 && pass == 9) { + int i, j, k = 1; + + for (i = 0; i < 5; i++) { + for (j = 0; j < 10; j++) { + if (failed_pages[i][j] == page) + k++; + } + } + + printk(KERN_ERR "%s[%d]: %d fails\n", __func__, __LINE__, k); + return 1; + } + + return 0; +} + +#ifdef CONFIG_BLOCK +/* Returns true if all buffers are successfully locked */ +static bool buffer_migrate_lock_buffers(struct buffer_head *head, enum migrate_mode mode) +{ + struct buffer_head *bh = head; + + /* Simple case, sync compaction */ + if (mode != MIGRATE_ASYNC) { + do { + get_bh(bh); + lock_buffer(bh); + bh = bh->b_this_page; + + } while (bh != head); + + return true; + } + + /* async case, we cannot block on lock_buffer so use trylock_buffer */ + do { + get_bh(bh); + if (!trylock_buffer(bh)) { + /* + * We failed to lock the buffer and cannot stall in + * async migration. Release the taken locks + */ + struct buffer_head *failed_bh = bh; + put_bh(failed_bh); + bh = head; + while (bh != failed_bh) { + unlock_buffer(bh); + put_bh(bh); + bh = bh->b_this_page; + } + return false; + } + + bh = bh->b_this_page; + } while (bh != head); + return true; +} +#else +static inline bool buffer_migrate_lock_buffers(struct buffer_head *head, + enum migrate_mode mode) +{ + return true; +} +#endif /* CONFIG_BLOCK */ + +/* + * Replace the page in the mapping. + * + * The number of remaining references must be: + * 1 for anonymous pages without a mapping + * 2 for pages with a mapping + * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. + */ +static int migrate_page_move_mapping(struct address_space *mapping, + struct page *newpage, struct page *page, + struct buffer_head *head, enum migrate_mode mode, + int pass, int tries) +{ + int expected_count; + void **pslot; + + if (!mapping) { + /* Anonymous page without mapping */ + if (page_count(page) != 1) { + if (is_failed_page(page, pass, tries)) { + printk(KERN_ERR "%s[%d]: 1 ", + __func__, __LINE__); + dump_page(page); + } + return -EAGAIN; + } + return 0; + } + + spin_lock_irq(&mapping->tree_lock); + + pslot = radix_tree_lookup_slot(&mapping->page_tree, page_index(page)); + + expected_count = 2 + page_has_private(page); + if (page_count(page) != expected_count || + radix_tree_deref_slot_protected(pslot, + &mapping->tree_lock) != page) { + spin_unlock_irq(&mapping->tree_lock); + if (is_failed_page(page, pass, tries)) { + printk("%s[%d]: 2 ", __func__, __LINE__); + dump_page(page); + } + return -EAGAIN; + } + + if (!page_freeze_refs(page, expected_count)) { + spin_unlock_irq(&mapping->tree_lock); + if (is_failed_page(page, pass, tries)) { + printk("%s[%d]: 3 ", __func__, __LINE__); + dump_page(page); + } + return -EAGAIN; + } + + /* + * In the async migration case of moving a page with buffers, lock the + * buffers using trylock before the mapping is moved. If the mapping + * was moved, we later failed to lock the buffers and could not move + * the mapping back due to an elevated page count, we would have to + * block waiting on other references to be dropped. + */ + if (mode == MIGRATE_ASYNC && head && !buffer_migrate_lock_buffers(head, mode)) { + page_unfreeze_refs(page, expected_count); + spin_unlock_irq(&mapping->tree_lock); + return -EAGAIN; + } + + /* + * Now we know that no one else is looking at the page. + */ + get_page(newpage); /* add cache reference */ + if (PageSwapCache(page)) { + SetPageSwapCache(newpage); + set_page_private(newpage, page_private(page)); + } + + radix_tree_replace_slot(pslot, newpage); + + page_unfreeze_refs(page, expected_count); + /* + * Drop cache reference from old page. + * We know this isn't the last reference. + */ + __put_page(page); + + /* + * If moved to a different zone then also account + * the page for that zone. Other VM counters will be + * taken care of when we establish references to the + * new page and drop references to the old page. + * + * Note that anonymous pages are accounted for + * via NR_FILE_PAGES and NR_ANON_PAGES if they + * are mapped to swap space. + */ + __dec_zone_page_state(page, NR_FILE_PAGES); + __inc_zone_page_state(newpage, NR_FILE_PAGES); + if (!PageSwapCache(page) && PageSwapBacked(page)) { + __dec_zone_page_state(page, NR_SHMEM); + __inc_zone_page_state(newpage, NR_SHMEM); + } + spin_unlock_irq(&mapping->tree_lock); + + return 0; +} + +/* + * The expected number of remaining references is the same as that + * of migrate_page_move_mapping(). + */ +int migrate_huge_page_move_mapping(struct address_space *mapping, + struct page *newpage, struct page *page) +{ + int expected_count; + void **pslot; + + if (!mapping) { + if (page_count(page) != 1) + return -EAGAIN; + return 0; + } + + spin_lock_irq(&mapping->tree_lock); + + pslot = radix_tree_lookup_slot(&mapping->page_tree, + page_index(page)); + + expected_count = 2 + page_has_private(page); + if (page_count(page) != expected_count || + radix_tree_deref_slot_protected(pslot, + &mapping->tree_lock) != page) { + spin_unlock_irq(&mapping->tree_lock); + return -EAGAIN; + } + + if (!page_freeze_refs(page, expected_count)) { + spin_unlock_irq(&mapping->tree_lock); + return -EAGAIN; + } + + get_page(newpage); + + radix_tree_replace_slot(pslot, newpage); + + page_unfreeze_refs(page, expected_count); + + __put_page(page); + + spin_unlock_irq(&mapping->tree_lock); + return 0; +} + +/* + * Copy the page to its new location + */ +void migrate_page_copy(struct page *newpage, struct page *page) +{ + if (PageHuge(page)) + copy_huge_page(newpage, page); + else + copy_highpage(newpage, page); + + if (PageError(page)) + SetPageError(newpage); + if (PageReferenced(page)) + SetPageReferenced(newpage); + if (PageUptodate(page)) + SetPageUptodate(newpage); + if (TestClearPageActive(page)) { + VM_BUG_ON(PageUnevictable(page)); + SetPageActive(newpage); + } else if (TestClearPageUnevictable(page)) + SetPageUnevictable(newpage); + if (PageChecked(page)) + SetPageChecked(newpage); + if (PageMappedToDisk(page)) + SetPageMappedToDisk(newpage); + + if (PageDirty(page)) { + clear_page_dirty_for_io(page); + /* + * Want to mark the page and the radix tree as dirty, and + * redo the accounting that clear_page_dirty_for_io undid, + * but we can't use set_page_dirty because that function + * is actually a signal that all of the page has become dirty. + * Whereas only part of our page may be dirty. + */ + __set_page_dirty_nobuffers(newpage); + } + + mlock_migrate_page(newpage, page); + ksm_migrate_page(newpage, page); + + ClearPageSwapCache(page); + ClearPagePrivate(page); + set_page_private(page, 0); + page->mapping = NULL; + + /* + * If any waiters have accumulated on the new page then + * wake them up. + */ + if (PageWriteback(newpage)) + end_page_writeback(newpage); +} + +/************************************************************ + * Migration functions + ***********************************************************/ + +/* Always fail migration. Used for mappings that are not movable */ +int fail_migrate_page(struct address_space *mapping, + struct page *newpage, struct page *page) +{ + return -EIO; +} +EXPORT_SYMBOL(fail_migrate_page); + +/* + * Common logic to directly migrate a single page suitable for + * pages that do not use PagePrivate/PagePrivate2. + * + * Pages are locked upon entry and exit. + */ +static int __migrate_page(struct address_space *mapping, + struct page *newpage, struct page *page, + enum migrate_mode mode, int pass, int tries) +{ + int rc; + + BUG_ON(PageWriteback(page)); /* Writeback must be complete */ + + rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode, pass, tries); + + if (rc) { + if (is_failed_page(page, pass, tries)) { + printk("%s[%d]: 1 ", __func__, __LINE__); + dump_page(page); + } + return rc; + } + + migrate_page_copy(newpage, page); + return 0; +} + +int migrate_page(struct address_space *mapping, + struct page *newpage, struct page *page, enum migrate_mode mode) +{ + return __migrate_page(mapping, newpage, page, mode, 0, 0); +} +EXPORT_SYMBOL(migrate_page); + +#ifdef CONFIG_BLOCK +/* + * Migration function for pages with buffers. This function can only be used + * if the underlying filesystem guarantees that no other references to "page" + * exist. + */ +int buffer_migrate_page(struct address_space *mapping, + struct page *newpage, struct page *page, enum migrate_mode mode) +{ + struct buffer_head *bh, *head; + int rc; + + if (!page_has_buffers(page)) + return migrate_page(mapping, newpage, page, mode); + + head = page_buffers(page); + + rc = migrate_page_move_mapping(mapping, newpage, page, head, mode, 0, 0); + + if (rc) + return rc; + + /* + * In the async case, migrate_page_move_mapping locked the buffers + * with an IRQ-safe spinlock held. In the sync case, the buffers + * need to be locked now + */ + if (mode != MIGRATE_ASYNC) + BUG_ON(!buffer_migrate_lock_buffers(head, mode)); + + ClearPagePrivate(page); + set_page_private(newpage, page_private(page)); + set_page_private(page, 0); + put_page(page); + get_page(newpage); + + bh = head; + do { + set_bh_page(bh, newpage, bh_offset(bh)); + bh = bh->b_this_page; + + } while (bh != head); + + SetPagePrivate(newpage); + + migrate_page_copy(newpage, page); + + bh = head; + do { + unlock_buffer(bh); + put_bh(bh); + bh = bh->b_this_page; + + } while (bh != head); + + return 0; +} +EXPORT_SYMBOL(buffer_migrate_page); +#endif + +/* + * Writeback a page to clean the dirty state + */ +static int writeout(struct address_space *mapping, struct page *page) +{ + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + .nr_to_write = 1, + .range_start = 0, + .range_end = LLONG_MAX, + .for_reclaim = 1 + }; + int rc; + + if (!mapping->a_ops->writepage) + /* No write method for the address space */ + return -EINVAL; + + if (!clear_page_dirty_for_io(page)) + /* Someone else already triggered a write */ + return -EAGAIN; + + /* + * A dirty page may imply that the underlying filesystem has + * the page on some queue. So the page must be clean for + * migration. Writeout may mean we loose the lock and the + * page state is no longer what we checked for earlier. + * At this point we know that the migration attempt cannot + * be successful. + */ + remove_migration_ptes(page, page); + + rc = mapping->a_ops->writepage(page, &wbc); + + if (rc != AOP_WRITEPAGE_ACTIVATE) + /* unlocked. Relock */ + lock_page(page); + + return (rc < 0) ? -EIO : -EAGAIN; +} + +/* + * Default handling if a filesystem does not provide a migration function. + */ +static int fallback_migrate_page(struct address_space *mapping, + struct page *newpage, struct page *page, enum migrate_mode mode, int pass, int tries) +{ + int rc; + + if (PageDirty(page)) { + if(mode != MIGRATE_SYNC) + return -EBUSY; + rc = writeout(mapping, page); + if (is_failed_page(page, pass, tries)) { + printk("%s[%d] 1 ", __func__, __LINE__); + dump_page(page); + } + return rc; + } + + /* + * Buffers may be managed in a filesystem specific way. + * We must have no buffers or drop them. + */ + if (page_has_private(page) && + !try_to_release_page(page, GFP_KERNEL)) { + if (is_failed_page(page, pass, tries)) { + printk("%s[%d] 2 ", __func__, __LINE__); + dump_page(page); + } + return -EAGAIN; + } + + rc = __migrate_page(mapping, newpage, page, mode, pass, tries); + if (rc) { + if (is_failed_page(page, pass, tries)) { + printk("%s[%d] 3 ", __func__, __LINE__); + dump_page(page); + } + } + return rc; +} + +/* + * Move a page to a newly allocated page + * The page is locked and all ptes have been successfully removed. + * + * The new page will have replaced the old page if this function + * is successful. + * + * Return value: + * < 0 - error code + * == 0 - success + */ +static int move_to_new_page(struct page *newpage, struct page *page, + int remap_swapcache, enum migrate_mode mode, + int pass, int tries) +{ + struct address_space *mapping; + int rc = 0; + + /* + * Block others from accessing the page when we get around to + * establishing additional references. We are the only one + * holding a reference to the new page at this point. + */ + if (!trylock_page(newpage)) + BUG(); + + /* Prepare mapping for the new page.*/ + newpage->index = page->index; + newpage->mapping = page->mapping; + if (PageSwapBacked(page)) + SetPageSwapBacked(newpage); + + mapping = page_mapping(page); + if (!mapping) { + rc = __migrate_page(mapping, newpage, page, mode, pass, tries); + if (rc) { + if (is_failed_page(page, pass, tries)) { + printk("%s[%d]: 1 ", __func__, __LINE__); + dump_page(page); + } + } + } else if (mapping->a_ops->migratepage) { + /* + * Most pages have a mapping and most filesystems + * should provide a migration function. Anonymous + * pages are part of swap space which also has its + * own migration function. This is the most common + * path for page migration. + */ + rc = mapping->a_ops->migratepage(mapping, + newpage, page, mode); + if (rc) { + if (is_failed_page(page, pass, tries)) { + printk(KERN_ERR "%s[%d]: 3 ", + __func__, __LINE__); + dump_page(page); + } + } + } else { + rc = fallback_migrate_page(mapping, newpage, page, mode, + pass, tries); + if (rc) { + if (is_failed_page(page, pass, tries)) { + printk(KERN_ERR "%s[%d]: 4 ", + __func__, __LINE__); + dump_page(page); + } + } + } + + if (rc) { + newpage->mapping = NULL; + } else { + if (remap_swapcache) + remove_migration_ptes(page, newpage); + } + + unlock_page(newpage); + + return rc; +} + +/* + * Obtain the lock on page, remove all ptes and migrate the page + * to the newly allocated page in newpage. + */ +static int unmap_and_move(new_page_t get_new_page, unsigned long private, + struct page *page, int force, bool offlining, enum migrate_mode mode, + int pass, int tries) +{ + int rc = 0; + int *result = NULL; + struct page *newpage = get_new_page(page, private, &result); + int remap_swapcache = 1; + int charge = 0; + struct mem_cgroup *mem; + struct anon_vma *anon_vma = NULL; + + if (!newpage) + return -ENOMEM; + + if (page_count(page) == 1) { + /* page was freed from under us. So we are done. */ + goto move_newpage; + } + if (unlikely(PageTransHuge(page))) + if (unlikely(split_huge_page(page))) + goto move_newpage; + + /* prepare cgroup just returns 0 or -ENOMEM */ + rc = -EAGAIN; + + if (!trylock_page(page)) { + if (!force || mode == MIGRATE_ASYNC) { + if (is_failed_page(page, pass, tries)) { + printk("%s[%d] 1 ", __func__, __LINE__); + dump_page(page); + } + goto move_newpage; + } + + /* + * It's not safe for direct compaction to call lock_page. + * For example, during page readahead pages are added locked + * to the LRU. Later, when the IO completes the pages are + * marked uptodate and unlocked. However, the queueing + * could be merging multiple pages for one bio (e.g. + * mpage_readpages). If an allocation happens for the + * second or third page, the process can end up locking + * the same page twice and deadlocking. Rather than + * trying to be clever about what pages can be locked, + * avoid the use of lock_page for direct compaction + * altogether. + */ + if (current->flags & PF_MEMALLOC) { + if (is_failed_page(page, pass, tries)) { + printk("%s[%d] 2 ", __func__, __LINE__); + dump_page(page); + } + goto move_newpage; + } + + lock_page(page); + } + + /* + * Only memory hotplug's offline_pages() caller has locked out KSM, + * and can safely migrate a KSM page. The other cases have skipped + * PageKsm along with PageReserved - but it is only now when we have + * the page lock that we can be certain it will not go KSM beneath us + * (KSM will not upgrade a page from PageAnon to PageKsm when it sees + * its pagecount raised, but only here do we take the page lock which + * serializes that). + */ + if (PageKsm(page) && !offlining) { + rc = -EBUSY; + goto unlock; + } + + if (invalidate_inode_page(page)) { + rc = 0; + goto unlock; + } + + /* charge against new page */ + charge = mem_cgroup_prepare_migration(page, newpage, &mem, GFP_KERNEL); + if (charge == -ENOMEM) { + rc = -ENOMEM; + goto unlock; + } + BUG_ON(charge); + + if (PageWriteback(page)) { + /* + * Only in the case of a full syncronous migration is it + * necessary to wait for PageWriteback. In the async case, + * the retry loop is too short and in the sync-light case, + * the overhead of stalling is too much + */ + if (mode != MIGRATE_SYNC) { + rc = -EBUSY; + goto uncharge; + } + if (!force) + goto uncharge; + wait_on_page_writeback(page); + } + /* + * By try_to_unmap(), page->mapcount goes down to 0 here. In this case, + * we cannot notice that anon_vma is freed while we migrates a page. + * This get_anon_vma() delays freeing anon_vma pointer until the end + * of migration. File cache pages are no problem because of page_lock() + * File Caches may use write_page() or lock_page() in migration, then, + * just care Anon page here. + */ + if (PageAnon(page)) { + /* + * Only page_lock_anon_vma() understands the subtleties of + * getting a hold on an anon_vma from outside one of its mms. + */ + anon_vma = page_get_anon_vma(page); + if (anon_vma) { + /* + * Anon page + */ + } else if (PageSwapCache(page)) { + /* + * We cannot be sure that the anon_vma of an unmapped + * swapcache page is safe to use because we don't + * know in advance if the VMA that this page belonged + * to still exists. If the VMA and others sharing the + * data have been freed, then the anon_vma could + * already be invalid. + * + * To avoid this possibility, swapcache pages get + * migrated but are not remapped when migration + * completes + */ + remap_swapcache = 0; + } else { + if (is_failed_page(page, pass, tries)) { + printk("%s[%d] 3 ", __func__, __LINE__); + dump_page(page); + } + goto uncharge; + } + } + + /* + * Corner case handling: + * 1. When a new swap-cache page is read into, it is added to the LRU + * and treated as swapcache but it has no rmap yet. + * Calling try_to_unmap() against a page->mapping==NULL page will + * trigger a BUG. So handle it here. + * 2. An orphaned page (see truncate_complete_page) might have + * fs-private metadata. The page can be picked up due to memory + * offlining. Everywhere else except page reclaim, the page is + * invisible to the vm, so the page can not be migrated. So try to + * free the metadata, so the page can be freed. + */ + if (!page->mapping) { + VM_BUG_ON(PageAnon(page)); + if (page_has_private(page)) { + try_to_free_buffers(page); + if (is_failed_page(page, pass, tries)) { + printk("%s[%d] 4 ", __func__, __LINE__); + dump_page(page); + } + goto uncharge; + } + goto skip_unmap; + } + + /* Establish migration ptes or remove ptes */ + try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); + +skip_unmap: + if (!page_mapped(page)) { + rc = move_to_new_page(newpage, page, remap_swapcache, mode, + pass, tries); + if (rc) { + if (is_failed_page(page, pass, tries)) { + printk("%s[%d] 5 ", __func__, __LINE__); + dump_page(page); + } + } + } + + if (rc && remap_swapcache) + remove_migration_ptes(page, page); + + /* Drop an anon_vma reference if we took one */ + if (anon_vma) + put_anon_vma(anon_vma); + +uncharge: + if (!charge) + mem_cgroup_end_migration(mem, page, newpage, rc == 0); +unlock: + unlock_page(page); + +move_newpage: + if (rc != -EAGAIN) { + /* + * A page that has been migrated has all references + * removed and will be freed. A page that has not been + * migrated will have kepts its references and be + * restored. + */ + list_del(&page->lru); + dec_zone_page_state(page, NR_ISOLATED_ANON + + page_is_file_cache(page)); + putback_lru_page(page); + } + + /* + * Move the new page to the LRU. If migration was not successful + * then this will free the page. + */ + putback_lru_page(newpage); + + if (result) { + if (rc) + *result = rc; + else + *result = page_to_nid(newpage); + } + return rc; +} + +/* + * Counterpart of unmap_and_move_page() for hugepage migration. + * + * This function doesn't wait the completion of hugepage I/O + * because there is no race between I/O and migration for hugepage. + * Note that currently hugepage I/O occurs only in direct I/O + * where no lock is held and PG_writeback is irrelevant, + * and writeback status of all subpages are counted in the reference + * count of the head page (i.e. if all subpages of a 2MB hugepage are + * under direct I/O, the reference of the head page is 512 and a bit more.) + * This means that when we try to migrate hugepage whose subpages are + * doing direct I/O, some references remain after try_to_unmap() and + * hugepage migration fails without data corruption. + * + * There is also no race when direct I/O is issued on the page under migration, + * because then pte is replaced with migration swap entry and direct I/O code + * will wait in the page fault for migration to complete. + */ +static int unmap_and_move_huge_page(new_page_t get_new_page, + unsigned long private, struct page *hpage, + int force, bool offlining, enum migrate_mode mode) +{ + int rc = 0; + int *result = NULL; + struct page *new_hpage = get_new_page(hpage, private, &result); + struct anon_vma *anon_vma = NULL; + + if (!new_hpage) + return -ENOMEM; + + rc = -EAGAIN; + + if (!trylock_page(hpage)) { + if (!force || mode != MIGRATE_SYNC) + goto out; + lock_page(hpage); + } + + if (PageAnon(hpage)) + anon_vma = page_get_anon_vma(hpage); + + try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); + + if (!page_mapped(hpage)) + rc = move_to_new_page(new_hpage, hpage, 1, mode, 0, 0); + + if (rc) + remove_migration_ptes(hpage, hpage); + + if (anon_vma) + put_anon_vma(anon_vma); +out: + unlock_page(hpage); + + if (rc != -EAGAIN) { + list_del(&hpage->lru); + put_page(hpage); + } + + put_page(new_hpage); + + if (result) { + if (rc) + *result = rc; + else + *result = page_to_nid(new_hpage); + } + return rc; +} + +struct page *migrate_pages_current = NULL; + +/* + * migrate_pages + * + * The function takes one list of pages to migrate and a function + * that determines from the page to be migrated and the private data + * the target of the move and allocates the page. + * + * The function returns after 10 attempts or if no pages + * are movable anymore because to has become empty + * or no retryable pages exist anymore. + * Caller should call putback_lru_pages to return pages to the LRU + * or free list only if ret != 0. + * + * Return: Number of pages not migrated or error code. + */ +int migrate_pages(struct list_head *from, + new_page_t get_new_page, unsigned long private, bool offlining, + enum migrate_mode mode, int tries) +{ + int retry = 1; + int nr_failed = 0; + int pass = 0; + struct page *page; + struct page *page2; + int swapwrite = current->flags & PF_SWAPWRITE; + int rc; + + if (!swapwrite) + current->flags |= PF_SWAPWRITE; + + for (pass = 0; pass < 10 && retry; pass++) { + retry = 0; + + list_for_each_entry_safe(page, page2, from, lru) { + cond_resched(); + + rc = unmap_and_move(get_new_page, private, + page, pass > 2, offlining, + mode, pass, tries); + if (rc) + failed_pages[tries][pass] = page; + + switch (rc) { + case -ENOMEM: + goto out; + case -EAGAIN: + if (0) { + migrate_pages_current = page; + printk(KERN_INFO "%s:%d ", __func__, + __LINE__); + dump_page(page); + } + retry++; + break; + case 0: + break; + default: + /* Permanent failure */ + nr_failed++; + break; + } + } + } + rc = 0; +out: + if (!swapwrite) + current->flags &= ~PF_SWAPWRITE; + + if (rc) + return rc; + + return nr_failed + retry; +} + +int migrate_huge_pages(struct list_head *from, + new_page_t get_new_page, unsigned long private, bool offlining, + enum migrate_mode mode) +{ + int retry = 1; + int nr_failed = 0; + int pass = 0; + struct page *page; + struct page *page2; + int rc; + + for (pass = 0; pass < 10 && retry; pass++) { + retry = 0; + + list_for_each_entry_safe(page, page2, from, lru) { + cond_resched(); + + rc = unmap_and_move_huge_page(get_new_page, + private, page, pass > 2, offlining, + mode); + + switch (rc) { + case -ENOMEM: + goto out; + case -EAGAIN: + retry++; + break; + case 0: + break; + default: + /* Permanent failure */ + nr_failed++; + break; + } + } + } + rc = 0; +out: + if (rc) + return rc; + + return nr_failed + retry; +} + +static struct page * +__migrate_replace_alloc(struct page *page, unsigned long private, + int **resultp) +{ + struct page **newpage = (struct page **)private; + *newpage = alloc_page(GFP_USER); + + return *newpage; +} + +/* + * migrate_replace_single_page + * + * The function takes one single page (oldpage) and a target page + * (newpage) and tries to migrate data to the target page. The caller + * must ensure that the source page is locked with one additional + * get_page() call, which will be freed during the migration. + * + * Return: error code or 0 on success. + */ +int migrate_replace_cma_page(struct page *oldpage, struct page **newpage) +{ + /* This function is based on compact_zone() from compaction.c. */ + unsigned long pfn = page_to_pfn(oldpage); + int ret; + struct compact_control cc = { + .nr_migratepages = 0, + .order = 0, + .zone = page_zone(oldpage), + .sync = true, + }; + INIT_LIST_HEAD(&cc.migratepages); + + migrate_prep_local(); + + pfn = isolate_migratepages_range(cc.zone, &cc, pfn, pfn+1); + if (!pfn || list_empty(&cc.migratepages)) + goto putback; + + /* + * Put the additional reference to the old page, now migration code + * owns it + */ + put_page(oldpage); + + ret = migrate_pages(&cc.migratepages, __migrate_replace_alloc, + (unsigned long)newpage, false, true, 0); + + if (ret == 0) { + /* + * Do the same as follow_page() did with oldpage and + * return + */ + get_page_foll(*newpage); + return 0; + } + + if (is_failed_page(oldpage, 0, 0)) + dump_page(oldpage); + + /* + * Restore additional reference to the old page before giving it back + * to lru + */ + get_page(oldpage); +putback: + putback_lru_pages(&cc.migratepages); + return -EBUSY; +} + +#ifdef CONFIG_NUMA +/* + * Move a list of individual pages + */ +struct page_to_node { + unsigned long addr; + struct page *page; + int node; + int status; +}; + +static struct page *new_page_node(struct page *p, unsigned long private, + int **result) +{ + struct page_to_node *pm = (struct page_to_node *)private; + + while (pm->node != MAX_NUMNODES && pm->page != p) + pm++; + + if (pm->node == MAX_NUMNODES) + return NULL; + + *result = &pm->status; + + return alloc_pages_exact_node(pm->node, + GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0); +} + +/* + * Move a set of pages as indicated in the pm array. The addr + * field must be set to the virtual address of the page to be moved + * and the node number must contain a valid target node. + * The pm array ends with node = MAX_NUMNODES. + */ +static int do_move_page_to_node_array(struct mm_struct *mm, + struct page_to_node *pm, + int migrate_all) +{ + int err; + struct page_to_node *pp; + LIST_HEAD(pagelist); + + down_read(&mm->mmap_sem); + + /* + * Build a list of pages to migrate + */ + for (pp = pm; pp->node != MAX_NUMNODES; pp++) { + struct vm_area_struct *vma; + struct page *page; + + err = -EFAULT; + vma = find_vma(mm, pp->addr); + if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma)) + goto set_status; + + page = follow_page(vma, pp->addr, FOLL_GET|FOLL_SPLIT); + + err = PTR_ERR(page); + if (IS_ERR(page)) + goto set_status; + + err = -ENOENT; + if (!page) + goto set_status; + + /* Use PageReserved to check for zero page */ + if (PageReserved(page) || PageKsm(page)) + goto put_and_set; + + pp->page = page; + err = page_to_nid(page); + + if (err == pp->node) + /* + * Node already in the right place + */ + goto put_and_set; + + err = -EACCES; + if (page_mapcount(page) > 1 && + !migrate_all) + goto put_and_set; + + err = isolate_lru_page(page); + if (!err) { + list_add_tail(&page->lru, &pagelist); + inc_zone_page_state(page, NR_ISOLATED_ANON + + page_is_file_cache(page)); + } +put_and_set: + /* + * Either remove the duplicate refcount from + * isolate_lru_page() or drop the page ref if it was + * not isolated. + */ + put_page(page); +set_status: + pp->status = err; + } + + err = 0; + if (!list_empty(&pagelist)) { + err = migrate_pages(&pagelist, new_page_node, + (unsigned long)pm, 0, MIGRATE_SYNC, 0); + if (err) + putback_lru_pages(&pagelist); + } + + up_read(&mm->mmap_sem); + return err; +} + +/* + * Migrate an array of page address onto an array of nodes and fill + * the corresponding array of status. + */ +static int do_pages_move(struct mm_struct *mm, struct task_struct *task, + unsigned long nr_pages, + const void __user * __user *pages, + const int __user *nodes, + int __user *status, int flags) +{ + struct page_to_node *pm; + nodemask_t task_nodes; + unsigned long chunk_nr_pages; + unsigned long chunk_start; + int err; + + task_nodes = cpuset_mems_allowed(task); + + err = -ENOMEM; + pm = (struct page_to_node *)__get_free_page(GFP_KERNEL); + if (!pm) + goto out; + + migrate_prep(); + + /* + * Store a chunk of page_to_node array in a page, + * but keep the last one as a marker + */ + chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1; + + for (chunk_start = 0; + chunk_start < nr_pages; + chunk_start += chunk_nr_pages) { + int j; + + if (chunk_start + chunk_nr_pages > nr_pages) + chunk_nr_pages = nr_pages - chunk_start; + + /* fill the chunk pm with addrs and nodes from user-space */ + for (j = 0; j < chunk_nr_pages; j++) { + const void __user *p; + int node; + + err = -EFAULT; + if (get_user(p, pages + j + chunk_start)) + goto out_pm; + pm[j].addr = (unsigned long) p; + + if (get_user(node, nodes + j + chunk_start)) + goto out_pm; + + err = -ENODEV; + if (node < 0 || node >= MAX_NUMNODES) + goto out_pm; + + if (!node_state(node, N_HIGH_MEMORY)) + goto out_pm; + + err = -EACCES; + if (!node_isset(node, task_nodes)) + goto out_pm; + + pm[j].node = node; + } + + /* End marker for this chunk */ + pm[chunk_nr_pages].node = MAX_NUMNODES; + + /* Migrate this chunk */ + err = do_move_page_to_node_array(mm, pm, + flags & MPOL_MF_MOVE_ALL); + if (err < 0) + goto out_pm; + + /* Return status information */ + for (j = 0; j < chunk_nr_pages; j++) + if (put_user(pm[j].status, status + j + chunk_start)) { + err = -EFAULT; + goto out_pm; + } + } + err = 0; + +out_pm: + free_page((unsigned long)pm); +out: + return err; +} + +/* + * Determine the nodes of an array of pages and store it in an array of status. + */ +static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, + const void __user **pages, int *status) +{ + unsigned long i; + + down_read(&mm->mmap_sem); + + for (i = 0; i < nr_pages; i++) { + unsigned long addr = (unsigned long)(*pages); + struct vm_area_struct *vma; + struct page *page; + int err = -EFAULT; + + vma = find_vma(mm, addr); + if (!vma || addr < vma->vm_start) + goto set_status; + + page = follow_page(vma, addr, 0); + + err = PTR_ERR(page); + if (IS_ERR(page)) + goto set_status; + + err = -ENOENT; + /* Use PageReserved to check for zero page */ + if (!page || PageReserved(page) || PageKsm(page)) + goto set_status; + + err = page_to_nid(page); +set_status: + *status = err; + + pages++; + status++; + } + + up_read(&mm->mmap_sem); +} + +/* + * Determine the nodes of a user array of pages and store it in + * a user array of status. + */ +static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages, + const void __user * __user *pages, + int __user *status) +{ +#define DO_PAGES_STAT_CHUNK_NR 16 + const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR]; + int chunk_status[DO_PAGES_STAT_CHUNK_NR]; + + while (nr_pages) { + unsigned long chunk_nr; + + chunk_nr = nr_pages; + if (chunk_nr > DO_PAGES_STAT_CHUNK_NR) + chunk_nr = DO_PAGES_STAT_CHUNK_NR; + + if (copy_from_user(chunk_pages, pages, + chunk_nr * sizeof(*chunk_pages))) + break; + + do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status); + + if (copy_to_user(status, chunk_status, + chunk_nr * sizeof(*status))) + break; + + pages += chunk_nr; + status += chunk_nr; + nr_pages -= chunk_nr; + } + return nr_pages ? -EFAULT : 0; +} + +/* + * Move a list of pages in the address space of the currently executing + * process. + */ +SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages, + const void __user * __user *, pages, + const int __user *, nodes, + int __user *, status, int, flags) +{ + const struct cred *cred = current_cred(), *tcred; + struct task_struct *task; + struct mm_struct *mm; + int err; + + /* Check flags */ + if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) + return -EINVAL; + + if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) + return -EPERM; + + /* Find the mm_struct */ + rcu_read_lock(); + task = pid ? find_task_by_vpid(pid) : current; + if (!task) { + rcu_read_unlock(); + return -ESRCH; + } + mm = get_task_mm(task); + rcu_read_unlock(); + + if (!mm) + return -EINVAL; + + /* + * Check if this process has the right to modify the specified + * process. The right exists if the process has administrative + * capabilities, superuser privileges or the same + * userid as the target process. + */ + rcu_read_lock(); + tcred = __task_cred(task); + if (cred->euid != tcred->suid && cred->euid != tcred->uid && + cred->uid != tcred->suid && cred->uid != tcred->uid && + !capable(CAP_SYS_NICE)) { + rcu_read_unlock(); + err = -EPERM; + goto out; + } + rcu_read_unlock(); + + err = security_task_movememory(task); + if (err) + goto out; + + if (nodes) + err = do_pages_move(mm, task, nr_pages, pages, nodes, status, + flags); + else + err = do_pages_stat(mm, nr_pages, pages, status); + +out: + mmput(mm); + return err; +} + +/* + * Call migration functions in the vma_ops that may prepare + * memory in a vm for migration. migration functions may perform + * the migration for vmas that do not have an underlying page struct. + */ +int migrate_vmas(struct mm_struct *mm, const nodemask_t *to, + const nodemask_t *from, unsigned long flags) +{ + struct vm_area_struct *vma; + int err = 0; + + for (vma = mm->mmap; vma && !err; vma = vma->vm_next) { + if (vma->vm_ops && vma->vm_ops->migrate) { + err = vma->vm_ops->migrate(vma, to, from, flags); + if (err) + break; + } + } + return err; +} +#endif diff --git a/mm/migrate.c b/mm/migrate.c index 7d26ea5..3e315a7 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -13,7 +13,7 @@ */ #include <linux/migrate.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/swap.h> #include <linux/swapops.h> #include <linux/pagemap.h> @@ -141,11 +141,8 @@ static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, get_page(new); pte = pte_mkold(mk_pte(new, vma->vm_page_prot)); - - /* Recheck VMA as permissions can change since migration started */ if (is_write_migration_entry(entry)) - pte = maybe_mkwrite(pte, vma); - + pte = pte_mkwrite(pte); #ifdef CONFIG_HUGETLB_PAGE if (PageHuge(new)) pte = pte_mkhuge(pte); @@ -950,9 +947,9 @@ static int unmap_and_move_huge_page(new_page_t get_new_page, if (anon_vma) put_anon_vma(anon_vma); +out: unlock_page(hpage); -out: if (rc != -EAGAIN) { list_del(&hpage->lru); put_page(hpage); diff --git a/mm/mincore.c b/mm/mincore.c index 936b4ce..117ff54 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -69,15 +69,12 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t pgoff) * file will not get a swp_entry_t in its pte, but rather it is like * any other file mapping (ie. marked !present and faulted in with * tmpfs's .fault). So swapped out tmpfs mappings are tested here. + * + * However when tmpfs moves the page from pagecache and into swapcache, + * it is still in core, but the find_get_page below won't find it. + * No big deal, but make a note of it. */ page = find_get_page(mapping, pgoff); -#ifdef CONFIG_SWAP - /* shmem/tmpfs may return swap: account for swapcache page too. */ - if (radix_tree_exceptional_entry(page)) { - swp_entry_t swap = radix_to_swp_entry(page); - page = find_get_page(&swapper_space, swap.val); - } -#endif if (page) { present = PageUptodate(page); page_cache_release(page); @@ -14,7 +14,7 @@ #include <linux/mempolicy.h> #include <linux/syscalls.h> #include <linux/sched.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/rmap.h> #include <linux/mmzone.h> #include <linux/hugetlb.h> @@ -78,7 +78,6 @@ void __clear_page_mlock(struct page *page) */ void mlock_vma_page(struct page *page) { - /* Serialize with page migration */ BUG_ON(!PageLocked(page)); if (!TestSetPageMlocked(page)) { @@ -106,21 +105,12 @@ void mlock_vma_page(struct page *page) */ void munlock_vma_page(struct page *page) { - /* For try_to_munlock() and to serialize with page migration */ BUG_ON(!PageLocked(page)); if (TestClearPageMlocked(page)) { dec_zone_page_state(page, NR_MLOCK); if (!isolate_lru_page(page)) { - int ret = SWAP_AGAIN; - - /* - * Optimization: if the page was mapped just once, - * that's our mapping and we don't need to check all the - * other vmas. - */ - if (page_mapcount(page) > 1) - ret = try_to_munlock(page); + int ret = try_to_munlock(page); /* * did try_to_unlock() succeed or punt? */ @@ -559,8 +549,7 @@ SYSCALL_DEFINE1(mlockall, int, flags) if (!can_do_mlock()) goto out; - if (flags & MCL_CURRENT) - lru_add_drain_all(); /* flush pagevec */ + lru_add_drain_all(); /* flush pagevec */ down_write(¤t->mm->mmap_sem); diff --git a/mm/mm_init.c b/mm/mm_init.c index 1ffd97a..4e0e265 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -8,7 +8,7 @@ #include <linux/kernel.h> #include <linux/init.h> #include <linux/kobject.h> -#include <linux/export.h> +#include <linux/module.h> #include "internal.h" #ifdef CONFIG_DEBUG_MEMORY_INIT @@ -22,7 +22,7 @@ #include <linux/security.h> #include <linux/hugetlb.h> #include <linux/profile.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/mount.h> #include <linux/mempolicy.h> #include <linux/rmap.h> @@ -111,7 +111,7 @@ struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp; */ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) { - long free, allowed; + unsigned long free, allowed; vm_acct_memory(pages); @@ -122,17 +122,9 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) return 0; if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { - free = global_page_state(NR_FREE_PAGES); - free += global_page_state(NR_FILE_PAGES); - - /* - * shmem pages shouldn't be counted as free in this - * case, they can't be purged, only swapped out, and - * that won't affect the overall amount of available - * memory in the system. - */ - free -= global_page_state(NR_SHMEM); + unsigned long n; + free = global_page_state(NR_FILE_PAGES); free += nr_swap_pages; /* @@ -144,18 +136,34 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) free += global_page_state(NR_SLAB_RECLAIMABLE); /* + * Leave the last 3% for root + */ + if (!cap_sys_admin) + free -= free / 32; + + if (free > pages) + return 0; + + /* + * nr_free_pages() is very expensive on large systems, + * only call if we're about to fail. + */ + n = nr_free_pages(); + + /* * Leave reserved pages. The pages are not for anonymous pages. */ - if (free <= totalreserve_pages) + if (n <= totalreserve_pages) goto error; else - free -= totalreserve_pages; + n -= totalreserve_pages; /* * Leave the last 3% for root */ if (!cap_sys_admin) - free -= free / 32; + n -= n / 32; + free += n; if (free > pages) return 0; @@ -537,12 +545,9 @@ again: remove_next = 1 + (end > next->vm_end); * shrinking vma had, to cover any anon pages imported. */ if (exporter && exporter->anon_vma && !importer->anon_vma) { - int error; - + if (anon_vma_clone(importer, exporter)) + return -ENOMEM; importer->anon_vma = exporter->anon_vma; - error = anon_vma_clone(importer, exporter); - if (error) - return error; } } @@ -799,7 +804,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, end, prev->vm_pgoff, NULL); if (err) return NULL; - khugepaged_enter_vma_merge(prev, vm_flags); + khugepaged_enter_vma_merge(prev); return prev; } @@ -818,7 +823,7 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm, next->vm_pgoff - pglen, NULL); if (err) return NULL; - khugepaged_enter_vma_merge(area, vm_flags); + khugepaged_enter_vma_merge(area); return area; } @@ -1371,7 +1376,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, struct vm_area_struct *vma; unsigned long start_addr; - if (len > TASK_SIZE - mmap_min_addr) + if (len > TASK_SIZE) return -ENOMEM; if (flags & MAP_FIXED) @@ -1380,7 +1385,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, if (addr) { addr = PAGE_ALIGN(addr); vma = find_vma(mm, addr); - if (TASK_SIZE - len >= addr && addr >= mmap_min_addr && + if (TASK_SIZE - len >= addr && (!vma || addr + len <= vma->vm_start)) return addr; } @@ -1445,10 +1450,9 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, struct vm_area_struct *vma; struct mm_struct *mm = current->mm; unsigned long addr = addr0; - unsigned long low_limit = max(PAGE_SIZE, mmap_min_addr); /* requested length too big for entire address space */ - if (len > TASK_SIZE - mmap_min_addr) + if (len > TASK_SIZE) return -ENOMEM; if (flags & MAP_FIXED) @@ -1458,7 +1462,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, if (addr) { addr = PAGE_ALIGN(addr); vma = find_vma(mm, addr); - if (TASK_SIZE - len >= addr && addr >= mmap_min_addr && + if (TASK_SIZE - len >= addr && (!vma || addr + len <= vma->vm_start)) return addr; } @@ -1473,14 +1477,14 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, addr = mm->free_area_cache; /* make sure it can fit in the remaining address space */ - if (addr >= low_limit + len) { + if (addr > len) { vma = find_vma(mm, addr-len); if (!vma || addr <= vma->vm_start) /* remember the address as a hint for next time */ return (mm->free_area_cache = addr-len); } - if (mm->mmap_base < low_limit + len) + if (mm->mmap_base < len) goto bottomup; addr = mm->mmap_base-len; @@ -1502,7 +1506,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, /* try just below the current vma->vm_start */ addr = vma->vm_start-len; - } while (vma->vm_start >= low_limit + len); + } while (len < vma->vm_start); bottomup: /* @@ -1651,17 +1655,14 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns { struct mm_struct *mm = vma->vm_mm; struct rlimit *rlim = current->signal->rlim; - unsigned long new_start, actual_size; + unsigned long new_start; /* address space limit tests */ if (!may_expand_vm(mm, grow)) return -ENOMEM; /* Stack limit test */ - actual_size = size; - if (size && (vma->vm_flags & (VM_GROWSUP | VM_GROWSDOWN))) - actual_size -= PAGE_SIZE; - if (actual_size > ACCESS_ONCE(rlim[RLIMIT_STACK].rlim_cur)) + if (size > ACCESS_ONCE(rlim[RLIMIT_STACK].rlim_cur)) return -ENOMEM; /* mlock limit tests */ @@ -1747,7 +1748,7 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) } } vma_unlock_anon_vma(vma); - khugepaged_enter_vma_merge(vma, vma->vm_flags); + khugepaged_enter_vma_merge(vma); return error; } #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */ @@ -1798,7 +1799,7 @@ int expand_downwards(struct vm_area_struct *vma, } } vma_unlock_anon_vma(vma); - khugepaged_enter_vma_merge(vma, vma->vm_flags); + khugepaged_enter_vma_merge(vma); return error; } @@ -2565,6 +2566,7 @@ int mm_take_all_locks(struct mm_struct *mm) { struct vm_area_struct *vma; struct anon_vma_chain *avc; + int ret = -EINTR; BUG_ON(down_read_trylock(&mm->mmap_sem)); @@ -2585,11 +2587,13 @@ int mm_take_all_locks(struct mm_struct *mm) vm_lock_anon_vma(mm, avc->anon_vma); } - return 0; + ret = 0; out_unlock: - mm_drop_all_locks(mm); - return -EINTR; + if (ret) + mm_drop_all_locks(mm); + + return ret; } static void vm_unlock_anon_vma(struct anon_vma *anon_vma) diff --git a/mm/mmu_context.c b/mm/mmu_context.c index cf332bc..9e82e93 100644 --- a/mm/mmu_context.c +++ b/mm/mmu_context.c @@ -5,7 +5,7 @@ #include <linux/mm.h> #include <linux/mmu_context.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/sched.h> #include <asm/mmu_context.h> diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index a160ec8..2b52947 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -11,7 +11,7 @@ #include <linux/rculist.h> #include <linux/mmu_notifier.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/mm.h> #include <linux/err.h> #include <linux/srcu.h> @@ -37,7 +37,7 @@ static struct srcu_struct srcu; void __mmu_notifier_release(struct mm_struct *mm) { struct mmu_notifier *mn; - struct hlist_node *n; + struct hlist_node *node; int id; /* @@ -45,7 +45,7 @@ void __mmu_notifier_release(struct mm_struct *mm) * ->release returns. */ id = srcu_read_lock(&srcu); - hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) + hlist_for_each_entry_rcu(mn, node, &mm->mmu_notifier_mm->list, hlist) /* * If ->release runs before mmu_notifier_unregister it must be * handled, as it's the only way for the driver to flush all diff --git a/mm/mmzone.c b/mm/mmzone.c index 7cf7b7d..f5b7d17 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -8,6 +8,7 @@ #include <linux/stddef.h> #include <linux/mm.h> #include <linux/mmzone.h> +#include <linux/module.h> struct pglist_data *first_online_pgdat(void) { diff --git a/mm/mremap.c b/mm/mremap.c index d6959cb..506fa44 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -41,7 +41,8 @@ static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) return NULL; pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) + split_huge_page_pmd(mm, pmd); + if (pmd_none_or_clear_bad(pmd)) return NULL; return pmd; @@ -64,6 +65,8 @@ static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma, return NULL; VM_BUG_ON(pmd_trans_huge(*pmd)); + if (pmd_none(*pmd) && __pte_alloc(mm, vma, pmd, addr)) + return NULL; return pmd; } @@ -77,7 +80,11 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, struct mm_struct *mm = vma->vm_mm; pte_t *old_pte, *new_pte, pte; spinlock_t *old_ptl, *new_ptl; + unsigned long old_start; + old_start = old_addr; + mmu_notifier_invalidate_range_start(vma->vm_mm, + old_start, old_end); if (vma->vm_file) { /* * Subtle point from Rajesh Venkatasubramanian: before @@ -104,7 +111,7 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, new_pte++, new_addr += PAGE_SIZE) { if (pte_none(*old_pte)) continue; - pte = ptep_get_and_clear(mm, old_addr, old_pte); + pte = ptep_clear_flush(vma, old_addr, old_pte); pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr); set_pte_at(mm, new_addr, new_pte, pte); } @@ -116,6 +123,7 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, pte_unmap_unlock(old_pte - 1, old_ptl); if (mapping) mutex_unlock(&mapping->i_mmap_mutex); + mmu_notifier_invalidate_range_end(vma->vm_mm, old_start, old_end); } #define LATENCY_LIMIT (64 * PAGE_SIZE) @@ -126,43 +134,22 @@ unsigned long move_page_tables(struct vm_area_struct *vma, { unsigned long extent, next, old_end; pmd_t *old_pmd, *new_pmd; - bool need_flush = false; old_end = old_addr + len; flush_cache_range(vma, old_addr, old_end); - mmu_notifier_invalidate_range_start(vma->vm_mm, old_addr, old_end); - for (; old_addr < old_end; old_addr += extent, new_addr += extent) { cond_resched(); next = (old_addr + PMD_SIZE) & PMD_MASK; - /* even if next overflowed, extent below will be ok */ + if (next - 1 > old_end) + next = old_end; extent = next - old_addr; - if (extent > old_end - old_addr) - extent = old_end - old_addr; old_pmd = get_old_pmd(vma->vm_mm, old_addr); if (!old_pmd) continue; new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr); if (!new_pmd) break; - if (pmd_trans_huge(*old_pmd)) { - int err = 0; - if (extent == HPAGE_PMD_SIZE) - err = move_huge_pmd(vma, new_vma, old_addr, - new_addr, old_end, - old_pmd, new_pmd); - if (err > 0) { - need_flush = true; - continue; - } else if (!err) { - split_huge_page_pmd(vma->vm_mm, old_pmd); - } - VM_BUG_ON(pmd_trans_huge(*old_pmd)); - } - if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma, - new_pmd, new_addr)) - break; next = (new_addr + PMD_SIZE) & PMD_MASK; if (extent > next - new_addr) extent = next - new_addr; @@ -170,12 +157,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma, extent = LATENCY_LIMIT; move_ptes(vma, old_pmd, old_addr, old_addr + extent, new_vma, new_pmd, new_addr); - need_flush = true; } - if (likely(need_flush)) - flush_tlb_range(vma, old_end-len, old_addr); - - mmu_notifier_invalidate_range_end(vma->vm_mm, old_end-len, old_end); return len + old_addr - old_end; /* how much done */ } diff --git a/mm/nobootmem.c b/mm/nobootmem.c index 07c08c4..e39e3ef 100644 --- a/mm/nobootmem.c +++ b/mm/nobootmem.c @@ -12,7 +12,7 @@ #include <linux/pfn.h> #include <linux/slab.h> #include <linux/bootmem.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/kmemleak.h> #include <linux/range.h> #include <linux/memblock.h> @@ -13,7 +13,7 @@ * Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org> */ -#include <linux/export.h> +#include <linux/module.h> #include <linux/mm.h> #include <linux/mman.h> #include <linux/swap.h> @@ -22,6 +22,7 @@ #include <linux/pagemap.h> #include <linux/slab.h> #include <linux/vmalloc.h> +#include <linux/tracehook.h> #include <linux/blkdev.h> #include <linux/backing-dev.h> #include <linux/mount.h> @@ -454,7 +455,7 @@ void __attribute__((weak)) vmalloc_sync_all(void) * between processes, it syncs the pagetable across all * processes. */ -struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) +struct vm_struct *alloc_vm_area(size_t size) { BUG(); return NULL; @@ -1088,7 +1089,7 @@ static unsigned long determine_vm_flags(struct file *file, * it's being traced - otherwise breakpoints set in it may interfere * with another untraced process */ - if ((flags & MAP_PRIVATE) && current->ptrace) + if ((flags & MAP_PRIVATE) && tracehook_expect_breakpoints(current)) vm_flags &= ~VM_MAYSHARE; return vm_flags; @@ -1885,7 +1886,7 @@ EXPORT_SYMBOL(unmap_mapping_range); */ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) { - long free, allowed; + unsigned long free, allowed; vm_acct_memory(pages); @@ -1896,17 +1897,9 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) return 0; if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { - free = global_page_state(NR_FREE_PAGES); - free += global_page_state(NR_FILE_PAGES); - - /* - * shmem pages shouldn't be counted as free in this - * case, they can't be purged, only swapped out, and - * that won't affect the overall amount of available - * memory in the system. - */ - free -= global_page_state(NR_SHMEM); + unsigned long n; + free = global_page_state(NR_FILE_PAGES); free += nr_swap_pages; /* @@ -1918,18 +1911,34 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) free += global_page_state(NR_SLAB_RECLAIMABLE); /* + * Leave the last 3% for root + */ + if (!cap_sys_admin) + free -= free / 32; + + if (free > pages) + return 0; + + /* + * nr_free_pages() is very expensive on large systems, + * only call if we're about to fail. + */ + n = nr_free_pages(); + + /* * Leave reserved pages. The pages are not for anonymous pages. */ - if (free <= totalreserve_pages) + if (n <= totalreserve_pages) goto error; else - free -= totalreserve_pages; + n -= totalreserve_pages; /* * Leave the last 3% for root */ if (!cap_sys_admin) - free -= free / 32; + n -= n / 32; + free += n; if (free > pages) return 0; diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 4dda948..7c72487 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -26,38 +26,18 @@ #include <linux/timex.h> #include <linux/jiffies.h> #include <linux/cpuset.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/notifier.h> #include <linux/memcontrol.h> #include <linux/mempolicy.h> #include <linux/security.h> #include <linux/ptrace.h> -#include <linux/freezer.h> int sysctl_panic_on_oom; int sysctl_oom_kill_allocating_task; int sysctl_oom_dump_tasks = 1; static DEFINE_SPINLOCK(zone_scan_lock); -/* - * compare_swap_oom_score_adj() - compare and swap current's oom_score_adj - * @old_val: old oom_score_adj for compare - * @new_val: new oom_score_adj for swap - * - * Sets the oom_score_adj value for current to @new_val iff its present value is - * @old_val. Usually used to reinstate a previous value to prevent racing with - * userspacing tuning the value in the interim. - */ -void compare_swap_oom_score_adj(int old_val, int new_val) -{ - struct sighand_struct *sighand = current->sighand; - - spin_lock_irq(&sighand->siglock); - if (current->signal->oom_score_adj == old_val) - current->signal->oom_score_adj = new_val; - spin_unlock_irq(&sighand->siglock); -} - /** * test_set_oom_score_adj() - set current's oom_score_adj and return old value * @new_val: new oom_score_adj value @@ -73,7 +53,13 @@ int test_set_oom_score_adj(int new_val) spin_lock_irq(&sighand->siglock); old_val = current->signal->oom_score_adj; - current->signal->oom_score_adj = new_val; + if (new_val != old_val) { + if (new_val == OOM_SCORE_ADJ_MIN) + atomic_inc(¤t->mm->oom_disable_count); + else if (old_val == OOM_SCORE_ADJ_MIN) + atomic_dec(¤t->mm->oom_disable_count); + current->signal->oom_score_adj = new_val; + } spin_unlock_irq(&sighand->siglock); return old_val; @@ -185,7 +171,12 @@ unsigned int oom_badness(struct task_struct *p, struct mem_cgroup *mem, if (!p) return 0; - if (p->signal->oom_score_adj == OOM_SCORE_ADJ_MIN) { + /* + * Shortcut check for a thread sharing p->mm that is OOM_SCORE_ADJ_MIN + * so the entire heuristic doesn't need to be executed for something + * that cannot be killed. + */ + if (atomic_read(&p->mm->oom_disable_count)) { task_unlock(p); return 0; } @@ -213,7 +204,7 @@ unsigned int oom_badness(struct task_struct *p, struct mem_cgroup *mem, * implementation used by LSMs. */ if (has_capability_noaudit(p, CAP_SYS_ADMIN)) - points -= (points * 3) / 100; + points -= 30; /* * /proc/pid/oom_score_adj ranges from -1000 to +1000 such that it may @@ -326,11 +317,8 @@ static struct task_struct *select_bad_process(unsigned int *ppoints, * blocked waiting for another task which itself is waiting * for memory. Is there a better alternative? */ - if (test_tsk_thread_flag(p, TIF_MEMDIE)) { - if (unlikely(frozen(p))) - thaw_process(p); + if (test_tsk_thread_flag(p, TIF_MEMDIE)) return ERR_PTR(-1UL); - } if (!p->mm) continue; @@ -353,7 +341,8 @@ static struct task_struct *select_bad_process(unsigned int *ppoints, * then wait for it to finish before killing * some other task unnecessarily. */ - if (!(p->group_leader->ptrace & PT_TRACE_EXIT)) + if (!(task_ptrace(p->group_leader) & + PT_TRACE_EXIT)) return ERR_PTR(-1UL); } } @@ -447,7 +436,7 @@ static int oom_kill_task(struct task_struct *p, struct mem_cgroup *mem) task_unlock(p); /* - * Kill all user processes sharing p->mm in other thread groups, if any. + * Kill all processes sharing p->mm in other thread groups, if any. * They don't get access to memory reserves or a higher scheduler * priority, though, to avoid depletion of all memory or task * starvation. This prevents mm->mmap_sem livelock when an oom killed @@ -457,11 +446,7 @@ static int oom_kill_task(struct task_struct *p, struct mem_cgroup *mem) * signal. */ for_each_process(q) - if (q->mm == mm && !same_thread_group(q, p) && - !(q->flags & PF_KTHREAD)) { - if (q->signal->oom_score_adj == OOM_SCORE_ADJ_MIN) - continue; - + if (q->mm == mm && !same_thread_group(q, p)) { task_lock(q); /* Protect ->comm from prctl() */ pr_err("Kill process %d (%s) sharing same memory\n", task_pid_nr(q), q->comm); @@ -505,7 +490,7 @@ static int oom_kill_process(struct task_struct *p, gfp_t gfp_mask, int order, /* * If any of p's children has a different mm and is eligible for kill, - * the one with the highest oom_badness() score is sacrificed for its + * the one with the highest badness() score is sacrificed for its * parent. This attempts to lose the minimal amount of work done while * still freeing memory. */ @@ -738,7 +723,7 @@ void out_of_memory(struct zonelist *zonelist, gfp_t gfp_mask, read_lock(&tasklist_lock); if (sysctl_oom_kill_allocating_task && !oom_unkillable_task(current, NULL, nodemask) && - current->mm) { + current->mm && !atomic_read(¤t->mm->oom_disable_count)) { /* * oom_kill_process() needs tasklist_lock held. If it returns * non-zero, current could not be killed so we must fallback to diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 62bfbd9..955fe35 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -12,7 +12,7 @@ */ #include <linux/kernel.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/spinlock.h> #include <linux/fs.h> #include <linux/mm.h> @@ -37,22 +37,24 @@ #include <trace/events/writeback.h> /* - * Sleep at most 200ms at a time in balance_dirty_pages(). + * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited + * will look to see if it needs to force writeback or throttling. */ -#define MAX_PAUSE max(HZ/5, 1) +static long ratelimit_pages = 32; /* - * Estimate write bandwidth at 200ms intervals. + * When balance_dirty_pages decides that the caller needs to perform some + * non-background writeback, this is how many pages it will attempt to write. + * It should be somewhat larger than dirtied pages to ensure that reasonably + * large amounts of I/O are submitted. */ -#define BANDWIDTH_INTERVAL max(HZ/5, 1) - -#define RATELIMIT_CALC_SHIFT 10 +static inline long sync_writeback_pages(unsigned long dirtied) +{ + if (dirtied < ratelimit_pages) + dirtied = ratelimit_pages; -/* - * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited - * will look to see if it needs to force writeback or throttling. - */ -static long ratelimit_pages = 32; + return dirtied + dirtied / 2; +} /* The following parameters are exported via /proc/sys/vm */ @@ -109,7 +111,6 @@ EXPORT_SYMBOL(laptop_mode); /* End of sysctl-exported parameters */ -unsigned long global_dirty_limit; /* * Scale the writeback cache size proportional to the relative writeout speeds. @@ -128,6 +129,7 @@ unsigned long global_dirty_limit; * */ static struct prop_descriptor vm_completions; +static struct prop_descriptor vm_dirties; /* * couple the period to the dirty_ratio: @@ -153,8 +155,7 @@ static void update_completion_period(void) { int shift = calc_period_shift(); prop_change_shift(&vm_completions, shift); - - writeback_set_ratelimit(); + prop_change_shift(&vm_dirties, shift); } int dirty_background_ratio_handler(struct ctl_table *table, int write, @@ -218,7 +219,6 @@ int dirty_bytes_handler(struct ctl_table *table, int write, */ static inline void __bdi_writeout_inc(struct backing_dev_info *bdi) { - __inc_bdi_stat(bdi, BDI_WRITTEN); __prop_inc_percpu_max(&vm_completions, &bdi->completions, bdi->max_prop_frac); } @@ -233,20 +233,65 @@ void bdi_writeout_inc(struct backing_dev_info *bdi) } EXPORT_SYMBOL_GPL(bdi_writeout_inc); +void task_dirty_inc(struct task_struct *tsk) +{ + prop_inc_single(&vm_dirties, &tsk->dirties); +} + /* * Obtain an accurate fraction of the BDI's portion. */ static void bdi_writeout_fraction(struct backing_dev_info *bdi, long *numerator, long *denominator) { - prop_fraction_percpu(&vm_completions, &bdi->completions, + if (bdi_cap_writeback_dirty(bdi)) { + prop_fraction_percpu(&vm_completions, &bdi->completions, + numerator, denominator); + } else { + *numerator = 0; + *denominator = 1; + } +} + +static inline void task_dirties_fraction(struct task_struct *tsk, + long *numerator, long *denominator) +{ + prop_fraction_single(&vm_dirties, &tsk->dirties, numerator, denominator); } /* - * bdi_min_ratio keeps the sum of the minimum dirty shares of all - * registered backing devices, which, for obvious reasons, can not - * exceed 100%. + * task_dirty_limit - scale down dirty throttling threshold for one task + * + * task specific dirty limit: + * + * dirty -= (dirty/8) * p_{t} + * + * To protect light/slow dirtying tasks from heavier/fast ones, we start + * throttling individual tasks before reaching the bdi dirty limit. + * Relatively low thresholds will be allocated to heavy dirtiers. So when + * dirty pages grow large, heavy dirtiers will be throttled first, which will + * effectively curb the growth of dirty pages. Light dirtiers with high enough + * dirty threshold may never get throttled. + */ +static unsigned long task_dirty_limit(struct task_struct *tsk, + unsigned long bdi_dirty) +{ + long numerator, denominator; + unsigned long dirty = bdi_dirty; + u64 inv = dirty >> 3; + + task_dirties_fraction(tsk, &numerator, &denominator); + inv *= numerator; + do_div(inv, denominator); + + dirty -= inv; + + return max(dirty, bdi_dirty/2); +} + +/* + * */ static unsigned int bdi_min_ratio; @@ -352,17 +397,6 @@ unsigned long determine_dirtyable_memory(void) return x + 1; /* Ensure that we never return 0 */ } -static unsigned long dirty_freerun_ceiling(unsigned long thresh, - unsigned long bg_thresh) -{ - return (thresh + bg_thresh) / 2; -} - -static unsigned long hard_dirty_limit(unsigned long thresh) -{ - return max(thresh, global_dirty_limit); -} - /* * global_dirty_limits - background-writeback and dirty-throttling thresholds * @@ -401,25 +435,12 @@ void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty) } *pbackground = background; *pdirty = dirty; - trace_global_dirty_state(background, dirty); } -/** +/* * bdi_dirty_limit - @bdi's share of dirty throttling threshold - * @bdi: the backing_dev_info to query - * @dirty: global dirty limit in pages - * - * Returns @bdi's dirty limit in pages. The term "dirty" in the context of - * dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages. * - * Note that balance_dirty_pages() will only seriously take it as a hard limit - * when sleeping max_pause per page is not enough to keep the dirty pages under - * control. For example, when the device is completely stalled due to some error - * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key. - * In the other normal situations, it acts more gently by throttling the tasks - * more (rather than completely block them) when the bdi dirty pages go high. - * - * It allocates high/low dirty limits to fast/slow devices, in order to prevent + * Allocate high/low dirty limits to fast/slow devices, in order to prevent * - starving fast devices * - piling up dirty pages (that will take long time to sync) on slow devices * @@ -448,596 +469,36 @@ unsigned long bdi_dirty_limit(struct backing_dev_info *bdi, unsigned long dirty) } /* - * Dirty position control. - * - * (o) global/bdi setpoints - * - * We want the dirty pages be balanced around the global/bdi setpoints. - * When the number of dirty pages is higher/lower than the setpoint, the - * dirty position control ratio (and hence task dirty ratelimit) will be - * decreased/increased to bring the dirty pages back to the setpoint. - * - * pos_ratio = 1 << RATELIMIT_CALC_SHIFT - * - * if (dirty < setpoint) scale up pos_ratio - * if (dirty > setpoint) scale down pos_ratio - * - * if (bdi_dirty < bdi_setpoint) scale up pos_ratio - * if (bdi_dirty > bdi_setpoint) scale down pos_ratio - * - * task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT - * - * (o) global control line - * - * ^ pos_ratio - * | - * | |<===== global dirty control scope ======>| - * 2.0 .............* - * | .* - * | . * - * | . * - * | . * - * | . * - * | . * - * 1.0 ................................* - * | . . * - * | . . * - * | . . * - * | . . * - * | . . * - * 0 +------------.------------------.----------------------*-------------> - * freerun^ setpoint^ limit^ dirty pages - * - * (o) bdi control line - * - * ^ pos_ratio - * | - * | * - * | * - * | * - * | * - * | * |<=========== span ============>| - * 1.0 .......................* - * | . * - * | . * - * | . * - * | . * - * | . * - * | . * - * | . * - * | . * - * | . * - * | . * - * | . * - * 1/4 ...............................................* * * * * * * * * * * * - * | . . - * | . . - * | . . - * 0 +----------------------.-------------------------------.-------------> - * bdi_setpoint^ x_intercept^ - * - * The bdi control line won't drop below pos_ratio=1/4, so that bdi_dirty can - * be smoothly throttled down to normal if it starts high in situations like - * - start writing to a slow SD card and a fast disk at the same time. The SD - * card's bdi_dirty may rush to many times higher than bdi_setpoint. - * - the bdi dirty thresh drops quickly due to change of JBOD workload - */ -static unsigned long bdi_position_ratio(struct backing_dev_info *bdi, - unsigned long thresh, - unsigned long bg_thresh, - unsigned long dirty, - unsigned long bdi_thresh, - unsigned long bdi_dirty) -{ - unsigned long write_bw = bdi->avg_write_bandwidth; - unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh); - unsigned long limit = hard_dirty_limit(thresh); - unsigned long x_intercept; - unsigned long setpoint; /* dirty pages' target balance point */ - unsigned long bdi_setpoint; - unsigned long span; - long long pos_ratio; /* for scaling up/down the rate limit */ - long x; - - if (unlikely(dirty >= limit)) - return 0; - - /* - * global setpoint - * - * setpoint - dirty 3 - * f(dirty) := 1.0 + (----------------) - * limit - setpoint - * - * it's a 3rd order polynomial that subjects to - * - * (1) f(freerun) = 2.0 => rampup dirty_ratelimit reasonably fast - * (2) f(setpoint) = 1.0 => the balance point - * (3) f(limit) = 0 => the hard limit - * (4) df/dx <= 0 => negative feedback control - * (5) the closer to setpoint, the smaller |df/dx| (and the reverse) - * => fast response on large errors; small oscillation near setpoint - */ - setpoint = (freerun + limit) / 2; - x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT, - (limit - setpoint) | 1); - pos_ratio = x; - pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; - pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; - pos_ratio += 1 << RATELIMIT_CALC_SHIFT; - - /* - * We have computed basic pos_ratio above based on global situation. If - * the bdi is over/under its share of dirty pages, we want to scale - * pos_ratio further down/up. That is done by the following mechanism. - */ - - /* - * bdi setpoint - * - * f(bdi_dirty) := 1.0 + k * (bdi_dirty - bdi_setpoint) - * - * x_intercept - bdi_dirty - * := -------------------------- - * x_intercept - bdi_setpoint - * - * The main bdi control line is a linear function that subjects to - * - * (1) f(bdi_setpoint) = 1.0 - * (2) k = - 1 / (8 * write_bw) (in single bdi case) - * or equally: x_intercept = bdi_setpoint + 8 * write_bw - * - * For single bdi case, the dirty pages are observed to fluctuate - * regularly within range - * [bdi_setpoint - write_bw/2, bdi_setpoint + write_bw/2] - * for various filesystems, where (2) can yield in a reasonable 12.5% - * fluctuation range for pos_ratio. - * - * For JBOD case, bdi_thresh (not bdi_dirty!) could fluctuate up to its - * own size, so move the slope over accordingly and choose a slope that - * yields 100% pos_ratio fluctuation on suddenly doubled bdi_thresh. - */ - if (unlikely(bdi_thresh > thresh)) - bdi_thresh = thresh; - /* - * It's very possible that bdi_thresh is close to 0 not because the - * device is slow, but that it has remained inactive for long time. - * Honour such devices a reasonable good (hopefully IO efficient) - * threshold, so that the occasional writes won't be blocked and active - * writes can rampup the threshold quickly. - */ - bdi_thresh = max(bdi_thresh, (limit - dirty) / 8); - /* - * scale global setpoint to bdi's: - * bdi_setpoint = setpoint * bdi_thresh / thresh - */ - x = div_u64((u64)bdi_thresh << 16, thresh | 1); - bdi_setpoint = setpoint * (u64)x >> 16; - /* - * Use span=(8*write_bw) in single bdi case as indicated by - * (thresh - bdi_thresh ~= 0) and transit to bdi_thresh in JBOD case. - * - * bdi_thresh thresh - bdi_thresh - * span = ---------- * (8 * write_bw) + ------------------- * bdi_thresh - * thresh thresh - */ - span = (thresh - bdi_thresh + 8 * write_bw) * (u64)x >> 16; - x_intercept = bdi_setpoint + span; - - if (bdi_dirty < x_intercept - span / 4) { - pos_ratio = div64_u64(pos_ratio * (x_intercept - bdi_dirty), - (x_intercept - bdi_setpoint) | 1); - } else - pos_ratio /= 4; - - /* - * bdi reserve area, safeguard against dirty pool underrun and disk idle - * It may push the desired control point of global dirty pages higher - * than setpoint. - */ - x_intercept = bdi_thresh / 2; - if (bdi_dirty < x_intercept) { - if (bdi_dirty > x_intercept / 8) - pos_ratio = div_u64(pos_ratio * x_intercept, bdi_dirty); - else - pos_ratio *= 8; - } - - return pos_ratio; -} - -static void bdi_update_write_bandwidth(struct backing_dev_info *bdi, - unsigned long elapsed, - unsigned long written) -{ - const unsigned long period = roundup_pow_of_two(3 * HZ); - unsigned long avg = bdi->avg_write_bandwidth; - unsigned long old = bdi->write_bandwidth; - u64 bw; - - /* - * bw = written * HZ / elapsed - * - * bw * elapsed + write_bandwidth * (period - elapsed) - * write_bandwidth = --------------------------------------------------- - * period - * - * @written may have decreased due to account_page_redirty(). - * Avoid underflowing @bw calculation. - */ - bw = written - min(written, bdi->written_stamp); - bw *= HZ; - if (unlikely(elapsed > period)) { - do_div(bw, elapsed); - avg = bw; - goto out; - } - bw += (u64)bdi->write_bandwidth * (period - elapsed); - bw >>= ilog2(period); - - /* - * one more level of smoothing, for filtering out sudden spikes - */ - if (avg > old && old >= (unsigned long)bw) - avg -= (avg - old) >> 3; - - if (avg < old && old <= (unsigned long)bw) - avg += (old - avg) >> 3; - -out: - bdi->write_bandwidth = bw; - bdi->avg_write_bandwidth = avg; -} - -/* - * The global dirtyable memory and dirty threshold could be suddenly knocked - * down by a large amount (eg. on the startup of KVM in a swapless system). - * This may throw the system into deep dirty exceeded state and throttle - * heavy/light dirtiers alike. To retain good responsiveness, maintain - * global_dirty_limit for tracking slowly down to the knocked down dirty - * threshold. - */ -static void update_dirty_limit(unsigned long thresh, unsigned long dirty) -{ - unsigned long limit = global_dirty_limit; - - /* - * Follow up in one step. - */ - if (limit < thresh) { - limit = thresh; - goto update; - } - - /* - * Follow down slowly. Use the higher one as the target, because thresh - * may drop below dirty. This is exactly the reason to introduce - * global_dirty_limit which is guaranteed to lie above the dirty pages. - */ - thresh = max(thresh, dirty); - if (limit > thresh) { - limit -= (limit - thresh) >> 5; - goto update; - } - return; -update: - global_dirty_limit = limit; -} - -static void global_update_bandwidth(unsigned long thresh, - unsigned long dirty, - unsigned long now) -{ - static DEFINE_SPINLOCK(dirty_lock); - static unsigned long update_time = INITIAL_JIFFIES; - - /* - * check locklessly first to optimize away locking for the most time - */ - if (time_before(now, update_time + BANDWIDTH_INTERVAL)) - return; - - spin_lock(&dirty_lock); - if (time_after_eq(now, update_time + BANDWIDTH_INTERVAL)) { - update_dirty_limit(thresh, dirty); - update_time = now; - } - spin_unlock(&dirty_lock); -} - -/* - * Maintain bdi->dirty_ratelimit, the base dirty throttle rate. - * - * Normal bdi tasks will be curbed at or below it in long term. - * Obviously it should be around (write_bw / N) when there are N dd tasks. - */ -static void bdi_update_dirty_ratelimit(struct backing_dev_info *bdi, - unsigned long thresh, - unsigned long bg_thresh, - unsigned long dirty, - unsigned long bdi_thresh, - unsigned long bdi_dirty, - unsigned long dirtied, - unsigned long elapsed) -{ - unsigned long freerun = dirty_freerun_ceiling(thresh, bg_thresh); - unsigned long limit = hard_dirty_limit(thresh); - unsigned long setpoint = (freerun + limit) / 2; - unsigned long write_bw = bdi->avg_write_bandwidth; - unsigned long dirty_ratelimit = bdi->dirty_ratelimit; - unsigned long dirty_rate; - unsigned long task_ratelimit; - unsigned long balanced_dirty_ratelimit; - unsigned long pos_ratio; - unsigned long step; - unsigned long x; - - /* - * The dirty rate will match the writeout rate in long term, except - * when dirty pages are truncated by userspace or re-dirtied by FS. - */ - dirty_rate = (dirtied - bdi->dirtied_stamp) * HZ / elapsed; - - pos_ratio = bdi_position_ratio(bdi, thresh, bg_thresh, dirty, - bdi_thresh, bdi_dirty); - /* - * task_ratelimit reflects each dd's dirty rate for the past 200ms. - */ - task_ratelimit = (u64)dirty_ratelimit * - pos_ratio >> RATELIMIT_CALC_SHIFT; - task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */ - - /* - * A linear estimation of the "balanced" throttle rate. The theory is, - * if there are N dd tasks, each throttled at task_ratelimit, the bdi's - * dirty_rate will be measured to be (N * task_ratelimit). So the below - * formula will yield the balanced rate limit (write_bw / N). - * - * Note that the expanded form is not a pure rate feedback: - * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) (1) - * but also takes pos_ratio into account: - * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio (2) - * - * (1) is not realistic because pos_ratio also takes part in balancing - * the dirty rate. Consider the state - * pos_ratio = 0.5 (3) - * rate = 2 * (write_bw / N) (4) - * If (1) is used, it will stuck in that state! Because each dd will - * be throttled at - * task_ratelimit = pos_ratio * rate = (write_bw / N) (5) - * yielding - * dirty_rate = N * task_ratelimit = write_bw (6) - * put (6) into (1) we get - * rate_(i+1) = rate_(i) (7) - * - * So we end up using (2) to always keep - * rate_(i+1) ~= (write_bw / N) (8) - * regardless of the value of pos_ratio. As long as (8) is satisfied, - * pos_ratio is able to drive itself to 1.0, which is not only where - * the dirty count meet the setpoint, but also where the slope of - * pos_ratio is most flat and hence task_ratelimit is least fluctuated. - */ - balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw, - dirty_rate | 1); - - /* - * We could safely do this and return immediately: - * - * bdi->dirty_ratelimit = balanced_dirty_ratelimit; - * - * However to get a more stable dirty_ratelimit, the below elaborated - * code makes use of task_ratelimit to filter out sigular points and - * limit the step size. - * - * The below code essentially only uses the relative value of - * - * task_ratelimit - dirty_ratelimit - * = (pos_ratio - 1) * dirty_ratelimit - * - * which reflects the direction and size of dirty position error. - */ - - /* - * dirty_ratelimit will follow balanced_dirty_ratelimit iff - * task_ratelimit is on the same side of dirty_ratelimit, too. - * For example, when - * - dirty_ratelimit > balanced_dirty_ratelimit - * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint) - * lowering dirty_ratelimit will help meet both the position and rate - * control targets. Otherwise, don't update dirty_ratelimit if it will - * only help meet the rate target. After all, what the users ultimately - * feel and care are stable dirty rate and small position error. - * - * |task_ratelimit - dirty_ratelimit| is used to limit the step size - * and filter out the sigular points of balanced_dirty_ratelimit. Which - * keeps jumping around randomly and can even leap far away at times - * due to the small 200ms estimation period of dirty_rate (we want to - * keep that period small to reduce time lags). - */ - step = 0; - if (dirty < setpoint) { - x = min(bdi->balanced_dirty_ratelimit, - min(balanced_dirty_ratelimit, task_ratelimit)); - if (dirty_ratelimit < x) - step = x - dirty_ratelimit; - } else { - x = max(bdi->balanced_dirty_ratelimit, - max(balanced_dirty_ratelimit, task_ratelimit)); - if (dirty_ratelimit > x) - step = dirty_ratelimit - x; - } - - /* - * Don't pursue 100% rate matching. It's impossible since the balanced - * rate itself is constantly fluctuating. So decrease the track speed - * when it gets close to the target. Helps eliminate pointless tremors. - */ - step >>= dirty_ratelimit / (2 * step + 1); - /* - * Limit the tracking speed to avoid overshooting. - */ - step = (step + 7) / 8; - - if (dirty_ratelimit < balanced_dirty_ratelimit) - dirty_ratelimit += step; - else - dirty_ratelimit -= step; - - bdi->dirty_ratelimit = max(dirty_ratelimit, 1UL); - bdi->balanced_dirty_ratelimit = balanced_dirty_ratelimit; - - trace_bdi_dirty_ratelimit(bdi, dirty_rate, task_ratelimit); -} - -void __bdi_update_bandwidth(struct backing_dev_info *bdi, - unsigned long thresh, - unsigned long bg_thresh, - unsigned long dirty, - unsigned long bdi_thresh, - unsigned long bdi_dirty, - unsigned long start_time) -{ - unsigned long now = jiffies; - unsigned long elapsed = now - bdi->bw_time_stamp; - unsigned long dirtied; - unsigned long written; - - /* - * rate-limit, only update once every 200ms. - */ - if (elapsed < BANDWIDTH_INTERVAL) - return; - - dirtied = percpu_counter_read(&bdi->bdi_stat[BDI_DIRTIED]); - written = percpu_counter_read(&bdi->bdi_stat[BDI_WRITTEN]); - - /* - * Skip quiet periods when disk bandwidth is under-utilized. - * (at least 1s idle time between two flusher runs) - */ - if (elapsed > HZ && time_before(bdi->bw_time_stamp, start_time)) - goto snapshot; - - if (thresh) { - global_update_bandwidth(thresh, dirty, now); - bdi_update_dirty_ratelimit(bdi, thresh, bg_thresh, dirty, - bdi_thresh, bdi_dirty, - dirtied, elapsed); - } - bdi_update_write_bandwidth(bdi, elapsed, written); - -snapshot: - bdi->dirtied_stamp = dirtied; - bdi->written_stamp = written; - bdi->bw_time_stamp = now; -} - -static void bdi_update_bandwidth(struct backing_dev_info *bdi, - unsigned long thresh, - unsigned long bg_thresh, - unsigned long dirty, - unsigned long bdi_thresh, - unsigned long bdi_dirty, - unsigned long start_time) -{ - if (time_is_after_eq_jiffies(bdi->bw_time_stamp + BANDWIDTH_INTERVAL)) - return; - spin_lock(&bdi->wb.list_lock); - __bdi_update_bandwidth(bdi, thresh, bg_thresh, dirty, - bdi_thresh, bdi_dirty, start_time); - spin_unlock(&bdi->wb.list_lock); -} - -/* - * After a task dirtied this many pages, balance_dirty_pages_ratelimited_nr() - * will look to see if it needs to start dirty throttling. - * - * If dirty_poll_interval is too low, big NUMA machines will call the expensive - * global_page_state() too often. So scale it near-sqrt to the safety margin - * (the number of pages we may dirty without exceeding the dirty limits). - */ -static unsigned long dirty_poll_interval(unsigned long dirty, - unsigned long thresh) -{ - if (thresh > dirty) - return 1UL << (ilog2(thresh - dirty) >> 1); - - return 1; -} - -static unsigned long bdi_max_pause(struct backing_dev_info *bdi, - unsigned long bdi_dirty) -{ - unsigned long bw = bdi->avg_write_bandwidth; - unsigned long hi = ilog2(bw); - unsigned long lo = ilog2(bdi->dirty_ratelimit); - unsigned long t; - - /* target for 20ms max pause on 1-dd case */ - t = HZ / 50; - - /* - * Scale up pause time for concurrent dirtiers in order to reduce CPU - * overheads. - * - * (N * 20ms) on 2^N concurrent tasks. - */ - if (hi > lo) - t += (hi - lo) * (20 * HZ) / 1024; - - /* - * Limit pause time for small memory systems. If sleeping for too long - * time, a small pool of dirty/writeback pages may go empty and disk go - * idle. - * - * 8 serves as the safety ratio. - */ - t = min(t, bdi_dirty * HZ / (8 * bw + 1)); - - /* - * The pause time will be settled within range (max_pause/4, max_pause). - * Apply a minimal value of 4 to get a non-zero max_pause/4. - */ - return clamp_val(t, 4, MAX_PAUSE); -} - -/* * balance_dirty_pages() must be called by processes which are generating dirty * data. It looks at the number of dirty pages in the machine and will force - * the caller to wait once crossing the (background_thresh + dirty_thresh) / 2. + * the caller to perform writeback if the system is over `vm_dirty_ratio'. * If we're over `background_thresh' then the writeback threads are woken to * perform some writeout. */ static void balance_dirty_pages(struct address_space *mapping, - unsigned long pages_dirtied) + unsigned long write_chunk) { - unsigned long nr_reclaimable; /* = file_dirty + unstable_nfs */ - unsigned long bdi_reclaimable; - unsigned long nr_dirty; /* = file_dirty + writeback + unstable_nfs */ - unsigned long bdi_dirty; - unsigned long freerun; + long nr_reclaimable, bdi_nr_reclaimable; + long nr_writeback, bdi_nr_writeback; unsigned long background_thresh; unsigned long dirty_thresh; unsigned long bdi_thresh; - long pause = 0; - long uninitialized_var(max_pause); + unsigned long pages_written = 0; + unsigned long pause = 1; bool dirty_exceeded = false; - unsigned long task_ratelimit; - unsigned long uninitialized_var(dirty_ratelimit); - unsigned long pos_ratio; struct backing_dev_info *bdi = mapping->backing_dev_info; - unsigned long start_time = jiffies; for (;;) { - /* - * Unstable writes are a feature of certain networked - * filesystems (i.e. NFS) in which data may have been - * written to the server's write cache, but has not yet - * been flushed to permanent storage. - */ + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + .older_than_this = NULL, + .nr_to_write = write_chunk, + .range_cyclic = 1, + }; + nr_reclaimable = global_page_state(NR_FILE_DIRTY) + global_page_state(NR_UNSTABLE_NFS); - nr_dirty = nr_reclaimable + global_page_state(NR_WRITEBACK); + nr_writeback = global_page_state(NR_WRITEBACK); global_dirty_limits(&background_thresh, &dirty_thresh); @@ -1046,28 +507,12 @@ static void balance_dirty_pages(struct address_space *mapping, * catch-up. This avoids (excessively) small writeouts * when the bdi limits are ramping up. */ - freerun = dirty_freerun_ceiling(dirty_thresh, - background_thresh); - if (nr_dirty <= freerun) + if (nr_reclaimable + nr_writeback <= + (background_thresh + dirty_thresh) / 2) break; - if (unlikely(!writeback_in_progress(bdi))) - bdi_start_background_writeback(bdi); - - /* - * bdi_thresh is not treated as some limiting factor as - * dirty_thresh, due to reasons - * - in JBOD setup, bdi_thresh can fluctuate a lot - * - in a system with HDD and USB key, the USB key may somehow - * go into state (bdi_dirty >> bdi_thresh) either because - * bdi_dirty starts high, or because bdi_thresh drops low. - * In this case we don't want to hard throttle the USB key - * dirtiers for 100 seconds until bdi_dirty drops under - * bdi_thresh. Instead the auxiliary bdi control line in - * bdi_position_ratio() will let the dirtier task progress - * at some rate <= (write_bw / 2) for bringing down bdi_dirty. - */ bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh); + bdi_thresh = task_dirty_limit(current, bdi_thresh); /* * In order to avoid the stacked BDI deadlock we need @@ -1079,114 +524,63 @@ static void balance_dirty_pages(struct address_space *mapping, * actually dirty; with m+n sitting in the percpu * deltas. */ - if (bdi_thresh < 2 * bdi_stat_error(bdi)) { - bdi_reclaimable = bdi_stat_sum(bdi, BDI_RECLAIMABLE); - bdi_dirty = bdi_reclaimable + - bdi_stat_sum(bdi, BDI_WRITEBACK); + if (bdi_thresh < 2*bdi_stat_error(bdi)) { + bdi_nr_reclaimable = bdi_stat_sum(bdi, BDI_RECLAIMABLE); + bdi_nr_writeback = bdi_stat_sum(bdi, BDI_WRITEBACK); } else { - bdi_reclaimable = bdi_stat(bdi, BDI_RECLAIMABLE); - bdi_dirty = bdi_reclaimable + - bdi_stat(bdi, BDI_WRITEBACK); + bdi_nr_reclaimable = bdi_stat(bdi, BDI_RECLAIMABLE); + bdi_nr_writeback = bdi_stat(bdi, BDI_WRITEBACK); } - dirty_exceeded = (bdi_dirty > bdi_thresh) || - (nr_dirty > dirty_thresh); - if (dirty_exceeded && !bdi->dirty_exceeded) - bdi->dirty_exceeded = 1; + /* + * The bdi thresh is somehow "soft" limit derived from the + * global "hard" limit. The former helps to prevent heavy IO + * bdi or process from holding back light ones; The latter is + * the last resort safeguard. + */ + dirty_exceeded = + (bdi_nr_reclaimable + bdi_nr_writeback > bdi_thresh) + || (nr_reclaimable + nr_writeback > dirty_thresh); - bdi_update_bandwidth(bdi, dirty_thresh, background_thresh, - nr_dirty, bdi_thresh, bdi_dirty, - start_time); - - max_pause = bdi_max_pause(bdi, bdi_dirty); - - dirty_ratelimit = bdi->dirty_ratelimit; - pos_ratio = bdi_position_ratio(bdi, dirty_thresh, - background_thresh, nr_dirty, - bdi_thresh, bdi_dirty); - task_ratelimit = ((u64)dirty_ratelimit * pos_ratio) >> - RATELIMIT_CALC_SHIFT; - if (unlikely(task_ratelimit == 0)) { - pause = max_pause; - goto pause; - } - pause = HZ * pages_dirtied / task_ratelimit; - if (unlikely(pause <= 0)) { - trace_balance_dirty_pages(bdi, - dirty_thresh, - background_thresh, - nr_dirty, - bdi_thresh, - bdi_dirty, - dirty_ratelimit, - task_ratelimit, - pages_dirtied, - pause, - start_time); - pause = 1; /* avoid resetting nr_dirtied_pause below */ + if (!dirty_exceeded) break; - } - pause = min(pause, max_pause); - -pause: - trace_balance_dirty_pages(bdi, - dirty_thresh, - background_thresh, - nr_dirty, - bdi_thresh, - bdi_dirty, - dirty_ratelimit, - task_ratelimit, - pages_dirtied, - pause, - start_time); - __set_current_state(TASK_KILLABLE); - io_schedule_timeout(pause); - /* - * This is typically equal to (nr_dirty < dirty_thresh) and can - * also keep "1000+ dd on a slow USB stick" under control. + if (!bdi->dirty_exceeded) + bdi->dirty_exceeded = 1; + + /* Note: nr_reclaimable denotes nr_dirty + nr_unstable. + * Unstable writes are a feature of certain networked + * filesystems (i.e. NFS) in which data may have been + * written to the server's write cache, but has not yet + * been flushed to permanent storage. + * Only move pages to writeback if this bdi is over its + * threshold otherwise wait until the disk writes catch + * up. */ - if (task_ratelimit) - break; + trace_wbc_balance_dirty_start(&wbc, bdi); + if (bdi_nr_reclaimable > bdi_thresh) { + writeback_inodes_wb(&bdi->wb, &wbc); + pages_written += write_chunk - wbc.nr_to_write; + trace_wbc_balance_dirty_written(&wbc, bdi); + if (pages_written >= write_chunk) + break; /* We've done our duty */ + } + trace_wbc_balance_dirty_wait(&wbc, bdi); + __set_current_state(TASK_UNINTERRUPTIBLE); + io_schedule_timeout(pause); /* - * In the case of an unresponding NFS server and the NFS dirty - * pages exceeds dirty_thresh, give the other good bdi's a pipe - * to go through, so that tasks on them still remain responsive. - * - * In theory 1 page is enough to keep the comsumer-producer - * pipe going: the flusher cleans 1 page => the task dirties 1 - * more page. However bdi_dirty has accounting errors. So use - * the larger and more IO friendly bdi_stat_error. + * Increase the delay for each loop, up to our previous + * default of taking a 100ms nap. */ - if (bdi_dirty <= bdi_stat_error(bdi)) - break; - - if (fatal_signal_pending(current)) - break; + pause <<= 1; + if (pause > HZ / 10) + pause = HZ / 10; } if (!dirty_exceeded && bdi->dirty_exceeded) bdi->dirty_exceeded = 0; - current->nr_dirtied = 0; - if (pause == 0) { /* in freerun area */ - current->nr_dirtied_pause = - dirty_poll_interval(nr_dirty, dirty_thresh); - } else if (pause <= max_pause / 4 && - pages_dirtied >= current->nr_dirtied_pause) { - current->nr_dirtied_pause = clamp_val( - dirty_ratelimit * (max_pause / 2) / HZ, - pages_dirtied + pages_dirtied / 8, - pages_dirtied * 4); - } else if (pause >= max_pause) { - current->nr_dirtied_pause = 1 | clamp_val( - dirty_ratelimit * (max_pause / 2) / HZ, - pages_dirtied / 4, - pages_dirtied - pages_dirtied / 8); - } - if (writeback_in_progress(bdi)) return; @@ -1198,14 +592,22 @@ pause: * In normal mode, we start background writeout at the lower * background_thresh, to keep the amount of dirty memory low. */ - if (laptop_mode) - return; - - if (nr_reclaimable > background_thresh) + if ((laptop_mode && pages_written) || + (!laptop_mode && (nr_reclaimable > background_thresh))) bdi_start_background_writeback(bdi); } -static DEFINE_PER_CPU(int, bdp_ratelimits); +void set_page_dirty_balance(struct page *page, int page_mkwrite) +{ + if (set_page_dirty(page) || page_mkwrite) { + struct address_space *mapping = page_mapping(page); + + if (mapping) + balance_dirty_pages_ratelimited(mapping); + } +} + +static DEFINE_PER_CPU(unsigned long, bdp_ratelimits) = 0; /** * balance_dirty_pages_ratelimited_nr - balance dirty memory state @@ -1224,40 +626,28 @@ static DEFINE_PER_CPU(int, bdp_ratelimits); void balance_dirty_pages_ratelimited_nr(struct address_space *mapping, unsigned long nr_pages_dirtied) { - struct backing_dev_info *bdi = mapping->backing_dev_info; - int ratelimit; - int *p; - - if (!bdi_cap_account_dirty(bdi)) - return; - - ratelimit = current->nr_dirtied_pause; - if (bdi->dirty_exceeded) - ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10)); + unsigned long ratelimit; + unsigned long *p; - current->nr_dirtied += nr_pages_dirtied; + ratelimit = ratelimit_pages; + if (mapping->backing_dev_info->dirty_exceeded) + ratelimit = 8; - preempt_disable(); /* - * This prevents one CPU to accumulate too many dirtied pages without - * calling into balance_dirty_pages(), which can happen when there are - * 1000+ tasks, all of them start dirtying pages at exactly the same - * time, hence all honoured too large initial task->nr_dirtied_pause. + * Check the rate limiting. Also, we do not want to throttle real-time + * tasks in balance_dirty_pages(). Period. */ + preempt_disable(); p = &__get_cpu_var(bdp_ratelimits); - if (unlikely(current->nr_dirtied >= ratelimit)) + *p += nr_pages_dirtied; + if (unlikely(*p >= ratelimit)) { + ratelimit = sync_writeback_pages(*p); *p = 0; - else { - *p += nr_pages_dirtied; - if (unlikely(*p >= ratelimit_pages)) { - *p = 0; - ratelimit = 0; - } + preempt_enable(); + balance_dirty_pages(mapping, ratelimit); + return; } preempt_enable(); - - if (unlikely(current->nr_dirtied >= ratelimit)) - balance_dirty_pages(mapping, current->nr_dirtied); } EXPORT_SYMBOL(balance_dirty_pages_ratelimited_nr); @@ -1313,8 +703,7 @@ void laptop_mode_timer_fn(unsigned long data) * threshold */ if (bdi_has_dirty_io(&q->backing_dev_info)) - bdi_start_writeback(&q->backing_dev_info, nr_pages, - WB_REASON_LAPTOP_TIMER); + bdi_start_writeback(&q->backing_dev_info, nr_pages); } /* @@ -1353,17 +742,22 @@ void laptop_sync_completion(void) * * Here we set ratelimit_pages to a level which ensures that when all CPUs are * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory - * thresholds. + * thresholds before writeback cuts in. + * + * But the limit should not be set too high. Because it also controls the + * amount of memory which the balance_dirty_pages() caller has to write back. + * If this is too large then the caller will block on the IO queue all the + * time. So limit it to four megabytes - the balance_dirty_pages() caller + * will write six megabyte chunks, max. */ void writeback_set_ratelimit(void) { - unsigned long background_thresh; - unsigned long dirty_thresh; - global_dirty_limits(&background_thresh, &dirty_thresh); - ratelimit_pages = dirty_thresh / (num_online_cpus() * 32); + ratelimit_pages = vm_total_pages / (num_online_cpus() * 32); if (ratelimit_pages < 16) ratelimit_pages = 16; + if (ratelimit_pages * PAGE_CACHE_SIZE > 4096 * 1024) + ratelimit_pages = (4096 * 1024) / PAGE_CACHE_SIZE; } static int __cpuinit @@ -1405,6 +799,7 @@ void __init page_writeback_init(void) shift = calc_period_shift(); prop_descriptor_init(&vm_completions, shift); + prop_descriptor_init(&vm_dirties, shift); } /** @@ -1732,7 +1127,7 @@ void account_page_dirtied(struct page *page, struct address_space *mapping) __inc_zone_page_state(page, NR_FILE_DIRTY); __inc_zone_page_state(page, NR_DIRTIED); __inc_bdi_stat(mapping->backing_dev_info, BDI_RECLAIMABLE); - __inc_bdi_stat(mapping->backing_dev_info, BDI_DIRTIED); + task_dirty_inc(current); task_io_account_write(PAGE_CACHE_SIZE); } } @@ -1746,6 +1141,7 @@ EXPORT_SYMBOL(account_page_dirtied); void account_page_writeback(struct page *page) { inc_zone_page_state(page, NR_WRITEBACK); + inc_zone_page_state(page, NR_WRITTEN); } EXPORT_SYMBOL(account_page_writeback); @@ -1757,26 +1153,32 @@ EXPORT_SYMBOL(account_page_writeback); * page dirty in that case, but not all the buffers. This is a "bottom-up" * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. * - * The caller must ensure this doesn't race with truncation. Most will simply - * hold the page lock, but e.g. zap_pte_range() calls with the page mapped and - * the pte lock held, which also locks out truncation. + * Most callers have locked the page, which pins the address_space in memory. + * But zap_pte_range() does not lock the page, however in that case the + * mapping is pinned by the vma's ->vm_file reference. + * + * We take care to handle the case where the page was truncated from the + * mapping by re-checking page_mapping() inside tree_lock. */ int __set_page_dirty_nobuffers(struct page *page) { if (!TestSetPageDirty(page)) { struct address_space *mapping = page_mapping(page); - unsigned long flags; + struct address_space *mapping2; if (!mapping) return 1; - spin_lock_irqsave(&mapping->tree_lock, flags); - BUG_ON(page_mapping(page) != mapping); - WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); - account_page_dirtied(page, mapping); - radix_tree_tag_set(&mapping->page_tree, page_index(page), - PAGECACHE_TAG_DIRTY); - spin_unlock_irqrestore(&mapping->tree_lock, flags); + spin_lock_irq(&mapping->tree_lock); + mapping2 = page_mapping(page); + if (mapping2) { /* Race with truncate? */ + BUG_ON(mapping2 != mapping); + WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); + account_page_dirtied(page, mapping); + radix_tree_tag_set(&mapping->page_tree, + page_index(page), PAGECACHE_TAG_DIRTY); + } + spin_unlock_irq(&mapping->tree_lock); if (mapping->host) { /* !PageAnon && !swapper_space */ __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); @@ -1788,24 +1190,6 @@ int __set_page_dirty_nobuffers(struct page *page) EXPORT_SYMBOL(__set_page_dirty_nobuffers); /* - * Call this whenever redirtying a page, to de-account the dirty counters - * (NR_DIRTIED, BDI_DIRTIED, tsk->nr_dirtied), so that they match the written - * counters (NR_WRITTEN, BDI_WRITTEN) in long term. The mismatches will lead to - * systematic errors in balanced_dirty_ratelimit and the dirty pages position - * control. - */ -void account_page_redirty(struct page *page) -{ - struct address_space *mapping = page->mapping; - if (mapping && mapping_cap_account_dirty(mapping)) { - current->nr_dirtied--; - dec_zone_page_state(page, NR_DIRTIED); - dec_bdi_stat(mapping->backing_dev_info, BDI_DIRTIED); - } -} -EXPORT_SYMBOL(account_page_redirty); - -/* * When a writepage implementation decides that it doesn't want to write this * page for some reason, it should redirty the locked page via * redirty_page_for_writepage() and it should then unlock the page and return 0 @@ -1813,7 +1197,6 @@ EXPORT_SYMBOL(account_page_redirty); int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page) { wbc->pages_skipped++; - account_page_redirty(page); return __set_page_dirty_nobuffers(page); } EXPORT_SYMBOL(redirty_page_for_writepage); @@ -1932,10 +1315,12 @@ int clear_page_dirty_for_io(struct page *page) /* * We carefully synchronise fault handlers against * installing a dirty pte and marking the page dirty - * at this point. We do this by having them hold the - * page lock while dirtying the page, and pages are - * always locked coming in here, so we get the desired - * exclusion. + * at this point. We do this by having them hold the + * page lock at some point after installing their + * pte, but before marking the page dirty. + * Pages are always locked coming in here, so we get + * the desired exclusion. See mm/memory.c:do_wp_page() + * for more comments. */ if (TestClearPageDirty(page)) { dec_zone_page_state(page, NR_FILE_DIRTY); @@ -1973,10 +1358,8 @@ int test_clear_page_writeback(struct page *page) } else { ret = TestClearPageWriteback(page); } - if (ret) { + if (ret) dec_zone_page_state(page, NR_WRITEBACK); - inc_zone_page_state(page, NR_WRITTEN); - } return ret; } @@ -2022,6 +1405,10 @@ EXPORT_SYMBOL(test_set_page_writeback); */ int mapping_tagged(struct address_space *mapping, int tag) { - return radix_tree_tagged(&mapping->page_tree, tag); + int ret; + rcu_read_lock(); + ret = radix_tree_tagged(&mapping->page_tree, tag); + rcu_read_unlock(); + return ret; } EXPORT_SYMBOL(mapping_tagged); diff --git a/mm/page_alloc-cma.c b/mm/page_alloc-cma.c new file mode 100644 index 0000000..fc675bd --- /dev/null +++ b/mm/page_alloc-cma.c @@ -0,0 +1,6442 @@ +/* + * linux/mm/page_alloc.c + * + * Manages the free list, the system allocates free pages here. + * Note that kmalloc() lives in slab.c + * + * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds + * Swap reorganised 29.12.95, Stephen Tweedie + * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 + * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 + * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 + * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 + * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 + * (lots of bits borrowed from Ingo Molnar & Andrew Morton) + */ + +#include <linux/stddef.h> +#include <linux/mm.h> +#include <linux/swap.h> +#include <linux/interrupt.h> +#include <linux/pagemap.h> +#include <linux/jiffies.h> +#include <linux/bootmem.h> +#include <linux/memblock.h> +#include <linux/compiler.h> +#include <linux/kernel.h> +#include <linux/kmemcheck.h> +#include <linux/module.h> +#include <linux/suspend.h> +#include <linux/pagevec.h> +#include <linux/blkdev.h> +#include <linux/slab.h> +#include <linux/ratelimit.h> +#include <linux/oom.h> +#include <linux/notifier.h> +#include <linux/topology.h> +#include <linux/sysctl.h> +#include <linux/cpu.h> +#include <linux/cpuset.h> +#include <linux/memory_hotplug.h> +#include <linux/nodemask.h> +#include <linux/vmalloc.h> +#include <linux/vmstat.h> +#include <linux/mempolicy.h> +#include <linux/stop_machine.h> +#include <linux/sort.h> +#include <linux/pfn.h> +#include <linux/backing-dev.h> +#include <linux/fault-inject.h> +#include <linux/page-isolation.h> +#include <linux/page_cgroup.h> +#include <linux/debugobjects.h> +#include <linux/kmemleak.h> +#include <linux/memory.h> +#include <linux/compaction.h> +#include <trace/events/kmem.h> +#include <linux/ftrace_event.h> +#include <linux/memcontrol.h> +#include <linux/prefetch.h> +#include <linux/migrate.h> +#include <linux/delay.h> +#include <linux/dma-contiguous.h> + +#include <asm/tlbflush.h> +#include <asm/div64.h> +#include "internal.h" + +#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID +DEFINE_PER_CPU(int, numa_node); +EXPORT_PER_CPU_SYMBOL(numa_node); +#endif + +#ifdef CONFIG_HAVE_MEMORYLESS_NODES +/* + * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. + * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. + * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() + * defined in <linux/topology.h>. + */ +DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ +EXPORT_PER_CPU_SYMBOL(_numa_mem_); +#endif + +struct rw_semaphore page_alloc_slow_rwsem; + +/* + * Array of node states. + */ +nodemask_t node_states[NR_NODE_STATES] __read_mostly = { + [N_POSSIBLE] = NODE_MASK_ALL, + [N_ONLINE] = { { [0] = 1UL } }, +#ifndef CONFIG_NUMA + [N_NORMAL_MEMORY] = { { [0] = 1UL } }, +#ifdef CONFIG_HIGHMEM + [N_HIGH_MEMORY] = { { [0] = 1UL } }, +#endif + [N_CPU] = { { [0] = 1UL } }, +#endif /* NUMA */ +}; +EXPORT_SYMBOL(node_states); + +unsigned long totalram_pages __read_mostly; +EXPORT_SYMBOL(totalram_pages); +unsigned long totalreserve_pages __read_mostly; +int percpu_pagelist_fraction; +gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; + +#ifdef CONFIG_COMPACTION_RETRY_DEBUG +static inline void show_buddy_info(void); +#else +static inline void show_buddy_info(void) +{ +} +#endif + +#ifdef CONFIG_PM_SLEEP +/* + * The following functions are used by the suspend/hibernate code to temporarily + * change gfp_allowed_mask in order to avoid using I/O during memory allocations + * while devices are suspended. To avoid races with the suspend/hibernate code, + * they should always be called with pm_mutex held (gfp_allowed_mask also should + * only be modified with pm_mutex held, unless the suspend/hibernate code is + * guaranteed not to run in parallel with that modification). + */ + +static gfp_t saved_gfp_mask; + +void pm_restore_gfp_mask(void) +{ + WARN_ON(!mutex_is_locked(&pm_mutex)); + if (saved_gfp_mask) { + gfp_allowed_mask = saved_gfp_mask; + saved_gfp_mask = 0; + } +} + +void pm_restrict_gfp_mask(void) +{ + WARN_ON(!mutex_is_locked(&pm_mutex)); + WARN_ON(saved_gfp_mask); + saved_gfp_mask = gfp_allowed_mask; + gfp_allowed_mask &= ~GFP_IOFS; +} + +static bool pm_suspending(void) +{ + if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS) + return false; + return true; +} + +#else + +static bool pm_suspending(void) +{ + return false; +} +#endif /* CONFIG_PM_SLEEP */ + +#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE +int pageblock_order __read_mostly; +#endif + +static void __free_pages_ok(struct page *page, unsigned int order); + +/* + * results with 256, 32 in the lowmem_reserve sysctl: + * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) + * 1G machine -> (16M dma, 784M normal, 224M high) + * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA + * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL + * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA + * + * TBD: should special case ZONE_DMA32 machines here - in those we normally + * don't need any ZONE_NORMAL reservation + */ +int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { +#ifdef CONFIG_ZONE_DMA + 256, +#endif +#ifdef CONFIG_ZONE_DMA32 + 256, +#endif +#ifdef CONFIG_HIGHMEM + 32, +#endif + 32, +}; + +static char * const zone_names[MAX_NR_ZONES] = { +#ifdef CONFIG_ZONE_DMA + "DMA", +#endif +#ifdef CONFIG_ZONE_DMA32 + "DMA32", +#endif + "Normal", +#ifdef CONFIG_HIGHMEM + "HighMem", +#endif + "Movable", +}; + +int min_free_kbytes = 1024; +int min_free_order_shift = 1; + +static unsigned long __meminitdata nr_kernel_pages; +static unsigned long __meminitdata nr_all_pages; +static unsigned long __meminitdata dma_reserve; + +#ifdef CONFIG_ARCH_POPULATES_NODE_MAP + /* + * MAX_ACTIVE_REGIONS determines the maximum number of distinct + * ranges of memory(RAM) that may be registered with add_active_range(). + * Ranges passed to add_active_range() will be merged if possible + * so the number of times add_active_range() can be called is + * related to the number of nodes and the number of holes + */ +#ifdef CONFIG_MAX_ACTIVE_REGIONS + /* Allow an architecture to set MAX_ACTIVE_REGIONS to save memory */ + #define MAX_ACTIVE_REGIONS CONFIG_MAX_ACTIVE_REGIONS +#else +#if MAX_NUMNODES >= 32 + /* If there can be many nodes, allow up to 50 holes per node */ + #define MAX_ACTIVE_REGIONS (MAX_NUMNODES*50) +#else + /* By default, allow up to 256 distinct regions */ + #define MAX_ACTIVE_REGIONS 256 +#endif +#endif + + static struct node_active_region __meminitdata + early_node_map[MAX_ACTIVE_REGIONS]; + static int __meminitdata nr_nodemap_entries; + static unsigned long __meminitdata + arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; + static unsigned long __meminitdata + arch_zone_highest_possible_pfn[MAX_NR_ZONES]; + static unsigned long __initdata required_kernelcore; + static unsigned long __initdata required_movablecore; + static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES]; + + /* + * movable_zone is the "real" zone pages in ZONE_MOVABLE + * are taken from. + */ + int movable_zone; + EXPORT_SYMBOL(movable_zone); +#endif /* CONFIG_ARCH_POPULATES_NODE_MAP */ + +#if MAX_NUMNODES > 1 +int nr_node_ids __read_mostly = MAX_NUMNODES; +EXPORT_SYMBOL(nr_node_ids); +int nr_online_nodes __read_mostly = 1; +EXPORT_SYMBOL(nr_online_nodes); +#endif + +int page_group_by_mobility_disabled __read_mostly; + +static void set_pageblock_migratetype(struct page *page, int migratetype) +{ + + if (unlikely(page_group_by_mobility_disabled)) + migratetype = MIGRATE_UNMOVABLE; + + set_pageblock_flags_group(page, (unsigned long)migratetype, + PB_migrate, PB_migrate_end); +} + +bool oom_killer_disabled __read_mostly; + +#ifdef CONFIG_DEBUG_VM +static int page_outside_zone_boundaries(struct zone *zone, struct page *page) +{ + int ret = 0; + unsigned seq; + unsigned long pfn = page_to_pfn(page); + + do { + seq = zone_span_seqbegin(zone); + if (pfn >= zone->zone_start_pfn + zone->spanned_pages) + ret = 1; + else if (pfn < zone->zone_start_pfn) + ret = 1; + } while (zone_span_seqretry(zone, seq)); + + return ret; +} + +static int page_is_consistent(struct zone *zone, struct page *page) +{ + if (!pfn_valid_within(page_to_pfn(page))) + return 0; + if (zone != page_zone(page)) + return 0; + + return 1; +} +/* + * Temporary debugging check for pages not lying within a given zone. + */ +static int bad_range(struct zone *zone, struct page *page) +{ + if (page_outside_zone_boundaries(zone, page)) + return 1; + if (!page_is_consistent(zone, page)) + return 1; + + return 0; +} +#else +static inline int bad_range(struct zone *zone, struct page *page) +{ + return 0; +} +#endif + +static void bad_page(struct page *page) +{ + static unsigned long resume; + static unsigned long nr_shown; + static unsigned long nr_unshown; + + /* Don't complain about poisoned pages */ + if (PageHWPoison(page)) { + reset_page_mapcount(page); /* remove PageBuddy */ + return; + } + + /* + * Allow a burst of 60 reports, then keep quiet for that minute; + * or allow a steady drip of one report per second. + */ + if (nr_shown == 60) { + if (time_before(jiffies, resume)) { + nr_unshown++; + goto out; + } + if (nr_unshown) { + printk(KERN_ALERT + "BUG: Bad page state: %lu messages suppressed\n", + nr_unshown); + nr_unshown = 0; + } + nr_shown = 0; + } + if (nr_shown++ == 0) + resume = jiffies + 60 * HZ; + + printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n", + current->comm, page_to_pfn(page)); + dump_page(page); + + dump_stack(); +out: + /* Leave bad fields for debug, except PageBuddy could make trouble */ + reset_page_mapcount(page); /* remove PageBuddy */ + add_taint(TAINT_BAD_PAGE); +} + +/* + * Higher-order pages are called "compound pages". They are structured thusly: + * + * The first PAGE_SIZE page is called the "head page". + * + * The remaining PAGE_SIZE pages are called "tail pages". + * + * All pages have PG_compound set. All pages have their ->private pointing at + * the head page (even the head page has this). + * + * The first tail page's ->lru.next holds the address of the compound page's + * put_page() function. Its ->lru.prev holds the order of allocation. + * This usage means that zero-order pages may not be compound. + */ + +static void free_compound_page(struct page *page) +{ + __free_pages_ok(page, compound_order(page)); +} + +void prep_compound_page(struct page *page, unsigned long order) +{ + int i; + int nr_pages = 1 << order; + + set_compound_page_dtor(page, free_compound_page); + set_compound_order(page, order); + __SetPageHead(page); + for (i = 1; i < nr_pages; i++) { + struct page *p = page + i; + __SetPageTail(p); + set_page_count(p, 0); + p->first_page = page; + } +} + +/* update __split_huge_page_refcount if you change this function */ +static int destroy_compound_page(struct page *page, unsigned long order) +{ + int i; + int nr_pages = 1 << order; + int bad = 0; + + if (unlikely(compound_order(page) != order) || + unlikely(!PageHead(page))) { + bad_page(page); + bad++; + } + + __ClearPageHead(page); + + for (i = 1; i < nr_pages; i++) { + struct page *p = page + i; + + if (unlikely(!PageTail(p) || (p->first_page != page))) { + bad_page(page); + bad++; + } + __ClearPageTail(p); + } + + return bad; +} + +static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags) +{ + int i; + + /* + * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO + * and __GFP_HIGHMEM from hard or soft interrupt context. + */ + VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt()); + for (i = 0; i < (1 << order); i++) + clear_highpage(page + i); +} + +static inline void set_page_order(struct page *page, int order) +{ + set_page_private(page, order); + __SetPageBuddy(page); +} + +static inline void rmv_page_order(struct page *page) +{ + __ClearPageBuddy(page); + set_page_private(page, 0); +} + +/* + * Locate the struct page for both the matching buddy in our + * pair (buddy1) and the combined O(n+1) page they form (page). + * + * 1) Any buddy B1 will have an order O twin B2 which satisfies + * the following equation: + * B2 = B1 ^ (1 << O) + * For example, if the starting buddy (buddy2) is #8 its order + * 1 buddy is #10: + * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10 + * + * 2) Any buddy B will have an order O+1 parent P which + * satisfies the following equation: + * P = B & ~(1 << O) + * + * Assumption: *_mem_map is contiguous at least up to MAX_ORDER + */ +static inline unsigned long +__find_buddy_index(unsigned long page_idx, unsigned int order) +{ + return page_idx ^ (1 << order); +} + +/* + * This function checks whether a page is free && is the buddy + * we can do coalesce a page and its buddy if + * (a) the buddy is not in a hole && + * (b) the buddy is in the buddy system && + * (c) a page and its buddy have the same order && + * (d) a page and its buddy are in the same zone. + * + * For recording whether a page is in the buddy system, we set ->_mapcount -2. + * Setting, clearing, and testing _mapcount -2 is serialized by zone->lock. + * + * For recording page's order, we use page_private(page). + */ +static inline int page_is_buddy(struct page *page, struct page *buddy, + int order) +{ + if (!pfn_valid_within(page_to_pfn(buddy))) + return 0; + + if (page_zone_id(page) != page_zone_id(buddy)) + return 0; + + if (PageBuddy(buddy) && page_order(buddy) == order) { + VM_BUG_ON(page_count(buddy) != 0); + return 1; + } + return 0; +} + +/* + * Freeing function for a buddy system allocator. + * + * The concept of a buddy system is to maintain direct-mapped table + * (containing bit values) for memory blocks of various "orders". + * The bottom level table contains the map for the smallest allocatable + * units of memory (here, pages), and each level above it describes + * pairs of units from the levels below, hence, "buddies". + * At a high level, all that happens here is marking the table entry + * at the bottom level available, and propagating the changes upward + * as necessary, plus some accounting needed to play nicely with other + * parts of the VM system. + * At each level, we keep a list of pages, which are heads of continuous + * free pages of length of (1 << order) and marked with _mapcount -2. Page's + * order is recorded in page_private(page) field. + * So when we are allocating or freeing one, we can derive the state of the + * other. That is, if we allocate a small block, and both were + * free, the remainder of the region must be split into blocks. + * If a block is freed, and its buddy is also free, then this + * triggers coalescing into a block of larger size. + * + * -- wli + */ + +static inline void __free_one_page(struct page *page, + struct zone *zone, unsigned int order, + int migratetype) +{ + unsigned long page_idx; + unsigned long combined_idx; + unsigned long uninitialized_var(buddy_idx); + struct page *buddy; + + if (unlikely(PageCompound(page))) + if (unlikely(destroy_compound_page(page, order))) + return; + + VM_BUG_ON(migratetype == -1); + + page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1); + + VM_BUG_ON(page_idx & ((1 << order) - 1)); + VM_BUG_ON(bad_range(zone, page)); + + while (order < MAX_ORDER-1) { + buddy_idx = __find_buddy_index(page_idx, order); + buddy = page + (buddy_idx - page_idx); + if (!page_is_buddy(page, buddy, order)) + break; + + /* Our buddy is free, merge with it and move up one order. */ + list_del(&buddy->lru); + zone->free_area[order].nr_free--; + rmv_page_order(buddy); + combined_idx = buddy_idx & page_idx; + page = page + (combined_idx - page_idx); + page_idx = combined_idx; + order++; + } + set_page_order(page, order); + + /* + * If this is not the largest possible page, check if the buddy + * of the next-highest order is free. If it is, it's possible + * that pages are being freed that will coalesce soon. In case, + * that is happening, add the free page to the tail of the list + * so it's less likely to be used soon and more likely to be merged + * as a higher order page + */ + if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { + struct page *higher_page, *higher_buddy; + combined_idx = buddy_idx & page_idx; + higher_page = page + (combined_idx - page_idx); + buddy_idx = __find_buddy_index(combined_idx, order + 1); + higher_buddy = page + (buddy_idx - combined_idx); + if (page_is_buddy(higher_page, higher_buddy, order + 1)) { + list_add_tail(&page->lru, + &zone->free_area[order].free_list[migratetype]); + goto out; + } + } + + list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); +out: + zone->free_area[order].nr_free++; +} + +/* + * free_page_mlock() -- clean up attempts to free and mlocked() page. + * Page should not be on lru, so no need to fix that up. + * free_pages_check() will verify... + */ +static inline void free_page_mlock(struct page *page) +{ + __dec_zone_page_state(page, NR_MLOCK); + __count_vm_event(UNEVICTABLE_MLOCKFREED); +} + +static inline int free_pages_check(struct page *page) +{ + if (unlikely(page_mapcount(page) | + (page->mapping != NULL) | + (atomic_read(&page->_count) != 0) | + (page->flags & PAGE_FLAGS_CHECK_AT_FREE) | + (mem_cgroup_bad_page_check(page)))) { + bad_page(page); + return 1; + } + if (page->flags & PAGE_FLAGS_CHECK_AT_PREP) + page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; + return 0; +} + +/* + * Frees a number of pages from the PCP lists + * Assumes all pages on list are in same zone, and of same order. + * count is the number of pages to free. + * + * If the zone was previously in an "all pages pinned" state then look to + * see if this freeing clears that state. + * + * And clear the zone's pages_scanned counter, to hold off the "all pages are + * pinned" detection logic. + */ +static void free_pcppages_bulk(struct zone *zone, int count, + struct per_cpu_pages *pcp) +{ + int migratetype = 0; + int batch_free = 0; + int to_free = count; + + spin_lock(&zone->lock); + zone->all_unreclaimable = 0; + zone->pages_scanned = 0; + + while (to_free) { + struct page *page; + struct list_head *list; + + /* + * Remove pages from lists in a round-robin fashion. A + * batch_free count is maintained that is incremented when an + * empty list is encountered. This is so more pages are freed + * off fuller lists instead of spinning excessively around empty + * lists + */ + do { + batch_free++; + if (++migratetype == MIGRATE_PCPTYPES) + migratetype = 0; + list = &pcp->lists[migratetype]; + } while (list_empty(list)); + + /* This is the only non-empty list. Free them all. */ + if (batch_free == MIGRATE_PCPTYPES) + batch_free = to_free; + + do { + int mt; + + page = list_entry(list->prev, struct page, lru); + /* must delete as __free_one_page list manipulates */ + list_del(&page->lru); + + mt = page_private(page); + /* + * cached MIGRATE_CMA pageblock type may have changed + * during isolation + */ + if (is_migrate_cma(mt) && + get_pageblock_migratetype(page) == MIGRATE_ISOLATE) + mt = MIGRATE_ISOLATE; + + /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ + __free_one_page(page, zone, 0, mt); + trace_mm_page_pcpu_drain(page, 0, mt); + if (is_cma_pageblock(page)) + __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, + 1); + } while (--to_free && --batch_free && !list_empty(list)); + } + __mod_zone_page_state(zone, NR_FREE_PAGES, count); + spin_unlock(&zone->lock); +} + +static void free_one_page(struct zone *zone, struct page *page, int order, + int migratetype) +{ + spin_lock(&zone->lock); + zone->all_unreclaimable = 0; + zone->pages_scanned = 0; + + __free_one_page(page, zone, order, migratetype); + __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order); + if (is_cma_pageblock(page)) + __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, 1 << order); + spin_unlock(&zone->lock); +} + +static bool free_pages_prepare(struct page *page, unsigned int order) +{ + int i; + int bad = 0; + + trace_mm_page_free_direct(page, order); + kmemcheck_free_shadow(page, order); + + if (PageAnon(page)) + page->mapping = NULL; + for (i = 0; i < (1 << order); i++) + bad += free_pages_check(page + i); + if (bad) + return false; + + if (!PageHighMem(page)) { + debug_check_no_locks_freed(page_address(page), + PAGE_SIZE<<order); + debug_check_no_obj_freed(page_address(page), + PAGE_SIZE << order); + } + arch_free_page(page, order); + kernel_map_pages(page, 1 << order, 0); + + return true; +} + +static void __free_pages_ok(struct page *page, unsigned int order) +{ + unsigned long flags; + int wasMlocked = __TestClearPageMlocked(page); + + if (!free_pages_prepare(page, order)) + return; + + local_irq_save(flags); + if (unlikely(wasMlocked)) + free_page_mlock(page); + __count_vm_events(PGFREE, 1 << order); + free_one_page(page_zone(page), page, order, + get_pageblock_migratetype(page)); + local_irq_restore(flags); +} + +/* + * permit the bootmem allocator to evade page validation on high-order frees + */ +void __meminit __free_pages_bootmem(struct page *page, unsigned int order) +{ + if (order == 0) { + __ClearPageReserved(page); + set_page_count(page, 0); + set_page_refcounted(page); + __free_page(page); + } else { + int loop; + + prefetchw(page); + for (loop = 0; loop < BITS_PER_LONG; loop++) { + struct page *p = &page[loop]; + + if (loop + 1 < BITS_PER_LONG) + prefetchw(p + 1); + __ClearPageReserved(p); + set_page_count(p, 0); + } + + set_page_refcounted(page); + __free_pages(page, order); + } +} + +#ifdef CONFIG_DMA_CMA + +struct cma_pageblock { + unsigned long start_pfn, end_pfn; +}; + +static struct cma_pageblock cma_pageblocks[MAX_CMA_AREAS]; + +static unsigned int cma_pageblocks_count; + +void set_cma_pageblock(struct page *page) +{ + unsigned long pfn = page_to_pfn(page); + int i; + + BUG_ON(cma_pageblocks_count >= MAX_CMA_AREAS); + + for (i = 0; i < cma_pageblocks_count; i++) { + unsigned long end = cma_pageblocks[i].end_pfn; + + if (pfn == end + 1) { + cma_pageblocks[i].end_pfn += pageblock_nr_pages; + return; + } + } + + cma_pageblocks[i].start_pfn = pfn; + cma_pageblocks[i].end_pfn = pfn + pageblock_nr_pages - 1; + + cma_pageblocks_count++; +} + +bool is_cma_pageblock(struct page *page) +{ + unsigned long pfn = page_to_pfn(page); + int i; + + for (i = 0; i < cma_pageblocks_count; i++) { + unsigned long start, end; + + start = cma_pageblocks[i].start_pfn; + end = cma_pageblocks[i].end_pfn; + + if (pfn >= start && pfn <= end) + return true; + } + + return false; +} + +/* Free whole pageblock and set it's migration type to MIGRATE_CMA. */ +void __init init_cma_reserved_pageblock(struct page *page) +{ + unsigned i = pageblock_nr_pages; + struct page *p = page; + + do { + __ClearPageReserved(p); + set_page_count(p, 0); + } while (++p, --i); + + set_page_refcounted(page); + set_pageblock_migratetype(page, MIGRATE_CMA); + set_cma_pageblock(page); + __free_pages(page, pageblock_order); + totalram_pages += pageblock_nr_pages; +} +#endif + +/* + * The order of subdivision here is critical for the IO subsystem. + * Please do not alter this order without good reasons and regression + * testing. Specifically, as large blocks of memory are subdivided, + * the order in which smaller blocks are delivered depends on the order + * they're subdivided in this function. This is the primary factor + * influencing the order in which pages are delivered to the IO + * subsystem according to empirical testing, and this is also justified + * by considering the behavior of a buddy system containing a single + * large block of memory acted on by a series of small allocations. + * This behavior is a critical factor in sglist merging's success. + * + * -- wli + */ +static inline void expand(struct zone *zone, struct page *page, + int low, int high, struct free_area *area, + int migratetype) +{ + unsigned long size = 1 << high; + + while (high > low) { + area--; + high--; + size >>= 1; + VM_BUG_ON(bad_range(zone, &page[size])); + list_add(&page[size].lru, &area->free_list[migratetype]); + area->nr_free++; + set_page_order(&page[size], high); + } +} + +/* + * This page is about to be returned from the page allocator + */ +static inline int check_new_page(struct page *page) +{ + if (unlikely(page_mapcount(page) | + (page->mapping != NULL) | + (atomic_read(&page->_count) != 0) | + (page->flags & PAGE_FLAGS_CHECK_AT_PREP) | + (mem_cgroup_bad_page_check(page)))) { + bad_page(page); + return 1; + } + return 0; +} + +static int prep_new_page(struct page *page, int order, gfp_t gfp_flags) +{ + int i; + + for (i = 0; i < (1 << order); i++) { + struct page *p = page + i; + if (unlikely(check_new_page(p))) + return 1; + } + + set_page_private(page, 0); + set_page_refcounted(page); + + arch_alloc_page(page, order); + kernel_map_pages(page, 1 << order, 1); + + if (gfp_flags & __GFP_ZERO) + prep_zero_page(page, order, gfp_flags); + + if (order && (gfp_flags & __GFP_COMP)) + prep_compound_page(page, order); + + return 0; +} + +/* + * Go through the free lists for the given migratetype and remove + * the smallest available page from the freelists + */ +static inline +struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, + int migratetype) +{ + unsigned int current_order; + struct free_area *area; + struct page *page; + + /* Find a page of the appropriate size in the preferred list */ + for (current_order = order; current_order < MAX_ORDER; ++current_order) { + area = &(zone->free_area[current_order]); + if (list_empty(&area->free_list[migratetype])) + continue; + + page = list_entry(area->free_list[migratetype].next, + struct page, lru); + list_del(&page->lru); + rmv_page_order(page); + area->nr_free--; + expand(zone, page, order, current_order, area, migratetype); + return page; + } + + return NULL; +} + + +/* + * This array describes the order lists are fallen back to when + * the free lists for the desirable migrate type are depleted + */ +static int fallbacks[MIGRATE_TYPES][4] = { + [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE }, + [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE }, +#ifdef CONFIG_DMA_CMA + [MIGRATE_MOVABLE] = { MIGRATE_CMA, MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE }, + [MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */ +#else + [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE }, +#endif + [MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */ + [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */ +}; + +/* + * Move the free pages in a range to the free lists of the requested type. + * Note that start_page and end_pages are not aligned on a pageblock + * boundary. If alignment is required, use move_freepages_block() + */ +static int move_freepages(struct zone *zone, + struct page *start_page, struct page *end_page, + int migratetype) +{ + struct page *page; + unsigned long order; + int pages_moved = 0; + +#ifndef CONFIG_HOLES_IN_ZONE + /* + * page_zone is not safe to call in this context when + * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant + * anyway as we check zone boundaries in move_freepages_block(). + * Remove at a later date when no bug reports exist related to + * grouping pages by mobility + */ + BUG_ON(page_zone(start_page) != page_zone(end_page)); +#endif + + for (page = start_page; page <= end_page;) { + /* Make sure we are not inadvertently changing nodes */ + VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone)); + + if (!pfn_valid_within(page_to_pfn(page))) { + page++; + continue; + } + + if (!PageBuddy(page)) { + page++; + continue; + } + + order = page_order(page); + list_move(&page->lru, + &zone->free_area[order].free_list[migratetype]); + page += 1 << order; + pages_moved += 1 << order; + } + + return pages_moved; +} + +static int move_freepages_block(struct zone *zone, struct page *page, + int migratetype) +{ + unsigned long start_pfn, end_pfn; + struct page *start_page, *end_page; + + start_pfn = page_to_pfn(page); + start_pfn = start_pfn & ~(pageblock_nr_pages-1); + start_page = pfn_to_page(start_pfn); + end_page = start_page + pageblock_nr_pages - 1; + end_pfn = start_pfn + pageblock_nr_pages - 1; + + /* Do not cross zone boundaries */ + if (start_pfn < zone->zone_start_pfn) + start_page = page; + if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages) + return 0; + + return move_freepages(zone, start_page, end_page, migratetype); +} + +static void change_pageblock_range(struct page *pageblock_page, + int start_order, int migratetype) +{ + int nr_pageblocks = 1 << (start_order - pageblock_order); + + while (nr_pageblocks--) { + set_pageblock_migratetype(pageblock_page, migratetype); + pageblock_page += pageblock_nr_pages; + } +} + +/* Remove an element from the buddy allocator from the fallback list */ +static inline struct page * +__rmqueue_fallback(struct zone *zone, int order, int start_migratetype) +{ + struct free_area *area; + int current_order; + struct page *page; + int migratetype, i; + + /* Find the largest possible block of pages in the other list */ + for (current_order = MAX_ORDER-1; current_order >= order; + --current_order) { + for (i = 0;; i++) { + migratetype = fallbacks[start_migratetype][i]; + + /* MIGRATE_RESERVE handled later if necessary */ + if (migratetype == MIGRATE_RESERVE) + break; + + area = &(zone->free_area[current_order]); + if (list_empty(&area->free_list[migratetype])) + continue; + + page = list_entry(area->free_list[migratetype].next, + struct page, lru); + area->nr_free--; + + /* + * If breaking a large block of pages, move all free + * pages to the preferred allocation list. If falling + * back for a reclaimable kernel allocation, be more + * aggressive about taking ownership of free pages + * + * On the other hand, never change migration + * type of MIGRATE_CMA pageblocks nor move CMA + * pages on different free lists. We don't + * want unmovable pages to be allocated from + * MIGRATE_CMA areas. + */ + if (!is_migrate_cma(migratetype) && + (unlikely(current_order >= pageblock_order / 2) || + start_migratetype == MIGRATE_RECLAIMABLE || + page_group_by_mobility_disabled)) { + int pages; + pages = move_freepages_block(zone, page, + start_migratetype); + + /* Claim the whole block if over half of it is free */ + if (pages >= (1 << (pageblock_order-1)) || + page_group_by_mobility_disabled) + set_pageblock_migratetype(page, + start_migratetype); + + migratetype = start_migratetype; + } + + /* Remove the page from the freelists */ + list_del(&page->lru); + rmv_page_order(page); + + /* Take ownership for orders >= pageblock_order */ + if (current_order >= pageblock_order && + !is_migrate_cma(migratetype)) + change_pageblock_range(page, current_order, + start_migratetype); + + expand(zone, page, order, current_order, area, + is_migrate_cma(migratetype) + ? migratetype : start_migratetype); + + trace_mm_page_alloc_extfrag(page, order, current_order, + start_migratetype, migratetype); + + return page; + } + } + + return NULL; +} + +/* + * Do the hard work of removing an element from the buddy allocator. + * Call me with the zone->lock already held. + */ +static struct page *__rmqueue(struct zone *zone, unsigned int order, + int migratetype) +{ + struct page *page; + +retry_reserve: + page = __rmqueue_smallest(zone, order, migratetype); + + if (unlikely(!page) && migratetype != MIGRATE_RESERVE) { + page = __rmqueue_fallback(zone, order, migratetype); + + /* + * Use MIGRATE_RESERVE rather than fail an allocation. goto + * is used because __rmqueue_smallest is an inline function + * and we want just one call site + */ + if (!page) { + migratetype = MIGRATE_RESERVE; + goto retry_reserve; + } + } + + trace_mm_page_alloc_zone_locked(page, order, migratetype); + return page; +} + +static struct page *__rmqueue_cma(struct zone *zone, unsigned int order, + int migratetype) +{ + struct page *page = 0; + + if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc) + page = __rmqueue_smallest(zone, order, MIGRATE_CMA); + else +retry_reserve : + page = __rmqueue_smallest(zone, order, migratetype); + + + if (unlikely(!page) && migratetype != MIGRATE_RESERVE) { + page = __rmqueue_fallback(zone, order, migratetype); + + /* + * Use MIGRATE_RESERVE rather than fail an allocation. goto + * is used because __rmqueue_smallest is an inline function + * and we want just one call site + */ + if (!page) { + migratetype = MIGRATE_RESERVE; + goto retry_reserve; + } + } + + trace_mm_page_alloc_zone_locked(page, order, migratetype); + return page; +} + +/* + * Obtain a specified number of elements from the buddy allocator, all under + * a single hold of the lock, for efficiency. Add them to the supplied list. + * Returns the number of new pages which were placed at *list. + */ +static int rmqueue_bulk(struct zone *zone, unsigned int order, + unsigned long count, struct list_head *list, + int migratetype, int cold, int cma) +{ + int mt = migratetype, i; + + spin_lock(&zone->lock); + for (i = 0; i < count; ++i) { + struct page *page; + if (cma) + page = __rmqueue_cma(zone, order, migratetype); + else + page = __rmqueue(zone, order, migratetype); + if (unlikely(page == NULL)) + break; + + /* + * Split buddy pages returned by expand() are received here + * in physical page order. The page is added to the callers and + * list and the list head then moves forward. From the callers + * perspective, the linked list is ordered by page number in + * some conditions. This is useful for IO devices that can + * merge IO requests if the physical pages are ordered + * properly. + */ + if (likely(cold == 0)) + list_add(&page->lru, list); + else + list_add_tail(&page->lru, list); + + mt = get_pageblock_migratetype(page); + if (!is_migrate_cma(mt) && mt != MIGRATE_ISOLATE) + mt = migratetype; + + set_page_private(page, mt); + list = &page->lru; + if (is_cma_pageblock(page)) + __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, + -(1 << order)); + } + __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); + spin_unlock(&zone->lock); + return i; +} + +#ifdef CONFIG_NUMA +/* + * Called from the vmstat counter updater to drain pagesets of this + * currently executing processor on remote nodes after they have + * expired. + * + * Note that this function must be called with the thread pinned to + * a single processor. + */ +void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) +{ + unsigned long flags; + int to_drain; + + local_irq_save(flags); + if (pcp->count >= pcp->batch) + to_drain = pcp->batch; + else + to_drain = pcp->count; + free_pcppages_bulk(zone, to_drain, pcp); + pcp->count -= to_drain; + local_irq_restore(flags); +} +#endif + +/* + * Drain pages of the indicated processor. + * + * The processor must either be the current processor and the + * thread pinned to the current processor or a processor that + * is not online. + */ +static void drain_pages(unsigned int cpu) +{ + unsigned long flags; + struct zone *zone; + + for_each_populated_zone(zone) { + struct per_cpu_pageset *pset; + struct per_cpu_pages *pcp; + + local_irq_save(flags); + pset = per_cpu_ptr(zone->pageset, cpu); + + pcp = &pset->pcp; + if (pcp->count) { + free_pcppages_bulk(zone, pcp->count, pcp); + pcp->count = 0; + } + local_irq_restore(flags); + } +} + +/* + * Spill all of this CPU's per-cpu pages back into the buddy allocator. + */ +void drain_local_pages(void *arg) +{ + drain_pages(smp_processor_id()); +} + +/* + * Spill all the per-cpu pages from all CPUs back into the buddy allocator + */ +void drain_all_pages(void) +{ + on_each_cpu(drain_local_pages, NULL, 1); +} + +#ifdef CONFIG_HIBERNATION + +void mark_free_pages(struct zone *zone) +{ + unsigned long pfn, max_zone_pfn; + unsigned long flags; + int order, t; + struct list_head *curr; + + if (!zone->spanned_pages) + return; + + spin_lock_irqsave(&zone->lock, flags); + + max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; + for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) + if (pfn_valid(pfn)) { + struct page *page = pfn_to_page(pfn); + + if (!swsusp_page_is_forbidden(page)) + swsusp_unset_page_free(page); + } + + for_each_migratetype_order(order, t) { + list_for_each(curr, &zone->free_area[order].free_list[t]) { + unsigned long i; + + pfn = page_to_pfn(list_entry(curr, struct page, lru)); + for (i = 0; i < (1UL << order); i++) + swsusp_set_page_free(pfn_to_page(pfn + i)); + } + } + spin_unlock_irqrestore(&zone->lock, flags); +} +#endif /* CONFIG_PM */ + +/* + * Free a 0-order page + * cold == 1 ? free a cold page : free a hot page + */ +void free_hot_cold_page(struct page *page, int cold) +{ + struct zone *zone = page_zone(page); + struct per_cpu_pages *pcp; + unsigned long flags; + int migratetype; + int wasMlocked = __TestClearPageMlocked(page); + + if (!free_pages_prepare(page, 0)) + return; + + migratetype = get_pageblock_migratetype(page); + set_page_private(page, migratetype); + local_irq_save(flags); + if (unlikely(wasMlocked)) + free_page_mlock(page); + __count_vm_event(PGFREE); + + /* + * We only track unmovable, reclaimable and movable on pcp lists. + * Free ISOLATE pages back to the allocator because they are being + * offlined but treat RESERVE as movable pages so we can get those + * areas back if necessary. Otherwise, we may have to free + * excessively into the page allocator + */ + if (migratetype >= MIGRATE_PCPTYPES) { + if (unlikely(migratetype == MIGRATE_ISOLATE) || + is_migrate_cma(migratetype)) { + free_one_page(zone, page, 0, migratetype); + goto out; + } + migratetype = MIGRATE_MOVABLE; + } + + pcp = &this_cpu_ptr(zone->pageset)->pcp; + if (cold) + list_add_tail(&page->lru, &pcp->lists[migratetype]); + else + list_add(&page->lru, &pcp->lists[migratetype]); + pcp->count++; + if (pcp->count >= pcp->high) { + free_pcppages_bulk(zone, pcp->batch, pcp); + pcp->count -= pcp->batch; + } + +out: + local_irq_restore(flags); +} + +/* + * split_page takes a non-compound higher-order page, and splits it into + * n (1<<order) sub-pages: page[0..n] + * Each sub-page must be freed individually. + * + * Note: this is probably too low level an operation for use in drivers. + * Please consult with lkml before using this in your driver. + */ +void split_page(struct page *page, unsigned int order) +{ + int i; + + VM_BUG_ON(PageCompound(page)); + VM_BUG_ON(!page_count(page)); + +#ifdef CONFIG_KMEMCHECK + /* + * Split shadow pages too, because free(page[0]) would + * otherwise free the whole shadow. + */ + if (kmemcheck_page_is_tracked(page)) + split_page(virt_to_page(page[0].shadow), order); +#endif + + for (i = 1; i < (1 << order); i++) + set_page_refcounted(page + i); +} + +/* + * Similar to split_page except the page is already free. As this is only + * being used for migration, the migratetype of the block also changes. + * As this is called with interrupts disabled, the caller is responsible + * for calling arch_alloc_page() and kernel_map_page() after interrupts + * are enabled. + * + * Note: this is probably too low level an operation for use in drivers. + * Please consult with lkml before using this in your driver. + */ +int split_free_page(struct page *page, bool for_cma) +{ + unsigned int order; + struct zone *zone; + + BUG_ON(!PageBuddy(page)); + + zone = page_zone(page); + order = page_order(page); + + /* Obey watermarks as if the page was being allocated */ + + if (!for_cma) { + unsigned long watermark; + + watermark = low_wmark_pages(zone) + (1 << order); + if (!zone_watermark_ok(zone, 0, watermark, 0, 0)) + return 0; + } + + /* Remove page from free list */ + list_del(&page->lru); + zone->free_area[order].nr_free--; + rmv_page_order(page); + __mod_zone_page_state(zone, NR_FREE_PAGES, -(1UL << order)); + if (is_cma_pageblock(page)) + __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, -(1UL << order)); + + /* Split into individual pages */ + set_page_refcounted(page); + split_page(page, order); + + if (order >= pageblock_order - 1) { + struct page *endpage = page + (1 << order) - 1; + for (; page < endpage; page += pageblock_nr_pages) { + int mt = get_pageblock_migratetype(page); + if (mt != MIGRATE_ISOLATE && !is_migrate_cma(mt)) + set_pageblock_migratetype(page, + MIGRATE_MOVABLE); + } + } + + return 1 << order; +} + +/* + * Really, prep_compound_page() should be called from __rmqueue_bulk(). But + * we cheat by calling it from here, in the order > 0 path. Saves a branch + * or two. + */ +static inline +struct page *buffered_rmqueue(struct zone *preferred_zone, + struct zone *zone, int order, gfp_t gfp_flags, + int migratetype) +{ + unsigned long flags; + struct page *page; + int cold = !!(gfp_flags & __GFP_COLD); + +again: + if (likely(order == 0)) { + struct per_cpu_pages *pcp; + struct list_head *list; + int mt; + + local_irq_save(flags); + pcp = &this_cpu_ptr(zone->pageset)->pcp; + list = &pcp->lists[migratetype]; + if (list_empty(list)) { + pcp->count += rmqueue_bulk(zone, 0, + pcp->batch, list, + migratetype, cold, + gfp_flags & __GFP_CMA); + if (unlikely(list_empty(list))) + goto failed; + } + + if (cold) + page = list_entry(list->prev, struct page, lru); + else + page = list_entry(list->next, struct page, lru); + + list_del(&page->lru); + pcp->count--; + + spin_lock(&zone->lock); + mt = page_private(page); + /* + * cached MIGRATE_CMA pageblock type may have changed + * during isolation + */ + if ((is_migrate_cma(mt) && + get_pageblock_migratetype(page) == MIGRATE_ISOLATE) || + mt == MIGRATE_ISOLATE) { + mt = MIGRATE_ISOLATE; + + zone->all_unreclaimable = 0; + zone->pages_scanned = 0; + + __free_one_page(page, zone, 0, mt); + __mod_zone_page_state(zone, NR_FREE_PAGES, 1); + spin_unlock(&zone->lock); + goto again; + } else + spin_unlock(&zone->lock); + } else { + if (unlikely(gfp_flags & __GFP_NOFAIL)) { + /* + * __GFP_NOFAIL is not to be used in new code. + * + * All __GFP_NOFAIL callers should be fixed so that they + * properly detect and handle allocation failures. + * + * We most definitely don't want callers attempting to + * allocate greater than order-1 page units with + * __GFP_NOFAIL. + */ + WARN_ON_ONCE(order > 1); + } + spin_lock_irqsave(&zone->lock, flags); + if (gfp_flags & __GFP_CMA) + page = __rmqueue_cma(zone, order, migratetype); + else + page = __rmqueue(zone, order, migratetype); + spin_unlock(&zone->lock); + if (!page) + goto failed; + if (is_cma_pageblock(page)) + __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, + -(1 << order)); + __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order)); + } + + __count_zone_vm_events(PGALLOC, zone, 1 << order); + zone_statistics(preferred_zone, zone, gfp_flags); + local_irq_restore(flags); + + VM_BUG_ON(bad_range(zone, page)); + if (prep_new_page(page, order, gfp_flags)) + goto again; + return page; + +failed: + local_irq_restore(flags); + return NULL; +} + +/* The ALLOC_WMARK bits are used as an index to zone->watermark */ +#define ALLOC_WMARK_MIN WMARK_MIN +#define ALLOC_WMARK_LOW WMARK_LOW +#define ALLOC_WMARK_HIGH WMARK_HIGH +#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */ + +/* Mask to get the watermark bits */ +#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1) + +#define ALLOC_HARDER 0x10 /* try to alloc harder */ +#define ALLOC_HIGH 0x20 /* __GFP_HIGH set */ +#define ALLOC_CPUSET 0x40 /* check for correct cpuset */ + +#ifdef CONFIG_FAIL_PAGE_ALLOC + +static struct fail_page_alloc_attr { + struct fault_attr attr; + + u32 ignore_gfp_highmem; + u32 ignore_gfp_wait; + u32 min_order; + +#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS + + struct dentry *ignore_gfp_highmem_file; + struct dentry *ignore_gfp_wait_file; + struct dentry *min_order_file; + +#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ + +} fail_page_alloc = { + .attr = FAULT_ATTR_INITIALIZER, + .ignore_gfp_wait = 1, + .ignore_gfp_highmem = 1, + .min_order = 1, +}; + +static int __init setup_fail_page_alloc(char *str) +{ + return setup_fault_attr(&fail_page_alloc.attr, str); +} +__setup("fail_page_alloc=", setup_fail_page_alloc); + +static int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) +{ + if (order < fail_page_alloc.min_order) + return 0; + if (gfp_mask & __GFP_NOFAIL) + return 0; + if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) + return 0; + if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT)) + return 0; + + return should_fail(&fail_page_alloc.attr, 1 << order); +} + +#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS + +static int __init fail_page_alloc_debugfs(void) +{ + mode_t mode = S_IFREG | S_IRUSR | S_IWUSR; + struct dentry *dir; + int err; + + err = init_fault_attr_dentries(&fail_page_alloc.attr, + "fail_page_alloc"); + if (err) + return err; + dir = fail_page_alloc.attr.dentries.dir; + + fail_page_alloc.ignore_gfp_wait_file = + debugfs_create_bool("ignore-gfp-wait", mode, dir, + &fail_page_alloc.ignore_gfp_wait); + + fail_page_alloc.ignore_gfp_highmem_file = + debugfs_create_bool("ignore-gfp-highmem", mode, dir, + &fail_page_alloc.ignore_gfp_highmem); + fail_page_alloc.min_order_file = + debugfs_create_u32("min-order", mode, dir, + &fail_page_alloc.min_order); + + if (!fail_page_alloc.ignore_gfp_wait_file || + !fail_page_alloc.ignore_gfp_highmem_file || + !fail_page_alloc.min_order_file) { + err = -ENOMEM; + debugfs_remove(fail_page_alloc.ignore_gfp_wait_file); + debugfs_remove(fail_page_alloc.ignore_gfp_highmem_file); + debugfs_remove(fail_page_alloc.min_order_file); + cleanup_fault_attr_dentries(&fail_page_alloc.attr); + } + + return err; +} + +late_initcall(fail_page_alloc_debugfs); + +#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ + +#else /* CONFIG_FAIL_PAGE_ALLOC */ + +static inline int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) +{ + return 0; +} + +#endif /* CONFIG_FAIL_PAGE_ALLOC */ + +/* + * Return true if free pages are above 'mark'. This takes into account the order + * of the allocation. + */ +static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark, + int classzone_idx, int alloc_flags, + long free_pages, long free_cma_pages) +{ + /* free_pages my go negative - that's OK */ + long min = mark; + long lowmem_res = z->lowmem_reserve[classzone_idx]; + int o; + + free_pages -= (1 << order) + 1; + if (alloc_flags & ALLOC_HIGH) + min -= min / 2; + if (alloc_flags & ALLOC_HARDER) + min -= min / 4; + + /* do not account cma pages twice during alloc_contig_migrate_range() */ + if (free_cma_pages) + free_cma_pages -= z->min_cma_pages; + + if (free_pages - free_cma_pages <= min + lowmem_res) + return false; + for (o = 0; o < order; o++) { + /* At the next order, this order's pages become unavailable */ + free_pages -= z->free_area[o].nr_free << o; + + /* Require fewer higher order pages to be free */ + min >>= min_free_order_shift; + + if (free_pages <= min) + return false; + } + return true; +} + +bool zone_watermark_ok(struct zone *z, int order, unsigned long mark, + int classzone_idx, int alloc_flags) +{ + return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, + zone_page_state(z, NR_FREE_PAGES), + zone_page_state(z, NR_FREE_CMA_PAGES)); +} + +bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark, + int classzone_idx, int alloc_flags) +{ + long free_pages = zone_page_state(z, NR_FREE_PAGES); + long free_cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES); + + if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) + free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); + + return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, + free_pages, free_cma_pages); +} + +#ifdef CONFIG_NUMA +/* + * zlc_setup - Setup for "zonelist cache". Uses cached zone data to + * skip over zones that are not allowed by the cpuset, or that have + * been recently (in last second) found to be nearly full. See further + * comments in mmzone.h. Reduces cache footprint of zonelist scans + * that have to skip over a lot of full or unallowed zones. + * + * If the zonelist cache is present in the passed in zonelist, then + * returns a pointer to the allowed node mask (either the current + * tasks mems_allowed, or node_states[N_HIGH_MEMORY].) + * + * If the zonelist cache is not available for this zonelist, does + * nothing and returns NULL. + * + * If the fullzones BITMAP in the zonelist cache is stale (more than + * a second since last zap'd) then we zap it out (clear its bits.) + * + * We hold off even calling zlc_setup, until after we've checked the + * first zone in the zonelist, on the theory that most allocations will + * be satisfied from that first zone, so best to examine that zone as + * quickly as we can. + */ +static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) +{ + struct zonelist_cache *zlc; /* cached zonelist speedup info */ + nodemask_t *allowednodes; /* zonelist_cache approximation */ + + zlc = zonelist->zlcache_ptr; + if (!zlc) + return NULL; + + if (time_after(jiffies, zlc->last_full_zap + HZ)) { + bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); + zlc->last_full_zap = jiffies; + } + + allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ? + &cpuset_current_mems_allowed : + &node_states[N_HIGH_MEMORY]; + return allowednodes; +} + +/* + * Given 'z' scanning a zonelist, run a couple of quick checks to see + * if it is worth looking at further for free memory: + * 1) Check that the zone isn't thought to be full (doesn't have its + * bit set in the zonelist_cache fullzones BITMAP). + * 2) Check that the zones node (obtained from the zonelist_cache + * z_to_n[] mapping) is allowed in the passed in allowednodes mask. + * Return true (non-zero) if zone is worth looking at further, or + * else return false (zero) if it is not. + * + * This check -ignores- the distinction between various watermarks, + * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is + * found to be full for any variation of these watermarks, it will + * be considered full for up to one second by all requests, unless + * we are so low on memory on all allowed nodes that we are forced + * into the second scan of the zonelist. + * + * In the second scan we ignore this zonelist cache and exactly + * apply the watermarks to all zones, even it is slower to do so. + * We are low on memory in the second scan, and should leave no stone + * unturned looking for a free page. + */ +static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z, + nodemask_t *allowednodes) +{ + struct zonelist_cache *zlc; /* cached zonelist speedup info */ + int i; /* index of *z in zonelist zones */ + int n; /* node that zone *z is on */ + + zlc = zonelist->zlcache_ptr; + if (!zlc) + return 1; + + i = z - zonelist->_zonerefs; + n = zlc->z_to_n[i]; + + /* This zone is worth trying if it is allowed but not full */ + return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones); +} + +/* + * Given 'z' scanning a zonelist, set the corresponding bit in + * zlc->fullzones, so that subsequent attempts to allocate a page + * from that zone don't waste time re-examining it. + */ +static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z) +{ + struct zonelist_cache *zlc; /* cached zonelist speedup info */ + int i; /* index of *z in zonelist zones */ + + zlc = zonelist->zlcache_ptr; + if (!zlc) + return; + + i = z - zonelist->_zonerefs; + + set_bit(i, zlc->fullzones); +} + +/* + * clear all zones full, called after direct reclaim makes progress so that + * a zone that was recently full is not skipped over for up to a second + */ +static void zlc_clear_zones_full(struct zonelist *zonelist) +{ + struct zonelist_cache *zlc; /* cached zonelist speedup info */ + + zlc = zonelist->zlcache_ptr; + if (!zlc) + return; + + bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); +} + +#else /* CONFIG_NUMA */ + +static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) +{ + return NULL; +} + +static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z, + nodemask_t *allowednodes) +{ + return 1; +} + +static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z) +{ +} + +static void zlc_clear_zones_full(struct zonelist *zonelist) +{ +} +#endif /* CONFIG_NUMA */ + +/* + * get_page_from_freelist goes through the zonelist trying to allocate + * a page. + */ +static struct page * +get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order, + struct zonelist *zonelist, int high_zoneidx, int alloc_flags, + struct zone *preferred_zone, int migratetype) +{ + struct zoneref *z; + struct page *page = NULL; + int classzone_idx; + struct zone *zone; + nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */ + int zlc_active = 0; /* set if using zonelist_cache */ + int did_zlc_setup = 0; /* just call zlc_setup() one time */ + + classzone_idx = zone_idx(preferred_zone); +zonelist_scan: + /* + * Scan zonelist, looking for a zone with enough free. + * See also cpuset_zone_allowed() comment in kernel/cpuset.c. + */ + for_each_zone_zonelist_nodemask(zone, z, zonelist, + high_zoneidx, nodemask) { + if (NUMA_BUILD && zlc_active && + !zlc_zone_worth_trying(zonelist, z, allowednodes)) + continue; + if ((alloc_flags & ALLOC_CPUSET) && + !cpuset_zone_allowed_softwall(zone, gfp_mask)) + continue; + + BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); + if (!(alloc_flags & ALLOC_NO_WATERMARKS)) { + unsigned long mark; + int ret; + + mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; + if (zone_watermark_ok(zone, order, mark, + classzone_idx, alloc_flags)) + goto try_this_zone; + + if (NUMA_BUILD && !did_zlc_setup + && nr_online_nodes > 1) { + /* + * we do zlc_setup if there are multiple nodes + * and before considering the first zone allowed + * by the cpuset. + */ + allowednodes = zlc_setup(zonelist, alloc_flags); + zlc_active = 1; + did_zlc_setup = 1; + } + + if (zone_reclaim_mode == 0) + goto this_zone_full; + + /* + * As we may have just activated ZLC, check if the first + * eligible zone has failed zone_reclaim recently. + */ + if (NUMA_BUILD && zlc_active && + !zlc_zone_worth_trying(zonelist, z, allowednodes)) + continue; + + ret = zone_reclaim(zone, gfp_mask, order); + switch (ret) { + case ZONE_RECLAIM_NOSCAN: + /* did not scan */ + continue; + case ZONE_RECLAIM_FULL: + /* scanned but unreclaimable */ + continue; + default: + /* did we reclaim enough */ + if (!zone_watermark_ok(zone, order, mark, + classzone_idx, alloc_flags)) + goto this_zone_full; + } + } + +try_this_zone: + page = buffered_rmqueue(preferred_zone, zone, order, + gfp_mask, migratetype); + if (page) + break; +this_zone_full: + if (NUMA_BUILD) + zlc_mark_zone_full(zonelist, z); + } + + if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) { + /* Disable zlc cache for second zonelist scan */ + zlc_active = 0; + goto zonelist_scan; + } + return page; +} + +/* + * Large machines with many possible nodes should not always dump per-node + * meminfo in irq context. + */ +static inline bool should_suppress_show_mem(void) +{ + bool ret = false; + +#if NODES_SHIFT > 8 + ret = in_interrupt(); +#endif + return ret; +} + +static DEFINE_RATELIMIT_STATE(nopage_rs, + DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + +void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...) +{ + va_list args; + unsigned int filter = SHOW_MEM_FILTER_NODES; + + if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) + return; + + /* + * This documents exceptions given to allocations in certain + * contexts that are allowed to allocate outside current's set + * of allowed nodes. + */ + if (!(gfp_mask & __GFP_NOMEMALLOC)) + if (test_thread_flag(TIF_MEMDIE) || + (current->flags & (PF_MEMALLOC | PF_EXITING))) + filter &= ~SHOW_MEM_FILTER_NODES; + if (in_interrupt() || !(gfp_mask & __GFP_WAIT)) + filter &= ~SHOW_MEM_FILTER_NODES; + + if (fmt) { + printk(KERN_WARNING); + va_start(args, fmt); + vprintk(fmt, args); + va_end(args); + } + + pr_warning("%s: page allocation failure: order:%d, mode:0x%x\n", + current->comm, order, gfp_mask); + + dump_stack(); + if (!should_suppress_show_mem()) + show_mem(filter); +} + +static inline int +should_alloc_retry(gfp_t gfp_mask, unsigned int order, + unsigned long pages_reclaimed) +{ + /* Do not loop if specifically requested */ + if (gfp_mask & __GFP_NORETRY) + return 0; + + /* + * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER + * means __GFP_NOFAIL, but that may not be true in other + * implementations. + */ + if (order <= PAGE_ALLOC_COSTLY_ORDER) + return 1; + + /* + * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is + * specified, then we retry until we no longer reclaim any pages + * (above), or we've reclaimed an order of pages at least as + * large as the allocation's order. In both cases, if the + * allocation still fails, we stop retrying. + */ + if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order)) + return 1; + + /* + * Don't let big-order allocations loop unless the caller + * explicitly requests that. + */ + if (gfp_mask & __GFP_NOFAIL) + return 1; + + return 0; +} + +static inline struct page * +__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, + struct zonelist *zonelist, enum zone_type high_zoneidx, + nodemask_t *nodemask, struct zone *preferred_zone, + int migratetype) +{ + struct page *page; +#ifdef CONFIG_COMPACTION_RETRY + struct zoneref *z; + struct zone *zone; +#endif + + /* Acquire the OOM killer lock for the zones in zonelist */ + if (!try_set_zonelist_oom(zonelist, gfp_mask)) { + schedule_timeout_uninterruptible(1); + return NULL; + } + + /* + * Go through the zonelist yet one more time, keep very high watermark + * here, this is only to catch a parallel oom killing, we must fail if + * we're still under heavy pressure. + */ + page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, + order, zonelist, high_zoneidx, + ALLOC_WMARK_HIGH|ALLOC_CPUSET, + preferred_zone, migratetype); + if (page) + goto out; + +#ifdef CONFIG_COMPACTION_RETRY + /* + * When we reach here, we already tried direct reclaim. + * Therefore it might be possible that we have enough + * free memory but extremely fragmented. + * So we give it a last chance to try memory compaction and get pages. + */ + if (order) { + pr_info("reclaim before oom : retry compaction.\n"); + show_buddy_info(); + + for_each_zone_zonelist_nodemask(zone, z, zonelist, + high_zoneidx, nodemask) + compact_zone_order(zone, -1, gfp_mask, true); + + show_buddy_info(); + pr_info("reclaim :end\n"); + page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, + nodemask, order, zonelist, high_zoneidx, + ALLOC_WMARK_HIGH|ALLOC_CPUSET, + preferred_zone, migratetype); + if (page) + goto out; + } +#endif + + if (!(gfp_mask & __GFP_NOFAIL)) { + /* The OOM killer will not help higher order allocs */ + if (order > PAGE_ALLOC_COSTLY_ORDER) + goto out; + /* The OOM killer does not needlessly kill tasks for lowmem */ + if (high_zoneidx < ZONE_NORMAL) + goto out; + /* + * GFP_THISNODE contains __GFP_NORETRY and we never hit this. + * Sanity check for bare calls of __GFP_THISNODE, not real OOM. + * The caller should handle page allocation failure by itself if + * it specifies __GFP_THISNODE. + * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER. + */ + if (gfp_mask & __GFP_THISNODE) + goto out; + } + /* Exhausted what can be done so it's blamo time */ + out_of_memory(zonelist, gfp_mask, order, nodemask); + +out: + clear_zonelist_oom(zonelist, gfp_mask); + return page; +} + +#ifdef CONFIG_COMPACTION +/* Try memory compaction for high-order allocations before reclaim */ +static struct page * +__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, + struct zonelist *zonelist, enum zone_type high_zoneidx, + nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone, + int migratetype, unsigned long *did_some_progress, + bool sync_migration) +{ + struct page *page; + + if (!order || compaction_deferred(preferred_zone)) + return NULL; + + current->flags |= PF_MEMALLOC; + *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask, + nodemask, sync_migration); + current->flags &= ~PF_MEMALLOC; + if (*did_some_progress != COMPACT_SKIPPED) { + + /* Page migration frees to the PCP lists but we want merging */ + drain_pages(get_cpu()); + put_cpu(); + + page = get_page_from_freelist(gfp_mask, nodemask, + order, zonelist, high_zoneidx, + alloc_flags, preferred_zone, + migratetype); + if (page) { + preferred_zone->compact_considered = 0; + preferred_zone->compact_defer_shift = 0; + count_vm_event(COMPACTSUCCESS); + return page; + } + + /* + * It's bad if compaction run occurs and fails. + * The most likely reason is that pages exist, + * but not enough to satisfy watermarks. + */ + count_vm_event(COMPACTFAIL); + defer_compaction(preferred_zone); + + cond_resched(); + } + + return NULL; +} +#else +static inline struct page * +__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, + struct zonelist *zonelist, enum zone_type high_zoneidx, + nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone, + int migratetype, unsigned long *did_some_progress, + bool sync_migration) +{ + return NULL; +} +#endif /* CONFIG_COMPACTION */ + +/* Perform direct synchronous page reclaim */ +static int +__perform_reclaim(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist, + nodemask_t *nodemask) +{ + struct reclaim_state reclaim_state; + int progress; + + cond_resched(); + + /* We now go into synchronous reclaim */ + cpuset_memory_pressure_bump(); + current->flags |= PF_MEMALLOC; + lockdep_set_current_reclaim_state(gfp_mask); + reclaim_state.reclaimed_slab = 0; + current->reclaim_state = &reclaim_state; + + progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask); + + current->reclaim_state = NULL; + lockdep_clear_current_reclaim_state(); + current->flags &= ~PF_MEMALLOC; + + cond_resched(); + + return progress; +} + +/* The really slow allocator path where we enter direct reclaim */ +static inline struct page * +__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, + struct zonelist *zonelist, enum zone_type high_zoneidx, + nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone, + int migratetype, unsigned long *did_some_progress) +{ + struct page *page = NULL; + bool drained = false; + + *did_some_progress = __perform_reclaim(gfp_mask, order, zonelist, + nodemask); + if (unlikely(!(*did_some_progress))) + return NULL; + + /* After successful reclaim, reconsider all zones for allocation */ + if (NUMA_BUILD) + zlc_clear_zones_full(zonelist); + +retry: + page = get_page_from_freelist(gfp_mask, nodemask, order, + zonelist, high_zoneidx, + alloc_flags, preferred_zone, + migratetype); + + /* + * If an allocation failed after direct reclaim, it could be because + * pages are pinned on the per-cpu lists. Drain them and try again + */ + if (!page && !drained) { + drain_all_pages(); + drained = true; + goto retry; + } + + return page; +} + +/* + * This is called in the allocator slow-path if the allocation request is of + * sufficient urgency to ignore watermarks and take other desperate measures + */ +static inline struct page * +__alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order, + struct zonelist *zonelist, enum zone_type high_zoneidx, + nodemask_t *nodemask, struct zone *preferred_zone, + int migratetype) +{ + struct page *page; + + do { + page = get_page_from_freelist(gfp_mask, nodemask, order, + zonelist, high_zoneidx, ALLOC_NO_WATERMARKS, + preferred_zone, migratetype); + + if (!page && gfp_mask & __GFP_NOFAIL) + wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50); + } while (!page && (gfp_mask & __GFP_NOFAIL)); + + return page; +} + +static inline +void wake_all_kswapd(unsigned int order, struct zonelist *zonelist, + enum zone_type high_zoneidx, + enum zone_type classzone_idx) +{ + struct zoneref *z; + struct zone *zone; + + for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) + wakeup_kswapd(zone, order, classzone_idx); +} + +static inline int +gfp_to_alloc_flags(gfp_t gfp_mask) +{ + int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; + const gfp_t wait = gfp_mask & __GFP_WAIT; + + /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ + BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); + + /* + * The caller may dip into page reserves a bit more if the caller + * cannot run direct reclaim, or if the caller has realtime scheduling + * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will + * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH). + */ + alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); + + if (!wait) { + /* + * Not worth trying to allocate harder for + * __GFP_NOMEMALLOC even if it can't schedule. + */ + if (!(gfp_mask & __GFP_NOMEMALLOC)) + alloc_flags |= ALLOC_HARDER; + /* + * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc. + * See also cpuset_zone_allowed() comment in kernel/cpuset.c. + */ + alloc_flags &= ~ALLOC_CPUSET; + } else if (unlikely(rt_task(current)) && !in_interrupt()) + alloc_flags |= ALLOC_HARDER; + + if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) { + if (!in_interrupt() && + ((current->flags & PF_MEMALLOC) || + unlikely(test_thread_flag(TIF_MEMDIE)))) + alloc_flags |= ALLOC_NO_WATERMARKS; + } + + return alloc_flags; +} + +static inline struct page * +__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, + struct zonelist *zonelist, enum zone_type high_zoneidx, + nodemask_t *nodemask, struct zone *preferred_zone, + int migratetype) +{ + const gfp_t wait = gfp_mask & __GFP_WAIT; + struct page *page = NULL; + int alloc_flags; + unsigned long pages_reclaimed = 0; + unsigned long did_some_progress; + bool sync_migration = false; +#ifdef CONFIG_ANDROID_WIP + unsigned long start_tick = jiffies; +#endif + + /* + * In the slowpath, we sanity check order to avoid ever trying to + * reclaim >= MAX_ORDER areas which will never succeed. Callers may + * be using allocators in order of preference for an area that is + * too large. + */ + if (order >= MAX_ORDER) { + WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); + return NULL; + } + + if (gfp_mask & __GFP_WAIT) + down_read(&page_alloc_slow_rwsem); + + /* + * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and + * __GFP_NOWARN set) should not cause reclaim since the subsystem + * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim + * using a larger set of nodes after it has established that the + * allowed per node queues are empty and that nodes are + * over allocated. + */ + if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE) + goto nopage; + +restart: + if (!(gfp_mask & __GFP_NO_KSWAPD)) + wake_all_kswapd(order, zonelist, high_zoneidx, + zone_idx(preferred_zone)); + + /* + * OK, we're below the kswapd watermark and have kicked background + * reclaim. Now things get more complex, so set up alloc_flags according + * to how we want to proceed. + */ + alloc_flags = gfp_to_alloc_flags(gfp_mask); + + /* + * Find the true preferred zone if the allocation is unconstrained by + * cpusets. + */ + if (!(alloc_flags & ALLOC_CPUSET) && !nodemask) + first_zones_zonelist(zonelist, high_zoneidx, NULL, + &preferred_zone); + +rebalance: + /* This is the last chance, in general, before the goto nopage. */ + page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist, + high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS, + preferred_zone, migratetype); + if (page) + goto got_pg; + + /* Allocate without watermarks if the context allows */ + if (alloc_flags & ALLOC_NO_WATERMARKS) { + page = __alloc_pages_high_priority(gfp_mask, order, + zonelist, high_zoneidx, nodemask, + preferred_zone, migratetype); + if (page) + goto got_pg; + } + + /* Atomic allocations - we can't balance anything */ + if (!wait) + goto nopage; + + /* Avoid recursion of direct reclaim */ + if (current->flags & PF_MEMALLOC) + goto nopage; + + /* Avoid allocations with no watermarks from looping endlessly */ + if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL)) + goto nopage; + + /* + * Try direct compaction. The first pass is asynchronous. Subsequent + * attempts after direct reclaim are synchronous + */ + page = __alloc_pages_direct_compact(gfp_mask, order, + zonelist, high_zoneidx, + nodemask, + alloc_flags, preferred_zone, + migratetype, &did_some_progress, + sync_migration); + if (page) + goto got_pg; + sync_migration = true; + + /* Try direct reclaim and then allocating */ + page = __alloc_pages_direct_reclaim(gfp_mask, order, + zonelist, high_zoneidx, + nodemask, + alloc_flags, preferred_zone, + migratetype, &did_some_progress); + if (page) + goto got_pg; + + /* + * If we failed to make any progress reclaiming, then we are + * running out of options and have to consider going OOM + * ANDROID_WIP: If we are looping more than 1 second, consider OOM + */ +#ifdef CONFIG_ANDROID_WIP +#define SHOULD_CONSIDER_OOM (!did_some_progress || time_after(jiffies, start_tick + HZ)) +#else +#define SHOULD_CONSIDER_OOM (!did_some_progress) +#endif + if (SHOULD_CONSIDER_OOM) { + if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) { + if (oom_killer_disabled) + goto nopage; +#ifdef CONFIG_ANDROID_WIP + if (did_some_progress) + pr_info("time's up : calling " + "__alloc_pages_may_oom\n"); +#endif + page = __alloc_pages_may_oom(gfp_mask, order, + zonelist, high_zoneidx, + nodemask, preferred_zone, + migratetype); + if (page) + goto got_pg; + + if (!(gfp_mask & __GFP_NOFAIL)) { + /* + * The oom killer is not called for high-order + * allocations that may fail, so if no progress + * is being made, there are no other options and + * retrying is unlikely to help. + */ + if (order > PAGE_ALLOC_COSTLY_ORDER) + goto nopage; + /* + * The oom killer is not called for lowmem + * allocations to prevent needlessly killing + * innocent tasks. + */ + if (high_zoneidx < ZONE_NORMAL) + goto nopage; + } + + goto restart; + } + + /* + * Suspend converts GFP_KERNEL to __GFP_WAIT which can + * prevent reclaim making forward progress without + * invoking OOM. Bail if we are suspending + */ + if (pm_suspending()) + goto nopage; + } + + /* Check if we should retry the allocation */ + pages_reclaimed += did_some_progress; + if (should_alloc_retry(gfp_mask, order, pages_reclaimed)) { + /* Wait for some write requests to complete then retry */ + wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50); + goto rebalance; + } else { + /* + * High-order allocations do not necessarily loop after + * direct reclaim and reclaim/compaction depends on compaction + * being called after reclaim so call directly if necessary + */ + page = __alloc_pages_direct_compact(gfp_mask, order, + zonelist, high_zoneidx, + nodemask, + alloc_flags, preferred_zone, + migratetype, &did_some_progress, + sync_migration); + if (page) + goto got_pg; + } + +nopage: + warn_alloc_failed(gfp_mask, order, NULL); + if (gfp_mask & __GFP_WAIT) + up_read(&page_alloc_slow_rwsem); + return page; +got_pg: + if (kmemcheck_enabled) + kmemcheck_pagealloc_alloc(page, order, gfp_mask); + if (gfp_mask & __GFP_WAIT) + up_read(&page_alloc_slow_rwsem); + return page; + +} + +/* + * This is the 'heart' of the zoned buddy allocator. + */ +struct page * +__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, + struct zonelist *zonelist, nodemask_t *nodemask) +{ + enum zone_type high_zoneidx = gfp_zone(gfp_mask); + struct zone *preferred_zone; + struct page *page; + int migratetype = allocflags_to_migratetype(gfp_mask); + unsigned int cpuset_mems_cookie; + + gfp_mask &= gfp_allowed_mask; + + lockdep_trace_alloc(gfp_mask); + + might_sleep_if(gfp_mask & __GFP_WAIT); + + if (should_fail_alloc_page(gfp_mask, order)) + return NULL; + + /* + * Check the zones suitable for the gfp_mask contain at least one + * valid zone. It's possible to have an empty zonelist as a result + * of GFP_THISNODE and a memoryless node + */ + if (unlikely(!zonelist->_zonerefs->zone)) + return NULL; + + retry_cpuset: + cpuset_mems_cookie = get_mems_allowed(); + /* The preferred zone is used for statistics later */ + first_zones_zonelist(zonelist, high_zoneidx, + nodemask ? : &cpuset_current_mems_allowed, + &preferred_zone); + if (!preferred_zone) + goto out; + + /* First allocation attempt */ + page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order, + zonelist, high_zoneidx, ALLOC_WMARK_LOW|ALLOC_CPUSET, + preferred_zone, migratetype); + if (unlikely(!page)) + page = __alloc_pages_slowpath(gfp_mask, order, + zonelist, high_zoneidx, nodemask, + preferred_zone, migratetype); + trace_mm_page_alloc(page, order, gfp_mask, migratetype); + +out: + /* + * When updating a task's mems_allowed, it is possible to race with + * parallel threads in such a way that an allocation can fail while + * the mask is being updated. If a page allocation is about to fail, + * check if the cpuset changed during allocation and if so, retry. + */ + if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page)) + goto retry_cpuset; + + return page; +} +EXPORT_SYMBOL(__alloc_pages_nodemask); + +/* + * Common helper functions. + */ +unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) +{ + struct page *page; + + /* + * __get_free_pages() returns a 32-bit address, which cannot represent + * a highmem page + */ + VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0); + + page = alloc_pages(gfp_mask, order); + if (!page) + return 0; + return (unsigned long) page_address(page); +} +EXPORT_SYMBOL(__get_free_pages); + +unsigned long get_zeroed_page(gfp_t gfp_mask) +{ + return __get_free_pages(gfp_mask | __GFP_ZERO, 0); +} +EXPORT_SYMBOL(get_zeroed_page); + +void __pagevec_free(struct pagevec *pvec) +{ + int i = pagevec_count(pvec); + + while (--i >= 0) { + trace_mm_pagevec_free(pvec->pages[i], pvec->cold); + free_hot_cold_page(pvec->pages[i], pvec->cold); + } +} + +void __free_pages(struct page *page, unsigned int order) +{ + if (put_page_testzero(page)) { + if (order == 0) + free_hot_cold_page(page, 0); + else + __free_pages_ok(page, order); + } +} +EXPORT_SYMBOL(__free_pages); + +void free_pages(unsigned long addr, unsigned int order) +{ + if (addr != 0) { + VM_BUG_ON(!virt_addr_valid((void *)addr)); + __free_pages(virt_to_page((void *)addr), order); + } +} +EXPORT_SYMBOL(free_pages); + +static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size) +{ + if (addr) { + unsigned long alloc_end = addr + (PAGE_SIZE << order); + unsigned long used = addr + PAGE_ALIGN(size); + + split_page(virt_to_page((void *)addr), order); + while (used < alloc_end) { + free_page(used); + used += PAGE_SIZE; + } + } + return (void *)addr; +} + +/** + * alloc_pages_exact - allocate an exact number physically-contiguous pages. + * @size: the number of bytes to allocate + * @gfp_mask: GFP flags for the allocation + * + * This function is similar to alloc_pages(), except that it allocates the + * minimum number of pages to satisfy the request. alloc_pages() can only + * allocate memory in power-of-two pages. + * + * This function is also limited by MAX_ORDER. + * + * Memory allocated by this function must be released by free_pages_exact(). + */ +void *alloc_pages_exact(size_t size, gfp_t gfp_mask) +{ + unsigned int order = get_order(size); + unsigned long addr; + + addr = __get_free_pages(gfp_mask, order); + return make_alloc_exact(addr, order, size); +} +EXPORT_SYMBOL(alloc_pages_exact); + +/** + * alloc_pages_exact_nid - allocate an exact number of physically-contiguous + * pages on a node. + * @nid: the preferred node ID where memory should be allocated + * @size: the number of bytes to allocate + * @gfp_mask: GFP flags for the allocation + * + * Like alloc_pages_exact(), but try to allocate on node nid first + * before falling back. + * Note this is not alloc_pages_exact_node() which allocates + * on a specific node, but is not exact. + */ +void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) +{ + unsigned order = get_order(size); + struct page *p = alloc_pages_node(nid, gfp_mask, order); + if (!p) + return NULL; + return make_alloc_exact((unsigned long)page_address(p), order, size); +} +EXPORT_SYMBOL(alloc_pages_exact_nid); + +/** + * free_pages_exact - release memory allocated via alloc_pages_exact() + * @virt: the value returned by alloc_pages_exact. + * @size: size of allocation, same value as passed to alloc_pages_exact(). + * + * Release the memory allocated by a previous call to alloc_pages_exact. + */ +void free_pages_exact(void *virt, size_t size) +{ + unsigned long addr = (unsigned long)virt; + unsigned long end = addr + PAGE_ALIGN(size); + + while (addr < end) { + free_page(addr); + addr += PAGE_SIZE; + } +} +EXPORT_SYMBOL(free_pages_exact); + +static unsigned int nr_free_zone_pages(int offset) +{ + struct zoneref *z; + struct zone *zone; + + /* Just pick one node, since fallback list is circular */ + unsigned int sum = 0; + + struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); + + for_each_zone_zonelist(zone, z, zonelist, offset) { + unsigned long size = zone->present_pages; + unsigned long high = high_wmark_pages(zone); + if (size > high) + sum += size - high; + } + + return sum; +} + +/* + * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL + */ +unsigned int nr_free_buffer_pages(void) +{ + return nr_free_zone_pages(gfp_zone(GFP_USER)); +} +EXPORT_SYMBOL_GPL(nr_free_buffer_pages); + +/* + * Amount of free RAM allocatable within all zones + */ +unsigned int nr_free_pagecache_pages(void) +{ + return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); +} + +static inline void show_node(struct zone *zone) +{ + if (NUMA_BUILD) + printk(KERN_INFO "Node %d ", zone_to_nid(zone)); +} + +void si_meminfo(struct sysinfo *val) +{ + val->totalram = totalram_pages; + val->sharedram = 0; + val->freeram = global_page_state(NR_FREE_PAGES); + val->bufferram = nr_blockdev_pages(); + val->totalhigh = totalhigh_pages; + val->freehigh = nr_free_highpages(); + val->mem_unit = PAGE_SIZE; +} +EXPORT_SYMBOL(si_meminfo); + +#ifdef CONFIG_NUMA +void si_meminfo_node(struct sysinfo *val, int nid) +{ + pg_data_t *pgdat = NODE_DATA(nid); + + val->totalram = pgdat->node_present_pages; + val->freeram = node_page_state(nid, NR_FREE_PAGES); +#ifdef CONFIG_HIGHMEM + val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages; + val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM], + NR_FREE_PAGES); +#else + val->totalhigh = 0; + val->freehigh = 0; +#endif + val->mem_unit = PAGE_SIZE; +} +#endif + +/* + * Determine whether the node should be displayed or not, depending on whether + * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). + */ +bool skip_free_areas_node(unsigned int flags, int nid) +{ + bool ret = false; + unsigned int cpuset_mems_cookie; + + if (!(flags & SHOW_MEM_FILTER_NODES)) + goto out; + + do { + cpuset_mems_cookie = get_mems_allowed(); + ret = !node_isset(nid, cpuset_current_mems_allowed); + } while (!put_mems_allowed(cpuset_mems_cookie)); +out: + return ret; +} + +#define K(x) ((x) << (PAGE_SHIFT-10)) + +/* + * Show free area list (used inside shift_scroll-lock stuff) + * We also calculate the percentage fragmentation. We do this by counting the + * memory on each free list with the exception of the first item on the list. + * Suppresses nodes that are not allowed by current's cpuset if + * SHOW_MEM_FILTER_NODES is passed. + */ +void show_free_areas(unsigned int filter) +{ + int cpu; + struct zone *zone; + + for_each_populated_zone(zone) { + if (skip_free_areas_node(filter, zone_to_nid(zone))) + continue; + show_node(zone); + printk(KERN_INFO "%s per-cpu:\n", zone->name); + + for_each_online_cpu(cpu) { + struct per_cpu_pageset *pageset; + + pageset = per_cpu_ptr(zone->pageset, cpu); + + printk(KERN_INFO "CPU %4d: hi:%5d, btch:%4d usd:%4d\n", + cpu, pageset->pcp.high, + pageset->pcp.batch, pageset->pcp.count); + } + } + + printk(KERN_INFO "active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" + " active_file:%lu inactive_file:%lu isolated_file:%lu\n" + " unevictable:%lu" + " dirty:%lu writeback:%lu unstable:%lu\n" + " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n" + " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" + " free_cma:%lu\n", + global_page_state(NR_ACTIVE_ANON), + global_page_state(NR_INACTIVE_ANON), + global_page_state(NR_ISOLATED_ANON), + global_page_state(NR_ACTIVE_FILE), + global_page_state(NR_INACTIVE_FILE), + global_page_state(NR_ISOLATED_FILE), + global_page_state(NR_UNEVICTABLE), + global_page_state(NR_FILE_DIRTY), + global_page_state(NR_WRITEBACK), + global_page_state(NR_UNSTABLE_NFS), + global_page_state(NR_FREE_PAGES), + global_page_state(NR_SLAB_RECLAIMABLE), + global_page_state(NR_SLAB_UNRECLAIMABLE), + global_page_state(NR_FILE_MAPPED), + global_page_state(NR_SHMEM), + global_page_state(NR_PAGETABLE), + global_page_state(NR_BOUNCE), + global_page_state(NR_FREE_CMA_PAGES)); + + for_each_populated_zone(zone) { + int i; + + if (skip_free_areas_node(filter, zone_to_nid(zone))) + continue; + show_node(zone); + printk(KERN_INFO "%s" + " free:%lukB" + " min:%lukB" + " low:%lukB" + " high:%lukB" + " active_anon:%lukB" + " inactive_anon:%lukB" + " active_file:%lukB" + " inactive_file:%lukB" + " unevictable:%lukB" + " isolated(anon):%lukB" + " isolated(file):%lukB" + " present:%lukB" + " mlocked:%lukB" + " dirty:%lukB" + " writeback:%lukB" + " mapped:%lukB" + " shmem:%lukB" + " slab_reclaimable:%lukB" + " slab_unreclaimable:%lukB" + " kernel_stack:%lukB" + " pagetables:%lukB" + " unstable:%lukB" + " bounce:%lukB" + " free_cma:%lukB" + " writeback_tmp:%lukB" + " pages_scanned:%lu" + " all_unreclaimable? %s" + "\n", + zone->name, + K(zone_page_state(zone, NR_FREE_PAGES)), + K(min_wmark_pages(zone)), + K(low_wmark_pages(zone)), + K(high_wmark_pages(zone)), + K(zone_page_state(zone, NR_ACTIVE_ANON)), + K(zone_page_state(zone, NR_INACTIVE_ANON)), + K(zone_page_state(zone, NR_ACTIVE_FILE)), + K(zone_page_state(zone, NR_INACTIVE_FILE)), + K(zone_page_state(zone, NR_UNEVICTABLE)), + K(zone_page_state(zone, NR_ISOLATED_ANON)), + K(zone_page_state(zone, NR_ISOLATED_FILE)), + K(zone->present_pages), + K(zone_page_state(zone, NR_MLOCK)), + K(zone_page_state(zone, NR_FILE_DIRTY)), + K(zone_page_state(zone, NR_WRITEBACK)), + K(zone_page_state(zone, NR_FILE_MAPPED)), + K(zone_page_state(zone, NR_SHMEM)), + K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)), + K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)), + zone_page_state(zone, NR_KERNEL_STACK) * + THREAD_SIZE / 1024, + K(zone_page_state(zone, NR_PAGETABLE)), + K(zone_page_state(zone, NR_UNSTABLE_NFS)), + K(zone_page_state(zone, NR_BOUNCE)), + K(zone_page_state(zone, NR_FREE_CMA_PAGES)), + K(zone_page_state(zone, NR_WRITEBACK_TEMP)), + zone->pages_scanned, + (zone->all_unreclaimable ? "yes" : "no") + ); + printk(KERN_INFO "lowmem_reserve[]:"); + for (i = 0; i < MAX_NR_ZONES; i++) + printk(KERN_INFO " %lu", zone->lowmem_reserve[i]); + printk(KERN_INFO "\n"); + } + + for_each_populated_zone(zone) { + unsigned long nr[MAX_ORDER], flags, order, total = 0; + + if (skip_free_areas_node(filter, zone_to_nid(zone))) + continue; + show_node(zone); + printk(KERN_INFO "%s: ", zone->name); + + spin_lock_irqsave(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) { + nr[order] = zone->free_area[order].nr_free; + total += nr[order] << order; + } + spin_unlock_irqrestore(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) + printk(KERN_INFO "%lu*%lukB ", nr[order], + K(1UL) << order); + printk(KERN_INFO "= %lukB\n", K(total)); + } + + printk(KERN_INFO "%ld total pagecache pages\n", + global_page_state(NR_FILE_PAGES)); + + show_swap_cache_info(); +} + +#ifdef CONFIG_COMPACTION_RETRY_DEBUG +void show_buddy_info(void) +{ + struct zone *zone; + unsigned long nr[MAX_ORDER], flags, order, total = 0; + char buf[256]; + int len; + + for_each_populated_zone(zone) { + + if (skip_free_areas_node(SHOW_MEM_FILTER_NODES, + zone_to_nid(zone))) + continue; + show_node(zone); + len = sprintf(buf, "%s: ", zone->name); + + spin_lock_irqsave(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) { + nr[order] = zone->free_area[order].nr_free; + total += nr[order] << order; + } + spin_unlock_irqrestore(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) + len += sprintf(buf + len, "%lu*%lukB ", + nr[order], K(1UL) << order); + len += sprintf(buf + len, "= %lukB", K(total)); + pr_err("%s\n", buf); + } +} +#endif + +static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) +{ + zoneref->zone = zone; + zoneref->zone_idx = zone_idx(zone); +} + +/* + * Builds allocation fallback zone lists. + * + * Add all populated zones of a node to the zonelist. + */ +static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, + int nr_zones, enum zone_type zone_type) +{ + struct zone *zone; + + BUG_ON(zone_type >= MAX_NR_ZONES); + zone_type++; + + do { + zone_type--; + zone = pgdat->node_zones + zone_type; + if (populated_zone(zone)) { + zoneref_set_zone(zone, + &zonelist->_zonerefs[nr_zones++]); + check_highest_zone(zone_type); + } + + } while (zone_type); + return nr_zones; +} + + +/* + * zonelist_order: + * 0 = automatic detection of better ordering. + * 1 = order by ([node] distance, -zonetype) + * 2 = order by (-zonetype, [node] distance) + * + * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create + * the same zonelist. So only NUMA can configure this param. + */ +#define ZONELIST_ORDER_DEFAULT 0 +#define ZONELIST_ORDER_NODE 1 +#define ZONELIST_ORDER_ZONE 2 + +/* zonelist order in the kernel. + * set_zonelist_order() will set this to NODE or ZONE. + */ +static int current_zonelist_order = ZONELIST_ORDER_DEFAULT; +static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"}; + + +#ifdef CONFIG_NUMA +/* The value user specified ....changed by config */ +static int user_zonelist_order = ZONELIST_ORDER_DEFAULT; +/* string for sysctl */ +#define NUMA_ZONELIST_ORDER_LEN 16 +char numa_zonelist_order[16] = "default"; + +/* + * interface for configure zonelist ordering. + * command line option "numa_zonelist_order" + * = "[dD]efault - default, automatic configuration. + * = "[nN]ode - order by node locality, then by zone within node + * = "[zZ]one - order by zone, then by locality within zone + */ + +static int __parse_numa_zonelist_order(char *s) +{ + if (*s == 'd' || *s == 'D') { + user_zonelist_order = ZONELIST_ORDER_DEFAULT; + } else if (*s == 'n' || *s == 'N') { + user_zonelist_order = ZONELIST_ORDER_NODE; + } else if (*s == 'z' || *s == 'Z') { + user_zonelist_order = ZONELIST_ORDER_ZONE; + } else { + printk(KERN_WARNING + "Ignoring invalid numa_zonelist_order value: " + "%s\n", s); + return -EINVAL; + } + return 0; +} + +static __init int setup_numa_zonelist_order(char *s) +{ + int ret; + + if (!s) + return 0; + + ret = __parse_numa_zonelist_order(s); + if (ret == 0) + strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN); + + return ret; +} +early_param("numa_zonelist_order", setup_numa_zonelist_order); + +/* + * sysctl handler for numa_zonelist_order + */ +int numa_zonelist_order_handler(ctl_table *table, int write, + void __user *buffer, size_t *length, + loff_t *ppos) +{ + char saved_string[NUMA_ZONELIST_ORDER_LEN]; + int ret; + static DEFINE_MUTEX(zl_order_mutex); + + mutex_lock(&zl_order_mutex); + if (write) + strcpy(saved_string, (char *)table->data); + ret = proc_dostring(table, write, buffer, length, ppos); + if (ret) + goto out; + if (write) { + int oldval = user_zonelist_order; + if (__parse_numa_zonelist_order((char *)table->data)) { + /* + * bogus value. restore saved string + */ + strncpy((char *)table->data, saved_string, + NUMA_ZONELIST_ORDER_LEN); + user_zonelist_order = oldval; + } else if (oldval != user_zonelist_order) { + mutex_lock(&zonelists_mutex); + build_all_zonelists(NULL); + mutex_unlock(&zonelists_mutex); + } + } +out: + mutex_unlock(&zl_order_mutex); + return ret; +} + + +#define MAX_NODE_LOAD (nr_online_nodes) +static int node_load[MAX_NUMNODES]; + +/** + * find_next_best_node - find the next node that should appear in a given node's fallback list + * @node: node whose fallback list we're appending + * @used_node_mask: nodemask_t of already used nodes + * + * We use a number of factors to determine which is the next node that should + * appear on a given node's fallback list. The node should not have appeared + * already in @node's fallback list, and it should be the next closest node + * according to the distance array (which contains arbitrary distance values + * from each node to each node in the system), and should also prefer nodes + * with no CPUs, since presumably they'll have very little allocation pressure + * on them otherwise. + * It returns -1 if no node is found. + */ +static int find_next_best_node(int node, nodemask_t *used_node_mask) +{ + int n, val; + int min_val = INT_MAX; + int best_node = -1; + const struct cpumask *tmp = cpumask_of_node(0); + + /* Use the local node if we haven't already */ + if (!node_isset(node, *used_node_mask)) { + node_set(node, *used_node_mask); + return node; + } + + for_each_node_state(n, N_HIGH_MEMORY) { + + /* Don't want a node to appear more than once */ + if (node_isset(n, *used_node_mask)) + continue; + + /* Use the distance array to find the distance */ + val = node_distance(node, n); + + /* Penalize nodes under us ("prefer the next node") */ + val += (n < node); + + /* Give preference to headless and unused nodes */ + tmp = cpumask_of_node(n); + if (!cpumask_empty(tmp)) + val += PENALTY_FOR_NODE_WITH_CPUS; + + /* Slight preference for less loaded node */ + val *= (MAX_NODE_LOAD*MAX_NUMNODES); + val += node_load[n]; + + if (val < min_val) { + min_val = val; + best_node = n; + } + } + + if (best_node >= 0) + node_set(best_node, *used_node_mask); + + return best_node; +} + + +/* + * Build zonelists ordered by node and zones within node. + * This results in maximum locality--normal zone overflows into local + * DMA zone, if any--but risks exhausting DMA zone. + */ +static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) +{ + int j; + struct zonelist *zonelist; + + zonelist = &pgdat->node_zonelists[0]; + for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++) + ; + j = build_zonelists_node(NODE_DATA(node), zonelist, j, + MAX_NR_ZONES - 1); + zonelist->_zonerefs[j].zone = NULL; + zonelist->_zonerefs[j].zone_idx = 0; +} + +/* + * Build gfp_thisnode zonelists + */ +static void build_thisnode_zonelists(pg_data_t *pgdat) +{ + int j; + struct zonelist *zonelist; + + zonelist = &pgdat->node_zonelists[1]; + j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1); + zonelist->_zonerefs[j].zone = NULL; + zonelist->_zonerefs[j].zone_idx = 0; +} + +/* + * Build zonelists ordered by zone and nodes within zones. + * This results in conserving DMA zone[s] until all Normal memory is + * exhausted, but results in overflowing to remote node while memory + * may still exist in local DMA zone. + */ +static int node_order[MAX_NUMNODES]; + +static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes) +{ + int pos, j, node; + int zone_type; /* needs to be signed */ + struct zone *z; + struct zonelist *zonelist; + + zonelist = &pgdat->node_zonelists[0]; + pos = 0; + for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) { + for (j = 0; j < nr_nodes; j++) { + node = node_order[j]; + z = &NODE_DATA(node)->node_zones[zone_type]; + if (populated_zone(z)) { + zoneref_set_zone(z, + &zonelist->_zonerefs[pos++]); + check_highest_zone(zone_type); + } + } + } + zonelist->_zonerefs[pos].zone = NULL; + zonelist->_zonerefs[pos].zone_idx = 0; +} + +static int default_zonelist_order(void) +{ + int nid, zone_type; + unsigned long low_kmem_size, total_size; + struct zone *z; + int average_size; + /* + * ZONE_DMA and ZONE_DMA32 can be very small area in the system. + * If they are really small and used heavily, the system can fall + * into OOM very easily. + * This function detect ZONE_DMA/DMA32 size and configures zone order. + */ + /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */ + low_kmem_size = 0; + total_size = 0; + for_each_online_node(nid) { + for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { + z = &NODE_DATA(nid)->node_zones[zone_type]; + if (populated_zone(z)) { + if (zone_type < ZONE_NORMAL) + low_kmem_size += z->present_pages; + total_size += z->present_pages; + } else if (zone_type == ZONE_NORMAL) { + /* + * If any node has only lowmem, then node order + * is preferred to allow kernel allocations + * locally; otherwise, they can easily infringe + * on other nodes when there is an abundance of + * lowmem available to allocate from. + */ + return ZONELIST_ORDER_NODE; + } + } + } + if (!low_kmem_size || /* there are no DMA area. */ + low_kmem_size > total_size/2) /* DMA/DMA32 is big. */ + return ZONELIST_ORDER_NODE; + /* + * look into each node's config. + * If there is a node whose DMA/DMA32 memory is very big area on + * local memory, NODE_ORDER may be suitable. + */ + average_size = total_size / + (nodes_weight(node_states[N_HIGH_MEMORY]) + 1); + for_each_online_node(nid) { + low_kmem_size = 0; + total_size = 0; + for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) { + z = &NODE_DATA(nid)->node_zones[zone_type]; + if (populated_zone(z)) { + if (zone_type < ZONE_NORMAL) + low_kmem_size += z->present_pages; + total_size += z->present_pages; + } + } + if (low_kmem_size && + total_size > average_size && /* ignore small node */ + low_kmem_size > total_size * 70/100) + return ZONELIST_ORDER_NODE; + } + return ZONELIST_ORDER_ZONE; +} + +static void set_zonelist_order(void) +{ + if (user_zonelist_order == ZONELIST_ORDER_DEFAULT) + current_zonelist_order = default_zonelist_order(); + else + current_zonelist_order = user_zonelist_order; +} + +static void build_zonelists(pg_data_t *pgdat) +{ + int j, node, load; + enum zone_type i; + nodemask_t used_mask; + int local_node, prev_node; + struct zonelist *zonelist; + int order = current_zonelist_order; + + /* initialize zonelists */ + for (i = 0; i < MAX_ZONELISTS; i++) { + zonelist = pgdat->node_zonelists + i; + zonelist->_zonerefs[0].zone = NULL; + zonelist->_zonerefs[0].zone_idx = 0; + } + + /* NUMA-aware ordering of nodes */ + local_node = pgdat->node_id; + load = nr_online_nodes; + prev_node = local_node; + nodes_clear(used_mask); + + memset(node_order, 0, sizeof(node_order)); + j = 0; + + while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { + int distance = node_distance(local_node, node); + + /* + * If another node is sufficiently far away then it is better + * to reclaim pages in a zone before going off node. + */ + if (distance > RECLAIM_DISTANCE) + zone_reclaim_mode = 1; + + /* + * We don't want to pressure a particular node. + * So adding penalty to the first node in same + * distance group to make it round-robin. + */ + if (distance != node_distance(local_node, prev_node)) + node_load[node] = load; + + prev_node = node; + load--; + if (order == ZONELIST_ORDER_NODE) + build_zonelists_in_node_order(pgdat, node); + else + node_order[j++] = node; /* remember order */ + } + + if (order == ZONELIST_ORDER_ZONE) { + /* calculate node order -- i.e., DMA last! */ + build_zonelists_in_zone_order(pgdat, j); + } + + build_thisnode_zonelists(pgdat); +} + +/* Construct the zonelist performance cache - see further mmzone.h */ +static void build_zonelist_cache(pg_data_t *pgdat) +{ + struct zonelist *zonelist; + struct zonelist_cache *zlc; + struct zoneref *z; + + zonelist = &pgdat->node_zonelists[0]; + zonelist->zlcache_ptr = zlc = &zonelist->zlcache; + bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); + for (z = zonelist->_zonerefs; z->zone; z++) + zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z); +} + +#ifdef CONFIG_HAVE_MEMORYLESS_NODES +/* + * Return node id of node used for "local" allocations. + * I.e., first node id of first zone in arg node's generic zonelist. + * Used for initializing percpu 'numa_mem', which is used primarily + * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. + */ +int local_memory_node(int node) +{ + struct zone *zone; + + (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL), + gfp_zone(GFP_KERNEL), + NULL, + &zone); + return zone->node; +} +#endif + +#else /* CONFIG_NUMA */ + +static void set_zonelist_order(void) +{ + current_zonelist_order = ZONELIST_ORDER_ZONE; +} + +static void build_zonelists(pg_data_t *pgdat) +{ + int node, local_node; + enum zone_type j; + struct zonelist *zonelist; + + local_node = pgdat->node_id; + + zonelist = &pgdat->node_zonelists[0]; + j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1); + + /* + * Now we build the zonelist so that it contains the zones + * of all the other nodes. + * We don't want to pressure a particular node, so when + * building the zones for node N, we make sure that the + * zones coming right after the local ones are those from + * node N+1 (modulo N) + */ + for (node = local_node + 1; node < MAX_NUMNODES; node++) { + if (!node_online(node)) + continue; + j = build_zonelists_node(NODE_DATA(node), zonelist, j, + MAX_NR_ZONES - 1); + } + for (node = 0; node < local_node; node++) { + if (!node_online(node)) + continue; + j = build_zonelists_node(NODE_DATA(node), zonelist, j, + MAX_NR_ZONES - 1); + } + + zonelist->_zonerefs[j].zone = NULL; + zonelist->_zonerefs[j].zone_idx = 0; +} + +/* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */ +static void build_zonelist_cache(pg_data_t *pgdat) +{ + pgdat->node_zonelists[0].zlcache_ptr = NULL; +} + +#endif /* CONFIG_NUMA */ + +/* + * Boot pageset table. One per cpu which is going to be used for all + * zones and all nodes. The parameters will be set in such a way + * that an item put on a list will immediately be handed over to + * the buddy list. This is safe since pageset manipulation is done + * with interrupts disabled. + * + * The boot_pagesets must be kept even after bootup is complete for + * unused processors and/or zones. They do play a role for bootstrapping + * hotplugged processors. + * + * zoneinfo_show() and maybe other functions do + * not check if the processor is online before following the pageset pointer. + * Other parts of the kernel may not check if the zone is available. + */ +static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); +static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); +static void setup_zone_pageset(struct zone *zone); + +/* + * Global mutex to protect against size modification of zonelists + * as well as to serialize pageset setup for the new populated zone. + */ +DEFINE_MUTEX(zonelists_mutex); + +/* return values int ....just for stop_machine() */ +static __init_refok int __build_all_zonelists(void *data) +{ + int nid; + int cpu; + +#ifdef CONFIG_NUMA + memset(node_load, 0, sizeof(node_load)); +#endif + for_each_online_node(nid) { + pg_data_t *pgdat = NODE_DATA(nid); + + build_zonelists(pgdat); + build_zonelist_cache(pgdat); + } + + /* + * Initialize the boot_pagesets that are going to be used + * for bootstrapping processors. The real pagesets for + * each zone will be allocated later when the per cpu + * allocator is available. + * + * boot_pagesets are used also for bootstrapping offline + * cpus if the system is already booted because the pagesets + * are needed to initialize allocators on a specific cpu too. + * F.e. the percpu allocator needs the page allocator which + * needs the percpu allocator in order to allocate its pagesets + * (a chicken-egg dilemma). + */ + for_each_possible_cpu(cpu) { + setup_pageset(&per_cpu(boot_pageset, cpu), 0); + +#ifdef CONFIG_HAVE_MEMORYLESS_NODES + /* + * We now know the "local memory node" for each node-- + * i.e., the node of the first zone in the generic zonelist. + * Set up numa_mem percpu variable for on-line cpus. During + * boot, only the boot cpu should be on-line; we'll init the + * secondary cpus' numa_mem as they come on-line. During + * node/memory hotplug, we'll fixup all on-line cpus. + */ + if (cpu_online(cpu)) + set_cpu_numa_mem(cpu, + local_memory_node(cpu_to_node(cpu))); +#endif + } + + return 0; +} + +/* + * Called with zonelists_mutex held always + * unless system_state == SYSTEM_BOOTING. + */ +void __ref build_all_zonelists(void *data) +{ + set_zonelist_order(); + + if (system_state == SYSTEM_BOOTING) { + __build_all_zonelists(NULL); + mminit_verify_zonelist(); + cpuset_init_current_mems_allowed(); + } else { + /* we have to stop all cpus to guarantee there is no user + of zonelist */ +#ifdef CONFIG_MEMORY_HOTPLUG + if (data) + setup_zone_pageset((struct zone *)data); +#endif + stop_machine(__build_all_zonelists, NULL, NULL); + /* cpuset refresh routine should be here */ + } + vm_total_pages = nr_free_pagecache_pages(); + /* + * Disable grouping by mobility if the number of pages in the + * system is too low to allow the mechanism to work. It would be + * more accurate, but expensive to check per-zone. This check is + * made on memory-hotadd so a system can start with mobility + * disabled and enable it later + */ + if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) + page_group_by_mobility_disabled = 1; + else + page_group_by_mobility_disabled = 0; + + printk(KERN_INFO "Built %i zonelists in %s order, mobility grouping %s." + "Total pages: %ld\n", + nr_online_nodes, + zonelist_order_name[current_zonelist_order], + page_group_by_mobility_disabled ? "off" : "on", + vm_total_pages); +#ifdef CONFIG_NUMA + printk(KERN_INFO "Policy zone: %s\n", zone_names[policy_zone]); +#endif +} + +/* + * Helper functions to size the waitqueue hash table. + * Essentially these want to choose hash table sizes sufficiently + * large so that collisions trying to wait on pages are rare. + * But in fact, the number of active page waitqueues on typical + * systems is ridiculously low, less than 200. So this is even + * conservative, even though it seems large. + * + * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to + * waitqueues, i.e. the size of the waitq table given the number of pages. + */ +#define PAGES_PER_WAITQUEUE 256 + +#ifndef CONFIG_MEMORY_HOTPLUG +static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) +{ + unsigned long size = 1; + + pages /= PAGES_PER_WAITQUEUE; + + while (size < pages) + size <<= 1; + + /* + * Once we have dozens or even hundreds of threads sleeping + * on IO we've got bigger problems than wait queue collision. + * Limit the size of the wait table to a reasonable size. + */ + size = min(size, 4096UL); + + return max(size, 4UL); +} +#else +/* + * A zone's size might be changed by hot-add, so it is not possible to determine + * a suitable size for its wait_table. So we use the maximum size now. + * + * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie: + * + * i386 (preemption config) : 4096 x 16 = 64Kbyte. + * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte. + * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte. + * + * The maximum entries are prepared when a zone's memory is (512K + 256) pages + * or more by the traditional way. (See above). It equals: + * + * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte. + * ia64(16K page size) : = ( 8G + 4M)byte. + * powerpc (64K page size) : = (32G +16M)byte. + */ +static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) +{ + return 4096UL; +} +#endif + +/* + * This is an integer logarithm so that shifts can be used later + * to extract the more random high bits from the multiplicative + * hash function before the remainder is taken. + */ +static inline unsigned long wait_table_bits(unsigned long size) +{ + return ffz(~size); +} + +#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) + +/* + * Check if a pageblock contains reserved pages + */ +static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn) +{ + unsigned long pfn; + + for (pfn = start_pfn; pfn < end_pfn; pfn++) { + if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn))) + return 1; + } + return 0; +} + +/* + * Mark a number of pageblocks as MIGRATE_RESERVE. The number + * of blocks reserved is based on min_wmark_pages(zone). The memory within + * the reserve will tend to store contiguous free pages. Setting min_free_kbytes + * higher will lead to a bigger reserve which will get freed as contiguous + * blocks as reclaim kicks in + */ +static void setup_zone_migrate_reserve(struct zone *zone) +{ + unsigned long start_pfn, pfn, end_pfn, block_end_pfn; + struct page *page; + unsigned long block_migratetype; + int reserve; + + /* + * Get the start pfn, end pfn and the number of blocks to reserve + * We have to be careful to be aligned to pageblock_nr_pages to + * make sure that we always check pfn_valid for the first page in + * the block. + */ + start_pfn = zone->zone_start_pfn; + end_pfn = start_pfn + zone->spanned_pages; + start_pfn = roundup(start_pfn, pageblock_nr_pages); + reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >> + pageblock_order; + + /* + * Reserve blocks are generally in place to help high-order atomic + * allocations that are short-lived. A min_free_kbytes value that + * would result in more than 2 reserve blocks for atomic allocations + * is assumed to be in place to help anti-fragmentation for the + * future allocation of hugepages at runtime. + */ + reserve = min(2, reserve); + + for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { + if (!pfn_valid(pfn)) + continue; + page = pfn_to_page(pfn); + + /* Watch out for overlapping nodes */ + if (page_to_nid(page) != zone_to_nid(zone)) + continue; + + /* Blocks with reserved pages will never free, skip them. */ + block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn); + if (pageblock_is_reserved(pfn, block_end_pfn)) + continue; + + block_migratetype = get_pageblock_migratetype(page); + + /* If this block is reserved, account for it */ + if (reserve > 0 && block_migratetype == MIGRATE_RESERVE) { + reserve--; + continue; + } + + /* Suitable for reserving if this block is movable */ + if (reserve > 0 && block_migratetype == MIGRATE_MOVABLE) { + set_pageblock_migratetype(page, MIGRATE_RESERVE); + move_freepages_block(zone, page, MIGRATE_RESERVE); + reserve--; + continue; + } + + /* + * If the reserve is met and this is a previous reserved block, + * take it back + */ + if (block_migratetype == MIGRATE_RESERVE) { + set_pageblock_migratetype(page, MIGRATE_MOVABLE); + move_freepages_block(zone, page, MIGRATE_MOVABLE); + } + } +} + +/* + * Initially all pages are reserved - free ones are freed + * up by free_all_bootmem() once the early boot process is + * done. Non-atomic initialization, single-pass. + */ +void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, + unsigned long start_pfn, enum memmap_context context) +{ + struct page *page; + unsigned long end_pfn = start_pfn + size; + unsigned long pfn; + struct zone *z; + + if (highest_memmap_pfn < end_pfn - 1) + highest_memmap_pfn = end_pfn - 1; + + z = &NODE_DATA(nid)->node_zones[zone]; + for (pfn = start_pfn; pfn < end_pfn; pfn++) { + /* + * There can be holes in boot-time mem_map[]s + * handed to this function. They do not + * exist on hotplugged memory. + */ + if (context == MEMMAP_EARLY) { + if (!early_pfn_valid(pfn)) + continue; + if (!early_pfn_in_nid(pfn, nid)) + continue; + } + page = pfn_to_page(pfn); + set_page_links(page, zone, nid, pfn); + mminit_verify_page_links(page, zone, nid, pfn); + init_page_count(page); + reset_page_mapcount(page); + SetPageReserved(page); + /* + * Mark the block movable so that blocks are reserved for + * movable at startup. This will force kernel allocations + * to reserve their blocks rather than leaking throughout + * the address space during boot when many long-lived + * kernel allocations are made. Later some blocks near + * the start are marked MIGRATE_RESERVE by + * setup_zone_migrate_reserve() + * + * bitmap is created for zone's valid pfn range. but memmap + * can be created for invalid pages (for alignment) + * check here not to call set_pageblock_migratetype() against + * pfn out of zone. + */ + if ((z->zone_start_pfn <= pfn) + && (pfn < z->zone_start_pfn + z->spanned_pages) + && !(pfn & (pageblock_nr_pages - 1))) + set_pageblock_migratetype(page, MIGRATE_MOVABLE); + + INIT_LIST_HEAD(&page->lru); +#ifdef WANT_PAGE_VIRTUAL + /* The shift won't overflow because ZONE_NORMAL is below 4G. */ + if (!is_highmem_idx(zone)) + set_page_address(page, __va(pfn << PAGE_SHIFT)); +#endif + } +} + +static void __meminit zone_init_free_lists(struct zone *zone) +{ + int order, t; + for_each_migratetype_order(order, t) { + INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); + zone->free_area[order].nr_free = 0; + } +} + +#ifndef __HAVE_ARCH_MEMMAP_INIT +#define memmap_init(size, nid, zone, start_pfn) \ + memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY) +#endif + +static int zone_batchsize(struct zone *zone) +{ +#ifdef CONFIG_MMU + int batch; + + /* + * The per-cpu-pages pools are set to around 1000th of the + * size of the zone. But no more than 1/2 of a meg. + * + * OK, so we don't know how big the cache is. So guess. + */ + batch = zone->present_pages / 1024; + if (batch * PAGE_SIZE > 512 * 1024) + batch = (512 * 1024) / PAGE_SIZE; + batch /= 4; /* We effectively *= 4 below */ + if (batch < 1) + batch = 1; + + /* + * Clamp the batch to a 2^n - 1 value. Having a power + * of 2 value was found to be more likely to have + * suboptimal cache aliasing properties in some cases. + * + * For example if 2 tasks are alternately allocating + * batches of pages, one task can end up with a lot + * of pages of one half of the possible page colors + * and the other with pages of the other colors. + */ + batch = rounddown_pow_of_two(batch + batch/2) - 1; + + return batch; + +#else + /* The deferral and batching of frees should be suppressed under NOMMU + * conditions. + * + * The problem is that NOMMU needs to be able to allocate large chunks + * of contiguous memory as there's no hardware page translation to + * assemble apparent contiguous memory from discontiguous pages. + * + * Queueing large contiguous runs of pages for batching, however, + * causes the pages to actually be freed in smaller chunks. As there + * can be a significant delay between the individual batches being + * recycled, this leads to the once large chunks of space being + * fragmented and becoming unavailable for high-order allocations. + */ + return 0; +#endif +} + +static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) +{ + struct per_cpu_pages *pcp; + int migratetype; + + memset(p, 0, sizeof(*p)); + + pcp = &p->pcp; + pcp->count = 0; + pcp->high = 6 * batch; + pcp->batch = max(1UL, 1 * batch); + for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) + INIT_LIST_HEAD(&pcp->lists[migratetype]); +} + +/* + * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist + * to the value high for the pageset p. + */ + +static void setup_pagelist_highmark(struct per_cpu_pageset *p, + unsigned long high) +{ + struct per_cpu_pages *pcp; + + pcp = &p->pcp; + pcp->high = high; + pcp->batch = max(1UL, high/4); + if ((high/4) > (PAGE_SHIFT * 8)) + pcp->batch = PAGE_SHIFT * 8; +} + +static void setup_zone_pageset(struct zone *zone) +{ + int cpu; + + zone->pageset = alloc_percpu(struct per_cpu_pageset); + + for_each_possible_cpu(cpu) { + struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); + + setup_pageset(pcp, zone_batchsize(zone)); + + if (percpu_pagelist_fraction) + setup_pagelist_highmark(pcp, + (zone->present_pages / + percpu_pagelist_fraction)); + } +} + +/* + * Allocate per cpu pagesets and initialize them. + * Before this call only boot pagesets were available. + */ +void __init setup_per_cpu_pageset(void) +{ + struct zone *zone; + + for_each_populated_zone(zone) + setup_zone_pageset(zone); +} + +static noinline __init_refok +int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) +{ + int i; + struct pglist_data *pgdat = zone->zone_pgdat; + size_t alloc_size; + + /* + * The per-page waitqueue mechanism uses hashed waitqueues + * per zone. + */ + zone->wait_table_hash_nr_entries = + wait_table_hash_nr_entries(zone_size_pages); + zone->wait_table_bits = + wait_table_bits(zone->wait_table_hash_nr_entries); + alloc_size = zone->wait_table_hash_nr_entries + * sizeof(wait_queue_head_t); + + if (!slab_is_available()) { + zone->wait_table = (wait_queue_head_t *) + alloc_bootmem_node_nopanic(pgdat, alloc_size); + } else { + /* + * This case means that a zone whose size was 0 gets new memory + * via memory hot-add. + * But it may be the case that a new node was hot-added. In + * this case vmalloc() will not be able to use this new node's + * memory - this wait_table must be initialized to use this new + * node itself as well. + * To use this new node's memory, further consideration will be + * necessary. + */ + zone->wait_table = vmalloc(alloc_size); + } + if (!zone->wait_table) + return -ENOMEM; + + for (i = 0; i < zone->wait_table_hash_nr_entries; ++i) + init_waitqueue_head(zone->wait_table + i); + + return 0; +} + +static int __zone_pcp_update(void *data) +{ + struct zone *zone = data; + int cpu; + unsigned long batch = zone_batchsize(zone), flags; + + for_each_possible_cpu(cpu) { + struct per_cpu_pageset *pset; + struct per_cpu_pages *pcp; + + pset = per_cpu_ptr(zone->pageset, cpu); + pcp = &pset->pcp; + + local_irq_save(flags); + free_pcppages_bulk(zone, pcp->count, pcp); + setup_pageset(pset, batch); + local_irq_restore(flags); + } + return 0; +} + +void zone_pcp_update(struct zone *zone) +{ + stop_machine(__zone_pcp_update, zone, NULL); +} + +static __meminit void zone_pcp_init(struct zone *zone) +{ + /* + * per cpu subsystem is not up at this point. The following code + * relies on the ability of the linker to provide the + * offset of a (static) per cpu variable into the per cpu area. + */ + zone->pageset = &boot_pageset; + + if (zone->present_pages) + printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", + zone->name, zone->present_pages, + zone_batchsize(zone)); +} + +__meminit int init_currently_empty_zone(struct zone *zone, + unsigned long zone_start_pfn, + unsigned long size, + enum memmap_context context) +{ + struct pglist_data *pgdat = zone->zone_pgdat; + int ret; + ret = zone_wait_table_init(zone, size); + if (ret) + return ret; + pgdat->nr_zones = zone_idx(zone) + 1; + + zone->zone_start_pfn = zone_start_pfn; + + mminit_dprintk(MMINIT_TRACE, "memmap_init", + "Initialising map node %d zone %lu pfns %lu -> %lu\n", + pgdat->node_id, + (unsigned long)zone_idx(zone), + zone_start_pfn, (zone_start_pfn + size)); + + zone_init_free_lists(zone); + + return 0; +} + +#ifdef CONFIG_ARCH_POPULATES_NODE_MAP +/* + * Basic iterator support. Return the first range of PFNs for a node + * Note: nid == MAX_NUMNODES returns first region regardless of node + */ +static int __meminit first_active_region_index_in_nid(int nid) +{ + int i; + + for (i = 0; i < nr_nodemap_entries; i++) + if (nid == MAX_NUMNODES || early_node_map[i].nid == nid) + return i; + + return -1; +} + +/* + * Basic iterator support. Return the next active range of PFNs for a node + * Note: nid == MAX_NUMNODES returns next region regardless of node + */ +static int __meminit next_active_region_index_in_nid(int index, int nid) +{ + for (index = index + 1; index < nr_nodemap_entries; index++) + if (nid == MAX_NUMNODES || early_node_map[index].nid == nid) + return index; + + return -1; +} + +#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID +/* + * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. + * Architectures may implement their own version but if add_active_range() + * was used and there are no special requirements, this is a convenient + * alternative + */ +int __meminit __early_pfn_to_nid(unsigned long pfn) +{ + int i; + + for (i = 0; i < nr_nodemap_entries; i++) { + unsigned long start_pfn = early_node_map[i].start_pfn; + unsigned long end_pfn = early_node_map[i].end_pfn; + + if (start_pfn <= pfn && pfn < end_pfn) + return early_node_map[i].nid; + } + /* This is a memory hole */ + return -1; +} +#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ + +int __meminit early_pfn_to_nid(unsigned long pfn) +{ + int nid; + + nid = __early_pfn_to_nid(pfn); + if (nid >= 0) + return nid; + /* just returns 0 */ + return 0; +} + +#ifdef CONFIG_NODES_SPAN_OTHER_NODES +bool __meminit early_pfn_in_nid(unsigned long pfn, int node) +{ + int nid; + + nid = __early_pfn_to_nid(pfn); + if (nid >= 0 && nid != node) + return false; + return true; +} +#endif + +/* Basic iterator support to walk early_node_map[] */ +#define for_each_active_range_index_in_nid(i, nid) \ + for (i = first_active_region_index_in_nid(nid); i != -1; \ + i = next_active_region_index_in_nid(i, nid)) + +/** + * free_bootmem_with_active_regions - Call free_bootmem_node for each active range + * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. + * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node + * + * If an architecture guarantees that all ranges registered with + * add_active_ranges() contain no holes and may be freed, this + * this function may be used instead of calling free_bootmem() manually. + */ +void __init free_bootmem_with_active_regions(int nid, + unsigned long max_low_pfn) +{ + int i; + + for_each_active_range_index_in_nid(i, nid) { + unsigned long size_pages = 0; + unsigned long end_pfn = early_node_map[i].end_pfn; + + if (early_node_map[i].start_pfn >= max_low_pfn) + continue; + + if (end_pfn > max_low_pfn) + end_pfn = max_low_pfn; + + size_pages = end_pfn - early_node_map[i].start_pfn; + free_bootmem_node(NODE_DATA(early_node_map[i].nid), + PFN_PHYS(early_node_map[i].start_pfn), + size_pages << PAGE_SHIFT); + } +} + +#ifdef CONFIG_HAVE_MEMBLOCK +/* + * Basic iterator support. Return the last range of PFNs for a node + * Note: nid == MAX_NUMNODES returns last region regardless of node + */ +static int __meminit last_active_region_index_in_nid(int nid) +{ + int i; + + for (i = nr_nodemap_entries - 1; i >= 0; i--) + if (nid == MAX_NUMNODES || early_node_map[i].nid == nid) + return i; + + return -1; +} + +/* + * Basic iterator support. Return the previous active range of PFNs for a node + * Note: nid == MAX_NUMNODES returns next region regardless of node + */ +static int __meminit previous_active_region_index_in_nid(int index, int nid) +{ + for (index = index - 1; index >= 0; index--) + if (nid == MAX_NUMNODES || early_node_map[index].nid == nid) + return index; + + return -1; +} + +#define for_each_active_range_index_in_nid_reverse(i, nid) \ + for (i = last_active_region_index_in_nid(nid); i != -1; \ + i = previous_active_region_index_in_nid(i, nid)) + +u64 __init find_memory_core_early(int nid, u64 size, u64 align, + u64 goal, u64 limit) +{ + int i; + + /* Need to go over early_node_map to find out good range for node */ + for_each_active_range_index_in_nid_reverse(i, nid) { + u64 addr; + u64 ei_start, ei_last; + u64 final_start, final_end; + + ei_last = early_node_map[i].end_pfn; + ei_last <<= PAGE_SHIFT; + ei_start = early_node_map[i].start_pfn; + ei_start <<= PAGE_SHIFT; + + final_start = max(ei_start, goal); + final_end = min(ei_last, limit); + + if (final_start >= final_end) + continue; + + addr = memblock_find_in_range(final_start, final_end, + size, align); + if (addr == MEMBLOCK_ERROR) + continue; + + return addr; + } + + return MEMBLOCK_ERROR; +} +#endif + +int __init add_from_early_node_map(struct range *range, int az, + int nr_range, int nid) +{ + int i; + u64 start, end; + + /* need to go over early_node_map to find out good range for node */ + for_each_active_range_index_in_nid(i, nid) { + start = early_node_map[i].start_pfn; + end = early_node_map[i].end_pfn; + nr_range = add_range(range, az, nr_range, start, end); + } + return nr_range; +} + +void __init work_with_active_regions(int nid, work_fn_t work_fn, void *data) +{ + int i; + int ret; + + for_each_active_range_index_in_nid(i, nid) { + ret = work_fn(early_node_map[i].start_pfn, + early_node_map[i].end_pfn, data); + if (ret) + break; + } +} +/** + * sparse_memory_present_with_active_regions + * - Call memory_present for each active range + * @nid: The node to call memory_present for. + * If MAX_NUMNODES, all nodes will be used. + * + * If an architecture guarantees that all ranges registered with + * add_active_ranges() contain no holes and may be freed, this + * function may be used instead of calling memory_present() manually. + */ +void __init sparse_memory_present_with_active_regions(int nid) +{ + int i; + + for_each_active_range_index_in_nid(i, nid) + memory_present(early_node_map[i].nid, + early_node_map[i].start_pfn, + early_node_map[i].end_pfn); +} + +/** + * get_pfn_range_for_nid - Return the start and end page frames for a node + * @nid: The nid to return the range for. + * If MAX_NUMNODES, the min and max PFN are returned. + * @start_pfn: Passed by reference. On return, it will have the node start_pfn. + * @end_pfn: Passed by reference. On return, it will have the node end_pfn. + * + * It returns the start and end page frame of a node based on information + * provided by an arch calling add_active_range(). If called for a node + * with no available memory, a warning is printed and the start and end + * PFNs will be 0. + */ +void __meminit get_pfn_range_for_nid(unsigned int nid, + unsigned long *start_pfn, unsigned long *end_pfn) +{ + int i; + *start_pfn = -1UL; + *end_pfn = 0; + + for_each_active_range_index_in_nid(i, nid) { + *start_pfn = min(*start_pfn, early_node_map[i].start_pfn); + *end_pfn = max(*end_pfn, early_node_map[i].end_pfn); + } + + if (*start_pfn == -1UL) + *start_pfn = 0; +} + +/* + * This finds a zone that can be used for ZONE_MOVABLE pages. The + * assumption is made that zones within a node are ordered in monotonic + * increasing memory addresses so that the "highest" populated zone is used + */ +static void __init find_usable_zone_for_movable(void) +{ + int zone_index; + for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { + if (zone_index == ZONE_MOVABLE) + continue; + + if (arch_zone_highest_possible_pfn[zone_index] > + arch_zone_lowest_possible_pfn[zone_index]) + break; + } + + VM_BUG_ON(zone_index == -1); + movable_zone = zone_index; +} + +/* + * The zone ranges provided by the architecture do not include ZONE_MOVABLE + * because it is sized independent of architecture. Unlike the other zones, + * the starting point for ZONE_MOVABLE is not fixed. It may be different + * in each node depending on the size of each node and how evenly kernelcore + * is distributed. This helper function adjusts the zone ranges + * provided by the architecture for a given node by using the end of the + * highest usable zone for ZONE_MOVABLE. This preserves the assumption that + * zones within a node are in order of monotonic increases memory addresses + */ +static void __meminit adjust_zone_range_for_zone_movable(int nid, + unsigned long zone_type, + unsigned long node_start_pfn, + unsigned long node_end_pfn, + unsigned long *zone_start_pfn, + unsigned long *zone_end_pfn) +{ + /* Only adjust if ZONE_MOVABLE is on this node */ + if (zone_movable_pfn[nid]) { + /* Size ZONE_MOVABLE */ + if (zone_type == ZONE_MOVABLE) { + *zone_start_pfn = zone_movable_pfn[nid]; + *zone_end_pfn = min(node_end_pfn, + arch_zone_highest_possible_pfn[movable_zone]); + + /* Adjust for ZONE_MOVABLE starting within this range */ + } else if (*zone_start_pfn < zone_movable_pfn[nid] && + *zone_end_pfn > zone_movable_pfn[nid]) { + *zone_end_pfn = zone_movable_pfn[nid]; + + /* Check if this whole range is within ZONE_MOVABLE */ + } else if (*zone_start_pfn >= zone_movable_pfn[nid]) + *zone_start_pfn = *zone_end_pfn; + } +} + +/* + * Return the number of pages a zone spans in a node, including holes + * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() + */ +static unsigned long __meminit zone_spanned_pages_in_node(int nid, + unsigned long zone_type, + unsigned long *ignored) +{ + unsigned long node_start_pfn, node_end_pfn; + unsigned long zone_start_pfn, zone_end_pfn; + + /* Get the start and end of the node and zone */ + get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn); + zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; + zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; + adjust_zone_range_for_zone_movable(nid, zone_type, + node_start_pfn, node_end_pfn, + &zone_start_pfn, &zone_end_pfn); + + /* Check that this node has pages within the zone's required range */ + if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn) + return 0; + + /* Move the zone boundaries inside the node if necessary */ + zone_end_pfn = min(zone_end_pfn, node_end_pfn); + zone_start_pfn = max(zone_start_pfn, node_start_pfn); + + /* Return the spanned pages */ + return zone_end_pfn - zone_start_pfn; +} + +/* + * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, + * then all holes in the requested range will be accounted for. + */ +unsigned long __meminit __absent_pages_in_range(int nid, + unsigned long range_start_pfn, + unsigned long range_end_pfn) +{ + int i = 0; + unsigned long prev_end_pfn = 0, hole_pages = 0; + unsigned long start_pfn; + + /* Find the end_pfn of the first active range of pfns in the node */ + i = first_active_region_index_in_nid(nid); + if (i == -1) + return 0; + + prev_end_pfn = min(early_node_map[i].start_pfn, range_end_pfn); + + /* Account for ranges before physical memory on this node */ + if (early_node_map[i].start_pfn > range_start_pfn) + hole_pages = prev_end_pfn - range_start_pfn; + + /* Find all holes for the zone within the node */ + for (; i != -1; i = next_active_region_index_in_nid(i, nid)) { + + /* No need to continue if prev_end_pfn is outside the zone */ + if (prev_end_pfn >= range_end_pfn) + break; + + /* Make sure the end of the zone is not within the hole */ + start_pfn = min(early_node_map[i].start_pfn, range_end_pfn); + prev_end_pfn = max(prev_end_pfn, range_start_pfn); + + /* Update the hole size cound and move on */ + if (start_pfn > range_start_pfn) { + BUG_ON(prev_end_pfn > start_pfn); + hole_pages += start_pfn - prev_end_pfn; + } + prev_end_pfn = early_node_map[i].end_pfn; + } + + /* Account for ranges past physical memory on this node */ + if (range_end_pfn > prev_end_pfn) + hole_pages += range_end_pfn - + max(range_start_pfn, prev_end_pfn); + + return hole_pages; +} + +/** + * absent_pages_in_range - Return number of page frames in holes within a range + * @start_pfn: The start PFN to start searching for holes + * @end_pfn: The end PFN to stop searching for holes + * + * It returns the number of pages frames in memory holes within a range. + */ +unsigned long __init absent_pages_in_range(unsigned long start_pfn, + unsigned long end_pfn) +{ + return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); +} + +/* Return the number of page frames in holes in a zone on a node */ +static unsigned long __meminit zone_absent_pages_in_node(int nid, + unsigned long zone_type, + unsigned long *ignored) +{ + unsigned long node_start_pfn, node_end_pfn; + unsigned long zone_start_pfn, zone_end_pfn; + + get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn); + zone_start_pfn = max(arch_zone_lowest_possible_pfn[zone_type], + node_start_pfn); + zone_end_pfn = min(arch_zone_highest_possible_pfn[zone_type], + node_end_pfn); + + adjust_zone_range_for_zone_movable(nid, zone_type, + node_start_pfn, node_end_pfn, + &zone_start_pfn, &zone_end_pfn); + return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); +} + +#else +static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, + unsigned long zone_type, + unsigned long *zones_size) +{ + return zones_size[zone_type]; +} + +static inline unsigned long __meminit zone_absent_pages_in_node(int nid, + unsigned long zone_type, + unsigned long *zholes_size) +{ + if (!zholes_size) + return 0; + + return zholes_size[zone_type]; +} + +#endif + +static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, + unsigned long *zones_size, unsigned long *zholes_size) +{ + unsigned long realtotalpages, totalpages = 0; + enum zone_type i; + + for (i = 0; i < MAX_NR_ZONES; i++) + totalpages += zone_spanned_pages_in_node(pgdat->node_id, i, + zones_size); + pgdat->node_spanned_pages = totalpages; + + realtotalpages = totalpages; + for (i = 0; i < MAX_NR_ZONES; i++) + realtotalpages -= + zone_absent_pages_in_node(pgdat->node_id, i, + zholes_size); + pgdat->node_present_pages = realtotalpages; + printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, + realtotalpages); +} + +#ifndef CONFIG_SPARSEMEM +/* + * Calculate the size of the zone->blockflags rounded to an unsigned long + * Start by making sure zonesize is a multiple of pageblock_order by rounding + * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally + * round what is now in bits to nearest long in bits, then return it in + * bytes. + */ +static unsigned long __init usemap_size(unsigned long zonesize) +{ + unsigned long usemapsize; + + usemapsize = roundup(zonesize, pageblock_nr_pages); + usemapsize = usemapsize >> pageblock_order; + usemapsize *= NR_PAGEBLOCK_BITS; + usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); + + return usemapsize / 8; +} + +static void __init setup_usemap(struct pglist_data *pgdat, + struct zone *zone, unsigned long zonesize) +{ + unsigned long usemapsize = usemap_size(zonesize); + zone->pageblock_flags = NULL; + if (usemapsize) + zone->pageblock_flags = alloc_bootmem_node_nopanic(pgdat, + usemapsize); +} +#else +static inline void setup_usemap(struct pglist_data *pgdat, + struct zone *zone, unsigned long zonesize) {} +#endif /* CONFIG_SPARSEMEM */ + +#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE + +/* Return a sensible default order for the pageblock size. */ +static inline int pageblock_default_order(void) +{ + if (HPAGE_SHIFT > PAGE_SHIFT) + return HUGETLB_PAGE_ORDER; + + return MAX_ORDER-1; +} + +/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ +static inline void __init set_pageblock_order(unsigned int order) +{ + /* Check that pageblock_nr_pages has not already been setup */ + if (pageblock_order) + return; + + /* + * Assume the largest contiguous order of interest is a huge page. + * This value may be variable depending on boot parameters on IA64 + */ + pageblock_order = order; +} +#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ + +/* + * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() + * and pageblock_default_order() are unused as pageblock_order is set + * at compile-time. See include/linux/pageblock-flags.h for the values of + * pageblock_order based on the kernel config + */ +static inline int pageblock_default_order(unsigned int order) +{ + return MAX_ORDER-1; +} +#define set_pageblock_order(x) do {} while (0) + +#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ + +/* + * Set up the zone data structures: + * - mark all pages reserved + * - mark all memory queues empty + * - clear the memory bitmaps + */ +static void __paginginit free_area_init_core(struct pglist_data *pgdat, + unsigned long *zones_size, unsigned long *zholes_size) +{ + enum zone_type j; + int nid = pgdat->node_id; + unsigned long zone_start_pfn = pgdat->node_start_pfn; + int ret; + + pgdat_resize_init(pgdat); + pgdat->nr_zones = 0; + init_waitqueue_head(&pgdat->kswapd_wait); + pgdat->kswapd_max_order = 0; + pgdat_page_cgroup_init(pgdat); + + for (j = 0; j < MAX_NR_ZONES; j++) { + struct zone *zone = pgdat->node_zones + j; + unsigned long size, realsize, memmap_pages; + enum lru_list l; + + size = zone_spanned_pages_in_node(nid, j, zones_size); + realsize = size - zone_absent_pages_in_node(nid, j, + zholes_size); + + /* + * Adjust realsize so that it accounts for how much memory + * is used by this zone for memmap. This affects the watermark + * and per-cpu initialisations + */ + memmap_pages = + PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT; + if (realsize >= memmap_pages) { + realsize -= memmap_pages; + if (memmap_pages) + printk(KERN_DEBUG + " %s zone: %lu pages used for memmap\n", + zone_names[j], memmap_pages); + } else + printk(KERN_WARNING + " %s zone: %lu pages exceeds realsize %lu\n", + zone_names[j], memmap_pages, realsize); + + /* Account for reserved pages */ + if (j == 0 && realsize > dma_reserve) { + realsize -= dma_reserve; + printk(KERN_DEBUG " %s zone: %lu pages reserved\n", + zone_names[0], dma_reserve); + } + + if (!is_highmem_idx(j)) + nr_kernel_pages += realsize; + nr_all_pages += realsize; + + zone->spanned_pages = size; + zone->present_pages = realsize; +#ifdef CONFIG_NUMA + zone->node = nid; + zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio) + / 100; + zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100; +#endif + zone->name = zone_names[j]; + spin_lock_init(&zone->lock); + spin_lock_init(&zone->lru_lock); + zone_seqlock_init(zone); + zone->zone_pgdat = pgdat; + + zone_pcp_init(zone); + for_each_lru(l) + INIT_LIST_HEAD(&zone->lru[l].list); + zone->reclaim_stat.recent_rotated[0] = 0; + zone->reclaim_stat.recent_rotated[1] = 0; + zone->reclaim_stat.recent_scanned[0] = 0; + zone->reclaim_stat.recent_scanned[1] = 0; + zap_zone_vm_stats(zone); + zone->flags = 0; + if (!size) + continue; + + set_pageblock_order(pageblock_default_order()); + setup_usemap(pgdat, zone, size); + ret = init_currently_empty_zone(zone, zone_start_pfn, + size, MEMMAP_EARLY); + BUG_ON(ret); + memmap_init(size, nid, j, zone_start_pfn); + zone_start_pfn += size; + } +} + +static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat) +{ + /* Skip empty nodes */ + if (!pgdat->node_spanned_pages) + return; + +#ifdef CONFIG_FLAT_NODE_MEM_MAP + /* ia64 gets its own node_mem_map, before this, without bootmem */ + if (!pgdat->node_mem_map) { + unsigned long size, start, end; + struct page *map; + + /* + * The zone's endpoints aren't required to be MAX_ORDER + * aligned but the node_mem_map endpoints must be in order + * for the buddy allocator to function correctly. + */ + start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); + end = pgdat->node_start_pfn + pgdat->node_spanned_pages; + end = ALIGN(end, MAX_ORDER_NR_PAGES); + size = (end - start) * sizeof(struct page); + map = alloc_remap(pgdat->node_id, size); + if (!map) + map = alloc_bootmem_node_nopanic(pgdat, size); + pgdat->node_mem_map = map + (pgdat->node_start_pfn - start); + } +#ifndef CONFIG_NEED_MULTIPLE_NODES + /* + * With no DISCONTIG, the global mem_map is just set as node 0's + */ + if (pgdat == NODE_DATA(0)) { + mem_map = NODE_DATA(0)->node_mem_map; +#ifdef CONFIG_ARCH_POPULATES_NODE_MAP + if (page_to_pfn(mem_map) != pgdat->node_start_pfn) + mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET); +#endif /* CONFIG_ARCH_POPULATES_NODE_MAP */ + } +#endif +#endif /* CONFIG_FLAT_NODE_MEM_MAP */ +} + +void __paginginit free_area_init_node(int nid, unsigned long *zones_size, + unsigned long node_start_pfn, unsigned long *zholes_size) +{ + pg_data_t *pgdat = NODE_DATA(nid); + + pgdat->node_id = nid; + pgdat->node_start_pfn = node_start_pfn; + calculate_node_totalpages(pgdat, zones_size, zholes_size); + + alloc_node_mem_map(pgdat); +#ifdef CONFIG_FLAT_NODE_MEM_MAP + printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx," + " node_mem_map %08lx\n", + nid, (unsigned long)pgdat, + (unsigned long)pgdat->node_mem_map); +#endif + + free_area_init_core(pgdat, zones_size, zholes_size); +} + +#ifdef CONFIG_ARCH_POPULATES_NODE_MAP + +#if MAX_NUMNODES > 1 +/* + * Figure out the number of possible node ids. + */ +static void __init setup_nr_node_ids(void) +{ + unsigned int node; + unsigned int highest = 0; + + for_each_node_mask(node, node_possible_map) + highest = node; + nr_node_ids = highest + 1; +} +#else +static inline void setup_nr_node_ids(void) +{ +} +#endif + +/** + * add_active_range - Register a range of PFNs backed by physical memory + * @nid: The node ID the range resides on + * @start_pfn: The start PFN of the available physical memory + * @end_pfn: The end PFN of the available physical memory + * + * These ranges are stored in an early_node_map[] and later used by + * free_area_init_nodes() to calculate zone sizes and holes. If the + * range spans a memory hole, it is up to the architecture to ensure + * the memory is not freed by the bootmem allocator. If possible + * the range being registered will be merged with existing ranges. + */ +void __init add_active_range(unsigned int nid, unsigned long start_pfn, + unsigned long end_pfn) +{ + int i; + + mminit_dprintk(MMINIT_TRACE, "memory_register", + "Entering add_active_range(%d, %#lx, %#lx) " + "%d entries of %d used\n", + nid, start_pfn, end_pfn, + nr_nodemap_entries, MAX_ACTIVE_REGIONS); + + mminit_validate_memmodel_limits(&start_pfn, &end_pfn); + + /* Merge with existing active regions if possible */ + for (i = 0; i < nr_nodemap_entries; i++) { + if (early_node_map[i].nid != nid) + continue; + + /* Skip if an existing region covers this new one */ + if (start_pfn >= early_node_map[i].start_pfn && + end_pfn <= early_node_map[i].end_pfn) + return; + + /* Merge forward if suitable */ + if (start_pfn <= early_node_map[i].end_pfn && + end_pfn > early_node_map[i].end_pfn) { + early_node_map[i].end_pfn = end_pfn; + return; + } + + /* Merge backward if suitable */ + if (start_pfn < early_node_map[i].start_pfn && + end_pfn >= early_node_map[i].start_pfn) { + early_node_map[i].start_pfn = start_pfn; + return; + } + } + + /* Check that early_node_map is large enough */ + if (i >= MAX_ACTIVE_REGIONS) { + printk(KERN_CRIT "More than %d memory regions, truncating\n", + MAX_ACTIVE_REGIONS); + return; + } + + early_node_map[i].nid = nid; + early_node_map[i].start_pfn = start_pfn; + early_node_map[i].end_pfn = end_pfn; + nr_nodemap_entries = i + 1; +} + +/** + * remove_active_range - Shrink an existing registered range of PFNs + * @nid: The node id the range is on that should be shrunk + * @start_pfn: The new PFN of the range + * @end_pfn: The new PFN of the range + * + * i386 with NUMA use alloc_remap() to store a node_mem_map on a local node. + * The map is kept near the end physical page range that has already been + * registered. This function allows an arch to shrink an existing registered + * range. + */ +void __init remove_active_range(unsigned int nid, unsigned long start_pfn, + unsigned long end_pfn) +{ + int i, j; + int removed = 0; + + printk(KERN_DEBUG "remove_active_range (%d, %lu, %lu)\n", + nid, start_pfn, end_pfn); + + /* Find the old active region end and shrink */ + for_each_active_range_index_in_nid(i, nid) { + if (early_node_map[i].start_pfn >= start_pfn && + early_node_map[i].end_pfn <= end_pfn) { + /* clear it */ + early_node_map[i].start_pfn = 0; + early_node_map[i].end_pfn = 0; + removed = 1; + continue; + } + if (early_node_map[i].start_pfn < start_pfn && + early_node_map[i].end_pfn > start_pfn) { + unsigned long temp_end_pfn = early_node_map[i].end_pfn; + early_node_map[i].end_pfn = start_pfn; + if (temp_end_pfn > end_pfn) + add_active_range(nid, end_pfn, temp_end_pfn); + continue; + } + if (early_node_map[i].start_pfn >= start_pfn && + early_node_map[i].end_pfn > end_pfn && + early_node_map[i].start_pfn < end_pfn) { + early_node_map[i].start_pfn = end_pfn; + continue; + } + } + + if (!removed) + return; + + /* remove the blank ones */ + for (i = nr_nodemap_entries - 1; i > 0; i--) { + if (early_node_map[i].nid != nid) + continue; + if (early_node_map[i].end_pfn) + continue; + /* we found it, get rid of it */ + for (j = i; j < nr_nodemap_entries - 1; j++) + memcpy(&early_node_map[j], &early_node_map[j+1], + sizeof(early_node_map[j])); + j = nr_nodemap_entries - 1; + memset(&early_node_map[j], 0, sizeof(early_node_map[j])); + nr_nodemap_entries--; + } +} + +/** + * remove_all_active_ranges - Remove all currently registered regions + * + * During discovery, it may be found that a table like SRAT is invalid + * and an alternative discovery method must be used. This function removes + * all currently registered regions. + */ +void __init remove_all_active_ranges(void) +{ + memset(early_node_map, 0, sizeof(early_node_map)); + nr_nodemap_entries = 0; +} + +/* Compare two active node_active_regions */ +static int __init cmp_node_active_region(const void *a, const void *b) +{ + struct node_active_region *arange = (struct node_active_region *)a; + struct node_active_region *brange = (struct node_active_region *)b; + + /* Done this way to avoid overflows */ + if (arange->start_pfn > brange->start_pfn) + return 1; + if (arange->start_pfn < brange->start_pfn) + return -1; + + return 0; +} + +/* sort the node_map by start_pfn */ +void __init sort_node_map(void) +{ + sort(early_node_map, (size_t)nr_nodemap_entries, + sizeof(struct node_active_region), + cmp_node_active_region, NULL); +} + +/* Find the lowest pfn for a node */ +static unsigned long __init find_min_pfn_for_node(int nid) +{ + int i; + unsigned long min_pfn = ULONG_MAX; + + /* Assuming a sorted map, the first range found has the starting pfn */ + for_each_active_range_index_in_nid(i, nid) + min_pfn = min(min_pfn, early_node_map[i].start_pfn); + + if (min_pfn == ULONG_MAX) { + printk(KERN_WARNING + "Could not find start_pfn for node %d\n", nid); + return 0; + } + + return min_pfn; +} + +/** + * find_min_pfn_with_active_regions - Find the minimum PFN registered + * + * It returns the minimum PFN based on information provided via + * add_active_range(). + */ +unsigned long __init find_min_pfn_with_active_regions(void) +{ + return find_min_pfn_for_node(MAX_NUMNODES); +} + +/* + * early_calculate_totalpages() + * Sum pages in active regions for movable zone. + * Populate N_HIGH_MEMORY for calculating usable_nodes. + */ +static unsigned long __init early_calculate_totalpages(void) +{ + int i; + unsigned long totalpages = 0; + + for (i = 0; i < nr_nodemap_entries; i++) { + unsigned long pages = early_node_map[i].end_pfn - + early_node_map[i].start_pfn; + totalpages += pages; + if (pages) + node_set_state(early_node_map[i].nid, N_HIGH_MEMORY); + } + return totalpages; +} + +/* + * Find the PFN the Movable zone begins in each node. Kernel memory + * is spread evenly between nodes as long as the nodes have enough + * memory. When they don't, some nodes will have more kernelcore than + * others + */ +static void __init find_zone_movable_pfns_for_nodes(unsigned long *movable_pfn) +{ + int i, nid; + unsigned long usable_startpfn; + unsigned long kernelcore_node, kernelcore_remaining; + /* save the state before borrow the nodemask */ + nodemask_t saved_node_state = node_states[N_HIGH_MEMORY]; + unsigned long totalpages = early_calculate_totalpages(); + int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]); + + /* + * If movablecore was specified, calculate what size of + * kernelcore that corresponds so that memory usable for + * any allocation type is evenly spread. If both kernelcore + * and movablecore are specified, then the value of kernelcore + * will be used for required_kernelcore if it's greater than + * what movablecore would have allowed. + */ + if (required_movablecore) { + unsigned long corepages; + + /* + * Round-up so that ZONE_MOVABLE is at least as large as what + * was requested by the user + */ + required_movablecore = + roundup(required_movablecore, MAX_ORDER_NR_PAGES); + corepages = totalpages - required_movablecore; + + required_kernelcore = max(required_kernelcore, corepages); + } + + /* If kernelcore was not specified, there is no ZONE_MOVABLE */ + if (!required_kernelcore) + goto out; + + /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ + find_usable_zone_for_movable(); + usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; + +restart: + /* Spread kernelcore memory as evenly as possible throughout nodes */ + kernelcore_node = required_kernelcore / usable_nodes; + for_each_node_state(nid, N_HIGH_MEMORY) { + /* + * Recalculate kernelcore_node if the division per node + * now exceeds what is necessary to satisfy the requested + * amount of memory for the kernel + */ + if (required_kernelcore < kernelcore_node) + kernelcore_node = required_kernelcore / usable_nodes; + + /* + * As the map is walked, we track how much memory is usable + * by the kernel using kernelcore_remaining. When it is + * 0, the rest of the node is usable by ZONE_MOVABLE + */ + kernelcore_remaining = kernelcore_node; + + /* Go through each range of PFNs within this node */ + for_each_active_range_index_in_nid(i, nid) { + unsigned long start_pfn, end_pfn; + unsigned long size_pages; + + start_pfn = max(early_node_map[i].start_pfn, + zone_movable_pfn[nid]); + end_pfn = early_node_map[i].end_pfn; + if (start_pfn >= end_pfn) + continue; + + /* Account for what is only usable for kernelcore */ + if (start_pfn < usable_startpfn) { + unsigned long kernel_pages; + kernel_pages = min(end_pfn, usable_startpfn) + - start_pfn; + + kernelcore_remaining -= min(kernel_pages, + kernelcore_remaining); + required_kernelcore -= min(kernel_pages, + required_kernelcore); + + /* Continue if range is now fully accounted */ + if (end_pfn <= usable_startpfn) { + + /* + * Push zone_movable_pfn to the end so + * that if we have to rebalance + * kernelcore across nodes, we will + * not double account here + */ + zone_movable_pfn[nid] = end_pfn; + continue; + } + start_pfn = usable_startpfn; + } + + /* + * The usable PFN range for ZONE_MOVABLE is from + * start_pfn->end_pfn. Calculate size_pages as the + * number of pages used as kernelcore + */ + size_pages = end_pfn - start_pfn; + if (size_pages > kernelcore_remaining) + size_pages = kernelcore_remaining; + zone_movable_pfn[nid] = start_pfn + size_pages; + + /* + * Some kernelcore has been met, update counts and + * break if the kernelcore for this node has been + * satisified + */ + required_kernelcore -= min(required_kernelcore, + size_pages); + kernelcore_remaining -= size_pages; + if (!kernelcore_remaining) + break; + } + } + + /* + * If there is still required_kernelcore, we do another pass with one + * less node in the count. This will push zone_movable_pfn[nid] further + * along on the nodes that still have memory until kernelcore is + * satisified + */ + usable_nodes--; + if (usable_nodes && required_kernelcore > usable_nodes) + goto restart; + + /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ + for (nid = 0; nid < MAX_NUMNODES; nid++) + zone_movable_pfn[nid] = + roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); + +out: + /* restore the node_state */ + node_states[N_HIGH_MEMORY] = saved_node_state; +} + +/* Any regular memory on that node ? */ +static void check_for_regular_memory(pg_data_t *pgdat) +{ +#ifdef CONFIG_HIGHMEM + enum zone_type zone_type; + + for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) { + struct zone *zone = &pgdat->node_zones[zone_type]; + if (zone->present_pages) + node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY); + } +#endif +} + +/** + * free_area_init_nodes - Initialise all pg_data_t and zone data + * @max_zone_pfn: an array of max PFNs for each zone + * + * This will call free_area_init_node() for each active node in the system. + * Using the page ranges provided by add_active_range(), the size of each + * zone in each node and their holes is calculated. If the maximum PFN + * between two adjacent zones match, it is assumed that the zone is empty. + * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed + * that arch_max_dma32_pfn has no pages. It is also assumed that a zone + * starts where the previous one ended. For example, ZONE_DMA32 starts + * at arch_max_dma_pfn. + */ +void __init free_area_init_nodes(unsigned long *max_zone_pfn) +{ + unsigned long nid; + int i; + + /* Sort early_node_map as initialisation assumes it is sorted */ + sort_node_map(); + + /* Record where the zone boundaries are */ + memset(arch_zone_lowest_possible_pfn, 0, + sizeof(arch_zone_lowest_possible_pfn)); + memset(arch_zone_highest_possible_pfn, 0, + sizeof(arch_zone_highest_possible_pfn)); + arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions(); + arch_zone_highest_possible_pfn[0] = max_zone_pfn[0]; + for (i = 1; i < MAX_NR_ZONES; i++) { + if (i == ZONE_MOVABLE) + continue; + arch_zone_lowest_possible_pfn[i] = + arch_zone_highest_possible_pfn[i-1]; + arch_zone_highest_possible_pfn[i] = + max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]); + } + arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0; + arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0; + + /* Find the PFNs that ZONE_MOVABLE begins at in each node */ + memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); + find_zone_movable_pfns_for_nodes(zone_movable_pfn); + + /* Print out the zone ranges */ + printk(KERN_INFO "Zone PFN ranges:\n"); + for (i = 0; i < MAX_NR_ZONES; i++) { + if (i == ZONE_MOVABLE) + continue; + printk(KERN_INFO " %-8s ", zone_names[i]); + if (arch_zone_lowest_possible_pfn[i] == + arch_zone_highest_possible_pfn[i]) + printk(KERN_INFO "empty\n"); + else + printk(KERN_INFO "%0#10lx -> %0#10lx\n", + arch_zone_lowest_possible_pfn[i], + arch_zone_highest_possible_pfn[i]); + } + + /* Print out the PFNs ZONE_MOVABLE begins at in each node */ + printk(KERN_INFO "Movable zone start PFN for each node\n"); + for (i = 0; i < MAX_NUMNODES; i++) { + if (zone_movable_pfn[i]) + printk(KERN_INFO " Node %d: %lu\n", i, + zone_movable_pfn[i]); + } + + /* Print out the early_node_map[] */ + printk(KERN_INFO "early_node_map[%d] active PFN ranges\n", + nr_nodemap_entries); + for (i = 0; i < nr_nodemap_entries; i++) + printk(KERN_INFO " %3d: %0#10lx -> %0#10lx\n", + early_node_map[i].nid, + early_node_map[i].start_pfn, + early_node_map[i].end_pfn); + + /* Initialise every node */ + mminit_verify_pageflags_layout(); + setup_nr_node_ids(); + for_each_online_node(nid) { + pg_data_t *pgdat = NODE_DATA(nid); + free_area_init_node(nid, NULL, + find_min_pfn_for_node(nid), NULL); + + /* Any memory on that node */ + if (pgdat->node_present_pages) + node_set_state(nid, N_HIGH_MEMORY); + check_for_regular_memory(pgdat); + } +} + +static int __init cmdline_parse_core(char *p, unsigned long *core) +{ + unsigned long long coremem; + if (!p) + return -EINVAL; + + coremem = memparse(p, &p); + *core = coremem >> PAGE_SHIFT; + + /* Paranoid check that UL is enough for the coremem value */ + WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); + + return 0; +} + +/* + * kernelcore=size sets the amount of memory for use for allocations that + * cannot be reclaimed or migrated. + */ +static int __init cmdline_parse_kernelcore(char *p) +{ + return cmdline_parse_core(p, &required_kernelcore); +} + +/* + * movablecore=size sets the amount of memory for use for allocations that + * can be reclaimed or migrated. + */ +static int __init cmdline_parse_movablecore(char *p) +{ + return cmdline_parse_core(p, &required_movablecore); +} + +early_param("kernelcore", cmdline_parse_kernelcore); +early_param("movablecore", cmdline_parse_movablecore); + +#endif /* CONFIG_ARCH_POPULATES_NODE_MAP */ + +/** + * set_dma_reserve - set the specified number of pages reserved in the first zone + * @new_dma_reserve: The number of pages to mark reserved + * + * The per-cpu batchsize and zone watermarks are determined by present_pages. + * In the DMA zone, a significant percentage may be consumed by kernel image + * and other unfreeable allocations which can skew the watermarks badly. This + * function may optionally be used to account for unfreeable pages in the + * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and + * smaller per-cpu batchsize. + */ +void __init set_dma_reserve(unsigned long new_dma_reserve) +{ + dma_reserve = new_dma_reserve; +} + +void __init free_area_init(unsigned long *zones_size) +{ + free_area_init_node(0, zones_size, + __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); +} + +static int page_alloc_cpu_notify(struct notifier_block *self, + unsigned long action, void *hcpu) +{ + int cpu = (unsigned long)hcpu; + + if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { + drain_pages(cpu); + + /* + * Spill the event counters of the dead processor + * into the current processors event counters. + * This artificially elevates the count of the current + * processor. + */ + vm_events_fold_cpu(cpu); + + /* + * Zero the differential counters of the dead processor + * so that the vm statistics are consistent. + * + * This is only okay since the processor is dead and cannot + * race with what we are doing. + */ + refresh_cpu_vm_stats(cpu); + } + return NOTIFY_OK; +} + +void __init page_alloc_init(void) +{ + hotcpu_notifier(page_alloc_cpu_notify, 0); + init_rwsem(&page_alloc_slow_rwsem); +} + +/* + * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio + * or min_free_kbytes changes. + */ +static void calculate_totalreserve_pages(void) +{ + struct pglist_data *pgdat; + unsigned long reserve_pages = 0; + enum zone_type i, j; + + for_each_online_pgdat(pgdat) { + for (i = 0; i < MAX_NR_ZONES; i++) { + struct zone *zone = pgdat->node_zones + i; + unsigned long max = 0; + + /* Find valid and maximum lowmem_reserve in the zone */ + for (j = i; j < MAX_NR_ZONES; j++) { + if (zone->lowmem_reserve[j] > max) + max = zone->lowmem_reserve[j]; + } + + /* we treat the high watermark as reserved pages. */ + max += high_wmark_pages(zone); + + if (max > zone->present_pages) + max = zone->present_pages; + reserve_pages += max; + } + } + totalreserve_pages = reserve_pages; +} + +/* + * setup_per_zone_lowmem_reserve - called whenever + * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone + * has a correct pages reserved value, so an adequate number of + * pages are left in the zone after a successful __alloc_pages(). + */ +static void setup_per_zone_lowmem_reserve(void) +{ + struct pglist_data *pgdat; + enum zone_type j, idx; + + for_each_online_pgdat(pgdat) { + for (j = 0; j < MAX_NR_ZONES; j++) { + struct zone *zone = pgdat->node_zones + j; + unsigned long present_pages = zone->present_pages; + + zone->lowmem_reserve[j] = 0; + + idx = j; + while (idx) { + struct zone *lower_zone; + + idx--; + + if (sysctl_lowmem_reserve_ratio[idx] < 1) + sysctl_lowmem_reserve_ratio[idx] = 1; + + lower_zone = pgdat->node_zones + idx; + lower_zone->lowmem_reserve[j] = present_pages / + sysctl_lowmem_reserve_ratio[idx]; + present_pages += lower_zone->present_pages; + } + } + } + + /* update totalreserve_pages */ + calculate_totalreserve_pages(); +} + +static void __setup_per_zone_wmarks(void) +{ + unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); + unsigned long lowmem_pages = 0; + struct zone *zone; + unsigned long flags; + + /* Calculate total number of !ZONE_HIGHMEM pages */ + for_each_zone(zone) { + if (!is_highmem(zone)) + lowmem_pages += zone->present_pages; + } + + for_each_zone(zone) { + u64 tmp; + + spin_lock_irqsave(&zone->lock, flags); + tmp = (u64)pages_min * zone->present_pages; + do_div(tmp, lowmem_pages); + if (is_highmem(zone)) { + /* + * __GFP_HIGH and PF_MEMALLOC allocations usually don't + * need highmem pages, so cap pages_min to a small + * value here. + * + * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) + * deltas controls asynch page reclaim, and so should + * not be capped for highmem. + */ + int min_pages; + + min_pages = zone->present_pages / 1024; + if (min_pages < SWAP_CLUSTER_MAX) + min_pages = SWAP_CLUSTER_MAX; + if (min_pages > 128) + min_pages = 128; + zone->watermark[WMARK_MIN] = min_pages; + } else { + /* + * If it's a lowmem zone, reserve a number of pages + * proportionate to the zone's size. + */ + zone->watermark[WMARK_MIN] = tmp; + } + + zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + + (tmp >> 2); + zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + + (tmp >> 1); + + zone->watermark[WMARK_MIN] += cma_wmark_pages(zone); + zone->watermark[WMARK_LOW] += cma_wmark_pages(zone); + zone->watermark[WMARK_HIGH] += cma_wmark_pages(zone); + + setup_zone_migrate_reserve(zone); + spin_unlock_irqrestore(&zone->lock, flags); + } + + /* update totalreserve_pages */ + calculate_totalreserve_pages(); +} + +/** + * setup_per_zone_wmarks - called when min_free_kbytes changes + * or when memory is hot-{added|removed} + * + * Ensures that the watermark[min,low,high] values for each zone are set + * correctly with respect to min_free_kbytes. + */ +void setup_per_zone_wmarks(void) +{ + mutex_lock(&zonelists_mutex); + __setup_per_zone_wmarks(); + mutex_unlock(&zonelists_mutex); +} + +/* + * The inactive anon list should be small enough that the VM never has to + * do too much work, but large enough that each inactive page has a chance + * to be referenced again before it is swapped out. + * + * The inactive_anon ratio is the target ratio of ACTIVE_ANON to + * INACTIVE_ANON pages on this zone's LRU, maintained by the + * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of + * the anonymous pages are kept on the inactive list. + * + * total target max + * memory ratio inactive anon + * ------------------------------------- + * 10MB 1 5MB + * 100MB 1 50MB + * 1GB 3 250MB + * 10GB 10 0.9GB + * 100GB 31 3GB + * 1TB 101 10GB + * 10TB 320 32GB + */ +static void __meminit calculate_zone_inactive_ratio(struct zone *zone) +{ + unsigned int gb, ratio; + + /* Zone size in gigabytes */ + gb = zone->present_pages >> (30 - PAGE_SHIFT); + if (gb) + ratio = int_sqrt(10 * gb); + else + ratio = 1; + + zone->inactive_ratio = ratio; +} + +static void __meminit setup_per_zone_inactive_ratio(void) +{ + struct zone *zone; + + for_each_zone(zone) + calculate_zone_inactive_ratio(zone); +} + +/* + * Initialise min_free_kbytes. + * + * For small machines we want it small (128k min). For large machines + * we want it large (64MB max). But it is not linear, because network + * bandwidth does not increase linearly with machine size. We use + * + * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: + * min_free_kbytes = sqrt(lowmem_kbytes * 16) + * + * which yields + * + * 16MB: 512k + * 32MB: 724k + * 64MB: 1024k + * 128MB: 1448k + * 256MB: 2048k + * 512MB: 2896k + * 1024MB: 4096k + * 2048MB: 5792k + * 4096MB: 8192k + * 8192MB: 11584k + * 16384MB: 16384k + */ +int __meminit init_per_zone_wmark_min(void) +{ + unsigned long lowmem_kbytes; + + lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); + + min_free_kbytes = int_sqrt(lowmem_kbytes * 16); + if (min_free_kbytes < 128) + min_free_kbytes = 128; + if (min_free_kbytes > 65536) + min_free_kbytes = 65536; + setup_per_zone_wmarks(); + refresh_zone_stat_thresholds(); + setup_per_zone_lowmem_reserve(); + setup_per_zone_inactive_ratio(); + return 0; +} +module_init(init_per_zone_wmark_min) + +/* + * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so + * that we can call two helper functions whenever min_free_kbytes + * changes. + */ +int min_free_kbytes_sysctl_handler(ctl_table *table, int write, + void __user *buffer, size_t *length, loff_t *ppos) +{ + proc_dointvec(table, write, buffer, length, ppos); + if (write) + setup_per_zone_wmarks(); + return 0; +} + +#ifdef CONFIG_NUMA +int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write, + void __user *buffer, size_t *length, loff_t *ppos) +{ + struct zone *zone; + int rc; + + rc = proc_dointvec_minmax(table, write, buffer, length, ppos); + if (rc) + return rc; + + for_each_zone(zone) + zone->min_unmapped_pages = (zone->present_pages * + sysctl_min_unmapped_ratio) / 100; + return 0; +} + +int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write, + void __user *buffer, size_t *length, loff_t *ppos) +{ + struct zone *zone; + int rc; + + rc = proc_dointvec_minmax(table, write, buffer, length, ppos); + if (rc) + return rc; + + for_each_zone(zone) + zone->min_slab_pages = (zone->present_pages * + sysctl_min_slab_ratio) / 100; + return 0; +} +#endif + +/* + * lowmem_reserve_ratio_sysctl_handler - just a wrapper around + * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() + * whenever sysctl_lowmem_reserve_ratio changes. + * + * The reserve ratio obviously has absolutely no relation with the + * minimum watermarks. The lowmem reserve ratio can only make sense + * if in function of the boot time zone sizes. + */ +int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write, + void __user *buffer, size_t *length, loff_t *ppos) +{ + proc_dointvec_minmax(table, write, buffer, length, ppos); + setup_per_zone_lowmem_reserve(); + return 0; +} + +/* + * percpu_pagelist_fraction - changes the pcp->high for each zone on each + * cpu. It is the fraction of total pages in each zone that a hot per cpu + * pagelist can have before it gets flushed back to buddy allocator. + */ + +int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write, + void __user *buffer, size_t *length, loff_t *ppos) +{ + struct zone *zone; + unsigned int cpu; + int ret; + + ret = proc_dointvec_minmax(table, write, buffer, length, ppos); + if (!write || (ret == -EINVAL)) + return ret; + for_each_populated_zone(zone) { + for_each_possible_cpu(cpu) { + unsigned long high; + high = zone->present_pages / percpu_pagelist_fraction; + setup_pagelist_highmark( + per_cpu_ptr(zone->pageset, cpu), high); + } + } + return 0; +} + +int hashdist = HASHDIST_DEFAULT; + +#ifdef CONFIG_NUMA +static int __init set_hashdist(char *str) +{ + if (!str) + return 0; + hashdist = simple_strtoul(str, &str, 0); + return 1; +} +__setup("hashdist=", set_hashdist); +#endif + +/* + * allocate a large system hash table from bootmem + * - it is assumed that the hash table must contain an exact power-of-2 + * quantity of entries + * - limit is the number of hash buckets, not the total allocation size + */ +void *__init alloc_large_system_hash(const char *tablename, + unsigned long bucketsize, + unsigned long numentries, + int scale, + int flags, + unsigned int *_hash_shift, + unsigned int *_hash_mask, + unsigned long limit) +{ + unsigned long long max = limit; + unsigned long log2qty, size; + void *table = NULL; + + /* allow the kernel cmdline to have a say */ + if (!numentries) { + /* round applicable memory size up to nearest megabyte */ + numentries = nr_kernel_pages; + numentries += (1UL << (20 - PAGE_SHIFT)) - 1; + numentries >>= 20 - PAGE_SHIFT; + numentries <<= 20 - PAGE_SHIFT; + + /* limit to 1 bucket per 2^scale bytes of low memory */ + if (scale > PAGE_SHIFT) + numentries >>= (scale - PAGE_SHIFT); + else + numentries <<= (PAGE_SHIFT - scale); + + /* Make sure we've got at least a 0-order allocation.. */ + if (unlikely(flags & HASH_SMALL)) { + /* Makes no sense without HASH_EARLY */ + WARN_ON(!(flags & HASH_EARLY)); + if (!(numentries >> *_hash_shift)) { + numentries = 1UL << *_hash_shift; + BUG_ON(!numentries); + } + } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) + numentries = PAGE_SIZE / bucketsize; + } + numentries = roundup_pow_of_two(numentries); + + /* limit allocation size to 1/16 total memory by default */ + if (max == 0) { + max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; + do_div(max, bucketsize); + } + + if (numentries > max) + numentries = max; + + log2qty = ilog2(numentries); + + do { + size = bucketsize << log2qty; + if (flags & HASH_EARLY) + table = alloc_bootmem_nopanic(size); + else if (hashdist) + table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); + else { + /* + * If bucketsize is not a power-of-two, we may free + * some pages at the end of hash table which + * alloc_pages_exact() automatically does + */ + if (get_order(size) < MAX_ORDER) { + table = alloc_pages_exact(size, GFP_ATOMIC); + kmemleak_alloc(table, size, 1, GFP_ATOMIC); + } + } + } while (!table && size > PAGE_SIZE && --log2qty); + + if (!table) + panic("Failed to allocate %s hash table\n", tablename); + + printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n", + tablename, + (1UL << log2qty), + ilog2(size) - PAGE_SHIFT, + size); + + if (_hash_shift) + *_hash_shift = log2qty; + if (_hash_mask) + *_hash_mask = (1 << log2qty) - 1; + + return table; +} + +/* Return a pointer to the bitmap storing bits affecting a block of pages */ +static inline unsigned long *get_pageblock_bitmap(struct zone *zone, + unsigned long pfn) +{ +#ifdef CONFIG_SPARSEMEM + return __pfn_to_section(pfn)->pageblock_flags; +#else + return zone->pageblock_flags; +#endif /* CONFIG_SPARSEMEM */ +} + +static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn) +{ +#ifdef CONFIG_SPARSEMEM + pfn &= (PAGES_PER_SECTION-1); + return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; +#else + pfn = pfn - zone->zone_start_pfn; + return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; +#endif /* CONFIG_SPARSEMEM */ +} + +/** + * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages + * @page: The page within the block of interest + * @start_bitidx: The first bit of interest to retrieve + * @end_bitidx: The last bit of interest + * returns pageblock_bits flags + */ +unsigned long get_pageblock_flags_group(struct page *page, + int start_bitidx, int end_bitidx) +{ + struct zone *zone; + unsigned long *bitmap; + unsigned long pfn, bitidx; + unsigned long flags = 0; + unsigned long value = 1; + + zone = page_zone(page); + pfn = page_to_pfn(page); + bitmap = get_pageblock_bitmap(zone, pfn); + bitidx = pfn_to_bitidx(zone, pfn); + + for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1) + if (test_bit(bitidx + start_bitidx, bitmap)) + flags |= value; + + return flags; +} + +/** + * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages + * @page: The page within the block of interest + * @start_bitidx: The first bit of interest + * @end_bitidx: The last bit of interest + * @flags: The flags to set + */ +void set_pageblock_flags_group(struct page *page, unsigned long flags, + int start_bitidx, int end_bitidx) +{ + struct zone *zone; + unsigned long *bitmap; + unsigned long pfn, bitidx; + unsigned long value = 1; + + zone = page_zone(page); + pfn = page_to_pfn(page); + bitmap = get_pageblock_bitmap(zone, pfn); + bitidx = pfn_to_bitidx(zone, pfn); + VM_BUG_ON(pfn < zone->zone_start_pfn); + VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages); + + for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1) + if (flags & value) + __set_bit(bitidx + start_bitidx, bitmap); + else + __clear_bit(bitidx + start_bitidx, bitmap); +} + +/* + * This is designed as sub function...plz see page_isolation.c also. + * set/clear page block's type to be ISOLATE. + * page allocater never alloc memory from ISOLATE block. + */ + +static int +__count_immobile_pages(struct zone *zone, struct page *page, int count) +{ + unsigned long pfn, iter, found; + int mt; + + /* + * For avoiding noise data, lru_add_drain_all() should be called + * If ZONE_MOVABLE, the zone never contains immobile pages + */ + if (zone_idx(zone) == ZONE_MOVABLE) + return true; + mt = get_pageblock_migratetype(page); + if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt)) + return true; + + pfn = page_to_pfn(page); + for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) { + unsigned long check = pfn + iter; + + if (!pfn_valid_within(check)) + continue; + + page = pfn_to_page(check); + if (!page_count(page)) { + if (PageBuddy(page)) + iter += (1 << page_order(page)) - 1; + continue; + } + if (!PageLRU(page)) + found++; + /* + * If there are RECLAIMABLE pages, we need to check it. + * But now, memory offline itself doesn't call shrink_slab() + * and it still to be fixed. + */ + /* + * If the page is not RAM, page_count()should be 0. + * we don't need more check. This is an _used_ not-movable page. + * + * The problematic thing here is PG_reserved pages. PG_reserved + * is set to both of a memory hole page and a _used_ kernel + * page at boot. + */ + if (found > count) + return false; + } + return true; +} + +bool is_pageblock_removable_nolock(struct page *page) +{ + struct zone *zone = page_zone(page); + return __count_immobile_pages(zone, page, 0); +} + +int set_migratetype_isolate(struct page *page) +{ + struct zone *zone; + unsigned long flags, pfn; + struct memory_isolate_notify arg; + int notifier_ret; + int ret = -EBUSY; + + zone = page_zone(page); + + spin_lock_irqsave(&zone->lock, flags); + + pfn = page_to_pfn(page); + arg.start_pfn = pfn; + arg.nr_pages = pageblock_nr_pages; + arg.pages_found = 0; + + /* + * It may be possible to isolate a pageblock even if the + * migratetype is not MIGRATE_MOVABLE. The memory isolation + * notifier chain is used by balloon drivers to return the + * number of pages in a range that are held by the balloon + * driver to shrink memory. If all the pages are accounted for + * by balloons, are free, or on the LRU, isolation can continue. + * Later, for example, when memory hotplug notifier runs, these + * pages reported as "can be isolated" should be isolated(freed) + * by the balloon driver through the memory notifier chain. + */ + notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg); + notifier_ret = notifier_to_errno(notifier_ret); + if (notifier_ret) + goto out; + /* + * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself. + * We just check MOVABLE pages. + */ + if (__count_immobile_pages(zone, page, arg.pages_found)) + ret = 0; + + /* + * immobile means "not-on-lru" paes. If immobile is larger than + * removable-by-driver pages reported by notifier, we'll fail. + */ + +out: + if (!ret) { + set_pageblock_migratetype(page, MIGRATE_ISOLATE); + move_freepages_block(zone, page, MIGRATE_ISOLATE); + } + + spin_unlock_irqrestore(&zone->lock, flags); + if (!ret) + drain_all_pages(); + return ret; +} + +void unset_migratetype_isolate(struct page *page, unsigned migratetype) +{ + struct zone *zone; + unsigned long flags; + zone = page_zone(page); + spin_lock_irqsave(&zone->lock, flags); + if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE) + goto out; + set_pageblock_migratetype(page, migratetype); + move_freepages_block(zone, page, migratetype); +out: + spin_unlock_irqrestore(&zone->lock, flags); +} + +#ifdef CONFIG_DMA_CMA + +static unsigned long pfn_max_align_down(unsigned long pfn) +{ + return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, + pageblock_nr_pages) - 1); +} + +static unsigned long pfn_max_align_up(unsigned long pfn) +{ + return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, + pageblock_nr_pages)); +} + +static struct page * +__alloc_contig_migrate_alloc(struct page *page, unsigned long private, + int **resultp) +{ + gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE; + + if (PageHighMem(page)) + printk(KERN_INFO "%s[%d] gfp_mask 0x%x\n", + __func__, __LINE__, (unsigned int) gfp_mask); + + return alloc_page(gfp_mask); +} + +struct page *failed_pages[5][10]; /* FIXME: locking */ + +/* [start, end) must belong to a single zone. */ +static int __alloc_contig_migrate_range(unsigned long start, unsigned long end) +{ + /* This function is based on compact_zone() from compaction.c. */ + + unsigned long pfn = start; + int tries = 0; + int ret = 0; + int i, j; + + struct compact_control cc = { + .nr_migratepages = 0, + .order = -1, + .zone = page_zone(pfn_to_page(start)), + .sync = true, + }; + INIT_LIST_HEAD(&cc.migratepages); + + for (i = 0; i < 5; i++) { + for (j = 0; j < 10; j++) + failed_pages[i][j] = NULL; + } + + migrate_prep(); + + while (pfn < end || !list_empty(&cc.migratepages)) { + if (fatal_signal_pending(current)) { + ret = -EINTR; + break; + } + + if (list_empty(&cc.migratepages)) { + cc.nr_migratepages = 0; + pfn = isolate_migratepages_range(cc.zone, &cc, + pfn, end); + if (!pfn) { + ret = -EINTR; + break; + } + tries = 0; + } else if (++tries == 5) { + ret = ret < 0 ? ret : -EBUSY; + break; + } + + ret = migrate_pages(&cc.migratepages, + __alloc_contig_migrate_alloc, + 0, false, true, tries); + } + + putback_lru_pages(&cc.migratepages); + return ret > 0 ? 0 : ret; +} + +/* + * Update zone's cma pages counter used for watermark level calculation. + */ +static inline void __update_cma_watermarks(struct zone *zone, int count) +{ + unsigned long flags; + spin_lock_irqsave(&zone->lock, flags); + zone->min_cma_pages += count; + spin_unlock_irqrestore(&zone->lock, flags); + setup_per_zone_wmarks(); +} + +/* + * Trigger memory pressure bump to reclaim some pages in order to be able to + * allocate 'count' pages in single page units. Does similar work as + *__alloc_pages_slowpath() function. + */ +static int __reclaim_pages(struct zone *zone, gfp_t gfp_mask, int count) +{ + enum zone_type high_zoneidx = gfp_zone(gfp_mask); + struct zonelist *zonelist = node_zonelist(0, gfp_mask); + int did_some_progress = 0; + int order = 1; + int retry = 0; + + /* + * Increase level of watermarks to force kswapd do his job + * to stabilise at new watermark level. + */ + __update_cma_watermarks(zone, count); + + /* Obey watermarks as if the page was being allocated */ + while (!zone_watermark_ok(zone, 0, low_wmark_pages(zone), 0, 0) + && retry++ < 16) { + wake_all_kswapd(order, zonelist, high_zoneidx, zone_idx(zone)); + + did_some_progress = __perform_reclaim(gfp_mask, order, zonelist, + NULL); + if (!did_some_progress) { + /* Exhausted what can be done so it's blamo time */ + out_of_memory(zonelist, gfp_mask, order, NULL); + } + } + + /* Restore original watermark levels. */ + __update_cma_watermarks(zone, -count); + + return count; +} + +/** + * alloc_contig_range() -- tries to allocate given range of pages + * @start: start PFN to allocate + * @end: one-past-the-last PFN to allocate + * @migratetype: migratetype of the underlaying pageblocks (either + * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks + * in range must have the same migratetype and it must + * be either of the two. + * + * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES + * aligned, however it's the caller's responsibility to guarantee that + * we are the only thread that changes migrate type of pageblocks the + * pages fall in. + * + * The PFN range must belong to a single zone. + * + * Returns zero on success or negative error code. On success all + * pages which PFN is in [start, end) are allocated for the caller and + * need to be freed with free_contig_range(). + */ +int alloc_contig_range(unsigned long start, unsigned long end, + unsigned migratetype) +{ + struct zone *zone = page_zone(pfn_to_page(start)); + unsigned long outer_start, outer_end; + int ret = 0, order, retry = 0; + + /* + * What we do here is we mark all pageblocks in range as + * MIGRATE_ISOLATE. Because pageblock and max order pages may + * have different sizes, and due to the way page allocator + * work, we align the range to biggest of the two pages so + * that page allocator won't try to merge buddies from + * different pageblocks and change MIGRATE_ISOLATE to some + * other migration type. + * + * Once the pageblocks are marked as MIGRATE_ISOLATE, we + * migrate the pages from an unaligned range (ie. pages that + * we are interested in). This will put all the pages in + * range back to page allocator as MIGRATE_ISOLATE. + * + * When this is done, we take the pages in range from page + * allocator removing them from the buddy system. This way + * page allocator will never consider using them. + * + * This lets us mark the pageblocks back as + * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the + * aligned range but not in the unaligned, original range are + * put back to page allocator so that buddy can use them. + */ + + ret = start_isolate_page_range(pfn_max_align_down(start), + pfn_max_align_up(end), migratetype); + if (ret) { + printk(KERN_ERR "%s : start_isolate_page_range failed\n", + __func__); + goto done; + } + + drain_all_pages(); + zone->cma_alloc = 1; + +migrate: + printk(KERN_DEBUG "migrating range %lx %lx, retry (%d)\n", + start, end, retry); + ret = __alloc_contig_migrate_range(start, end); + if (ret) { + printk(KERN_ERR "__alloc_contig_migrate_range failed\n"); + goto done; + } + + /* + * Pages from [start, end) are within a MAX_ORDER_NR_PAGES + * aligned blocks that are marked as MIGRATE_ISOLATE. What's + * more, all pages in [start, end) are free in page allocator. + * What we are going to do is to allocate all pages from + * [start, end) (that is remove them from page allocator). + * + * The only problem is that pages at the beginning and at the + * end of interesting range may be not aligned with pages that + * page allocator holds, ie. they can be part of higher order + * pages. Because of this, we reserve the bigger range and + * once this is done free the pages we are not interested in. + * + * We don't have to hold zone->lock here because the pages are + * isolated thus they won't get removed from buddy. + */ + + outer_start = start; + for (order = 0; order < MAX_ORDER; ++order) { + unsigned long pfn = start & (~0UL << order); + struct page *page = pfn_to_page(pfn); + + if (PageBuddy(page)) { + if (page_order(page) >= order) + outer_start = pfn; + break; + } + } + + /* Make sure the range is really isolated. */ + if (test_pages_isolated(outer_start, end)) { + printk(KERN_ERR "%s: test_pages_isolated(%lx, %lx) failed\n", + __func__, outer_start, end); + ret = -EBUSY; + goto done; + } + + /* + * Reclaim enough pages to make sure that contiguous allocation + * will not starve the system. + */ + __reclaim_pages(zone, GFP_USER | __GFP_MOVABLE, end-start); + + /* Grab isolated pages from freelists. */ + outer_end = isolate_freepages_range(outer_start, end, true); + if (!outer_end) { + ret = -EBUSY; + printk(KERN_ERR "%s : isolate_freepages_range failed %lu\n", + __func__, outer_end); + goto done; + } + + /* Free head and tail (if any) */ + if (start != outer_start) + free_contig_range(outer_start, start - outer_start); + if (end != outer_end) + free_contig_range(end, outer_end - end); + +done: + if ((ret == -EBUSY || ret == -EAGAIN) && retry++ < 5) { + unsigned long count, cf; + /* FIXME kmpark: temporaily drop the cma free pages */ + count = zone_page_state(zone, NR_FREE_CMA_PAGES); + cf = zone_page_state(zone, NR_FREE_PAGES); + printk(KERN_ERR "%s[%d] free %lu, cma count %lu\n", + __func__, __LINE__, cf, count); + msleep(100); + count = zone_page_state(zone, NR_FREE_CMA_PAGES); + cf = zone_page_state(zone, NR_FREE_PAGES); + printk(KERN_ERR "%s[%d] free %lu, cma count %lu\n", + __func__, __LINE__, cf, count); + goto migrate; + } + + undo_isolate_page_range(pfn_max_align_down(start), + pfn_max_align_up(end), migratetype); + zone->cma_alloc = 0; + return ret; +} + +void free_contig_range(unsigned long pfn, unsigned nr_pages) +{ + for (; nr_pages--; ++pfn) + __free_page(pfn_to_page(pfn)); +} +#endif + +#ifdef CONFIG_MEMORY_HOTREMOVE +/* + * All pages in the range must be isolated before calling this. + */ +void +__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) +{ + struct page *page; + struct zone *zone; + int order, i; + unsigned long pfn; + unsigned long flags; + /* find the first valid pfn */ + for (pfn = start_pfn; pfn < end_pfn; pfn++) + if (pfn_valid(pfn)) + break; + if (pfn == end_pfn) + return; + zone = page_zone(pfn_to_page(pfn)); + spin_lock_irqsave(&zone->lock, flags); + pfn = start_pfn; + while (pfn < end_pfn) { + if (!pfn_valid(pfn)) { + pfn++; + continue; + } + page = pfn_to_page(pfn); + BUG_ON(page_count(page)); + BUG_ON(!PageBuddy(page)); + order = page_order(page); +#ifdef CONFIG_DEBUG_VM + printk(KERN_INFO "remove from free list %lx %d %lx\n", + pfn, 1 << order, end_pfn); +#endif + list_del(&page->lru); + rmv_page_order(page); + zone->free_area[order].nr_free--; + __mod_zone_page_state(zone, NR_FREE_PAGES, + -(1UL << order)); + for (i = 0; i < (1 << order); i++) + SetPageReserved((page+i)); + pfn += (1 << order); + } + spin_unlock_irqrestore(&zone->lock, flags); +} +#endif + +#ifdef CONFIG_MEMORY_FAILURE +bool is_free_buddy_page(struct page *page) +{ + struct zone *zone = page_zone(page); + unsigned long pfn = page_to_pfn(page); + unsigned long flags; + int order; + + spin_lock_irqsave(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) { + struct page *page_head = page - (pfn & ((1 << order) - 1)); + + if (PageBuddy(page_head) && page_order(page_head) >= order) + break; + } + spin_unlock_irqrestore(&zone->lock, flags); + + return order < MAX_ORDER; +} +#endif + +static struct trace_print_flags pageflag_names[] = { + {1UL << PG_locked, "locked" }, + {1UL << PG_error, "error" }, + {1UL << PG_referenced, "referenced" }, + {1UL << PG_uptodate, "uptodate" }, + {1UL << PG_dirty, "dirty" }, + {1UL << PG_lru, "lru" }, + {1UL << PG_active, "active" }, + {1UL << PG_slab, "slab" }, + {1UL << PG_owner_priv_1, "owner_priv_1" }, + {1UL << PG_arch_1, "arch_1" }, + {1UL << PG_reserved, "reserved" }, + {1UL << PG_private, "private" }, + {1UL << PG_private_2, "private_2" }, + {1UL << PG_writeback, "writeback" }, +#ifdef CONFIG_PAGEFLAGS_EXTENDED + {1UL << PG_head, "head" }, + {1UL << PG_tail, "tail" }, +#else + {1UL << PG_compound, "compound" }, +#endif + {1UL << PG_swapcache, "swapcache" }, + {1UL << PG_mappedtodisk, "mappedtodisk" }, + {1UL << PG_reclaim, "reclaim" }, + {1UL << PG_swapbacked, "swapbacked" }, + {1UL << PG_unevictable, "unevictable" }, +#ifdef CONFIG_MMU + {1UL << PG_mlocked, "mlocked" }, +#endif +#ifdef CONFIG_ARCH_USES_PG_UNCACHED + {1UL << PG_uncached, "uncached" }, +#endif +#ifdef CONFIG_MEMORY_FAILURE + {1UL << PG_hwpoison, "hwpoison" }, +#endif + {-1UL, NULL }, +}; + +static void dump_page_flags(unsigned long flags) +{ + const char *delim = ""; + unsigned long mask; + int i; + + printk(KERN_ALERT "page flags: %#lx(", flags); + + /* remove zone id */ + flags &= (1UL << NR_PAGEFLAGS) - 1; + + for (i = 0; pageflag_names[i].name && flags; i++) { + + mask = pageflag_names[i].mask; + if ((flags & mask) != mask) + continue; + + flags &= ~mask; + printk("%s%s", delim, pageflag_names[i].name); + delim = "|"; + } + + /* check for left over flags */ + if (flags) + printk("%s%#lx", delim, flags); + + printk(")\n"); +} + +void dump_page(struct page *page) +{ + printk(KERN_ALERT + "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n", + page, atomic_read(&page->_count), page_mapcount(page), + page->mapping, page->index); + dump_page_flags(page->flags); + mem_cgroup_print_bad_page(page); +} diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 62a7fa2..9450436 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -78,6 +78,8 @@ DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ EXPORT_PER_CPU_SYMBOL(_numa_mem_); #endif +struct rw_semaphore page_alloc_slow_rwsem; + /* * Array of node states. */ @@ -99,6 +101,14 @@ unsigned long totalreserve_pages __read_mostly; int percpu_pagelist_fraction; gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; +#ifdef CONFIG_COMPACTION_RETRY_DEBUG +static inline void show_buddy_info(void); +#else +static inline void show_buddy_info(void) +{ +} +#endif + #ifdef CONFIG_PM_SLEEP /* * The following functions are used by the suspend/hibernate code to temporarily @@ -127,6 +137,20 @@ void pm_restrict_gfp_mask(void) saved_gfp_mask = gfp_allowed_mask; gfp_allowed_mask &= ~GFP_IOFS; } + +static bool pm_suspending(void) +{ + if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS) + return false; + return true; +} + +#else + +static bool pm_suspending(void) +{ + return false; +} #endif /* CONFIG_PM_SLEEP */ #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE @@ -176,6 +200,7 @@ static char * const zone_names[MAX_NR_ZONES] = { }; int min_free_kbytes = 1024; +int min_free_order_shift = 1; static unsigned long __meminitdata nr_kernel_pages; static unsigned long __meminitdata nr_all_pages; @@ -318,7 +343,6 @@ static void bad_page(struct page *page) current->comm, page_to_pfn(page)); dump_page(page); - print_modules(); dump_stack(); out: /* Leave bad fields for debug, except PageBuddy could make trouble */ @@ -1371,12 +1395,21 @@ failed: #ifdef CONFIG_FAIL_PAGE_ALLOC -static struct { +static struct fail_page_alloc_attr { struct fault_attr attr; u32 ignore_gfp_highmem; u32 ignore_gfp_wait; u32 min_order; + +#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS + + struct dentry *ignore_gfp_highmem_file; + struct dentry *ignore_gfp_wait_file; + struct dentry *min_order_file; + +#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ + } fail_page_alloc = { .attr = FAULT_ATTR_INITIALIZER, .ignore_gfp_wait = 1, @@ -1410,27 +1443,36 @@ static int __init fail_page_alloc_debugfs(void) { mode_t mode = S_IFREG | S_IRUSR | S_IWUSR; struct dentry *dir; + int err; + + err = init_fault_attr_dentries(&fail_page_alloc.attr, + "fail_page_alloc"); + if (err) + return err; + dir = fail_page_alloc.attr.dentries.dir; + + fail_page_alloc.ignore_gfp_wait_file = + debugfs_create_bool("ignore-gfp-wait", mode, dir, + &fail_page_alloc.ignore_gfp_wait); + + fail_page_alloc.ignore_gfp_highmem_file = + debugfs_create_bool("ignore-gfp-highmem", mode, dir, + &fail_page_alloc.ignore_gfp_highmem); + fail_page_alloc.min_order_file = + debugfs_create_u32("min-order", mode, dir, + &fail_page_alloc.min_order); + + if (!fail_page_alloc.ignore_gfp_wait_file || + !fail_page_alloc.ignore_gfp_highmem_file || + !fail_page_alloc.min_order_file) { + err = -ENOMEM; + debugfs_remove(fail_page_alloc.ignore_gfp_wait_file); + debugfs_remove(fail_page_alloc.ignore_gfp_highmem_file); + debugfs_remove(fail_page_alloc.min_order_file); + cleanup_fault_attr_dentries(&fail_page_alloc.attr); + } - dir = fault_create_debugfs_attr("fail_page_alloc", NULL, - &fail_page_alloc.attr); - if (IS_ERR(dir)) - return PTR_ERR(dir); - - if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, - &fail_page_alloc.ignore_gfp_wait)) - goto fail; - if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir, - &fail_page_alloc.ignore_gfp_highmem)) - goto fail; - if (!debugfs_create_u32("min-order", mode, dir, - &fail_page_alloc.min_order)) - goto fail; - - return 0; -fail: - debugfs_remove_recursive(dir); - - return -ENOMEM; + return err; } late_initcall(fail_page_alloc_debugfs); @@ -1470,7 +1512,7 @@ static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark, free_pages -= z->free_area[o].nr_free << o; /* Require fewer higher order pages to be free */ - min >>= 1; + min >>= min_free_order_shift; if (free_pages <= min) return false; @@ -1754,19 +1796,13 @@ static DEFINE_RATELIMIT_STATE(nopage_rs, void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...) { + va_list args; unsigned int filter = SHOW_MEM_FILTER_NODES; if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs)) return; /* - * Walking all memory to count page types is very expensive and should - * be inhibited in non-blockable contexts. - */ - if (!(gfp_mask & __GFP_WAIT)) - filter |= SHOW_MEM_FILTER_PAGE_COUNT; - - /* * This documents exceptions given to allocations in certain * contexts that are allowed to allocate outside current's set * of allowed nodes. @@ -1779,21 +1815,14 @@ void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...) filter &= ~SHOW_MEM_FILTER_NODES; if (fmt) { - struct va_format vaf; - va_list args; - + printk(KERN_WARNING); va_start(args, fmt); - - vaf.fmt = fmt; - vaf.va = &args; - - pr_warn("%pV", &vaf); - + vprintk(fmt, args); va_end(args); } - pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n", - current->comm, order, gfp_mask); + pr_warning("%s: page allocation failure: order:%d, mode:0x%x\n", + current->comm, order, gfp_mask); dump_stack(); if (!should_suppress_show_mem()) @@ -1843,6 +1872,10 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, int migratetype) { struct page *page; +#ifdef CONFIG_COMPACTION_RETRY + struct zoneref *z; + struct zone *zone; +#endif /* Acquire the OOM killer lock for the zones in zonelist */ if (!try_set_zonelist_oom(zonelist, gfp_mask)) { @@ -1862,6 +1895,32 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, if (page) goto out; +#ifdef CONFIG_COMPACTION_RETRY + /* + * When we reach here, we already tried direct reclaim. + * Therefore it might be possible that we have enough + * free memory but extremely fragmented. + * So we give it a last chance to try memory compaction and get pages. + */ + if (order) { + pr_info("reclaim before oom : retry compaction.\n"); + show_buddy_info(); + + for_each_zone_zonelist_nodemask(zone, z, zonelist, + high_zoneidx, nodemask) + compact_zone_order(zone, -1, gfp_mask, true); + + show_buddy_info(); + pr_info("reclaim :end\n"); + page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, + nodemask, order, zonelist, high_zoneidx, + ALLOC_WMARK_HIGH|ALLOC_CPUSET, + preferred_zone, migratetype); + if (page) + goto out; + } +#endif + if (!(gfp_mask & __GFP_NOFAIL)) { /* The OOM killer will not help higher order allocs */ if (order > PAGE_ALLOC_COSTLY_ORDER) @@ -2054,7 +2113,7 @@ static inline int gfp_to_alloc_flags(gfp_t gfp_mask) { int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; - const bool atomic = !(gfp_mask & (__GFP_WAIT | __GFP_NO_KSWAPD)); + const gfp_t wait = gfp_mask & __GFP_WAIT; /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); @@ -2063,20 +2122,20 @@ gfp_to_alloc_flags(gfp_t gfp_mask) * The caller may dip into page reserves a bit more if the caller * cannot run direct reclaim, or if the caller has realtime scheduling * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will - * set both ALLOC_HARDER (atomic == true) and ALLOC_HIGH (__GFP_HIGH). + * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH). */ alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); - if (atomic) { + if (!wait) { /* - * Not worth trying to allocate harder for __GFP_NOMEMALLOC even - * if it can't schedule. + * Not worth trying to allocate harder for + * __GFP_NOMEMALLOC even if it can't schedule. */ - if (!(gfp_mask & __GFP_NOMEMALLOC)) + if (!(gfp_mask & __GFP_NOMEMALLOC)) alloc_flags |= ALLOC_HARDER; /* - * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the - * comment for __cpuset_node_allowed_softwall(). + * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc. + * See also cpuset_zone_allowed() comment in kernel/cpuset.c. */ alloc_flags &= ~ALLOC_CPUSET; } else if (unlikely(rt_task(current)) && !in_interrupt()) @@ -2104,6 +2163,9 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, unsigned long pages_reclaimed = 0; unsigned long did_some_progress; bool sync_migration = false; +#ifdef CONFIG_ANDROID_WIP + unsigned long start_tick = jiffies; +#endif bool deferred_compaction = false; /* @@ -2117,6 +2179,9 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, return NULL; } + if (gfp_mask & __GFP_WAIT) + down_read(&page_alloc_slow_rwsem); + /* * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and * __GFP_NOWARN set) should not cause reclaim since the subsystem @@ -2213,11 +2278,22 @@ rebalance: /* * If we failed to make any progress reclaiming, then we are * running out of options and have to consider going OOM + * ANDROID_WIP: If we are looping more than 1 second, consider OOM */ - if (!did_some_progress) { +#ifdef CONFIG_ANDROID_WIP +#define SHOULD_CONSIDER_OOM !did_some_progress || time_after(jiffies, start_tick + HZ) +#else +#define SHOULD_CONSIDER_OOM !did_some_progress +#endif + if (SHOULD_CONSIDER_OOM) { if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) { if (oom_killer_disabled) goto nopage; +#ifdef CONFIG_ANDROID_WIP + if (did_some_progress) + pr_info("time's up : calling " + "__alloc_pages_may_oom\n"); +#endif page = __alloc_pages_may_oom(gfp_mask, order, zonelist, high_zoneidx, nodemask, preferred_zone, @@ -2245,6 +2321,14 @@ rebalance: goto restart; } + + /* + * Suspend converts GFP_KERNEL to __GFP_WAIT which can + * prevent reclaim making forward progress without + * invoking OOM. Bail if we are suspending + */ + if (pm_suspending()) + goto nopage; } /* Check if we should retry the allocation */ @@ -2272,10 +2356,14 @@ rebalance: nopage: warn_alloc_failed(gfp_mask, order, NULL); + if (gfp_mask & __GFP_WAIT) + up_read(&page_alloc_slow_rwsem); return page; got_pg: if (kmemcheck_enabled) kmemcheck_pagealloc_alloc(page, order, gfp_mask); + if (gfp_mask & __GFP_WAIT) + up_read(&page_alloc_slow_rwsem); return page; } @@ -2725,6 +2813,37 @@ void show_free_areas(unsigned int filter) show_swap_cache_info(); } +#ifdef CONFIG_COMPACTION_RETRY_DEBUG +void show_buddy_info(void) +{ + struct zone *zone; + unsigned long nr[MAX_ORDER], flags, order, total = 0; + char buf[256]; + int len; + + for_each_populated_zone(zone) { + + if (skip_free_areas_node(SHOW_MEM_FILTER_NODES, + zone_to_nid(zone))) + continue; + show_node(zone); + len = sprintf(buf, "%s: ", zone->name); + + spin_lock_irqsave(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) { + nr[order] = zone->free_area[order].nr_free; + total += nr[order] << order; + } + spin_unlock_irqrestore(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) + len += sprintf(buf + len, "%lu*%lukB ", + nr[order], K(1UL) << order); + len += sprintf(buf + len, "= %lukB", K(total)); + pr_err("%s\n", buf); + } +} +#endif + static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) { zoneref->zone = zone; @@ -4291,24 +4410,25 @@ static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE -/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ -void __init set_pageblock_order(void) +/* Return a sensible default order for the pageblock size. */ +static inline int pageblock_default_order(void) { - unsigned int order; + if (HPAGE_SHIFT > PAGE_SHIFT) + return HUGETLB_PAGE_ORDER; + + return MAX_ORDER-1; +} +/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ +static inline void __init set_pageblock_order(unsigned int order) +{ /* Check that pageblock_nr_pages has not already been setup */ if (pageblock_order) return; - if (HPAGE_SHIFT > PAGE_SHIFT) - order = HUGETLB_PAGE_ORDER; - else - order = MAX_ORDER - 1; - /* * Assume the largest contiguous order of interest is a huge page. - * This value may be variable depending on boot parameters on IA64 and - * powerpc. + * This value may be variable depending on boot parameters on IA64 */ pageblock_order = order; } @@ -4316,13 +4436,15 @@ void __init set_pageblock_order(void) /* * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() - * is unused as pageblock_order is set at compile-time. See - * include/linux/pageblock-flags.h for the values of pageblock_order based on - * the kernel config + * and pageblock_default_order() are unused as pageblock_order is set + * at compile-time. See include/linux/pageblock-flags.h for the values of + * pageblock_order based on the kernel config */ -void __init set_pageblock_order(void) +static inline int pageblock_default_order(unsigned int order) { + return MAX_ORDER-1; } +#define set_pageblock_order(x) do {} while (0) #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ @@ -4410,7 +4532,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat, if (!size) continue; - set_pageblock_order(); + set_pageblock_order(pageblock_default_order()); setup_usemap(pgdat, zone, zone_start_pfn, size); ret = init_currently_empty_zone(zone, zone_start_pfn, size, MEMMAP_EARLY); @@ -4665,60 +4787,6 @@ void __init sort_node_map(void) cmp_node_active_region, NULL); } -/** - * node_map_pfn_alignment - determine the maximum internode alignment - * - * This function should be called after node map is populated and sorted. - * It calculates the maximum power of two alignment which can distinguish - * all the nodes. - * - * For example, if all nodes are 1GiB and aligned to 1GiB, the return value - * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the - * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is - * shifted, 1GiB is enough and this function will indicate so. - * - * This is used to test whether pfn -> nid mapping of the chosen memory - * model has fine enough granularity to avoid incorrect mapping for the - * populated node map. - * - * Returns the determined alignment in pfn's. 0 if there is no alignment - * requirement (single node). - */ -unsigned long __init node_map_pfn_alignment(void) -{ - unsigned long accl_mask = 0, last_end = 0; - int last_nid = -1; - int i; - - for_each_active_range_index_in_nid(i, MAX_NUMNODES) { - int nid = early_node_map[i].nid; - unsigned long start = early_node_map[i].start_pfn; - unsigned long end = early_node_map[i].end_pfn; - unsigned long mask; - - if (!start || last_nid < 0 || last_nid == nid) { - last_nid = nid; - last_end = end; - continue; - } - - /* - * Start with a mask granular enough to pin-point to the - * start pfn and tick off bits one-by-one until it becomes - * too coarse to separate the current node from the last. - */ - mask = ~((1 << __ffs(start)) - 1); - while (mask && last_end <= (start & (mask << 1))) - mask <<= 1; - - /* accumulate all internode masks */ - accl_mask |= mask; - } - - /* convert mask to number of pages */ - return ~accl_mask + 1; -} - /* Find the lowest pfn for a node */ static unsigned long __init find_min_pfn_for_node(int nid) { @@ -5106,6 +5174,7 @@ static int page_alloc_cpu_notify(struct notifier_block *self, void __init page_alloc_init(void) { hotcpu_notifier(page_alloc_cpu_notify, 0); + init_rwsem(&page_alloc_slow_rwsem); } /* diff --git a/mm/page_cgroup.c b/mm/page_cgroup.c index 6f4ef53..53bffc6 100644 --- a/mm/page_cgroup.c +++ b/mm/page_cgroup.c @@ -133,13 +133,10 @@ struct page *lookup_cgroup_page(struct page_cgroup *pc) static void *__meminit alloc_page_cgroup(size_t size, int nid) { void *addr = NULL; - gfp_t flags = GFP_KERNEL | __GFP_NOWARN; - addr = alloc_pages_exact_nid(nid, size, flags); - if (addr) { - kmemleak_alloc(addr, size, 1, flags); + addr = alloc_pages_exact_nid(nid, size, GFP_KERNEL | __GFP_NOWARN); + if (addr) return addr; - } if (node_state(nid, N_HIGH_MEMORY)) addr = vmalloc_node(size, nid); @@ -160,7 +157,6 @@ static void free_page_cgroup(void *addr) sizeof(struct page_cgroup) * PAGES_PER_SECTION; BUG_ON(PageReserved(page)); - kmemleak_free(addr); free_pages_exact(addr, table_size); } } @@ -229,8 +225,8 @@ int __meminit online_page_cgroup(unsigned long start_pfn, unsigned long start, end, pfn; int fail = 0; - start = SECTION_ALIGN_DOWN(start_pfn); - end = SECTION_ALIGN_UP(start_pfn + nr_pages); + start = start_pfn & ~(PAGES_PER_SECTION - 1); + end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); if (nid == -1) { /* @@ -262,8 +258,8 @@ int __meminit offline_page_cgroup(unsigned long start_pfn, { unsigned long start, end, pfn; - start = SECTION_ALIGN_DOWN(start_pfn); - end = SECTION_ALIGN_UP(start_pfn + nr_pages); + start = start_pfn & ~(PAGES_PER_SECTION - 1); + end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION); for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) __free_page_cgroup(pfn); @@ -361,7 +357,7 @@ struct swap_cgroup_ctrl { spinlock_t lock; }; -static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES]; +struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES]; struct swap_cgroup { unsigned short id; @@ -517,10 +513,11 @@ int swap_cgroup_swapon(int type, unsigned long max_pages) length = DIV_ROUND_UP(max_pages, SC_PER_PAGE); array_size = length * sizeof(void *); - array = vzalloc(array_size); + array = vmalloc(array_size); if (!array) goto nomem; + memset(array, 0, array_size); ctrl = &swap_cgroup_ctrl[type]; mutex_lock(&swap_cgroup_mutex); ctrl->length = length; @@ -540,7 +537,7 @@ int swap_cgroup_swapon(int type, unsigned long max_pages) nomem: printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n"); printk(KERN_INFO - "swap_cgroup can be disabled by swapaccount=0 boot option\n"); + "swap_cgroup can be disabled by noswapaccount boot option\n"); return -ENOMEM; } diff --git a/mm/page_isolation-cma.c b/mm/page_isolation-cma.c new file mode 100644 index 0000000..07de2e0 --- /dev/null +++ b/mm/page_isolation-cma.c @@ -0,0 +1,147 @@ +/* + * linux/mm/page_isolation.c + */ + +#include <linux/mm.h> +#include <linux/page-isolation.h> +#include <linux/pageblock-flags.h> +#include "internal.h" + +static inline struct page * +__first_valid_page(unsigned long pfn, unsigned long nr_pages) +{ + int i; + for (i = 0; i < nr_pages; i++) + if (pfn_valid_within(pfn + i)) + break; + if (unlikely(i == nr_pages)) + return NULL; + return pfn_to_page(pfn + i); +} + +/* + * start_isolate_page_range() -- make page-allocation-type of range of pages + * to be MIGRATE_ISOLATE. + * @start_pfn: The lower PFN of the range to be isolated. + * @end_pfn: The upper PFN of the range to be isolated. + * @migratetype: migrate type to set in error recovery. + * + * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in + * the range will never be allocated. Any free pages and pages freed in the + * future will not be allocated again. + * + * start_pfn/end_pfn must be aligned to pageblock_order. + * Returns 0 on success and -EBUSY if any part of range cannot be isolated. + */ +int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, + unsigned migratetype) +{ + unsigned long pfn; + unsigned long undo_pfn; + struct page *page; + + BUG_ON((start_pfn) & (pageblock_nr_pages - 1)); + BUG_ON((end_pfn) & (pageblock_nr_pages - 1)); + + for (pfn = start_pfn; + pfn < end_pfn; + pfn += pageblock_nr_pages) { + page = __first_valid_page(pfn, pageblock_nr_pages); + if (page && set_migratetype_isolate(page)) { + undo_pfn = pfn; + goto undo; + } + } + return 0; +undo: + for (pfn = start_pfn; + pfn < undo_pfn; + pfn += pageblock_nr_pages) + unset_migratetype_isolate(pfn_to_page(pfn), migratetype); + + return -EBUSY; +} + +/* + * Make isolated pages available again. + */ +int undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, + unsigned migratetype) +{ + unsigned long pfn; + struct page *page; + BUG_ON((start_pfn) & (pageblock_nr_pages - 1)); + BUG_ON((end_pfn) & (pageblock_nr_pages - 1)); + for (pfn = start_pfn; + pfn < end_pfn; + pfn += pageblock_nr_pages) { + page = __first_valid_page(pfn, pageblock_nr_pages); + if (!page || get_pageblock_migratetype(page) != MIGRATE_ISOLATE) + continue; + unset_migratetype_isolate(page, migratetype); + } + return 0; +} +/* + * Test all pages in the range is free(means isolated) or not. + * all pages in [start_pfn...end_pfn) must be in the same zone. + * zone->lock must be held before call this. + * + * Returns 1 if all pages in the range are isolated. + */ +static int +__test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn) +{ + struct page *page; + + while (pfn < end_pfn) { + if (!pfn_valid_within(pfn)) { + pfn++; + continue; + } + page = pfn_to_page(pfn); + if (PageBuddy(page)) + pfn += 1 << page_order(page); + else if (page_count(page) == 0 && + page_private(page) == MIGRATE_ISOLATE) { + pfn += 1; + printk(KERN_INFO "%s:%d ", __func__, __LINE__); + dump_page(page); + } else { + printk(KERN_INFO "%s:%d ", __func__, __LINE__); + dump_page(page); + break; + } + } + if (pfn < end_pfn) + return 0; + return 1; +} + +int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn) +{ + unsigned long pfn, flags; + struct page *page; + struct zone *zone; + int ret; + + /* + * Note: pageblock_nr_page != MAX_ORDER. Then, chunks of free page + * is not aligned to pageblock_nr_pages. + * Then we just check pagetype fist. + */ + for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { + page = __first_valid_page(pfn, pageblock_nr_pages); + if (page && get_pageblock_migratetype(page) != MIGRATE_ISOLATE) + break; + } + page = __first_valid_page(start_pfn, end_pfn - start_pfn); + if ((pfn < end_pfn) || !page) + return -EBUSY; + /* Check all pages are free or Marked as ISOLATED */ + zone = page_zone(page); + spin_lock_irqsave(&zone->lock, flags); + ret = __test_page_isolated_in_pageblock(start_pfn, end_pfn); + spin_unlock_irqrestore(&zone->lock, flags); + return ret ? 0 : -EBUSY; +} diff --git a/mm/pagewalk.c b/mm/pagewalk.c index fdc255e..87eac0e 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -126,18 +126,7 @@ static int walk_hugetlb_range(struct vm_area_struct *vma, return 0; } - -#else /* CONFIG_HUGETLB_PAGE */ -static int walk_hugetlb_range(struct vm_area_struct *vma, - unsigned long addr, unsigned long end, - struct mm_walk *walk) -{ - return 0; -} - -#endif /* CONFIG_HUGETLB_PAGE */ - - +#endif /** * walk_page_range - walk a memory map's page tables with a callback @@ -155,15 +144,11 @@ static int walk_hugetlb_range(struct vm_area_struct *vma, * associated range, and a copy of the original mm_walk for access to * the ->private or ->mm fields. * - * Usually no locks are taken, but splitting transparent huge page may - * take page table lock. And the bottom level iterator will map PTE + * No locks are taken, but the bottom level iterator will map PTE * directories from highmem if necessary. * * If any callback returns a non-zero value, the walk is aborted and * the return value is propagated back to the caller. Otherwise 0 is returned. - * - * walk->mm->mmap_sem must be held for at least read if walk->hugetlb_entry - * is !NULL. */ int walk_page_range(unsigned long addr, unsigned long end, struct mm_walk *walk) @@ -178,58 +163,33 @@ int walk_page_range(unsigned long addr, unsigned long end, if (!walk->mm) return -EINVAL; - VM_BUG_ON(!rwsem_is_locked(&walk->mm->mmap_sem)); - pgd = pgd_offset(walk->mm, addr); do { - struct vm_area_struct *vma = NULL; + struct vm_area_struct *uninitialized_var(vma); next = pgd_addr_end(addr, end); +#ifdef CONFIG_HUGETLB_PAGE /* - * This function was not intended to be vma based. - * But there are vma special cases to be handled: - * - hugetlb vma's - * - VM_PFNMAP vma's + * handle hugetlb vma individually because pagetable walk for + * the hugetlb page is dependent on the architecture and + * we can't handled it in the same manner as non-huge pages. */ vma = find_vma(walk->mm, addr); - if (vma) { - /* - * There are no page structures backing a VM_PFNMAP - * range, so do not allow split_huge_page_pmd(). - */ - if ((vma->vm_start <= addr) && - (vma->vm_flags & VM_PFNMAP)) { - if (walk->pte_hole) - err = walk->pte_hole(addr, next, walk); - if (err) - break; - pgd = pgd_offset(walk->mm, next); - continue; - } + if (vma && is_vm_hugetlb_page(vma)) { + if (vma->vm_end < next) + next = vma->vm_end; /* - * Handle hugetlb vma individually because pagetable - * walk for the hugetlb page is dependent on the - * architecture and we can't handled it in the same - * manner as non-huge pages. + * Hugepage is very tightly coupled with vma, so + * walk through hugetlb entries within a given vma. */ - if (walk->hugetlb_entry && (vma->vm_start <= addr) && - is_vm_hugetlb_page(vma)) { - if (vma->vm_end < next) - next = vma->vm_end; - /* - * Hugepage is very tightly coupled with vma, - * so walk through hugetlb entries within a - * given vma. - */ - err = walk_hugetlb_range(vma, addr, next, walk); - if (err) - break; - pgd = pgd_offset(walk->mm, next); - continue; - } + err = walk_hugetlb_range(vma, addr, next, walk); + if (err) + break; + pgd = pgd_offset(walk->mm, next); + continue; } - +#endif if (pgd_none_or_clear_bad(pgd)) { if (walk->pte_hole) err = walk->pte_hole(addr, next, walk); diff --git a/mm/percpu-vm.c b/mm/percpu-vm.c index 0539f6a..bfad724 100644 --- a/mm/percpu-vm.c +++ b/mm/percpu-vm.c @@ -50,13 +50,14 @@ static struct page **pcpu_get_pages_and_bitmap(struct pcpu_chunk *chunk, if (!pages || !bitmap) { if (may_alloc && !pages) - pages = pcpu_mem_zalloc(pages_size); + pages = pcpu_mem_alloc(pages_size); if (may_alloc && !bitmap) - bitmap = pcpu_mem_zalloc(bitmap_size); + bitmap = pcpu_mem_alloc(bitmap_size); if (!pages || !bitmap) return NULL; } + memset(pages, 0, pages_size); bitmap_copy(bitmap, chunk->populated, pcpu_unit_pages); *bitmapp = bitmap; @@ -108,7 +109,7 @@ static int pcpu_alloc_pages(struct pcpu_chunk *chunk, int page_start, int page_end) { const gfp_t gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_COLD; - unsigned int cpu, tcpu; + unsigned int cpu; int i; for_each_possible_cpu(cpu) { @@ -116,23 +117,14 @@ static int pcpu_alloc_pages(struct pcpu_chunk *chunk, struct page **pagep = &pages[pcpu_page_idx(cpu, i)]; *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0); - if (!*pagep) - goto err; + if (!*pagep) { + pcpu_free_pages(chunk, pages, populated, + page_start, page_end); + return -ENOMEM; + } } } return 0; - -err: - while (--i >= page_start) - __free_page(pages[pcpu_page_idx(cpu, i)]); - - for_each_possible_cpu(tcpu) { - if (tcpu == cpu) - break; - for (i = page_start; i < page_end; i++) - __free_page(pages[pcpu_page_idx(tcpu, i)]); - } - return -ENOMEM; } /** @@ -273,7 +265,6 @@ err: __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start), page_end - page_start); } - pcpu_post_unmap_tlb_flush(chunk, page_start, page_end); return err; } diff --git a/mm/percpu.c b/mm/percpu.c index e29a1c4..af0cc7a 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -273,11 +273,11 @@ static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk, (rs) = (re) + 1, pcpu_next_pop((chunk), &(rs), &(re), (end))) /** - * pcpu_mem_zalloc - allocate memory + * pcpu_mem_alloc - allocate memory * @size: bytes to allocate * * Allocate @size bytes. If @size is smaller than PAGE_SIZE, - * kzalloc() is used; otherwise, vzalloc() is used. The returned + * kzalloc() is used; otherwise, vmalloc() is used. The returned * memory is always zeroed. * * CONTEXT: @@ -286,7 +286,7 @@ static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk, * RETURNS: * Pointer to the allocated area on success, NULL on failure. */ -static void *pcpu_mem_zalloc(size_t size) +static void *pcpu_mem_alloc(size_t size) { if (WARN_ON_ONCE(!slab_is_available())) return NULL; @@ -302,7 +302,7 @@ static void *pcpu_mem_zalloc(size_t size) * @ptr: memory to free * @size: size of the area * - * Free @ptr. @ptr should have been allocated using pcpu_mem_zalloc(). + * Free @ptr. @ptr should have been allocated using pcpu_mem_alloc(). */ static void pcpu_mem_free(void *ptr, size_t size) { @@ -384,7 +384,7 @@ static int pcpu_extend_area_map(struct pcpu_chunk *chunk, int new_alloc) size_t old_size = 0, new_size = new_alloc * sizeof(new[0]); unsigned long flags; - new = pcpu_mem_zalloc(new_size); + new = pcpu_mem_alloc(new_size); if (!new) return -ENOMEM; @@ -604,12 +604,11 @@ static struct pcpu_chunk *pcpu_alloc_chunk(void) { struct pcpu_chunk *chunk; - chunk = pcpu_mem_zalloc(pcpu_chunk_struct_size); + chunk = pcpu_mem_alloc(pcpu_chunk_struct_size); if (!chunk) return NULL; - chunk->map = pcpu_mem_zalloc(PCPU_DFL_MAP_ALLOC * - sizeof(chunk->map[0])); + chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0])); if (!chunk->map) { kfree(chunk); return NULL; @@ -978,17 +977,6 @@ bool is_kernel_percpu_address(unsigned long addr) * address. The caller is responsible for ensuring @addr stays valid * until this function finishes. * - * percpu allocator has special setup for the first chunk, which currently - * supports either embedding in linear address space or vmalloc mapping, - * and, from the second one, the backing allocator (currently either vm or - * km) provides translation. - * - * The addr can be tranlated simply without checking if it falls into the - * first chunk. But the current code reflects better how percpu allocator - * actually works, and the verification can discover both bugs in percpu - * allocator itself and per_cpu_ptr_to_phys() callers. So we keep current - * code. - * * RETURNS: * The physical address for @addr. */ @@ -1895,8 +1883,6 @@ void __init setup_per_cpu_areas(void) if (pcpu_setup_first_chunk(ai, fc) < 0) panic("Failed to initialize percpu areas."); - - pcpu_free_alloc_info(ai); } #endif /* CONFIG_SMP */ @@ -1921,7 +1907,7 @@ void __init percpu_init_late(void) BUILD_BUG_ON(size > PAGE_SIZE); - map = pcpu_mem_zalloc(size); + map = pcpu_mem_alloc(size); BUG_ON(!map); spin_lock_irqsave(&pcpu_lock, flags); diff --git a/mm/quicklist.c b/mm/quicklist.c index 9422129..2876349 100644 --- a/mm/quicklist.c +++ b/mm/quicklist.c @@ -17,6 +17,7 @@ #include <linux/gfp.h> #include <linux/mm.h> #include <linux/mmzone.h> +#include <linux/module.h> #include <linux/quicklist.h> DEFINE_PER_CPU(struct quicklist [CONFIG_NR_QUICK], quicklist); diff --git a/mm/readahead.c b/mm/readahead.c index cbcbb02..867f9dd 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -11,7 +11,7 @@ #include <linux/fs.h> #include <linux/gfp.h> #include <linux/mm.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/blkdev.h> #include <linux/backing-dev.h> #include <linux/task_io_accounting_ops.h> @@ -21,6 +21,7 @@ * Lock ordering in mm: * * inode->i_mutex (while writing or truncating, not reading or faulting) + * inode->i_alloc_sem (vmtruncate_range) * mm->mmap_sem * page->flags PG_locked (lock_page) * mapping->i_mmap_mutex @@ -31,11 +32,11 @@ * mmlist_lock (in mmput, drain_mmlist and others) * mapping->private_lock (in __set_page_dirty_buffers) * inode->i_lock (in set_page_dirty's __mark_inode_dirty) - * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) + * inode_wb_list_lock (in set_page_dirty's __mark_inode_dirty) * sb_lock (within inode_lock in fs/fs-writeback.c) * mapping->tree_lock (widely used, in set_page_dirty, * in arch-dependent flush_dcache_mmap_lock, - * within bdi.wb->list_lock in __sync_single_inode) + * within inode_wb_list_lock in __sync_single_inode) * * anon_vma->mutex,mapping->i_mutex (memory_failure, collect_procs_anon) * ->tasklist_lock @@ -51,7 +52,7 @@ #include <linux/ksm.h> #include <linux/rmap.h> #include <linux/rcupdate.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/memcontrol.h> #include <linux/mmu_notifier.h> #include <linux/migrate.h> @@ -72,8 +73,6 @@ static inline struct anon_vma *anon_vma_alloc(void) anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); if (anon_vma) { atomic_set(&anon_vma->refcount, 1); - anon_vma->degree = 1; /* Reference for first vma */ - anon_vma->parent = anon_vma; /* * Initialise the anon_vma root to point to itself. If called * from fork, the root will be reset to the parents anon_vma. @@ -105,7 +104,6 @@ static inline void anon_vma_free(struct anon_vma *anon_vma) * LOCK should suffice since the actual taking of the lock must * happen _before_ what follows. */ - might_sleep(); if (mutex_is_locked(&anon_vma->root->mutex)) { anon_vma_lock(anon_vma); anon_vma_unlock(anon_vma); @@ -183,8 +181,6 @@ int anon_vma_prepare(struct vm_area_struct *vma) avc->vma = vma; list_add(&avc->same_vma, &vma->anon_vma_chain); list_add_tail(&avc->same_anon_vma, &anon_vma->head); - /* vma reference or self-parent link for new root */ - anon_vma->degree++; allocated = NULL; avc = NULL; } @@ -248,14 +244,6 @@ static void anon_vma_chain_link(struct vm_area_struct *vma, /* * Attach the anon_vmas from src to dst. * Returns 0 on success, -ENOMEM on failure. - * - * If dst->anon_vma is NULL this function tries to find and reuse existing - * anon_vma which has no vmas and only one child anon_vma. This prevents - * degradation of anon_vma hierarchy to endless linear chain in case of - * constantly forking task. On the other hand, an anon_vma with more than one - * child isn't reused even if there was no alive vma, thus rmap walker has a - * good chance of avoiding scanning the whole hierarchy when it searches where - * page is mapped. */ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) { @@ -276,32 +264,11 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) anon_vma = pavc->anon_vma; root = lock_anon_vma_root(root, anon_vma); anon_vma_chain_link(dst, avc, anon_vma); - - /* - * Reuse existing anon_vma if its degree lower than two, - * that means it has no vma and only one anon_vma child. - * - * Do not chose parent anon_vma, otherwise first child - * will always reuse it. Root anon_vma is never reused: - * it has self-parent reference and at least one child. - */ - if (!dst->anon_vma && anon_vma != src->anon_vma && - anon_vma->degree < 2) - dst->anon_vma = anon_vma; } - if (dst->anon_vma) - dst->anon_vma->degree++; unlock_anon_vma_root(root); return 0; enomem_failure: - /* - * dst->anon_vma is dropped here otherwise its degree can be incorrectly - * decremented in unlink_anon_vmas(). - * We can safely do this because callers of anon_vma_clone() don't care - * about dst->anon_vma if anon_vma_clone() failed. - */ - dst->anon_vma = NULL; unlink_anon_vmas(dst); return -ENOMEM; } @@ -320,9 +287,6 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) if (!pvma->anon_vma) return 0; - /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ - vma->anon_vma = NULL; - /* * First, attach the new VMA to the parent VMA's anon_vmas, * so rmap can find non-COWed pages in child processes. @@ -330,10 +294,6 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) if (anon_vma_clone(vma, pvma)) return -ENOMEM; - /* An existing anon_vma has been reused, all done then. */ - if (vma->anon_vma) - return 0; - /* Then add our own anon_vma. */ anon_vma = anon_vma_alloc(); if (!anon_vma) @@ -347,7 +307,6 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) * lock any of the anon_vmas in this anon_vma tree. */ anon_vma->root = pvma->anon_vma->root; - anon_vma->parent = pvma->anon_vma; /* * With refcounts, an anon_vma can stay around longer than the * process it belongs to. The root anon_vma needs to be pinned until @@ -358,7 +317,6 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) vma->anon_vma = anon_vma; anon_vma_lock(anon_vma); anon_vma_chain_link(vma, avc, anon_vma); - anon_vma->parent->degree++; anon_vma_unlock(anon_vma); return 0; @@ -389,16 +347,12 @@ void unlink_anon_vmas(struct vm_area_struct *vma) * Leave empty anon_vmas on the list - we'll need * to free them outside the lock. */ - if (list_empty(&anon_vma->head)) { - anon_vma->parent->degree--; + if (list_empty(&anon_vma->head)) continue; - } list_del(&avc->same_vma); anon_vma_chain_free(avc); } - if (vma->anon_vma) - vma->anon_vma->degree--; unlock_anon_vma_root(root); /* @@ -409,7 +363,6 @@ void unlink_anon_vmas(struct vm_area_struct *vma) list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; - BUG_ON(anon_vma->degree); put_anon_vma(anon_vma); list_del(&avc->same_vma); @@ -482,9 +435,8 @@ struct anon_vma *page_get_anon_vma(struct page *page) * above cannot corrupt). */ if (!page_mapped(page)) { - rcu_read_unlock(); put_anon_vma(anon_vma); - return NULL; + anon_vma = NULL; } out: rcu_read_unlock(); @@ -534,9 +486,9 @@ struct anon_vma *page_lock_anon_vma(struct page *page) } if (!page_mapped(page)) { - rcu_read_unlock(); put_anon_vma(anon_vma); - return NULL; + anon_vma = NULL; + goto out; } /* we pinned the anon_vma, its safe to sleep */ @@ -630,11 +582,7 @@ pte_t *__page_check_address(struct page *page, struct mm_struct *mm, spinlock_t *ptl; if (unlikely(PageHuge(page))) { - /* when pud is not present, pte will be NULL */ pte = huge_pte_offset(mm, address); - if (!pte) - return NULL; - ptl = &mm->page_table_lock; goto check; } @@ -923,11 +871,11 @@ int page_referenced(struct page *page, vm_flags); if (we_locked) unlock_page(page); - - if (page_test_and_clear_young(page_to_pfn(page))) - referenced++; } out: + if (page_test_and_clear_young(page_to_pfn(page))) + referenced++; + return referenced; } @@ -1228,7 +1176,7 @@ void page_remove_rmap(struct page *page) /* * Subfunctions of try_to_unmap: try_to_unmap_one called - * repeatedly from try_to_unmap_ksm, try_to_unmap_anon or try_to_unmap_file. + * repeatedly from either try_to_unmap_anon or try_to_unmap_file. */ int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, unsigned long address, enum ttu_flags flags) @@ -1434,19 +1382,9 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount, BUG_ON(!page || PageAnon(page)); if (locked_vma) { - if (page == check_page) { - /* we know we have check_page locked */ - mlock_vma_page(page); + mlock_vma_page(page); /* no-op if already mlocked */ + if (page == check_page) ret = SWAP_MLOCK; - } else if (trylock_page(page)) { - /* - * If we can lock the page, perform mlock. - * Otherwise leave the page alone, it will be - * eventually encountered again later. - */ - mlock_vma_page(page); - unlock_page(page); - } continue; /* don't unmap */ } @@ -1718,9 +1656,10 @@ void __put_anon_vma(struct anon_vma *anon_vma) { struct anon_vma *root = anon_vma->root; - anon_vma_free(anon_vma); if (root != anon_vma && atomic_dec_and_test(&root->refcount)) anon_vma_free(root); + + anon_vma_free(anon_vma); } #ifdef CONFIG_MIGRATION @@ -6,8 +6,7 @@ * 2000-2001 Christoph Rohland * 2000-2001 SAP AG * 2002 Red Hat Inc. - * Copyright (C) 2002-2011 Hugh Dickins. - * Copyright (C) 2011 Google Inc. + * Copyright (C) 2002-2005 Hugh Dickins. * Copyright (C) 2002-2005 VERITAS Software Corporation. * Copyright (C) 2004 Andi Kleen, SuSE Labs * @@ -28,7 +27,8 @@ #include <linux/pagemap.h> #include <linux/file.h> #include <linux/mm.h> -#include <linux/export.h> +#include <linux/module.h> +#include <linux/percpu_counter.h> #include <linux/swap.h> static struct vfsmount *shm_mnt; @@ -51,9 +51,6 @@ static struct vfsmount *shm_mnt; #include <linux/shmem_fs.h> #include <linux/writeback.h> #include <linux/blkdev.h> -#include <linux/pagevec.h> -#include <linux/percpu_counter.h> -#include <linux/splice.h> #include <linux/security.h> #include <linux/swapops.h> #include <linux/mempolicy.h> @@ -65,27 +62,42 @@ static struct vfsmount *shm_mnt; #include <linux/magic.h> #include <asm/uaccess.h> +#include <asm/div64.h> #include <asm/pgtable.h> +/* + * The maximum size of a shmem/tmpfs file is limited by the maximum size of + * its triple-indirect swap vector - see illustration at shmem_swp_entry(). + * + * With 4kB page size, maximum file size is just over 2TB on a 32-bit kernel, + * but one eighth of that on a 64-bit kernel. With 8kB page size, maximum + * file size is just over 4TB on a 64-bit kernel, but 16TB on a 32-bit kernel, + * MAX_LFS_FILESIZE being then more restrictive than swap vector layout. + * + * We use / and * instead of shifts in the definitions below, so that the swap + * vector can be tested with small even values (e.g. 20) for ENTRIES_PER_PAGE. + */ +#define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long)) +#define ENTRIES_PER_PAGEPAGE ((unsigned long long)ENTRIES_PER_PAGE*ENTRIES_PER_PAGE) + +#define SHMSWP_MAX_INDEX (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1)) +#define SHMSWP_MAX_BYTES (SHMSWP_MAX_INDEX << PAGE_CACHE_SHIFT) + +#define SHMEM_MAX_BYTES min_t(unsigned long long, SHMSWP_MAX_BYTES, MAX_LFS_FILESIZE) +#define SHMEM_MAX_INDEX ((unsigned long)((SHMEM_MAX_BYTES+1) >> PAGE_CACHE_SHIFT)) + #define BLOCKS_PER_PAGE (PAGE_CACHE_SIZE/512) #define VM_ACCT(size) (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT) -/* Pretend that each entry is of this size in directory's i_size */ -#define BOGO_DIRENT_SIZE 20 +/* info->flags needs VM_flags to handle pagein/truncate races efficiently */ +#define SHMEM_PAGEIN VM_READ +#define SHMEM_TRUNCATE VM_WRITE -/* Symlink up to this size is kmalloc'ed instead of using a swappable page */ -#define SHORT_SYMLINK_LEN 128 +/* Definition to limit shmem_truncate's steps between cond_rescheds */ +#define LATENCY_LIMIT 64 -/* - * vmtruncate_range() communicates with shmem_fault via - * inode->i_private (with i_mutex making sure that it has only one user at - * a time): we would prefer not to enlarge the shmem inode just for that. - */ -struct shmem_falloc { - wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ - pgoff_t start; /* start of range currently being fallocated */ - pgoff_t next; /* the next page offset to be fallocated */ -}; +/* Pretend that each entry is of this size in directory's i_size */ +#define BOGO_DIRENT_SIZE 20 struct shmem_xattr { struct list_head list; /* anchored by shmem_inode_info->xattr_list */ @@ -94,7 +106,7 @@ struct shmem_xattr { char value[0]; }; -/* Flag allocation requirements to shmem_getpage */ +/* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */ enum sgp_type { SGP_READ, /* don't exceed i_size, don't allocate page */ SGP_CACHE, /* don't exceed i_size, may allocate page */ @@ -114,14 +126,57 @@ static unsigned long shmem_default_max_inodes(void) } #endif -static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, - struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type); +static int shmem_getpage(struct inode *inode, unsigned long idx, + struct page **pagep, enum sgp_type sgp, int *type); + +static inline struct page *shmem_dir_alloc(gfp_t gfp_mask) +{ + /* + * The above definition of ENTRIES_PER_PAGE, and the use of + * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE: + * might be reconsidered if it ever diverges from PAGE_SIZE. + * + * Mobility flags are masked out as swap vectors cannot move + */ + return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO, + PAGE_CACHE_SHIFT-PAGE_SHIFT); +} -static inline int shmem_getpage(struct inode *inode, pgoff_t index, - struct page **pagep, enum sgp_type sgp, int *fault_type) +static inline void shmem_dir_free(struct page *page) { - return shmem_getpage_gfp(inode, index, pagep, sgp, - mapping_gfp_mask(inode->i_mapping), fault_type); + __free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT); +} + +static struct page **shmem_dir_map(struct page *page) +{ + return (struct page **)kmap_atomic(page, KM_USER0); +} + +static inline void shmem_dir_unmap(struct page **dir) +{ + kunmap_atomic(dir, KM_USER0); +} + +static swp_entry_t *shmem_swp_map(struct page *page) +{ + return (swp_entry_t *)kmap_atomic(page, KM_USER1); +} + +static inline void shmem_swp_balance_unmap(void) +{ + /* + * When passing a pointer to an i_direct entry, to code which + * also handles indirect entries and so will shmem_swp_unmap, + * we must arrange for the preempt count to remain in balance. + * What kmap_atomic of a lowmem page does depends on config + * and architecture, so pretend to kmap_atomic some lowmem page. + */ + (void) kmap_atomic(ZERO_PAGE(0), KM_USER1); +} + +static inline void shmem_swp_unmap(swp_entry_t *entry) +{ + kunmap_atomic(entry, KM_USER1); } static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) @@ -181,6 +236,17 @@ static struct backing_dev_info shmem_backing_dev_info __read_mostly = { static LIST_HEAD(shmem_swaplist); static DEFINE_MUTEX(shmem_swaplist_mutex); +static void shmem_free_blocks(struct inode *inode, long pages) +{ + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + if (sbinfo->max_blocks) { + percpu_counter_add(&sbinfo->used_blocks, -pages); + spin_lock(&inode->i_lock); + inode->i_blocks -= pages*BLOCKS_PER_PAGE; + spin_unlock(&inode->i_lock); + } +} + static int shmem_reserve_inode(struct super_block *sb) { struct shmem_sb_info *sbinfo = SHMEM_SB(sb); @@ -207,7 +273,7 @@ static void shmem_free_inode(struct super_block *sb) } /** - * shmem_recalc_inode - recalculate the block usage of an inode + * shmem_recalc_inode - recalculate the size of an inode * @inode: inode to recalc * * We have to calculate the free blocks since the mm can drop @@ -225,335 +291,474 @@ static void shmem_recalc_inode(struct inode *inode) freed = info->alloced - info->swapped - inode->i_mapping->nrpages; if (freed > 0) { - struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); - if (sbinfo->max_blocks) - percpu_counter_add(&sbinfo->used_blocks, -freed); info->alloced -= freed; - inode->i_blocks -= freed * BLOCKS_PER_PAGE; shmem_unacct_blocks(info->flags, freed); + shmem_free_blocks(inode, freed); } } -/* - * Replace item expected in radix tree by a new item, while holding tree lock. - */ -static int shmem_radix_tree_replace(struct address_space *mapping, - pgoff_t index, void *expected, void *replacement) -{ - void **pslot; - void *item = NULL; - - VM_BUG_ON(!expected); - pslot = radix_tree_lookup_slot(&mapping->page_tree, index); - if (pslot) - item = radix_tree_deref_slot_protected(pslot, - &mapping->tree_lock); - if (item != expected) - return -ENOENT; - if (replacement) - radix_tree_replace_slot(pslot, replacement); - else - radix_tree_delete(&mapping->page_tree, index); - return 0; -} - -/* - * Like add_to_page_cache_locked, but error if expected item has gone. +/** + * shmem_swp_entry - find the swap vector position in the info structure + * @info: info structure for the inode + * @index: index of the page to find + * @page: optional page to add to the structure. Has to be preset to + * all zeros + * + * If there is no space allocated yet it will return NULL when + * page is NULL, else it will use the page for the needed block, + * setting it to NULL on return to indicate that it has been used. + * + * The swap vector is organized the following way: + * + * There are SHMEM_NR_DIRECT entries directly stored in the + * shmem_inode_info structure. So small files do not need an addional + * allocation. + * + * For pages with index > SHMEM_NR_DIRECT there is the pointer + * i_indirect which points to a page which holds in the first half + * doubly indirect blocks, in the second half triple indirect blocks: + * + * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the + * following layout (for SHMEM_NR_DIRECT == 16): + * + * i_indirect -> dir --> 16-19 + * | +-> 20-23 + * | + * +-->dir2 --> 24-27 + * | +-> 28-31 + * | +-> 32-35 + * | +-> 36-39 + * | + * +-->dir3 --> 40-43 + * +-> 44-47 + * +-> 48-51 + * +-> 52-55 */ -static int shmem_add_to_page_cache(struct page *page, - struct address_space *mapping, - pgoff_t index, gfp_t gfp, void *expected) +static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page) { - int error = 0; - - VM_BUG_ON(!PageLocked(page)); - VM_BUG_ON(!PageSwapBacked(page)); + unsigned long offset; + struct page **dir; + struct page *subdir; - if (!expected) - error = radix_tree_preload(gfp & GFP_RECLAIM_MASK); - if (!error) { - page_cache_get(page); - page->mapping = mapping; - page->index = index; - - spin_lock_irq(&mapping->tree_lock); - if (!expected) - error = radix_tree_insert(&mapping->page_tree, - index, page); - else - error = shmem_radix_tree_replace(mapping, index, - expected, page); - if (!error) { - mapping->nrpages++; - __inc_zone_page_state(page, NR_FILE_PAGES); - __inc_zone_page_state(page, NR_SHMEM); - spin_unlock_irq(&mapping->tree_lock); - } else { - page->mapping = NULL; - spin_unlock_irq(&mapping->tree_lock); - page_cache_release(page); + if (index < SHMEM_NR_DIRECT) { + shmem_swp_balance_unmap(); + return info->i_direct+index; + } + if (!info->i_indirect) { + if (page) { + info->i_indirect = *page; + *page = NULL; } - if (!expected) - radix_tree_preload_end(); + return NULL; /* need another page */ } - if (error) - mem_cgroup_uncharge_cache_page(page); - return error; -} - -/* - * Like delete_from_page_cache, but substitutes swap for page. - */ -static void shmem_delete_from_page_cache(struct page *page, void *radswap) -{ - struct address_space *mapping = page->mapping; - int error; - spin_lock_irq(&mapping->tree_lock); - error = shmem_radix_tree_replace(mapping, page->index, page, radswap); - page->mapping = NULL; - mapping->nrpages--; - __dec_zone_page_state(page, NR_FILE_PAGES); - __dec_zone_page_state(page, NR_SHMEM); - spin_unlock_irq(&mapping->tree_lock); - page_cache_release(page); - BUG_ON(error); -} - -/* - * Like find_get_pages, but collecting swap entries as well as pages. - */ -static unsigned shmem_find_get_pages_and_swap(struct address_space *mapping, - pgoff_t start, unsigned int nr_pages, - struct page **pages, pgoff_t *indices) -{ - unsigned int i; - unsigned int ret; - unsigned int nr_found; - - rcu_read_lock(); -restart: - nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree, - (void ***)pages, indices, start, nr_pages); - ret = 0; - for (i = 0; i < nr_found; i++) { - struct page *page; -repeat: - page = radix_tree_deref_slot((void **)pages[i]); - if (unlikely(!page)) - continue; - if (radix_tree_exception(page)) { - if (radix_tree_deref_retry(page)) - goto restart; - /* - * Otherwise, we must be storing a swap entry - * here as an exceptional entry: so return it - * without attempting to raise page count. - */ - goto export; + index -= SHMEM_NR_DIRECT; + offset = index % ENTRIES_PER_PAGE; + index /= ENTRIES_PER_PAGE; + dir = shmem_dir_map(info->i_indirect); + + if (index >= ENTRIES_PER_PAGE/2) { + index -= ENTRIES_PER_PAGE/2; + dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE; + index %= ENTRIES_PER_PAGE; + subdir = *dir; + if (!subdir) { + if (page) { + *dir = *page; + *page = NULL; + } + shmem_dir_unmap(dir); + return NULL; /* need another page */ } - if (!page_cache_get_speculative(page)) - goto repeat; + shmem_dir_unmap(dir); + dir = shmem_dir_map(subdir); + } - /* Has the page moved? */ - if (unlikely(page != *((void **)pages[i]))) { - page_cache_release(page); - goto repeat; + dir += index; + subdir = *dir; + if (!subdir) { + if (!page || !(subdir = *page)) { + shmem_dir_unmap(dir); + return NULL; /* need a page */ } -export: - indices[ret] = indices[i]; - pages[ret] = page; - ret++; - } - if (unlikely(!ret && nr_found)) - goto restart; - rcu_read_unlock(); - return ret; + *dir = subdir; + *page = NULL; + } + shmem_dir_unmap(dir); + return shmem_swp_map(subdir) + offset; } -/* - * Remove swap entry from radix tree, free the swap and its page cache. - */ -static int shmem_free_swap(struct address_space *mapping, - pgoff_t index, void *radswap) +static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value) { - int error; - - spin_lock_irq(&mapping->tree_lock); - error = shmem_radix_tree_replace(mapping, index, radswap, NULL); - spin_unlock_irq(&mapping->tree_lock); - if (!error) - free_swap_and_cache(radix_to_swp_entry(radswap)); - return error; -} + long incdec = value? 1: -1; -/* - * Pagevec may contain swap entries, so shuffle up pages before releasing. - */ -static void shmem_deswap_pagevec(struct pagevec *pvec) -{ - int i, j; - - for (i = 0, j = 0; i < pagevec_count(pvec); i++) { - struct page *page = pvec->pages[i]; - if (!radix_tree_exceptional_entry(page)) - pvec->pages[j++] = page; + entry->val = value; + info->swapped += incdec; + if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) { + struct page *page = kmap_atomic_to_page(entry); + set_page_private(page, page_private(page) + incdec); } - pvec->nr = j; } -/* - * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. +/** + * shmem_swp_alloc - get the position of the swap entry for the page. + * @info: info structure for the inode + * @index: index of the page to find + * @sgp: check and recheck i_size? skip allocation? + * + * If the entry does not exist, allocate it. */ -void shmem_unlock_mapping(struct address_space *mapping) +static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp) { - struct pagevec pvec; - pgoff_t indices[PAGEVEC_SIZE]; - pgoff_t index = 0; + struct inode *inode = &info->vfs_inode; + struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); + struct page *page = NULL; + swp_entry_t *entry; - pagevec_init(&pvec, 0); - /* - * Minor point, but we might as well stop if someone else SHM_LOCKs it. - */ - while (!mapping_unevictable(mapping)) { + if (sgp != SGP_WRITE && + ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) + return ERR_PTR(-EINVAL); + + while (!(entry = shmem_swp_entry(info, index, &page))) { + if (sgp == SGP_READ) + return shmem_swp_map(ZERO_PAGE(0)); /* - * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it - * has finished, if it hits a row of PAGEVEC_SIZE swap entries. + * Test used_blocks against 1 less max_blocks, since we have 1 data + * page (and perhaps indirect index pages) yet to allocate: + * a waste to allocate index if we cannot allocate data. */ - pvec.nr = shmem_find_get_pages_and_swap(mapping, index, - PAGEVEC_SIZE, pvec.pages, indices); - if (!pvec.nr) + if (sbinfo->max_blocks) { + if (percpu_counter_compare(&sbinfo->used_blocks, + sbinfo->max_blocks - 1) >= 0) + return ERR_PTR(-ENOSPC); + percpu_counter_inc(&sbinfo->used_blocks); + spin_lock(&inode->i_lock); + inode->i_blocks += BLOCKS_PER_PAGE; + spin_unlock(&inode->i_lock); + } + + spin_unlock(&info->lock); + page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping)); + spin_lock(&info->lock); + + if (!page) { + shmem_free_blocks(inode, 1); + return ERR_PTR(-ENOMEM); + } + if (sgp != SGP_WRITE && + ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) { + entry = ERR_PTR(-EINVAL); break; - index = indices[pvec.nr - 1] + 1; - shmem_deswap_pagevec(&pvec); - check_move_unevictable_pages(pvec.pages, pvec.nr); - pagevec_release(&pvec); - cond_resched(); + } + if (info->next_index <= index) + info->next_index = index + 1; + } + if (page) { + /* another task gave its page, or truncated the file */ + shmem_free_blocks(inode, 1); + shmem_dir_free(page); } + if (info->next_index <= index && !IS_ERR(entry)) + info->next_index = index + 1; + return entry; } -/* - * Remove range of pages and swap entries from radix tree, and free them. +/** + * shmem_free_swp - free some swap entries in a directory + * @dir: pointer to the directory + * @edir: pointer after last entry of the directory + * @punch_lock: pointer to spinlock when needed for the holepunch case */ -void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) +static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir, + spinlock_t *punch_lock) +{ + spinlock_t *punch_unlock = NULL; + swp_entry_t *ptr; + int freed = 0; + + for (ptr = dir; ptr < edir; ptr++) { + if (ptr->val) { + if (unlikely(punch_lock)) { + punch_unlock = punch_lock; + punch_lock = NULL; + spin_lock(punch_unlock); + if (!ptr->val) + continue; + } + free_swap_and_cache(*ptr); + *ptr = (swp_entry_t){0}; + freed++; + } + } + if (punch_unlock) + spin_unlock(punch_unlock); + return freed; +} + +static int shmem_map_and_free_swp(struct page *subdir, int offset, + int limit, struct page ***dir, spinlock_t *punch_lock) +{ + swp_entry_t *ptr; + int freed = 0; + + ptr = shmem_swp_map(subdir); + for (; offset < limit; offset += LATENCY_LIMIT) { + int size = limit - offset; + if (size > LATENCY_LIMIT) + size = LATENCY_LIMIT; + freed += shmem_free_swp(ptr+offset, ptr+offset+size, + punch_lock); + if (need_resched()) { + shmem_swp_unmap(ptr); + if (*dir) { + shmem_dir_unmap(*dir); + *dir = NULL; + } + cond_resched(); + ptr = shmem_swp_map(subdir); + } + } + shmem_swp_unmap(ptr); + return freed; +} + +static void shmem_free_pages(struct list_head *next) +{ + struct page *page; + int freed = 0; + + do { + page = container_of(next, struct page, lru); + next = next->next; + shmem_dir_free(page); + freed++; + if (freed >= LATENCY_LIMIT) { + cond_resched(); + freed = 0; + } + } while (next); +} + +void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end) { - struct address_space *mapping = inode->i_mapping; struct shmem_inode_info *info = SHMEM_I(inode); - pgoff_t start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; - unsigned partial = lstart & (PAGE_CACHE_SIZE - 1); - pgoff_t end = (lend >> PAGE_CACHE_SHIFT); - struct pagevec pvec; - pgoff_t indices[PAGEVEC_SIZE]; + unsigned long idx; + unsigned long size; + unsigned long limit; + unsigned long stage; + unsigned long diroff; + struct page **dir; + struct page *topdir; + struct page *middir; + struct page *subdir; + swp_entry_t *ptr; + LIST_HEAD(pages_to_free); + long nr_pages_to_free = 0; long nr_swaps_freed = 0; - pgoff_t index; - int i; - - BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1)); - - pagevec_init(&pvec, 0); - index = start; - while (index <= end) { - pvec.nr = shmem_find_get_pages_and_swap(mapping, index, - min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1, - pvec.pages, indices); - if (!pvec.nr) - break; - mem_cgroup_uncharge_start(); - for (i = 0; i < pagevec_count(&pvec); i++) { - struct page *page = pvec.pages[i]; + int offset; + int freed; + int punch_hole; + spinlock_t *needs_lock; + spinlock_t *punch_lock; + unsigned long upper_limit; - index = indices[i]; - if (index > end) - break; + truncate_inode_pages_range(inode->i_mapping, start, end); - if (radix_tree_exceptional_entry(page)) { - nr_swaps_freed += !shmem_free_swap(mapping, - index, page); - continue; - } + inode->i_ctime = inode->i_mtime = CURRENT_TIME; + idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; + if (idx >= info->next_index) + return; - if (!trylock_page(page)) - continue; - if (page->mapping == mapping) { - VM_BUG_ON(PageWriteback(page)); - truncate_inode_page(mapping, page); - } - unlock_page(page); + spin_lock(&info->lock); + info->flags |= SHMEM_TRUNCATE; + if (likely(end == (loff_t) -1)) { + limit = info->next_index; + upper_limit = SHMEM_MAX_INDEX; + info->next_index = idx; + needs_lock = NULL; + punch_hole = 0; + } else { + if (end + 1 >= inode->i_size) { /* we may free a little more */ + limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >> + PAGE_CACHE_SHIFT; + upper_limit = SHMEM_MAX_INDEX; + } else { + limit = (end + 1) >> PAGE_CACHE_SHIFT; + upper_limit = limit; } - shmem_deswap_pagevec(&pvec); - pagevec_release(&pvec); - mem_cgroup_uncharge_end(); - cond_resched(); - index++; + needs_lock = &info->lock; + punch_hole = 1; } - if (partial) { - struct page *page = NULL; - shmem_getpage(inode, start - 1, &page, SGP_READ, NULL); - if (page) { - zero_user_segment(page, partial, PAGE_CACHE_SIZE); - set_page_dirty(page); - unlock_page(page); - page_cache_release(page); - } + topdir = info->i_indirect; + if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) { + info->i_indirect = NULL; + nr_pages_to_free++; + list_add(&topdir->lru, &pages_to_free); } + spin_unlock(&info->lock); - index = start; - while (index <= end) { - cond_resched(); - pvec.nr = shmem_find_get_pages_and_swap(mapping, index, - min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1, - pvec.pages, indices); - if (!pvec.nr) { - /* If all gone or hole-punch, we're done */ - if (index == start || end != -1) - break; - /* But if truncating, restart to make sure all gone */ - index = start; - continue; - } - mem_cgroup_uncharge_start(); - for (i = 0; i < pagevec_count(&pvec); i++) { - struct page *page = pvec.pages[i]; + if (info->swapped && idx < SHMEM_NR_DIRECT) { + ptr = info->i_direct; + size = limit; + if (size > SHMEM_NR_DIRECT) + size = SHMEM_NR_DIRECT; + nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock); + } - index = indices[i]; - if (index > end) - break; + /* + * If there are no indirect blocks or we are punching a hole + * below indirect blocks, nothing to be done. + */ + if (!topdir || limit <= SHMEM_NR_DIRECT) + goto done2; - if (radix_tree_exceptional_entry(page)) { - if (shmem_free_swap(mapping, index, page)) { - /* Swap was replaced by page: retry */ - index--; - break; - } - nr_swaps_freed++; - continue; + /* + * The truncation case has already dropped info->lock, and we're safe + * because i_size and next_index have already been lowered, preventing + * access beyond. But in the punch_hole case, we still need to take + * the lock when updating the swap directory, because there might be + * racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or + * shmem_writepage. However, whenever we find we can remove a whole + * directory page (not at the misaligned start or end of the range), + * we first NULLify its pointer in the level above, and then have no + * need to take the lock when updating its contents: needs_lock and + * punch_lock (either pointing to info->lock or NULL) manage this. + */ + + upper_limit -= SHMEM_NR_DIRECT; + limit -= SHMEM_NR_DIRECT; + idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0; + offset = idx % ENTRIES_PER_PAGE; + idx -= offset; + + dir = shmem_dir_map(topdir); + stage = ENTRIES_PER_PAGEPAGE/2; + if (idx < ENTRIES_PER_PAGEPAGE/2) { + middir = topdir; + diroff = idx/ENTRIES_PER_PAGE; + } else { + dir += ENTRIES_PER_PAGE/2; + dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE; + while (stage <= idx) + stage += ENTRIES_PER_PAGEPAGE; + middir = *dir; + if (*dir) { + diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) % + ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE; + if (!diroff && !offset && upper_limit >= stage) { + if (needs_lock) { + spin_lock(needs_lock); + *dir = NULL; + spin_unlock(needs_lock); + needs_lock = NULL; + } else + *dir = NULL; + nr_pages_to_free++; + list_add(&middir->lru, &pages_to_free); } + shmem_dir_unmap(dir); + dir = shmem_dir_map(middir); + } else { + diroff = 0; + offset = 0; + idx = stage; + } + } - lock_page(page); - if (page->mapping == mapping) { - VM_BUG_ON(PageWriteback(page)); - truncate_inode_page(mapping, page); - } else { - /* Page was replaced by swap: retry */ - unlock_page(page); - index--; - break; + for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) { + if (unlikely(idx == stage)) { + shmem_dir_unmap(dir); + dir = shmem_dir_map(topdir) + + ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE; + while (!*dir) { + dir++; + idx += ENTRIES_PER_PAGEPAGE; + if (idx >= limit) + goto done1; } - unlock_page(page); + stage = idx + ENTRIES_PER_PAGEPAGE; + middir = *dir; + if (punch_hole) + needs_lock = &info->lock; + if (upper_limit >= stage) { + if (needs_lock) { + spin_lock(needs_lock); + *dir = NULL; + spin_unlock(needs_lock); + needs_lock = NULL; + } else + *dir = NULL; + nr_pages_to_free++; + list_add(&middir->lru, &pages_to_free); + } + shmem_dir_unmap(dir); + cond_resched(); + dir = shmem_dir_map(middir); + diroff = 0; } - shmem_deswap_pagevec(&pvec); - pagevec_release(&pvec); - mem_cgroup_uncharge_end(); - index++; + punch_lock = needs_lock; + subdir = dir[diroff]; + if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) { + if (needs_lock) { + spin_lock(needs_lock); + dir[diroff] = NULL; + spin_unlock(needs_lock); + punch_lock = NULL; + } else + dir[diroff] = NULL; + nr_pages_to_free++; + list_add(&subdir->lru, &pages_to_free); + } + if (subdir && page_private(subdir) /* has swap entries */) { + size = limit - idx; + if (size > ENTRIES_PER_PAGE) + size = ENTRIES_PER_PAGE; + freed = shmem_map_and_free_swp(subdir, + offset, size, &dir, punch_lock); + if (!dir) + dir = shmem_dir_map(middir); + nr_swaps_freed += freed; + if (offset || punch_lock) { + spin_lock(&info->lock); + set_page_private(subdir, + page_private(subdir) - freed); + spin_unlock(&info->lock); + } else + BUG_ON(page_private(subdir) != freed); + } + offset = 0; + } +done1: + shmem_dir_unmap(dir); +done2: + if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) { + /* + * Call truncate_inode_pages again: racing shmem_unuse_inode + * may have swizzled a page in from swap since + * truncate_pagecache or generic_delete_inode did it, before we + * lowered next_index. Also, though shmem_getpage checks + * i_size before adding to cache, no recheck after: so fix the + * narrow window there too. + */ + truncate_inode_pages_range(inode->i_mapping, start, end); } spin_lock(&info->lock); + info->flags &= ~SHMEM_TRUNCATE; info->swapped -= nr_swaps_freed; + if (nr_pages_to_free) + shmem_free_blocks(inode, nr_pages_to_free); shmem_recalc_inode(inode); spin_unlock(&info->lock); - inode->i_ctime = inode->i_mtime = CURRENT_TIME; + /* + * Empty swap vector directory pages to be freed? + */ + if (!list_empty(&pages_to_free)) { + pages_to_free.prev->next = NULL; + shmem_free_pages(pages_to_free.next); + } } EXPORT_SYMBOL_GPL(shmem_truncate_range); @@ -569,7 +774,37 @@ static int shmem_setattr(struct dentry *dentry, struct iattr *attr) if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { loff_t oldsize = inode->i_size; loff_t newsize = attr->ia_size; + struct page *page = NULL; + if (newsize < oldsize) { + /* + * If truncating down to a partial page, then + * if that page is already allocated, hold it + * in memory until the truncation is over, so + * truncate_partial_page cannot miss it were + * it assigned to swap. + */ + if (newsize & (PAGE_CACHE_SIZE-1)) { + (void) shmem_getpage(inode, + newsize >> PAGE_CACHE_SHIFT, + &page, SGP_READ, NULL); + if (page) + unlock_page(page); + } + /* + * Reset SHMEM_PAGEIN flag so that shmem_truncate can + * detect if any pages might have been added to cache + * after truncate_inode_pages. But we needn't bother + * if it's being fully truncated to zero-length: the + * nrpages check is efficient enough in that case. + */ + if (newsize) { + struct shmem_inode_info *info = SHMEM_I(inode); + spin_lock(&info->lock); + info->flags &= ~SHMEM_PAGEIN; + spin_unlock(&info->lock); + } + } if (newsize != oldsize) { i_size_write(inode, newsize); inode->i_ctime = inode->i_mtime = CURRENT_TIME; @@ -581,6 +816,8 @@ static int shmem_setattr(struct dentry *dentry, struct iattr *attr) /* unmap again to remove racily COWed private pages */ unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); } + if (page) + page_cache_release(page); } setattr_copy(inode, attr); @@ -605,39 +842,117 @@ static void shmem_evict_inode(struct inode *inode) list_del_init(&info->swaplist); mutex_unlock(&shmem_swaplist_mutex); } - } else - kfree(info->symlink); + } list_for_each_entry_safe(xattr, nxattr, &info->xattr_list, list) { kfree(xattr->name); kfree(xattr); } - WARN_ON(inode->i_blocks); + BUG_ON(inode->i_blocks); shmem_free_inode(inode->i_sb); end_writeback(inode); } -/* - * If swap found in inode, free it and move page from swapcache to filecache. - */ -static int shmem_unuse_inode(struct shmem_inode_info *info, - swp_entry_t swap, struct page *page) +static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir) { - struct address_space *mapping = info->vfs_inode.i_mapping; - void *radswap; - pgoff_t index; + swp_entry_t *ptr; + + for (ptr = dir; ptr < edir; ptr++) { + if (ptr->val == entry.val) + return ptr - dir; + } + return -1; +} + +static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page) +{ + struct address_space *mapping; + unsigned long idx; + unsigned long size; + unsigned long limit; + unsigned long stage; + struct page **dir; + struct page *subdir; + swp_entry_t *ptr; + int offset; int error; - radswap = swp_to_radix_entry(swap); - index = radix_tree_locate_item(&mapping->page_tree, radswap); - if (index == -1) - return 0; + idx = 0; + ptr = info->i_direct; + spin_lock(&info->lock); + if (!info->swapped) { + list_del_init(&info->swaplist); + goto lost2; + } + limit = info->next_index; + size = limit; + if (size > SHMEM_NR_DIRECT) + size = SHMEM_NR_DIRECT; + offset = shmem_find_swp(entry, ptr, ptr+size); + if (offset >= 0) { + shmem_swp_balance_unmap(); + goto found; + } + if (!info->i_indirect) + goto lost2; + + dir = shmem_dir_map(info->i_indirect); + stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2; + + for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) { + if (unlikely(idx == stage)) { + shmem_dir_unmap(dir-1); + if (cond_resched_lock(&info->lock)) { + /* check it has not been truncated */ + if (limit > info->next_index) { + limit = info->next_index; + if (idx >= limit) + goto lost2; + } + } + dir = shmem_dir_map(info->i_indirect) + + ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE; + while (!*dir) { + dir++; + idx += ENTRIES_PER_PAGEPAGE; + if (idx >= limit) + goto lost1; + } + stage = idx + ENTRIES_PER_PAGEPAGE; + subdir = *dir; + shmem_dir_unmap(dir); + dir = shmem_dir_map(subdir); + } + subdir = *dir; + if (subdir && page_private(subdir)) { + ptr = shmem_swp_map(subdir); + size = limit - idx; + if (size > ENTRIES_PER_PAGE) + size = ENTRIES_PER_PAGE; + offset = shmem_find_swp(entry, ptr, ptr+size); + shmem_swp_unmap(ptr); + if (offset >= 0) { + shmem_dir_unmap(dir); + ptr = shmem_swp_map(subdir); + goto found; + } + } + } +lost1: + shmem_dir_unmap(dir-1); +lost2: + spin_unlock(&info->lock); + return 0; +found: + idx += offset; + ptr += offset; /* * Move _head_ to start search for next from here. * But be careful: shmem_evict_inode checks list_empty without taking * mutex, and there's an instant in list_move_tail when info->swaplist - * would appear empty, if it were the only one on shmem_swaplist. + * would appear empty, if it were the only one on shmem_swaplist. We + * could avoid doing it if inode NULL; or use this minor optimization. */ if (shmem_swaplist.next != &info->swaplist) list_move_tail(&shmem_swaplist, &info->swaplist); @@ -647,34 +962,42 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, * but also to hold up shmem_evict_inode(): so inode cannot be freed * beneath us (pagelock doesn't help until the page is in pagecache). */ - error = shmem_add_to_page_cache(page, mapping, index, - GFP_NOWAIT, radswap); + mapping = info->vfs_inode.i_mapping; + error = add_to_page_cache_locked(page, mapping, idx, GFP_NOWAIT); /* which does mem_cgroup_uncharge_cache_page on error */ - if (error != -ENOMEM) { - /* - * Truncation and eviction use free_swap_and_cache(), which - * only does trylock page: if we raced, best clean up here. - */ + if (error == -EEXIST) { + struct page *filepage = find_get_page(mapping, idx); + error = 1; + if (filepage) { + /* + * There might be a more uptodate page coming down + * from a stacked writepage: forget our swappage if so. + */ + if (PageUptodate(filepage)) + error = 0; + page_cache_release(filepage); + } + } + if (!error) { delete_from_swap_cache(page); set_page_dirty(page); - if (!error) { - spin_lock(&info->lock); - info->swapped--; - spin_unlock(&info->lock); - swap_free(swap); - } + info->flags |= SHMEM_PAGEIN; + shmem_swp_set(info, ptr, 0); + swap_free(entry); error = 1; /* not an error, but entry was found */ } + shmem_swp_unmap(ptr); + spin_unlock(&info->lock); return error; } /* - * Search through swapped inodes to find and replace swap by page. + * shmem_unuse() search for an eventually swapped out shmem page. */ -int shmem_unuse(swp_entry_t swap, struct page *page) +int shmem_unuse(swp_entry_t entry, struct page *page) { - struct list_head *this, *next; + struct list_head *p, *next; struct shmem_inode_info *info; int found = 0; int error; @@ -683,25 +1006,32 @@ int shmem_unuse(swp_entry_t swap, struct page *page) * Charge page using GFP_KERNEL while we can wait, before taking * the shmem_swaplist_mutex which might hold up shmem_writepage(). * Charged back to the user (not to caller) when swap account is used. + * add_to_page_cache() will be called with GFP_NOWAIT. */ error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL); if (error) goto out; - /* No radix_tree_preload: swap entry keeps a place for page in tree */ + /* + * Try to preload while we can wait, to not make a habit of + * draining atomic reserves; but don't latch on to this cpu, + * it's okay if sometimes we get rescheduled after this. + */ + error = radix_tree_preload(GFP_KERNEL); + if (error) + goto uncharge; + radix_tree_preload_end(); mutex_lock(&shmem_swaplist_mutex); - list_for_each_safe(this, next, &shmem_swaplist) { - info = list_entry(this, struct shmem_inode_info, swaplist); - if (info->swapped) - found = shmem_unuse_inode(info, swap, page); - else - list_del_init(&info->swaplist); + list_for_each_safe(p, next, &shmem_swaplist) { + info = list_entry(p, struct shmem_inode_info, swaplist); + found = shmem_unuse_inode(info, entry, page); cond_resched(); if (found) break; } mutex_unlock(&shmem_swaplist_mutex); +uncharge: if (!found) mem_cgroup_uncharge_cache_page(page); if (found < 0) @@ -718,10 +1048,10 @@ out: static int shmem_writepage(struct page *page, struct writeback_control *wbc) { struct shmem_inode_info *info; + swp_entry_t *entry, swap; struct address_space *mapping; + unsigned long index; struct inode *inode; - swp_entry_t swap; - pgoff_t index; BUG_ON(!PageLocked(page)); mapping = page->mapping; @@ -730,52 +1060,84 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc) info = SHMEM_I(inode); if (info->flags & VM_LOCKED) goto redirty; +#ifdef CONFIG_ZRAM_FOR_ANDROID + /* + * Modification for compcache + * shmem_writepage can be reason of kernel panic when using swap. + * This modification prevent using swap by shmem. + */ + goto redirty; +#else if (!total_swap_pages) goto redirty; +#endif /* * shmem_backing_dev_info's capabilities prevent regular writeback or * sync from ever calling shmem_writepage; but a stacking filesystem - * might use ->writepage of its underlying filesystem, in which case + * may use the ->writepage of its underlying filesystem, in which case * tmpfs should write out to swap only in response to memory pressure, - * and not for the writeback threads or sync. + * and not for the writeback threads or sync. However, in those cases, + * we do still want to check if there's a redundant swappage to be + * discarded. */ - if (!wbc->for_reclaim) { - WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ - goto redirty; - } - swap = get_swap_page(); - if (!swap.val) - goto redirty; + if (wbc->for_reclaim) + swap = get_swap_page(); + else + swap.val = 0; /* * Add inode to shmem_unuse()'s list of swapped-out inodes, - * if it's not already there. Do it now before the page is - * moved to swap cache, when its pagelock no longer protects + * if it's not already there. Do it now because we cannot take + * mutex while holding spinlock, and must do so before the page + * is moved to swap cache, when its pagelock no longer protects * the inode from eviction. But don't unlock the mutex until - * we've incremented swapped, because shmem_unuse_inode() will - * prune a !swapped inode from the swaplist under this mutex. + * we've taken the spinlock, because shmem_unuse_inode() will + * prune a !swapped inode from the swaplist under both locks. */ - mutex_lock(&shmem_swaplist_mutex); - if (list_empty(&info->swaplist)) - list_add_tail(&info->swaplist, &shmem_swaplist); + if (swap.val) { + mutex_lock(&shmem_swaplist_mutex); + if (list_empty(&info->swaplist)) + list_add_tail(&info->swaplist, &shmem_swaplist); + } - if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { - swap_shmem_alloc(swap); - shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); + spin_lock(&info->lock); + if (swap.val) + mutex_unlock(&shmem_swaplist_mutex); - spin_lock(&info->lock); - info->swapped++; - shmem_recalc_inode(inode); - spin_unlock(&info->lock); + if (index >= info->next_index) { + BUG_ON(!(info->flags & SHMEM_TRUNCATE)); + goto unlock; + } + entry = shmem_swp_entry(info, index, NULL); + if (entry->val) { + /* + * The more uptodate page coming down from a stacked + * writepage should replace our old swappage. + */ + free_swap_and_cache(*entry); + shmem_swp_set(info, entry, 0); + } + shmem_recalc_inode(inode); - mutex_unlock(&shmem_swaplist_mutex); + if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { + delete_from_page_cache(page); + shmem_swp_set(info, entry, swap.val); + shmem_swp_unmap(entry); + swap_shmem_alloc(swap); + spin_unlock(&info->lock); BUG_ON(page_mapped(page)); swap_writepage(page, wbc); return 0; } - mutex_unlock(&shmem_swaplist_mutex); + shmem_swp_unmap(entry); +unlock: + spin_unlock(&info->lock); + /* + * add_to_swap_cache() doesn't return -EEXIST, so we can safely + * clear SWAP_HAS_CACHE flag. + */ swapcache_free(swap, NULL); redirty: set_page_dirty(page); @@ -812,19 +1174,19 @@ static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) } #endif /* CONFIG_TMPFS */ -static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, - struct shmem_inode_info *info, pgoff_t index) +static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp, + struct shmem_inode_info *info, unsigned long idx) { struct vm_area_struct pvma; struct page *page; /* Create a pseudo vma that just contains the policy */ pvma.vm_start = 0; - pvma.vm_pgoff = index; + pvma.vm_pgoff = idx; pvma.vm_ops = NULL; - pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index); + pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx); - page = swapin_readahead(swap, gfp, &pvma, 0); + page = swapin_readahead(entry, gfp, &pvma, 0); /* Drop reference taken by mpol_shared_policy_lookup() */ mpol_cond_put(pvma.vm_policy); @@ -833,16 +1195,16 @@ static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, } static struct page *shmem_alloc_page(gfp_t gfp, - struct shmem_inode_info *info, pgoff_t index) + struct shmem_inode_info *info, unsigned long idx) { struct vm_area_struct pvma; struct page *page; /* Create a pseudo vma that just contains the policy */ pvma.vm_start = 0; - pvma.vm_pgoff = index; + pvma.vm_pgoff = idx; pvma.vm_ops = NULL; - pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, index); + pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx); page = alloc_page_vma(gfp, &pvma, 0); @@ -853,19 +1215,19 @@ static struct page *shmem_alloc_page(gfp_t gfp, } #else /* !CONFIG_NUMA */ #ifdef CONFIG_TMPFS -static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) +static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *p) { } #endif /* CONFIG_TMPFS */ -static inline struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, - struct shmem_inode_info *info, pgoff_t index) +static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp, + struct shmem_inode_info *info, unsigned long idx) { - return swapin_readahead(swap, gfp, NULL, 0); + return swapin_readahead(entry, gfp, NULL, 0); } static inline struct page *shmem_alloc_page(gfp_t gfp, - struct shmem_inode_info *info, pgoff_t index) + struct shmem_inode_info *info, unsigned long idx) { return alloc_page(gfp); } @@ -879,195 +1241,311 @@ static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) #endif /* - * shmem_getpage_gfp - find page in cache, or get from swap, or allocate + * shmem_getpage - either get the page from swap or allocate a new one * * If we allocate a new one we do not mark it dirty. That's up to the * vm. If we swap it in we mark it dirty since we also free the swap * entry since a page cannot live in both the swap and page cache */ -static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, - struct page **pagep, enum sgp_type sgp, gfp_t gfp, int *fault_type) +static int shmem_getpage(struct inode *inode, unsigned long idx, + struct page **pagep, enum sgp_type sgp, int *type) { struct address_space *mapping = inode->i_mapping; - struct shmem_inode_info *info; + struct shmem_inode_info *info = SHMEM_I(inode); struct shmem_sb_info *sbinfo; - struct page *page; + struct page *filepage = *pagep; + struct page *swappage; + struct page *prealloc_page = NULL; + swp_entry_t *entry; swp_entry_t swap; + gfp_t gfp; int error; - int once = 0; - if (index > (MAX_LFS_FILESIZE >> PAGE_CACHE_SHIFT)) + if (idx >= SHMEM_MAX_INDEX) return -EFBIG; -repeat: - swap.val = 0; - page = find_lock_page(mapping, index); - if (radix_tree_exceptional_entry(page)) { - swap = radix_to_swp_entry(page); - page = NULL; - } - if (sgp != SGP_WRITE && - ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) { - error = -EINVAL; - goto failed; - } + if (type) + *type = 0; - if (page || (sgp == SGP_READ && !swap.val)) { + /* + * Normally, filepage is NULL on entry, and either found + * uptodate immediately, or allocated and zeroed, or read + * in under swappage, which is then assigned to filepage. + * But shmem_readpage (required for splice) passes in a locked + * filepage, which may be found not uptodate by other callers + * too, and may need to be copied from the swappage read in. + */ +repeat: + if (!filepage) + filepage = find_lock_page(mapping, idx); + if (filepage && PageUptodate(filepage)) + goto done; + gfp = mapping_gfp_mask(mapping); + if (!filepage) { /* - * Once we can get the page lock, it must be uptodate: - * if there were an error in reading back from swap, - * the page would not be inserted into the filecache. + * Try to preload while we can wait, to not make a habit of + * draining atomic reserves; but don't latch on to this cpu. */ - BUG_ON(page && !PageUptodate(page)); - *pagep = page; - return 0; + error = radix_tree_preload(gfp & ~__GFP_HIGHMEM); + if (error) + goto failed; + radix_tree_preload_end(); + if (sgp != SGP_READ && !prealloc_page) { + /* We don't care if this fails */ + prealloc_page = shmem_alloc_page(gfp, info, idx); + if (prealloc_page) { + if (mem_cgroup_cache_charge(prealloc_page, + current->mm, GFP_KERNEL)) { + page_cache_release(prealloc_page); + prealloc_page = NULL; + } + } + } } + error = 0; - /* - * Fast cache lookup did not find it: - * bring it back from swap or allocate. - */ - info = SHMEM_I(inode); - sbinfo = SHMEM_SB(inode->i_sb); + spin_lock(&info->lock); + shmem_recalc_inode(inode); + entry = shmem_swp_alloc(info, idx, sgp); + if (IS_ERR(entry)) { + spin_unlock(&info->lock); + error = PTR_ERR(entry); + goto failed; + } + swap = *entry; if (swap.val) { /* Look it up and read it in.. */ - page = lookup_swap_cache(swap); - if (!page) { + swappage = lookup_swap_cache(swap); + if (!swappage) { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); /* here we actually do the io */ - if (fault_type) - *fault_type |= VM_FAULT_MAJOR; - page = shmem_swapin(swap, gfp, info, index); - if (!page) { - error = -ENOMEM; - goto failed; + if (type) + *type |= VM_FAULT_MAJOR; + swappage = shmem_swapin(swap, gfp, info, idx); + if (!swappage) { + spin_lock(&info->lock); + entry = shmem_swp_alloc(info, idx, sgp); + if (IS_ERR(entry)) + error = PTR_ERR(entry); + else { + if (entry->val == swap.val) + error = -ENOMEM; + shmem_swp_unmap(entry); + } + spin_unlock(&info->lock); + if (error) + goto failed; + goto repeat; } + wait_on_page_locked(swappage); + page_cache_release(swappage); + goto repeat; } /* We have to do this with page locked to prevent races */ - lock_page(page); - if (!PageUptodate(page)) { - error = -EIO; - goto failed; + if (!trylock_page(swappage)) { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + wait_on_page_locked(swappage); + page_cache_release(swappage); + goto repeat; } - wait_on_page_writeback(page); - - /* Someone may have already done it for us */ - if (page->mapping) { - if (page->mapping == mapping && - page->index == index) - goto done; - error = -EEXIST; - goto failed; + if (PageWriteback(swappage)) { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + wait_on_page_writeback(swappage); + unlock_page(swappage); + page_cache_release(swappage); + goto repeat; } - - error = mem_cgroup_cache_charge(page, current->mm, - gfp & GFP_RECLAIM_MASK); - if (!error) - error = shmem_add_to_page_cache(page, mapping, index, - gfp, swp_to_radix_entry(swap)); - if (error) + if (!PageUptodate(swappage)) { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + unlock_page(swappage); + page_cache_release(swappage); + error = -EIO; goto failed; + } - spin_lock(&info->lock); - info->swapped--; - shmem_recalc_inode(inode); + if (filepage) { + shmem_swp_set(info, entry, 0); + shmem_swp_unmap(entry); + delete_from_swap_cache(swappage); + spin_unlock(&info->lock); + copy_highpage(filepage, swappage); + unlock_page(swappage); + page_cache_release(swappage); + flush_dcache_page(filepage); + SetPageUptodate(filepage); + set_page_dirty(filepage); + swap_free(swap); + } else if (!(error = add_to_page_cache_locked(swappage, mapping, + idx, GFP_NOWAIT))) { + info->flags |= SHMEM_PAGEIN; + shmem_swp_set(info, entry, 0); + shmem_swp_unmap(entry); + delete_from_swap_cache(swappage); + spin_unlock(&info->lock); + filepage = swappage; + set_page_dirty(filepage); + swap_free(swap); + } else { + shmem_swp_unmap(entry); + spin_unlock(&info->lock); + if (error == -ENOMEM) { + /* + * reclaim from proper memory cgroup and + * call memcg's OOM if needed. + */ + error = mem_cgroup_shmem_charge_fallback( + swappage, + current->mm, + gfp); + if (error) { + unlock_page(swappage); + page_cache_release(swappage); + goto failed; + } + } + unlock_page(swappage); + page_cache_release(swappage); + goto repeat; + } + } else if (sgp == SGP_READ && !filepage) { + shmem_swp_unmap(entry); + filepage = find_get_page(mapping, idx); + if (filepage && + (!PageUptodate(filepage) || !trylock_page(filepage))) { + spin_unlock(&info->lock); + wait_on_page_locked(filepage); + page_cache_release(filepage); + filepage = NULL; + goto repeat; + } spin_unlock(&info->lock); - - delete_from_swap_cache(page); - set_page_dirty(page); - swap_free(swap); - } else { - if (shmem_acct_block(info->flags)) { - error = -ENOSPC; - goto failed; - } + shmem_swp_unmap(entry); + sbinfo = SHMEM_SB(inode->i_sb); if (sbinfo->max_blocks) { if (percpu_counter_compare(&sbinfo->used_blocks, - sbinfo->max_blocks) >= 0) { - error = -ENOSPC; - goto unacct; - } + sbinfo->max_blocks) >= 0 || + shmem_acct_block(info->flags)) + goto nospace; percpu_counter_inc(&sbinfo->used_blocks); - } + spin_lock(&inode->i_lock); + inode->i_blocks += BLOCKS_PER_PAGE; + spin_unlock(&inode->i_lock); + } else if (shmem_acct_block(info->flags)) + goto nospace; + + if (!filepage) { + int ret; + + if (!prealloc_page) { + spin_unlock(&info->lock); + filepage = shmem_alloc_page(gfp, info, idx); + if (!filepage) { + shmem_unacct_blocks(info->flags, 1); + shmem_free_blocks(inode, 1); + error = -ENOMEM; + goto failed; + } + SetPageSwapBacked(filepage); + + /* + * Precharge page while we can wait, compensate + * after + */ + error = mem_cgroup_cache_charge(filepage, + current->mm, GFP_KERNEL); + if (error) { + page_cache_release(filepage); + shmem_unacct_blocks(info->flags, 1); + shmem_free_blocks(inode, 1); + filepage = NULL; + goto failed; + } - page = shmem_alloc_page(gfp, info, index); - if (!page) { - error = -ENOMEM; - goto decused; - } + spin_lock(&info->lock); + } else { + filepage = prealloc_page; + prealloc_page = NULL; + SetPageSwapBacked(filepage); + } - SetPageSwapBacked(page); - __set_page_locked(page); - error = mem_cgroup_cache_charge(page, current->mm, - gfp & GFP_RECLAIM_MASK); - if (!error) - error = shmem_add_to_page_cache(page, mapping, index, - gfp, NULL); - if (error) - goto decused; - lru_cache_add_anon(page); + entry = shmem_swp_alloc(info, idx, sgp); + if (IS_ERR(entry)) + error = PTR_ERR(entry); + else { + swap = *entry; + shmem_swp_unmap(entry); + } + ret = error || swap.val; + if (ret) + mem_cgroup_uncharge_cache_page(filepage); + else + ret = add_to_page_cache_lru(filepage, mapping, + idx, GFP_NOWAIT); + /* + * At add_to_page_cache_lru() failure, uncharge will + * be done automatically. + */ + if (ret) { + spin_unlock(&info->lock); + page_cache_release(filepage); + shmem_unacct_blocks(info->flags, 1); + shmem_free_blocks(inode, 1); + filepage = NULL; + if (error) + goto failed; + goto repeat; + } + info->flags |= SHMEM_PAGEIN; + } - spin_lock(&info->lock); info->alloced++; - inode->i_blocks += BLOCKS_PER_PAGE; - shmem_recalc_inode(inode); spin_unlock(&info->lock); - - clear_highpage(page); - flush_dcache_page(page); - SetPageUptodate(page); + clear_highpage(filepage); + flush_dcache_page(filepage); + SetPageUptodate(filepage); if (sgp == SGP_DIRTY) - set_page_dirty(page); + set_page_dirty(filepage); } done: - /* Perhaps the file has been truncated since we checked */ - if (sgp != SGP_WRITE && - ((loff_t)index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) { - error = -EINVAL; - goto trunc; - } - *pagep = page; - return 0; + *pagep = filepage; + error = 0; + goto out; +nospace: /* - * Error recovery. + * Perhaps the page was brought in from swap between find_lock_page + * and taking info->lock? We allow for that at add_to_page_cache_lru, + * but must also avoid reporting a spurious ENOSPC while working on a + * full tmpfs. (When filepage has been passed in to shmem_getpage, it + * is already in page cache, which prevents this race from occurring.) */ -trunc: - ClearPageDirty(page); - delete_from_page_cache(page); - spin_lock(&info->lock); - info->alloced--; - inode->i_blocks -= BLOCKS_PER_PAGE; + if (!filepage) { + struct page *page = find_get_page(mapping, idx); + if (page) { + spin_unlock(&info->lock); + page_cache_release(page); + goto repeat; + } + } spin_unlock(&info->lock); -decused: - if (sbinfo->max_blocks) - percpu_counter_add(&sbinfo->used_blocks, -1); -unacct: - shmem_unacct_blocks(info->flags, 1); + error = -ENOSPC; failed: - if (swap.val && error != -EINVAL) { - struct page *test = find_get_page(mapping, index); - if (test && !radix_tree_exceptional_entry(test)) - page_cache_release(test); - /* Have another try if the entry has changed */ - if (test != swp_to_radix_entry(swap)) - error = -EEXIST; + if (*pagep != filepage) { + unlock_page(filepage); + page_cache_release(filepage); } - if (page) { - unlock_page(page); - page_cache_release(page); - } - if (error == -ENOSPC && !once++) { - info = SHMEM_I(inode); - spin_lock(&info->lock); - shmem_recalc_inode(inode); - spin_unlock(&info->lock); - goto repeat; +out: + if (prealloc_page) { + mem_cgroup_uncharge_cache_page(prealloc_page); + page_cache_release(prealloc_page); } - if (error == -EEXIST) - goto repeat; return error; } @@ -1075,132 +1553,36 @@ static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct inode *inode = vma->vm_file->f_path.dentry->d_inode; int error; - int ret = VM_FAULT_LOCKED; - - /* - * Trinity finds that probing a hole which tmpfs is punching can - * prevent the hole-punch from ever completing: which in turn - * locks writers out with its hold on i_mutex. So refrain from - * faulting pages into the hole while it's being punched. Although - * shmem_truncate_range() does remove the additions, it may be unable to - * keep up, as each new page needs its own unmap_mapping_range() call, - * and the i_mmap tree grows ever slower to scan if new vmas are added. - * - * It does not matter if we sometimes reach this check just before the - * hole-punch begins, so that one fault then races with the punch: - * we just need to make racing faults a rare case. - * - * The implementation below would be much simpler if we just used a - * standard mutex or completion: but we cannot take i_mutex in fault, - * and bloating every shmem inode for this unlikely case would be sad. - */ - if (unlikely(inode->i_private)) { - struct shmem_falloc *shmem_falloc; - - spin_lock(&inode->i_lock); - shmem_falloc = inode->i_private; - if (shmem_falloc && - vmf->pgoff >= shmem_falloc->start && - vmf->pgoff < shmem_falloc->next) { - wait_queue_head_t *shmem_falloc_waitq; - DEFINE_WAIT(shmem_fault_wait); - - ret = VM_FAULT_NOPAGE; - if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && - !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { - /* It's polite to up mmap_sem if we can */ - up_read(&vma->vm_mm->mmap_sem); - ret = VM_FAULT_RETRY; - } - - shmem_falloc_waitq = shmem_falloc->waitq; - prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, - TASK_UNINTERRUPTIBLE); - spin_unlock(&inode->i_lock); - schedule(); + int ret; - /* - * shmem_falloc_waitq points into the vmtruncate_range() - * stack of the hole-punching task: shmem_falloc_waitq - * is usually invalid by the time we reach here, but - * finish_wait() does not dereference it in that case; - * though i_lock needed lest racing with wake_up_all(). - */ - spin_lock(&inode->i_lock); - finish_wait(shmem_falloc_waitq, &shmem_fault_wait); - spin_unlock(&inode->i_lock); - return ret; - } - spin_unlock(&inode->i_lock); - } + if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) + return VM_FAULT_SIGBUS; error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret); if (error) return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS); - if (ret & VM_FAULT_MAJOR) { count_vm_event(PGMAJFAULT); mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); } - return ret; -} - -int vmtruncate_range(struct inode *inode, loff_t lstart, loff_t lend) -{ - /* - * If the underlying filesystem is not going to provide - * a way to truncate a range of blocks (punch a hole) - - * we should return failure right now. - * Only CONFIG_SHMEM shmem.c ever supported i_op->truncate_range(). - */ - if (inode->i_op->truncate_range != shmem_truncate_range) - return -ENOSYS; - - mutex_lock(&inode->i_mutex); - { - struct shmem_falloc shmem_falloc; - struct address_space *mapping = inode->i_mapping; - loff_t unmap_start = round_up(lstart, PAGE_SIZE); - loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1; - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); - - shmem_falloc.waitq = &shmem_falloc_waitq; - shmem_falloc.start = unmap_start >> PAGE_SHIFT; - shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; - spin_lock(&inode->i_lock); - inode->i_private = &shmem_falloc; - spin_unlock(&inode->i_lock); - - if ((u64)unmap_end > (u64)unmap_start) - unmap_mapping_range(mapping, unmap_start, - 1 + unmap_end - unmap_start, 0); - shmem_truncate_range(inode, lstart, lend); - /* No need to unmap again: hole-punching leaves COWed pages */ - - spin_lock(&inode->i_lock); - inode->i_private = NULL; - wake_up_all(&shmem_falloc_waitq); - spin_unlock(&inode->i_lock); - } - mutex_unlock(&inode->i_mutex); - return 0; + return ret | VM_FAULT_LOCKED; } #ifdef CONFIG_NUMA -static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) +static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new) { - struct inode *inode = vma->vm_file->f_path.dentry->d_inode; - return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); + struct inode *i = vma->vm_file->f_path.dentry->d_inode; + return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new); } static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, unsigned long addr) { - struct inode *inode = vma->vm_file->f_path.dentry->d_inode; - pgoff_t index; + struct inode *i = vma->vm_file->f_path.dentry->d_inode; + unsigned long idx; - index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; - return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); + idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx); } #endif @@ -1221,6 +1603,7 @@ int shmem_lock(struct file *file, int lock, struct user_struct *user) user_shm_unlock(inode->i_size, user); info->flags &= ~VM_LOCKED; mapping_clear_unevictable(file->f_mapping); + scan_mapping_unevictable_pages(file->f_mapping); } retval = 0; @@ -1297,7 +1680,20 @@ static struct inode *shmem_get_inode(struct super_block *sb, const struct inode #ifdef CONFIG_TMPFS static const struct inode_operations shmem_symlink_inode_operations; -static const struct inode_operations shmem_short_symlink_operations; +static const struct inode_operations shmem_symlink_inline_operations; + +/* + * Normally tmpfs avoids the use of shmem_readpage and shmem_write_begin; + * but providing them allows a tmpfs file to be used for splice, sendfile, and + * below the loop driver, in the generic fashion that many filesystems support. + */ +static int shmem_readpage(struct file *file, struct page *page) +{ + struct inode *inode = page->mapping->host; + int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL); + unlock_page(page); + return error; +} static int shmem_write_begin(struct file *file, struct address_space *mapping, @@ -1306,6 +1702,7 @@ shmem_write_begin(struct file *file, struct address_space *mapping, { struct inode *inode = mapping->host; pgoff_t index = pos >> PAGE_CACHE_SHIFT; + *pagep = NULL; return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL); } @@ -1330,8 +1727,7 @@ static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_ { struct inode *inode = filp->f_path.dentry->d_inode; struct address_space *mapping = inode->i_mapping; - pgoff_t index; - unsigned long offset; + unsigned long index, offset; enum sgp_type sgp = SGP_READ; /* @@ -1347,8 +1743,7 @@ static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_ for (;;) { struct page *page = NULL; - pgoff_t end_index; - unsigned long nr, ret; + unsigned long end_index, nr, ret; loff_t i_size = i_size_read(inode); end_index = i_size >> PAGE_CACHE_SHIFT; @@ -1464,120 +1859,6 @@ static ssize_t shmem_file_aio_read(struct kiocb *iocb, return retval; } -static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos, - struct pipe_inode_info *pipe, size_t len, - unsigned int flags) -{ - struct address_space *mapping = in->f_mapping; - struct inode *inode = mapping->host; - unsigned int loff, nr_pages, req_pages; - struct page *pages[PIPE_DEF_BUFFERS]; - struct partial_page partial[PIPE_DEF_BUFFERS]; - struct page *page; - pgoff_t index, end_index; - loff_t isize, left; - int error, page_nr; - struct splice_pipe_desc spd = { - .pages = pages, - .partial = partial, - .nr_pages_max = PIPE_DEF_BUFFERS, - .flags = flags, - .ops = &page_cache_pipe_buf_ops, - .spd_release = spd_release_page, - }; - - isize = i_size_read(inode); - if (unlikely(*ppos >= isize)) - return 0; - - left = isize - *ppos; - if (unlikely(left < len)) - len = left; - - if (splice_grow_spd(pipe, &spd)) - return -ENOMEM; - - index = *ppos >> PAGE_CACHE_SHIFT; - loff = *ppos & ~PAGE_CACHE_MASK; - req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; - nr_pages = min(req_pages, pipe->buffers); - - spd.nr_pages = find_get_pages_contig(mapping, index, - nr_pages, spd.pages); - index += spd.nr_pages; - error = 0; - - while (spd.nr_pages < nr_pages) { - error = shmem_getpage(inode, index, &page, SGP_CACHE, NULL); - if (error) - break; - unlock_page(page); - spd.pages[spd.nr_pages++] = page; - index++; - } - - index = *ppos >> PAGE_CACHE_SHIFT; - nr_pages = spd.nr_pages; - spd.nr_pages = 0; - - for (page_nr = 0; page_nr < nr_pages; page_nr++) { - unsigned int this_len; - - if (!len) - break; - - this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff); - page = spd.pages[page_nr]; - - if (!PageUptodate(page) || page->mapping != mapping) { - error = shmem_getpage(inode, index, &page, - SGP_CACHE, NULL); - if (error) - break; - unlock_page(page); - page_cache_release(spd.pages[page_nr]); - spd.pages[page_nr] = page; - } - - isize = i_size_read(inode); - end_index = (isize - 1) >> PAGE_CACHE_SHIFT; - if (unlikely(!isize || index > end_index)) - break; - - if (end_index == index) { - unsigned int plen; - - plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; - if (plen <= loff) - break; - - this_len = min(this_len, plen - loff); - len = this_len; - } - - spd.partial[page_nr].offset = loff; - spd.partial[page_nr].len = this_len; - len -= this_len; - loff = 0; - spd.nr_pages++; - index++; - } - - while (page_nr < nr_pages) - page_cache_release(spd.pages[page_nr++]); - - if (spd.nr_pages) - error = splice_to_pipe(pipe, &spd); - - splice_shrink_spd(&spd); - - if (error > 0) { - *ppos += error; - file_accessed(in); - } - return error; -} - static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) { struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); @@ -1587,9 +1868,8 @@ static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) buf->f_namelen = NAME_MAX; if (sbinfo->max_blocks) { buf->f_blocks = sbinfo->max_blocks; - buf->f_bavail = - buf->f_bfree = sbinfo->max_blocks - - percpu_counter_sum(&sbinfo->used_blocks); + buf->f_bavail = buf->f_bfree = + sbinfo->max_blocks - percpu_counter_sum(&sbinfo->used_blocks); } if (sbinfo->max_inodes) { buf->f_files = sbinfo->max_inodes; @@ -1611,7 +1891,7 @@ shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev) inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); if (inode) { error = security_inode_init_security(inode, dir, - &dentry->d_name, + &dentry->d_name, NULL, NULL, NULL); if (error) { if (error != -EOPNOTSUPP) { @@ -1719,10 +1999,8 @@ static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct if (new_dentry->d_inode) { (void) shmem_unlink(new_dir, new_dentry); - if (they_are_dirs) { - drop_nlink(new_dentry->d_inode); + if (they_are_dirs) drop_nlink(old_dir); - } } else if (they_are_dirs) { drop_nlink(old_dir); inc_nlink(new_dir); @@ -1741,7 +2019,7 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s int error; int len; struct inode *inode; - struct page *page; + struct page *page = NULL; char *kaddr; struct shmem_inode_info *info; @@ -1753,7 +2031,7 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s if (!inode) return -ENOSPC; - error = security_inode_init_security(inode, dir, &dentry->d_name, + error = security_inode_init_security(inode, dir, &dentry->d_name, NULL, NULL, NULL); if (error) { if (error != -EOPNOTSUPP) { @@ -1765,13 +2043,10 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s info = SHMEM_I(inode); inode->i_size = len-1; - if (len <= SHORT_SYMLINK_LEN) { - info->symlink = kmemdup(symname, len, GFP_KERNEL); - if (!info->symlink) { - iput(inode); - return -ENOMEM; - } - inode->i_op = &shmem_short_symlink_operations; + if (len <= SHMEM_SYMLINK_INLINE_LEN) { + /* do it inline */ + memcpy(info->inline_symlink, symname, len); + inode->i_op = &shmem_symlink_inline_operations; } else { error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL); if (error) { @@ -1794,17 +2069,17 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s return 0; } -static void *shmem_follow_short_symlink(struct dentry *dentry, struct nameidata *nd) +static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd) { - nd_set_link(nd, SHMEM_I(dentry->d_inode)->symlink); + nd_set_link(nd, SHMEM_I(dentry->d_inode)->inline_symlink); return NULL; } static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd) { struct page *page = NULL; - int error = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL); - nd_set_link(nd, error ? ERR_PTR(error) : kmap(page)); + int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL); + nd_set_link(nd, res ? ERR_PTR(res) : kmap(page)); if (page) unlock_page(page); return page; @@ -1915,6 +2190,7 @@ out: return err; } + static const struct xattr_handler *shmem_xattr_handlers[] = { #ifdef CONFIG_TMPFS_POSIX_ACL &generic_acl_access_handler, @@ -2044,9 +2320,9 @@ static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) } #endif /* CONFIG_TMPFS_XATTR */ -static const struct inode_operations shmem_short_symlink_operations = { +static const struct inode_operations shmem_symlink_inline_operations = { .readlink = generic_readlink, - .follow_link = shmem_follow_short_symlink, + .follow_link = shmem_follow_link_inline, #ifdef CONFIG_TMPFS_XATTR .setxattr = shmem_setxattr, .getxattr = shmem_getxattr, @@ -2249,7 +2525,8 @@ static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) if (config.max_inodes < inodes) goto out; /* - * Those tests disallow limited->unlimited while any are in use; + * Those tests also disallow limited->unlimited while any are in + * use, so i_blocks will always be zero when max_blocks is zero; * but we must separately disallow unlimited->limited, because * in that case we have no record of how much is already in use. */ @@ -2346,7 +2623,7 @@ int shmem_fill_super(struct super_block *sb, void *data, int silent) goto failed; sbinfo->free_inodes = sbinfo->max_inodes; - sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_maxbytes = SHMEM_MAX_BYTES; sb->s_blocksize = PAGE_CACHE_SIZE; sb->s_blocksize_bits = PAGE_CACHE_SHIFT; sb->s_magic = TMPFS_MAGIC; @@ -2381,14 +2658,14 @@ static struct kmem_cache *shmem_inode_cachep; static struct inode *shmem_alloc_inode(struct super_block *sb) { - struct shmem_inode_info *info; - info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); - if (!info) + struct shmem_inode_info *p; + p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); + if (!p) return NULL; - return &info->vfs_inode; + return &p->vfs_inode; } -static void shmem_destroy_callback(struct rcu_head *head) +static void shmem_i_callback(struct rcu_head *head) { struct inode *inode = container_of(head, struct inode, i_rcu); INIT_LIST_HEAD(&inode->i_dentry); @@ -2397,26 +2674,29 @@ static void shmem_destroy_callback(struct rcu_head *head) static void shmem_destroy_inode(struct inode *inode) { - if ((inode->i_mode & S_IFMT) == S_IFREG) + if ((inode->i_mode & S_IFMT) == S_IFREG) { + /* only struct inode is valid if it's an inline symlink */ mpol_free_shared_policy(&SHMEM_I(inode)->policy); - call_rcu(&inode->i_rcu, shmem_destroy_callback); + } + call_rcu(&inode->i_rcu, shmem_i_callback); } -static void shmem_init_inode(void *foo) +static void init_once(void *foo) { - struct shmem_inode_info *info = foo; - inode_init_once(&info->vfs_inode); + struct shmem_inode_info *p = (struct shmem_inode_info *) foo; + + inode_init_once(&p->vfs_inode); } -static int shmem_init_inodecache(void) +static int init_inodecache(void) { shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", sizeof(struct shmem_inode_info), - 0, SLAB_PANIC, shmem_init_inode); + 0, SLAB_PANIC, init_once); return 0; } -static void shmem_destroy_inodecache(void) +static void destroy_inodecache(void) { kmem_cache_destroy(shmem_inode_cachep); } @@ -2425,6 +2705,7 @@ static const struct address_space_operations shmem_aops = { .writepage = shmem_writepage, .set_page_dirty = __set_page_dirty_no_writeback, #ifdef CONFIG_TMPFS + .readpage = shmem_readpage, .write_begin = shmem_write_begin, .write_end = shmem_write_end, #endif @@ -2441,7 +2722,7 @@ static const struct file_operations shmem_file_operations = { .aio_read = shmem_file_aio_read, .aio_write = generic_file_aio_write, .fsync = noop_fsync, - .splice_read = shmem_file_splice_read, + .splice_read = generic_file_splice_read, .splice_write = generic_file_splice_write, #endif }; @@ -2455,6 +2736,10 @@ static const struct inode_operations shmem_inode_operations = { .listxattr = shmem_listxattr, .removexattr = shmem_removexattr, #endif +#ifdef CONFIG_TMPFS_POSIX_ACL + .check_acl = generic_check_acl, +#endif + }; static const struct inode_operations shmem_dir_inode_operations = { @@ -2477,6 +2762,7 @@ static const struct inode_operations shmem_dir_inode_operations = { #endif #ifdef CONFIG_TMPFS_POSIX_ACL .setattr = shmem_setattr, + .check_acl = generic_check_acl, #endif }; @@ -2489,6 +2775,7 @@ static const struct inode_operations shmem_special_inode_operations = { #endif #ifdef CONFIG_TMPFS_POSIX_ACL .setattr = shmem_setattr, + .check_acl = generic_check_acl, #endif }; @@ -2513,20 +2800,21 @@ static const struct vm_operations_struct shmem_vm_ops = { #endif }; + static struct dentry *shmem_mount(struct file_system_type *fs_type, int flags, const char *dev_name, void *data) { return mount_nodev(fs_type, flags, data, shmem_fill_super); } -static struct file_system_type shmem_fs_type = { +static struct file_system_type tmpfs_fs_type = { .owner = THIS_MODULE, .name = "tmpfs", .mount = shmem_mount, .kill_sb = kill_litter_super, }; -int __init shmem_init(void) +int __init init_tmpfs(void) { int error; @@ -2534,18 +2822,18 @@ int __init shmem_init(void) if (error) goto out4; - error = shmem_init_inodecache(); + error = init_inodecache(); if (error) goto out3; - error = register_filesystem(&shmem_fs_type); + error = register_filesystem(&tmpfs_fs_type); if (error) { printk(KERN_ERR "Could not register tmpfs\n"); goto out2; } - shm_mnt = vfs_kern_mount(&shmem_fs_type, MS_NOUSER, - shmem_fs_type.name, NULL); + shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER, + tmpfs_fs_type.name, NULL); if (IS_ERR(shm_mnt)) { error = PTR_ERR(shm_mnt); printk(KERN_ERR "Could not kern_mount tmpfs\n"); @@ -2554,9 +2842,9 @@ int __init shmem_init(void) return 0; out1: - unregister_filesystem(&shmem_fs_type); + unregister_filesystem(&tmpfs_fs_type); out2: - shmem_destroy_inodecache(); + destroy_inodecache(); out3: bdi_destroy(&shmem_backing_dev_info); out4: @@ -2564,6 +2852,45 @@ out4: return error; } +#ifdef CONFIG_CGROUP_MEM_RES_CTLR +/** + * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file + * @inode: the inode to be searched + * @pgoff: the offset to be searched + * @pagep: the pointer for the found page to be stored + * @ent: the pointer for the found swap entry to be stored + * + * If a page is found, refcount of it is incremented. Callers should handle + * these refcount. + */ +void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff, + struct page **pagep, swp_entry_t *ent) +{ + swp_entry_t entry = { .val = 0 }, *ptr; + struct page *page = NULL; + struct shmem_inode_info *info = SHMEM_I(inode); + + if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) + goto out; + + spin_lock(&info->lock); + ptr = shmem_swp_entry(info, pgoff, NULL); +#ifdef CONFIG_SWAP + if (ptr && ptr->val) { + entry.val = ptr->val; + page = find_get_page(&swapper_space, entry.val); + } else +#endif + page = find_get_page(inode->i_mapping, pgoff); + if (ptr) + shmem_swp_unmap(ptr); + spin_unlock(&info->lock); +out: + *pagep = page; + *ent = entry; +} +#endif + #else /* !CONFIG_SHMEM */ /* @@ -2577,23 +2904,23 @@ out4: #include <linux/ramfs.h> -static struct file_system_type shmem_fs_type = { +static struct file_system_type tmpfs_fs_type = { .name = "tmpfs", .mount = ramfs_mount, .kill_sb = kill_litter_super, }; -int __init shmem_init(void) +int __init init_tmpfs(void) { - BUG_ON(register_filesystem(&shmem_fs_type) != 0); + BUG_ON(register_filesystem(&tmpfs_fs_type) != 0); - shm_mnt = kern_mount(&shmem_fs_type); + shm_mnt = kern_mount(&tmpfs_fs_type); BUG_ON(IS_ERR(shm_mnt)); return 0; } -int shmem_unuse(swp_entry_t swap, struct page *page) +int shmem_unuse(swp_entry_t entry, struct page *page) { return 0; } @@ -2603,27 +2930,43 @@ int shmem_lock(struct file *file, int lock, struct user_struct *user) return 0; } -void shmem_unlock_mapping(struct address_space *mapping) -{ -} - -void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) +void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end) { - truncate_inode_pages_range(inode->i_mapping, lstart, lend); + truncate_inode_pages_range(inode->i_mapping, start, end); } EXPORT_SYMBOL_GPL(shmem_truncate_range); -int vmtruncate_range(struct inode *inode, loff_t lstart, loff_t lend) +#ifdef CONFIG_CGROUP_MEM_RES_CTLR +/** + * mem_cgroup_get_shmem_target - find a page or entry assigned to the shmem file + * @inode: the inode to be searched + * @pgoff: the offset to be searched + * @pagep: the pointer for the found page to be stored + * @ent: the pointer for the found swap entry to be stored + * + * If a page is found, refcount of it is incremented. Callers should handle + * these refcount. + */ +void mem_cgroup_get_shmem_target(struct inode *inode, pgoff_t pgoff, + struct page **pagep, swp_entry_t *ent) { - /* Only CONFIG_SHMEM shmem.c ever supported i_op->truncate_range(). */ - return -ENOSYS; + struct page *page = NULL; + + if ((pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode)) + goto out; + page = find_get_page(inode->i_mapping, pgoff); +out: + *pagep = page; + *ent = (swp_entry_t){ .val = 0 }; } +#endif #define shmem_vm_ops generic_file_vm_ops #define shmem_file_operations ramfs_file_operations #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) #define shmem_acct_size(flags, size) 0 #define shmem_unacct_size(flags, size) do {} while (0) +#define SHMEM_MAX_BYTES MAX_LFS_FILESIZE #endif /* CONFIG_SHMEM */ @@ -2647,7 +2990,7 @@ struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags if (IS_ERR(shm_mnt)) return (void *)shm_mnt; - if (size < 0 || size > MAX_LFS_FILESIZE) + if (size < 0 || size > SHMEM_MAX_BYTES) return ERR_PTR(-EINVAL); if (shmem_acct_size(flags, size)) @@ -2670,7 +3013,7 @@ struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags d_instantiate(path.dentry, inode); inode->i_size = size; - clear_nlink(inode); /* It is unlinked */ + inode->i_nlink = 0; /* It is unlinked */ #ifndef CONFIG_MMU error = ramfs_nommu_expand_for_mapping(inode, size); if (error) @@ -2693,6 +3036,15 @@ put_memory: } EXPORT_SYMBOL_GPL(shmem_file_setup); +void shmem_set_file(struct vm_area_struct *vma, struct file *file) +{ + if (vma->vm_file) + fput(vma->vm_file); + vma->vm_file = file; + vma->vm_ops = &shmem_vm_ops; + vma->vm_flags |= VM_CAN_NONLINEAR; +} + /** * shmem_zero_setup - setup a shared anonymous mapping * @vma: the vma to be mmapped is prepared by do_mmap_pgoff @@ -2706,11 +3058,7 @@ int shmem_zero_setup(struct vm_area_struct *vma) if (IS_ERR(file)) return PTR_ERR(file); - if (vma->vm_file) - fput(vma->vm_file); - vma->vm_file = file; - vma->vm_ops = &shmem_vm_ops; - vma->vm_flags |= VM_CAN_NONLINEAR; + shmem_set_file(vma, file); return 0; } @@ -2726,29 +3074,13 @@ int shmem_zero_setup(struct vm_area_struct *vma) * suit tmpfs, since it may have pages in swapcache, and needs to find those * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. * - * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in - * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. + * Provide a stub for those callers to start using now, then later + * flesh it out to call shmem_getpage() with additional gfp mask, when + * shmem_file_splice_read() is added and shmem_readpage() is removed. */ struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp) { -#ifdef CONFIG_SHMEM - struct inode *inode = mapping->host; - struct page *page; - int error; - - BUG_ON(mapping->a_ops != &shmem_aops); - error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, gfp, NULL); - if (error) - page = ERR_PTR(error); - else - unlock_page(page); - return page; -#else - /* - * The tiny !SHMEM case uses ramfs without swap - */ return read_cache_page_gfp(mapping, index, gfp); -#endif } EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); @@ -574,9 +574,7 @@ static struct arraycache_init initarray_generic = { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; /* internal cache of cache description objs */ -static struct kmem_list3 *cache_cache_nodelists[MAX_NUMNODES]; static struct kmem_cache cache_cache = { - .nodelists = cache_cache_nodelists, .batchcount = 1, .limit = BOOT_CPUCACHE_ENTRIES, .shared = 1, @@ -595,7 +593,6 @@ static enum { PARTIAL_AC, PARTIAL_L3, EARLY, - LATE, FULL } g_cpucache_up; @@ -623,67 +620,37 @@ int slab_is_available(void) static struct lock_class_key on_slab_l3_key; static struct lock_class_key on_slab_alc_key; -static struct lock_class_key debugobj_l3_key; -static struct lock_class_key debugobj_alc_key; - -static void slab_set_lock_classes(struct kmem_cache *cachep, - struct lock_class_key *l3_key, struct lock_class_key *alc_key, - int q) -{ - struct array_cache **alc; - struct kmem_list3 *l3; - int r; - - l3 = cachep->nodelists[q]; - if (!l3) - return; - - lockdep_set_class(&l3->list_lock, l3_key); - alc = l3->alien; - /* - * FIXME: This check for BAD_ALIEN_MAGIC - * should go away when common slab code is taught to - * work even without alien caches. - * Currently, non NUMA code returns BAD_ALIEN_MAGIC - * for alloc_alien_cache, - */ - if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC) - return; - for_each_node(r) { - if (alc[r]) - lockdep_set_class(&alc[r]->lock, alc_key); - } -} - -static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node) -{ - slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node); -} - -static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) -{ - int node; - - for_each_online_node(node) - slab_set_debugobj_lock_classes_node(cachep, node); -} - static void init_node_lock_keys(int q) { struct cache_sizes *s = malloc_sizes; - if (g_cpucache_up < LATE) + if (g_cpucache_up != FULL) return; for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) { + struct array_cache **alc; struct kmem_list3 *l3; + int r; l3 = s->cs_cachep->nodelists[q]; if (!l3 || OFF_SLAB(s->cs_cachep)) continue; - - slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key, - &on_slab_alc_key, q); + lockdep_set_class(&l3->list_lock, &on_slab_l3_key); + alc = l3->alien; + /* + * FIXME: This check for BAD_ALIEN_MAGIC + * should go away when common slab code is taught to + * work even without alien caches. + * Currently, non NUMA code returns BAD_ALIEN_MAGIC + * for alloc_alien_cache, + */ + if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC) + continue; + for_each_node(r) { + if (alc[r]) + lockdep_set_class(&alc[r]->lock, + &on_slab_alc_key); + } } } @@ -702,14 +669,6 @@ static void init_node_lock_keys(int q) static inline void init_lock_keys(void) { } - -static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node) -{ -} - -static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) -{ -} #endif /* @@ -1303,8 +1262,6 @@ static int __cpuinit cpuup_prepare(long cpu) spin_unlock_irq(&l3->list_lock); kfree(shared); free_alien_cache(alien); - if (cachep->flags & SLAB_DEBUG_OBJECTS) - slab_set_debugobj_lock_classes_node(cachep, node); } init_node_lock_keys(node); @@ -1535,10 +1492,11 @@ void __init kmem_cache_init(void) cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node]; /* - * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids + * struct kmem_cache size depends on nr_node_ids, which + * can be less than MAX_NUMNODES. */ - cache_cache.buffer_size = offsetof(struct kmem_cache, array[nr_cpu_ids]) + - nr_node_ids * sizeof(struct kmem_list3 *); + cache_cache.buffer_size = offsetof(struct kmem_cache, nodelists) + + nr_node_ids * sizeof(struct kmem_list3 *); #if DEBUG cache_cache.obj_size = cache_cache.buffer_size; #endif @@ -1667,8 +1625,6 @@ void __init kmem_cache_init_late(void) { struct kmem_cache *cachep; - g_cpucache_up = LATE; - /* 6) resize the head arrays to their final sizes */ mutex_lock(&cache_chain_mutex); list_for_each_entry(cachep, &cache_chain, next) @@ -1676,12 +1632,12 @@ void __init kmem_cache_init_late(void) BUG(); mutex_unlock(&cache_chain_mutex); - /* Annotate slab for lockdep -- annotate the malloc caches */ - init_lock_keys(); - /* Done! */ g_cpucache_up = FULL; + /* Annotate slab for lockdep -- annotate the malloc caches */ + init_lock_keys(); + /* * Register a cpu startup notifier callback that initializes * cpu_cache_get for all new cpus @@ -1854,15 +1810,15 @@ static void dump_line(char *data, int offset, int limit) unsigned char error = 0; int bad_count = 0; - printk(KERN_ERR "%03x: ", offset); + printk(KERN_ERR "%03x:", offset); for (i = 0; i < limit; i++) { if (data[offset + i] != POISON_FREE) { error = data[offset + i]; bad_count++; } + printk(" %02x", (unsigned char)data[offset + i]); } - print_hex_dump(KERN_CONT, "", 0, 16, 1, - &data[offset], limit, 1); + printk("\n"); if (bad_count == 1) { error ^= POISON_FREE; @@ -2352,7 +2308,6 @@ kmem_cache_create (const char *name, size_t size, size_t align, if (!cachep) goto oops; - cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids]; #if DEBUG cachep->obj_size = size; @@ -2469,16 +2424,6 @@ kmem_cache_create (const char *name, size_t size, size_t align, goto oops; } - if (flags & SLAB_DEBUG_OBJECTS) { - /* - * Would deadlock through slab_destroy()->call_rcu()-> - * debug_object_activate()->kmem_cache_alloc(). - */ - WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU); - - slab_set_debugobj_lock_classes(cachep); - } - /* cache setup completed, link it into the list */ list_add(&cachep->next, &cache_chain); oops: @@ -3042,9 +2987,14 @@ bad: printk(KERN_ERR "slab: Internal list corruption detected in " "cache '%s'(%d), slabp %p(%d). Hexdump:\n", cachep->name, cachep->num, slabp, slabp->inuse); - print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, slabp, - sizeof(*slabp) + cachep->num * sizeof(kmem_bufctl_t), - 1); + for (i = 0; + i < sizeof(*slabp) + cachep->num * sizeof(kmem_bufctl_t); + i++) { + if (i % 16 == 0) + printk("\n%03x:", i); + printk(" %02x", ((unsigned char *)slabp)[i]); + } + printk("\n"); BUG(); } } @@ -3203,11 +3153,12 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, objp += obj_offset(cachep); if (cachep->ctor && cachep->flags & SLAB_POISON) cachep->ctor(objp); - if (ARCH_SLAB_MINALIGN && - ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) { +#if ARCH_SLAB_MINALIGN + if ((u32)objp & (ARCH_SLAB_MINALIGN-1)) { printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n", - objp, (int)ARCH_SLAB_MINALIGN); + objp, ARCH_SLAB_MINALIGN); } +#endif return objp; } #else @@ -3270,7 +3221,7 @@ static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) nid_alloc = cpuset_slab_spread_node(); else if (current->mempolicy) - nid_alloc = slab_node(); + nid_alloc = slab_node(current->mempolicy); if (nid_alloc != nid_here) return ____cache_alloc_node(cachep, flags, nid_alloc); return NULL; @@ -3302,7 +3253,7 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) retry_cpuset: cpuset_mems_cookie = get_mems_allowed(); - zonelist = node_zonelist(slab_node(), flags); + zonelist = node_zonelist(slab_node(current->mempolicy), flags); retry: /* @@ -3454,7 +3405,7 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, cache_alloc_debugcheck_before(cachep, flags); local_irq_save(save_flags); - if (nodeid == NUMA_NO_NODE) + if (nodeid == -1) nodeid = slab_node; if (unlikely(!cachep->nodelists[nodeid])) { @@ -3985,7 +3936,7 @@ fail: struct ccupdate_struct { struct kmem_cache *cachep; - struct array_cache *new[0]; + struct array_cache *new[NR_CPUS]; }; static void do_ccupdate_local(void *info) @@ -4007,8 +3958,7 @@ static int do_tune_cpucache(struct kmem_cache *cachep, int limit, struct ccupdate_struct *new; int i; - new = kzalloc(sizeof(*new) + nr_cpu_ids * sizeof(struct array_cache *), - gfp); + new = kzalloc(sizeof(*new), gfp); if (!new) return -ENOMEM; @@ -4585,7 +4535,7 @@ static const struct file_operations proc_slabstats_operations = { static int __init slab_proc_init(void) { - proc_create("slabinfo",S_IWUSR|S_IRUSR,NULL,&proc_slabinfo_operations); + proc_create("slabinfo",S_IWUSR|S_IRUGO,NULL,&proc_slabinfo_operations); #ifdef CONFIG_DEBUG_SLAB_LEAK proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations); #endif @@ -63,14 +63,14 @@ #include <linux/swap.h> /* struct reclaim_state */ #include <linux/cache.h> #include <linux/init.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/rcupdate.h> #include <linux/list.h> #include <linux/kmemleak.h> #include <trace/events/kmem.h> -#include <linux/atomic.h> +#include <asm/atomic.h> /* * slob_block has a field 'units', which indicates size of block if +ve, @@ -482,8 +482,6 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node) int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); void *ret; - gfp &= gfp_allowed_mask; - lockdep_trace_alloc(gfp); if (size < PAGE_SIZE - align) { @@ -610,10 +608,6 @@ void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node) { void *b; - flags &= gfp_allowed_mask; - - lockdep_trace_alloc(flags); - if (c->size < PAGE_SIZE) { b = slob_alloc(c->size, flags, c->align, node); trace_kmem_cache_alloc_node(_RET_IP_, b, c->size, @@ -2,11 +2,10 @@ * SLUB: A slab allocator that limits cache line use instead of queuing * objects in per cpu and per node lists. * - * The allocator synchronizes using per slab locks or atomic operatios - * and only uses a centralized lock to manage a pool of partial slabs. + * The allocator synchronizes using per slab locks and only + * uses a centralized lock to manage a pool of partial slabs. * * (C) 2007 SGI, Christoph Lameter - * (C) 2011 Linux Foundation, Christoph Lameter */ #include <linux/mm.h> @@ -28,33 +27,20 @@ #include <linux/memory.h> #include <linux/math64.h> #include <linux/fault-inject.h> -#include <linux/stacktrace.h> #include <trace/events/kmem.h> /* * Lock order: - * 1. slub_lock (Global Semaphore) - * 2. node->list_lock - * 3. slab_lock(page) (Only on some arches and for debugging) + * 1. slab_lock(page) + * 2. slab->list_lock * - * slub_lock - * - * The role of the slub_lock is to protect the list of all the slabs - * and to synchronize major metadata changes to slab cache structures. - * - * The slab_lock is only used for debugging and on arches that do not - * have the ability to do a cmpxchg_double. It only protects the second - * double word in the page struct. Meaning - * A. page->freelist -> List of object free in a page - * B. page->counters -> Counters of objects - * C. page->frozen -> frozen state - * - * If a slab is frozen then it is exempt from list management. It is not - * on any list. The processor that froze the slab is the one who can - * perform list operations on the page. Other processors may put objects - * onto the freelist but the processor that froze the slab is the only - * one that can retrieve the objects from the page's freelist. + * The slab_lock protects operations on the object of a particular + * slab and its metadata in the page struct. If the slab lock + * has been taken then no allocations nor frees can be performed + * on the objects in the slab nor can the slab be added or removed + * from the partial or full lists since this would mean modifying + * the page_struct of the slab. * * The list_lock protects the partial and full list on each node and * the partial slab counter. If taken then no new slabs may be added or @@ -67,6 +53,20 @@ * slabs, operations can continue without any centralized lock. F.e. * allocating a long series of objects that fill up slabs does not require * the list lock. + * + * The lock order is sometimes inverted when we are trying to get a slab + * off a list. We take the list_lock and then look for a page on the list + * to use. While we do that objects in the slabs may be freed. We can + * only operate on the slab if we have also taken the slab_lock. So we use + * a slab_trylock() on the slab. If trylock was successful then no frees + * can occur anymore and we can use the slab for allocations etc. If the + * slab_trylock() does not succeed then frees are in progress in the slab and + * we must stay away from it for a while since we may cause a bouncing + * cacheline if we try to acquire the lock. So go onto the next slab. + * If all pages are busy then we may allocate a new slab instead of reusing + * a partial slab. A new slab has no one operating on it and thus there is + * no danger of cacheline contention. + * * Interrupts are disabled during allocation and deallocation in order to * make the slab allocator safe to use in the context of an irq. In addition * interrupts are disabled to ensure that the processor does not change @@ -131,9 +131,6 @@ static inline int kmem_cache_debug(struct kmem_cache *s) /* Enable to test recovery from slab corruption on boot */ #undef SLUB_RESILIENCY_TEST -/* Enable to log cmpxchg failures */ -#undef SLUB_DEBUG_CMPXCHG - /* * Mininum number of partial slabs. These will be left on the partial * lists even if they are empty. kmem_cache_shrink may reclaim them. @@ -169,11 +166,10 @@ static inline int kmem_cache_debug(struct kmem_cache *s) #define OO_SHIFT 16 #define OO_MASK ((1 << OO_SHIFT) - 1) -#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */ +#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */ /* Internal SLUB flags */ #define __OBJECT_POISON 0x80000000UL /* Poison object */ -#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */ static int kmem_size = sizeof(struct kmem_cache); @@ -195,12 +191,8 @@ static LIST_HEAD(slab_caches); /* * Tracking user of a slab. */ -#define TRACK_ADDRS_COUNT 16 struct track { unsigned long addr; /* Called from address */ -#ifdef CONFIG_STACKTRACE - unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */ -#endif int cpu; /* Was running on cpu */ int pid; /* Pid context */ unsigned long when; /* When did the operation occur */ @@ -346,99 +338,11 @@ static inline int oo_objects(struct kmem_cache_order_objects x) return x.x & OO_MASK; } -/* - * Per slab locking using the pagelock - */ -static __always_inline void slab_lock(struct page *page) -{ - bit_spin_lock(PG_locked, &page->flags); -} - -static __always_inline void slab_unlock(struct page *page) -{ - __bit_spin_unlock(PG_locked, &page->flags); -} - -/* Interrupts must be disabled (for the fallback code to work right) */ -static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, - void *freelist_old, unsigned long counters_old, - void *freelist_new, unsigned long counters_new, - const char *n) -{ - VM_BUG_ON(!irqs_disabled()); -#ifdef CONFIG_CMPXCHG_DOUBLE - if (s->flags & __CMPXCHG_DOUBLE) { - if (cmpxchg_double(&page->freelist, - freelist_old, counters_old, - freelist_new, counters_new)) - return 1; - } else -#endif - { - slab_lock(page); - if (page->freelist == freelist_old && page->counters == counters_old) { - page->freelist = freelist_new; - page->counters = counters_new; - slab_unlock(page); - return 1; - } - slab_unlock(page); - } - - cpu_relax(); - stat(s, CMPXCHG_DOUBLE_FAIL); - -#ifdef SLUB_DEBUG_CMPXCHG - printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name); -#endif - - return 0; -} - -static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, - void *freelist_old, unsigned long counters_old, - void *freelist_new, unsigned long counters_new, - const char *n) -{ -#ifdef CONFIG_CMPXCHG_DOUBLE - if (s->flags & __CMPXCHG_DOUBLE) { - if (cmpxchg_double(&page->freelist, - freelist_old, counters_old, - freelist_new, counters_new)) - return 1; - } else -#endif - { - unsigned long flags; - - local_irq_save(flags); - slab_lock(page); - if (page->freelist == freelist_old && page->counters == counters_old) { - page->freelist = freelist_new; - page->counters = counters_new; - slab_unlock(page); - local_irq_restore(flags); - return 1; - } - slab_unlock(page); - local_irq_restore(flags); - } - - cpu_relax(); - stat(s, CMPXCHG_DOUBLE_FAIL); - -#ifdef SLUB_DEBUG_CMPXCHG - printk(KERN_INFO "%s %s: cmpxchg double redo ", n, s->name); -#endif - - return 0; -} - #ifdef CONFIG_SLUB_DEBUG /* * Determine a map of object in use on a page. * - * Node listlock must be held to guarantee that the page does + * Slab lock or node listlock must be held to guarantee that the page does * not vanish from under us. */ static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map) @@ -467,8 +371,34 @@ static int disable_higher_order_debug; */ static void print_section(char *text, u8 *addr, unsigned int length) { - print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr, - length, 1); + int i, offset; + int newline = 1; + char ascii[17]; + + ascii[16] = 0; + + for (i = 0; i < length; i++) { + if (newline) { + printk(KERN_ERR "%8s 0x%p: ", text, addr + i); + newline = 0; + } + printk(KERN_CONT " %02x", addr[i]); + offset = i % 16; + ascii[offset] = isgraph(addr[i]) ? addr[i] : '.'; + if (offset == 15) { + printk(KERN_CONT " %s\n", ascii); + newline = 1; + } + } + if (!newline) { + i %= 16; + while (i < 16) { + printk(KERN_CONT " "); + ascii[i] = ' '; + i++; + } + printk(KERN_CONT " %s\n", ascii); + } } static struct track *get_track(struct kmem_cache *s, void *object, @@ -490,24 +420,6 @@ static void set_track(struct kmem_cache *s, void *object, struct track *p = get_track(s, object, alloc); if (addr) { -#ifdef CONFIG_STACKTRACE - struct stack_trace trace; - int i; - - trace.nr_entries = 0; - trace.max_entries = TRACK_ADDRS_COUNT; - trace.entries = p->addrs; - trace.skip = 3; - save_stack_trace(&trace); - - /* See rant in lockdep.c */ - if (trace.nr_entries != 0 && - trace.entries[trace.nr_entries - 1] == ULONG_MAX) - trace.nr_entries--; - - for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++) - p->addrs[i] = 0; -#endif p->addr = addr; p->cpu = smp_processor_id(); p->pid = current->pid; @@ -532,16 +444,6 @@ static void print_track(const char *s, struct track *t) printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n", s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid); -#ifdef CONFIG_STACKTRACE - { - int i; - for (i = 0; i < TRACK_ADDRS_COUNT; i++) - if (t->addrs[i]) - printk(KERN_ERR "\t%pS\n", (void *)t->addrs[i]); - else - break; - } -#endif } static void print_tracking(struct kmem_cache *s, void *object) @@ -599,12 +501,12 @@ static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) p, p - addr, get_freepointer(s, p)); if (p > addr + 16) - print_section("Bytes b4 ", p - 16, 16); + print_section("Bytes b4", p - 16, 16); + + print_section("Object", p, min_t(unsigned long, s->objsize, PAGE_SIZE)); - print_section("Object ", p, min_t(unsigned long, s->objsize, - PAGE_SIZE)); if (s->flags & SLAB_RED_ZONE) - print_section("Redzone ", p + s->objsize, + print_section("Redzone", p + s->objsize, s->inuse - s->objsize); if (s->offset) @@ -617,7 +519,7 @@ static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) if (off != s->size) /* Beginning of the filler is the free pointer */ - print_section("Padding ", p + off, s->size - off); + print_section("Padding", p + off, s->size - off); dump_stack(); } @@ -627,6 +529,9 @@ static void object_err(struct kmem_cache *s, struct page *page, { slab_bug(s, "%s", reason); print_trailer(s, page, object); + + if(slub_debug) + panic("SLUB ERROR: object_err"); } static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...) @@ -640,6 +545,9 @@ static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...) slab_bug(s, "%s", buf); print_page_info(page); dump_stack(); + + if(slub_debug) + panic("SLUB ERROR: slab_err"); } static void init_object(struct kmem_cache *s, void *object, u8 val) @@ -655,6 +563,17 @@ static void init_object(struct kmem_cache *s, void *object, u8 val) memset(p + s->objsize, val, s->inuse - s->objsize); } +static u8 *check_bytes(u8 *start, unsigned int value, unsigned int bytes) +{ + while (bytes) { + if (*start != (u8)value) + return start; + start++; + bytes--; + } + return NULL; +} + static void restore_bytes(struct kmem_cache *s, char *message, u8 data, void *from, void *to) { @@ -669,7 +588,7 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, u8 *fault; u8 *end; - fault = memchr_inv(start, value, bytes); + fault = check_bytes(start, value, bytes); if (!fault) return 1; @@ -683,6 +602,10 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, print_trailer(s, page, object); restore_bytes(s, what, value, fault, end); + + if(slub_debug) + panic("SLUB ERROR: check_bytes_and_report. Can it be restored?"); + return 0; } @@ -762,14 +685,14 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page) if (!remainder) return 1; - fault = memchr_inv(end - remainder, POISON_INUSE, remainder); + fault = check_bytes(end - remainder, POISON_INUSE, remainder); if (!fault) return 1; while (end > fault && end[-1] == POISON_INUSE) end--; slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1); - print_section("Padding ", end - remainder, remainder); + print_section("Padding", end - remainder, remainder); restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end); return 0; @@ -860,11 +783,10 @@ static int check_slab(struct kmem_cache *s, struct page *page) static int on_freelist(struct kmem_cache *s, struct page *page, void *search) { int nr = 0; - void *fp; + void *fp = page->freelist; void *object = NULL; unsigned long max_objects; - fp = page->freelist; while (fp && nr <= page->objects) { if (fp == search) return 1; @@ -918,7 +840,7 @@ static void trace(struct kmem_cache *s, struct page *page, void *object, page->freelist); if (!alloc) - print_section("Object ", (void *)object, s->objsize); + print_section("Object", (void *)object, s->objsize); dump_stack(); } @@ -969,27 +891,26 @@ static inline void slab_free_hook(struct kmem_cache *s, void *x) /* * Tracking of fully allocated slabs for debugging purposes. - * - * list_lock must be held. */ -static void add_full(struct kmem_cache *s, - struct kmem_cache_node *n, struct page *page) +static void add_full(struct kmem_cache_node *n, struct page *page) { - if (!(s->flags & SLAB_STORE_USER)) - return; - + spin_lock(&n->list_lock); list_add(&page->lru, &n->full); + spin_unlock(&n->list_lock); } -/* - * list_lock must be held. - */ static void remove_full(struct kmem_cache *s, struct page *page) { + struct kmem_cache_node *n; + if (!(s->flags & SLAB_STORE_USER)) return; + n = get_node(s, page_to_nid(page)); + + spin_lock(&n->list_lock); list_del(&page->lru); + spin_unlock(&n->list_lock); } /* Tracking of the number of slabs for debugging purposes */ @@ -1045,6 +966,11 @@ static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *pa if (!check_slab(s, page)) goto bad; + if (!on_freelist(s, page, object)) { + object_err(s, page, object, "Object already allocated"); + goto bad; + } + if (!check_valid_pointer(s, page, object)) { object_err(s, page, object, "Freelist Pointer check fails"); goto bad; @@ -1077,12 +1003,6 @@ bad: static noinline int free_debug_processing(struct kmem_cache *s, struct page *page, void *object, unsigned long addr) { - unsigned long flags; - int rc = 0; - - local_irq_save(flags); - slab_lock(page); - if (!check_slab(s, page)) goto fail; @@ -1097,7 +1017,7 @@ static noinline int free_debug_processing(struct kmem_cache *s, } if (!check_object(s, page, object, SLUB_RED_ACTIVE)) - goto out; + return 0; if (unlikely(s != page->slab)) { if (!PageSlab(page)) { @@ -1114,19 +1034,18 @@ static noinline int free_debug_processing(struct kmem_cache *s, goto fail; } + /* Special debug activities for freeing objects */ + if (!PageSlubFrozen(page) && !page->freelist) + remove_full(s, page); if (s->flags & SLAB_STORE_USER) set_track(s, object, TRACK_FREE, addr); trace(s, page, object, 0); init_object(s, object, SLUB_RED_INACTIVE); - rc = 1; -out: - slab_unlock(page); - local_irq_restore(flags); - return rc; + return 1; fail: slab_fix(s, "Object at 0x%p not freed", object); - goto out; + return 0; } static int __init setup_slub_debug(char *str) @@ -1226,9 +1145,7 @@ static inline int slab_pad_check(struct kmem_cache *s, struct page *page) { return 1; } static inline int check_object(struct kmem_cache *s, struct page *page, void *object, u8 val) { return 1; } -static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, - struct page *page) {} -static inline void remove_full(struct kmem_cache *s, struct page *page) {} +static inline void add_full(struct kmem_cache_node *n, struct page *page) {} static inline unsigned long kmem_cache_flags(unsigned long objsize, unsigned long flags, const char *name, void (*ctor)(void *)) @@ -1280,11 +1197,6 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) struct kmem_cache_order_objects oo = s->oo; gfp_t alloc_gfp; - flags &= gfp_allowed_mask; - - if (flags & __GFP_WAIT) - local_irq_enable(); - flags |= s->allocflags; /* @@ -1301,17 +1213,12 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) * Try a lower order alloc if possible */ page = alloc_slab_page(flags, node, oo); + if (!page) + return NULL; - if (page) - stat(s, ORDER_FALLBACK); + stat(s, ORDER_FALLBACK); } - if (flags & __GFP_WAIT) - local_irq_disable(); - - if (!page) - return NULL; - if (kmemcheck_enabled && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { int pages = 1 << oo_order(oo); @@ -1378,8 +1285,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) set_freepointer(s, last, NULL); page->freelist = start; - page->inuse = page->objects; - page->frozen = 1; + page->inuse = 0; out: return page; } @@ -1457,84 +1363,79 @@ static void discard_slab(struct kmem_cache *s, struct page *page) } /* - * Management of partially allocated slabs. - * - * list_lock must be held. + * Per slab locking using the pagelock */ -static inline void add_partial(struct kmem_cache_node *n, +static __always_inline void slab_lock(struct page *page) +{ + bit_spin_lock(PG_locked, &page->flags); +} + +static __always_inline void slab_unlock(struct page *page) +{ + __bit_spin_unlock(PG_locked, &page->flags); +} + +static __always_inline int slab_trylock(struct page *page) +{ + int rc = 1; + + rc = bit_spin_trylock(PG_locked, &page->flags); + return rc; +} + +/* + * Management of partially allocated slabs + */ +static void add_partial(struct kmem_cache_node *n, struct page *page, int tail) { + spin_lock(&n->list_lock); n->nr_partial++; - if (tail == DEACTIVATE_TO_TAIL) + if (tail) list_add_tail(&page->lru, &n->partial); else list_add(&page->lru, &n->partial); + spin_unlock(&n->list_lock); } -/* - * list_lock must be held. - */ -static inline void remove_partial(struct kmem_cache_node *n, +static inline void __remove_partial(struct kmem_cache_node *n, struct page *page) { list_del(&page->lru); n->nr_partial--; } +static void remove_partial(struct kmem_cache *s, struct page *page) +{ + struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + + spin_lock(&n->list_lock); + __remove_partial(n, page); + spin_unlock(&n->list_lock); +} + /* - * Lock slab, remove from the partial list and put the object into the - * per cpu freelist. - * - * Returns a list of objects or NULL if it fails. + * Lock slab and remove from the partial list. * * Must hold list_lock. */ -static inline void *acquire_slab(struct kmem_cache *s, - struct kmem_cache_node *n, struct page *page, - int mode) +static inline int lock_and_freeze_slab(struct kmem_cache_node *n, + struct page *page) { - void *freelist; - unsigned long counters; - struct page new; - - /* - * Zap the freelist and set the frozen bit. - * The old freelist is the list of objects for the - * per cpu allocation list. - */ - do { - freelist = page->freelist; - counters = page->counters; - new.counters = counters; - if (mode) { - new.inuse = page->objects; - new.freelist = NULL; - } else { - new.freelist = freelist; - } - - VM_BUG_ON(new.frozen); - new.frozen = 1; - - } while (!__cmpxchg_double_slab(s, page, - freelist, counters, - new.freelist, new.counters, - "lock and freeze")); - - remove_partial(n, page); - return freelist; + if (slab_trylock(page)) { + __remove_partial(n, page); + __SetPageSlubFrozen(page); + return 1; + } + return 0; } -static int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain); - /* * Try to allocate a partial slab from a specific node. */ -static void *get_partial_node(struct kmem_cache *s, - struct kmem_cache_node *n, struct kmem_cache_cpu *c) +static struct page *get_partial_node(struct kmem_cache_node *n) { - struct page *page, *page2; - void *object = NULL; + struct page *page; /* * Racy check. If we mistakenly see no partial slabs then we @@ -1546,42 +1447,26 @@ static void *get_partial_node(struct kmem_cache *s, return NULL; spin_lock(&n->list_lock); - list_for_each_entry_safe(page, page2, &n->partial, lru) { - void *t = acquire_slab(s, n, page, object == NULL); - int available; - - if (!t) - break; - - if (!object) { - c->page = page; - c->node = page_to_nid(page); - stat(s, ALLOC_FROM_PARTIAL); - object = t; - available = page->objects - page->inuse; - } else { - available = put_cpu_partial(s, page, 0); - } - if (kmem_cache_debug(s) || available > s->cpu_partial / 2) - break; - - } + list_for_each_entry(page, &n->partial, lru) + if (lock_and_freeze_slab(n, page)) + goto out; + page = NULL; +out: spin_unlock(&n->list_lock); - return object; + return page; } /* * Get a page from somewhere. Search in increasing NUMA distances. */ -static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags, - struct kmem_cache_cpu *c) +static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags) { #ifdef CONFIG_NUMA struct zonelist *zonelist; struct zoneref *z; struct zone *zone; enum zone_type high_zoneidx = gfp_zone(flags); - void *object; + struct page *page; unsigned int cpuset_mems_cookie; /* @@ -1608,7 +1493,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags, do { cpuset_mems_cookie = get_mems_allowed(); - zonelist = node_zonelist(slab_node(), flags); + zonelist = node_zonelist(slab_node(current->mempolicy), flags); for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { struct kmem_cache_node *n; @@ -1616,8 +1501,8 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags, if (n && cpuset_zone_allowed_hardwall(zone, flags) && n->nr_partial > s->min_partial) { - object = get_partial_node(s, n, c); - if (object) { + page = get_partial_node(n); + if (page) { /* * Return the object even if * put_mems_allowed indicated that @@ -1627,7 +1512,7 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags, * and the cpuset update. */ put_mems_allowed(cpuset_mems_cookie); - return object; + return page; } } } @@ -1639,17 +1524,63 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags, /* * Get a partial page, lock it and return it. */ -static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, - struct kmem_cache_cpu *c) +static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node) { - void *object; + struct page *page; int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node; - object = get_partial_node(s, get_node(s, searchnode), c); - if (object || node != NUMA_NO_NODE) - return object; + page = get_partial_node(get_node(s, searchnode)); + if (page || node != NUMA_NO_NODE) + return page; - return get_any_partial(s, flags, c); + return get_any_partial(s, flags); +} + +/* + * Move a page back to the lists. + * + * Must be called with the slab lock held. + * + * On exit the slab lock will have been dropped. + */ +static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) + __releases(bitlock) +{ + struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + + __ClearPageSlubFrozen(page); + if (page->inuse) { + + if (page->freelist) { + add_partial(n, page, tail); + stat(s, tail ? DEACTIVATE_TO_TAIL : DEACTIVATE_TO_HEAD); + } else { + stat(s, DEACTIVATE_FULL); + if (kmem_cache_debug(s) && (s->flags & SLAB_STORE_USER)) + add_full(n, page); + } + slab_unlock(page); + } else { + stat(s, DEACTIVATE_EMPTY); + if (n->nr_partial < s->min_partial) { + /* + * Adding an empty slab to the partial slabs in order + * to avoid page allocator overhead. This slab needs + * to come after the other slabs with objects in + * so that the others get filled first. That way the + * size of the partial list stays small. + * + * kmem_cache_shrink can reclaim any empty slabs from + * the partial list. + */ + add_partial(n, page, 1); + slab_unlock(page); + } else { + slab_unlock(page); + stat(s, FREE_SLAB); + discard_slab(s, page); + } + } } #ifdef CONFIG_PREEMPT @@ -1718,270 +1649,45 @@ void init_kmem_cache_cpus(struct kmem_cache *s) for_each_possible_cpu(cpu) per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); } - /* * Remove the cpu slab */ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) + __releases(bitlock) { - enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; struct page *page = c->page; - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); - int lock = 0; - enum slab_modes l = M_NONE, m = M_NONE; - void *freelist; - void *nextfree; - int tail = DEACTIVATE_TO_HEAD; - struct page new; - struct page old; - - if (page->freelist) { - stat(s, DEACTIVATE_REMOTE_FREES); - tail = DEACTIVATE_TO_TAIL; - } - - c->tid = next_tid(c->tid); - c->page = NULL; - freelist = c->freelist; - c->freelist = NULL; - - /* - * Stage one: Free all available per cpu objects back - * to the page freelist while it is still frozen. Leave the - * last one. - * - * There is no need to take the list->lock because the page - * is still frozen. - */ - while (freelist && (nextfree = get_freepointer(s, freelist))) { - void *prior; - unsigned long counters; - - do { - prior = page->freelist; - counters = page->counters; - set_freepointer(s, freelist, prior); - new.counters = counters; - new.inuse--; - VM_BUG_ON(!new.frozen); - - } while (!__cmpxchg_double_slab(s, page, - prior, counters, - freelist, new.counters, - "drain percpu freelist")); - - freelist = nextfree; - } + int tail = 1; + if (page->freelist) + stat(s, DEACTIVATE_REMOTE_FREES); /* - * Stage two: Ensure that the page is unfrozen while the - * list presence reflects the actual number of objects - * during unfreeze. - * - * We setup the list membership and then perform a cmpxchg - * with the count. If there is a mismatch then the page - * is not unfrozen but the page is on the wrong list. - * - * Then we restart the process which may have to remove - * the page from the list that we just put it on again - * because the number of objects in the slab may have - * changed. + * Merge cpu freelist into slab freelist. Typically we get here + * because both freelists are empty. So this is unlikely + * to occur. */ -redo: - - old.freelist = page->freelist; - old.counters = page->counters; - VM_BUG_ON(!old.frozen); - - /* Determine target state of the slab */ - new.counters = old.counters; - if (freelist) { - new.inuse--; - set_freepointer(s, freelist, old.freelist); - new.freelist = freelist; - } else - new.freelist = old.freelist; - - new.frozen = 0; + while (unlikely(c->freelist)) { + void **object; - if (!new.inuse && n->nr_partial > s->min_partial) - m = M_FREE; - else if (new.freelist) { - m = M_PARTIAL; - if (!lock) { - lock = 1; - /* - * Taking the spinlock removes the possiblity - * that acquire_slab() will see a slab page that - * is frozen - */ - spin_lock(&n->list_lock); - } - } else { - m = M_FULL; - if (kmem_cache_debug(s) && !lock) { - lock = 1; - /* - * This also ensures that the scanning of full - * slabs from diagnostic functions will not see - * any frozen slabs. - */ - spin_lock(&n->list_lock); - } - } - - if (l != m) { + tail = 0; /* Hot objects. Put the slab first */ - if (l == M_PARTIAL) + /* Retrieve object from cpu_freelist */ + object = c->freelist; + c->freelist = get_freepointer(s, c->freelist); - remove_partial(n, page); - - else if (l == M_FULL) - - remove_full(s, page); - - if (m == M_PARTIAL) { - - add_partial(n, page, tail); - stat(s, tail); - - } else if (m == M_FULL) { - - stat(s, DEACTIVATE_FULL); - add_full(s, n, page); - - } - } - - l = m; - if (!__cmpxchg_double_slab(s, page, - old.freelist, old.counters, - new.freelist, new.counters, - "unfreezing slab")) - goto redo; - - if (lock) - spin_unlock(&n->list_lock); - - if (m == M_FREE) { - stat(s, DEACTIVATE_EMPTY); - discard_slab(s, page); - stat(s, FREE_SLAB); + /* And put onto the regular freelist */ + set_freepointer(s, object, page->freelist); + page->freelist = object; + page->inuse--; } -} - -/* Unfreeze all the cpu partial slabs */ -static void unfreeze_partials(struct kmem_cache *s) -{ - struct kmem_cache_node *n = NULL, *n2 = NULL; - struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab); - struct page *page, *discard_page = NULL; - - while ((page = c->partial)) { - struct page new; - struct page old; - - c->partial = page->next; - - n2 = get_node(s, page_to_nid(page)); - if (n != n2) { - if (n) - spin_unlock(&n->list_lock); - - n = n2; - spin_lock(&n->list_lock); - } - - do { - - old.freelist = page->freelist; - old.counters = page->counters; - VM_BUG_ON(!old.frozen); - - new.counters = old.counters; - new.freelist = old.freelist; - - new.frozen = 0; - - } while (!cmpxchg_double_slab(s, page, - old.freelist, old.counters, - new.freelist, new.counters, - "unfreezing slab")); - - if (unlikely(!new.inuse && n->nr_partial > s->min_partial)) { - page->next = discard_page; - discard_page = page; - } else { - add_partial(n, page, DEACTIVATE_TO_TAIL); - stat(s, FREE_ADD_PARTIAL); - } - } - - if (n) - spin_unlock(&n->list_lock); - - while (discard_page) { - page = discard_page; - discard_page = discard_page->next; - - stat(s, DEACTIVATE_EMPTY); - discard_slab(s, page); - stat(s, FREE_SLAB); - } -} - -/* - * Put a page that was just frozen (in __slab_free) into a partial page - * slot if available. This is done without interrupts disabled and without - * preemption disabled. The cmpxchg is racy and may put the partial page - * onto a random cpus partial slot. - * - * If we did not find a slot then simply move all the partials to the - * per node partial list. - */ -int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) -{ - struct page *oldpage; - int pages; - int pobjects; - - do { - pages = 0; - pobjects = 0; - oldpage = this_cpu_read(s->cpu_slab->partial); - - if (oldpage) { - pobjects = oldpage->pobjects; - pages = oldpage->pages; - if (drain && pobjects > s->cpu_partial) { - unsigned long flags; - /* - * partial array is full. Move the existing - * set to the per node partial list. - */ - local_irq_save(flags); - unfreeze_partials(s); - local_irq_restore(flags); - pobjects = 0; - pages = 0; - } - } - - pages++; - pobjects += page->objects - page->inuse; - - page->pages = pages; - page->pobjects = pobjects; - page->next = oldpage; - - } while (irqsafe_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != oldpage); - stat(s, CPU_PARTIAL_FREE); - return pobjects; + c->page = NULL; + c->tid = next_tid(c->tid); + unfreeze_slab(s, page, tail); } static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) { stat(s, CPUSLAB_FLUSH); + slab_lock(c->page); deactivate_slab(s, c); } @@ -1994,12 +1700,8 @@ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) { struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); - if (likely(c)) { - if (c->page) - flush_slab(s, c); - - unfreeze_partials(s); - } + if (likely(c && c->page)) + flush_slab(s, c); } static void flush_cpu_slab(void *d) @@ -2090,39 +1792,12 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) } } -static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, - int node, struct kmem_cache_cpu **pc) -{ - void *object; - struct kmem_cache_cpu *c; - struct page *page = new_slab(s, flags, node); - - if (page) { - c = __this_cpu_ptr(s->cpu_slab); - if (c->page) - flush_slab(s, c); - - /* - * No other reference to the page yet so we can - * muck around with it freely without cmpxchg - */ - object = page->freelist; - page->freelist = NULL; - - stat(s, ALLOC_SLAB); - c->node = page_to_nid(page); - c->page = page; - *pc = c; - } else - object = NULL; - - return object; -} - /* * Slow path. The lockless freelist is empty or we need to perform * debugging duties. * + * Interrupts are disabled. + * * Processing is still very fast if new objects have been freed to the * regular freelist. In that case we simply take over the regular freelist * as the lockless freelist and zap the regular freelist. @@ -2139,9 +1814,8 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr, struct kmem_cache_cpu *c) { void **object; + struct page *page; unsigned long flags; - struct page new; - unsigned long counters; local_irq_save(flags); #ifdef CONFIG_PREEMPT @@ -2153,96 +1827,87 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, c = this_cpu_ptr(s->cpu_slab); #endif - if (!c->page) - goto new_slab; -redo: - if (unlikely(!node_match(c, node))) { - stat(s, ALLOC_NODE_MISMATCH); - deactivate_slab(s, c); + /* We handle __GFP_ZERO in the caller */ + gfpflags &= ~__GFP_ZERO; + + page = c->page; + if (!page) goto new_slab; - } + + slab_lock(page); + if (unlikely(!node_match(c, node))) + goto another_slab; /* must check again c->freelist in case of cpu migration or IRQ */ object = c->freelist; if (object) - goto load_freelist; - - stat(s, ALLOC_SLOWPATH); - - do { - object = c->page->freelist; - counters = c->page->counters; - new.counters = counters; - VM_BUG_ON(!new.frozen); - - /* - * If there is no object left then we use this loop to - * deactivate the slab which is simple since no objects - * are left in the slab and therefore we do not need to - * put the page back onto the partial list. - * - * If there are objects left then we retrieve them - * and use them to refill the per cpu queue. - */ - - new.inuse = c->page->objects; - new.frozen = object != NULL; - - } while (!__cmpxchg_double_slab(s, c->page, - object, counters, - NULL, new.counters, - "__slab_alloc")); - - if (!object) { - c->page = NULL; - stat(s, DEACTIVATE_BYPASS); - goto new_slab; - } + goto update_freelist; stat(s, ALLOC_REFILL); load_freelist: + object = page->freelist; + if (unlikely(!object)) + goto another_slab; + if (kmem_cache_debug(s)) + goto debug; + +update_freelist: c->freelist = get_freepointer(s, object); + page->inuse = page->objects; + page->freelist = NULL; + + slab_unlock(page); c->tid = next_tid(c->tid); local_irq_restore(flags); + stat(s, ALLOC_SLOWPATH); return object; -new_slab: +another_slab: + deactivate_slab(s, c); - if (c->partial) { - c->page = c->partial; - c->partial = c->page->next; - c->node = page_to_nid(c->page); - stat(s, CPU_PARTIAL_ALLOC); - c->freelist = NULL; - goto redo; +new_slab: + page = get_partial(s, gfpflags, node); + if (page) { + stat(s, ALLOC_FROM_PARTIAL); + c->node = page_to_nid(page); + c->page = page; + goto load_freelist; } - /* Then do expensive stuff like retrieving pages from the partial lists */ - object = get_partial(s, gfpflags, node, c); - - if (unlikely(!object)) { + gfpflags &= gfp_allowed_mask; + if (gfpflags & __GFP_WAIT) + local_irq_enable(); - object = new_slab_objects(s, gfpflags, node, &c); + page = new_slab(s, gfpflags, node); - if (unlikely(!object)) { - if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) - slab_out_of_memory(s, gfpflags, node); + if (gfpflags & __GFP_WAIT) + local_irq_disable(); - local_irq_restore(flags); - return NULL; - } - } + if (page) { + c = __this_cpu_ptr(s->cpu_slab); + stat(s, ALLOC_SLAB); + if (c->page) + flush_slab(s, c); - if (likely(!kmem_cache_debug(s))) + slab_lock(page); + __SetPageSlubFrozen(page); + c->node = page_to_nid(page); + c->page = page; goto load_freelist; + } + if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) + slab_out_of_memory(s, gfpflags, node); + local_irq_restore(flags); + return NULL; +debug: + if (!alloc_debug_processing(s, page, object, addr)) + goto another_slab; - /* Only entered in the debug case */ - if (!alloc_debug_processing(s, c->page, object, addr)) - goto new_slab; /* Slab failed checks. Next slab needed */ - - c->freelist = get_freepointer(s, object); + page->inuse++; + page->freelist = get_freepointer(s, object); deactivate_slab(s, c); + c->page = NULL; c->node = NUMA_NO_NODE; local_irq_restore(flags); return object; @@ -2392,110 +2057,52 @@ static void __slab_free(struct kmem_cache *s, struct page *page, { void *prior; void **object = (void *)x; - int was_frozen; - int inuse; - struct page new; - unsigned long counters; - struct kmem_cache_node *n = NULL; - unsigned long uninitialized_var(flags); + unsigned long flags; + local_irq_save(flags); + slab_lock(page); stat(s, FREE_SLOWPATH); if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr)) - return; - - do { - prior = page->freelist; - counters = page->counters; - set_freepointer(s, object, prior); - new.counters = counters; - was_frozen = new.frozen; - new.inuse--; - if ((!new.inuse || !prior) && !was_frozen && !n) { - - if (!kmem_cache_debug(s) && !prior) - - /* - * Slab was on no list before and will be partially empty - * We can defer the list move and instead freeze it. - */ - new.frozen = 1; - - else { /* Needs to be taken off a list */ - - n = get_node(s, page_to_nid(page)); - /* - * Speculatively acquire the list_lock. - * If the cmpxchg does not succeed then we may - * drop the list_lock without any processing. - * - * Otherwise the list_lock will synchronize with - * other processors updating the list of slabs. - */ - spin_lock_irqsave(&n->list_lock, flags); - - } - } - inuse = new.inuse; - - } while (!cmpxchg_double_slab(s, page, - prior, counters, - object, new.counters, - "__slab_free")); + goto out_unlock; - if (likely(!n)) { + prior = page->freelist; + set_freepointer(s, object, prior); + page->freelist = object; + page->inuse--; - /* - * If we just froze the page then put it onto the - * per cpu partial list. - */ - if (new.frozen && !was_frozen) - put_cpu_partial(s, page, 1); + if (unlikely(PageSlubFrozen(page))) { + stat(s, FREE_FROZEN); + goto out_unlock; + } - /* - * The list lock was not taken therefore no list - * activity can be necessary. - */ - if (was_frozen) - stat(s, FREE_FROZEN); - return; - } + if (unlikely(!page->inuse)) + goto slab_empty; /* - * was_frozen may have been set after we acquired the list_lock in - * an earlier loop. So we need to check it here again. + * Objects left in the slab. If it was not on the partial list before + * then add it. */ - if (was_frozen) - stat(s, FREE_FROZEN); - else { - if (unlikely(!inuse && n->nr_partial > s->min_partial)) - goto slab_empty; - - /* - * Objects left in the slab. If it was not on the partial list before - * then add it. - */ - if (unlikely(!prior)) { - remove_full(s, page); - add_partial(n, page, DEACTIVATE_TO_TAIL); - stat(s, FREE_ADD_PARTIAL); - } + if (unlikely(!prior)) { + add_partial(get_node(s, page_to_nid(page)), page, 1); + stat(s, FREE_ADD_PARTIAL); } - spin_unlock_irqrestore(&n->list_lock, flags); + +out_unlock: + slab_unlock(page); + local_irq_restore(flags); return; slab_empty: if (prior) { /* - * Slab on the partial list. + * Slab still on the partial list. */ - remove_partial(n, page); + remove_partial(s, page); stat(s, FREE_REMOVE_PARTIAL); - } else - /* Slab must be on the full list */ - remove_full(s, page); - - spin_unlock_irqrestore(&n->list_lock, flags); + } + slab_unlock(page); + local_irq_restore(flags); stat(s, FREE_SLAB); discard_slab(s, page); } @@ -2521,6 +2128,7 @@ static __always_inline void slab_free(struct kmem_cache *s, slab_free_hook(s, x); redo: + /* * Determine the currently cpus per cpu slab. * The cpu may change afterward. However that does not matter since @@ -2768,6 +2376,7 @@ static void early_kmem_cache_node_alloc(int node) { struct page *page; struct kmem_cache_node *n; + unsigned long flags; BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); @@ -2784,8 +2393,7 @@ static void early_kmem_cache_node_alloc(int node) n = page->freelist; BUG_ON(!n); page->freelist = get_freepointer(kmem_cache_node, n); - page->inuse = 1; - page->frozen = 0; + page->inuse++; kmem_cache_node->node[node] = n; #ifdef CONFIG_SLUB_DEBUG init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); @@ -2794,7 +2402,14 @@ static void early_kmem_cache_node_alloc(int node) init_kmem_cache_node(n, kmem_cache_node); inc_slabs_node(kmem_cache_node, node, page->objects); - add_partial(n, page, DEACTIVATE_TO_HEAD); + /* + * lockdep requires consistent irq usage for each lock + * so even though there cannot be a race this early in + * the boot sequence, we still disable irqs. + */ + local_irq_save(flags); + add_partial(n, page, 0); + local_irq_restore(flags); } static void free_kmem_cache_nodes(struct kmem_cache *s) @@ -3000,44 +2615,11 @@ static int kmem_cache_open(struct kmem_cache *s, } } -#ifdef CONFIG_CMPXCHG_DOUBLE - if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0) - /* Enable fast mode */ - s->flags |= __CMPXCHG_DOUBLE; -#endif - /* * The larger the object size is, the more pages we want on the partial * list to avoid pounding the page allocator excessively. */ - set_min_partial(s, ilog2(s->size) / 2); - - /* - * cpu_partial determined the maximum number of objects kept in the - * per cpu partial lists of a processor. - * - * Per cpu partial lists mainly contain slabs that just have one - * object freed. If they are used for allocation then they can be - * filled up again with minimal effort. The slab will never hit the - * per node partial lists and therefore no locking will be required. - * - * This setting also determines - * - * A) The number of objects from per cpu partial slabs dumped to the - * per node list when we reach the limit. - * B) The number of objects in cpu partial slabs to extract from the - * per node list when we run out of per cpu objects. We only fetch 50% - * to keep some capacity around for frees. - */ - if (s->size >= PAGE_SIZE) - s->cpu_partial = 2; - else if (s->size >= 1024) - s->cpu_partial = 6; - else if (s->size >= 256) - s->cpu_partial = 13; - else - s->cpu_partial = 30; - + set_min_partial(s, ilog2(s->size)); s->refcount = 1; #ifdef CONFIG_NUMA s->remote_node_defrag_ratio = 1000; @@ -3096,22 +2678,23 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page, /* * Attempt to free all partial slabs on a node. - * This is called from kmem_cache_close(). We must be the last thread - * using the cache and therefore we do not need to lock anymore. */ static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) { + unsigned long flags; struct page *page, *h; + spin_lock_irqsave(&n->list_lock, flags); list_for_each_entry_safe(page, h, &n->partial, lru) { if (!page->inuse) { - remove_partial(n, page); + __remove_partial(n, page); discard_slab(s, page); } else { list_slab_objects(s, page, "Objects remaining on kmem_cache_close()"); } } + spin_unlock_irqrestore(&n->list_lock, flags); } /* @@ -3145,7 +2728,6 @@ void kmem_cache_destroy(struct kmem_cache *s) s->refcount--; if (!s->refcount) { list_del(&s->list); - up_write(&slub_lock); if (kmem_cache_close(s)) { printk(KERN_ERR "SLUB %s: %s called for cache that " "still has objects.\n", s->name, __func__); @@ -3154,8 +2736,8 @@ void kmem_cache_destroy(struct kmem_cache *s) if (s->flags & SLAB_DESTROY_BY_RCU) rcu_barrier(); sysfs_slab_remove(s); - } else - up_write(&slub_lock); + } + up_write(&slub_lock); } EXPORT_SYMBOL(kmem_cache_destroy); @@ -3372,42 +2954,6 @@ size_t ksize(const void *object) } EXPORT_SYMBOL(ksize); -#ifdef CONFIG_SLUB_DEBUG -bool verify_mem_not_deleted(const void *x) -{ - struct page *page; - void *object = (void *)x; - unsigned long flags; - bool rv; - - if (unlikely(ZERO_OR_NULL_PTR(x))) - return false; - - local_irq_save(flags); - - page = virt_to_head_page(x); - if (unlikely(!PageSlab(page))) { - /* maybe it was from stack? */ - rv = true; - goto out_unlock; - } - - slab_lock(page); - if (on_freelist(page->slab, page, object)) { - object_err(page->slab, page, object, "Object is on free-list"); - rv = false; - } else { - rv = true; - } - slab_unlock(page); - -out_unlock: - local_irq_restore(flags); - return rv; -} -EXPORT_SYMBOL(verify_mem_not_deleted); -#endif - void kfree(const void *x) { struct page *page; @@ -3473,23 +3019,29 @@ int kmem_cache_shrink(struct kmem_cache *s) * list_lock. page->inuse here is the upper limit. */ list_for_each_entry_safe(page, t, &n->partial, lru) { - list_move(&page->lru, slabs_by_inuse + page->inuse); - if (!page->inuse) - n->nr_partial--; + if (!page->inuse && slab_trylock(page)) { + /* + * Must hold slab lock here because slab_free + * may have freed the last object and be + * waiting to release the slab. + */ + __remove_partial(n, page); + slab_unlock(page); + discard_slab(s, page); + } else { + list_move(&page->lru, + slabs_by_inuse + page->inuse); + } } /* * Rebuild the partial list with the slabs filled up most * first and the least used slabs at the end. */ - for (i = objects - 1; i > 0; i--) + for (i = objects - 1; i >= 0; i--) list_splice(slabs_by_inuse + i, n->partial.prev); spin_unlock_irqrestore(&n->list_lock, flags); - - /* Release empty slabs */ - list_for_each_entry_safe(page, t, slabs_by_inuse, lru) - discard_slab(s, page); } kfree(slabs_by_inuse); @@ -4063,9 +3615,12 @@ static int validate_slab(struct kmem_cache *s, struct page *page, static void validate_slab_slab(struct kmem_cache *s, struct page *page, unsigned long *map) { - slab_lock(page); - validate_slab(s, page, map); - slab_unlock(page); + if (slab_trylock(page)) { + validate_slab(s, page, map); + slab_unlock(page); + } else + printk(KERN_INFO "SLUB %s: Skipped busy slab 0x%p\n", + s->name, page); } static int validate_slab_node(struct kmem_cache *s, @@ -4446,37 +4001,22 @@ static ssize_t show_slab_objects(struct kmem_cache *s, for_each_possible_cpu(cpu) { struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); - int node = ACCESS_ONCE(c->node); - struct page *page; - if (node < 0) + if (!c || c->node < 0) continue; - page = ACCESS_ONCE(c->page); - if (page) { - if (flags & SO_TOTAL) - x = page->objects; + + if (c->page) { + if (flags & SO_TOTAL) + x = c->page->objects; else if (flags & SO_OBJECTS) - x = page->inuse; + x = c->page->inuse; else x = 1; total += x; - nodes[node] += x; + nodes[c->node] += x; } - page = c->partial; - - if (page) { - node = page_to_nid(page); - if (flags & SO_TOTAL) - WARN_ON_ONCE(1); - else if (flags & SO_OBJECTS) - WARN_ON_ONCE(1); - else - x = page->pages; - total += x; - nodes[node] += x; - } - per_cpu[node]++; + per_cpu[c->node]++; } } @@ -4545,7 +4085,7 @@ static int any_slab_objects(struct kmem_cache *s) #endif #define to_slab_attr(n) container_of(n, struct slab_attribute, attr) -#define to_slab(n) container_of(n, struct kmem_cache, kobj) +#define to_slab(n) container_of(n, struct kmem_cache, kobj); struct slab_attribute { struct attribute attr; @@ -4554,12 +4094,11 @@ struct slab_attribute { }; #define SLAB_ATTR_RO(_name) \ - static struct slab_attribute _name##_attr = \ - __ATTR(_name, 0400, _name##_show, NULL) + static struct slab_attribute _name##_attr = __ATTR_RO(_name) #define SLAB_ATTR(_name) \ static struct slab_attribute _name##_attr = \ - __ATTR(_name, 0600, _name##_show, _name##_store) + __ATTR(_name, 0644, _name##_show, _name##_store) static ssize_t slab_size_show(struct kmem_cache *s, char *buf) { @@ -4628,27 +4167,6 @@ static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, } SLAB_ATTR(min_partial); -static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) -{ - return sprintf(buf, "%u\n", s->cpu_partial); -} - -static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, - size_t length) -{ - unsigned long objects; - int err; - - err = strict_strtoul(buf, 10, &objects); - if (err) - return err; - - s->cpu_partial = objects; - flush_all(s); - return length; -} -SLAB_ATTR(cpu_partial); - static ssize_t ctor_show(struct kmem_cache *s, char *buf) { if (!s->ctor) @@ -4687,37 +4205,6 @@ static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) } SLAB_ATTR_RO(objects_partial); -static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) -{ - int objects = 0; - int pages = 0; - int cpu; - int len; - - for_each_online_cpu(cpu) { - struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial; - - if (page) { - pages += page->pages; - objects += page->pobjects; - } - } - - len = sprintf(buf, "%d(%d)", objects, pages); - -#ifdef CONFIG_SMP - for_each_online_cpu(cpu) { - struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial; - - if (page && len < PAGE_SIZE - 20) - len += sprintf(buf + len, " C%d=%d(%d)", cpu, - page->pobjects, page->pages); - } -#endif - return len + sprintf(buf + len, "\n"); -} -SLAB_ATTR_RO(slabs_cpu_partial); - static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); @@ -4781,10 +4268,8 @@ static ssize_t sanity_checks_store(struct kmem_cache *s, const char *buf, size_t length) { s->flags &= ~SLAB_DEBUG_FREE; - if (buf[0] == '1') { - s->flags &= ~__CMPXCHG_DOUBLE; + if (buf[0] == '1') s->flags |= SLAB_DEBUG_FREE; - } return length; } SLAB_ATTR(sanity_checks); @@ -4798,10 +4283,8 @@ static ssize_t trace_store(struct kmem_cache *s, const char *buf, size_t length) { s->flags &= ~SLAB_TRACE; - if (buf[0] == '1') { - s->flags &= ~__CMPXCHG_DOUBLE; + if (buf[0] == '1') s->flags |= SLAB_TRACE; - } return length; } SLAB_ATTR(trace); @@ -4818,10 +4301,8 @@ static ssize_t red_zone_store(struct kmem_cache *s, return -EBUSY; s->flags &= ~SLAB_RED_ZONE; - if (buf[0] == '1') { - s->flags &= ~__CMPXCHG_DOUBLE; + if (buf[0] == '1') s->flags |= SLAB_RED_ZONE; - } calculate_sizes(s, -1); return length; } @@ -4839,10 +4320,8 @@ static ssize_t poison_store(struct kmem_cache *s, return -EBUSY; s->flags &= ~SLAB_POISON; - if (buf[0] == '1') { - s->flags &= ~__CMPXCHG_DOUBLE; + if (buf[0] == '1') s->flags |= SLAB_POISON; - } calculate_sizes(s, -1); return length; } @@ -4860,10 +4339,8 @@ static ssize_t store_user_store(struct kmem_cache *s, return -EBUSY; s->flags &= ~SLAB_STORE_USER; - if (buf[0] == '1') { - s->flags &= ~__CMPXCHG_DOUBLE; + if (buf[0] == '1') s->flags |= SLAB_STORE_USER; - } calculate_sizes(s, -1); return length; } @@ -5028,7 +4505,6 @@ STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); STAT_ATTR(ALLOC_SLAB, alloc_slab); STAT_ATTR(ALLOC_REFILL, alloc_refill); -STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); STAT_ATTR(FREE_SLAB, free_slab); STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); STAT_ATTR(DEACTIVATE_FULL, deactivate_full); @@ -5036,12 +4512,7 @@ STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); -STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); STAT_ATTR(ORDER_FALLBACK, order_fallback); -STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); -STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); -STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); -STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); #endif static struct attribute *slab_attrs[] = { @@ -5050,7 +4521,6 @@ static struct attribute *slab_attrs[] = { &objs_per_slab_attr.attr, &order_attr.attr, &min_partial_attr.attr, - &cpu_partial_attr.attr, &objects_attr.attr, &objects_partial_attr.attr, &partial_attr.attr, @@ -5063,7 +4533,6 @@ static struct attribute *slab_attrs[] = { &destroy_by_rcu_attr.attr, &shrink_attr.attr, &reserved_attr.attr, - &slabs_cpu_partial_attr.attr, #ifdef CONFIG_SLUB_DEBUG &total_objects_attr.attr, &slabs_attr.attr, @@ -5093,7 +4562,6 @@ static struct attribute *slab_attrs[] = { &alloc_from_partial_attr.attr, &alloc_slab_attr.attr, &alloc_refill_attr.attr, - &alloc_node_mismatch_attr.attr, &free_slab_attr.attr, &cpuslab_flush_attr.attr, &deactivate_full_attr.attr, @@ -5101,12 +4569,7 @@ static struct attribute *slab_attrs[] = { &deactivate_to_head_attr.attr, &deactivate_to_tail_attr.attr, &deactivate_remote_frees_attr.attr, - &deactivate_bypass_attr.attr, &order_fallback_attr.attr, - &cmpxchg_double_fail_attr.attr, - &cmpxchg_double_cpu_fail_attr.attr, - &cpu_partial_alloc_attr.attr, - &cpu_partial_free_attr.attr, #endif #ifdef CONFIG_FAILSLAB &failslab_attr.attr, @@ -5458,7 +4921,7 @@ static const struct file_operations proc_slabinfo_operations = { static int __init slab_proc_init(void) { - proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations); + proc_create("slabinfo", S_IRUGO, NULL, &proc_slabinfo_operations); return 0; } module_init(slab_proc_init); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 1b7e22a..64b9840 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -21,6 +21,7 @@ #include <linux/mmzone.h> #include <linux/bootmem.h> #include <linux/highmem.h> +#include <linux/module.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/vmalloc.h> diff --git a/mm/sparse.c b/mm/sparse.c index 42935b5..9054f83 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -6,7 +6,7 @@ #include <linux/mmzone.h> #include <linux/bootmem.h> #include <linux/highmem.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/spinlock.h> #include <linux/vmalloc.h> #include "internal.h" @@ -40,7 +40,7 @@ static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; #endif -int page_to_nid(const struct page *page) +int page_to_nid(struct page *page) { return section_to_node_table[page_to_section(page)]; } @@ -486,9 +486,6 @@ void __init sparse_init(void) struct page **map_map; #endif - /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ - set_pageblock_order(); - /* * map is using big page (aka 2M in x86 64 bit) * usemap is less one page (aka 24 bytes) @@ -21,7 +21,7 @@ #include <linux/pagemap.h> #include <linux/pagevec.h> #include <linux/init.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/mm_inline.h> #include <linux/buffer_head.h> /* for try_to_release_page() */ #include <linux/percpu_counter.h> @@ -31,7 +31,6 @@ #include <linux/backing-dev.h> #include <linux/memcontrol.h> #include <linux/gfp.h> -#include <linux/hugetlb.h> #include "internal.h" @@ -70,8 +69,7 @@ static void __put_compound_page(struct page *page) { compound_page_dtor *dtor; - if (!PageHuge(page)) - __page_cache_release(page); + __page_cache_release(page); dtor = get_compound_page_dtor(page); (*dtor)(page); } @@ -85,35 +83,6 @@ static void put_compound_page(struct page *page) if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; - - if (PageHeadHuge(page_head)) { - if (likely(PageTail(page))) { - /* - * __split_huge_page_refcount - * cannot race here. - */ - VM_BUG_ON(!PageHead(page_head)); - atomic_dec(&page->_mapcount); - if (put_page_testzero(page_head)) - VM_BUG_ON(1); - if (put_page_testzero(page_head)) - __put_compound_page(page_head); - return; - } else { - /* - * __split_huge_page_refcount - * run before us, "page" was a - * THP tail. The split - * page_head has been freed - * and reallocated as slab or - * hugetlbfs page of smaller - * order (only possible if - * reallocated as slab on - * x86). - */ - goto skip_lock; - } - } /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time @@ -125,29 +94,9 @@ static void put_compound_page(struct page *page) /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); VM_BUG_ON(PageHead(page_head)); -skip_lock: - if (put_page_testzero(page_head)) { - /* - * The head page may have been - * freed and reallocated as a - * compound page of smaller - * order and then freed again. - * All we know is that it - * cannot have become: a THP - * page, a compound page of - * higher order, a tail page. - * That is because we still - * hold the refcount of the - * split THP tail and - * page_head was the THP head - * before the split. - */ - if (PageHead(page_head)) - __put_compound_page(page_head); - else - __put_single_page(page_head); - } -out_put_single: + if (put_page_testzero(page_head)) + __put_single_page(page_head); + out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; @@ -214,31 +163,6 @@ bool __get_page_tail(struct page *page) struct page *page_head = compound_trans_head(page); if (likely(page != page_head && get_page_unless_zero(page_head))) { - /* Ref to put_compound_page() comment. */ - if (PageHeadHuge(page_head)) { - if (likely(PageTail(page))) { - /* - * This is a hugetlbfs - * page. __split_huge_page_refcount - * cannot race here. - */ - VM_BUG_ON(!PageHead(page_head)); - __get_page_tail_foll(page, false); - return true; - } else { - /* - * __split_huge_page_refcount run - * before us, "page" was a THP - * tail. The split page_head has been - * freed and reallocated as slab or - * hugetlbfs page of smaller order - * (only possible if reallocated as - * slab on x86). - */ - put_page(page_head); - return false; - } - } /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time @@ -449,12 +373,27 @@ EXPORT_SYMBOL(mark_page_accessed); void __lru_cache_add(struct page *page, enum lru_list lru) { +#ifndef CONFIG_DMA_CMA struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru]; page_cache_get(page); if (!pagevec_add(pvec, page)) ____pagevec_lru_add(pvec, lru); put_cpu_var(lru_add_pvecs); +#else + struct pagevec *pvec; + int is_cma; + + /* FIXME: too slow */ + is_cma = is_cma_pageblock(page); + + pvec = &get_cpu_var(lru_add_pvecs)[lru]; + + page_cache_get(page); + if (!pagevec_add(pvec, page) || is_cma) + ____pagevec_lru_add(pvec, lru); + put_cpu_var(lru_add_pvecs); +#endif } EXPORT_SYMBOL(__lru_cache_add); diff --git a/mm/swap_state.c b/mm/swap_state.c index 7b3dadd..43b957e 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -6,6 +6,7 @@ * * Rewritten to use page cache, (C) 1998 Stephen Tweedie */ +#include <linux/module.h> #include <linux/mm.h> #include <linux/gfp.h> #include <linux/kernel_stat.h> diff --git a/mm/swapfile.c b/mm/swapfile.c index dbd2b67..3e5a3a7 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -21,6 +21,7 @@ #include <linux/proc_fs.h> #include <linux/seq_file.h> #include <linux/init.h> +#include <linux/module.h> #include <linux/ksm.h> #include <linux/rmap.h> #include <linux/security.h> @@ -1614,7 +1615,7 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) oom_score_adj = test_set_oom_score_adj(OOM_SCORE_ADJ_MAX); err = try_to_unuse(type); - compare_swap_oom_score_adj(OOM_SCORE_ADJ_MAX, oom_score_adj); + test_set_oom_score_adj(oom_score_adj); if (err) { /* @@ -1649,6 +1650,7 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) p->max = 0; swap_map = p->swap_map; p->swap_map = NULL; + p->flags = 0; spin_unlock(&swap_lock); mutex_unlock(&swapon_mutex); vfree(swap_map); @@ -1666,16 +1668,6 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) mutex_unlock(&inode->i_mutex); } filp_close(swap_file, NULL); - - /* - * Clear the SWP_USED flag after all resources are freed so that swapon - * can reuse this swap_info in alloc_swap_info() safely. It is ok to - * not hold p->lock after we cleared its SWP_WRITEOK. - */ - spin_lock(&swap_lock); - p->flags = 0; - spin_unlock(&swap_lock); - err = 0; atomic_inc(&proc_poll_event); wake_up_interruptible(&proc_poll_wait); @@ -1687,14 +1679,19 @@ out: } #ifdef CONFIG_PROC_FS +struct proc_swaps { + struct seq_file seq; + int event; +}; + static unsigned swaps_poll(struct file *file, poll_table *wait) { - struct seq_file *seq = file->private_data; + struct proc_swaps *s = file->private_data; poll_wait(file, &proc_poll_wait, wait); - if (seq->poll_event != atomic_read(&proc_poll_event)) { - seq->poll_event = atomic_read(&proc_poll_event); + if (s->event != atomic_read(&proc_poll_event)) { + s->event = atomic_read(&proc_poll_event); return POLLIN | POLLRDNORM | POLLERR | POLLPRI; } @@ -1784,16 +1781,24 @@ static const struct seq_operations swaps_op = { static int swaps_open(struct inode *inode, struct file *file) { - struct seq_file *seq; + struct proc_swaps *s; int ret; + s = kmalloc(sizeof(struct proc_swaps), GFP_KERNEL); + if (!s) + return -ENOMEM; + + file->private_data = s; + ret = seq_open(file, &swaps_op); - if (ret) + if (ret) { + kfree(s); return ret; + } - seq = file->private_data; - seq->poll_event = atomic_read(&proc_poll_event); - return 0; + s->seq.private = s; + s->event = atomic_read(&proc_poll_event); + return ret; } static const struct file_operations proc_swaps_operations = { @@ -1930,13 +1935,13 @@ static unsigned long read_swap_header(struct swap_info_struct *p, /* * Find out how many pages are allowed for a single swap - * device. There are two limiting factors: 1) the number - * of bits for the swap offset in the swp_entry_t type, and - * 2) the number of bits in the swap pte as defined by the - * different architectures. In order to find the - * largest possible bit mask, a swap entry with swap type 0 + * device. There are two limiting factors: 1) the number of + * bits for the swap offset in the swp_entry_t type and + * 2) the number of bits in the a swap pte as defined by + * the different architectures. In order to find the + * largest possible bit mask a swap entry with swap type 0 * and swap offset ~0UL is created, encoded to a swap pte, - * decoded to a swp_entry_t again, and finally the swap + * decoded to a swp_entry_t again and finally the swap * offset is extracted. This will mask all the bits from * the initial ~0UL mask that can't be encoded in either * the swp_entry_t or the architecture definition of a @@ -2007,6 +2012,159 @@ static int setup_swap_map_and_extents(struct swap_info_struct *p, return nr_extents; } +#ifdef CONFIG_ZRAM_FOR_ANDROID +int swapon(char *name, int swap_flags) +{ + struct swap_info_struct *p; + + struct file *swap_file = NULL; + struct address_space *mapping; + int i; + int prio; + int error; + union swap_header *swap_header; + int nr_extents; + sector_t span; + unsigned long maxpages; + unsigned char *swap_map = NULL; + struct page *page = NULL; + struct inode *inode = NULL; + + p = alloc_swap_info(); + if (IS_ERR(p)) + return PTR_ERR(p); + + swap_file = filp_open(name, O_RDWR | O_LARGEFILE, 0); + if (IS_ERR(swap_file)) { + error = PTR_ERR(swap_file); + swap_file = NULL; + printk("zfqin, filp_open failed\n"); + goto bad_swap; + } + + printk("zfqin, filp_open succeeded\n"); + p->swap_file = swap_file; + mapping = swap_file->f_mapping; + + for (i = 0; i < nr_swapfiles; i++) { + struct swap_info_struct *q = swap_info[i]; + + if (q == p || !q->swap_file) + continue; + if (mapping == q->swap_file->f_mapping) { + error = -EBUSY; + goto bad_swap; + } + } + + inode = mapping->host; + /* If S_ISREG(inode->i_mode) will do mutex_lock(&inode->i_mutex); */ + error = claim_swapfile(p, inode); + if (unlikely(error)) + goto bad_swap; + + /* + * Read the swap header. + */ + if (!mapping->a_ops->readpage) { + error = -EINVAL; + goto bad_swap; + } + page = read_mapping_page(mapping, 0, swap_file); + if (IS_ERR(page)) { + error = PTR_ERR(page); + goto bad_swap; + } + swap_header = kmap(page); + + maxpages = read_swap_header(p, swap_header, inode); + if (unlikely(!maxpages)) { + error = -EINVAL; + goto bad_swap; + } + + /* OK, set up the swap map and apply the bad block list */ + swap_map = vzalloc(maxpages); + if (!swap_map) { + error = -ENOMEM; + goto bad_swap; + } + + error = swap_cgroup_swapon(p->type, maxpages); + if (error) + goto bad_swap; + + nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map, + maxpages, &span); + if (unlikely(nr_extents < 0)) { + error = nr_extents; + goto bad_swap; + } + + if (p->bdev) { + if (blk_queue_nonrot(bdev_get_queue(p->bdev))) { + p->flags |= SWP_SOLIDSTATE; + p->cluster_next = 1 + (random32() % p->highest_bit); + } + if (discard_swap(p) == 0 && (swap_flags & SWAP_FLAG_DISCARD)) + p->flags |= SWP_DISCARDABLE; + } + + mutex_lock(&swapon_mutex); + prio = -1; + if (swap_flags & SWAP_FLAG_PREFER) + prio = + (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT; + enable_swap_info(p, prio, swap_map); + + printk(KERN_INFO "Adding %uk swap on %s. " + "Priority:%d extents:%d across:%lluk %s%s\n", + p->pages << (PAGE_SHIFT - 10), name, p->prio, + nr_extents, (unsigned long long)span << (PAGE_SHIFT - 10), + (p->flags & SWP_SOLIDSTATE) ? "SS" : "", + (p->flags & SWP_DISCARDABLE) ? "D" : ""); + + mutex_unlock(&swapon_mutex); + atomic_inc(&proc_poll_event); + wake_up_interruptible(&proc_poll_wait); + + if (S_ISREG(inode->i_mode)) + inode->i_flags |= S_SWAPFILE; + error = 0; + goto out; + bad_swap: + if (inode && S_ISBLK(inode->i_mode) && p->bdev) { + set_blocksize(p->bdev, p->old_block_size); + blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); + } + destroy_swap_extents(p); + swap_cgroup_swapoff(p->type); + spin_lock(&swap_lock); + p->swap_file = NULL; + p->flags = 0; + spin_unlock(&swap_lock); + vfree(swap_map); + if (swap_file) { + if (inode && S_ISREG(inode->i_mode)) { + mutex_unlock(&inode->i_mutex); + inode = NULL; + } + filp_close(swap_file, NULL); + } + out: + if (page && !IS_ERR(page)) { + kunmap(page); + page_cache_release(page); + } + + if (inode && S_ISREG(inode->i_mode)) + mutex_unlock(&inode->i_mutex); + return error; +} + +EXPORT_SYMBOL(swapon); +#endif /* CONFIG_ZRAM_FOR_ANDROID */ + SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) { struct swap_info_struct *p; @@ -2107,7 +2265,7 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) p->flags |= SWP_SOLIDSTATE; p->cluster_next = 1 + (random32() % p->highest_bit); } - if ((swap_flags & SWAP_FLAG_DISCARD) && discard_swap(p) == 0) + if (discard_swap(p) == 0 && (swap_flags & SWAP_FLAG_DISCARD)) p->flags |= SWP_DISCARDABLE; } diff --git a/mm/thrash.c b/mm/thrash.c index 57ad495..fabf2d0 100644 --- a/mm/thrash.c +++ b/mm/thrash.c @@ -6,7 +6,7 @@ * Released under the GPL, see the file COPYING for details. * * Simple token based thrashing protection, using the algorithm - * described in: http://www.cse.ohio-state.edu/hpcs/WWW/HTML/publications/abs05-1.html + * described in: http://www.cs.wm.edu/~sjiang/token.pdf * * Sep 2006, Ashwin Chaugule <ashwin.chaugule@celunite.com> * Improved algorithm to pass token: @@ -29,7 +29,9 @@ static DEFINE_SPINLOCK(swap_token_lock); struct mm_struct *swap_token_mm; -static struct mem_cgroup *swap_token_memcg; +struct mem_cgroup *swap_token_memcg; +static unsigned int global_faults; +static unsigned int last_aging; #ifdef CONFIG_CGROUP_MEM_RES_CTLR static struct mem_cgroup *swap_token_memcg_from_mm(struct mm_struct *mm) @@ -53,8 +55,6 @@ void grab_swap_token(struct mm_struct *mm) { int current_interval; unsigned int old_prio = mm->token_priority; - static unsigned int global_faults; - static unsigned int last_aging; global_faults++; @@ -67,17 +67,6 @@ void grab_swap_token(struct mm_struct *mm) if (!swap_token_mm) goto replace_token; - /* - * Usually, we don't need priority aging because long interval faults - * makes priority decrease quickly. But there is one exception. If the - * token owner task is sleeping, it never make long interval faults. - * Thus, we need a priority aging mechanism instead. The requirements - * of priority aging are - * 1) An aging interval is reasonable enough long. Too short aging - * interval makes quick swap token lost and decrease performance. - * 2) The swap token owner task have to get priority aging even if - * it's under sleep. - */ if ((global_faults - last_aging) > TOKEN_AGING_INTERVAL) { swap_token_mm->token_priority /= 2; last_aging = global_faults; diff --git a/mm/truncate.c b/mm/truncate.c index 143883a..6a11acb 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -12,7 +12,7 @@ #include <linux/gfp.h> #include <linux/mm.h> #include <linux/swap.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/pagemap.h> #include <linux/highmem.h> #include <linux/pagevec.h> @@ -20,7 +20,6 @@ #include <linux/buffer_head.h> /* grr. try_to_release_page, do_invalidatepage */ #include <linux/cleancache.h> -#include <linux/rmap.h> #include "internal.h" @@ -139,12 +138,18 @@ invalidate_complete_page(struct address_space *mapping, struct page *page) return ret; } +#ifdef CONFIG_MACH_P4NOTE +static int unmap_mapcount = -99; +#endif int truncate_inode_page(struct address_space *mapping, struct page *page) { if (page_mapped(page)) { unmap_mapping_range(mapping, (loff_t)page->index << PAGE_CACHE_SHIFT, PAGE_CACHE_SIZE, 0); +#ifdef CONFIG_MACH_P4NOTE + unmap_mapcount = atomic_read(&(page)->_mapcount); +#endif } return truncate_complete_page(mapping, page); } @@ -200,6 +205,9 @@ int invalidate_inode_page(struct page *page) * The first pass will remove most pages, so the search cost of the second pass * is low. * + * When looking at page->index outside the page lock we need to be careful to + * copy it into a local to avoid races (it could change at any time). + * * We pass down the cache-hot hint to the page freeing code. Even if the * mapping is large, it is probably the case that the final pages are the most * recently touched, and freeing happens in ascending file offset order. @@ -208,10 +216,10 @@ void truncate_inode_pages_range(struct address_space *mapping, loff_t lstart, loff_t lend) { const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT; + pgoff_t end; const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1); struct pagevec pvec; - pgoff_t index; - pgoff_t end; + pgoff_t next; int i; cleancache_flush_inode(mapping); @@ -222,21 +230,24 @@ void truncate_inode_pages_range(struct address_space *mapping, end = (lend >> PAGE_CACHE_SHIFT); pagevec_init(&pvec, 0); - index = start; - while (index <= end && pagevec_lookup(&pvec, mapping, index, - min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) { + next = start; + while (next <= end && + pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { mem_cgroup_uncharge_start(); for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; + pgoff_t page_index = page->index; - /* We rely upon deletion not changing page->index */ - index = page->index; - if (index > end) + if (page_index > end) { + next = page_index; break; + } + if (page_index > next) + next = page_index; + next++; if (!trylock_page(page)) continue; - WARN_ON(page->index != index); if (PageWriteback(page)) { unlock_page(page); continue; @@ -247,7 +258,6 @@ void truncate_inode_pages_range(struct address_space *mapping, pagevec_release(&pvec); mem_cgroup_uncharge_end(); cond_resched(); - index++; } if (partial) { @@ -260,17 +270,16 @@ void truncate_inode_pages_range(struct address_space *mapping, } } - index = start; + next = start; for ( ; ; ) { cond_resched(); - if (!pagevec_lookup(&pvec, mapping, index, - min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) { - if (index == start) + if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { + if (next == start) break; - index = start; + next = start; continue; } - if (index == start && pvec.pages[0]->index > end) { + if (pvec.pages[0]->index > end) { pagevec_release(&pvec); break; } @@ -278,20 +287,18 @@ void truncate_inode_pages_range(struct address_space *mapping, for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; - /* We rely upon deletion not changing page->index */ - index = page->index; - if (index > end) + if (page->index > end) break; - lock_page(page); - WARN_ON(page->index != index); wait_on_page_writeback(page); truncate_inode_page(mapping, page); + if (page->index > next) + next = page->index; + next++; unlock_page(page); } pagevec_release(&pvec); mem_cgroup_uncharge_end(); - index++; } cleancache_flush_inode(mapping); } @@ -332,34 +339,35 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping, pgoff_t start, pgoff_t end) { struct pagevec pvec; - pgoff_t index = start; + pgoff_t next = start; unsigned long ret; unsigned long count = 0; int i; - /* - * Note: this function may get called on a shmem/tmpfs mapping: - * pagevec_lookup() might then return 0 prematurely (because it - * got a gangful of swap entries); but it's hardly worth worrying - * about - it can rarely have anything to free from such a mapping - * (most pages are dirty), and already skips over any difficulties. - */ - pagevec_init(&pvec, 0); - while (index <= end && pagevec_lookup(&pvec, mapping, index, - min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) { + while (next <= end && + pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { mem_cgroup_uncharge_start(); for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; + pgoff_t index; + int lock_failed; - /* We rely upon deletion not changing page->index */ - index = page->index; - if (index > end) - break; + lock_failed = !trylock_page(page); - if (!trylock_page(page)) + /* + * We really shouldn't be looking at the ->index of an + * unlocked page. But we're not allowed to lock these + * pages. So we rely upon nobody altering the ->index + * of this (pinned-by-us) page. + */ + index = page->index; + if (index > next) + next = index; + next++; + if (lock_failed) continue; - WARN_ON(page->index != index); + ret = invalidate_inode_page(page); unlock_page(page); /* @@ -369,11 +377,12 @@ unsigned long invalidate_mapping_pages(struct address_space *mapping, if (!ret) deactivate_page(page); count += ret; + if (next > end) + break; } pagevec_release(&pvec); mem_cgroup_uncharge_end(); cond_resched(); - index++; } return count; } @@ -440,32 +449,37 @@ int invalidate_inode_pages2_range(struct address_space *mapping, pgoff_t start, pgoff_t end) { struct pagevec pvec; - pgoff_t index; + pgoff_t next; int i; int ret = 0; int ret2 = 0; int did_range_unmap = 0; + int wrapped = 0; cleancache_flush_inode(mapping); pagevec_init(&pvec, 0); - index = start; - while (index <= end && pagevec_lookup(&pvec, mapping, index, - min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) { + next = start; + while (next <= end && !wrapped && + pagevec_lookup(&pvec, mapping, next, + min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) { mem_cgroup_uncharge_start(); for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; - - /* We rely upon deletion not changing page->index */ - index = page->index; - if (index > end) - break; + pgoff_t page_index; lock_page(page); - WARN_ON(page->index != index); if (page->mapping != mapping) { unlock_page(page); continue; } + page_index = page->index; + next = page_index + 1; + if (next == 0) + wrapped = 1; + if (page_index > end) { + unlock_page(page); + break; + } wait_on_page_writeback(page); if (page_mapped(page)) { if (!did_range_unmap) { @@ -473,9 +487,9 @@ int invalidate_inode_pages2_range(struct address_space *mapping, * Zap the rest of the file in one hit. */ unmap_mapping_range(mapping, - (loff_t)index << PAGE_CACHE_SHIFT, - (loff_t)(1 + end - index) - << PAGE_CACHE_SHIFT, + (loff_t)page_index<<PAGE_CACHE_SHIFT, + (loff_t)(end - page_index + 1) + << PAGE_CACHE_SHIFT, 0); did_range_unmap = 1; } else { @@ -483,8 +497,8 @@ int invalidate_inode_pages2_range(struct address_space *mapping, * Just zap this page */ unmap_mapping_range(mapping, - (loff_t)index << PAGE_CACHE_SHIFT, - PAGE_CACHE_SIZE, 0); + (loff_t)page_index<<PAGE_CACHE_SHIFT, + PAGE_CACHE_SIZE, 0); } } BUG_ON(page_mapped(page)); @@ -500,7 +514,6 @@ int invalidate_inode_pages2_range(struct address_space *mapping, pagevec_release(&pvec); mem_cgroup_uncharge_end(); cond_resched(); - index++; } cleancache_flush_inode(mapping); return ret; @@ -525,8 +538,8 @@ EXPORT_SYMBOL_GPL(invalidate_inode_pages2); /** * truncate_pagecache - unmap and remove pagecache that has been truncated * @inode: inode - * @oldsize: old file size - * @newsize: new file size + * @old: old file offset + * @new: new file offset * * inode's new i_size must already be written before truncate_pagecache * is called. @@ -538,10 +551,9 @@ EXPORT_SYMBOL_GPL(invalidate_inode_pages2); * situations such as writepage being called for a page that has already * had its underlying blocks deallocated. */ -void truncate_pagecache(struct inode *inode, loff_t oldsize, loff_t newsize) +void truncate_pagecache(struct inode *inode, loff_t old, loff_t new) { struct address_space *mapping = inode->i_mapping; - loff_t holebegin = round_up(newsize, PAGE_SIZE); /* * unmap_mapping_range is called twice, first simply for @@ -552,9 +564,9 @@ void truncate_pagecache(struct inode *inode, loff_t oldsize, loff_t newsize) * truncate_inode_pages finishes, hence the second * unmap_mapping_range call must be made for correctness. */ - unmap_mapping_range(mapping, holebegin, 0, 1); - truncate_inode_pages(mapping, newsize); - unmap_mapping_range(mapping, holebegin, 0, 1); + unmap_mapping_range(mapping, new + PAGE_SIZE - 1, 0, 1); + truncate_inode_pages(mapping, new); + unmap_mapping_range(mapping, new + PAGE_SIZE - 1, 0, 1); } EXPORT_SYMBOL(truncate_pagecache); @@ -576,82 +588,54 @@ void truncate_setsize(struct inode *inode, loff_t newsize) oldsize = inode->i_size; i_size_write(inode, newsize); - if (newsize > oldsize) - pagecache_isize_extended(inode, oldsize, newsize); + truncate_pagecache(inode, oldsize, newsize); } EXPORT_SYMBOL(truncate_setsize); /** - * pagecache_isize_extended - update pagecache after extension of i_size - * @inode: inode for which i_size was extended - * @from: original inode size - * @to: new inode size - * - * Handle extension of inode size either caused by extending truncate or by - * write starting after current i_size. We mark the page straddling current - * i_size RO so that page_mkwrite() is called on the nearest write access to - * the page. This way filesystem can be sure that page_mkwrite() is called on - * the page before user writes to the page via mmap after the i_size has been - * changed. - * - * The function must be called after i_size is updated so that page fault - * coming after we unlock the page will already see the new i_size. - * The function must be called while we still hold i_mutex - this not only - * makes sure i_size is stable but also that userspace cannot observe new - * i_size value before we are prepared to store mmap writes at new inode size. - */ -void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to) -{ - int bsize = 1 << inode->i_blkbits; - loff_t rounded_from; - struct page *page; - pgoff_t index; - - WARN_ON(to > inode->i_size); - - if (from >= to || bsize == PAGE_CACHE_SIZE) - return; - /* Page straddling @from will not have any hole block created? */ - rounded_from = round_up(from, bsize); - if (to <= rounded_from || !(rounded_from & (PAGE_CACHE_SIZE - 1))) - return; - - index = from >> PAGE_CACHE_SHIFT; - page = find_lock_page(inode->i_mapping, index); - /* Page not cached? Nothing to do */ - if (!page) - return; - /* - * See clear_page_dirty_for_io() for details why set_page_dirty() - * is needed. - */ - if (page_mkclean(page)) - set_page_dirty(page); - unlock_page(page); - page_cache_release(page); -} -EXPORT_SYMBOL(pagecache_isize_extended); - -/** * vmtruncate - unmap mappings "freed" by truncate() syscall * @inode: inode of the file used - * @newsize: file offset to start truncating + * @offset: file offset to start truncating * * This function is deprecated and truncate_setsize or truncate_pagecache * should be used instead, together with filesystem specific block truncation. */ -int vmtruncate(struct inode *inode, loff_t newsize) +int vmtruncate(struct inode *inode, loff_t offset) { int error; - error = inode_newsize_ok(inode, newsize); + error = inode_newsize_ok(inode, offset); if (error) return error; - truncate_setsize(inode, newsize); + truncate_setsize(inode, offset); if (inode->i_op->truncate) inode->i_op->truncate(inode); return 0; } EXPORT_SYMBOL(vmtruncate); + +int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end) +{ + struct address_space *mapping = inode->i_mapping; + + /* + * If the underlying filesystem is not going to provide + * a way to truncate a range of blocks (punch a hole) - + * we should return failure right now. + */ + if (!inode->i_op->truncate_range) + return -ENOSYS; + + mutex_lock(&inode->i_mutex); + down_write(&inode->i_alloc_sem); + unmap_mapping_range(mapping, offset, (end - offset), 1); + inode->i_op->truncate_range(inode, offset, end); + /* unmap again to remove racily COWed private pages */ + unmap_mapping_range(mapping, offset, (end - offset), 1); + up_write(&inode->i_alloc_sem); + mutex_unlock(&inode->i_mutex); + + return 0; +} @@ -1,7 +1,7 @@ #include <linux/mm.h> #include <linux/slab.h> #include <linux/string.h> -#include <linux/export.h> +#include <linux/module.h> #include <linux/err.h> #include <linux/sched.h> #include <asm/uaccess.h> diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 1431458..e9c4611 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -26,7 +26,7 @@ #include <linux/rcupdate.h> #include <linux/pfn.h> #include <linux/kmemleak.h> -#include <linux/atomic.h> +#include <asm/atomic.h> #include <asm/uaccess.h> #include <asm/tlbflush.h> #include <asm/shmparam.h> @@ -349,12 +349,6 @@ static struct vmap_area *alloc_vmap_area(unsigned long size, if (unlikely(!va)) return ERR_PTR(-ENOMEM); - /* - * Only scan the relevant parts containing pointers to other objects - * to avoid false negatives. - */ - kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK); - retry: spin_lock(&vmap_area_lock); /* @@ -458,6 +452,13 @@ overflow: return ERR_PTR(-EBUSY); } +static void rcu_free_va(struct rcu_head *head) +{ + struct vmap_area *va = container_of(head, struct vmap_area, rcu_head); + + kfree(va); +} + static void __free_vmap_area(struct vmap_area *va) { BUG_ON(RB_EMPTY_NODE(&va->rb_node)); @@ -490,7 +491,7 @@ static void __free_vmap_area(struct vmap_area *va) if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END) vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end); - kfree_rcu(va, rcu_head); + call_rcu(&va->rcu_head, rcu_free_va); } /* @@ -837,6 +838,13 @@ static struct vmap_block *new_vmap_block(gfp_t gfp_mask) return vb; } +static void rcu_free_vb(struct rcu_head *head) +{ + struct vmap_block *vb = container_of(head, struct vmap_block, rcu_head); + + kfree(vb); +} + static void free_vmap_block(struct vmap_block *vb) { struct vmap_block *tmp; @@ -849,7 +857,7 @@ static void free_vmap_block(struct vmap_block *vb) BUG_ON(tmp != vb); free_vmap_area_noflush(vb->va); - kfree_rcu(vb, rcu_head); + call_rcu(&vb->rcu_head, rcu_free_vb); } static void purge_fragmented_blocks(int cpu) @@ -1267,6 +1275,10 @@ static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, vm->addr = (void *)va->va_start; vm->size = va->va_end - va->va_start; vm->caller = caller; +#ifdef CONFIG_DEBUG_VMALLOC + vm->pid = current->pid; + vm->task_name = current->comm; +#endif va->vm = vm; va->flags |= VM_VM_AREA; } @@ -1297,7 +1309,7 @@ static struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long align, unsigned long flags, unsigned long start, unsigned long end, int node, gfp_t gfp_mask, void *caller) { - struct vmap_area *va; + static struct vmap_area *va; struct vm_struct *area; BUG_ON(in_interrupt()); @@ -1572,6 +1584,10 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, } area->pages = pages; area->caller = caller; +#ifdef CONFIG_DEBUG_VMALLOC + area->pid = current->pid; + area->task_name = current->comm; +#endif if (!area->pages) { remove_vm_area(area->addr); kfree(area); @@ -1600,8 +1616,8 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, return area->addr; fail: - warn_alloc_failed(gfp_mask, order, - "vmalloc: allocation failure, allocated %ld of %ld bytes\n", + warn_alloc_failed(gfp_mask, order, "vmalloc: allocation failure, " + "allocated %ld of %ld bytes\n", (area->nr_pages*PAGE_SIZE), area->size); vfree(area->addr); return NULL; @@ -1632,12 +1648,13 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, size = PAGE_ALIGN(size); if (!size || (size >> PAGE_SHIFT) > totalram_pages) - goto fail; + return NULL; area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNLIST, start, end, node, gfp_mask, caller); + if (!area) - goto fail; + return NULL; addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller); if (!addr) @@ -1650,19 +1667,13 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align, insert_vmalloc_vmlist(area); /* - * A ref_count = 2 is needed because vm_struct allocated in - * __get_vm_area_node() contains a reference to the virtual address of - * the vmalloc'ed block. + * A ref_count = 3 is needed because the vm_struct and vmap_area + * structures allocated in the __get_vm_area_node() function contain + * references to the virtual address of the vmalloc'ed block. */ - kmemleak_alloc(addr, real_size, 2, gfp_mask); + kmemleak_alloc(addr, real_size, 3, gfp_mask); return addr; - -fail: - warn_alloc_failed(gfp_mask, 0, - "vmalloc: allocation failure: %lu bytes\n", - real_size); - return NULL; } /** @@ -2150,30 +2161,23 @@ void __attribute__((weak)) vmalloc_sync_all(void) static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data) { - pte_t ***p = data; - - if (p) { - *(*p) = pte; - (*p)++; - } + /* apply_to_page_range() does all the hard work. */ return 0; } /** * alloc_vm_area - allocate a range of kernel address space * @size: size of the area - * @ptes: returns the PTEs for the address space * * Returns: NULL on failure, vm_struct on success * * This function reserves a range of kernel address space, and * allocates pagetables to map that range. No actual mappings - * are created. - * - * If @ptes is non-NULL, pointers to the PTEs (in init_mm) - * allocated for the VM area are returned. + * are created. If the kernel address space is not shared + * between processes, it syncs the pagetable across all + * processes. */ -struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) +struct vm_struct *alloc_vm_area(size_t size) { struct vm_struct *area; @@ -2187,11 +2191,19 @@ struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) * of kernel virtual address space and mapped into init_mm. */ if (apply_to_page_range(&init_mm, (unsigned long)area->addr, - size, f, ptes ? &ptes : NULL)) { + area->size, f, NULL)) { free_vm_area(area); return NULL; } + /* + * If the allocated address space is passed to a hypercall + * before being used then we cannot rely on a page fault to + * trigger an update of the page tables. So sync all the page + * tables here. + */ + vmalloc_sync_all(); + return area; } EXPORT_SYMBOL_GPL(alloc_vm_area); @@ -2569,6 +2581,14 @@ static int s_show(struct seq_file *m, void *p) if (v->flags & VM_VPAGES) seq_printf(m, " vpages"); +#ifdef CONFIG_DEBUG_VMALLOC + if (v->pid) + seq_printf(m, " pid=%d", v->pid); + + if (v->task_name) + seq_printf(m, " task name=%s", v->task_name); +#endif + show_numa_info(m, v); seq_putc(m, '\n'); return 0; diff --git a/mm/vmscan.c b/mm/vmscan.c index ab98dc6..99082fa 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -95,6 +95,8 @@ struct scan_control { /* Can pages be swapped as part of reclaim? */ int may_swap; + int swappiness; + int order; /* @@ -171,8 +173,7 @@ static unsigned long zone_nr_lru_pages(struct zone *zone, struct scan_control *sc, enum lru_list lru) { if (!scanning_global_lru(sc)) - return mem_cgroup_zone_nr_lru_pages(sc->mem_cgroup, - zone_to_nid(zone), zone_idx(zone), BIT(lru)); + return mem_cgroup_zone_nr_lru_pages(sc->mem_cgroup, zone, lru); return zone_page_state(zone, NR_LRU_BASE + lru); } @@ -183,7 +184,7 @@ static unsigned long zone_nr_lru_pages(struct zone *zone, */ void register_shrinker(struct shrinker *shrinker) { - atomic_long_set(&shrinker->nr_in_batch, 0); + shrinker->nr = 0; down_write(&shrinker_rwsem); list_add_tail(&shrinker->list, &shrinker_list); up_write(&shrinker_rwsem); @@ -252,8 +253,6 @@ unsigned long shrink_slab(struct shrink_control *shrink, int shrink_ret = 0; long nr; long new_nr; - long batch_size = shrinker->batch ? shrinker->batch - : SHRINK_BATCH; max_pass = do_shrinker_shrink(shrinker, shrink, 0); if (max_pass <= 0) @@ -264,7 +263,9 @@ unsigned long shrink_slab(struct shrink_control *shrink, * and zero it so that other concurrent shrinker invocations * don't also do this scanning work. */ - nr = atomic_long_xchg(&shrinker->nr_in_batch, 0); + do { + nr = shrinker->nr; + } while (cmpxchg(&shrinker->nr, nr, 0) != nr); total_scan = nr; delta = (4 * nr_pages_scanned) / shrinker->seeks; @@ -305,18 +306,19 @@ unsigned long shrink_slab(struct shrink_control *shrink, nr_pages_scanned, lru_pages, max_pass, delta, total_scan); - while (total_scan >= batch_size) { + while (total_scan >= SHRINK_BATCH) { + long this_scan = SHRINK_BATCH; int nr_before; nr_before = do_shrinker_shrink(shrinker, shrink, 0); shrink_ret = do_shrinker_shrink(shrinker, shrink, - batch_size); + this_scan); if (shrink_ret == -1) break; if (shrink_ret < nr_before) ret += nr_before - shrink_ret; - count_vm_events(SLABS_SCANNED, batch_size); - total_scan -= batch_size; + count_vm_events(SLABS_SCANNED, this_scan); + total_scan -= this_scan; cond_resched(); } @@ -326,11 +328,12 @@ unsigned long shrink_slab(struct shrink_control *shrink, * manner that handles concurrent updates. If we exhausted the * scan, there is no need to do an update. */ - if (total_scan > 0) - new_nr = atomic_long_add_return(total_scan, - &shrinker->nr_in_batch); - else - new_nr = atomic_long_read(&shrinker->nr_in_batch); + do { + nr = shrinker->nr; + new_nr = total_scan + nr; + if (total_scan <= 0) + break; + } while (cmpxchg(&shrinker->nr, nr, new_nr) != nr); trace_mm_shrink_slab_end(shrinker, shrink_ret, nr, new_nr); } @@ -495,6 +498,15 @@ static pageout_t pageout(struct page *page, struct address_space *mapping, return PAGE_ACTIVATE; } + /* + * Wait on writeback if requested to. This happens when + * direct reclaiming a large contiguous area and the + * first attempt to free a range of pages fails. + */ + if (PageWriteback(page) && + (sc->reclaim_mode & RECLAIM_MODE_SYNC)) + wait_on_page_writeback(page); + if (!PageWriteback(page)) { /* synchronous write or broken a_ops? */ ClearPageReclaim(page); @@ -633,14 +645,13 @@ redo: lru = LRU_UNEVICTABLE; add_page_to_unevictable_list(page); /* - * When racing with an mlock or AS_UNEVICTABLE clearing - * (page is unlocked) make sure that if the other thread - * does not observe our setting of PG_lru and fails - * isolation/check_move_unevictable_pages, - * we see PG_mlocked/AS_UNEVICTABLE cleared below and move + * When racing with an mlock clearing (page is + * unlocked), make sure that if the other thread does + * not observe our setting of PG_lru and fails + * isolation, we see PG_mlocked cleared below and move * the page back to the evictable list. * - * The other side is TestClearPageMlocked() or shmem_lock(). + * The other side is TestClearPageMlocked(). */ smp_mb(); } @@ -715,15 +726,19 @@ static enum page_references page_check_references(struct page *page, */ SetPageReferenced(page); +#ifndef CONFIG_DMA_CMA + if (referenced_page) + return PAGEREF_ACTIVATE; +#else if (referenced_page || referenced_ptes > 1) return PAGEREF_ACTIVATE; /* * Activate file-backed executable pages after first usage. - */ + */ if (vm_flags & VM_EXEC) return PAGEREF_ACTIVATE; - +#endif return PAGEREF_KEEP; } @@ -755,12 +770,12 @@ static noinline_for_stack void free_page_list(struct list_head *free_pages) /* * shrink_page_list() returns the number of reclaimed pages */ -static unsigned long shrink_page_list(struct list_head *page_list, +#ifndef CONFIG_ZRAM_FOR_ANDROID +static +#endif /* CONFIG_ZRAM_FOR_ANDROID */ +unsigned long shrink_page_list(struct list_head *page_list, struct zone *zone, - struct scan_control *sc, - int priority, - unsigned long *ret_nr_dirty, - unsigned long *ret_nr_writeback) + struct scan_control *sc) { LIST_HEAD(ret_pages); LIST_HEAD(free_pages); @@ -768,7 +783,6 @@ static unsigned long shrink_page_list(struct list_head *page_list, unsigned long nr_dirty = 0; unsigned long nr_congested = 0; unsigned long nr_reclaimed = 0; - unsigned long nr_writeback = 0; cond_resched(); @@ -805,12 +819,13 @@ static unsigned long shrink_page_list(struct list_head *page_list, (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO)); if (PageWriteback(page)) { - nr_writeback++; /* - * Synchronous reclaim cannot queue pages for - * writeback due to the possibility of stack overflow - * but if it encounters a page under writeback, wait - * for the IO to complete. + * Synchronous reclaim is performed in two passes, + * first an asynchronous pass over the list to + * start parallel writeback, and a second synchronous + * pass to wait for the IO to complete. Wait here + * for any page for which writeback has already + * started. */ if ((sc->reclaim_mode & RECLAIM_MODE_SYNC) && may_enter_fs) @@ -866,25 +881,6 @@ static unsigned long shrink_page_list(struct list_head *page_list, if (PageDirty(page)) { nr_dirty++; - /* - * Only kswapd can writeback filesystem pages to - * avoid risk of stack overflow but do not writeback - * unless under significant pressure. - */ - if (page_is_file_cache(page) && - (!current_is_kswapd() || priority >= DEF_PRIORITY - 2)) { - /* - * Immediately reclaim when written back. - * Similar in principal to deactivate_page() - * except we already have the page isolated - * and know it's dirty - */ - inc_zone_page_state(page, NR_VMSCAN_IMMEDIATE); - SetPageReclaim(page); - - goto keep_locked; - } - if (references == PAGEREF_RECLAIM_CLEAN) goto keep_locked; if (!may_enter_fs) @@ -1019,8 +1015,6 @@ keep_lumpy: list_splice(&ret_pages, page_list); count_vm_events(PGACTIVATE, pgactivate); - *ret_nr_dirty += nr_dirty; - *ret_nr_writeback += nr_writeback; return nr_reclaimed; } @@ -1063,8 +1057,12 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode, int file) * unevictable; only give shrink_page_list evictable pages. */ if (PageUnevictable(page)) +#ifndef CONFIG_DMA_CMA return ret; - +#else + printk(KERN_ERR "%s[%d] Unevictable page %p\n", + __func__, __LINE__, page); +#endif ret = -EBUSY; /* @@ -1281,7 +1279,10 @@ static unsigned long isolate_pages_global(unsigned long nr, * clear_active_flags() is a helper for shrink_active_list(), clearing * any active bits from the pages in the list. */ -static unsigned long clear_active_flags(struct list_head *page_list, +#ifndef CONFIG_ZRAM_FOR_ANDROID +static +#endif /* CONFIG_ZRAM_FOR_ANDROID */ +unsigned long clear_active_flags(struct list_head *page_list, unsigned int *count) { int nr_active = 0; @@ -1351,6 +1352,40 @@ int isolate_lru_page(struct page *page) return ret; } +#ifdef CONFIG_ZRAM_FOR_ANDROID +/** + * isolate_lru_page_compcache - tries to isolate a page for compcache + * @page: page to isolate from its LRU list + * + * Isolates a @page from an LRU list, clears PageLRU,but + * does not adjusts the vmstat statistic + * Returns 0 if the page was removed from an LRU list. + * Returns -EBUSY if the page was not on an LRU list. + */ +int isolate_lru_page_compcache(struct page *page) +{ + int ret = -EBUSY; + + VM_BUG_ON(!page_count(page)); + + if (PageLRU(page)) { + struct zone *zone = page_zone(page); + + spin_lock_irq(&zone->lru_lock); + if (PageLRU(page)) { + int lru = page_lru(page); + ret = 0; + get_page(page); + ClearPageLRU(page); + list_del(&page->lru); + mem_cgroup_del_lru_list(page, lru); + } + spin_unlock_irq(&zone->lru_lock); + } + return ret; +} +#endif + /* * Are there way too many processes in the direct reclaim path already? */ @@ -1458,7 +1493,7 @@ static noinline_for_stack void update_isolated_counts(struct zone *zone, } /* - * Returns true if a direct reclaim should wait on pages under writeback. + * Returns true if the caller should wait to clean dirty/writeback pages. * * If we are direct reclaiming for contiguous pages and we do not reclaim * everything in the list, try again and wait for writeback IO to complete. @@ -1480,7 +1515,7 @@ static inline bool should_reclaim_stall(unsigned long nr_taken, if (sc->reclaim_mode & RECLAIM_MODE_SINGLE) return false; - /* If we have reclaimed everything on the isolated list, no stall */ + /* If we have relaimed everything on the isolated list, no stall */ if (nr_freed == nr_taken) return false; @@ -1512,8 +1547,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone, unsigned long nr_taken; unsigned long nr_anon; unsigned long nr_file; - unsigned long nr_dirty = 0; - unsigned long nr_writeback = 0; isolate_mode_t reclaim_mode = ISOLATE_INACTIVE; while (unlikely(too_many_isolated(zone, file, sc))) { @@ -1566,14 +1599,12 @@ shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone, spin_unlock_irq(&zone->lru_lock); - nr_reclaimed = shrink_page_list(&page_list, zone, sc, priority, - &nr_dirty, &nr_writeback); + nr_reclaimed = shrink_page_list(&page_list, zone, sc); /* Check if we should syncronously wait for writeback */ if (should_reclaim_stall(nr_taken, nr_reclaimed, priority, sc)) { set_reclaim_mode(priority, sc, true); - nr_reclaimed += shrink_page_list(&page_list, zone, sc, - priority, &nr_dirty, &nr_writeback); + nr_reclaimed += shrink_page_list(&page_list, zone, sc); } local_irq_disable(); @@ -1583,32 +1614,6 @@ shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone, putback_lru_pages(zone, sc, nr_anon, nr_file, &page_list); - /* - * If reclaim is isolating dirty pages under writeback, it implies - * that the long-lived page allocation rate is exceeding the page - * laundering rate. Either the global limits are not being effective - * at throttling processes due to the page distribution throughout - * zones or there is heavy usage of a slow backing device. The - * only option is to throttle from reclaim context which is not ideal - * as there is no guarantee the dirtying process is throttled in the - * same way balance_dirty_pages() manages. - * - * This scales the number of dirty pages that must be under writeback - * before throttling depending on priority. It is a simple backoff - * function that has the most effect in the range DEF_PRIORITY to - * DEF_PRIORITY-2 which is the priority reclaim is considered to be - * in trouble and reclaim is considered to be in trouble. - * - * DEF_PRIORITY 100% isolated pages must be PageWriteback to throttle - * DEF_PRIORITY-1 50% must be PageWriteback - * DEF_PRIORITY-2 25% must be PageWriteback, kswapd in trouble - * ... - * DEF_PRIORITY-6 For SWAP_CLUSTER_MAX isolated pages, throttle if any - * isolated page is PageWriteback - */ - if (nr_writeback && nr_writeback >= (nr_taken >> (DEF_PRIORITY-priority))) - wait_iff_congested(zone, BLK_RW_ASYNC, HZ/10); - trace_mm_vmscan_lru_shrink_inactive(zone->zone_pgdat->node_id, zone_idx(zone), nr_scanned, nr_reclaimed, @@ -1617,6 +1622,44 @@ shrink_inactive_list(unsigned long nr_to_scan, struct zone *zone, return nr_reclaimed; } +#ifdef CONFIG_ZRAM_FOR_ANDROID +unsigned long +zone_id_shrink_pagelist(struct zone *zone, struct list_head *page_list) +{ + unsigned long nr_reclaimed = 0; + unsigned long nr_anon; + unsigned long nr_file; + + struct scan_control sc = { + .gfp_mask = GFP_USER, + .may_writepage = 1, + .nr_to_reclaim = SWAP_CLUSTER_MAX, + .may_unmap = 1, + .may_swap = 1, + .swappiness = vm_swappiness, + .order = 0, + .mem_cgroup = NULL, + .nodemask = NULL, + }; + + spin_lock_irq(&zone->lru_lock); + + update_isolated_counts(zone, &sc, &nr_anon, &nr_file, page_list); + + spin_unlock_irq(&zone->lru_lock); + + nr_reclaimed = shrink_page_list(page_list, zone, &sc); + + __count_zone_vm_events(PGSTEAL, zone, nr_reclaimed); + + putback_lru_pages(zone, &sc, nr_anon, nr_file, page_list); + + return nr_reclaimed; +} + +EXPORT_SYMBOL(zone_id_shrink_pagelist); +#endif /* CONFIG_ZRAM_FOR_ANDROID */ + /* * This moves pages from the active list to the inactive list. * @@ -1804,7 +1847,7 @@ static int inactive_anon_is_low(struct zone *zone, struct scan_control *sc) if (scanning_global_lru(sc)) low = inactive_anon_is_low_global(zone); else - low = mem_cgroup_inactive_anon_is_low(sc->mem_cgroup, zone); + low = mem_cgroup_inactive_anon_is_low(sc->mem_cgroup); return low; } #else @@ -1847,7 +1890,7 @@ static int inactive_file_is_low(struct zone *zone, struct scan_control *sc) if (scanning_global_lru(sc)) low = inactive_file_is_low_global(zone); else - low = mem_cgroup_inactive_file_is_low(sc->mem_cgroup, zone); + low = mem_cgroup_inactive_file_is_low(sc->mem_cgroup); return low; } @@ -1874,13 +1917,6 @@ static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan, return shrink_inactive_list(nr_to_scan, zone, sc, priority, file); } -static int vmscan_swappiness(struct scan_control *sc) -{ - if (scanning_global_lru(sc)) - return vm_swappiness; - return mem_cgroup_swappiness(sc->mem_cgroup); -} - /* * Determine how aggressively the anon and file LRU lists should be * scanned. The relative value of each set of LRU lists is determined @@ -1900,20 +1936,13 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc, enum lru_list l; int noswap = 0; bool force_scan = false; + unsigned long nr_force_scan[2]; - /* - * If the zone or memcg is small, nr[l] can be 0. This - * results in no scanning on this priority and a potential - * priority drop. Global direct reclaim can go to the next - * zone and tends to have no problems. Global kswapd is for - * zone balancing and it needs to scan a minimum amount. When - * reclaiming for a memcg, a priority drop can cause high - * latencies, so it's better to scan a minimum amount there as - * well. - */ + /* kswapd does zone balancing and needs to scan this zone */ if (scanning_global_lru(sc) && current_is_kswapd() && zone->all_unreclaimable) force_scan = true; + /* memcg may have small limit and need to avoid priority drop */ if (!scanning_global_lru(sc)) force_scan = true; @@ -1923,6 +1952,8 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc, fraction[0] = 0; fraction[1] = 1; denominator = 1; + nr_force_scan[0] = 0; + nr_force_scan[1] = SWAP_CLUSTER_MAX; goto out; } @@ -1939,6 +1970,8 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc, fraction[0] = 1; fraction[1] = 0; denominator = 1; + nr_force_scan[0] = SWAP_CLUSTER_MAX; + nr_force_scan[1] = 0; goto out; } } @@ -1947,8 +1980,8 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc, * With swappiness at 100, anonymous and file have the same priority. * This scanning priority is essentially the inverse of IO cost. */ - anon_prio = vmscan_swappiness(sc); - file_prio = 200 - vmscan_swappiness(sc); + anon_prio = sc->swappiness; + file_prio = 200 - sc->swappiness; /* * OK, so we have swap space and a fair amount of page cache @@ -1977,28 +2010,43 @@ static void get_scan_count(struct zone *zone, struct scan_control *sc, * proportional to the fraction of recently scanned pages on * each list that were recently referenced and in active use. */ - ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1); + ap = (anon_prio + 1) * (reclaim_stat->recent_scanned[0] + 1); ap /= reclaim_stat->recent_rotated[0] + 1; - fp = file_prio * (reclaim_stat->recent_scanned[1] + 1); + fp = (file_prio + 1) * (reclaim_stat->recent_scanned[1] + 1); fp /= reclaim_stat->recent_rotated[1] + 1; spin_unlock_irq(&zone->lru_lock); fraction[0] = ap; fraction[1] = fp; denominator = ap + fp + 1; + if (force_scan) { + unsigned long scan = SWAP_CLUSTER_MAX; + nr_force_scan[0] = div64_u64(scan * ap, denominator); + nr_force_scan[1] = div64_u64(scan * fp, denominator); + } out: for_each_evictable_lru(l) { int file = is_file_lru(l); unsigned long scan; scan = zone_nr_lru_pages(zone, sc, l); - if (priority || noswap || !vmscan_swappiness(sc)) { + if (priority || noswap) { scan >>= priority; - if (!scan && force_scan) - scan = SWAP_CLUSTER_MAX; scan = div64_u64(scan * fraction[file], denominator); } + + /* + * If zone is small or memcg is small, nr[l] can be 0. + * This results no-scan on this priority and priority drop down. + * For global direct reclaim, it can visit next zone and tend + * not to have problems. For global kswapd, it's for zone + * balancing and it need to scan a small amounts. When using + * memcg, priority drop can cause big latency. So, it's better + * to scan small amount. See may_noscan above. + */ + if (!scan && force_scan) + scan = nr_force_scan[file]; nr[l] = scan; } } @@ -2339,8 +2387,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, */ writeback_threshold = sc->nr_to_reclaim + sc->nr_to_reclaim / 2; if (total_scanned > writeback_threshold) { - wakeup_flusher_threads(laptop_mode ? 0 : total_scanned, - WB_REASON_TRY_TO_FREE_PAGES); + wakeup_flusher_threads(laptop_mode ? 0 : total_scanned); sc->may_writepage = 1; } @@ -2391,6 +2438,7 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order, .nr_to_reclaim = SWAP_CLUSTER_MAX, .may_unmap = 1, .may_swap = 1, + .swappiness = vm_swappiness, .order = order, .mem_cgroup = NULL, .nodemask = nodemask, @@ -2414,6 +2462,7 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order, unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *mem, gfp_t gfp_mask, bool noswap, + unsigned int swappiness, struct zone *zone, unsigned long *nr_scanned) { @@ -2423,6 +2472,7 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *mem, .may_writepage = !laptop_mode, .may_unmap = 1, .may_swap = !noswap, + .swappiness = swappiness, .order = 0, .mem_cgroup = mem, }; @@ -2451,7 +2501,8 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *mem, unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont, gfp_t gfp_mask, - bool noswap) + bool noswap, + unsigned int swappiness) { struct zonelist *zonelist; unsigned long nr_reclaimed; @@ -2461,6 +2512,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont, .may_unmap = 1, .may_swap = !noswap, .nr_to_reclaim = SWAP_CLUSTER_MAX, + .swappiness = swappiness, .order = 0, .mem_cgroup = mem_cont, .nodemask = NULL, /* we don't care the placement */ @@ -2611,6 +2663,7 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, * we want to put equal scanning pressure on each zone. */ .nr_to_reclaim = ULONG_MAX, + .swappiness = vm_swappiness, .order = order, .mem_cgroup = NULL, }; @@ -2852,8 +2905,6 @@ out: /* If balanced, clear the congested flag */ zone_clear_flag(zone, ZONE_CONGESTED); - if (i <= *classzone_idx) - balanced += zone->present_pages; } } @@ -3016,10 +3067,7 @@ static int kswapd(void *p) } } - tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD); current->reclaim_state = NULL; - lockdep_clear_current_reclaim_state(); - return 0; } @@ -3098,11 +3146,16 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) struct reclaim_state reclaim_state; struct scan_control sc = { .gfp_mask = GFP_HIGHUSER_MOVABLE, +#if defined(CONFIG_SLP) && defined(CONFIG_FULL_PAGE_RECLAIM) + .may_swap = 0, +#else .may_swap = 1, +#endif .may_unmap = 1, .may_writepage = 1, .nr_to_reclaim = nr_to_reclaim, .hibernation_mode = 1, + .swappiness = vm_swappiness, .order = 0, }; struct shrink_control shrink = { @@ -3292,6 +3345,7 @@ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) .nr_to_reclaim = max_t(unsigned long, nr_pages, SWAP_CLUSTER_MAX), .gfp_mask = gfp_mask, + .swappiness = vm_swappiness, .order = order, }; struct shrink_control shrink = { @@ -3440,66 +3494,158 @@ int page_evictable(struct page *page, struct vm_area_struct *vma) return 1; } -#ifdef CONFIG_SHMEM /** - * check_move_unevictable_pages - check pages for evictability and move to appropriate zone lru list - * @pages: array of pages to check - * @nr_pages: number of pages to check + * check_move_unevictable_page - check page for evictability and move to appropriate zone lru list + * @page: page to check evictability and move to appropriate lru list + * @zone: zone page is in * - * Checks pages for evictability and moves them to the appropriate lru list. + * Checks a page for evictability and moves the page to the appropriate + * zone lru list. * - * This function is only used for SysV IPC SHM_UNLOCK. + * Restrictions: zone->lru_lock must be held, page must be on LRU and must + * have PageUnevictable set. */ -void check_move_unevictable_pages(struct page **pages, int nr_pages) +static void check_move_unevictable_page(struct page *page, struct zone *zone) { - struct zone *zone = NULL; - int pgscanned = 0; - int pgrescued = 0; - int i; + VM_BUG_ON(PageActive(page)); - for (i = 0; i < nr_pages; i++) { - struct page *page = pages[i]; - struct zone *pagezone; +retry: + ClearPageUnevictable(page); + if (page_evictable(page, NULL)) { + enum lru_list l = page_lru_base_type(page); - pgscanned++; - pagezone = page_zone(page); - if (pagezone != zone) { - if (zone) - spin_unlock_irq(&zone->lru_lock); - zone = pagezone; - spin_lock_irq(&zone->lru_lock); - } + __dec_zone_state(zone, NR_UNEVICTABLE); + list_move(&page->lru, &zone->lru[l].list); + mem_cgroup_move_lists(page, LRU_UNEVICTABLE, l); + __inc_zone_state(zone, NR_INACTIVE_ANON + l); + __count_vm_event(UNEVICTABLE_PGRESCUED); + } else { + /* + * rotate unevictable list + */ + SetPageUnevictable(page); + list_move(&page->lru, &zone->lru[LRU_UNEVICTABLE].list); + mem_cgroup_rotate_lru_list(page, LRU_UNEVICTABLE); + if (page_evictable(page, NULL)) + goto retry; + } +} - if (!PageLRU(page) || !PageUnevictable(page)) - continue; +/** + * scan_mapping_unevictable_pages - scan an address space for evictable pages + * @mapping: struct address_space to scan for evictable pages + * + * Scan all pages in mapping. Check unevictable pages for + * evictability and move them to the appropriate zone lru list. + */ +void scan_mapping_unevictable_pages(struct address_space *mapping) +{ + pgoff_t next = 0; + pgoff_t end = (i_size_read(mapping->host) + PAGE_CACHE_SIZE - 1) >> + PAGE_CACHE_SHIFT; + struct zone *zone; + struct pagevec pvec; - if (page_evictable(page, NULL)) { - enum lru_list lru = page_lru_base_type(page); + if (mapping->nrpages == 0) + return; - VM_BUG_ON(PageActive(page)); - ClearPageUnevictable(page); - __dec_zone_state(zone, NR_UNEVICTABLE); - list_move(&page->lru, &zone->lru[lru].list); - mem_cgroup_move_lists(page, LRU_UNEVICTABLE, lru); - __inc_zone_state(zone, NR_INACTIVE_ANON + lru); - pgrescued++; + pagevec_init(&pvec, 0); + while (next < end && + pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) { + int i; + int pg_scanned = 0; + + zone = NULL; + + for (i = 0; i < pagevec_count(&pvec); i++) { + struct page *page = pvec.pages[i]; + pgoff_t page_index = page->index; + struct zone *pagezone = page_zone(page); + + pg_scanned++; + if (page_index > next) + next = page_index; + next++; + + if (pagezone != zone) { + if (zone) + spin_unlock_irq(&zone->lru_lock); + zone = pagezone; + spin_lock_irq(&zone->lru_lock); + } + + if (PageLRU(page) && PageUnevictable(page)) + check_move_unevictable_page(page, zone); } + if (zone) + spin_unlock_irq(&zone->lru_lock); + pagevec_release(&pvec); + + count_vm_events(UNEVICTABLE_PGSCANNED, pg_scanned); } - if (zone) { - __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued); - __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned); +} + +/** + * scan_zone_unevictable_pages - check unevictable list for evictable pages + * @zone - zone of which to scan the unevictable list + * + * Scan @zone's unevictable LRU lists to check for pages that have become + * evictable. Move those that have to @zone's inactive list where they + * become candidates for reclaim, unless shrink_inactive_zone() decides + * to reactivate them. Pages that are still unevictable are rotated + * back onto @zone's unevictable list. + */ +#define SCAN_UNEVICTABLE_BATCH_SIZE 16UL /* arbitrary lock hold batch size */ +static void scan_zone_unevictable_pages(struct zone *zone) +{ + struct list_head *l_unevictable = &zone->lru[LRU_UNEVICTABLE].list; + unsigned long scan; + unsigned long nr_to_scan = zone_page_state(zone, NR_UNEVICTABLE); + + while (nr_to_scan > 0) { + unsigned long batch_size = min(nr_to_scan, + SCAN_UNEVICTABLE_BATCH_SIZE); + + spin_lock_irq(&zone->lru_lock); + for (scan = 0; scan < batch_size; scan++) { + struct page *page = lru_to_page(l_unevictable); + + if (!trylock_page(page)) + continue; + + prefetchw_prev_lru_page(page, l_unevictable, flags); + + if (likely(PageLRU(page) && PageUnevictable(page))) + check_move_unevictable_page(page, zone); + + unlock_page(page); + } spin_unlock_irq(&zone->lru_lock); + + nr_to_scan -= batch_size; } } -#endif /* CONFIG_SHMEM */ -static void warn_scan_unevictable_pages(void) + +/** + * scan_all_zones_unevictable_pages - scan all unevictable lists for evictable pages + * + * A really big hammer: scan all zones' unevictable LRU lists to check for + * pages that have become evictable. Move those back to the zones' + * inactive list where they become candidates for reclaim. + * This occurs when, e.g., we have unswappable pages on the unevictable lists, + * and we add swap to the system. As such, it runs in the context of a task + * that has possibly/probably made some previously unevictable pages + * evictable. + */ +static void scan_all_zones_unevictable_pages(void) { - printk_once(KERN_WARNING - "The scan_unevictable_pages sysctl/node-interface has been " - "disabled for lack of a legitimate use case. If you have " - "one, please send an email to linux-mm@kvack.org.\n"); + struct zone *zone; + + for_each_zone(zone) { + scan_zone_unevictable_pages(zone); + } } /* @@ -3512,8 +3658,11 @@ int scan_unevictable_handler(struct ctl_table *table, int write, void __user *buffer, size_t *length, loff_t *ppos) { - warn_scan_unevictable_pages(); proc_doulongvec_minmax(table, write, buffer, length, ppos); + + if (write && *(unsigned long *)table->data) + scan_all_zones_unevictable_pages(); + scan_unevictable_pages = 0; return 0; } @@ -3528,7 +3677,6 @@ static ssize_t read_scan_unevictable_node(struct sys_device *dev, struct sysdev_attribute *attr, char *buf) { - warn_scan_unevictable_pages(); return sprintf(buf, "0\n"); /* always zero; should fit... */ } @@ -3536,7 +3684,19 @@ static ssize_t write_scan_unevictable_node(struct sys_device *dev, struct sysdev_attribute *attr, const char *buf, size_t count) { - warn_scan_unevictable_pages(); + struct zone *node_zones = NODE_DATA(dev->id)->node_zones; + struct zone *zone; + unsigned long res; + unsigned long req = strict_strtoul(buf, 10, &res); + + if (!req) + return 1; /* zero is no-op */ + + for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { + if (!populated_zone(zone)) + continue; + scan_zone_unevictable_pages(zone); + } return 1; } diff --git a/mm/vmstat.c b/mm/vmstat.c index 8fd603b..bdf534a 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -78,7 +78,11 @@ void vm_events_fold_cpu(int cpu) * * vm_stat contains the global counters */ +#ifndef CONFIG_DMA_CMA +atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; +#else atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp; +#endif EXPORT_SYMBOL(vm_stat); #ifdef CONFIG_SMP @@ -613,6 +617,9 @@ static char * const migratetype_names[MIGRATE_TYPES] = { "Reclaimable", "Movable", "Reserve", +#ifdef CONFIG_DMA_CMA + "CMA", +#endif "Isolate", }; @@ -659,7 +666,7 @@ static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat, } #endif -#if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || defined(CONFIG_NUMA) +#if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) #ifdef CONFIG_ZONE_DMA #define TEXT_FOR_DMA(xx) xx "_dma", #else @@ -702,7 +709,6 @@ const char * const vmstat_text[] = { "nr_unstable", "nr_bounce", "nr_vmscan_write", - "nr_vmscan_immediate_reclaim", "nr_writeback_temp", "nr_isolated_anon", "nr_isolated_file", @@ -719,6 +725,9 @@ const char * const vmstat_text[] = { "numa_other", #endif "nr_anon_transparent_hugepages", +#ifdef CONFIG_DMA_CMA + "nr_free_cma", +#endif "nr_dirty_threshold", "nr_dirty_background_threshold", @@ -789,7 +798,7 @@ const char * const vmstat_text[] = { #endif /* CONFIG_VM_EVENTS_COUNTERS */ }; -#endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA */ +#endif /* CONFIG_PROC_FS || CONFIG_SYSFS */ #ifdef CONFIG_PROC_FS |