From c6da2cfeb05178a11c6d062a06f8078150ee492f Mon Sep 17 00:00:00 2001 From: codeworkx Date: Sat, 2 Jun 2012 13:09:29 +0200 Subject: samsung update 1 --- mm/Kconfig | 52 ++ mm/Makefile | 10 +- mm/ashmem.c | 749 ++++++++++++++++++++++++++++ mm/cma-best-fit.c | 408 ++++++++++++++++ mm/cma.c | 1413 +++++++++++++++++++++++++++++++++++++++++++++++++++++ mm/compaction.c | 4 + mm/filemap.c | 8 +- mm/memory.c | 1 + mm/page_alloc.c | 47 +- mm/shmem.c | 15 +- mm/slub.c | 10 + mm/truncate.c | 6 + mm/vmscan.c | 3 + 13 files changed, 2712 insertions(+), 14 deletions(-) create mode 100644 mm/ashmem.c create mode 100644 mm/cma-best-fit.c create mode 100644 mm/cma.c (limited to 'mm') diff --git a/mm/Kconfig b/mm/Kconfig index 8ca47a5..3c2b673 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -370,3 +370,55 @@ config CLEANCACHE in a negligible performance hit. If unsure, say Y to enable cleancache + +config CMA + bool "Contiguous Memory Allocator framework" + # Currently there is only one allocator so force it on + select CMA_BEST_FIT + help + This enables the Contiguous Memory Allocator framework which + allows drivers to allocate big physically-contiguous blocks of + memory for use with hardware components that do not support I/O + map nor scatter-gather. + + If you select this option you will also have to select at least + one allocator algorithm below. + + To make use of CMA you need to specify the regions and + driver->region mapping on command line when booting the kernel. + +config CMA_DEVELOPEMENT + bool "Include CMA developement features" + depends on CMA + help + This lets you enable some developement features of the CMA + freamework. + +config CMA_DEBUG + bool "CMA debug messages" + depends on CMA_DEVELOPEMENT + help + Enable debug messages in CMA code. + +config CMA_SYSFS + bool "CMA SysFS interface support" + depends on CMA_DEVELOPEMENT + help + Enable support for SysFS interface. + +config CMA_CMDLINE + bool "CMA command line parameters support" + depends on CMA_DEVELOPEMENT + help + Enable support for cma, cma.map and cma.asterisk command line + parameters. + +config CMA_BEST_FIT + bool "CMA best-fit allocator" + depends on CMA + help + This is a best-fit algorithm running in O(n log n) time where + n is the number of existing holes (which is never greater then + the number of allocated regions and usually much smaller). It + allocates area from the smallest hole that is big enough for + allocation in question. diff --git a/mm/Makefile b/mm/Makefile index 836e416..4ade76a 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -8,11 +8,14 @@ mmu-$(CONFIG_MMU) := fremap.o highmem.o madvise.o memory.o mincore.o \ vmalloc.o pagewalk.o pgtable-generic.o obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ - maccess.o page_alloc.o page-writeback.o \ + maccess.o page-writeback.o \ readahead.o swap.o truncate.o vmscan.o shmem.o \ prio_tree.o util.o mmzone.o vmstat.o backing-dev.o \ - page_isolation.o mm_init.o mmu_context.o percpu.o \ + mm_init.o mmu_context.o percpu.o \ $(mmu-y) + +obj-y += page_alloc.o page_isolation.o + obj-y += init-mm.o ifdef CONFIG_NO_BOOTMEM @@ -30,6 +33,7 @@ obj-$(CONFIG_HUGETLBFS) += hugetlb.o obj-$(CONFIG_NUMA) += mempolicy.o obj-$(CONFIG_SPARSEMEM) += sparse.o obj-$(CONFIG_SPARSEMEM_VMEMMAP) += sparse-vmemmap.o +obj-$(CONFIG_ASHMEM) += ashmem.o obj-$(CONFIG_SLOB) += slob.o obj-$(CONFIG_COMPACTION) += compaction.o obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o @@ -50,3 +54,5 @@ obj-$(CONFIG_HWPOISON_INJECT) += hwpoison-inject.o obj-$(CONFIG_DEBUG_KMEMLEAK) += kmemleak.o obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak-test.o obj-$(CONFIG_CLEANCACHE) += cleancache.o +obj-$(CONFIG_CMA) += cma.o +obj-$(CONFIG_CMA_BEST_FIT) += cma-best-fit.o diff --git a/mm/ashmem.c b/mm/ashmem.c new file mode 100644 index 0000000..e98f358 --- /dev/null +++ b/mm/ashmem.c @@ -0,0 +1,749 @@ +/* mm/ashmem.c +** +** Anonymous Shared Memory Subsystem, ashmem +** +** Copyright (C) 2008 Google, Inc. +** +** Robert Love +** +** This software is licensed under the terms of the GNU General Public +** License version 2, as published by the Free Software Foundation, and +** may be copied, distributed, and modified under those terms. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ASHMEM_NAME_PREFIX "dev/ashmem/" +#define ASHMEM_NAME_PREFIX_LEN (sizeof(ASHMEM_NAME_PREFIX) - 1) +#define ASHMEM_FULL_NAME_LEN (ASHMEM_NAME_LEN + ASHMEM_NAME_PREFIX_LEN) + +/* + * ashmem_area - anonymous shared memory area + * Lifecycle: From our parent file's open() until its release() + * Locking: Protected by `ashmem_mutex' + * Big Note: Mappings do NOT pin this structure; it dies on close() + */ +struct ashmem_area { + char name[ASHMEM_FULL_NAME_LEN];/* optional name for /proc/pid/maps */ + struct list_head unpinned_list; /* list of all ashmem areas */ + struct file *file; /* the shmem-based backing file */ + size_t size; /* size of the mapping, in bytes */ + unsigned long prot_mask; /* allowed prot bits, as vm_flags */ +}; + +/* + * ashmem_range - represents an interval of unpinned (evictable) pages + * Lifecycle: From unpin to pin + * Locking: Protected by `ashmem_mutex' + */ +struct ashmem_range { + struct list_head lru; /* entry in LRU list */ + struct list_head unpinned; /* entry in its area's unpinned list */ + struct ashmem_area *asma; /* associated area */ + size_t pgstart; /* starting page, inclusive */ + size_t pgend; /* ending page, inclusive */ + unsigned int purged; /* ASHMEM_NOT or ASHMEM_WAS_PURGED */ +}; + +/* LRU list of unpinned pages, protected by ashmem_mutex */ +static LIST_HEAD(ashmem_lru_list); + +/* Count of pages on our LRU list, protected by ashmem_mutex */ +static unsigned long lru_count; + +/* + * ashmem_mutex - protects the list of and each individual ashmem_area + * + * Lock Ordering: ashmex_mutex -> i_mutex -> i_alloc_sem + */ +static DEFINE_MUTEX(ashmem_mutex); + +static struct kmem_cache *ashmem_area_cachep __read_mostly; +static struct kmem_cache *ashmem_range_cachep __read_mostly; + +#define range_size(range) \ + ((range)->pgend - (range)->pgstart + 1) + +#define range_on_lru(range) \ + ((range)->purged == ASHMEM_NOT_PURGED) + +#define page_range_subsumes_range(range, start, end) \ + (((range)->pgstart >= (start)) && ((range)->pgend <= (end))) + +#define page_range_subsumed_by_range(range, start, end) \ + (((range)->pgstart <= (start)) && ((range)->pgend >= (end))) + +#define page_in_range(range, page) \ + (((range)->pgstart <= (page)) && ((range)->pgend >= (page))) + +#define page_range_in_range(range, start, end) \ + (page_in_range(range, start) || page_in_range(range, end) || \ + page_range_subsumes_range(range, start, end)) + +#define range_before_page(range, page) \ + ((range)->pgend < (page)) + +#define PROT_MASK (PROT_EXEC | PROT_READ | PROT_WRITE) + +static inline void lru_add(struct ashmem_range *range) +{ + list_add_tail(&range->lru, &ashmem_lru_list); + lru_count += range_size(range); +} + +static inline void lru_del(struct ashmem_range *range) +{ + list_del(&range->lru); + lru_count -= range_size(range); +} + +/* + * range_alloc - allocate and initialize a new ashmem_range structure + * + * 'asma' - associated ashmem_area + * 'prev_range' - the previous ashmem_range in the sorted asma->unpinned list + * 'purged' - initial purge value (ASMEM_NOT_PURGED or ASHMEM_WAS_PURGED) + * 'start' - starting page, inclusive + * 'end' - ending page, inclusive + * + * Caller must hold ashmem_mutex. + */ +static int range_alloc(struct ashmem_area *asma, + struct ashmem_range *prev_range, unsigned int purged, + size_t start, size_t end) +{ + struct ashmem_range *range; + + range = kmem_cache_zalloc(ashmem_range_cachep, GFP_KERNEL); + if (unlikely(!range)) + return -ENOMEM; + + range->asma = asma; + range->pgstart = start; + range->pgend = end; + range->purged = purged; + + list_add_tail(&range->unpinned, &prev_range->unpinned); + + if (range_on_lru(range)) + lru_add(range); + + return 0; +} + +static void range_del(struct ashmem_range *range) +{ + list_del(&range->unpinned); + if (range_on_lru(range)) + lru_del(range); + kmem_cache_free(ashmem_range_cachep, range); +} + +/* + * range_shrink - shrinks a range + * + * Caller must hold ashmem_mutex. + */ +static inline void range_shrink(struct ashmem_range *range, + size_t start, size_t end) +{ + size_t pre = range_size(range); + + range->pgstart = start; + range->pgend = end; + + if (range_on_lru(range)) + lru_count -= pre - range_size(range); +} + +static int ashmem_open(struct inode *inode, struct file *file) +{ + struct ashmem_area *asma; + int ret; + + ret = generic_file_open(inode, file); + if (unlikely(ret)) + return ret; + + asma = kmem_cache_zalloc(ashmem_area_cachep, GFP_KERNEL); + if (unlikely(!asma)) + return -ENOMEM; + + INIT_LIST_HEAD(&asma->unpinned_list); + memcpy(asma->name, ASHMEM_NAME_PREFIX, ASHMEM_NAME_PREFIX_LEN); + asma->prot_mask = PROT_MASK; + file->private_data = asma; + + return 0; +} + +static int ashmem_release(struct inode *ignored, struct file *file) +{ + struct ashmem_area *asma = file->private_data; + struct ashmem_range *range, *next; + + mutex_lock(&ashmem_mutex); + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) + range_del(range); + mutex_unlock(&ashmem_mutex); + + if (asma->file) + fput(asma->file); + kmem_cache_free(ashmem_area_cachep, asma); + + return 0; +} + +static ssize_t ashmem_read(struct file *file, char __user *buf, + size_t len, loff_t *pos) +{ + struct ashmem_area *asma = file->private_data; + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* If size is not set, or set to 0, always return EOF. */ + if (asma->size == 0) { + goto out; + } + + if (!asma->file) { + ret = -EBADF; + goto out; + } + + ret = asma->file->f_op->read(asma->file, buf, len, pos); + if (ret < 0) { + goto out; + } + + /** Update backing file pos, since f_ops->read() doesn't */ + asma->file->f_pos = *pos; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +static loff_t ashmem_llseek(struct file *file, loff_t offset, int origin) +{ + struct ashmem_area *asma = file->private_data; + int ret; + + mutex_lock(&ashmem_mutex); + + if (asma->size == 0) { + ret = -EINVAL; + goto out; + } + + if (!asma->file) { + ret = -EBADF; + goto out; + } + + ret = asma->file->f_op->llseek(asma->file, offset, origin); + if (ret < 0) { + goto out; + } + + /** Copy f_pos from backing file, since f_ops->llseek() sets it */ + file->f_pos = asma->file->f_pos; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +static inline unsigned long +calc_vm_may_flags(unsigned long prot) +{ + return _calc_vm_trans(prot, PROT_READ, VM_MAYREAD ) | + _calc_vm_trans(prot, PROT_WRITE, VM_MAYWRITE) | + _calc_vm_trans(prot, PROT_EXEC, VM_MAYEXEC); +} + +static int ashmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct ashmem_area *asma = file->private_data; + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* user needs to SET_SIZE before mapping */ + if (unlikely(!asma->size)) { + ret = -EINVAL; + goto out; + } + + /* requested protection bits must match our allowed protection mask */ + if (unlikely((vma->vm_flags & ~calc_vm_prot_bits(asma->prot_mask)) & + calc_vm_prot_bits(PROT_MASK))) { + ret = -EPERM; + goto out; + } + vma->vm_flags &= ~calc_vm_may_flags(~asma->prot_mask); + + if (!asma->file) { + char *name = ASHMEM_NAME_DEF; + struct file *vmfile; + + if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0') + name = asma->name; + + /* ... and allocate the backing shmem file */ + vmfile = shmem_file_setup(name, asma->size, vma->vm_flags); + if (unlikely(IS_ERR(vmfile))) { + ret = PTR_ERR(vmfile); + goto out; + } + asma->file = vmfile; + } + get_file(asma->file); + + if (vma->vm_flags & VM_SHARED) + shmem_set_file(vma, asma->file); + else { + if (vma->vm_file) + fput(vma->vm_file); + vma->vm_file = asma->file; + } + vma->vm_flags |= VM_CAN_NONLINEAR; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +/* + * ashmem_shrink - our cache shrinker, called from mm/vmscan.c :: shrink_slab + * + * 'nr_to_scan' is the number of objects (pages) to prune, or 0 to query how + * many objects (pages) we have in total. + * + * 'gfp_mask' is the mask of the allocation that got us into this mess. + * + * Return value is the number of objects (pages) remaining, or -1 if we cannot + * proceed without risk of deadlock (due to gfp_mask). + * + * We approximate LRU via least-recently-unpinned, jettisoning unpinned partial + * chunks of ashmem regions LRU-wise one-at-a-time until we hit 'nr_to_scan' + * pages freed. + */ +static int ashmem_shrink(struct shrinker *s, struct shrink_control *sc) +{ + struct ashmem_range *range, *next; + + /* We might recurse into filesystem code, so bail out if necessary */ + if (sc->nr_to_scan && !(sc->gfp_mask & __GFP_FS)) + return -1; + if (!sc->nr_to_scan) + return lru_count; + + if (!mutex_trylock(&ashmem_mutex)) + return lru_count; + list_for_each_entry_safe(range, next, &ashmem_lru_list, lru) { + struct inode *inode = range->asma->file->f_dentry->d_inode; + loff_t start = range->pgstart * PAGE_SIZE; + loff_t end = (range->pgend + 1) * PAGE_SIZE - 1; + + vmtruncate_range(inode, start, end); + range->purged = ASHMEM_WAS_PURGED; + lru_del(range); + + sc->nr_to_scan -= range_size(range); + if (sc->nr_to_scan <= 0) + break; + } + mutex_unlock(&ashmem_mutex); + + return lru_count; +} + +static struct shrinker ashmem_shrinker = { + .shrink = ashmem_shrink, + .seeks = DEFAULT_SEEKS * 4, +}; + +static int set_prot_mask(struct ashmem_area *asma, unsigned long prot) +{ + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* the user can only remove, not add, protection bits */ + if (unlikely((asma->prot_mask & prot) != prot)) { + ret = -EINVAL; + goto out; + } + + /* does the application expect PROT_READ to imply PROT_EXEC? */ + if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) + prot |= PROT_EXEC; + + asma->prot_mask = prot; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +static int set_name(struct ashmem_area *asma, void __user *name) +{ + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* cannot change an existing mapping's name */ + if (unlikely(asma->file)) { + ret = -EINVAL; + goto out; + } + + if (unlikely(copy_from_user(asma->name + ASHMEM_NAME_PREFIX_LEN, + name, ASHMEM_NAME_LEN))) + ret = -EFAULT; + asma->name[ASHMEM_FULL_NAME_LEN-1] = '\0'; + +out: + mutex_unlock(&ashmem_mutex); + + return ret; +} + +static int get_name(struct ashmem_area *asma, void __user *name) +{ + int ret = 0; + + mutex_lock(&ashmem_mutex); + if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0') { + size_t len; + + /* + * Copying only `len', instead of ASHMEM_NAME_LEN, bytes + * prevents us from revealing one user's stack to another. + */ + len = strlen(asma->name + ASHMEM_NAME_PREFIX_LEN) + 1; + if (unlikely(copy_to_user(name, + asma->name + ASHMEM_NAME_PREFIX_LEN, len))) + ret = -EFAULT; + } else { + if (unlikely(copy_to_user(name, ASHMEM_NAME_DEF, + sizeof(ASHMEM_NAME_DEF)))) + ret = -EFAULT; + } + mutex_unlock(&ashmem_mutex); + + return ret; +} + +/* + * ashmem_pin - pin the given ashmem region, returning whether it was + * previously purged (ASHMEM_WAS_PURGED) or not (ASHMEM_NOT_PURGED). + * + * Caller must hold ashmem_mutex. + */ +static int ashmem_pin(struct ashmem_area *asma, size_t pgstart, size_t pgend) +{ + struct ashmem_range *range, *next; + int ret = ASHMEM_NOT_PURGED; + + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) { + /* moved past last applicable page; we can short circuit */ + if (range_before_page(range, pgstart)) + break; + + /* + * The user can ask us to pin pages that span multiple ranges, + * or to pin pages that aren't even unpinned, so this is messy. + * + * Four cases: + * 1. The requested range subsumes an existing range, so we + * just remove the entire matching range. + * 2. The requested range overlaps the start of an existing + * range, so we just update that range. + * 3. The requested range overlaps the end of an existing + * range, so we just update that range. + * 4. The requested range punches a hole in an existing range, + * so we have to update one side of the range and then + * create a new range for the other side. + */ + if (page_range_in_range(range, pgstart, pgend)) { + ret |= range->purged; + + /* Case #1: Easy. Just nuke the whole thing. */ + if (page_range_subsumes_range(range, pgstart, pgend)) { + range_del(range); + continue; + } + + /* Case #2: We overlap from the start, so adjust it */ + if (range->pgstart >= pgstart) { + range_shrink(range, pgend + 1, range->pgend); + continue; + } + + /* Case #3: We overlap from the rear, so adjust it */ + if (range->pgend <= pgend) { + range_shrink(range, range->pgstart, pgstart-1); + continue; + } + + /* + * Case #4: We eat a chunk out of the middle. A bit + * more complicated, we allocate a new range for the + * second half and adjust the first chunk's endpoint. + */ + range_alloc(asma, range, range->purged, + pgend + 1, range->pgend); + range_shrink(range, range->pgstart, pgstart - 1); + break; + } + } + + return ret; +} + +/* + * ashmem_unpin - unpin the given range of pages. Returns zero on success. + * + * Caller must hold ashmem_mutex. + */ +static int ashmem_unpin(struct ashmem_area *asma, size_t pgstart, size_t pgend) +{ + struct ashmem_range *range, *next; + unsigned int purged = ASHMEM_NOT_PURGED; + +restart: + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) { + /* short circuit: this is our insertion point */ + if (range_before_page(range, pgstart)) + break; + + /* + * The user can ask us to unpin pages that are already entirely + * or partially pinned. We handle those two cases here. + */ + if (page_range_subsumed_by_range(range, pgstart, pgend)) + return 0; + if (page_range_in_range(range, pgstart, pgend)) { + pgstart = min_t(size_t, range->pgstart, pgstart), + pgend = max_t(size_t, range->pgend, pgend); + purged |= range->purged; + range_del(range); + goto restart; + } + } + + return range_alloc(asma, range, purged, pgstart, pgend); +} + +/* + * ashmem_get_pin_status - Returns ASHMEM_IS_UNPINNED if _any_ pages in the + * given interval are unpinned and ASHMEM_IS_PINNED otherwise. + * + * Caller must hold ashmem_mutex. + */ +static int ashmem_get_pin_status(struct ashmem_area *asma, size_t pgstart, + size_t pgend) +{ + struct ashmem_range *range; + int ret = ASHMEM_IS_PINNED; + + list_for_each_entry(range, &asma->unpinned_list, unpinned) { + if (range_before_page(range, pgstart)) + break; + if (page_range_in_range(range, pgstart, pgend)) { + ret = ASHMEM_IS_UNPINNED; + break; + } + } + + return ret; +} + +static int ashmem_pin_unpin(struct ashmem_area *asma, unsigned long cmd, + void __user *p) +{ + struct ashmem_pin pin; + size_t pgstart, pgend; + int ret = -EINVAL; + + if (unlikely(!asma->file)) + return -EINVAL; + + if (unlikely(copy_from_user(&pin, p, sizeof(pin)))) + return -EFAULT; + + /* per custom, you can pass zero for len to mean "everything onward" */ + if (!pin.len) + pin.len = PAGE_ALIGN(asma->size) - pin.offset; + + if (unlikely((pin.offset | pin.len) & ~PAGE_MASK)) + return -EINVAL; + + if (unlikely(((__u32) -1) - pin.offset < pin.len)) + return -EINVAL; + + if (unlikely(PAGE_ALIGN(asma->size) < pin.offset + pin.len)) + return -EINVAL; + + pgstart = pin.offset / PAGE_SIZE; + pgend = pgstart + (pin.len / PAGE_SIZE) - 1; + + mutex_lock(&ashmem_mutex); + + switch (cmd) { + case ASHMEM_PIN: + ret = ashmem_pin(asma, pgstart, pgend); + break; + case ASHMEM_UNPIN: + ret = ashmem_unpin(asma, pgstart, pgend); + break; + case ASHMEM_GET_PIN_STATUS: + ret = ashmem_get_pin_status(asma, pgstart, pgend); + break; + } + + mutex_unlock(&ashmem_mutex); + + return ret; +} + +static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct ashmem_area *asma = file->private_data; + long ret = -ENOTTY; + + switch (cmd) { + case ASHMEM_SET_NAME: + ret = set_name(asma, (void __user *) arg); + break; + case ASHMEM_GET_NAME: + ret = get_name(asma, (void __user *) arg); + break; + case ASHMEM_SET_SIZE: + ret = -EINVAL; + if (!asma->file) { + ret = 0; + asma->size = (size_t) arg; + } + break; + case ASHMEM_GET_SIZE: + ret = asma->size; + break; + case ASHMEM_SET_PROT_MASK: + ret = set_prot_mask(asma, arg); + break; + case ASHMEM_GET_PROT_MASK: + ret = asma->prot_mask; + break; + case ASHMEM_PIN: + case ASHMEM_UNPIN: + case ASHMEM_GET_PIN_STATUS: + ret = ashmem_pin_unpin(asma, cmd, (void __user *) arg); + break; + case ASHMEM_PURGE_ALL_CACHES: + ret = -EPERM; + if (capable(CAP_SYS_ADMIN)) { + struct shrink_control sc = { + .gfp_mask = GFP_KERNEL, + .nr_to_scan = 0, + }; + ret = ashmem_shrink(&ashmem_shrinker, &sc); + sc.nr_to_scan = ret; + ashmem_shrink(&ashmem_shrinker, &sc); + } + break; + } + + return ret; +} + +static struct file_operations ashmem_fops = { + .owner = THIS_MODULE, + .open = ashmem_open, + .release = ashmem_release, + .read = ashmem_read, + .llseek = ashmem_llseek, + .mmap = ashmem_mmap, + .unlocked_ioctl = ashmem_ioctl, + .compat_ioctl = ashmem_ioctl, +}; + +static struct miscdevice ashmem_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "ashmem", + .fops = &ashmem_fops, +}; + +static int __init ashmem_init(void) +{ + int ret; + + ashmem_area_cachep = kmem_cache_create("ashmem_area_cache", + sizeof(struct ashmem_area), + 0, 0, NULL); + if (unlikely(!ashmem_area_cachep)) { + printk(KERN_ERR "ashmem: failed to create slab cache\n"); + return -ENOMEM; + } + + ashmem_range_cachep = kmem_cache_create("ashmem_range_cache", + sizeof(struct ashmem_range), + 0, 0, NULL); + if (unlikely(!ashmem_range_cachep)) { + printk(KERN_ERR "ashmem: failed to create slab cache\n"); + return -ENOMEM; + } + + ret = misc_register(&ashmem_misc); + if (unlikely(ret)) { + printk(KERN_ERR "ashmem: failed to register misc device!\n"); + return ret; + } + + register_shrinker(&ashmem_shrinker); + + printk(KERN_INFO "ashmem: initialized\n"); + + return 0; +} + +static void __exit ashmem_exit(void) +{ + int ret; + + unregister_shrinker(&ashmem_shrinker); + + ret = misc_deregister(&ashmem_misc); + if (unlikely(ret)) + printk(KERN_ERR "ashmem: failed to unregister misc device!\n"); + + kmem_cache_destroy(ashmem_range_cachep); + kmem_cache_destroy(ashmem_area_cachep); + + printk(KERN_INFO "ashmem: unloaded\n"); +} + +module_init(ashmem_init); +module_exit(ashmem_exit); + +MODULE_LICENSE("GPL"); diff --git a/mm/cma-best-fit.c b/mm/cma-best-fit.c new file mode 100644 index 0000000..24c27c8 --- /dev/null +++ b/mm/cma-best-fit.c @@ -0,0 +1,408 @@ +/* + * Contiguous Memory Allocator framework: Best Fit allocator + * Copyright (c) 2010 by Samsung Electronics. + * Written by Michal Nazarewicz (m.nazarewicz@samsung.com) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License or (at your optional) any later version of the license. + */ + +#define pr_fmt(fmt) "cma: bf: " fmt + +#ifdef CONFIG_CMA_DEBUG +# define DEBUG +#endif + +#include /* Error numbers */ +#include /* kmalloc() */ + +#include /* CMA structures */ + + +/************************* Data Types *************************/ + +struct cma_bf_item { + struct cma_chunk ch; + struct rb_node by_size; +}; + +struct cma_bf_private { + struct rb_root by_start_root; + struct rb_root by_size_root; +}; + + +/************************* Prototypes *************************/ + +/* + * Those are only for holes. They must be called whenever hole's + * properties change but also whenever chunk becomes a hole or hole + * becames a chunk. + */ +static void __cma_bf_hole_insert_by_size(struct cma_bf_item *item); +static void __cma_bf_hole_erase_by_size(struct cma_bf_item *item); +static int __must_check +__cma_bf_hole_insert_by_start(struct cma_bf_item *item); +static void __cma_bf_hole_erase_by_start(struct cma_bf_item *item); + +/** + * __cma_bf_hole_take - takes a chunk of memory out of a hole. + * @hole: hole to take chunk from + * @size: chunk's size + * @alignment: chunk's starting address alignment (must be power of two) + * + * Takes a @size bytes large chunk from hole @hole which must be able + * to hold the chunk. The "must be able" includes also alignment + * constraint. + * + * Returns allocated item or NULL on error (if kmalloc() failed). + */ +static struct cma_bf_item *__must_check +__cma_bf_hole_take(struct cma_bf_item *hole, size_t size, dma_addr_t alignment); + +/** + * __cma_bf_hole_merge_maybe - tries to merge hole with neighbours. + * @item: hole to try and merge + * + * Which items are preserved is undefined so you may not rely on it. + */ +static void __cma_bf_hole_merge_maybe(struct cma_bf_item *item); + + +/************************* Device API *************************/ + +int cma_bf_init(struct cma_region *reg) +{ + struct cma_bf_private *prv; + struct cma_bf_item *item; + + prv = kzalloc(sizeof *prv, GFP_KERNEL); + if (unlikely(!prv)) + return -ENOMEM; + + item = kzalloc(sizeof *item, GFP_KERNEL); + if (unlikely(!item)) { + kfree(prv); + return -ENOMEM; + } + + item->ch.start = reg->start; + item->ch.size = reg->size; + item->ch.reg = reg; + + rb_root_init(&prv->by_start_root, &item->ch.by_start); + rb_root_init(&prv->by_size_root, &item->by_size); + + reg->private_data = prv; + return 0; +} + +void cma_bf_cleanup(struct cma_region *reg) +{ + struct cma_bf_private *prv = reg->private_data; + struct cma_bf_item *item = + rb_entry(prv->by_size_root.rb_node, + struct cma_bf_item, by_size); + + /* We can assume there is only a single hole in the tree. */ + WARN_ON(item->by_size.rb_left || item->by_size.rb_right || + item->ch.by_start.rb_left || item->ch.by_start.rb_right); + + kfree(item); + kfree(prv); +} + +struct cma_chunk *cma_bf_alloc(struct cma_region *reg, + size_t size, dma_addr_t alignment) +{ + struct cma_bf_private *prv = reg->private_data; + struct rb_node *node = prv->by_size_root.rb_node; + struct cma_bf_item *item = NULL; + + /* First find hole that is large enough */ + while (node) { + struct cma_bf_item *i = + rb_entry(node, struct cma_bf_item, by_size); + + if (i->ch.size < size) { + node = node->rb_right; + } else if (i->ch.size >= size) { + node = node->rb_left; + item = i; + } + } + if (!item) + return NULL; + + /* Now look for items which can satisfy alignment requirements */ + node = &item->by_size; + for (;;) { + dma_addr_t start = ALIGN(item->ch.start, alignment); + dma_addr_t end = item->ch.start + item->ch.size; + if (start < end && end - start >= size) { + item = __cma_bf_hole_take(item, size, alignment); + return likely(item) ? &item->ch : NULL; + } + + node = rb_next(node); + if (!node) + return NULL; + + item = rb_entry(node, struct cma_bf_item, by_size); + } +} + +void cma_bf_free(struct cma_chunk *chunk) +{ + struct cma_bf_item *item = container_of(chunk, struct cma_bf_item, ch); + + /* Add new hole */ + if (unlikely(__cma_bf_hole_insert_by_start(item))) { + /* + * We're screwed... Just free the item and forget + * about it. Things are broken beyond repair so no + * sense in trying to recover. + */ + kfree(item); + } else { + __cma_bf_hole_insert_by_size(item); + + /* Merge with prev and next sibling */ + __cma_bf_hole_merge_maybe(item); + } +} + + +/************************* Basic Tree Manipulation *************************/ + +static void __cma_bf_hole_insert_by_size(struct cma_bf_item *item) +{ + struct cma_bf_private *prv = item->ch.reg->private_data; + struct rb_node **link = &prv->by_size_root.rb_node, *parent = NULL; + const typeof(item->ch.size) value = item->ch.size; + + while (*link) { + struct cma_bf_item *i; + parent = *link; + i = rb_entry(parent, struct cma_bf_item, by_size); + link = value <= i->ch.size + ? &parent->rb_left + : &parent->rb_right; + } + + rb_link_node(&item->by_size, parent, link); + rb_insert_color(&item->by_size, &prv->by_size_root); +} + +static void __cma_bf_hole_erase_by_size(struct cma_bf_item *item) +{ + struct cma_bf_private *prv = item->ch.reg->private_data; + rb_erase(&item->by_size, &prv->by_size_root); +} + +static int __must_check +__cma_bf_hole_insert_by_start(struct cma_bf_item *item) +{ + struct cma_bf_private *prv = item->ch.reg->private_data; + struct rb_node **link = &prv->by_start_root.rb_node, *parent = NULL; + const typeof(item->ch.start) value = item->ch.start; + + while (*link) { + struct cma_bf_item *i; + parent = *link; + i = rb_entry(parent, struct cma_bf_item, ch.by_start); + + if (WARN_ON(value == i->ch.start)) + /* + * This should *never* happen. And I mean + * *never*. We could even BUG on it but + * hopefully things are only a bit broken, + * ie. system can still run. We produce + * a warning and return an error. + */ + return -EBUSY; + + link = value <= i->ch.start + ? &parent->rb_left + : &parent->rb_right; + } + + rb_link_node(&item->ch.by_start, parent, link); + rb_insert_color(&item->ch.by_start, &prv->by_start_root); + return 0; +} + +static void __cma_bf_hole_erase_by_start(struct cma_bf_item *item) +{ + struct cma_bf_private *prv = item->ch.reg->private_data; + rb_erase(&item->ch.by_start, &prv->by_start_root); +} + + +/************************* More Tree Manipulation *************************/ + +static struct cma_bf_item *__must_check +__cma_bf_hole_take(struct cma_bf_item *hole, size_t size, size_t alignment) +{ + struct cma_bf_item *item; + + /* + * There are three cases: + * 1. the chunk takes the whole hole, + * 2. the chunk is at the beginning or at the end of the hole, or + * 3. the chunk is in the middle of the hole. + */ + + + /* Case 1, the whole hole */ + if (size == hole->ch.size) { + __cma_bf_hole_erase_by_size(hole); + __cma_bf_hole_erase_by_start(hole); + return hole; + } + + + /* Allocate */ + item = kmalloc(sizeof *item, GFP_KERNEL); + if (unlikely(!item)) + return NULL; + + item->ch.start = ALIGN(hole->ch.start, alignment); + item->ch.size = size; + + /* Case 3, in the middle */ + if (item->ch.start != hole->ch.start + && item->ch.start + item->ch.size != + hole->ch.start + hole->ch.size) { + struct cma_bf_item *tail; + + /* + * Space between the end of the chunk and the end of + * the region, ie. space left after the end of the + * chunk. If this is dividable by alignment we can + * move the chunk to the end of the hole. + */ + size_t left = + hole->ch.start + hole->ch.size - + (item->ch.start + item->ch.size); + if (left % alignment == 0) { + item->ch.start += left; + goto case_2; + } + + /* + * We are going to add a hole at the end. This way, + * we will reduce the problem to case 2 -- the chunk + * will be at the end of the hole. + */ + tail = kmalloc(sizeof *tail, GFP_KERNEL); + if (unlikely(!tail)) { + kfree(item); + return NULL; + } + + tail->ch.start = item->ch.start + item->ch.size; + tail->ch.size = + hole->ch.start + hole->ch.size - tail->ch.start; + tail->ch.reg = hole->ch.reg; + + if (unlikely(__cma_bf_hole_insert_by_start(tail))) { + /* + * Things are broken beyond repair... Abort + * inserting the hole but still continue with + * allocation (seems like the best we can do). + */ + + hole->ch.size = tail->ch.start - hole->ch.start; + kfree(tail); + } else { + __cma_bf_hole_insert_by_size(tail); + /* + * It's important that we first insert the new + * hole in the tree sorted by size and later + * reduce the size of the old hole. We will + * update the position of the old hole in the + * rb tree in code that handles case 2. + */ + hole->ch.size = tail->ch.start - hole->ch.start; + } + + /* Go to case 2 */ + } + + + /* Case 2, at the beginning or at the end */ +case_2: + /* No need to update the tree; order preserved. */ + if (item->ch.start == hole->ch.start) + hole->ch.start += item->ch.size; + + /* Alter hole's size */ + hole->ch.size -= size; + __cma_bf_hole_erase_by_size(hole); + __cma_bf_hole_insert_by_size(hole); + + return item; +} + + +static void __cma_bf_hole_merge_maybe(struct cma_bf_item *item) +{ + struct cma_bf_item *prev; + struct rb_node *node; + int twice = 2; + + node = rb_prev(&item->ch.by_start); + if (unlikely(!node)) + goto next; + prev = rb_entry(node, struct cma_bf_item, ch.by_start); + + for (;;) { + if (prev->ch.start + prev->ch.size == item->ch.start) { + /* Remove previous hole from trees */ + __cma_bf_hole_erase_by_size(prev); + __cma_bf_hole_erase_by_start(prev); + + /* Alter this hole */ + item->ch.size += prev->ch.size; + item->ch.start = prev->ch.start; + __cma_bf_hole_erase_by_size(item); + __cma_bf_hole_insert_by_size(item); + /* + * No need to update by start trees as we do + * not break sequence order + */ + + /* Free prev hole */ + kfree(prev); + } + +next: + if (!--twice) + break; + + node = rb_next(&item->ch.by_start); + if (unlikely(!node)) + break; + prev = item; + item = rb_entry(node, struct cma_bf_item, ch.by_start); + } +} + + + +/************************* Register *************************/ +static int cma_bf_module_init(void) +{ + static struct cma_allocator alloc = { + .name = "bf", + .init = cma_bf_init, + .cleanup = cma_bf_cleanup, + .alloc = cma_bf_alloc, + .free = cma_bf_free, + }; + return cma_allocator_register(&alloc); +} +module_init(cma_bf_module_init); diff --git a/mm/cma.c b/mm/cma.c new file mode 100644 index 0000000..546dd86 --- /dev/null +++ b/mm/cma.c @@ -0,0 +1,1413 @@ +/* + * Contiguous Memory Allocator framework + * Copyright (c) 2010 by Samsung Electronics. + * Written by Michal Nazarewicz (m.nazarewicz@samsung.com) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License or (at your optional) any later version of the license. + */ + +/* + * See Documentation/contiguous-memory.txt for details. + */ + +#define pr_fmt(fmt) "cma: " fmt + +#ifdef CONFIG_CMA_DEBUG +# define DEBUG +#endif + +#ifndef CONFIG_NO_BOOTMEM +# include /* alloc_bootmem_pages_nopanic() */ +#endif +#ifdef CONFIG_HAVE_MEMBLOCK +# include /* memblock*() */ +#endif +#include /* struct device, dev_name() */ +#include /* Error numbers */ +#include /* IS_ERR, PTR_ERR, etc. */ +#include /* PAGE_ALIGN() */ +#include /* EXPORT_SYMBOL_GPL() */ +#include /* mutex */ +#include /* kmalloc() */ +#include /* str*() */ + +#include +#include + +/* + * Protects cma_regions, cma_allocators, cma_map, cma_map_length, + * cma_kobj, cma_sysfs_regions and cma_chunks_by_start. + */ +static DEFINE_MUTEX(cma_mutex); + + + +/************************* Map attribute *************************/ + +static const char *cma_map; +static size_t cma_map_length; + +/* + * map-attr ::= [ rules [ ';' ] ] + * rules ::= rule [ ';' rules ] + * rule ::= patterns '=' regions + * patterns ::= pattern [ ',' patterns ] + * regions ::= REG-NAME [ ',' regions ] + * pattern ::= dev-pattern [ '/' TYPE-NAME ] | '/' TYPE-NAME + * + * See Documentation/contiguous-memory.txt for details. + */ +static ssize_t cma_map_validate(const char *param) +{ + const char *ch = param; + + if (*ch == '\0' || *ch == '\n') + return 0; + + for (;;) { + const char *start = ch; + + while (*ch && *ch != '\n' && *ch != ';' && *ch != '=') + ++ch; + + if (*ch != '=' || start == ch) { + pr_err("map: expecting \"=\" near %s\n", + start); + return -EINVAL; + } + + while (*++ch != ';') + if (*ch == '\0' || *ch == '\n') + return ch - param; + if (ch[1] == '\0' || ch[1] == '\n') + return ch - param; + ++ch; + } +} + +static int __init cma_map_param(char *param) +{ + ssize_t len; + + pr_debug("param: map: %s\n", param); + + len = cma_map_validate(param); + if (len < 0) + return len; + + cma_map = param; + cma_map_length = len; + return 0; +} + +#if defined CONFIG_CMA_CMDLINE + +early_param("cma.map", cma_map_param); + +#endif + + + +/************************* Early regions *************************/ + +struct list_head cma_early_regions __initdata = + LIST_HEAD_INIT(cma_early_regions); + +#ifdef CONFIG_CMA_CMDLINE + +/* + * regions-attr ::= [ regions [ ';' ] ] + * regions ::= region [ ';' regions ] + * + * region ::= [ '-' ] reg-name + * '=' size + * [ '@' start ] + * [ '/' alignment ] + * [ ':' alloc-name ] + * + * See Documentation/contiguous-memory.txt for details. + * + * Example: + * cma=reg1=64M:bf;reg2=32M@0x100000:bf;reg3=64M/1M:bf + * + * If allocator is ommited the first available allocater will be used. + */ + +#define NUMPARSE(cond_ch, type, cond) ({ \ + unsigned long long v = 0; \ + if (*param == (cond_ch)) { \ + const char *const msg = param + 1; \ + v = memparse(msg, ¶m); \ + if (!v || v > ~(type)0 || !(cond)) { \ + pr_err("param: invalid value near %s\n", msg); \ + ret = -EINVAL; \ + break; \ + } \ + } \ + v; \ + }) + +static int __init cma_param_parse(char *param) +{ + static struct cma_region regions[16]; + + size_t left = ARRAY_SIZE(regions); + struct cma_region *reg = regions; + int ret = 0; + + pr_debug("param: %s\n", param); + + for (; *param; ++reg) { + dma_addr_t start, alignment; + size_t size; + + if (unlikely(!--left)) { + pr_err("param: too many early regions\n"); + return -ENOSPC; + } + + /* Parse name */ + reg->name = param; + param = strchr(param, '='); + if (!param || param == reg->name) { + pr_err("param: expected \"=\" near %s\n", + reg->name); + ret = -EINVAL; + break; + } + *param = '\0'; + + /* Parse numbers */ + size = NUMPARSE('\0', size_t, true); + start = NUMPARSE('@', dma_addr_t, true); + alignment = NUMPARSE('/', dma_addr_t, (v & (v - 1)) == 0); + + alignment = max(alignment, (dma_addr_t)PAGE_SIZE); + start = ALIGN(start, alignment); + size = PAGE_ALIGN(size); + if (start + size < start) { + pr_err("param: invalid start, size combination\n"); + ret = -EINVAL; + break; + } + + /* Parse allocator */ + if (*param == ':') { + reg->alloc_name = ++param; + while (*param && *param != ';') + ++param; + if (param == reg->alloc_name) + reg->alloc_name = NULL; + } + + /* Go to next */ + if (*param == ';') { + *param = '\0'; + ++param; + } else if (*param) { + pr_err("param: expecting ';' or end of parameter near %s\n", + param); + ret = -EINVAL; + break; + } + + /* Add */ + reg->size = size; + reg->start = start; + reg->alignment = alignment; + reg->copy_name = 1; + + list_add_tail(®->list, &cma_early_regions); + + pr_debug("param: registering early region %s (%p@%p/%p)\n", + reg->name, (void *)reg->size, (void *)reg->start, + (void *)reg->alignment); + } + + return ret; +} +early_param("cma", cma_param_parse); + +#undef NUMPARSE + +#endif + + +int __init __must_check cma_early_region_register(struct cma_region *reg) +{ + dma_addr_t start, alignment; + size_t size; + + if (reg->alignment & (reg->alignment - 1)) + return -EINVAL; + + alignment = max(reg->alignment, (dma_addr_t)PAGE_SIZE); + start = ALIGN(reg->start, alignment); + size = PAGE_ALIGN(reg->size); + + if (start + size < start) + return -EINVAL; + + reg->size = size; + reg->start = start; + reg->alignment = alignment; + + list_add_tail(®->list, &cma_early_regions); + + pr_debug("param: registering early region %s (%p@%p/%p)\n", + reg->name, (void *)reg->size, (void *)reg->start, + (void *)reg->alignment); + + return 0; +} + + + +/************************* Regions & Allocators *************************/ + +static void __cma_sysfs_region_add(struct cma_region *reg); + +static int __cma_region_attach_alloc(struct cma_region *reg); +static void __maybe_unused __cma_region_detach_alloc(struct cma_region *reg); + + +/* List of all regions. Named regions are kept before unnamed. */ +static LIST_HEAD(cma_regions); + +#define cma_foreach_region(reg) \ + list_for_each_entry(reg, &cma_regions, list) + +int __must_check cma_region_register(struct cma_region *reg) +{ + const char *name, *alloc_name; + struct cma_region *r; + char *ch = NULL; + int ret = 0; + + if (!reg->size || reg->start + reg->size < reg->start) + return -EINVAL; + + reg->users = 0; + reg->used = 0; + reg->private_data = NULL; + reg->registered = 0; + reg->free_space = reg->size; + + /* Copy name and alloc_name */ + name = reg->name; + alloc_name = reg->alloc_name; + if (reg->copy_name && (reg->name || reg->alloc_name)) { + size_t name_size, alloc_size; + + name_size = reg->name ? strlen(reg->name) + 1 : 0; + alloc_size = reg->alloc_name ? strlen(reg->alloc_name) + 1 : 0; + + ch = kmalloc(name_size + alloc_size, GFP_KERNEL); + if (!ch) { + pr_err("%s: not enough memory to allocate name\n", + reg->name ?: "(private)"); + return -ENOMEM; + } + + if (name_size) { + memcpy(ch, reg->name, name_size); + name = ch; + ch += name_size; + } + + if (alloc_size) { + memcpy(ch, reg->alloc_name, alloc_size); + alloc_name = ch; + } + } + + mutex_lock(&cma_mutex); + + /* Don't let regions overlap */ + cma_foreach_region(r) + if (r->start + r->size > reg->start && + r->start < reg->start + reg->size) { + ret = -EADDRINUSE; + goto done; + } + + if (reg->alloc) { + ret = __cma_region_attach_alloc(reg); + if (unlikely(ret < 0)) + goto done; + } + + reg->name = name; + reg->alloc_name = alloc_name; + reg->registered = 1; + ch = NULL; + + /* + * Keep named at the beginning and unnamed (private) at the + * end. This helps in traversal when named region is looked + * for. + */ + if (name) + list_add(®->list, &cma_regions); + else + list_add_tail(®->list, &cma_regions); + + __cma_sysfs_region_add(reg); + +done: + mutex_unlock(&cma_mutex); + + pr_debug("%s: region %sregistered\n", + reg->name ?: "(private)", ret ? "not " : ""); + kfree(ch); + + return ret; +} +EXPORT_SYMBOL_GPL(cma_region_register); + +static struct cma_region *__must_check +__cma_region_find(const char **namep) +{ + struct cma_region *reg; + const char *ch, *name; + size_t n; + + ch = *namep; + while (*ch && *ch != ',' && *ch != ';') + ++ch; + name = *namep; + *namep = *ch == ',' ? ch + 1 : ch; + n = ch - name; + + /* + * Named regions are kept in front of unnamed so if we + * encounter unnamed region we can stop. + */ + cma_foreach_region(reg) + if (!reg->name) + break; + else if (!strncmp(name, reg->name, n) && !reg->name[n]) + return reg; + + return NULL; +} + + +/* List of all allocators. */ +static LIST_HEAD(cma_allocators); + +#define cma_foreach_allocator(alloc) \ + list_for_each_entry(alloc, &cma_allocators, list) + +int cma_allocator_register(struct cma_allocator *alloc) +{ + struct cma_region *reg; + int first; + + if (!alloc->alloc || !alloc->free) + return -EINVAL; + + mutex_lock(&cma_mutex); + + first = list_empty(&cma_allocators); + + list_add_tail(&alloc->list, &cma_allocators); + + /* + * Attach this allocator to all allocator-less regions that + * request this particular allocator (reg->alloc_name equals + * alloc->name) or if region wants the first available + * allocator and we are the first. + */ + cma_foreach_region(reg) { + if (reg->alloc) + continue; + if (reg->alloc_name + ? alloc->name && !strcmp(alloc->name, reg->alloc_name) + : (!reg->used && first)) + continue; + + reg->alloc = alloc; + __cma_region_attach_alloc(reg); + } + + mutex_unlock(&cma_mutex); + + pr_debug("%s: allocator registered\n", alloc->name ?: "(unnamed)"); + + return 0; +} +EXPORT_SYMBOL_GPL(cma_allocator_register); + +static struct cma_allocator *__must_check +__cma_allocator_find(const char *name) +{ + struct cma_allocator *alloc; + + if (!name) + return list_empty(&cma_allocators) + ? NULL + : list_entry(cma_allocators.next, + struct cma_allocator, list); + + cma_foreach_allocator(alloc) + if (alloc->name && !strcmp(name, alloc->name)) + return alloc; + + return NULL; +} + + + +/************************* Initialise CMA *************************/ + +int __init cma_set_defaults(struct cma_region *regions, const char *map) +{ + if (map) { + int ret = cma_map_param((char *)map); + if (unlikely(ret < 0)) + return ret; + } + + if (!regions) + return 0; + + for (; regions->size; ++regions) { + int ret = cma_early_region_register(regions); + if (unlikely(ret < 0)) + return ret; + } + + return 0; +} + + +int __init cma_early_region_reserve(struct cma_region *reg) +{ + int tried = 0; + + if (!reg->size || (reg->alignment & (reg->alignment - 1)) || + reg->reserved) + return -EINVAL; + +#ifndef CONFIG_NO_BOOTMEM + + tried = 1; + + { + void *ptr = __alloc_bootmem_nopanic(reg->size, reg->alignment, + reg->start); + if (ptr) { + reg->start = virt_to_phys(ptr); + reg->reserved = 1; + return 0; + } + } + +#endif + +#ifdef CONFIG_HAVE_MEMBLOCK + + tried = 1; + + if (reg->start) { + if (!memblock_is_region_reserved(reg->start, reg->size) && + memblock_reserve(reg->start, reg->size) >= 0) { + reg->reserved = 1; + return 0; + } + } else { + /* + * Use __memblock_alloc_base() since + * memblock_alloc_base() panic()s. + */ + u64 ret = __memblock_alloc_base(reg->size, reg->alignment, 0); + if (ret && + ret < ~(dma_addr_t)0 && + ret + reg->size < ~(dma_addr_t)0 && + ret + reg->size > ret) { + reg->start = ret; + reg->reserved = 1; + return 0; + } + + if (ret) + memblock_free(ret, reg->size); + } + +#endif + + return tried ? -ENOMEM : -EOPNOTSUPP; +} + +void __init cma_early_regions_reserve(int (*reserve)(struct cma_region *reg)) +{ + struct cma_region *reg; + + pr_debug("init: reserving early regions\n"); + + if (!reserve) + reserve = cma_early_region_reserve; + + list_for_each_entry(reg, &cma_early_regions, list) { + if (reg->reserved) { + /* nothing */ + } else if (reserve(reg) >= 0) { + pr_debug("init: %s: reserved %p@%p\n", + reg->name ?: "(private)", + (void *)reg->size, (void *)reg->start); + reg->reserved = 1; + } else { + pr_warn("init: %s: unable to reserve %p@%p/%p\n", + reg->name ?: "(private)", + (void *)reg->size, (void *)reg->start, + (void *)reg->alignment); + } + } +} + + +static int __init cma_init(void) +{ + struct cma_region *reg, *n; + + pr_debug("init: initialising\n"); + + if (cma_map) { + char *val = kmemdup(cma_map, cma_map_length + 1, GFP_KERNEL); + cma_map = val; + if (!val) + return -ENOMEM; + val[cma_map_length] = '\0'; + } + + list_for_each_entry_safe(reg, n, &cma_early_regions, list) { + INIT_LIST_HEAD(®->list); + /* + * We don't care if there was an error. It's a pity + * but there's not much we can do about it any way. + * If the error is on a region that was parsed from + * command line then it will stay and waste a bit of + * space; if it was registered using + * cma_early_region_register() it's caller's + * responsibility to do something about it. + */ + if (reg->reserved && cma_region_register(reg) < 0) + /* ignore error */; + } + + INIT_LIST_HEAD(&cma_early_regions); + + return 0; +} +/* + * We want to be initialised earlier than module_init/__initcall so + * that drivers that want to grab memory at boot time will get CMA + * ready. subsys_initcall() seems early enough and not too early at + * the same time. + */ +subsys_initcall(cma_init); + + + +/************************* SysFS *************************/ + +#if defined CONFIG_CMA_SYSFS + +static struct kobject cma_sysfs_regions; +static int cma_sysfs_regions_ready; + + +#define CMA_ATTR_INLINE(_type, _name) \ + (&((struct cma_ ## _type ## _attribute){ \ + .attr = { \ + .name = __stringify(_name), \ + .mode = 0644, \ + }, \ + .show = cma_sysfs_ ## _type ## _ ## _name ## _show, \ + .store = cma_sysfs_ ## _type ## _ ## _name ## _store, \ + }).attr) + +#define CMA_ATTR_RO_INLINE(_type, _name) \ + (&((struct cma_ ## _type ## _attribute){ \ + .attr = { \ + .name = __stringify(_name), \ + .mode = 0444, \ + }, \ + .show = cma_sysfs_ ## _type ## _ ## _name ## _show, \ + }).attr) + + +struct cma_root_attribute { + struct attribute attr; + ssize_t (*show)(char *buf); + int (*store)(const char *buf); +}; + +static ssize_t cma_sysfs_root_map_show(char *page) +{ + ssize_t len; + + len = cma_map_length; + if (!len) { + *page = 0; + len = 0; + } else { + if (len > (size_t)PAGE_SIZE - 1) + len = (size_t)PAGE_SIZE - 1; + memcpy(page, cma_map, len); + page[len++] = '\n'; + } + + return len; +} + +static int cma_sysfs_root_map_store(const char *page) +{ + ssize_t len = cma_map_validate(page); + char *val = NULL; + + if (len < 0) + return len; + + if (len) { + val = kmemdup(page, len + 1, GFP_KERNEL); + if (!val) + return -ENOMEM; + val[len] = '\0'; + } + + kfree(cma_map); + cma_map = val; + cma_map_length = len; + + return 0; +} + +static ssize_t cma_sysfs_root_allocators_show(char *page) +{ + struct cma_allocator *alloc; + size_t left = PAGE_SIZE; + char *ch = page; + + cma_foreach_allocator(alloc) { + ssize_t l = snprintf(ch, left, "%s ", alloc->name ?: "-"); + ch += l; + left -= l; + } + + if (ch != page) + ch[-1] = '\n'; + return ch - page; +} + +static ssize_t +cma_sysfs_root_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + struct cma_root_attribute *rattr = + container_of(attr, struct cma_root_attribute, attr); + ssize_t ret; + + mutex_lock(&cma_mutex); + ret = rattr->show(buf); + mutex_unlock(&cma_mutex); + + return ret; +} + +static ssize_t +cma_sysfs_root_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct cma_root_attribute *rattr = + container_of(attr, struct cma_root_attribute, attr); + int ret; + + mutex_lock(&cma_mutex); + ret = rattr->store(buf); + mutex_unlock(&cma_mutex); + + return ret < 0 ? ret : count; +} + +static struct kobj_type cma_sysfs_root_type = { + .sysfs_ops = &(const struct sysfs_ops){ + .show = cma_sysfs_root_show, + .store = cma_sysfs_root_store, + }, + .default_attrs = (struct attribute * []) { + CMA_ATTR_INLINE(root, map), + CMA_ATTR_RO_INLINE(root, allocators), + NULL + }, +}; + +static int __init cma_sysfs_init(void) +{ + static struct kobject root; + static struct kobj_type fake_type; + + struct cma_region *reg; + int ret; + + /* Root */ + ret = kobject_init_and_add(&root, &cma_sysfs_root_type, + mm_kobj, "contiguous"); + if (unlikely(ret < 0)) { + pr_err("init: unable to add root kobject: %d\n", ret); + return ret; + } + + /* Regions */ + ret = kobject_init_and_add(&cma_sysfs_regions, &fake_type, + &root, "regions"); + if (unlikely(ret < 0)) { + pr_err("init: unable to add regions kobject: %d\n", ret); + return ret; + } + + mutex_lock(&cma_mutex); + cma_sysfs_regions_ready = 1; + cma_foreach_region(reg) + __cma_sysfs_region_add(reg); + mutex_unlock(&cma_mutex); + + return 0; +} +device_initcall(cma_sysfs_init); + + + +struct cma_region_attribute { + struct attribute attr; + ssize_t (*show)(struct cma_region *reg, char *buf); + int (*store)(struct cma_region *reg, const char *buf); +}; + + +static ssize_t cma_sysfs_region_name_show(struct cma_region *reg, char *page) +{ + return reg->name ? snprintf(page, PAGE_SIZE, "%s\n", reg->name) : 0; +} + +static ssize_t cma_sysfs_region_start_show(struct cma_region *reg, char *page) +{ + return snprintf(page, PAGE_SIZE, "%p\n", (void *)reg->start); +} + +static ssize_t cma_sysfs_region_size_show(struct cma_region *reg, char *page) +{ + return snprintf(page, PAGE_SIZE, "%zu\n", reg->size); +} + +static ssize_t cma_sysfs_region_free_show(struct cma_region *reg, char *page) +{ + return snprintf(page, PAGE_SIZE, "%zu\n", reg->free_space); +} + +static ssize_t cma_sysfs_region_users_show(struct cma_region *reg, char *page) +{ + return snprintf(page, PAGE_SIZE, "%u\n", reg->users); +} + +static ssize_t cma_sysfs_region_alloc_show(struct cma_region *reg, char *page) +{ + if (reg->alloc) + return snprintf(page, PAGE_SIZE, "%s\n", + reg->alloc->name ?: "-"); + else if (reg->alloc_name) + return snprintf(page, PAGE_SIZE, "[%s]\n", reg->alloc_name); + else + return 0; +} + +static int +cma_sysfs_region_alloc_store(struct cma_region *reg, const char *page) +{ + char *s; + + if (reg->alloc && reg->users) + return -EBUSY; + + if (!*page || *page == '\n') { + s = NULL; + } else { + size_t len; + + for (s = (char *)page; *++s && *s != '\n'; ) + /* nop */; + + len = s - page; + s = kmemdup(page, len + 1, GFP_KERNEL); + if (!s) + return -ENOMEM; + s[len] = '\0'; + } + + if (reg->alloc) + __cma_region_detach_alloc(reg); + + if (reg->free_alloc_name) + kfree(reg->alloc_name); + + reg->alloc_name = s; + reg->free_alloc_name = !!s; + + return 0; +} + + +static ssize_t +cma_sysfs_region_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct cma_region *reg = container_of(kobj, struct cma_region, kobj); + struct cma_region_attribute *rattr = + container_of(attr, struct cma_region_attribute, attr); + ssize_t ret; + + mutex_lock(&cma_mutex); + ret = rattr->show(reg, buf); + mutex_unlock(&cma_mutex); + + return ret; +} + +static int +cma_sysfs_region_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + struct cma_region *reg = container_of(kobj, struct cma_region, kobj); + struct cma_region_attribute *rattr = + container_of(attr, struct cma_region_attribute, attr); + int ret; + + mutex_lock(&cma_mutex); + ret = rattr->store(reg, buf); + mutex_unlock(&cma_mutex); + + return ret < 0 ? ret : count; +} + +static struct kobj_type cma_sysfs_region_type = { + .sysfs_ops = &(const struct sysfs_ops){ + .show = cma_sysfs_region_show, + .store = cma_sysfs_region_store, + }, + .default_attrs = (struct attribute * []) { + CMA_ATTR_RO_INLINE(region, name), + CMA_ATTR_RO_INLINE(region, start), + CMA_ATTR_RO_INLINE(region, size), + CMA_ATTR_RO_INLINE(region, free), + CMA_ATTR_RO_INLINE(region, users), + CMA_ATTR_INLINE(region, alloc), + NULL + }, +}; + +static void __cma_sysfs_region_add(struct cma_region *reg) +{ + int ret; + + if (!cma_sysfs_regions_ready) + return; + + memset(®->kobj, 0, sizeof reg->kobj); + + ret = kobject_init_and_add(®->kobj, &cma_sysfs_region_type, + &cma_sysfs_regions, + "%p", (void *)reg->start); + + if (reg->name && + sysfs_create_link(&cma_sysfs_regions, ®->kobj, reg->name) < 0) + /* Ignore any errors. */; +} + +#else + +static void __cma_sysfs_region_add(struct cma_region *reg) +{ + /* nop */ +} + +#endif + + +/************************* Chunks *************************/ + +/* All chunks sorted by start address. */ +static struct rb_root cma_chunks_by_start; + +static struct cma_chunk *__must_check __cma_chunk_find(dma_addr_t addr) +{ + struct cma_chunk *chunk; + struct rb_node *n; + + for (n = cma_chunks_by_start.rb_node; n; ) { + chunk = rb_entry(n, struct cma_chunk, by_start); + if (addr < chunk->start) + n = n->rb_left; + else if (addr > chunk->start) + n = n->rb_right; + else + return chunk; + } + WARN(1, KERN_WARNING "no chunk starting at %p\n", (void *)addr); + return NULL; +} + +static int __must_check __cma_chunk_insert(struct cma_chunk *chunk) +{ + struct rb_node **new, *parent = NULL; + typeof(chunk->start) addr = chunk->start; + + for (new = &cma_chunks_by_start.rb_node; *new; ) { + struct cma_chunk *c = + container_of(*new, struct cma_chunk, by_start); + + parent = *new; + if (addr < c->start) { + new = &(*new)->rb_left; + } else if (addr > c->start) { + new = &(*new)->rb_right; + } else { + /* + * We should never be here. If we are it + * means allocator gave us an invalid chunk + * (one that has already been allocated) so we + * refuse to accept it. Our caller will + * recover by freeing the chunk. + */ + WARN_ON(1); + return -EADDRINUSE; + } + } + + rb_link_node(&chunk->by_start, parent, new); + rb_insert_color(&chunk->by_start, &cma_chunks_by_start); + + return 0; +} + +static void __cma_chunk_free(struct cma_chunk *chunk) +{ + rb_erase(&chunk->by_start, &cma_chunks_by_start); + + chunk->reg->free_space += chunk->size; + --chunk->reg->users; + + chunk->reg->alloc->free(chunk); +} + + +/************************* The Device API *************************/ + +static const char *__must_check +__cma_where_from(const struct device *dev, const char *type); + + +/* Allocate. */ + +static dma_addr_t __must_check +__cma_alloc_from_region(struct cma_region *reg, + size_t size, dma_addr_t alignment) +{ + struct cma_chunk *chunk; + + pr_debug("allocate %p/%p from %s\n", + (void *)size, (void *)alignment, + reg ? reg->name ?: "(private)" : "(null)"); + + if (!reg || reg->free_space < size) + return -ENOMEM; + + if (!reg->alloc) { + if (!reg->used) + __cma_region_attach_alloc(reg); + if (!reg->alloc) + return -ENOMEM; + } + + chunk = reg->alloc->alloc(reg, size, alignment); + if (!chunk) + return -ENOMEM; + + if (unlikely(__cma_chunk_insert(chunk) < 0)) { + /* We should *never* be here. */ + chunk->reg->alloc->free(chunk); + kfree(chunk); + return -EADDRINUSE; + } + + chunk->reg = reg; + ++reg->users; + reg->free_space -= chunk->size; + pr_debug("allocated at %p\n", (void *)chunk->start); + return chunk->start; +} + +dma_addr_t __must_check +cma_alloc_from_region(struct cma_region *reg, + size_t size, dma_addr_t alignment) +{ + dma_addr_t addr; + + pr_debug("allocate %p/%p from %s\n", + (void *)size, (void *)alignment, + reg ? reg->name ?: "(private)" : "(null)"); + + if (!size || alignment & (alignment - 1) || !reg) + return -EINVAL; + + mutex_lock(&cma_mutex); + + addr = reg->registered ? + __cma_alloc_from_region(reg, PAGE_ALIGN(size), + max(alignment, (dma_addr_t)PAGE_SIZE)) : + -EINVAL; + + mutex_unlock(&cma_mutex); + + return addr; +} +EXPORT_SYMBOL_GPL(cma_alloc_from_region); + +dma_addr_t __must_check +__cma_alloc(const struct device *dev, const char *type, + dma_addr_t size, dma_addr_t alignment) +{ + struct cma_region *reg; + const char *from; + dma_addr_t addr; + + if (dev) + pr_debug("allocate %p/%p for %s/%s\n", + (void *)size, (void *)alignment, + dev_name(dev), type ?: ""); + + if (!size || (alignment & ~alignment)) + return -EINVAL; + + if (alignment < PAGE_SIZE) + alignment = PAGE_SIZE; + + if (!IS_ALIGNED(size, alignment)) + size = ALIGN(size, alignment); + + mutex_lock(&cma_mutex); + + from = __cma_where_from(dev, type); + if (unlikely(IS_ERR(from))) { + addr = PTR_ERR(from); + goto done; + } + + pr_debug("allocate %p/%p from one of %s\n", + (void *)size, (void *)alignment, from); + + while (*from && *from != ';') { + reg = __cma_region_find(&from); + addr = __cma_alloc_from_region(reg, size, alignment); + if (!IS_ERR_VALUE(addr)) + goto done; + } + + pr_debug("not enough memory\n"); + addr = -ENOMEM; + +done: + mutex_unlock(&cma_mutex); + + return addr; +} +EXPORT_SYMBOL_GPL(__cma_alloc); + + +void *cma_get_virt(dma_addr_t phys, dma_addr_t size, int noncached) +{ + unsigned long num_pages, i; + struct page **pages; + void *virt; + + if (noncached) { + num_pages = size >> PAGE_SHIFT; + pages = kmalloc(num_pages * sizeof(struct page *), GFP_KERNEL); + + if (!pages) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < num_pages; i++) + pages[i] = pfn_to_page((phys >> PAGE_SHIFT) + i); + + virt = vmap(pages, num_pages, VM_MAP, + pgprot_writecombine(PAGE_KERNEL)); + + if (!virt) { + kfree(pages); + return ERR_PTR(-ENOMEM); + } + + kfree(pages); + } else { + virt = phys_to_virt((unsigned long)phys); + } + + return virt; +} +EXPORT_SYMBOL_GPL(cma_get_virt); + +/* Query information about regions. */ +static void __cma_info_add(struct cma_info *infop, struct cma_region *reg) +{ + infop->total_size += reg->size; + infop->free_size += reg->free_space; + if (infop->lower_bound > reg->start) + infop->lower_bound = reg->start; + if (infop->upper_bound < reg->start + reg->size) + infop->upper_bound = reg->start + reg->size; + ++infop->count; +} + +int +__cma_info(struct cma_info *infop, const struct device *dev, const char *type) +{ + struct cma_info info = { ~(dma_addr_t)0, 0, 0, 0, 0 }; + struct cma_region *reg; + const char *from; + int ret; + + if (unlikely(!infop)) + return -EINVAL; + + mutex_lock(&cma_mutex); + + from = __cma_where_from(dev, type); + if (IS_ERR(from)) { + ret = PTR_ERR(from); + info.lower_bound = 0; + goto done; + } + + while (*from && *from != ';') { + reg = __cma_region_find(&from); + if (reg) + __cma_info_add(&info, reg); + } + + ret = 0; +done: + mutex_unlock(&cma_mutex); + + memcpy(infop, &info, sizeof info); + return ret; +} +EXPORT_SYMBOL_GPL(__cma_info); + + +/* Freeing. */ +int cma_free(dma_addr_t addr) +{ + struct cma_chunk *c; + int ret; + + mutex_lock(&cma_mutex); + + c = __cma_chunk_find(addr); + + if (c) { + __cma_chunk_free(c); + ret = 0; + } else { + ret = -ENOENT; + } + + mutex_unlock(&cma_mutex); + + if (c) + pr_debug("free(%p): freed\n", (void *)addr); + else + pr_err("free(%p): not found\n", (void *)addr); + return ret; +} +EXPORT_SYMBOL_GPL(cma_free); + + +/************************* Miscellaneous *************************/ + +static int __cma_region_attach_alloc(struct cma_region *reg) +{ + struct cma_allocator *alloc; + int ret; + + /* + * If reg->alloc is set then caller wants us to use this + * allocator. Otherwise we need to find one by name. + */ + if (reg->alloc) { + alloc = reg->alloc; + } else { + alloc = __cma_allocator_find(reg->alloc_name); + if (!alloc) { + pr_warn("init: %s: %s: no such allocator\n", + reg->name ?: "(private)", + reg->alloc_name ?: "(default)"); + reg->used = 1; + return -ENOENT; + } + } + + /* Try to initialise the allocator. */ + reg->private_data = NULL; + ret = alloc->init ? alloc->init(reg) : 0; + if (unlikely(ret < 0)) { + pr_err("init: %s: %s: unable to initialise allocator\n", + reg->name ?: "(private)", alloc->name ?: "(unnamed)"); + reg->alloc = NULL; + reg->used = 1; + } else { + reg->alloc = alloc; + pr_debug("init: %s: %s: initialised allocator\n", + reg->name ?: "(private)", alloc->name ?: "(unnamed)"); + } + return ret; +} + +static void __cma_region_detach_alloc(struct cma_region *reg) +{ + if (!reg->alloc) + return; + + if (reg->alloc->cleanup) + reg->alloc->cleanup(reg); + + reg->alloc = NULL; + reg->used = 1; +} + + +/* + * s ::= rules + * rules ::= rule [ ';' rules ] + * rule ::= patterns '=' regions + * patterns ::= pattern [ ',' patterns ] + * regions ::= REG-NAME [ ',' regions ] + * pattern ::= dev-pattern [ '/' TYPE-NAME ] | '/' TYPE-NAME + */ +static const char *__must_check +__cma_where_from(const struct device *dev, const char *type) +{ + /* + * This function matches the pattern from the map attribute + * agains given device name and type. Type may be of course + * NULL or an emtpy string. + */ + + const char *s, *name; + int name_matched = 0; + + /* + * If dev is NULL we were called in alternative form where + * type is the from string. All we have to do is return it. + */ + if (!dev) + return type ?: ERR_PTR(-EINVAL); + + if (!cma_map) + return ERR_PTR(-ENOENT); + + name = dev_name(dev); + if (WARN_ON(!name || !*name)) + return ERR_PTR(-EINVAL); + + if (!type) + type = "common"; + + /* + * Now we go throught the cma_map attribute. + */ + for (s = cma_map; *s; ++s) { + const char *c; + + /* + * If the pattern starts with a slash, the device part of the + * pattern matches if it matched previously. + */ + if (*s == '/') { + if (!name_matched) + goto look_for_next; + goto match_type; + } + + /* + * We are now trying to match the device name. This also + * updates the name_matched variable. If, while reading the + * spec, we ecnounter comma it means that the pattern does not + * match and we need to start over with another pattern (the + * one afther the comma). If we encounter equal sign we need + * to start over with another rule. If there is a character + * that does not match, we neet to look for a comma (to get + * another pattern) or semicolon (to get another rule) and try + * again if there is one somewhere. + */ + + name_matched = 0; + + for (c = name; *s != '*' && *c; ++c, ++s) + if (*s == '=') + goto next_rule; + else if (*s == ',') + goto next_pattern; + else if (*s != '?' && *c != *s) + goto look_for_next; + if (*s == '*') + ++s; + + name_matched = 1; + + /* + * Now we need to match the type part of the pattern. If the + * pattern is missing it we match only if type points to an + * empty string. Otherwise wy try to match it just like name. + */ + if (*s == '/') { +match_type: /* s points to '/' */ + ++s; + + for (c = type; *s && *c; ++c, ++s) + if (*s == '=') + goto next_rule; + else if (*s == ',') + goto next_pattern; + else if (*c != *s) + goto look_for_next; + } + + /* Return the string behind the '=' sign of the rule. */ + if (*s == '=') + return s + 1; + else if (*s == ',') + return strchr(s, '=') + 1; + + /* Pattern did not match */ + +look_for_next: + do { + ++s; + } while (*s != ',' && *s != '='); + if (*s == ',') + continue; + +next_rule: /* s points to '=' */ + s = strchr(s, ';'); + if (!s) + break; + +next_pattern: + continue; + } + + return ERR_PTR(-ENOENT); +} diff --git a/mm/compaction.c b/mm/compaction.c index 6cc604b..9412abf 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -635,6 +635,10 @@ unsigned long try_to_compact_pages(struct zonelist *zonelist, if (!order || !may_enter_fs || !may_perform_io) return rc; +#ifdef CONFIG_MACH_Q1_BD + /* Temporary log to get information whether the compaction works well */ + printk(KERN_NOTICE "%s, order=%d, sync=%d\n", __func__, order, sync); +#endif count_vm_event(COMPACTSTALL); /* Compact each zone in the list */ diff --git a/mm/filemap.c b/mm/filemap.c index a8251a8..0245651 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1393,15 +1393,12 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, unsigned long seg = 0; size_t count; loff_t *ppos = &iocb->ki_pos; - struct blk_plug plug; count = 0; retval = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE); if (retval) return retval; - blk_start_plug(&plug); - /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */ if (filp->f_flags & O_DIRECT) { loff_t size; @@ -1417,8 +1414,12 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, retval = filemap_write_and_wait_range(mapping, pos, pos + iov_length(iov, nr_segs) - 1); if (!retval) { + struct blk_plug plug; + + blk_start_plug(&plug); retval = mapping->a_ops->direct_IO(READ, iocb, iov, pos, nr_segs); + blk_finish_plug(&plug); } if (retval > 0) { *ppos = pos + retval; @@ -1474,7 +1475,6 @@ generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov, break; } out: - blk_finish_plug(&plug); return retval; } EXPORT_SYMBOL(generic_file_aio_read); diff --git a/mm/memory.c b/mm/memory.c index 95a7799..27b7df1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -177,6 +177,7 @@ unsigned long get_mm_counter(struct mm_struct *mm, int member) return 0; return (unsigned long)val; } +EXPORT_SYMBOL(get_mm_counter); void sync_mm_rss(struct task_struct *task, struct mm_struct *mm) { diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 8439d2a..4d8f48c 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -78,6 +78,8 @@ DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ EXPORT_PER_CPU_SYMBOL(_numa_mem_); #endif +struct rw_semaphore page_alloc_slow_rwsem; + /* * Array of node states. */ @@ -127,6 +129,20 @@ void pm_restrict_gfp_mask(void) saved_gfp_mask = gfp_allowed_mask; gfp_allowed_mask &= ~GFP_IOFS; } + +static bool pm_suspending(void) +{ + if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS) + return false; + return true; +} + +#else + +static bool pm_suspending(void) +{ + return false; +} #endif /* CONFIG_PM_SLEEP */ #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE @@ -176,6 +192,7 @@ static char * const zone_names[MAX_NR_ZONES] = { }; int min_free_kbytes = 1024; +int min_free_order_shift = 1; static unsigned long __meminitdata nr_kernel_pages; static unsigned long __meminitdata nr_all_pages; @@ -1487,7 +1504,7 @@ static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark, free_pages -= z->free_area[o].nr_free << o; /* Require fewer higher order pages to be free */ - min >>= 1; + min >>= min_free_order_shift; if (free_pages <= min) return false; @@ -2095,7 +2112,9 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, unsigned long pages_reclaimed = 0; unsigned long did_some_progress; bool sync_migration = false; - +#ifdef CONFIG_ANDROID_WIP + unsigned long start_tick = jiffies; +#endif /* * In the slowpath, we sanity check order to avoid ever trying to * reclaim >= MAX_ORDER areas which will never succeed. Callers may @@ -2107,6 +2126,9 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, return NULL; } + if (gfp_mask & __GFP_WAIT) + down_read(&page_alloc_slow_rwsem); + /* * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and * __GFP_NOWARN set) should not cause reclaim since the subsystem @@ -2193,8 +2215,14 @@ rebalance: /* * If we failed to make any progress reclaiming, then we are * running out of options and have to consider going OOM + * ANDROID_WIP: If we are looping more than 1 second, consider OOM */ - if (!did_some_progress) { +#ifdef CONFIG_ANDROID_WIP +#define SHOULD_CONSIDER_OOM !did_some_progress || time_after(jiffies, start_tick + HZ) +#else +#define SHOULD_CONSIDER_OOM !did_some_progress +#endif + if (SHOULD_CONSIDER_OOM) { if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) { if (oom_killer_disabled) goto nopage; @@ -2225,6 +2253,14 @@ rebalance: goto restart; } + + /* + * Suspend converts GFP_KERNEL to __GFP_WAIT which can + * prevent reclaim making forward progress without + * invoking OOM. Bail if we are suspending + */ + if (pm_suspending()) + goto nopage; } /* Check if we should retry the allocation */ @@ -2251,10 +2287,14 @@ rebalance: nopage: warn_alloc_failed(gfp_mask, order, NULL); + if (gfp_mask & __GFP_WAIT) + up_read(&page_alloc_slow_rwsem); return page; got_pg: if (kmemcheck_enabled) kmemcheck_pagealloc_alloc(page, order, gfp_mask); + if (gfp_mask & __GFP_WAIT) + up_read(&page_alloc_slow_rwsem); return page; } @@ -5010,6 +5050,7 @@ static int page_alloc_cpu_notify(struct notifier_block *self, void __init page_alloc_init(void) { hotcpu_notifier(page_alloc_cpu_notify, 0); + init_rwsem(&page_alloc_slow_rwsem); } /* diff --git a/mm/shmem.c b/mm/shmem.c index fcedf54..883e98f 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -3015,6 +3015,15 @@ put_memory: } EXPORT_SYMBOL_GPL(shmem_file_setup); +void shmem_set_file(struct vm_area_struct *vma, struct file *file) +{ + if (vma->vm_file) + fput(vma->vm_file); + vma->vm_file = file; + vma->vm_ops = &shmem_vm_ops; + vma->vm_flags |= VM_CAN_NONLINEAR; +} + /** * shmem_zero_setup - setup a shared anonymous mapping * @vma: the vma to be mmapped is prepared by do_mmap_pgoff @@ -3028,11 +3037,7 @@ int shmem_zero_setup(struct vm_area_struct *vma) if (IS_ERR(file)) return PTR_ERR(file); - if (vma->vm_file) - fput(vma->vm_file); - vma->vm_file = file; - vma->vm_ops = &shmem_vm_ops; - vma->vm_flags |= VM_CAN_NONLINEAR; + shmem_set_file(vma, file); return 0; } diff --git a/mm/slub.c b/mm/slub.c index 35f351f..e609f0c 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -529,6 +529,9 @@ static void object_err(struct kmem_cache *s, struct page *page, { slab_bug(s, "%s", reason); print_trailer(s, page, object); + + if(slub_debug) + panic("SLUB ERROR: object_err"); } static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...) @@ -542,6 +545,9 @@ static void slab_err(struct kmem_cache *s, struct page *page, char *fmt, ...) slab_bug(s, "%s", buf); print_page_info(page); dump_stack(); + + if(slub_debug) + panic("SLUB ERROR: slab_err"); } static void init_object(struct kmem_cache *s, void *object, u8 val) @@ -596,6 +602,10 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page, print_trailer(s, page, object); restore_bytes(s, what, value, fault, end); + + if(slub_debug) + panic("SLUB ERROR: check_bytes_and_report. Can it be restored?"); + return 0; } diff --git a/mm/truncate.c b/mm/truncate.c index e13f22e..ca54477 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -138,12 +138,18 @@ invalidate_complete_page(struct address_space *mapping, struct page *page) return ret; } +#ifdef CONFIG_MACH_P4NOTE +static int unmap_mapcount = -99; +#endif int truncate_inode_page(struct address_space *mapping, struct page *page) { if (page_mapped(page)) { unmap_mapping_range(mapping, (loff_t)page->index << PAGE_CACHE_SHIFT, PAGE_CACHE_SIZE, 0); +#ifdef CONFIG_MACH_P4NOTE + unmap_mapcount = atomic_read(&(page)->_mapcount); +#endif } return truncate_complete_page(mapping, page); } diff --git a/mm/vmscan.c b/mm/vmscan.c index 6072d74..d14eda6 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1939,12 +1939,14 @@ static void shrink_zone(int priority, struct zone *zone, enum lru_list l; unsigned long nr_reclaimed, nr_scanned; unsigned long nr_to_reclaim = sc->nr_to_reclaim; + struct blk_plug plug; restart: nr_reclaimed = 0; nr_scanned = sc->nr_scanned; get_scan_count(zone, sc, nr, priority); + blk_start_plug(&plug); while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] || nr[LRU_INACTIVE_FILE]) { for_each_evictable_lru(l) { @@ -1968,6 +1970,7 @@ restart: if (nr_reclaimed >= nr_to_reclaim && priority < DEF_PRIORITY) break; } + blk_finish_plug(&plug); sc->nr_reclaimed += nr_reclaimed; /* -- cgit v1.1