From 7303f240981888884412a97ac742772527356880 Mon Sep 17 00:00:00 2001 From: Wu Fengguang Date: Mon, 11 May 2009 09:59:34 +0300 Subject: slob: use PG_slab for identifying SLOB pages For the sake of consistency. Signed-off-by: Wu Fengguang Cc: KOSAKI Motohiro Cc: Andi Kleen Acked-by: Matt Mackall Cc: Alexey Dobriyan Cc: Ingo Molnar Cc: Christoph Lameter Signed-off-by: Andrew Morton Signed-off-by: Pekka Enberg --- include/linux/page-flags.h | 2 -- mm/slob.c | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 62214c7..f036dfb 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -120,7 +120,6 @@ enum pageflags { PG_savepinned = PG_dirty, /* SLOB */ - PG_slob_page = PG_active, PG_slob_free = PG_private, /* SLUB */ @@ -203,7 +202,6 @@ PAGEFLAG(SavePinned, savepinned); /* Xen */ PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved) PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked) -__PAGEFLAG(SlobPage, slob_page) __PAGEFLAG(SlobFree, slob_free) __PAGEFLAG(SlubFrozen, slub_frozen) diff --git a/mm/slob.c b/mm/slob.c index a2d4ab3..aad9dad 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -131,17 +131,17 @@ static LIST_HEAD(free_slob_large); */ static inline int is_slob_page(struct slob_page *sp) { - return PageSlobPage((struct page *)sp); + return PageSlab((struct page *)sp); } static inline void set_slob_page(struct slob_page *sp) { - __SetPageSlobPage((struct page *)sp); + __SetPageSlab((struct page *)sp); } static inline void clear_slob_page(struct slob_page *sp) { - __ClearPageSlobPage((struct page *)sp); + __ClearPageSlab((struct page *)sp); } static inline struct slob_page *slob_page(const void *addr) -- cgit v1.1 From 6746136520cd0827320a83e62d0a023a5a433650 Mon Sep 17 00:00:00 2001 From: Ron Lee Date: Fri, 22 May 2009 04:58:22 +0930 Subject: slab: fix generic PAGE_POISONING conflict with SLAB_RED_ZONE A generic page poisoning mechanism was added with commit: 6a11f75b6a17b5d9ac5025f8d048382fd1f47377 which destructively poisons full pages with a bitpattern. On arches where PAGE_POISONING is used, this conflicts with the slab redzone checking enabled by DEBUG_SLAB, scribbling bits all over its magic words and making it complain about that quite emphatically. On x86 (and I presume at present all the other arches which set ARCH_SUPPORTS_DEBUG_PAGEALLOC too), the kernel_map_pages() operation is non destructive so it can coexist with the other DEBUG_SLAB mechanisms just fine. This patch favours the expensive full page destruction test for cases where there is a collision and it is explicitly selected. Signed-off-by: Ron Lee Signed-off-by: Pekka Enberg --- mm/slab.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mm/slab.c b/mm/slab.c index 9a90b00..1a6040d 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -2353,6 +2353,15 @@ kmem_cache_create (const char *name, size_t size, size_t align, /* really off slab. No need for manual alignment */ slab_size = cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab); + +#ifdef CONFIG_PAGE_POISONING + /* If we're going to use the generic kernel_map_pages() + * poisoning, then it's going to smash the contents of + * the redzone and userword anyhow, so switch them off. + */ + if (size % PAGE_SIZE == 0 && flags & SLAB_POISON) + flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); +#endif } cachep->colour_off = cache_line_size(); -- cgit v1.1 From a234bdc9aecc299ba41ffe8023b3ea110df9f51b Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Sun, 31 May 2009 13:50:38 +0300 Subject: slab: document kzfree() zeroing behavior As suggested by Alan Cox, document the fact that kzfree() can zero out a great deal more memory than the what the user requested from kmalloc(). Cc: Alan Cox Signed-off-by: Pekka Enberg --- mm/util.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/util.c b/mm/util.c index 55bef16..e79572b 100644 --- a/mm/util.c +++ b/mm/util.c @@ -166,6 +166,10 @@ EXPORT_SYMBOL(krealloc); * * The memory of the object @p points to is zeroed before freed. * If @p is %NULL, kzfree() does nothing. + * + * Note: this function zeroes the whole allocated buffer which can be a good + * deal bigger than the requested buffer size passed to kmalloc(). So be + * careful when using this function in performance sensitive code. */ void kzfree(const void *p) { -- cgit v1.1 From 781b2ba6eb5f22440afac9c79a89ebd6e3674a60 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 10 Jun 2009 18:50:32 +0300 Subject: SLUB: Out-of-memory diagnostics As suggested by Mel Gorman, add out-of-memory diagnostics to the SLUB allocator to make debugging OOM conditions easier. This patch helped hunt down a nasty OOM issue that popped up every now that was caused by SLUB debugging code which forced 4096 byte allocations to use order 1 pages even in the fallback case. An example print out looks like this: SLUB: Unable to allocate memory on node -1 (gfp=20) cache: kmalloc-4096, object size: 4096, buffer size: 4168, default order: 3, min order: 1 node 0: slabs: 95, objs: 665, free: 0 Acked-by: Christoph Lameter Acked-by: Mel Gorman Tested-by: Larry Finger Signed-off-by: Pekka Enberg --- mm/slub.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 65ffda5..a5a4ecf 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1484,6 +1484,56 @@ static inline int node_match(struct kmem_cache_cpu *c, int node) return 1; } +static int count_free(struct page *page) +{ + return page->objects - page->inuse; +} + +static unsigned long count_partial(struct kmem_cache_node *n, + int (*get_count)(struct page *)) +{ + unsigned long flags; + unsigned long x = 0; + struct page *page; + + spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(page, &n->partial, lru) + x += get_count(page); + spin_unlock_irqrestore(&n->list_lock, flags); + return x; +} + +static noinline void +slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) +{ + int node; + + printk(KERN_WARNING + "SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n", + nid, gfpflags); + printk(KERN_WARNING " cache: %s, object size: %d, buffer size: %d, " + "default order: %d, min order: %d\n", s->name, s->objsize, + s->size, oo_order(s->oo), oo_order(s->min)); + + for_each_online_node(node) { + struct kmem_cache_node *n = get_node(s, node); + unsigned long nr_slabs; + unsigned long nr_objs; + unsigned long nr_free; + + if (!n) + continue; + + nr_slabs = atomic_long_read(&n->nr_slabs); + nr_objs = atomic_long_read(&n->total_objects); + nr_free = count_partial(n, count_free); + + printk(KERN_WARNING + " node %d: slabs: %ld, objs: %ld, free: %ld\n", + node, nr_slabs, nr_objs, nr_free); + } +} + /* * Slow path. The lockless freelist is empty or we need to perform * debugging duties. @@ -1565,6 +1615,7 @@ new_slab: c->page = new; goto load_freelist; } + slab_out_of_memory(s, gfpflags, node); return NULL; debug: if (!alloc_debug_processing(s, c->page, object, addr)) @@ -3318,20 +3369,6 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, } #ifdef CONFIG_SLUB_DEBUG -static unsigned long count_partial(struct kmem_cache_node *n, - int (*get_count)(struct page *)) -{ - unsigned long flags; - unsigned long x = 0; - struct page *page; - - spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry(page, &n->partial, lru) - x += get_count(page); - spin_unlock_irqrestore(&n->list_lock, flags); - return x; -} - static int count_inuse(struct page *page) { return page->inuse; @@ -3342,11 +3379,6 @@ static int count_total(struct page *page) return page->objects; } -static int count_free(struct page *page) -{ - return page->objects - page->inuse; -} - static int validate_slab(struct kmem_cache *s, struct page *page, unsigned long *map) { -- cgit v1.1 From 26c02cf05ddadfee3952e829b841583794bf46f6 Mon Sep 17 00:00:00 2001 From: Alexander Beregalov Date: Thu, 11 Jun 2009 14:08:48 +0400 Subject: SLUB: fix build when !SLUB_DEBUG Fix this build error when CONFIG_SLUB_DEBUG is not set: mm/slub.c: In function 'slab_out_of_memory': mm/slub.c:1551: error: 'struct kmem_cache_node' has no member named 'nr_slabs' mm/slub.c:1552: error: 'struct kmem_cache_node' has no member named 'total_objects' [ penberg@cs.helsinki.fi: cleanups ] Signed-off-by: Alexander Beregalov Signed-off-by: Pekka Enberg --- mm/slub.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index a5a4ecf..9fb892b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -832,6 +832,11 @@ static inline unsigned long slabs_node(struct kmem_cache *s, int node) return atomic_long_read(&n->nr_slabs); } +static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) +{ + return atomic_long_read(&n->nr_slabs); +} + static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) { struct kmem_cache_node *n = get_node(s, node); @@ -1050,6 +1055,8 @@ static inline unsigned long kmem_cache_flags(unsigned long objsize, static inline unsigned long slabs_node(struct kmem_cache *s, int node) { return 0; } +static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) + { return 0; } static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) {} static inline void dec_slabs_node(struct kmem_cache *s, int node, @@ -1503,6 +1510,15 @@ static unsigned long count_partial(struct kmem_cache_node *n, return x; } +static inline unsigned long node_nr_objs(struct kmem_cache_node *n) +{ +#ifdef CONFIG_SLUB_DEBUG + return atomic_long_read(&n->total_objects); +#else + return 0; +#endif +} + static noinline void slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { @@ -1524,9 +1540,9 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) if (!n) continue; - nr_slabs = atomic_long_read(&n->nr_slabs); - nr_objs = atomic_long_read(&n->total_objects); - nr_free = count_partial(n, count_free); + nr_free = count_partial(n, count_free); + nr_slabs = node_nr_slabs(n); + nr_objs = node_nr_objs(n); printk(KERN_WARNING " node %d: slabs: %ld, objs: %ld, free: %ld\n", -- cgit v1.1 From 95f8598931bd86a5775073db2fa2004b892dd3d0 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Thu, 11 Jun 2009 16:18:09 +0300 Subject: SLUB: Don't print out OOM warning for __GFP_NOFAIL We must check for __GFP_NOFAIL like the page allocator does; otherwise we end up with false positives. While at it, add the printk_ratelimit() check in SLUB as well. Cc: Alexander Beregalov Signed-off-by: Pekka Enberg --- mm/slub.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index 9fb892b..0996b3b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1631,7 +1631,8 @@ new_slab: c->page = new; goto load_freelist; } - slab_out_of_memory(s, gfpflags, node); + if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) + slab_out_of_memory(s, gfpflags, node); return NULL; debug: if (!alloc_debug_processing(s, c->page, object, addr)) -- cgit v1.1 From 964cf35c88f93b4927dbc4e950dfa4d880c7f9d1 Mon Sep 17 00:00:00 2001 From: Nick Piggin Date: Mon, 15 Jun 2009 13:35:10 +0300 Subject: SLUB: Fix early boot GFP_DMA allocations Recent change to use slab allocations earlier exposed a bug where SLUB can call schedule_work and try to call sysfs before it is safe to do so. Reported-by: Heiko Carstens Tested-by: Heiko Carstens Signed-off-by: Nick Piggin Signed-off-by: Pekka Enberg --- mm/slub.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 30354bf..dcbfda0 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2610,6 +2610,7 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags) struct kmem_cache *s; char *text; size_t realsize; + unsigned long slabflags; s = kmalloc_caches_dma[index]; if (s) @@ -2631,9 +2632,18 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags) (unsigned int)realsize); s = kmalloc(kmem_size, flags & ~SLUB_DMA); + /* + * Must defer sysfs creation to a workqueue because we don't know + * what context we are called from. Before sysfs comes up, we don't + * need to do anything because our sysfs initcall will start by + * adding all existing slabs to sysfs. + */ + slabflags = SLAB_CACHE_DMA; + if (slab_state >= SYSFS) + slabflags |= __SYSFS_ADD_DEFERRED; + if (!s || !text || !kmem_cache_open(s, flags, text, - realsize, ARCH_KMALLOC_MINALIGN, - SLAB_CACHE_DMA|__SYSFS_ADD_DEFERRED, NULL)) { + realsize, ARCH_KMALLOC_MINALIGN, slabflags, NULL)) { kfree(s); kfree(text); goto unlock_out; @@ -2642,7 +2652,8 @@ static noinline struct kmem_cache *dma_kmalloc_cache(int index, gfp_t flags) list_add(&s->list, &slab_caches); kmalloc_caches_dma[index] = s; - schedule_work(&sysfs_add_work); + if (slab_state >= SYSFS) + schedule_work(&sysfs_add_work); unlock_out: up_write(&slub_lock); -- cgit v1.1