diff options
Diffstat (limited to 'mm/page_alloc.c')
| -rw-r--r-- | mm/page_alloc.c | 120 |
1 files changed, 118 insertions, 2 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 0ec869e..d1a7b6d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -78,6 +78,8 @@ DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ EXPORT_PER_CPU_SYMBOL(_numa_mem_); #endif +struct rw_semaphore page_alloc_slow_rwsem; + /* * Array of node states. */ @@ -99,6 +101,14 @@ unsigned long totalreserve_pages __read_mostly; int percpu_pagelist_fraction; gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; +#ifdef CONFIG_COMPACTION_RETRY_DEBUG +static inline void show_buddy_info(void); +#else +static inline void show_buddy_info(void) +{ +} +#endif + #ifdef CONFIG_PM_SLEEP /* * The following functions are used by the suspend/hibernate code to temporarily @@ -127,6 +137,20 @@ void pm_restrict_gfp_mask(void) saved_gfp_mask = gfp_allowed_mask; gfp_allowed_mask &= ~GFP_IOFS; } + +static bool pm_suspending(void) +{ + if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS) + return false; + return true; +} + +#else + +static bool pm_suspending(void) +{ + return false; +} #endif /* CONFIG_PM_SLEEP */ #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE @@ -176,6 +200,7 @@ static char * const zone_names[MAX_NR_ZONES] = { }; int min_free_kbytes = 1024; +int min_free_order_shift = 1; static unsigned long __meminitdata nr_kernel_pages; static unsigned long __meminitdata nr_all_pages; @@ -1487,7 +1512,7 @@ static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark, free_pages -= z->free_area[o].nr_free << o; /* Require fewer higher order pages to be free */ - min >>= 1; + min >>= min_free_order_shift; if (free_pages <= min) return false; @@ -1847,6 +1872,10 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, int migratetype) { struct page *page; +#ifdef CONFIG_COMPACTION_RETRY + struct zoneref *z; + struct zone *zone; +#endif /* Acquire the OOM killer lock for the zones in zonelist */ if (!try_set_zonelist_oom(zonelist, gfp_mask)) { @@ -1866,6 +1895,32 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, if (page) goto out; +#ifdef CONFIG_COMPACTION_RETRY + /* + * When we reach here, we already tried direct reclaim. + * Therefore it might be possible that we have enough + * free memory but extremely fragmented. + * So we give it a last chance to try memory compaction and get pages. + */ + if (order) { + pr_info("reclaim before oom : retry compaction.\n"); + show_buddy_info(); + + for_each_zone_zonelist_nodemask(zone, z, zonelist, + high_zoneidx, nodemask) + compact_zone_order(zone, -1, gfp_mask, true); + + show_buddy_info(); + pr_info("reclaim :end\n"); + page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, + nodemask, order, zonelist, high_zoneidx, + ALLOC_WMARK_HIGH|ALLOC_CPUSET, + preferred_zone, migratetype); + if (page) + goto out; + } +#endif + if (!(gfp_mask & __GFP_NOFAIL)) { /* The OOM killer will not help higher order allocs */ if (order > PAGE_ALLOC_COSTLY_ORDER) @@ -2108,6 +2163,9 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, unsigned long pages_reclaimed = 0; unsigned long did_some_progress; bool sync_migration = false; +#ifdef CONFIG_ANDROID_WIP + unsigned long start_tick = jiffies; +#endif bool deferred_compaction = false; /* @@ -2121,6 +2179,9 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, return NULL; } + if (gfp_mask & __GFP_WAIT) + down_read(&page_alloc_slow_rwsem); + /* * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and * __GFP_NOWARN set) should not cause reclaim since the subsystem @@ -2217,11 +2278,22 @@ rebalance: /* * If we failed to make any progress reclaiming, then we are * running out of options and have to consider going OOM + * ANDROID_WIP: If we are looping more than 1 second, consider OOM */ - if (!did_some_progress) { +#ifdef CONFIG_ANDROID_WIP +#define SHOULD_CONSIDER_OOM !did_some_progress || time_after(jiffies, start_tick + HZ) +#else +#define SHOULD_CONSIDER_OOM !did_some_progress +#endif + if (SHOULD_CONSIDER_OOM) { if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) { if (oom_killer_disabled) goto nopage; +#ifdef CONFIG_ANDROID_WIP + if (did_some_progress) + pr_info("time's up : calling " + "__alloc_pages_may_oom\n"); +#endif page = __alloc_pages_may_oom(gfp_mask, order, zonelist, high_zoneidx, nodemask, preferred_zone, @@ -2249,6 +2321,14 @@ rebalance: goto restart; } + + /* + * Suspend converts GFP_KERNEL to __GFP_WAIT which can + * prevent reclaim making forward progress without + * invoking OOM. Bail if we are suspending + */ + if (pm_suspending()) + goto nopage; } /* Check if we should retry the allocation */ @@ -2276,10 +2356,14 @@ rebalance: nopage: warn_alloc_failed(gfp_mask, order, NULL); + if (gfp_mask & __GFP_WAIT) + up_read(&page_alloc_slow_rwsem); return page; got_pg: if (kmemcheck_enabled) kmemcheck_pagealloc_alloc(page, order, gfp_mask); + if (gfp_mask & __GFP_WAIT) + up_read(&page_alloc_slow_rwsem); return page; } @@ -2729,6 +2813,37 @@ void show_free_areas(unsigned int filter) show_swap_cache_info(); } +#ifdef CONFIG_COMPACTION_RETRY_DEBUG +void show_buddy_info(void) +{ + struct zone *zone; + unsigned long nr[MAX_ORDER], flags, order, total = 0; + char buf[256]; + int len; + + for_each_populated_zone(zone) { + + if (skip_free_areas_node(SHOW_MEM_FILTER_NODES, + zone_to_nid(zone))) + continue; + show_node(zone); + len = sprintf(buf, "%s: ", zone->name); + + spin_lock_irqsave(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) { + nr[order] = zone->free_area[order].nr_free; + total += nr[order] << order; + } + spin_unlock_irqrestore(&zone->lock, flags); + for (order = 0; order < MAX_ORDER; order++) + len += sprintf(buf + len, "%lu*%lukB ", + nr[order], K(1UL) << order); + len += sprintf(buf + len, "= %lukB", K(total)); + pr_err("%s\n", buf); + } +} +#endif + static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) { zoneref->zone = zone; @@ -5056,6 +5171,7 @@ static int page_alloc_cpu_notify(struct notifier_block *self, void __init page_alloc_init(void) { hotcpu_notifier(page_alloc_cpu_notify, 0); + init_rwsem(&page_alloc_slow_rwsem); } /* |