From: William Lee Irwin III <wli@holomorphy.com>
The following patch consolidates redundant code in various hugetlb
implementations. I took the liberty of renaming a few things, since the
code was all moved anyway, and it has the benefit of helping to catch
missed conversions and/or consolidations.
Successfully tested on i386 and sparc64.
---
25-power4-akpm/arch/i386/mm/hugetlbpage.c | 264 ---------------------------
25-power4-akpm/arch/ia64/mm/hugetlbpage.c | 251 -------------------------
25-power4-akpm/arch/ppc64/mm/hugetlbpage.c | 258 --------------------------
25-power4-akpm/arch/sh/mm/hugetlbpage.c | 258 --------------------------
25-power4-akpm/arch/sparc64/mm/hugetlbpage.c | 259 --------------------------
25-power4-akpm/fs/hugetlbfs/inode.c | 2
25-power4-akpm/include/linux/hugetlb.h | 7
25-power4-akpm/kernel/sysctl.c | 6
25-power4-akpm/mm/Makefile | 1
25-power4-akpm/mm/hugetlb.c | 245 +++++++++++++++++++++++++
10 files changed, 263 insertions(+), 1288 deletions(-)
diff -puN arch/i386/mm/hugetlbpage.c~hugetlb-consolidation arch/i386/mm/hugetlbpage.c
--- 25-power4/arch/i386/mm/hugetlbpage.c~hugetlb-consolidation 2004-04-08 18:32:32.526455352 -0700
+++ 25-power4-akpm/arch/i386/mm/hugetlbpage.c 2004-04-08 18:32:32.540453224 -0700
@@ -20,68 +20,6 @@
#include <asm/tlb.h>
#include <asm/tlbflush.h>
-static long htlbpagemem;
-int htlbpage_max;
-static long htlbzone_pages;
-
-static struct list_head hugepage_freelists[MAX_NUMNODES];
-static spinlock_t htlbpage_lock = SPIN_LOCK_UNLOCKED;
-
-static void enqueue_huge_page(struct page *page)
-{
- list_add(&page->lru,
- &hugepage_freelists[page_zone(page)->zone_pgdat->node_id]);
-}
-
-static struct page *dequeue_huge_page(void)
-{
- int nid = numa_node_id();
- struct page *page = NULL;
-
- if (list_empty(&hugepage_freelists[nid])) {
- for (nid = 0; nid < MAX_NUMNODES; ++nid)
- if (!list_empty(&hugepage_freelists[nid]))
- break;
- }
- if (nid >= 0 && nid < MAX_NUMNODES && !list_empty(&hugepage_freelists[nid])) {
- page = list_entry(hugepage_freelists[nid].next, struct page, lru);
- list_del(&page->lru);
- }
- return page;
-}
-
-static struct page *alloc_fresh_huge_page(void)
-{
- static int nid = 0;
- struct page *page;
- page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP,
- HUGETLB_PAGE_ORDER);
- nid = (nid + 1) % numnodes;
- return page;
-}
-
-static void free_huge_page(struct page *page);
-
-static struct page *alloc_hugetlb_page(void)
-{
- int i;
- struct page *page;
-
- spin_lock(&htlbpage_lock);
- page = dequeue_huge_page();
- if (!page) {
- spin_unlock(&htlbpage_lock);
- return NULL;
- }
- htlbpagemem--;
- spin_unlock(&htlbpage_lock);
- set_page_count(page, 1);
- page->lru.prev = (void *)free_huge_page;
- for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i)
- clear_highpage(&page[i]);
- return page;
-}
-
static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
@@ -276,26 +214,6 @@ follow_huge_pmd(struct mm_struct *mm, un
}
#endif
-static void free_huge_page(struct page *page)
-{
- BUG_ON(page_count(page));
-
- INIT_LIST_HEAD(&page->lru);
-
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpagemem++;
- spin_unlock(&htlbpage_lock);
-}
-
-void huge_page_release(struct page *page)
-{
- if (!put_page_testzero(page))
- return;
-
- free_huge_page(page);
-}
-
void unmap_hugepage_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
@@ -319,16 +237,6 @@ void unmap_hugepage_range(struct vm_area
flush_tlb_range(vma, start, end);
}
-void
-zap_hugepage_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long length)
-{
- struct mm_struct *mm = vma->vm_mm;
- spin_lock(&mm->page_table_lock);
- unmap_hugepage_range(vma, start, start + length);
- spin_unlock(&mm->page_table_lock);
-}
-
int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)
{
struct mm_struct *mm = current->mm;
@@ -360,7 +268,7 @@ int hugetlb_prefault(struct address_spac
ret = -ENOMEM;
goto out;
}
- page = alloc_hugetlb_page();
+ page = alloc_huge_page();
if (!page) {
hugetlb_put_quota(mapping);
ret = -ENOMEM;
@@ -380,173 +288,3 @@ out:
spin_unlock(&mm->page_table_lock);
return ret;
}
-
-static void update_and_free_page(struct page *page)
-{
- int j;
- struct page *map;
-
- map = page;
- htlbzone_pages--;
- for (j = 0; j < (HPAGE_SIZE / PAGE_SIZE); j++) {
- map->flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
- 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
- 1 << PG_private | 1<< PG_writeback);
- set_page_count(map, 0);
- map++;
- }
- set_page_count(page, 1);
- __free_pages(page, HUGETLB_PAGE_ORDER);
-}
-
-static int try_to_free_low(int count)
-{
- struct list_head *p;
- struct page *page, *map;
-
- map = NULL;
- spin_lock(&htlbpage_lock);
- /* all lowmem is on node 0 */
- list_for_each(p, &hugepage_freelists[0]) {
- if (map) {
- list_del(&map->lru);
- update_and_free_page(map);
- htlbpagemem--;
- map = NULL;
- if (++count == 0)
- break;
- }
- page = list_entry(p, struct page, lru);
- if (!PageHighMem(page))
- map = page;
- }
- if (map) {
- list_del(&map->lru);
- update_and_free_page(map);
- htlbpagemem--;
- count++;
- }
- spin_unlock(&htlbpage_lock);
- return count;
-}
-
-static int set_hugetlb_mem_size(int count)
-{
- int lcount;
- struct page *page;
-
- if (count < 0)
- lcount = count;
- else
- lcount = count - htlbzone_pages;
-
- if (lcount == 0)
- return (int)htlbzone_pages;
- if (lcount > 0) { /* Increase the mem size. */
- while (lcount--) {
- page = alloc_fresh_huge_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpagemem++;
- htlbzone_pages++;
- spin_unlock(&htlbpage_lock);
- }
- return (int) htlbzone_pages;
- }
- /* Shrink the memory size. */
- lcount = try_to_free_low(lcount);
- while (lcount++) {
- page = alloc_hugetlb_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- update_and_free_page(page);
- spin_unlock(&htlbpage_lock);
- }
- return (int) htlbzone_pages;
-}
-
-int hugetlb_sysctl_handler(ctl_table *table, int write,
- struct file *file, void *buffer, size_t *length)
-{
- if (!cpu_has_pse)
- return -ENODEV;
- proc_dointvec(table, write, file, buffer, length);
- htlbpage_max = set_hugetlb_mem_size(htlbpage_max);
- return 0;
-}
-
-static int __init hugetlb_setup(char *s)
-{
- if (sscanf(s, "%d", &htlbpage_max) <= 0)
- htlbpage_max = 0;
- return 1;
-}
-__setup("hugepages=", hugetlb_setup);
-
-static int __init hugetlb_init(void)
-{
- int i;
- struct page *page;
-
- if (!cpu_has_pse)
- return -ENODEV;
-
- for (i = 0; i < MAX_NUMNODES; ++i)
- INIT_LIST_HEAD(&hugepage_freelists[i]);
-
- for (i = 0; i < htlbpage_max; ++i) {
- page = alloc_fresh_huge_page();
- if (!page)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- spin_unlock(&htlbpage_lock);
- }
- htlbpage_max = htlbpagemem = htlbzone_pages = i;
- printk("Total HugeTLB memory allocated, %ld\n", htlbpagemem);
- return 0;
-}
-module_init(hugetlb_init);
-
-int hugetlb_report_meminfo(char *buf)
-{
- return sprintf(buf,
- "HugePages_Total: %5lu\n"
- "HugePages_Free: %5lu\n"
- "Hugepagesize: %5lu kB\n",
- htlbzone_pages,
- htlbpagemem,
- HPAGE_SIZE/1024);
-}
-
-int is_hugepage_mem_enough(size_t size)
-{
- return (size + ~HPAGE_MASK)/HPAGE_SIZE <= htlbpagemem;
-}
-
-/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
-unsigned long hugetlb_total_pages(void)
-{
- return htlbzone_pages * (HPAGE_SIZE / PAGE_SIZE);
-}
-EXPORT_SYMBOL(hugetlb_total_pages);
-
-/*
- * We cannot handle pagefaults against hugetlb pages at all. They cause
- * handle_mm_fault() to try to instantiate regular-sized pages in the
- * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
- * this far.
- */
-static struct page *hugetlb_nopage(struct vm_area_struct *vma,
- unsigned long address, int *unused)
-{
- BUG();
- return NULL;
-}
-
-struct vm_operations_struct hugetlb_vm_ops = {
- .nopage = hugetlb_nopage,
-};
diff -puN arch/ia64/mm/hugetlbpage.c~hugetlb-consolidation arch/ia64/mm/hugetlbpage.c
--- 25-power4/arch/ia64/mm/hugetlbpage.c~hugetlb-consolidation 2004-04-08 18:32:32.527455200 -0700
+++ 25-power4-akpm/arch/ia64/mm/hugetlbpage.c 2004-04-08 18:32:32.542452920 -0700
@@ -22,69 +22,7 @@
#include <asm/tlb.h>
#include <asm/tlbflush.h>
-static long htlbpagemem;
-int htlbpage_max;
-static long htlbzone_pages;
-unsigned int hpage_shift=HPAGE_SHIFT_DEFAULT;
-
-static struct list_head hugepage_freelists[MAX_NUMNODES];
-static spinlock_t htlbpage_lock = SPIN_LOCK_UNLOCKED;
-
-static void enqueue_huge_page(struct page *page)
-{
- list_add(&page->lru,
- &hugepage_freelists[page_zone(page)->zone_pgdat->node_id]);
-}
-
-static struct page *dequeue_huge_page(void)
-{
- int nid = numa_node_id();
- struct page *page = NULL;
-
- if (list_empty(&hugepage_freelists[nid])) {
- for (nid = 0; nid < MAX_NUMNODES; ++nid)
- if (!list_empty(&hugepage_freelists[nid]))
- break;
- }
- if (nid >= 0 && nid < MAX_NUMNODES &&
- !list_empty(&hugepage_freelists[nid])) {
- page = list_entry(hugepage_freelists[nid].next, struct page, lru);
- list_del(&page->lru);
- }
- return page;
-}
-
-static struct page *alloc_fresh_huge_page(void)
-{
- static int nid = 0;
- struct page *page;
- page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP,
- HUGETLB_PAGE_ORDER);
- nid = (nid + 1) % numnodes;
- return page;
-}
-
-void free_huge_page(struct page *page);
-
-static struct page *alloc_hugetlb_page(void)
-{
- int i;
- struct page *page;
-
- spin_lock(&htlbpage_lock);
- page = dequeue_huge_page();
- if (!page) {
- spin_unlock(&htlbpage_lock);
- return NULL;
- }
- htlbpagemem--;
- spin_unlock(&htlbpage_lock);
- set_page_count(page, 1);
- page->lru.prev = (void *)free_huge_page;
- for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i)
- clear_highpage(&page[i]);
- return page;
-}
+unsigned int hpage_shift=HPAGE_SHIFT_DEFAULT;
static pte_t *
huge_pte_alloc (struct mm_struct *mm, unsigned long addr)
@@ -244,26 +182,6 @@ follow_huge_pmd(struct mm_struct *mm, un
return NULL;
}
-void free_huge_page(struct page *page)
-{
- BUG_ON(page_count(page));
-
- INIT_LIST_HEAD(&page->lru);
-
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpagemem++;
- spin_unlock(&htlbpage_lock);
-}
-
-void huge_page_release(struct page *page)
-{
- if (!put_page_testzero(page))
- return;
-
- free_huge_page(page);
-}
-
/*
* Same as generic free_pgtables(), except constant PGDIR_* and pgd_offset
* are hugetlb region specific.
@@ -339,14 +257,6 @@ void unmap_hugepage_range(struct vm_area
flush_tlb_range(vma, start, end);
}
-void zap_hugepage_range(struct vm_area_struct *vma, unsigned long start, unsigned long length)
-{
- struct mm_struct *mm = vma->vm_mm;
- spin_lock(&mm->page_table_lock);
- unmap_hugepage_range(vma, start, start + length);
- spin_unlock(&mm->page_table_lock);
-}
-
int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)
{
struct mm_struct *mm = current->mm;
@@ -378,7 +288,7 @@ int hugetlb_prefault(struct address_spac
ret = -ENOMEM;
goto out;
}
- page = alloc_hugetlb_page();
+ page = alloc_huge_page();
if (!page) {
hugetlb_put_quota(mapping);
ret = -ENOMEM;
@@ -422,106 +332,6 @@ unsigned long hugetlb_get_unmapped_area(
addr = ALIGN(vmm->vm_end, HPAGE_SIZE);
}
}
-void update_and_free_page(struct page *page)
-{
- int j;
- struct page *map;
-
- map = page;
- htlbzone_pages--;
- for (j = 0; j < (HPAGE_SIZE / PAGE_SIZE); j++) {
- map->flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
- 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
- 1 << PG_private | 1<< PG_writeback);
- set_page_count(map, 0);
- map++;
- }
- set_page_count(page, 1);
- __free_pages(page, HUGETLB_PAGE_ORDER);
-}
-
-int try_to_free_low(int count)
-{
- struct list_head *p;
- struct page *page, *map;
-
- map = NULL;
- spin_lock(&htlbpage_lock);
- list_for_each(p, &hugepage_freelists[0]) {
- if (map) {
- list_del(&map->lru);
- update_and_free_page(map);
- htlbpagemem--;
- map = NULL;
- if (++count == 0)
- break;
- }
- page = list_entry(p, struct page, lru);
- if (!PageHighMem(page))
- map = page;
- }
- if (map) {
- list_del(&map->lru);
- update_and_free_page(map);
- htlbpagemem--;
- count++;
- }
- spin_unlock(&htlbpage_lock);
- return count;
-}
-
-int set_hugetlb_mem_size(int count)
-{
- int lcount;
- struct page *page ;
-
- if (count < 0)
- lcount = count;
- else
- lcount = count - htlbzone_pages;
-
- if (lcount == 0)
- return (int)htlbzone_pages;
- if (lcount > 0) { /* Increase the mem size. */
- while (lcount--) {
- page = alloc_fresh_huge_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpagemem++;
- htlbzone_pages++;
- spin_unlock(&htlbpage_lock);
- }
- return (int) htlbzone_pages;
- }
- /* Shrink the memory size. */
- lcount = try_to_free_low(lcount);
- while (lcount++) {
- page = alloc_hugetlb_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- update_and_free_page(page);
- spin_unlock(&htlbpage_lock);
- }
- return (int) htlbzone_pages;
-}
-
-int hugetlb_sysctl_handler(ctl_table *table, int write, struct file *file, void *buffer, size_t *length)
-{
- proc_dointvec(table, write, file, buffer, length);
- htlbpage_max = set_hugetlb_mem_size(htlbpage_max);
- return 0;
-}
-
-static int __init hugetlb_setup(char *s)
-{
- if (sscanf(s, "%d", &htlbpage_max) <= 0)
- htlbpage_max = 0;
- return 1;
-}
-__setup("hugepages=", hugetlb_setup);
static int __init hugetlb_setup_sz(char *str)
{
@@ -551,60 +361,3 @@ static int __init hugetlb_setup_sz(char
return 1;
}
__setup("hugepagesz=", hugetlb_setup_sz);
-
-static int __init hugetlb_init(void)
-{
- int i;
- struct page *page;
-
- for (i = 0; i < MAX_NUMNODES; ++i)
- INIT_LIST_HEAD(&hugepage_freelists[i]);
-
- for (i = 0; i < htlbpage_max; ++i) {
- page = alloc_fresh_huge_page();
- if (!page)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- spin_unlock(&htlbpage_lock);
- }
- htlbpage_max = htlbpagemem = htlbzone_pages = i;
- printk("Total HugeTLB memory allocated, %ld\n", htlbpagemem);
- return 0;
-}
-__initcall(hugetlb_init);
-
-int hugetlb_report_meminfo(char *buf)
-{
- return sprintf(buf,
- "HugePages_Total: %5lu\n"
- "HugePages_Free: %5lu\n"
- "Hugepagesize: %5lu kB\n",
- htlbzone_pages,
- htlbpagemem,
- HPAGE_SIZE/1024);
-}
-
-int is_hugepage_mem_enough(size_t size)
-{
- if (size > (htlbpagemem << HPAGE_SHIFT))
- return 0;
- return 1;
-}
-
-/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
-unsigned long hugetlb_total_pages(void)
-{
- return htlbzone_pages * (HPAGE_SIZE / PAGE_SIZE);
-}
-EXPORT_SYMBOL(hugetlb_total_pages);
-
-static struct page *hugetlb_nopage(struct vm_area_struct * area, unsigned long address, int *unused)
-{
- BUG();
- return NULL;
-}
-
-struct vm_operations_struct hugetlb_vm_ops = {
- .nopage = hugetlb_nopage,
-};
diff -puN arch/ppc64/mm/hugetlbpage.c~hugetlb-consolidation arch/ppc64/mm/hugetlbpage.c
--- 25-power4/arch/ppc64/mm/hugetlbpage.c~hugetlb-consolidation 2004-04-08 18:32:32.528455048 -0700
+++ 25-power4-akpm/arch/ppc64/mm/hugetlbpage.c 2004-04-08 18:40:07.988214640 -0700
@@ -29,65 +29,6 @@
#include <linux/sysctl.h>
-int htlbpage_max;
-
-/* This lock protects the two counters and list below */
-static spinlock_t htlbpage_lock = SPIN_LOCK_UNLOCKED;
-
-static int htlbpage_free; /* = 0 */
-static int htlbpage_total; /* = 0 */
-static struct list_head hugepage_freelists[MAX_NUMNODES];
-
-static void enqueue_huge_page(struct page *page)
-{
- list_add(&page->lru,
- &hugepage_freelists[page_zone(page)->zone_pgdat->node_id]);
-}
-
-/* XXX make this a sysctl */
-unsigned long largepage_roundrobin = 1;
-
-static struct page *dequeue_huge_page(void)
-{
- static int nid = 0;
- struct page *page = NULL;
- int i;
-
- if (!largepage_roundrobin)
- nid = numa_node_id();
-
- for (i = 0; i < numnodes; i++) {
- if (!list_empty(&hugepage_freelists[nid]))
- break;
- nid = (nid + 1) % numnodes;
- }
-
- if (!list_empty(&hugepage_freelists[nid])) {
- page = list_entry(hugepage_freelists[nid].next, struct page, lru);
- list_del(&page->lru);
- }
-
- if (largepage_roundrobin)
- nid = (nid + 1) % numnodes;
-
- return page;
-}
-
-static struct page *alloc_fresh_huge_page(void)
-{
- static int nid = 0;
- struct page *page;
-
- page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP,
- HUGETLB_PAGE_ORDER);
- if (!page)
- return NULL;
-
- nid = page_zone(page)->zone_pgdat->node_id;
- nid = (nid + 1) % numnodes;
- return page;
-}
-
/* HugePTE layout:
*
* 31 30 ... 15 14 13 12 10 9 8 7 6 5 4 3 2 1 0
@@ -119,7 +60,6 @@ typedef struct {unsigned int val;} hugep
#define hugepte_none(x) (!(hugepte_val(x) & _HUGEPAGE_PFN))
-static void free_huge_page(struct page *page);
static void flush_hash_hugepage(mm_context_t context, unsigned long ea,
hugepte_t pte, int local);
@@ -146,27 +86,6 @@ static inline void set_hugepte(hugepte_t
hugepte_val(pte) & ~_HUGEPAGE_HPTEFLAGS);
}
-static struct page *alloc_hugetlb_page(void)
-{
- int i;
- struct page *page;
-
- spin_lock(&htlbpage_lock);
- page = dequeue_huge_page();
- if (!page) {
- spin_unlock(&htlbpage_lock);
- return NULL;
- }
-
- htlbpage_free--;
- spin_unlock(&htlbpage_lock);
- set_page_count(page, 1);
- page->lru.prev = (void *)free_huge_page;
- for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); ++i)
- clear_highpage(&page[i]);
- return page;
-}
-
static hugepte_t *hugepte_alloc(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
@@ -448,26 +367,6 @@ follow_huge_pmd(struct mm_struct *mm, un
return page;
}
-static void free_huge_page(struct page *page)
-{
- BUG_ON(page_count(page));
-
- INIT_LIST_HEAD(&page->lru);
-
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpage_free++;
- spin_unlock(&htlbpage_lock);
-}
-
-void huge_page_release(struct page *page)
-{
- if (!put_page_testzero(page))
- return;
-
- free_huge_page(page);
-}
-
void unmap_hugepage_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
@@ -510,16 +409,6 @@ void unmap_hugepage_range(struct vm_area
mm->rss -= (end - start) >> PAGE_SHIFT;
}
-void zap_hugepage_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long length)
-{
- struct mm_struct *mm = vma->vm_mm;
-
- spin_lock(&mm->page_table_lock);
- unmap_hugepage_range(vma, start, start + length);
- spin_unlock(&mm->page_table_lock);
-}
-
int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)
{
struct mm_struct *mm = current->mm;
@@ -554,7 +443,7 @@ int hugetlb_prefault(struct address_spac
ret = -ENOMEM;
goto out;
}
- page = alloc_hugetlb_page();
+ page = alloc_huge_page();
if (!page) {
hugetlb_put_quota(mapping);
ret = -ENOMEM;
@@ -876,148 +765,3 @@ static void flush_hash_hugepage(mm_conte
ppc_md.hpte_invalidate(slot, va, 1, local);
}
-
-static void split_and_free_hugepage(struct page *page)
-{
- int j;
- struct page *map;
-
- map = page;
- htlbpage_total--;
- for (j = 0; j < (HPAGE_SIZE / PAGE_SIZE); j++) {
- map->flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
- 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
- 1 << PG_private | 1<< PG_writeback);
- set_page_count(map, 0);
- map++;
- }
- set_page_count(page, 1);
- __free_pages(page, HUGETLB_PAGE_ORDER);
-}
-
-int set_hugetlb_mem_size(int count)
-{
- int lcount;
- struct page *page;
-
- if (!(cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE))
- return 0;
-
- if (count < 0)
- lcount = count;
- else
- lcount = count - htlbpage_total;
-
- if (lcount == 0)
- return htlbpage_total;
- if (lcount > 0) { /* Increase the mem size. */
- while (lcount--) {
- page = alloc_fresh_huge_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpage_free++;
- htlbpage_total++;
- spin_unlock(&htlbpage_lock);
- }
- return htlbpage_total;
- }
- /* Shrink the memory size. */
- while (lcount++) {
- page = alloc_hugetlb_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- split_and_free_hugepage(page);
- spin_unlock(&htlbpage_lock);
- }
- return htlbpage_total;
-}
-
-int hugetlb_sysctl_handler(ctl_table *table, int write,
- struct file *file, void *buffer, size_t *length)
-{
- proc_dointvec(table, write, file, buffer, length);
- htlbpage_max = set_hugetlb_mem_size(htlbpage_max);
- return 0;
-}
-
-static int __init hugetlb_setup(char *s)
-{
- if (sscanf(s, "%d", &htlbpage_max) <= 0)
- htlbpage_max = 0;
- return 1;
-}
-__setup("hugepages=", hugetlb_setup);
-
-static int __init hugetlb_init(void)
-{
- int i;
- struct page *page;
-
- if (cur_cpu_spec->cpu_features & CPU_FTR_16M_PAGE) {
- for (i = 0; i < MAX_NUMNODES; ++i)
- INIT_LIST_HEAD(&hugepage_freelists[i]);
-
- for (i = 0; i < htlbpage_max; ++i) {
- page = alloc_fresh_huge_page();
- if (!page)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- spin_unlock(&htlbpage_lock);
- }
- htlbpage_max = htlbpage_free = htlbpage_total = i;
- printk(KERN_INFO "Total HugeTLB memory allocated, %d\n",
- htlbpage_free);
- } else {
- htlbpage_max = 0;
- printk(KERN_INFO "CPU does not support HugeTLB\n");
- }
-
- return 0;
-}
-module_init(hugetlb_init);
-
-int hugetlb_report_meminfo(char *buf)
-{
- return sprintf(buf,
- "HugePages_Total: %5d\n"
- "HugePages_Free: %5d\n"
- "Hugepagesize: %5lu kB\n",
- htlbpage_total,
- htlbpage_free,
- HPAGE_SIZE/1024);
-}
-
-/* This is advisory only, so we can get away with accesing
- * htlbpage_free without taking the lock. */
-int is_hugepage_mem_enough(size_t size)
-{
- return (size + ~HPAGE_MASK)/HPAGE_SIZE <= htlbpage_free;
-}
-
-/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
-unsigned long hugetlb_total_pages(void)
-{
- return htlbpage_total * (HPAGE_SIZE / PAGE_SIZE);
-}
-EXPORT_SYMBOL(hugetlb_total_pages);
-
-/*
- * We cannot handle pagefaults against hugetlb pages at all. They cause
- * handle_mm_fault() to try to instantiate regular-sized pages in the
- * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
- * this far.
- */
-static struct page *hugetlb_nopage(struct vm_area_struct *vma,
- unsigned long address, int *unused)
-{
- BUG();
- return NULL;
-}
-
-struct vm_operations_struct hugetlb_vm_ops = {
- .nopage = hugetlb_nopage,
-};
diff -puN arch/sh/mm/hugetlbpage.c~hugetlb-consolidation arch/sh/mm/hugetlbpage.c
--- 25-power4/arch/sh/mm/hugetlbpage.c~hugetlb-consolidation 2004-04-08 18:32:32.529454896 -0700
+++ 25-power4-akpm/arch/sh/mm/hugetlbpage.c 2004-04-08 18:32:32.545452464 -0700
@@ -24,68 +24,6 @@
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>
-static long htlbpagemem;
-int htlbpage_max;
-static long htlbzone_pages;
-
-static struct list_head hugepage_freelists[MAX_NUMNODES];
-static spinlock_t htlbpage_lock = SPIN_LOCK_UNLOCKED;
-
-static void enqueue_huge_page(struct page *page)
-{
- list_add(&page->lru,
- &hugepage_freelists[page_zone(page)->zone_pgdat->node_id]);
-}
-
-static struct page *dequeue_huge_page(void)
-{
- int nid = numa_node_id();
- struct page *page = NULL;
-
- if (list_empty(&hugepage_freelists[nid])) {
- for (nid = 0; nid < MAX_NUMNODES; ++nid)
- if (!list_empty(&hugepage_freelists[nid]))
- break;
- }
- if (nid >= 0 && nid < MAX_NUMNODES &&
- !list_empty(&hugepage_freelists[nid])) {
- page = list_entry(hugepage_freelists[nid].next,
- struct page, list);
- list_del(&page->lru);
- }
- return page;
-}
-
-static struct page *alloc_fresh_huge_page(void)
-{
- static int nid = 0;
- struct page *page;
- page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP,
- HUGETLB_PAGE_ORDER);
- nid = (nid + 1) % numnodes;
- return page;
-}
-
-static void free_huge_page(struct page *page);
-
-static struct page *alloc_hugetlb_page(void)
-{
- struct page *page;
-
- spin_lock(&htlbpage_lock);
- page = dequeue_huge_page();
- if (!page) {
- spin_unlock(&htlbpage_lock);
- return NULL;
- }
- htlbpagemem--;
- spin_unlock(&htlbpage_lock);
- set_page_count(page, 1);
- page->lru.prev = (void *)free_huge_page;
- memset(page_address(page), 0, HPAGE_SIZE);
- return page;
-}
-
static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
@@ -250,25 +188,6 @@ struct page *follow_huge_pmd(struct mm_s
return NULL;
}
-static void free_huge_page(struct page *page)
-{
- BUG_ON(page_count(page));
- BUG_ON(page->mapping);
-
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpagemem++;
- spin_unlock(&htlbpage_lock);
-}
-
-void huge_page_release(struct page *page)
-{
- if (!put_page_testzero(page))
- return;
-
- free_huge_page(page);
-}
-
void unmap_hugepage_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
@@ -297,16 +216,6 @@ void unmap_hugepage_range(struct vm_area
flush_tlb_range(vma, start, end);
}
-void zap_hugepage_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long length)
-{
- struct mm_struct *mm = vma->vm_mm;
-
- spin_lock(&mm->page_table_lock);
- unmap_hugepage_range(vma, start, start + length);
- spin_unlock(&mm->page_table_lock);
-}
-
int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)
{
struct mm_struct *mm = current->mm;
@@ -338,7 +247,7 @@ int hugetlb_prefault(struct address_spac
ret = -ENOMEM;
goto out;
}
- page = alloc_hugetlb_page();
+ page = alloc_huge_page();
if (!page) {
hugetlb_put_quota(mapping);
ret = -ENOMEM;
@@ -358,168 +267,3 @@ out:
spin_unlock(&mm->page_table_lock);
return ret;
}
-
-static void update_and_free_page(struct page *page)
-{
- int j;
- struct page *map;
-
- map = page;
- htlbzone_pages--;
- for (j = 0; j < (HPAGE_SIZE / PAGE_SIZE); j++) {
- map->flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
- 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
- 1 << PG_private | 1<< PG_writeback);
- set_page_count(map, 0);
- map++;
- }
- set_page_count(page, 1);
- __free_pages(page, HUGETLB_PAGE_ORDER);
-}
-
-static int try_to_free_low(int count)
-{
- struct list_head *p;
- struct page *page, *map;
-
- map = NULL;
- spin_lock(&htlbpage_lock);
- /* all lowmem is on node 0 */
- list_for_each(p, &hugepage_freelists[0]) {
- if (map) {
- list_del(&map->lru);
- update_and_free_page(map);
- htlbpagemem--;
- map = NULL;
- if (++count == 0)
- break;
- }
- page = list_entry(p, struct page, list);
- if (!PageHighMem(page))
- map = page;
- }
- if (map) {
- list_del(&map->lru);
- update_and_free_page(map);
- htlbpagemem--;
- count++;
- }
- spin_unlock(&htlbpage_lock);
- return count;
-}
-
-static int set_hugetlb_mem_size(int count)
-{
- int lcount;
- struct page *page;
-
- if (count < 0)
- lcount = count;
- else
- lcount = count - htlbzone_pages;
-
- if (lcount == 0)
- return (int)htlbzone_pages;
- if (lcount > 0) { /* Increase the mem size. */
- while (lcount--) {
- page = alloc_fresh_huge_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpagemem++;
- htlbzone_pages++;
- spin_unlock(&htlbpage_lock);
- }
- return (int) htlbzone_pages;
- }
- /* Shrink the memory size. */
- lcount = try_to_free_low(lcount);
- while (lcount++) {
- page = alloc_hugetlb_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- update_and_free_page(page);
- spin_unlock(&htlbpage_lock);
- }
- return (int) htlbzone_pages;
-}
-
-int hugetlb_sysctl_handler(struct ctl_table *table, int write,
- struct file *file, void *buffer, size_t *length)
-{
- proc_dointvec(table, write, file, buffer, length);
- htlbpage_max = set_hugetlb_mem_size(htlbpage_max);
- return 0;
-}
-
-static int __init hugetlb_setup(char *s)
-{
- if (sscanf(s, "%d", &htlbpage_max) <= 0)
- htlbpage_max = 0;
- return 1;
-}
-__setup("hugepages=", hugetlb_setup);
-
-static int __init hugetlb_init(void)
-{
- int i;
- struct page *page;
-
- for (i = 0; i < MAX_NUMNODES; ++i)
- INIT_LIST_HEAD(&hugepage_freelists[i]);
-
- for (i = 0; i < htlbpage_max; ++i) {
- page = alloc_fresh_huge_page();
- if (!page)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- spin_unlock(&htlbpage_lock);
- }
- htlbpage_max = htlbpagemem = htlbzone_pages = i;
- printk("Total HugeTLB memory allocated, %ld\n", htlbpagemem);
- return 0;
-}
-module_init(hugetlb_init);
-
-int hugetlb_report_meminfo(char *buf)
-{
- return sprintf(buf,
- "HugePages_Total: %5lu\n"
- "HugePages_Free: %5lu\n"
- "Hugepagesize: %5lu kB\n",
- htlbzone_pages,
- htlbpagemem,
- HPAGE_SIZE/1024);
-}
-
-int is_hugepage_mem_enough(size_t size)
-{
- return (size + ~HPAGE_MASK)/HPAGE_SIZE <= htlbpagemem;
-}
-
-/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
-unsigned long hugetlb_total_pages(void)
-{
- return htlbzone_pages * (HPAGE_SIZE / PAGE_SIZE);
-}
-EXPORT_SYMBOL(hugetlb_total_pages);
-
-/*
- * We cannot handle pagefaults against hugetlb pages at all. They cause
- * handle_mm_fault() to try to instantiate regular-sized pages in the
- * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
- * this far.
- */
-static struct page *hugetlb_nopage(struct vm_area_struct *vma,
- unsigned long address, int *unused)
-{
- BUG();
- return NULL;
-}
-
-struct vm_operations_struct hugetlb_vm_ops = {
- .nopage = hugetlb_nopage,
-};
diff -puN arch/sparc64/mm/hugetlbpage.c~hugetlb-consolidation arch/sparc64/mm/hugetlbpage.c
--- 25-power4/arch/sparc64/mm/hugetlbpage.c~hugetlb-consolidation 2004-04-08 18:32:32.531454592 -0700
+++ 25-power4-akpm/arch/sparc64/mm/hugetlbpage.c 2004-04-08 18:32:32.546452312 -0700
@@ -21,68 +21,6 @@
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>
-static long htlbpagemem;
-int htlbpage_max;
-static long htlbzone_pages;
-
-static struct list_head hugepage_freelists[MAX_NUMNODES];
-static spinlock_t htlbpage_lock = SPIN_LOCK_UNLOCKED;
-
-static void enqueue_huge_page(struct page *page)
-{
- list_add(&page->lru,
- &hugepage_freelists[page_zone(page)->zone_pgdat->node_id]);
-}
-
-static struct page *dequeue_huge_page(void)
-{
- int nid = numa_node_id();
- struct page *page = NULL;
-
- if (list_empty(&hugepage_freelists[nid])) {
- for (nid = 0; nid < MAX_NUMNODES; ++nid)
- if (!list_empty(&hugepage_freelists[nid]))
- break;
- }
- if (nid >= 0 && nid < MAX_NUMNODES &&
- !list_empty(&hugepage_freelists[nid])) {
- page = list_entry(hugepage_freelists[nid].next,
- struct page, lru);
- list_del(&page->lru);
- }
- return page;
-}
-
-static struct page *alloc_fresh_huge_page(void)
-{
- static int nid = 0;
- struct page *page;
- page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP,
- HUGETLB_PAGE_ORDER);
- nid = (nid + 1) % numnodes;
- return page;
-}
-
-static void free_huge_page(struct page *page);
-
-static struct page *alloc_hugetlb_page(void)
-{
- struct page *page;
-
- spin_lock(&htlbpage_lock);
- page = dequeue_huge_page();
- if (!page) {
- spin_unlock(&htlbpage_lock);
- return NULL;
- }
- htlbpagemem--;
- spin_unlock(&htlbpage_lock);
- set_page_count(page, 1);
- page->lru.prev = (void *)free_huge_page;
- memset(page_address(page), 0, HPAGE_SIZE);
- return page;
-}
-
static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
@@ -247,26 +185,6 @@ struct page *follow_huge_pmd(struct mm_s
return NULL;
}
-static void free_huge_page(struct page *page)
-{
- BUG_ON(page_count(page));
-
- INIT_LIST_HEAD(&page->lru);
-
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpagemem++;
- spin_unlock(&htlbpage_lock);
-}
-
-void huge_page_release(struct page *page)
-{
- if (!put_page_testzero(page))
- return;
-
- free_huge_page(page);
-}
-
void unmap_hugepage_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
@@ -295,16 +213,6 @@ void unmap_hugepage_range(struct vm_area
flush_tlb_range(vma, start, end);
}
-void zap_hugepage_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long length)
-{
- struct mm_struct *mm = vma->vm_mm;
-
- spin_lock(&mm->page_table_lock);
- unmap_hugepage_range(vma, start, start + length);
- spin_unlock(&mm->page_table_lock);
-}
-
int hugetlb_prefault(struct address_space *mapping, struct vm_area_struct *vma)
{
struct mm_struct *mm = current->mm;
@@ -336,7 +244,7 @@ int hugetlb_prefault(struct address_spac
ret = -ENOMEM;
goto out;
}
- page = alloc_hugetlb_page();
+ page = alloc_huge_page();
if (!page) {
hugetlb_put_quota(mapping);
ret = -ENOMEM;
@@ -356,168 +264,3 @@ out:
spin_unlock(&mm->page_table_lock);
return ret;
}
-
-static void update_and_free_page(struct page *page)
-{
- int j;
- struct page *map;
-
- map = page;
- htlbzone_pages--;
- for (j = 0; j < (HPAGE_SIZE / PAGE_SIZE); j++) {
- map->flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
- 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
- 1 << PG_private | 1<< PG_writeback);
- set_page_count(map, 0);
- map++;
- }
- set_page_count(page, 1);
- __free_pages(page, HUGETLB_PAGE_ORDER);
-}
-
-static int try_to_free_low(int count)
-{
- struct list_head *p;
- struct page *page, *map;
-
- map = NULL;
- spin_lock(&htlbpage_lock);
- /* all lowmem is on node 0 */
- list_for_each(p, &hugepage_freelists[0]) {
- if (map) {
- list_del(&map->lru);
- update_and_free_page(map);
- htlbpagemem--;
- map = NULL;
- if (++count == 0)
- break;
- }
- page = list_entry(p, struct page, lru);
- if (!PageHighMem(page))
- map = page;
- }
- if (map) {
- list_del(&map->lru);
- update_and_free_page(map);
- htlbpagemem--;
- count++;
- }
- spin_unlock(&htlbpage_lock);
- return count;
-}
-
-static int set_hugetlb_mem_size(int count)
-{
- int lcount;
- struct page *page;
-
- if (count < 0)
- lcount = count;
- else
- lcount = count - htlbzone_pages;
-
- if (lcount == 0)
- return (int)htlbzone_pages;
- if (lcount > 0) { /* Increase the mem size. */
- while (lcount--) {
- page = alloc_fresh_huge_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- htlbpagemem++;
- htlbzone_pages++;
- spin_unlock(&htlbpage_lock);
- }
- return (int) htlbzone_pages;
- }
- /* Shrink the memory size. */
- lcount = try_to_free_low(lcount);
- while (lcount++) {
- page = alloc_hugetlb_page();
- if (page == NULL)
- break;
- spin_lock(&htlbpage_lock);
- update_and_free_page(page);
- spin_unlock(&htlbpage_lock);
- }
- return (int) htlbzone_pages;
-}
-
-int hugetlb_sysctl_handler(struct ctl_table *table, int write,
- struct file *file, void *buffer, size_t *length)
-{
- proc_dointvec(table, write, file, buffer, length);
- htlbpage_max = set_hugetlb_mem_size(htlbpage_max);
- return 0;
-}
-
-static int __init hugetlb_setup(char *s)
-{
- if (sscanf(s, "%d", &htlbpage_max) <= 0)
- htlbpage_max = 0;
- return 1;
-}
-__setup("hugepages=", hugetlb_setup);
-
-static int __init hugetlb_init(void)
-{
- int i;
- struct page *page;
-
- for (i = 0; i < MAX_NUMNODES; ++i)
- INIT_LIST_HEAD(&hugepage_freelists[i]);
-
- for (i = 0; i < htlbpage_max; ++i) {
- page = alloc_fresh_huge_page();
- if (!page)
- break;
- spin_lock(&htlbpage_lock);
- enqueue_huge_page(page);
- spin_unlock(&htlbpage_lock);
- }
- htlbpage_max = htlbpagemem = htlbzone_pages = i;
- printk("Total HugeTLB memory allocated, %ld\n", htlbpagemem);
- return 0;
-}
-module_init(hugetlb_init);
-
-int hugetlb_report_meminfo(char *buf)
-{
- return sprintf(buf,
- "HugePages_Total: %5lu\n"
- "HugePages_Free: %5lu\n"
- "Hugepagesize: %5lu kB\n",
- htlbzone_pages,
- htlbpagemem,
- HPAGE_SIZE/1024);
-}
-
-int is_hugepage_mem_enough(size_t size)
-{
- return (size + ~HPAGE_MASK)/HPAGE_SIZE <= htlbpagemem;
-}
-
-/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
-unsigned long hugetlb_total_pages(void)
-{
- return htlbzone_pages * (HPAGE_SIZE / PAGE_SIZE);
-}
-EXPORT_SYMBOL(hugetlb_total_pages);
-
-/*
- * We cannot handle pagefaults against hugetlb pages at all. They cause
- * handle_mm_fault() to try to instantiate regular-sized pages in the
- * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
- * this far.
- */
-static struct page *hugetlb_nopage(struct vm_area_struct *vma,
- unsigned long address, int *unused)
-{
- BUG();
- return NULL;
-}
-
-struct vm_operations_struct hugetlb_vm_ops = {
- .nopage = hugetlb_nopage,
-};
diff -puN fs/hugetlbfs/inode.c~hugetlb-consolidation fs/hugetlbfs/inode.c
--- 25-power4/fs/hugetlbfs/inode.c~hugetlb-consolidation 2004-04-08 18:32:32.532454440 -0700
+++ 25-power4-akpm/fs/hugetlbfs/inode.c 2004-04-08 18:41:23.814687264 -0700
@@ -573,7 +573,7 @@ hugetlbfs_parse_options(char *options, s
unsigned long long size = memparse(value, &rest);
if (*rest == '%') {
size <<= HPAGE_SHIFT;
- size *= htlbpage_max;
+ size *= max_huge_pages;
do_div(size, 100);
rest++;
}
diff -puN include/linux/hugetlb.h~hugetlb-consolidation include/linux/hugetlb.h
--- 25-power4/include/linux/hugetlb.h~hugetlb-consolidation 2004-04-08 18:32:32.533454288 -0700
+++ 25-power4-akpm/include/linux/hugetlb.h 2004-04-08 18:41:11.851505944 -0700
@@ -28,8 +28,11 @@ struct page *follow_huge_pmd(struct mm_s
pmd_t *pmd, int write);
int is_aligned_hugepage_range(unsigned long addr, unsigned long len);
int pmd_huge(pmd_t pmd);
+struct page *alloc_huge_page(void);
+void free_huge_page(struct page *);
-extern int htlbpage_max;
+extern unsigned long max_huge_pages;
+extern const unsigned long hugetlb_zero, hugetlb_infinity;
static inline void
mark_mm_hugetlb(struct mm_struct *mm, struct vm_area_struct *vma)
@@ -78,6 +81,8 @@ static inline unsigned long hugetlb_tota
#define pmd_huge(x) 0
#define is_hugepage_only_range(addr, len) 0
#define hugetlb_free_pgtables(tlb, prev, start, end) do { } while (0)
+#define alloc_huge_page() ({ NULL; })
+#define free_huge_page(p) ({ (void)(p); BUG(); })
#ifndef HPAGE_MASK
#define HPAGE_MASK 0 /* Keep the compiler happy */
diff -puN kernel/sysctl.c~hugetlb-consolidation kernel/sysctl.c
--- 25-power4/kernel/sysctl.c~hugetlb-consolidation 2004-04-08 18:32:32.535453984 -0700
+++ 25-power4-akpm/kernel/sysctl.c 2004-04-08 18:32:32.549451856 -0700
@@ -710,10 +710,12 @@ static ctl_table vm_table[] = {
{
.ctl_name = VM_HUGETLB_PAGES,
.procname = "nr_hugepages",
- .data = &htlbpage_max,
- .maxlen = sizeof(int),
+ .data = &max_huge_pages,
+ .maxlen = sizeof(unsigned long),
.mode = 0644,
.proc_handler = &hugetlb_sysctl_handler,
+ .extra1 = (void *)&hugetlb_zero,
+ .extra2 = (void *)&hugetlb_infinity,
},
#endif
{
diff -puN /dev/null mm/hugetlb.c
--- /dev/null 2003-09-15 06:40:47.000000000 -0700
+++ 25-power4-akpm/mm/hugetlb.c 2004-04-08 18:32:32.550451704 -0700
@@ -0,0 +1,245 @@
+/*
+ * Generic hugetlb support.
+ * (C) William Irwin, April 2004
+ */
+#include <linux/gfp.h>
+#include <linux/list.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/hugetlb.h>
+#include <linux/sysctl.h>
+
+const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
+static unsigned long nr_huge_pages, free_huge_pages;
+unsigned long max_huge_pages;
+static struct list_head hugepage_freelists[MAX_NUMNODES];
+static spinlock_t hugetlb_lock = SPIN_LOCK_UNLOCKED;
+
+static void enqueue_huge_page(struct page *page)
+{
+ list_add(&page->lru,
+ &hugepage_freelists[page_zone(page)->zone_pgdat->node_id]);
+}
+
+static struct page *dequeue_huge_page(void)
+{
+ int nid = numa_node_id();
+ struct page *page = NULL;
+
+ if (list_empty(&hugepage_freelists[nid])) {
+ for (nid = 0; nid < MAX_NUMNODES; ++nid)
+ if (!list_empty(&hugepage_freelists[nid]))
+ break;
+ }
+ if (nid >= 0 && nid < MAX_NUMNODES &&
+ !list_empty(&hugepage_freelists[nid])) {
+ page = list_entry(hugepage_freelists[nid].next,
+ struct page, lru);
+ list_del(&page->lru);
+ }
+ return page;
+}
+
+static struct page *alloc_fresh_huge_page(void)
+{
+ static int nid = 0;
+ struct page *page;
+ page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP,
+ HUGETLB_PAGE_ORDER);
+ nid = (nid + 1) % numnodes;
+ return page;
+}
+
+void free_huge_page(struct page *page)
+{
+ BUG_ON(page_count(page));
+
+ INIT_LIST_HEAD(&page->lru);
+
+ spin_lock(&hugetlb_lock);
+ enqueue_huge_page(page);
+ free_huge_pages++;
+ spin_unlock(&hugetlb_lock);
+}
+
+struct page *alloc_huge_page(void)
+{
+ struct page *page;
+
+ spin_lock(&hugetlb_lock);
+ page = dequeue_huge_page();
+ if (!page) {
+ spin_unlock(&hugetlb_lock);
+ return NULL;
+ }
+ free_huge_pages--;
+ spin_unlock(&hugetlb_lock);
+ set_page_count(page, 1);
+ page->lru.prev = (void *)free_huge_page;
+ memset(page_address(page), 0, HPAGE_SIZE);
+ return page;
+}
+
+void huge_page_release(struct page *page)
+{
+ if (!put_page_testzero(page))
+ return;
+
+ free_huge_page(page);
+}
+
+static int __init hugetlb_init(void)
+{
+ unsigned long i;
+ struct page *page;
+
+ for (i = 0; i < MAX_NUMNODES; ++i)
+ INIT_LIST_HEAD(&hugepage_freelists[i]);
+
+ for (i = 0; i < max_huge_pages; ++i) {
+ page = alloc_fresh_huge_page();
+ if (!page)
+ break;
+ spin_lock(&hugetlb_lock);
+ enqueue_huge_page(page);
+ spin_unlock(&hugetlb_lock);
+ }
+ max_huge_pages = free_huge_pages = nr_huge_pages = i;
+ printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
+ return 0;
+}
+module_init(hugetlb_init);
+
+static int __init hugetlb_setup(char *s)
+{
+ if (sscanf(s, "%lu", &max_huge_pages) <= 0)
+ max_huge_pages = 0;
+ return 1;
+}
+__setup("hugepages=", hugetlb_setup);
+
+static void update_and_free_page(struct page *page)
+{
+ int i;
+ nr_huge_pages--;
+ for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
+ page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
+ 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
+ 1 << PG_private | 1<< PG_writeback);
+ set_page_count(&page[i], 0);
+ }
+ set_page_count(page, 1);
+ __free_pages(page, HUGETLB_PAGE_ORDER);
+}
+
+#ifdef CONFIG_HIGHMEM
+static int try_to_free_low(unsigned long count)
+{
+ int i;
+ for (i = 0; i < MAX_NUMNODES; ++i) {
+ struct page *page;
+ list_for_each_entry(page, &hugepage_freelists[i], lru) {
+ if (PageHighmem(page))
+ continue;
+ list_del(&page->lru);
+ update_and_free_page(page);
+ --free_huge_pages
+ if (!--count)
+ return 0;
+ }
+ }
+ return count;
+}
+#else
+static inline int try_to_free_low(unsigned long count)
+{
+ return count;
+}
+#endif
+
+static unsigned long set_max_huge_pages(unsigned long count)
+{
+ while (count > nr_huge_pages) {
+ struct page *page = alloc_fresh_huge_page();
+ if (!page)
+ return nr_huge_pages;
+ spin_lock(&hugetlb_lock);
+ enqueue_huge_page(page);
+ free_huge_pages++;
+ nr_huge_pages++;
+ spin_unlock(&hugetlb_lock);
+ }
+ if (count >= nr_huge_pages)
+ return nr_huge_pages;
+
+ spin_lock(&hugetlb_lock);
+ for (count = try_to_free_low(count); count < nr_huge_pages; --free_huge_pages) {
+ struct page *page = dequeue_huge_page();
+ if (!page)
+ break;
+ update_and_free_page(page);
+ }
+ spin_unlock(&hugetlb_lock);
+ return nr_huge_pages;
+}
+
+#ifdef CONFIG_SYSCTL
+int hugetlb_sysctl_handler(struct ctl_table *table, int write,
+ struct file *file, void *buffer, size_t *length)
+{
+ proc_doulongvec_minmax(table, write, file, buffer, length);
+ max_huge_pages = set_max_huge_pages(max_huge_pages);
+ return 0;
+}
+#endif /* CONFIG_SYSCTL */
+
+int hugetlb_report_meminfo(char *buf)
+{
+ return sprintf(buf,
+ "HugePages_Total: %5lu\n"
+ "HugePages_Free: %5lu\n"
+ "Hugepagesize: %5lu kB\n",
+ nr_huge_pages,
+ free_huge_pages,
+ HPAGE_SIZE/1024);
+}
+
+int is_hugepage_mem_enough(size_t size)
+{
+ return (size + ~HPAGE_MASK)/HPAGE_SIZE <= free_huge_pages;
+}
+
+/* Return the number pages of memory we physically have, in PAGE_SIZE units. */
+unsigned long hugetlb_total_pages(void)
+{
+ return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
+}
+EXPORT_SYMBOL(hugetlb_total_pages);
+
+/*
+ * We cannot handle pagefaults against hugetlb pages at all. They cause
+ * handle_mm_fault() to try to instantiate regular-sized pages in the
+ * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
+ * this far.
+ */
+static struct page *hugetlb_nopage(struct vm_area_struct *vma,
+ unsigned long address, int *unused)
+{
+ BUG();
+ return NULL;
+}
+
+struct vm_operations_struct hugetlb_vm_ops = {
+ .nopage = hugetlb_nopage,
+};
+
+void zap_hugepage_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long length)
+{
+ struct mm_struct *mm = vma->vm_mm;
+
+ spin_lock(&mm->page_table_lock);
+ unmap_hugepage_range(vma, start, start + length);
+ spin_unlock(&mm->page_table_lock);
+}
diff -puN mm/Makefile~hugetlb-consolidation mm/Makefile
--- 25-power4/mm/Makefile~hugetlb-consolidation 2004-04-08 18:32:32.536453832 -0700
+++ 25-power4-akpm/mm/Makefile 2004-04-08 18:41:12.228448640 -0700
@@ -12,3 +12,4 @@ obj-y := bootmem.o filemap.o mempool.o
slab.o swap.o truncate.o vmscan.o $(mmu-y)
obj-$(CONFIG_SWAP) += page_io.o swap_state.o swapfile.o
+obj-$(CONFIG_HUGETLBFS) += hugetlb.o
_