diff --git a/drivers/block/as-iosched.c b/drivers/block/as-iosched.c
index 638db06de2be..3410b4d294b9 100644
--- a/drivers/block/as-iosched.c
+++ b/drivers/block/as-iosched.c
@@ -1871,20 +1871,22 @@ static int as_init_queue(request_queue_t *q, elevator_t *e)
if (!arq_pool)
return -ENOMEM;
- ad = kmalloc(sizeof(*ad), GFP_KERNEL);
+ ad = kmalloc_node(sizeof(*ad), GFP_KERNEL, q->node);
if (!ad)
return -ENOMEM;
memset(ad, 0, sizeof(*ad));
ad->q = q; /* Identify what queue the data belongs to */
- ad->hash = kmalloc(sizeof(struct list_head)*AS_HASH_ENTRIES,GFP_KERNEL);
+ ad->hash = kmalloc_node(sizeof(struct list_head)*AS_HASH_ENTRIES,
+ GFP_KERNEL, q->node);
if (!ad->hash) {
kfree(ad);
return -ENOMEM;
}
- ad->arq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, arq_pool);
+ ad->arq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+ mempool_free_slab, arq_pool, q->node);
if (!ad->arq_pool) {
kfree(ad->hash);
kfree(ad);
diff --git a/drivers/block/deadline-iosched.c b/drivers/block/deadline-iosched.c
index 7f79f3dd0165..4bc2fea73273 100644
--- a/drivers/block/deadline-iosched.c
+++ b/drivers/block/deadline-iosched.c
@@ -711,18 +711,20 @@ static int deadline_init_queue(request_queue_t *q, elevator_t *e)
if (!drq_pool)
return -ENOMEM;
- dd = kmalloc(sizeof(*dd), GFP_KERNEL);
+ dd = kmalloc_node(sizeof(*dd), GFP_KERNEL, q->node);
if (!dd)
return -ENOMEM;
memset(dd, 0, sizeof(*dd));
- dd->hash = kmalloc(sizeof(struct list_head)*DL_HASH_ENTRIES,GFP_KERNEL);
+ dd->hash = kmalloc_node(sizeof(struct list_head)*DL_HASH_ENTRIES,
+ GFP_KERNEL, q->node);
if (!dd->hash) {
kfree(dd);
return -ENOMEM;
}
- dd->drq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, drq_pool);
+ dd->drq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+ mempool_free_slab, drq_pool, q->node);
if (!dd->drq_pool) {
kfree(dd->hash);
kfree(dd);
diff --git a/drivers/block/genhd.c b/drivers/block/genhd.c
index 53f7d846b747..43805e4d31e9 100644
--- a/drivers/block/genhd.c
+++ b/drivers/block/genhd.c
@@ -582,10 +582,16 @@ struct seq_operations diskstats_op = {
.show = diskstats_show
};
-
struct gendisk *alloc_disk(int minors)
{
- struct gendisk *disk = kmalloc(sizeof(struct gendisk), GFP_KERNEL);
+ return alloc_disk_node(minors, -1);
+}
+
+struct gendisk *alloc_disk_node(int minors, int node_id)
+{
+ struct gendisk *disk;
+
+ disk = kmalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
if (disk) {
memset(disk, 0, sizeof(struct gendisk));
if (!init_disk_stats(disk)) {
@@ -594,7 +600,7 @@ struct gendisk *alloc_disk(int minors)
}
if (minors > 1) {
int size = (minors - 1) * sizeof(struct hd_struct *);
- disk->part = kmalloc(size, GFP_KERNEL);
+ disk->part = kmalloc_node(size, GFP_KERNEL, node_id);
if (!disk->part) {
kfree(disk);
return NULL;
@@ -610,6 +616,7 @@ struct gendisk *alloc_disk(int minors)
}
EXPORT_SYMBOL(alloc_disk);
+EXPORT_SYMBOL(alloc_disk_node);
struct kobject *get_disk(struct gendisk *disk)
{
diff --git a/drivers/block/ll_rw_blk.c b/drivers/block/ll_rw_blk.c
index 81fe3a0c1fe7..cd8cf302068c 100644
--- a/drivers/block/ll_rw_blk.c
+++ b/drivers/block/ll_rw_blk.c
@@ -28,6 +28,7 @@
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/writeback.h>
+#include <linux/blkdev.h>
/*
* for max sense size
@@ -1645,7 +1646,8 @@ static int blk_init_free_list(request_queue_t *q)
init_waitqueue_head(&rl->wait[WRITE]);
init_waitqueue_head(&rl->drain);
- rl->rq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, request_cachep);
+ rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
+ mempool_free_slab, request_cachep, q->node);
if (!rl->rq_pool)
return -ENOMEM;
@@ -1657,8 +1659,15 @@ static int __make_request(request_queue_t *, struct bio *);
request_queue_t *blk_alloc_queue(int gfp_mask)
{
- request_queue_t *q = kmem_cache_alloc(requestq_cachep, gfp_mask);
+ return blk_alloc_queue_node(gfp_mask, -1);
+}
+EXPORT_SYMBOL(blk_alloc_queue);
+request_queue_t *blk_alloc_queue_node(int gfp_mask, int node_id)
+{
+ request_queue_t *q;
+
+ q = kmem_cache_alloc_node(requestq_cachep, gfp_mask, node_id);
if (!q)
return NULL;
@@ -1671,8 +1680,7 @@ request_queue_t *blk_alloc_queue(int gfp_mask)
return q;
}
-
-EXPORT_SYMBOL(blk_alloc_queue);
+EXPORT_SYMBOL(blk_alloc_queue_node);
/**
* blk_init_queue - prepare a request queue for use with a block device
@@ -1705,13 +1713,22 @@ EXPORT_SYMBOL(blk_alloc_queue);
* blk_init_queue() must be paired with a blk_cleanup_queue() call
* when the block device is deactivated (such as at module unload).
**/
+
request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
{
- request_queue_t *q = blk_alloc_queue(GFP_KERNEL);
+ return blk_init_queue_node(rfn, lock, -1);
+}
+EXPORT_SYMBOL(blk_init_queue);
+
+request_queue_t *
+blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
+{
+ request_queue_t *q = blk_alloc_queue_node(GFP_KERNEL, node_id);
if (!q)
return NULL;
+ q->node = node_id;
if (blk_init_free_list(q))
goto out_init;
@@ -1754,8 +1771,7 @@ out_init:
kmem_cache_free(requestq_cachep, q);
return NULL;
}
-
-EXPORT_SYMBOL(blk_init_queue);
+EXPORT_SYMBOL(blk_init_queue_node);
int blk_get_queue(request_queue_t *q)
{
diff --git a/drivers/ide/ide-disk.c b/drivers/ide/ide-disk.c
index 3302cd8eab4c..d6f934886b04 100644
--- a/drivers/ide/ide-disk.c
+++ b/drivers/ide/ide-disk.c
@@ -1215,7 +1215,8 @@ static int ide_disk_probe(struct device *dev)
if (!idkp)
goto failed;
- g = alloc_disk(1 << PARTN_BITS);
+ g = alloc_disk_node(1 << PARTN_BITS,
+ pcibus_to_node(drive->hwif->pci_dev->bus));
if (!g)
goto out_free_idkp;
diff --git a/drivers/ide/ide-probe.c b/drivers/ide/ide-probe.c
index 5d876f53c697..7df85af75371 100644
--- a/drivers/ide/ide-probe.c
+++ b/drivers/ide/ide-probe.c
@@ -977,8 +977,9 @@ static int ide_init_queue(ide_drive_t *drive)
* limits and LBA48 we could raise it but as yet
* do not.
*/
-
- q = blk_init_queue(do_ide_request, &ide_lock);
+
+ q = blk_init_queue_node(do_ide_request, &ide_lock,
+ pcibus_to_node(drive->hwif->pci_dev->bus));
if (!q)
return 1;
@@ -1095,7 +1096,8 @@ static int init_irq (ide_hwif_t *hwif)
hwgroup->hwif->next = hwif;
spin_unlock_irq(&ide_lock);
} else {
- hwgroup = kmalloc(sizeof(ide_hwgroup_t),GFP_KERNEL);
+ hwgroup = kmalloc_node(sizeof(ide_hwgroup_t), GFP_KERNEL,
+ pcibus_to_node(hwif->drives[0].hwif->pci_dev->bus));
if (!hwgroup)
goto out_up;
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 4a99b76c5a33..235c3414d268 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -396,6 +396,7 @@ struct request_queue
*/
unsigned int sg_timeout;
unsigned int sg_reserved_size;
+ int node;
struct list_head drain_list;
@@ -615,6 +616,8 @@ static inline void blkdev_dequeue_request(struct request *req)
/*
* Access functions for manipulating queue properties
*/
+extern request_queue_t *blk_init_queue_node(request_fn_proc *rfn,
+ spinlock_t *lock, int node_id);
extern request_queue_t *blk_init_queue(request_fn_proc *, spinlock_t *);
extern void blk_cleanup_queue(request_queue_t *);
extern void blk_queue_make_request(request_queue_t *, make_request_fn *);
@@ -646,7 +649,8 @@ extern void blk_wait_queue_drained(request_queue_t *, int);
extern void blk_finish_queue_drain(request_queue_t *);
int blk_get_queue(request_queue_t *);
-request_queue_t *blk_alloc_queue(int);
+request_queue_t *blk_alloc_queue(int gfp_mask);
+request_queue_t *blk_alloc_queue_node(int,int);
#define blk_put_queue(q) blk_cleanup_queue((q))
/*
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 47dedaf971d6..af26dc718ef6 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -403,6 +403,7 @@ extern int rescan_partitions(struct gendisk *disk, struct block_device *bdev);
extern void add_partition(struct gendisk *, int, sector_t, sector_t);
extern void delete_partition(struct gendisk *, int);
+extern struct gendisk *alloc_disk_node(int minors, int node_id);
extern struct gendisk *alloc_disk(int minors);
extern struct kobject *get_disk(struct gendisk *disk);
extern void put_disk(struct gendisk *disk);
diff --git a/include/linux/ide.h b/include/linux/ide.h
index 336d6e509f59..92129078d4f3 100644
--- a/include/linux/ide.h
+++ b/include/linux/ide.h
@@ -917,7 +917,7 @@ typedef struct hwif_s {
unsigned dma;
void (*led_act)(void *data, int rw);
-} ide_hwif_t;
+} ____cacheline_maxaligned_in_smp ide_hwif_t;
/*
* internal ide interrupt handler type
diff --git a/include/linux/mempool.h b/include/linux/mempool.h
index 4a36edf1c974..796220ce47cc 100644
--- a/include/linux/mempool.h
+++ b/include/linux/mempool.h
@@ -20,9 +20,14 @@ typedef struct mempool_s {
mempool_free_t *free;
wait_queue_head_t wait;
} mempool_t;
-extern mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
- mempool_free_t *free_fn, void *pool_data);
-extern int mempool_resize(mempool_t *pool, int new_min_nr, unsigned int __nocast gfp_mask);
+
+extern mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
+ mempool_free_t *free_fn, void *pool_data);
+extern mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
+ mempool_free_t *free_fn, void *pool_data, int nid);
+
+extern int mempool_resize(mempool_t *pool, int new_min_nr,
+ unsigned int __nocast gfp_mask);
extern void mempool_destroy(mempool_t *pool);
extern void * mempool_alloc(mempool_t *pool, unsigned int __nocast gfp_mask);
extern void mempool_free(void *element, mempool_t *pool);
diff --git a/mm/mempool.c b/mm/mempool.c
index c9f3d4620428..920c8c3ab1b8 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -51,16 +51,23 @@ static void free_pool(mempool_t *pool)
* functions might sleep - as long as the mempool_alloc function is not called
* from IRQ contexts.
*/
-mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
+mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
mempool_free_t *free_fn, void *pool_data)
{
- mempool_t *pool;
+ return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1);
+}
+EXPORT_SYMBOL(mempool_create);
- pool = kmalloc(sizeof(*pool), GFP_KERNEL);
+mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
+ mempool_free_t *free_fn, void *pool_data, int node_id)
+{
+ mempool_t *pool;
+ pool = kmalloc_node(sizeof(*pool), GFP_KERNEL, node_id);
if (!pool)
return NULL;
memset(pool, 0, sizeof(*pool));
- pool->elements = kmalloc(min_nr * sizeof(void *), GFP_KERNEL);
+ pool->elements = kmalloc_node(min_nr * sizeof(void *),
+ GFP_KERNEL, node_id);
if (!pool->elements) {
kfree(pool);
return NULL;
@@ -87,7 +94,7 @@ mempool_t * mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
}
return pool;
}
-EXPORT_SYMBOL(mempool_create);
+EXPORT_SYMBOL(mempool_create_node);
/**
* mempool_resize - resize an existing memory pool