From fd0928df98b9578be8a786ac0cb78a47a5e17a20 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 24 Jan 2008 08:52:45 +0100 Subject: ioprio: move io priority from task_struct to io_context This is where it belongs and then it doesn't take up space for a process that doesn't do IO. Signed-off-by: Jens Axboe --- include/linux/iocontext.h | 79 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 include/linux/iocontext.h (limited to 'include/linux/iocontext.h') diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h new file mode 100644 index 000000000000..186807ea62e2 --- /dev/null +++ b/include/linux/iocontext.h @@ -0,0 +1,79 @@ +#ifndef IOCONTEXT_H +#define IOCONTEXT_H + +/* + * This is the per-process anticipatory I/O scheduler state. + */ +struct as_io_context { + spinlock_t lock; + + void (*dtor)(struct as_io_context *aic); /* destructor */ + void (*exit)(struct as_io_context *aic); /* called on task exit */ + + unsigned long state; + atomic_t nr_queued; /* queued reads & sync writes */ + atomic_t nr_dispatched; /* number of requests gone to the drivers */ + + /* IO History tracking */ + /* Thinktime */ + unsigned long last_end_request; + unsigned long ttime_total; + unsigned long ttime_samples; + unsigned long ttime_mean; + /* Layout pattern */ + unsigned int seek_samples; + sector_t last_request_pos; + u64 seek_total; + sector_t seek_mean; +}; + +struct cfq_queue; +struct cfq_io_context { + struct rb_node rb_node; + void *key; + + struct cfq_queue *cfqq[2]; + + struct io_context *ioc; + + unsigned long last_end_request; + sector_t last_request_pos; + + unsigned long ttime_total; + unsigned long ttime_samples; + unsigned long ttime_mean; + + unsigned int seek_samples; + u64 seek_total; + sector_t seek_mean; + + struct list_head queue_list; + + void (*dtor)(struct io_context *); /* destructor */ + void (*exit)(struct io_context *); /* called on task exit */ +}; + +/* + * This is the per-process I/O subsystem state. It is refcounted and + * kmalloc'ed. Currently all fields are modified in process io context + * (apart from the atomic refcount), so require no locking. + */ +struct io_context { + atomic_t refcount; + struct task_struct *task; + + unsigned short ioprio; + unsigned short ioprio_changed; + + /* + * For request batching + */ + unsigned long last_waited; /* Time last woken after wait for request */ + int nr_batch_requests; /* Number of requests left in the batch */ + + struct as_io_context *aic; + struct rb_root cic_root; + void *ioc_data; +}; + +#endif -- cgit v1.2.3 From d38ecf935fcb10264a6bc190855d9595165e6eeb Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 24 Jan 2008 08:53:35 +0100 Subject: io context sharing: preliminary support Detach task state from ioc, instead keep track of how many processes are accessing the ioc. Signed-off-by: Jens Axboe --- include/linux/iocontext.h | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'include/linux/iocontext.h') diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 186807ea62e2..cd44d458124a 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -54,13 +54,15 @@ struct cfq_io_context { }; /* - * This is the per-process I/O subsystem state. It is refcounted and - * kmalloc'ed. Currently all fields are modified in process io context - * (apart from the atomic refcount), so require no locking. + * I/O subsystem state of the associated processes. It is refcounted + * and kmalloc'ed. These could be shared between processes. */ struct io_context { atomic_t refcount; - struct task_struct *task; + atomic_t nr_tasks; + + /* all the fields below are protected by this lock */ + spinlock_t lock; unsigned short ioprio; unsigned short ioprio_changed; @@ -76,4 +78,16 @@ struct io_context { void *ioc_data; }; +static inline struct io_context *ioc_task_link(struct io_context *ioc) +{ + /* + * if ref count is zero, don't allow sharing (ioc is going away, it's + * a race). + */ + if (ioc && atomic_inc_not_zero(&ioc->refcount)) + return ioc; + + return NULL; +} + #endif -- cgit v1.2.3 From 4ac845a2e9a816ed5a7b301f56dcc0a3d0b1ba4d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 24 Jan 2008 08:44:49 +0100 Subject: block: cfq: make the io contect sharing lockless The io context sharing introduced a per-ioc spinlock, that would protect the cfq io context lookup. That is a regression from the original, since we never needed any locking there because the ioc/cic were process private. The cic lookup is changed from an rbtree construct to a radix tree, which we can then use RCU to make the reader side lockless. That is the performance critical path, modifying the radix tree is only done on process creation (when that process first does IO, actually) and on process exit (if that process has done IO). As it so happens, radix trees are also much faster for this type of lookup where the key is a pointer. It's a very sparse tree. Signed-off-by: Jens Axboe --- include/linux/iocontext.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux/iocontext.h') diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index cd44d458124a..593b222d9dcc 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -1,6 +1,8 @@ #ifndef IOCONTEXT_H #define IOCONTEXT_H +#include + /* * This is the per-process anticipatory I/O scheduler state. */ @@ -29,8 +31,8 @@ struct as_io_context { struct cfq_queue; struct cfq_io_context { - struct rb_node rb_node; void *key; + unsigned long dead_key; struct cfq_queue *cfqq[2]; @@ -74,7 +76,7 @@ struct io_context { int nr_batch_requests; /* Number of requests left in the batch */ struct as_io_context *aic; - struct rb_root cic_root; + struct radix_tree_root radix_root; void *ioc_data; }; -- cgit v1.2.3