diff --git a/fs/aio.c b/fs/aio.c index e4598d6..091bcb4 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -108,8 +108,8 @@ static int aio_setup_ring(struct kioctx *ctx) unsigned long size; int nr_pages; - /* Compensate for the ring buffer's head/tail overlap entry */ - nr_events += 2; /* 1 is required, 2 for good luck */ + /* round nr_event to next power of 2 */ + nr_events = roundup_pow_of_two(nr_events); size = sizeof(struct aio_ring); size += sizeof(struct io_event) * nr_events; @@ -118,8 +118,6 @@ static int aio_setup_ring(struct kioctx *ctx) if (nr_pages < 0) return -EINVAL; - nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) / sizeof(struct io_event); - info->nr = 0; info->ring_pages = info->internal_pages; if (nr_pages > AIO_RING_PAGES) { @@ -159,7 +157,8 @@ static int aio_setup_ring(struct kioctx *ctx) ring = kmap_atomic(info->ring_pages[0], KM_USER0); ring->nr = nr_events; /* user copy */ ring->id = ctx->user_id; - ring->head = ring->tail = 0; + atomic_set(&ring->head, 0); + ring->tail = 0; ring->magic = AIO_RING_MAGIC; ring->compat_features = AIO_RING_COMPAT_FEATURES; ring->incompat_features = AIO_RING_INCOMPAT_FEATURES; @@ -177,8 +176,8 @@ static int aio_setup_ring(struct kioctx *ctx) #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event)) #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE) -#define aio_ring_event(info, nr, km) ({ \ - unsigned pos = (nr) + AIO_EVENTS_OFFSET; \ +#define aio_ring_event(info, __nr, km) ({ \ + unsigned pos = ((__nr) & ((info)->nr - 1)) + AIO_EVENTS_OFFSET; \ struct io_event *__event; \ __event = kmap_atomic( \ (info)->ring_pages[pos / AIO_EVENTS_PER_PAGE], km); \ @@ -220,7 +219,6 @@ static struct kioctx *ioctx_alloc(unsigned nr_events) atomic_set(&ctx->users, 1); spin_lock_init(&ctx->ctx_lock); - spin_lock_init(&ctx->ring_info.ring_lock); init_waitqueue_head(&ctx->wait); INIT_LIST_HEAD(&ctx->active_reqs); @@ -927,7 +925,7 @@ int fastcall aio_complete(struct kiocb *iocb, long res, long res2) struct aio_ring *ring; struct io_event *event; unsigned long flags; - unsigned long tail; + unsigned tail; int ret; /* @@ -969,8 +967,6 @@ int fastcall aio_complete(struct kiocb *iocb, long res, long res2) tail = info->tail; event = aio_ring_event(info, tail, KM_IRQ0); - if (++tail >= info->nr) - tail = 0; event->obj = (u64)(unsigned long)iocb->ki_obj.user; event->data = iocb->ki_user_data; @@ -986,13 +982,14 @@ int fastcall aio_complete(struct kiocb *iocb, long res, long res2) */ smp_wmb(); /* make event visible before updating tail */ + tail++; info->tail = tail; ring->tail = tail; put_aio_ring_event(event, KM_IRQ0); kunmap_atomic(ring, KM_IRQ1); - pr_debug("added to ring %p at [%lu]\n", iocb, tail); + pr_debug("added to ring %p at [%u]\n", iocb, tail); put_rq: /* everything turned out well, dispose of the aiocb. */ ret = __aio_put_req(ctx, iocb); @@ -1007,42 +1004,36 @@ put_rq: /* aio_read_evt * Pull an event off of the ioctx's event ring. Returns the number of * events fetched (0 or 1 ;-) - * FIXME: make this use cmpxchg. * TODO: make the ringbuffer user mmap()able (requires FIXME). */ static int aio_read_evt(struct kioctx *ioctx, struct io_event *ent) { struct aio_ring_info *info = &ioctx->ring_info; struct aio_ring *ring; - unsigned long head; - int ret = 0; + struct io_event *evp; + unsigned head; + int ret; ring = kmap_atomic(info->ring_pages[0], KM_USER0); - dprintk("in aio_read_evt h%lu t%lu m%lu\n", - (unsigned long)ring->head, (unsigned long)ring->tail, - (unsigned long)ring->nr); - - if (ring->head == ring->tail) - goto out; - - spin_lock(&info->ring_lock); + dprintk("in aio_read_evt h%u t%u m%u\n", + atomic_read(&ring->head), ring->tail, ring->nr); - head = ring->head % info->nr; - if (head != ring->tail) { - struct io_event *evp = aio_ring_event(info, head, KM_USER1); + do { + head = atomic_read(&ring->head); + if (head == ring->tail) { + ret = 0; + break; + } + evp = aio_ring_event(info, head, KM_USER1); *ent = *evp; - head = (head + 1) % info->nr; smp_mb(); /* finish reading the event before updatng the head */ - ring->head = head; ret = 1; put_aio_ring_event(evp, KM_USER1); - } - spin_unlock(&info->ring_lock); + } while (head != atomic_cmpxchg(&ring->head, head, head + 1)); -out: kunmap_atomic(ring, KM_USER0); - dprintk("leaving aio_read_evt: %d h%lu t%lu\n", ret, - (unsigned long)ring->head, (unsigned long)ring->tail); + dprintk("leaving aio_read_evt: %d h%u t%u\n", ret, + atomic_read(&ring->head), ring->tail); return ret; } diff --git a/include/linux/aio.h b/include/linux/aio.h index a30ef13..aebb135 100644 --- a/include/linux/aio.h +++ b/include/linux/aio.h @@ -144,7 +144,7 @@ struct kiocb { struct aio_ring { unsigned id; /* kernel internal index number */ unsigned nr; /* number of io_events */ - unsigned head; + atomic_t head; unsigned tail; unsigned magic; @@ -152,11 +152,11 @@ struct aio_ring { unsigned incompat_features; unsigned header_length; /* size of aio_ring */ - - struct io_event io_events[0]; + struct io_event io_events[0]; }; /* 128 bytes + ring size */ -#define aio_ring_avail(info, ring) (((ring)->head + (info)->nr - 1 - (ring)->tail) % (info)->nr) +#define aio_ring_avail(info, ring) \ + ((info)->nr + (unsigned) atomic_read(&(ring)->head) - (ring)->tail) #define AIO_RING_PAGES 8 struct aio_ring_info { @@ -164,7 +164,6 @@ struct aio_ring_info { unsigned long mmap_size; struct page **ring_pages; - spinlock_t ring_lock; long nr_pages; unsigned nr, tail;