1 files changed, 204 insertions, 213 deletions

diff --git a/drivers/tty/tty_buffer.c b/drivers/tty/tty_buffer.c
index 9121c1f7aeef..c043136fbe51 100644
--- a/drivers/tty/tty_buffer.c
+++ b/drivers/tty/tty_buffer.c
@@ -18,31 +18,118 @@
 #include <linux/module.h>
 #include <linux/ratelimit.h>
 
+
+#define MIN_TTYB_SIZE	256
+#define TTYB_ALIGN_MASK	255
+
+/*
+ * Byte threshold to limit memory consumption for flip buffers.
+ * The actual memory limit is > 2x this amount.
+ */
+#define TTYB_MEM_LIMIT	65536
+
+/*
+ * We default to dicing tty buffer allocations to this many characters
+ * in order to avoid multiple page allocations. We know the size of
+ * tty_buffer itself but it must also be taken into account that the
+ * the buffer is 256 byte aligned. See tty_buffer_find for the allocation
+ * logic this must match
+ */
+
+#define TTY_BUFFER_PAGE	(((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~0xFF)
+
+
+/**
+ *	tty_buffer_lock_exclusive	-	gain exclusive access to buffer
+ *	tty_buffer_unlock_exclusive	-	release exclusive access
+ *
+ *	@port - tty_port owning the flip buffer
+ *
+ *	Guarantees safe use of the line discipline's receive_buf() method by
+ *	excluding the buffer work and any pending flush from using the flip
+ *	buffer. Data can continue to be added concurrently to the flip buffer
+ *	from the driver side.
+ *
+ *	On release, the buffer work is restarted if there is data in the
+ *	flip buffer
+ */
+
+void tty_buffer_lock_exclusive(struct tty_port *port)
+{
+	struct tty_bufhead *buf = &port->buf;
+
+	atomic_inc(&buf->priority);
+	mutex_lock(&buf->lock);
+}
+
+void tty_buffer_unlock_exclusive(struct tty_port *port)
+{
+	struct tty_bufhead *buf = &port->buf;
+	int restart;
+
+	restart = buf->head->commit != buf->head->read;
+
+	atomic_dec(&buf->priority);
+	mutex_unlock(&buf->lock);
+	if (restart)
+		queue_work(system_unbound_wq, &buf->work);
+}
+
+/**
+ *	tty_buffer_space_avail	-	return unused buffer space
+ *	@port - tty_port owning the flip buffer
+ *
+ *	Returns the # of bytes which can be written by the driver without
+ *	reaching the buffer limit.
+ *
+ *	Note: this does not guarantee that memory is available to write
+ *	the returned # of bytes (use tty_prepare_flip_string_xxx() to
+ *	pre-allocate if memory guarantee is required).
+ */
+
+int tty_buffer_space_avail(struct tty_port *port)
+{
+	int space = TTYB_MEM_LIMIT - atomic_read(&port->buf.memory_used);
+	return max(space, 0);
+}
+
+static void tty_buffer_reset(struct tty_buffer *p, size_t size)
+{
+	p->used = 0;
+	p->size = size;
+	p->next = NULL;
+	p->commit = 0;
+	p->read = 0;
+}
+
 /**
  *	tty_buffer_free_all		-	free buffers used by a tty
  *	@tty: tty to free from
  *
  *	Remove all the buffers pending on a tty whether queued with data
  *	or in the free ring. Must be called when the tty is no longer in use
- *
- *	Locking: none
  */
 
 void tty_buffer_free_all(struct tty_port *port)
 {
 	struct tty_bufhead *buf = &port->buf;
-	struct tty_buffer *thead;
+	struct tty_buffer *p, *next;
+	struct llist_node *llist;
 
-	while ((thead = buf->head) != NULL) {
-		buf->head = thead->next;
-		kfree(thead);
-	}
-	while ((thead = buf->free) != NULL) {
-		buf->free = thead->next;
-		kfree(thead);
+	while ((p = buf->head) != NULL) {
+		buf->head = p->next;
+		if (p->size > 0)
+			kfree(p);
 	}
-	buf->tail = NULL;
-	buf->memory_used = 0;
+	llist = llist_del_all(&buf->free);
+	llist_for_each_entry_safe(p, next, llist, free)
+		kfree(p);
+
+	tty_buffer_reset(&buf->sentinel, 0);
+	buf->head = &buf->sentinel;
+	buf->tail = &buf->sentinel;
+
+	atomic_set(&buf->memory_used, 0);
 }
 
 /**
@@ -51,29 +138,39 @@ void tty_buffer_free_all(struct tty_port *port)
  *	@size: desired size (characters)
  *
  *	Allocate a new tty buffer to hold the desired number of characters.
+ *	We round our buffers off in 256 character chunks to get better
+ *	allocation behaviour.
  *	Return NULL if out of memory or the allocation would exceed the
  *	per device queue
- *
- *	Locking: Caller must hold tty->buf.lock
  */
 
 static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size)
 {
+	struct llist_node *free;
 	struct tty_buffer *p;
 
-	if (port->buf.memory_used + size > 65536)
+	/* Round the buffer size out */
+	size = __ALIGN_MASK(size, TTYB_ALIGN_MASK);
+
+	if (size <= MIN_TTYB_SIZE) {
+		free = llist_del_first(&port->buf.free);
+		if (free) {
+			p = llist_entry(free, struct tty_buffer, free);
+			goto found;
+		}
+	}
+
+	/* Should possibly check if this fails for the largest buffer we
+	   have queued and recycle that ? */
+	if (atomic_read(&port->buf.memory_used) > TTYB_MEM_LIMIT)
 		return NULL;
 	p = kmalloc(sizeof(struct tty_buffer) + 2 * size, GFP_ATOMIC);
 	if (p == NULL)
 		return NULL;
-	p->used = 0;
-	p->size = size;
-	p->next = NULL;
-	p->commit = 0;
-	p->read = 0;
-	p->char_buf_ptr = (char *)(p->data);
-	p->flag_buf_ptr = (unsigned char *)p->char_buf_ptr + size;
-	port->buf.memory_used += size;
+
+found:
+	tty_buffer_reset(p, size);
+	atomic_add(size, &port->buf.memory_used);
 	return p;
 }
 
@@ -84,8 +181,6 @@ static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size)
  *
  *	Free a tty buffer, or add it to the free list according to our
  *	internal strategy
- *
- *	Locking: Caller must hold tty->buf.lock
  */
 
 static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b)
@@ -93,41 +188,12 @@ static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b)
 	struct tty_bufhead *buf = &port->buf;
 
 	/* Dumb strategy for now - should keep some stats */
-	buf->memory_used -= b->size;
-	WARN_ON(buf->memory_used < 0);
+	WARN_ON(atomic_sub_return(b->size, &buf->memory_used) < 0);
 
-	if (b->size >= 512)
+	if (b->size > MIN_TTYB_SIZE)
 		kfree(b);
-	else {
-		b->next = buf->free;
-		buf->free = b;
-	}
-}
-
-/**
- *	__tty_buffer_flush		-	flush full tty buffers
- *	@tty: tty to flush
- *
- *	flush all the buffers containing receive data. Caller must
- *	hold the buffer lock and must have ensured no parallel flush to
- *	ldisc is running.
- *
- *	Locking: Caller must hold tty->buf.lock
- */
-
-static void __tty_buffer_flush(struct tty_port *port)
-{
-	struct tty_bufhead *buf = &port->buf;
-	struct tty_buffer *thead;
-
-	if (unlikely(buf->head == NULL))
-		return;
-	while ((thead = buf->head->next) != NULL) {
-		tty_buffer_free(port, buf->head);
-		buf->head = thead;
-	}
-	WARN_ON(buf->head != buf->tail);
-	buf->head->read = buf->head->commit;
+	else if (b->size > 0)
+		llist_add(&b->free, &buf->free);
 }
 
 /**
@@ -138,65 +204,28 @@ static void __tty_buffer_flush(struct tty_port *port)
  *	being processed by flush_to_ldisc then we defer the processing
  *	to that function
  *
- *	Locking: none
+ *	Locking: takes buffer lock to ensure single-threaded flip buffer
+ *		 'consumer'
  */
 
 void tty_buffer_flush(struct tty_struct *tty)
 {
 	struct tty_port *port = tty->port;
 	struct tty_bufhead *buf = &port->buf;
-	unsigned long flags;
-
-	spin_lock_irqsave(&buf->lock, flags);
-
-	/* If the data is being pushed to the tty layer then we can't
-	   process it here. Instead set a flag and the flush_to_ldisc
-	   path will process the flush request before it exits */
-	if (test_bit(TTYP_FLUSHING, &port->iflags)) {
-		set_bit(TTYP_FLUSHPENDING, &port->iflags);
-		spin_unlock_irqrestore(&buf->lock, flags);
-		wait_event(tty->read_wait,
-				test_bit(TTYP_FLUSHPENDING, &port->iflags) == 0);
-		return;
-	} else
-		__tty_buffer_flush(port);
-	spin_unlock_irqrestore(&buf->lock, flags);
-}
+	struct tty_buffer *next;
 
-/**
- *	tty_buffer_find		-	find a free tty buffer
- *	@tty: tty owning the buffer
- *	@size: characters wanted
- *
- *	Locate an existing suitable tty buffer or if we are lacking one then
- *	allocate a new one. We round our buffers off in 256 character chunks
- *	to get better allocation behaviour.
- *
- *	Locking: Caller must hold tty->buf.lock
- */
+	atomic_inc(&buf->priority);
 
-static struct tty_buffer *tty_buffer_find(struct tty_port *port, size_t size)
-{
-	struct tty_buffer **tbh = &port->buf.free;
-	while ((*tbh) != NULL) {
-		struct tty_buffer *t = *tbh;
-		if (t->size >= size) {
-			*tbh = t->next;
-			t->next = NULL;
-			t->used = 0;
-			t->commit = 0;
-			t->read = 0;
-			port->buf.memory_used += t->size;
-			return t;
-		}
-		tbh = &((*tbh)->next);
+	mutex_lock(&buf->lock);
+	while ((next = buf->head->next) != NULL) {
+		tty_buffer_free(port, buf->head);
+		buf->head = next;
 	}
-	/* Round the buffer size out */
-	size = (size + 0xFF) & ~0xFF;
-	return tty_buffer_alloc(port, size);
-	/* Should possibly check if this fails for the largest buffer we
-	   have queued and recycle that ? */
+	buf->head->read = buf->head->commit;
+	atomic_dec(&buf->priority);
+	mutex_unlock(&buf->lock);
 }
+
 /**
  *	tty_buffer_request_room		-	grow tty buffer if needed
  *	@tty: tty structure
@@ -204,38 +233,26 @@ static struct tty_buffer *tty_buffer_find(struct tty_port *port, size_t size)
  *
  *	Make at least size bytes of linear space available for the tty
  *	buffer. If we fail return the size we managed to find.
- *
- *	Locking: Takes port->buf.lock
  */
 int tty_buffer_request_room(struct tty_port *port, size_t size)
 {
 	struct tty_bufhead *buf = &port->buf;
 	struct tty_buffer *b, *n;
 	int left;
-	unsigned long flags;
-	spin_lock_irqsave(&buf->lock, flags);
-	/* OPTIMISATION: We could keep a per tty "zero" sized buffer to
-	   remove this conditional if its worth it. This would be invisible
-	   to the callers */
+
 	b = buf->tail;
-	if (b != NULL)
-		left = b->size - b->used;
-	else
-		left = 0;
+	left = b->size - b->used;
 
 	if (left < size) {
 		/* This is the slow path - looking for new buffers to use */
-		if ((n = tty_buffer_find(port, size)) != NULL) {
-			if (b != NULL) {
-				b->next = n;
-				b->commit = b->used;
-			} else
-				buf->head = n;
+		if ((n = tty_buffer_alloc(port, size)) != NULL) {
 			buf->tail = n;
+			b->commit = b->used;
+			smp_mb();
+			b->next = n;
 		} else
 			size = left;
 	}
-	spin_unlock_irqrestore(&buf->lock, flags);
 	return size;
 }
 EXPORT_SYMBOL_GPL(tty_buffer_request_room);
@@ -249,8 +266,6 @@ EXPORT_SYMBOL_GPL(tty_buffer_request_room);
  *
  *	Queue a series of bytes to the tty buffering. All the characters
  *	passed are marked with the supplied flag. Returns the number added.
- *
- *	Locking: Called functions may take port->buf.lock
  */
 
 int tty_insert_flip_string_fixed_flag(struct tty_port *port,
@@ -261,12 +276,10 @@ int tty_insert_flip_string_fixed_flag(struct tty_port *port,
 		int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
 		int space = tty_buffer_request_room(port, goal);
 		struct tty_buffer *tb = port->buf.tail;
-		/* If there is no space then tb may be NULL */
-		if (unlikely(space == 0)) {
+		if (unlikely(space == 0))
 			break;
-		}
-		memcpy(tb->char_buf_ptr + tb->used, chars, space);
-		memset(tb->flag_buf_ptr + tb->used, flag, space);
+		memcpy(char_buf_ptr(tb, tb->used), chars, space);
+		memset(flag_buf_ptr(tb, tb->used), flag, space);
 		tb->used += space;
 		copied += space;
 		chars += space;
@@ -287,8 +300,6 @@ EXPORT_SYMBOL(tty_insert_flip_string_fixed_flag);
  *	Queue a series of bytes to the tty buffering. For each character
  *	the flags array indicates the status of the character. Returns the
  *	number added.
- *
- *	Locking: Called functions may take port->buf.lock
  */
 
 int tty_insert_flip_string_flags(struct tty_port *port,
@@ -299,12 +310,10 @@ int tty_insert_flip_string_flags(struct tty_port *port,
 		int goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE);
 		int space = tty_buffer_request_room(port, goal);
 		struct tty_buffer *tb = port->buf.tail;
-		/* If there is no space then tb may be NULL */
-		if (unlikely(space == 0)) {
+		if (unlikely(space == 0))
 			break;
-		}
-		memcpy(tb->char_buf_ptr + tb->used, chars, space);
-		memcpy(tb->flag_buf_ptr + tb->used, flags, space);
+		memcpy(char_buf_ptr(tb, tb->used), chars, space);
+		memcpy(flag_buf_ptr(tb, tb->used), flags, space);
 		tb->used += space;
 		copied += space;
 		chars += space;
@@ -325,20 +334,14 @@ EXPORT_SYMBOL(tty_insert_flip_string_flags);
  *	processing by the line discipline.
  *	Note that this function can only be used when the low_latency flag
  *	is unset. Otherwise the workqueue won't be flushed.
- *
- *	Locking: Takes port->buf.lock
  */
 
 void tty_schedule_flip(struct tty_port *port)
 {
 	struct tty_bufhead *buf = &port->buf;
-	unsigned long flags;
 	WARN_ON(port->low_latency);
 
-	spin_lock_irqsave(&buf->lock, flags);
-	if (buf->tail != NULL)
-		buf->tail->commit = buf->tail->used;
-	spin_unlock_irqrestore(&buf->lock, flags);
+	buf->tail->commit = buf->tail->used;
 	schedule_work(&buf->work);
 }
 EXPORT_SYMBOL(tty_schedule_flip);
@@ -354,8 +357,6 @@ EXPORT_SYMBOL(tty_schedule_flip);
  *	accounted for as ready for normal characters. This is used for drivers
  *	that need their own block copy routines into the buffer. There is no
  *	guarantee the buffer is a DMA target!
- *
- *	Locking: May call functions taking port->buf.lock
  */
 
 int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars,
@@ -364,8 +365,8 @@ int tty_prepare_flip_string(struct tty_port *port, unsigned char **chars,
 	int space = tty_buffer_request_room(port, size);
 	if (likely(space)) {
 		struct tty_buffer *tb = port->buf.tail;
-		*chars = tb->char_buf_ptr + tb->used;
-		memset(tb->flag_buf_ptr + tb->used, TTY_NORMAL, space);
+		*chars = char_buf_ptr(tb, tb->used);
+		memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space);
 		tb->used += space;
 	}
 	return space;
@@ -384,8 +385,6 @@ EXPORT_SYMBOL_GPL(tty_prepare_flip_string);
  *	accounted for as ready for characters. This is used for drivers
  *	that need their own block copy routines into the buffer. There is no
  *	guarantee the buffer is a DMA target!
- *
- *	Locking: May call functions taking port->buf.lock
  */
 
 int tty_prepare_flip_string_flags(struct tty_port *port,
@@ -394,8 +393,8 @@ int tty_prepare_flip_string_flags(struct tty_port *port,
 	int space = tty_buffer_request_room(port, size);
 	if (likely(space)) {
 		struct tty_buffer *tb = port->buf.tail;
-		*chars = tb->char_buf_ptr + tb->used;
-		*flags = tb->flag_buf_ptr + tb->used;
+		*chars = char_buf_ptr(tb, tb->used);
+		*flags = flag_buf_ptr(tb, tb->used);
 		tb->used += space;
 	}
 	return space;
@@ -403,6 +402,23 @@ int tty_prepare_flip_string_flags(struct tty_port *port,
 EXPORT_SYMBOL_GPL(tty_prepare_flip_string_flags);
 
 
+static int
+receive_buf(struct tty_struct *tty, struct tty_buffer *head, int count)
+{
+	struct tty_ldisc *disc = tty->ldisc;
+	unsigned char *p = char_buf_ptr(head, head->read);
+	char	      *f = flag_buf_ptr(head, head->read);
+
+	if (disc->ops->receive_buf2)
+		count = disc->ops->receive_buf2(tty, p, f, count);
+	else {
+		count = min_t(int, count, tty->receive_room);
+		if (count)
+			disc->ops->receive_buf(tty, p, f, count);
+	}
+	head->read += count;
+	return count;
+}
 
 /**
  *	flush_to_ldisc
@@ -411,9 +427,10 @@ EXPORT_SYMBOL_GPL(tty_prepare_flip_string_flags);
  *	This routine is called out of the software interrupt to flush data
  *	from the buffer chain to the line discipline.
  *
- *	Locking: holds tty->buf.lock to guard buffer list. Drops the lock
- *	while invoking the line discipline receive_buf method. The
- *	receive_buf method is single threaded for each tty instance.
+ *	The receive_buf method is single threaded for each tty instance.
+ *
+ *	Locking: takes buffer lock to ensure single-threaded flip buffer
+ *		 'consumer'
  */
 
 static void flush_to_ldisc(struct work_struct *work)
@@ -421,7 +438,6 @@ static void flush_to_ldisc(struct work_struct *work)
 	struct tty_port *port = container_of(work, struct tty_port, buf.work);
 	struct tty_bufhead *buf = &port->buf;
 	struct tty_struct *tty;
-	unsigned long 	flags;
 	struct tty_ldisc *disc;
 
 	tty = port->itty;
@@ -429,52 +445,34 @@ static void flush_to_ldisc(struct work_struct *work)
 		return;
 
 	disc = tty_ldisc_ref(tty);
-	if (disc == NULL)	/*  !TTY_LDISC */
+	if (disc == NULL)
 		return;
 
-	spin_lock_irqsave(&buf->lock, flags);
-
-	if (!test_and_set_bit(TTYP_FLUSHING, &port->iflags)) {
-		struct tty_buffer *head;
-		while ((head = buf->head) != NULL) {
-			int count;
-			char *char_buf;
-			unsigned char *flag_buf;
-
-			count = head->commit - head->read;
-			if (!count) {
-				if (head->next == NULL)
-					break;
-				buf->head = head->next;
-				tty_buffer_free(port, head);
-				continue;
-			}
-			if (!tty->receive_room)
-				break;
-			if (count > tty->receive_room)
-				count = tty->receive_room;
-			char_buf = head->char_buf_ptr + head->read;
-			flag_buf = head->flag_buf_ptr + head->read;
-			head->read += count;
-			spin_unlock_irqrestore(&buf->lock, flags);
-			disc->ops->receive_buf(tty, char_buf,
-							flag_buf, count);
-			spin_lock_irqsave(&buf->lock, flags);
-			/* Ldisc or user is trying to flush the buffers.
-			   We may have a deferred request to flush the
-			   input buffer, if so pull the chain under the lock
-			   and empty the queue */
-			if (test_bit(TTYP_FLUSHPENDING, &port->iflags)) {
-				__tty_buffer_flush(port);
-				clear_bit(TTYP_FLUSHPENDING, &port->iflags);
-				wake_up(&tty->read_wait);
+	mutex_lock(&buf->lock);
+
+	while (1) {
+		struct tty_buffer *head = buf->head;
+		int count;
+
+		/* Ldisc or user is trying to gain exclusive access */
+		if (atomic_read(&buf->priority))
+			break;
+
+		count = head->commit - head->read;
+		if (!count) {
+			if (head->next == NULL)
 				break;
-			}
+			buf->head = head->next;
+			tty_buffer_free(port, head);
+			continue;
 		}
-		clear_bit(TTYP_FLUSHING, &port->iflags);
+
+		count = receive_buf(tty, head, count);
+		if (!count)
+			break;
 	}
 
-	spin_unlock_irqrestore(&buf->lock, flags);
+	mutex_unlock(&buf->lock);
 
 	tty_ldisc_deref(disc);
 }
@@ -503,19 +501,13 @@ void tty_flush_to_ldisc(struct tty_struct *tty)
  *
  *	In the event of the queue being busy for flipping the work will be
  *	held off and retried later.
- *
- *	Locking: tty buffer lock. Driver locks in low latency mode.
  */
 
 void tty_flip_buffer_push(struct tty_port *port)
 {
 	struct tty_bufhead *buf = &port->buf;
-	unsigned long flags;
 
-	spin_lock_irqsave(&buf->lock, flags);
-	if (buf->tail != NULL)
-		buf->tail->commit = buf->tail->used;
-	spin_unlock_irqrestore(&buf->lock, flags);
+	buf->tail->commit = buf->tail->used;
 
 	if (port->low_latency)
 		flush_to_ldisc(&buf->work);
@@ -530,19 +522,18 @@ EXPORT_SYMBOL(tty_flip_buffer_push);
  *
  *	Set up the initial state of the buffer management for a tty device.
  *	Must be called before the other tty buffer functions are used.
- *
- *	Locking: none
  */
 
 void tty_buffer_init(struct tty_port *port)
 {
 	struct tty_bufhead *buf = &port->buf;
 
-	spin_lock_init(&buf->lock);
-	buf->head = NULL;
-	buf->tail = NULL;
-	buf->free = NULL;
-	buf->memory_used = 0;
+	mutex_init(&buf->lock);
+	tty_buffer_reset(&buf->sentinel, 0);
+	buf->head = &buf->sentinel;
+	buf->tail = &buf->sentinel;
+	init_llist_head(&buf->free);
+	atomic_set(&buf->memory_used, 0);
+	atomic_set(&buf->priority, 0);
 	INIT_WORK(&buf->work, flush_to_ldisc);
 }
-