clone

server/internal/circ/buffer.go

@@ -0,0 +1,212 @@
+package circ
+
+import (
+	"errors"
+	"io"
+	"sync"
+	"sync/atomic"
+)
+
+var (
+	// DefaultBufferSize is the default size of the buffer in bytes.
+	DefaultBufferSize int = 1024 * 256
+
+	// DefaultBlockSize is the default size per R/W block in bytes.
+	DefaultBlockSize int = 1024 * 8
+
+	// ErrOutOfRange indicates that the index was out of range.
+	ErrOutOfRange = errors.New("Indexes out of range")
+
+	// ErrInsufficientBytes indicates that there were not enough bytes to return.
+	ErrInsufficientBytes = errors.New("Insufficient bytes to return")
+)
+
+// Buffer is a circular buffer for reading and writing messages.
+type Buffer struct {
+	buf   []byte       // the bytes buffer.
+	tmp   []byte       // a temporary buffer.
+	Mu    sync.RWMutex // the buffer needs its own mutex to work properly.
+	ID    string       // the identifier of the buffer. This is used in debug output.
+	head  int64        // the current position in the sequence - a forever increasing index.
+	tail  int64        // the committed position in the sequence - a forever increasing index.
+	rcond *sync.Cond   // the sync condition for the buffer reader.
+	wcond *sync.Cond   // the sync condition for the buffer writer.
+	size  int          // the size of the buffer.
+	mask  int          // a bitmask of the buffer size (size-1).
+	block int          // the size of the R/W block.
+	done  uint32       // indicates that the buffer is closed.
+	State uint32       // indicates whether the buffer is reading from (1) or writing to (2).
+}
+
+// NewBuffer returns a new instance of buffer. You should call NewReader or
+// NewWriter instead of this function.
+func NewBuffer(size, block int) *Buffer {
+	if size == 0 {
+		size = DefaultBufferSize
+	}
+
+	if block == 0 {
+		block = DefaultBlockSize
+	}
+
+	if size < 2*block {
+		size = 2 * block
+	}
+
+	return &Buffer{
+		size:  size,
+		mask:  size - 1,
+		block: block,
+		buf:   make([]byte, size),
+		rcond: sync.NewCond(new(sync.Mutex)),
+		wcond: sync.NewCond(new(sync.Mutex)),
+	}
+}
+
+// NewBufferFromSlice returns a new instance of buffer using a
+// pre-existing byte slice.
+func NewBufferFromSlice(block int, buf []byte) *Buffer {
+	l := len(buf)
+
+	if block == 0 {
+		block = DefaultBlockSize
+	}
+
+	b := &Buffer{
+		size:  l,
+		mask:  l - 1,
+		block: block,
+		buf:   buf,
+		rcond: sync.NewCond(new(sync.Mutex)),
+		wcond: sync.NewCond(new(sync.Mutex)),
+	}
+
+	return b
+}
+
+// GetPos will return the tail and head positions of the buffer.
+// This method is for use with testing.
+func (b *Buffer) GetPos() (int64, int64) {
+	return atomic.LoadInt64(&b.tail), atomic.LoadInt64(&b.head)
+}
+
+// SetPos sets the head and tail of the buffer.
+func (b *Buffer) SetPos(tail, head int64) {
+	atomic.StoreInt64(&b.tail, tail)
+	atomic.StoreInt64(&b.head, head)
+}
+
+// Get returns the internal buffer.
+func (b *Buffer) Get() []byte {
+	b.Mu.Lock()
+	defer b.Mu.Unlock()
+	return b.buf
+}
+
+// Set writes bytes to a range of indexes in the byte buffer.
+func (b *Buffer) Set(p []byte, start, end int) error {
+	b.Mu.Lock()
+	defer b.Mu.Unlock()
+
+	if end > b.size || start > b.size {
+		return ErrOutOfRange
+	}
+
+	o := 0
+	for i := start; i < end; i++ {
+		b.buf[i] = p[o]
+		o++
+	}
+
+	return nil
+}
+
+// Index returns the buffer-relative index of an integer.
+func (b *Buffer) Index(i int64) int {
+	return b.mask & int(i)
+}
+
+// awaitEmpty will block until there is at least n bytes between
+// the head and the tail (looking forward).
+func (b *Buffer) awaitEmpty(n int) error {
+	// If the head has wrapped behind the tail, and next will overrun tail,
+	// then wait until tail has moved.
+	b.rcond.L.Lock()
+	for !b.checkEmpty(n) {
+		if atomic.LoadUint32(&b.done) == 1 {
+			b.rcond.L.Unlock()
+			return io.EOF
+		}
+		b.rcond.Wait()
+	}
+	b.rcond.L.Unlock()
+
+	return nil
+}
+
+// awaitFilled will block until there are at least n bytes between the
+// tail and the head (looking forward).
+func (b *Buffer) awaitFilled(n int) error {
+	// Because awaitCapacity prevents the head from overrunning the t
+	// able on write, we can simply ensure there is enough space
+	// the forever-incrementing tail and head integers.
+	b.wcond.L.Lock()
+	for !b.checkFilled(n) {
+		if atomic.LoadUint32(&b.done) == 1 {
+			b.wcond.L.Unlock()
+			return io.EOF
+		}
+
+		b.wcond.Wait()
+	}
+	b.wcond.L.Unlock()
+
+	return nil
+}
+
+// checkEmpty returns true if there are at least n bytes between the head and
+// the tail.
+func (b *Buffer) checkEmpty(n int) bool {
+	head := atomic.LoadInt64(&b.head)
+	next := head + int64(n)
+	tail := atomic.LoadInt64(&b.tail)
+	if next-tail > int64(b.size) {
+		return false
+	}
+
+	return true
+}
+
+// checkFilled returns true if there are at least n bytes between the tail and
+// the head.
+func (b *Buffer) checkFilled(n int) bool {
+	if atomic.LoadInt64(&b.tail)+int64(n) <= atomic.LoadInt64(&b.head) {
+		return true
+	}
+
+	return false
+}
+
+// CommitTail moves the tail position of the buffer n bytes.
+func (b *Buffer) CommitTail(n int) {
+	atomic.AddInt64(&b.tail, int64(n))
+	b.rcond.L.Lock()
+	b.rcond.Broadcast()
+	b.rcond.L.Unlock()
+}
+
+// CapDelta returns the difference between the head and tail.
+func (b *Buffer) CapDelta() int {
+	return int(atomic.LoadInt64(&b.head) - atomic.LoadInt64(&b.tail))
+}
+
+// Stop signals the buffer to stop processing.
+func (b *Buffer) Stop() {
+	atomic.StoreUint32(&b.done, 1)
+	b.rcond.L.Lock()
+	b.rcond.Broadcast()
+	b.rcond.L.Unlock()
+	b.wcond.L.Lock()
+	b.wcond.Broadcast()
+	b.wcond.L.Unlock()
+}

server/internal/circ/buffer_test.go

@@ -0,0 +1,317 @@
+package circ
+
+import (
+	//"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+	"unsafe"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewBuffer(t *testing.T) {
+	var size int = 16
+	var block int = 4
+	buf := NewBuffer(size, block)
+
+	require.NotNil(t, buf.buf)
+	require.NotNil(t, buf.rcond)
+	require.NotNil(t, buf.wcond)
+	require.Equal(t, size, len(buf.buf))
+	require.Equal(t, size, buf.size)
+	require.Equal(t, block, buf.block)
+}
+
+func TestNewBuffer0Size(t *testing.T) {
+	buf := NewBuffer(0, 0)
+	require.NotNil(t, buf.buf)
+	require.Equal(t, DefaultBufferSize, buf.size)
+	require.Equal(t, DefaultBlockSize, buf.block)
+}
+
+func TestNewBufferUndersize(t *testing.T) {
+	buf := NewBuffer(DefaultBlockSize+10, DefaultBlockSize)
+	require.NotNil(t, buf.buf)
+	require.Equal(t, DefaultBlockSize*2, buf.size)
+	require.Equal(t, DefaultBlockSize, buf.block)
+}
+
+func TestNewBufferFromSlice(t *testing.T) {
+	b := NewBytesPool(256)
+	buf := NewBufferFromSlice(DefaultBlockSize, b.Get())
+	require.NotNil(t, buf.buf)
+	require.Equal(t, 256, cap(buf.buf))
+}
+
+func TestNewBufferFromSlice0Size(t *testing.T) {
+	b := NewBytesPool(256)
+	buf := NewBufferFromSlice(0, b.Get())
+	require.NotNil(t, buf.buf)
+	require.Equal(t, 256, cap(buf.buf))
+}
+
+func TestAtomicAlignment(t *testing.T) {
+	var b Buffer
+
+	offset := unsafe.Offsetof(b.head)
+	require.Equalf(t, uintptr(0), offset%8,
+		"head requires 64-bit alignment for atomic: offset %d", offset)
+
+	offset = unsafe.Offsetof(b.tail)
+	require.Equalf(t, uintptr(0), offset%8,
+		"tail requires 64-bit alignment for atomic: offset %d", offset)
+}
+
+func TestGetPos(t *testing.T) {
+	buf := NewBuffer(16, 4)
+	tail, head := buf.GetPos()
+	require.Equal(t, int64(0), tail)
+	require.Equal(t, int64(0), head)
+
+	atomic.StoreInt64(&buf.tail, 3)
+	atomic.StoreInt64(&buf.head, 11)
+
+	tail, head = buf.GetPos()
+	require.Equal(t, int64(3), tail)
+	require.Equal(t, int64(11), head)
+}
+
+func TestGet(t *testing.T) {
+	buf := NewBuffer(16, 4)
+	require.Equal(t, make([]byte, 16), buf.Get())
+
+	buf.buf[0] = 1
+	buf.buf[15] = 1
+	require.Equal(t, []byte{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, buf.Get())
+}
+
+func TestSetPos(t *testing.T) {
+	buf := NewBuffer(16, 4)
+	require.Equal(t, int64(0), atomic.LoadInt64(&buf.tail))
+	require.Equal(t, int64(0), atomic.LoadInt64(&buf.head))
+
+	buf.SetPos(4, 8)
+	require.Equal(t, int64(4), atomic.LoadInt64(&buf.tail))
+	require.Equal(t, int64(8), atomic.LoadInt64(&buf.head))
+}
+
+func TestSet(t *testing.T) {
+	buf := NewBuffer(16, 4)
+	err := buf.Set([]byte{1, 1, 1, 1}, 17, 19)
+	require.Error(t, err)
+
+	err = buf.Set([]byte{1, 1, 1, 1}, 4, 8)
+	require.NoError(t, err)
+	require.Equal(t, []byte{0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0}, buf.buf)
+}
+
+func TestIndex(t *testing.T) {
+	buf := NewBuffer(1024, 4)
+	require.Equal(t, 512, buf.Index(512))
+	require.Equal(t, 0, buf.Index(1024))
+	require.Equal(t, 6, buf.Index(1030))
+	require.Equal(t, 6, buf.Index(61446))
+}
+
+func TestAwaitFilled(t *testing.T) {
+	tests := []struct {
+		tail  int64
+		head  int64
+		n     int
+		await int
+		desc  string
+	}{
+		{tail: 0, head: 4, n: 4, await: 1, desc: "OK 0, 4"},
+		{tail: 8, head: 11, n: 4, await: 1, desc: "OK 8, 11"},
+		{tail: 102, head: 103, n: 4, await: 3, desc: "OK 102, 103"},
+	}
+
+	for i, tt := range tests {
+		//fmt.Println(i)
+		buf := NewBuffer(16, 4)
+		buf.SetPos(tt.tail, tt.head)
+		o := make(chan error)
+		go func() {
+			o <- buf.awaitFilled(4)
+		}()
+
+		time.Sleep(time.Millisecond)
+		atomic.AddInt64(&buf.head, int64(tt.await))
+		buf.wcond.L.Lock()
+		buf.wcond.Broadcast()
+		buf.wcond.L.Unlock()
+
+		require.NoError(t, <-o, "Unexpected Error [i:%d] %s", i, tt.desc)
+	}
+}
+
+func TestAwaitFilledEnded(t *testing.T) {
+	buf := NewBuffer(16, 4)
+	o := make(chan error)
+	go func() {
+		o <- buf.awaitFilled(4)
+	}()
+	time.Sleep(time.Millisecond)
+	atomic.StoreUint32(&buf.done, 1)
+	buf.wcond.L.Lock()
+	buf.wcond.Broadcast()
+	buf.wcond.L.Unlock()
+
+	require.Error(t, <-o)
+}
+
+func TestAwaitEmptyOK(t *testing.T) {
+	tests := []struct {
+		tail  int64
+		head  int64
+		await int
+		desc  string
+	}{
+		{tail: 0, head: 0, await: 0, desc: "OK 0, 0"},
+		{tail: 0, head: 5, await: 0, desc: "OK 0, 5"},
+		{tail: 0, head: 14, await: 3, desc: "OK wrap 0, 14 "},
+		{tail: 22, head: 35, await: 2, desc: "OK wrap 0, 14 "},
+		{tail: 15, head: 17, await: 7, desc: "OK 15,2"},
+		{tail: 0, head: 10, await: 2, desc: "OK 0, 10"},
+		{tail: 1, head: 15, await: 4, desc: "OK 2, 14"},
+	}
+
+	for i, tt := range tests {
+		buf := NewBuffer(16, 4)
+		buf.SetPos(tt.tail, tt.head)
+		o := make(chan error)
+		go func() {
+			o <- buf.awaitEmpty(4)
+		}()
+
+		time.Sleep(time.Millisecond)
+		atomic.AddInt64(&buf.tail, int64(tt.await))
+		buf.rcond.L.Lock()
+		buf.rcond.Broadcast()
+		buf.rcond.L.Unlock()
+
+		require.NoError(t, <-o, "Unexpected Error [i:%d] %s", i, tt.desc)
+	}
+}
+
+func TestAwaitEmptyEnded(t *testing.T) {
+	buf := NewBuffer(16, 4)
+	buf.SetPos(1, 15)
+	o := make(chan error)
+	go func() {
+		o <- buf.awaitEmpty(4)
+	}()
+	time.Sleep(time.Millisecond)
+	atomic.StoreUint32(&buf.done, 1)
+	buf.rcond.L.Lock()
+	buf.rcond.Broadcast()
+	buf.rcond.L.Unlock()
+
+	require.Error(t, <-o)
+}
+
+func TestCheckEmpty(t *testing.T) {
+	buf := NewBuffer(16, 4)
+
+	tests := []struct {
+		head int64
+		tail int64
+		want bool
+		desc string
+	}{
+		{tail: 0, head: 0, want: true, desc: "0, 0 true"},
+		{tail: 3, head: 4, want: true, desc: "4, 3 true"},
+		{tail: 15, head: 17, want: true, desc: "15, 17(1) true"},
+		{tail: 1, head: 30, want: false, desc: "1, 30(14) false"},
+		{tail: 15, head: 30, want: false, desc: "15, 30(14) false; head has caught up to tail"},
+	}
+	for i, tt := range tests {
+		buf.SetPos(tt.tail, tt.head)
+		require.Equal(t, tt.want, buf.checkEmpty(4), "Mismatched bool wanted [i:%d] %s", i, tt.desc)
+	}
+}
+
+func TestCheckFilled(t *testing.T) {
+	buf := NewBuffer(16, 4)
+
+	tests := []struct {
+		head int64
+		tail int64
+		want bool
+		desc string
+	}{
+		{tail: 0, head: 0, want: false, desc: "0, 0 false"},
+		{tail: 0, head: 4, want: true, desc: "0, 4 true"},
+		{tail: 14, head: 16, want: false, desc: "14,16 false"},
+		{tail: 14, head: 18, want: true, desc: "14,16 true"},
+	}
+
+	for i, tt := range tests {
+		buf.SetPos(tt.tail, tt.head)
+		require.Equal(t, tt.want, buf.checkFilled(4), "Mismatched bool wanted [i:%d] %s", i, tt.desc)
+	}
+
+}
+
+func TestCommitTail(t *testing.T) {
+	tests := []struct {
+		tail  int64
+		head  int64
+		n     int
+		next  int64
+		await int
+		desc  string
+	}{
+		{tail: 0, head: 5, n: 4, next: 4, await: 0, desc: "OK 0, 4"},
+		{tail: 0, head: 5, n: 6, next: 6, await: 1, desc: "OK 0, 5"},
+	}
+
+	for i, tt := range tests {
+		buf := NewBuffer(16, 4)
+		buf.SetPos(tt.tail, tt.head)
+		go func() {
+			buf.CommitTail(tt.n)
+		}()
+
+		time.Sleep(time.Millisecond)
+		for j := 0; j < tt.await; j++ {
+			atomic.AddInt64(&buf.head, 1)
+			buf.wcond.L.Lock()
+			buf.wcond.Broadcast()
+			buf.wcond.L.Unlock()
+		}
+		require.Equal(t, tt.next, atomic.LoadInt64(&buf.tail), "Next tail mismatch [i:%d] %s", i, tt.desc)
+	}
+}
+
+/*
+func TestCommitTailEnded(t *testing.T) {
+	buf := NewBuffer(16, 4)
+	o := make(chan error)
+	go func() {
+		o <- buf.CommitTail(5)
+	}()
+	time.Sleep(time.Millisecond)
+	atomic.StoreUint32(&buf.done, 1)
+	buf.wcond.L.Lock()
+	buf.wcond.Broadcast()
+	buf.wcond.L.Unlock()
+
+	require.Error(t, <-o)
+}
+*/
+func TestCapDelta(t *testing.T) {
+	buf := NewBuffer(16, 4)
+
+	require.Equal(t, 0, buf.CapDelta())
+
+	buf.SetPos(10, 15)
+	require.Equal(t, 5, buf.CapDelta())
+}
+
+func TestStop(t *testing.T) {
+	buf := NewBuffer(16, 4)
+	buf.Stop()
+	require.Equal(t, uint32(1), buf.done)
+}

server/internal/circ/pool.go

@@ -0,0 +1,49 @@
+package circ
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// BytesPool is a pool of []byte.
+type BytesPool struct {
+	// int64/uint64 has to the first words in order
+	// to be 64-aligned on 32-bit architectures.
+	used int64 // access atomically
+	pool *sync.Pool
+}
+
+// NewBytesPool returns a sync.pool of []byte.
+func NewBytesPool(n int) *BytesPool {
+	if n == 0 {
+		n = DefaultBufferSize
+	}
+
+	return &BytesPool{
+		pool: &sync.Pool{
+			New: func() interface{} {
+				return make([]byte, n)
+			},
+		},
+	}
+}
+
+// Get returns a pooled bytes.Buffer.
+func (b *BytesPool) Get() []byte {
+	atomic.AddInt64(&b.used, 1)
+	return b.pool.Get().([]byte)
+}
+
+// Put puts the byte slice back into the pool.
+func (b *BytesPool) Put(x []byte) {
+	for i := range x {
+		x[i] = 0
+	}
+	b.pool.Put(x)
+	atomic.AddInt64(&b.used, -1)
+}
+
+// InUse returns the number of pool blocks in use.
+func (b *BytesPool) InUse() int64 {
+	return atomic.LoadInt64(&b.used)
+}

server/internal/circ/pool_test.go

@@ -0,0 +1,49 @@
+package circ
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewBytesPool(t *testing.T) {
+	bpool := NewBytesPool(256)
+	require.NotNil(t, bpool.pool)
+}
+
+func BenchmarkNewBytesPool(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		NewBytesPool(256)
+	}
+}
+
+func TestNewBytesPoolGet(t *testing.T) {
+	bpool := NewBytesPool(256)
+	buf := bpool.Get()
+
+	require.Equal(t, make([]byte, 256), buf)
+	require.Equal(t, int64(1), bpool.InUse())
+}
+
+func BenchmarkBytesPoolGet(b *testing.B) {
+	bpool := NewBytesPool(256)
+	for n := 0; n < b.N; n++ {
+		bpool.Get()
+	}
+}
+
+func TestNewBytesPoolPut(t *testing.T) {
+	bpool := NewBytesPool(256)
+	buf := bpool.Get()
+	require.Equal(t, int64(1), bpool.InUse())
+	bpool.Put(buf)
+	require.Equal(t, int64(0), bpool.InUse())
+}
+
+func BenchmarkBytesPoolPut(b *testing.B) {
+	bpool := NewBytesPool(256)
+	buf := bpool.Get()
+	for n := 0; n < b.N; n++ {
+		bpool.Put(buf)
+	}
+}

server/internal/circ/reader.go

@@ -0,0 +1,96 @@
+package circ
+
+import (
+	"io"
+	"sync/atomic"
+)
+
+// Reader is a circular buffer for reading data from an io.Reader.
+type Reader struct {
+	*Buffer
+}
+
+// NewReader returns a new Circular Reader.
+func NewReader(size, block int) *Reader {
+	b := NewBuffer(size, block)
+	b.ID = "\treader"
+	return &Reader{
+		b,
+	}
+}
+
+// NewReaderFromSlice returns a new Circular Reader using a pre-existing
+// byte slice.
+func NewReaderFromSlice(block int, p []byte) *Reader {
+	b := NewBufferFromSlice(block, p)
+	b.ID = "\treader"
+	return &Reader{
+		b,
+	}
+}
+
+// ReadFrom reads bytes from an io.Reader and commits them to the buffer when
+// there is sufficient capacity to do so.
+func (b *Reader) ReadFrom(r io.Reader) (total int64, err error) {
+	atomic.StoreUint32(&b.State, 1)
+	defer atomic.StoreUint32(&b.State, 0)
+	for {
+		if atomic.LoadUint32(&b.done) == 1 {
+			return total, nil
+		}
+
+		// Wait until there's enough capacity in the buffer before
+		// trying to read more bytes from the io.Reader.
+		err := b.awaitEmpty(b.block)
+		if err != nil {
+			// b.done is the only error condition for awaitCapacity
+			// so loop around and return properly.
+			continue
+		}
+
+		// If the block will overrun the circle end, just fill up
+		// and collect the rest on the next pass.
+		start := b.Index(atomic.LoadInt64(&b.head))
+		end := start + b.block
+		if end > b.size {
+			end = b.size
+		}
+
+		// Read into the buffer between the start and end indexes only.
+		n, err := r.Read(b.buf[start:end])
+		total += int64(n) // incr total bytes read.
+		if err != nil {
+			return total, err
+		}
+
+		// Move the head forward however many bytes were read.
+		atomic.AddInt64(&b.head, int64(n))
+
+		b.wcond.L.Lock()
+		b.wcond.Broadcast()
+		b.wcond.L.Unlock()
+	}
+}
+
+// Read reads n bytes from the buffer, and will block until at n bytes
+// exist in the buffer to read.
+func (b *Buffer) Read(n int) (p []byte, err error) {
+	err = b.awaitFilled(n)
+	if err != nil {
+		return
+	}
+
+	tail := atomic.LoadInt64(&b.tail)
+	next := tail + int64(n)
+
+	// If the read overruns the buffer, get everything until the end
+	// and then whatever is left from the start.
+	if b.Index(tail) > b.Index(next) {
+		b.tmp = b.buf[b.Index(tail):]
+		b.tmp = append(b.tmp, b.buf[:b.Index(next)]...)
+	} else {
+		b.tmp = b.buf[b.Index(tail):b.Index(next)] // Otherwise, simple tail:next read.
+	}
+
+	return b.tmp, nil
+}

server/internal/circ/reader_test.go

@@ -0,0 +1,129 @@
+package circ
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewReader(t *testing.T) {
+	var size = 16
+	var block = 4
+	buf := NewReader(size, block)
+
+	require.NotNil(t, buf.buf)
+	require.Equal(t, size, len(buf.buf))
+	require.Equal(t, size, buf.size)
+	require.Equal(t, block, buf.block)
+}
+
+func TestNewReaderFromSlice(t *testing.T) {
+	b := NewBytesPool(256)
+	buf := NewReaderFromSlice(DefaultBlockSize, b.Get())
+	require.NotNil(t, buf.buf)
+	require.Equal(t, 256, cap(buf.buf))
+}
+
+func TestReadFrom(t *testing.T) {
+	buf := NewReader(16, 4)
+
+	b4 := bytes.Repeat([]byte{'-'}, 4)
+	br := bytes.NewReader(b4)
+
+	_, err := buf.ReadFrom(br)
+	require.True(t, errors.Is(err, io.EOF))
+	require.Equal(t, bytes.Repeat([]byte{'-'}, 4), buf.buf[:4])
+	require.Equal(t, int64(4), buf.head)
+
+	br.Reset(b4)
+	_, err = buf.ReadFrom(br)
+	require.True(t, errors.Is(err, io.EOF))
+	require.Equal(t, int64(8), buf.head)
+
+	br.Reset(b4)
+	_, err = buf.ReadFrom(br)
+	require.True(t, errors.Is(err, io.EOF))
+	require.Equal(t, int64(12), buf.head)
+}
+
+func TestReadFromWrap(t *testing.T) {
+	buf := NewReader(16, 4)
+	buf.buf = bytes.Repeat([]byte{'-'}, 16)
+	buf.SetPos(8, 14)
+	br := bytes.NewReader(bytes.Repeat([]byte{'/'}, 8))
+
+	o := make(chan error)
+	go func() {
+		_, err := buf.ReadFrom(br)
+		o <- err
+	}()
+	time.Sleep(time.Millisecond * 100)
+	go func() {
+		atomic.StoreUint32(&buf.done, 1)
+		buf.rcond.L.Lock()
+		buf.rcond.Broadcast()
+		buf.rcond.L.Unlock()
+	}()
+	<-o
+	require.Equal(t, []byte{'/', '/', '/', '/', '/', '/', '-', '-', '-', '-', '-', '-', '-', '-', '/', '/'}, buf.Get())
+	require.Equal(t, int64(22), atomic.LoadInt64(&buf.head))
+	require.Equal(t, 6, buf.Index(atomic.LoadInt64(&buf.head)))
+}
+
+func TestReadOK(t *testing.T) {
+	buf := NewReader(16, 4)
+	buf.buf = []byte{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p'}
+
+	tests := []struct {
+		tail  int64
+		head  int64
+		n     int
+		bytes []byte
+		desc  string
+	}{
+		{tail: 0, head: 4, n: 4, bytes: []byte{'a', 'b', 'c', 'd'}, desc: "0, 4 OK"},
+		{tail: 3, head: 15, n: 8, bytes: []byte{'d', 'e', 'f', 'g', 'h', 'i', 'j', 'k'}, desc: "3, 15 OK"},
+		{tail: 14, head: 15, n: 6, bytes: []byte{'o', 'p', 'a', 'b', 'c', 'd'}, desc: "14, 2 wrapped OK"},
+	}
+
+	for i, tt := range tests {
+		buf.SetPos(tt.tail, tt.head)
+		o := make(chan []byte)
+		go func() {
+			p, _ := buf.Read(tt.n)
+			o <- p
+		}()
+
+		time.Sleep(time.Millisecond)
+		atomic.StoreInt64(&buf.head, buf.head+int64(tt.n))
+
+		buf.wcond.L.Lock()
+		buf.wcond.Broadcast()
+		buf.wcond.L.Unlock()
+
+		done := <-o
+		require.Equal(t, tt.bytes, done, "Peeked bytes mismatch [i:%d] %s", i, tt.desc)
+
+	}
+}
+
+func TestReadEnded(t *testing.T) {
+	buf := NewBuffer(16, 4)
+	o := make(chan error)
+	go func() {
+		_, err := buf.Read(4)
+		o <- err
+	}()
+	time.Sleep(time.Millisecond)
+	atomic.StoreUint32(&buf.done, 1)
+	buf.wcond.L.Lock()
+	buf.wcond.Broadcast()
+	buf.wcond.L.Unlock()
+
+	require.Error(t, <-o)
+}

server/internal/circ/writer.go

@@ -0,0 +1,106 @@
+package circ
+
+import (
+	"io"
+	"log"
+	"sync/atomic"
+)
+
+// Writer is a circular buffer for writing data to an io.Writer.
+type Writer struct {
+	*Buffer
+}
+
+// NewWriter returns a pointer to a new Circular Writer.
+func NewWriter(size, block int) *Writer {
+	b := NewBuffer(size, block)
+	b.ID = "writer"
+	return &Writer{
+		b,
+	}
+}
+
+// NewWriterFromSlice returns a new Circular Writer using a pre-existing
+// byte slice.
+func NewWriterFromSlice(block int, p []byte) *Writer {
+	b := NewBufferFromSlice(block, p)
+	b.ID = "writer"
+	return &Writer{
+		b,
+	}
+}
+
+// WriteTo writes the contents of the buffer to an io.Writer.
+func (b *Writer) WriteTo(w io.Writer) (total int64, err error) {
+	atomic.StoreUint32(&b.State, 2)
+	defer atomic.StoreUint32(&b.State, 0)
+	for {
+		if atomic.LoadUint32(&b.done) == 1 && b.CapDelta() == 0 {
+			return total, io.EOF
+		}
+
+		// Read from the buffer until there is at least 1 byte to write.
+		err = b.awaitFilled(1)
+		if err != nil {
+			return
+		}
+
+		// Get all the bytes between the tail and head, wrapping if necessary.
+		tail := atomic.LoadInt64(&b.tail)
+		rTail := b.Index(tail)
+		rHead := b.Index(atomic.LoadInt64(&b.head))
+		n := b.CapDelta()
+		p := make([]byte, 0, n)
+
+		if rTail > rHead {
+			p = append(p, b.buf[rTail:]...)
+			p = append(p, b.buf[:rHead]...)
+		} else {
+			p = append(p, b.buf[rTail:rHead]...)
+		}
+
+		n, err = w.Write(p)
+		total += int64(n)
+		if err != nil {
+			log.Println("error writing to buffer io.Writer;", err)
+			return
+		}
+
+		// Move the tail forward the bytes written and broadcast change.
+		atomic.StoreInt64(&b.tail, tail+int64(n))
+		b.rcond.L.Lock()
+		b.rcond.Broadcast()
+		b.rcond.L.Unlock()
+	}
+}
+
+// Write writes the buffer to the buffer p, returning the number of bytes written.
+// The bytes written to the buffer are picked up by WriteTo.
+func (b *Writer) Write(p []byte) (total int, err error) {
+	err = b.awaitEmpty(len(p))
+	if err != nil {
+		return
+	}
+
+	total = b.writeBytes(p)
+	atomic.AddInt64(&b.head, int64(total))
+	b.wcond.L.Lock()
+	b.wcond.Broadcast()
+	b.wcond.L.Unlock()
+	return
+}
+
+// writeBytes writes bytes to the buffer from the start position, and returns
+// the new head position. This function does not wait for capacity and will
+// overwrite any existing bytes.
+func (b *Writer) writeBytes(p []byte) int {
+	var o int
+	var n int
+	for i := 0; i < len(p); i++ {
+		o = b.Index(atomic.LoadInt64(&b.head) + int64(i))
+		b.buf[o] = p[i]
+		n++
+	}
+
+	return n
+}

server/internal/circ/writer_test.go

@@ -0,0 +1,155 @@
+package circ
+
+import (
+	"bufio"
+	"bytes"
+	"net"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewWriter(t *testing.T) {
+	var size = 16
+	var block = 4
+	buf := NewWriter(size, block)
+
+	require.NotNil(t, buf.buf)
+	require.Equal(t, size, len(buf.buf))
+	require.Equal(t, size, buf.size)
+	require.Equal(t, block, buf.block)
+}
+
+func TestNewWriterFromSlice(t *testing.T) {
+	b := NewBytesPool(256)
+	buf := NewWriterFromSlice(DefaultBlockSize, b.Get())
+	require.NotNil(t, buf.buf)
+	require.Equal(t, 256, cap(buf.buf))
+}
+
+func TestWriteTo(t *testing.T) {
+	tests := []struct {
+		tail  int64
+		head  int64
+		bytes []byte
+		await int
+		total int
+		err   error
+		desc  string
+	}{
+		{tail: 0, head: 5, bytes: []byte{'a', 'b', 'c', 'd', 'e'}, desc: "0,5 OK"},
+		{tail: 14, head: 21, bytes: []byte{'o', 'p', 'a', 'b', 'c', 'd', 'e'}, desc: "14,16(2) OK"},
+	}
+
+	for i, tt := range tests {
+		bb := []byte{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p'}
+		buf := NewWriter(16, 4)
+		buf.Set(bb, 0, 16)
+		buf.SetPos(tt.tail, tt.head)
+
+		var b bytes.Buffer
+		w := bufio.NewWriter(&b)
+
+		nc := make(chan int64)
+		go func() {
+			n, _ := buf.WriteTo(w)
+			nc <- n
+		}()
+
+		time.Sleep(time.Millisecond * 100)
+		atomic.StoreUint32(&buf.done, 1)
+		buf.wcond.L.Lock()
+		buf.wcond.Broadcast()
+		buf.wcond.L.Unlock()
+
+		w.Flush()
+		require.Equal(t, tt.bytes, b.Bytes(), "Written bytes mismatch [i:%d] %s", i, tt.desc)
+	}
+}
+
+func TestWriteToEndedFirst(t *testing.T) {
+	buf := NewWriter(16, 4)
+	buf.done = 1
+
+	var b bytes.Buffer
+	w := bufio.NewWriter(&b)
+	_, err := buf.WriteTo(w)
+	require.Error(t, err)
+}
+
+func TestWriteToBadWriter(t *testing.T) {
+	bb := []byte{'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p'}
+	buf := NewWriter(16, 4)
+	buf.Set(bb, 0, 16)
+	buf.SetPos(0, 6)
+	r, w := net.Pipe()
+
+	w.Close()
+	_, err := buf.WriteTo(w)
+	require.Error(t, err)
+	r.Close()
+}
+
+func TestWrite(t *testing.T) {
+	tests := []struct {
+		tail  int64
+		head  int64
+		rHead int64
+		bytes []byte
+		want  []byte
+		desc  string
+	}{
+		{tail: 0, head: 0, rHead: 4, bytes: []byte{'a', 'b', 'c', 'd'}, want: []byte{'a', 'b', 'c', 'd', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, desc: "0>4 OK"},
+		{tail: 4, head: 14, rHead: 2, bytes: []byte{'a', 'b', 'c', 'd'}, want: []byte{'c', 'd', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'a', 'b'}, desc: "14>2 OK"},
+	}
+
+	for i, tt := range tests {
+		buf := NewWriter(16, 4)
+		buf.SetPos(tt.tail, tt.head)
+
+		o := make(chan []interface{})
+		go func() {
+			nn, err := buf.Write(tt.bytes)
+			o <- []interface{}{nn, err}
+		}()
+
+		done := <-o
+		require.Equal(t, tt.want, buf.buf, "Wanted written mismatch [i:%d] %s", i, tt.desc)
+		require.Nil(t, done[1], "Unexpected Error [i:%d] %s", i, tt.desc)
+	}
+}
+
+func TestWriteEnded(t *testing.T) {
+	buf := NewWriter(16, 4)
+	buf.SetPos(15, 30)
+	buf.done = 1
+
+	_, err := buf.Write([]byte{'a', 'b', 'c', 'd'})
+	require.Error(t, err)
+}
+
+func TestWriteBytes(t *testing.T) {
+	tests := []struct {
+		tail  int64
+		head  int64
+		bytes []byte
+		want  []byte
+		start int
+		desc  string
+	}{
+		{tail: 0, head: 0, bytes: []byte{'a', 'b', 'c', 'd'}, want: []byte{'a', 'b', 'c', 'd', 0, 0, 0, 0}, desc: "0,4 OK"},
+		{tail: 6, head: 6, bytes: []byte{'a', 'b', 'c', 'd'}, want: []byte{'c', 'd', 0, 0, 0, 0, 'a', 'b'}, desc: "6,2 OK wrapped"},
+	}
+
+	for i, tt := range tests {
+		buf := NewWriter(8, 4)
+		buf.SetPos(tt.tail, tt.head)
+		n := buf.writeBytes(tt.bytes)
+
+		require.Equal(t, tt.want, buf.buf, "Buffer mistmatch [i:%d] %s", i, tt.desc)
+		require.Equal(t, len(tt.bytes), n)
+	}
+
+}